@team-agent/installer 0.5.128 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -4
- package/README.zh.md +1 -1
- package/package.json +4 -8
- package/skills/team-agent/SKILL.md +37 -20
- package/skills/team-agent/command-coverage.json +169 -183
- package/skills/team-agent/docs/reference/team-agent-operator.md +64 -30
- package/skills/team-agent/references/recovery-runbook.md +1 -1
- package/skills/team-agent/references/team-agent-operator.md +64 -30
- package/templates/team_state.md +1 -1
- package/Cargo.lock +0 -1063
- package/Cargo.toml +0 -36
- package/crates/team-agent/Cargo.toml +0 -69
- package/crates/team-agent/src/app_server_test_support.rs +0 -258
- package/crates/team-agent/src/cli/adapters.rs +0 -1288
- package/crates/team-agent/src/cli/arguments.rs +0 -405
- package/crates/team-agent/src/cli/attach_app_server_leader.rs +0 -98
- package/crates/team-agent/src/cli/diagnose.rs +0 -1721
- package/crates/team-agent/src/cli/emit.rs +0 -2930
- package/crates/team-agent/src/cli/grok_slot.rs +0 -384
- package/crates/team-agent/src/cli/helpers.rs +0 -91
- package/crates/team-agent/src/cli/intent/runtime.rs +0 -1216
- package/crates/team-agent/src/cli/intent/runtime_tests.rs +0 -912
- package/crates/team-agent/src/cli/intent/tests.rs +0 -689
- package/crates/team-agent/src/cli/intent.rs +0 -664
- package/crates/team-agent/src/cli/leader.rs +0 -300
- package/crates/team-agent/src/cli/leader_prompt.rs +0 -261
- package/crates/team-agent/src/cli/leaders.rs +0 -418
- package/crates/team-agent/src/cli/mod.rs +0 -6545
- package/crates/team-agent/src/cli/models.rs +0 -1206
- package/crates/team-agent/src/cli/named_address.rs +0 -1651
- package/crates/team-agent/src/cli/profile.rs +0 -342
- package/crates/team-agent/src/cli/route.rs +0 -212
- package/crates/team-agent/src/cli/schedule.rs +0 -175
- package/crates/team-agent/src/cli/send/coordinator.rs +0 -164
- package/crates/team-agent/src/cli/send/mailbox.rs +0 -145
- package/crates/team-agent/src/cli/send/persist.rs +0 -156
- package/crates/team-agent/src/cli/send/presentation.rs +0 -370
- package/crates/team-agent/src/cli/send/resolve.rs +0 -573
- package/crates/team-agent/src/cli/send.rs +0 -199
- package/crates/team-agent/src/cli/shutdown_socket.rs +0 -288
- package/crates/team-agent/src/cli/spec.rs +0 -165
- package/crates/team-agent/src/cli/status.rs +0 -280
- package/crates/team-agent/src/cli/status_port/agents.rs +0 -358
- package/crates/team-agent/src/cli/status_port/approvals.rs +0 -79
- package/crates/team-agent/src/cli/status_port/brief.rs +0 -1981
- package/crates/team-agent/src/cli/status_port/compact.rs +0 -240
- package/crates/team-agent/src/cli/status_port/format.rs +0 -146
- package/crates/team-agent/src/cli/status_port/inbox.rs +0 -62
- package/crates/team-agent/src/cli/status_port/runtime.rs +0 -197
- package/crates/team-agent/src/cli/status_port/snapshot.rs +0 -208
- package/crates/team-agent/src/cli/status_port/store.rs +0 -416
- package/crates/team-agent/src/cli/status_port/tests.rs +0 -90
- package/crates/team-agent/src/cli/status_port.rs +0 -92
- package/crates/team-agent/src/cli/tests/base.rs +0 -885
- package/crates/team-agent/src/cli/tests/divergence.rs +0 -518
- package/crates/team-agent/src/cli/tests/lane_c.rs +0 -453
- package/crates/team-agent/src/cli/tests/leader_watch.rs +0 -322
- package/crates/team-agent/src/cli/tests/main_preserved.rs +0 -430
- package/crates/team-agent/src/cli/tests/missing_subcommands.rs +0 -336
- package/crates/team-agent/src/cli/tests/mod.rs +0 -137
- package/crates/team-agent/src/cli/tests/named_address.rs +0 -761
- package/crates/team-agent/src/cli/tests/parser_safety.rs +0 -383
- package/crates/team-agent/src/cli/tests/run_delegation.rs +0 -484
- package/crates/team-agent/src/cli/tests/shutdown_kill_plan.rs +0 -2179
- package/crates/team-agent/src/cli/tests/status_brief.rs +0 -1514
- package/crates/team-agent/src/cli/tests/status_send.rs +0 -888
- package/crates/team-agent/src/cli/tests/verb_install_skill.rs +0 -93
- package/crates/team-agent/src/cli/tests/verb_profile.rs +0 -392
- package/crates/team-agent/src/cli/triage.rs +0 -234
- package/crates/team-agent/src/cli/types.rs +0 -591
- package/crates/team-agent/src/codex_app_server.rs +0 -546
- package/crates/team-agent/src/communication_mode/mod.rs +0 -56
- package/crates/team-agent/src/compiler/tests.rs +0 -827
- package/crates/team-agent/src/compiler.rs +0 -890
- package/crates/team-agent/src/conpty/backend.rs +0 -829
- package/crates/team-agent/src/conpty/mod.rs +0 -33
- package/crates/team-agent/src/contract_runtime/backend.rs +0 -1744
- package/crates/team-agent/src/contract_runtime/commands.rs +0 -671
- package/crates/team-agent/src/contract_runtime/config.rs +0 -364
- package/crates/team-agent/src/contract_runtime/config_tests.rs +0 -76
- package/crates/team-agent/src/contract_runtime/discovery.rs +0 -449
- package/crates/team-agent/src/contract_runtime/fork.rs +0 -361
- package/crates/team-agent/src/contract_runtime/forward.rs +0 -288
- package/crates/team-agent/src/contract_runtime/framework.rs +0 -367
- package/crates/team-agent/src/contract_runtime/mcp.rs +0 -394
- package/crates/team-agent/src/contract_runtime/mod.rs +0 -17
- package/crates/team-agent/src/contract_runtime/prompt.rs +0 -216
- package/crates/team-agent/src/contract_runtime/pump.rs +0 -271
- package/crates/team-agent/src/contract_runtime/registry.rs +0 -553
- package/crates/team-agent/src/contract_runtime/workspace.rs +0 -228
- package/crates/team-agent/src/coordinator/backoff.rs +0 -741
- package/crates/team-agent/src/coordinator/conpty_shim.rs +0 -805
- package/crates/team-agent/src/coordinator/health.rs +0 -1185
- package/crates/team-agent/src/coordinator/mod.rs +0 -119
- package/crates/team-agent/src/coordinator/orphan.rs +0 -110
- package/crates/team-agent/src/coordinator/runtime_detectors.rs +0 -798
- package/crates/team-agent/src/coordinator/runtime_observation.rs +0 -84
- package/crates/team-agent/src/coordinator/steps/abnormal.rs +0 -2309
- package/crates/team-agent/src/coordinator/steps/delivery.rs +0 -15
- package/crates/team-agent/src/coordinator/steps/health_sync.rs +0 -15
- package/crates/team-agent/src/coordinator/steps/mod.rs +0 -126
- package/crates/team-agent/src/coordinator/steps/persist.rs +0 -15
- package/crates/team-agent/src/coordinator/steps/runtime_prompts.rs +0 -15
- package/crates/team-agent/src/coordinator/steps/session_gate.rs +0 -16
- package/crates/team-agent/src/coordinator/tests/a0_lostupdate.rs +0 -158
- package/crates/team-agent/src/coordinator/tests/abnormal.rs +0 -312
- package/crates/team-agent/src/coordinator/tests/api_error_recovery.rs +0 -300
- package/crates/team-agent/src/coordinator/tests/basics.rs +0 -211
- package/crates/team-agent/src/coordinator/tests/daemon.rs +0 -514
- package/crates/team-agent/src/coordinator/tests/energy.rs +0 -549
- package/crates/team-agent/src/coordinator/tests/health_sync.rs +0 -503
- package/crates/team-agent/src/coordinator/tests/main_preserved.rs +0 -186
- package/crates/team-agent/src/coordinator/tests/mod.rs +0 -295
- package/crates/team-agent/src/coordinator/tests/s3_bounded.rs +0 -207
- package/crates/team-agent/src/coordinator/tests/spine.rs +0 -308
- package/crates/team-agent/src/coordinator/tests/takeover.rs +0 -286
- package/crates/team-agent/src/coordinator/tests/tick_core.rs +0 -237
- package/crates/team-agent/src/coordinator/tick.rs +0 -3024
- package/crates/team-agent/src/coordinator/types.rs +0 -540
- package/crates/team-agent/src/db/agent_health_capture.rs +0 -126
- package/crates/team-agent/src/db/message_store.rs +0 -1527
- package/crates/team-agent/src/db/migration.rs +0 -984
- package/crates/team-agent/src/db/mod.rs +0 -26
- package/crates/team-agent/src/db/schema.rs +0 -663
- package/crates/team-agent/src/diagnose/comms.rs +0 -242
- package/crates/team-agent/src/diagnose/mod.rs +0 -44
- package/crates/team-agent/src/diagnose/orphans.rs +0 -889
- package/crates/team-agent/src/direct_delivery.rs +0 -368
- package/crates/team-agent/src/event_log.rs +0 -502
- package/crates/team-agent/src/fake_worker.rs +0 -303
- package/crates/team-agent/src/inbox_store.rs +0 -169
- package/crates/team-agent/src/kill_audit.rs +0 -106
- package/crates/team-agent/src/layout/manager.rs +0 -248
- package/crates/team-agent/src/layout/mod.rs +0 -36
- package/crates/team-agent/src/layout/overlay.rs +0 -134
- package/crates/team-agent/src/layout/placement.rs +0 -52
- package/crates/team-agent/src/layout/recovery.rs +0 -106
- package/crates/team-agent/src/layout/runtime_sessions.rs +0 -317
- package/crates/team-agent/src/layout/sessions.rs +0 -294
- package/crates/team-agent/src/layout/tmux_endpoint.rs +0 -165
- package/crates/team-agent/src/layout/worker_env.rs +0 -721
- package/crates/team-agent/src/layout/worker_window_helpers.rs +0 -37
- package/crates/team-agent/src/leader/helpers.rs +0 -178
- package/crates/team-agent/src/leader/incident.rs +0 -6
- package/crates/team-agent/src/leader/lease.rs +0 -3356
- package/crates/team-agent/src/leader/mod.rs +0 -104
- package/crates/team-agent/src/leader/owner_bind.rs +0 -400
- package/crates/team-agent/src/leader/prompt.rs +0 -905
- package/crates/team-agent/src/leader/provider_attribution.rs +0 -247
- package/crates/team-agent/src/leader/rediscover/tests.rs +0 -689
- package/crates/team-agent/src/leader/rediscover.rs +0 -1192
- package/crates/team-agent/src/leader/registry.rs +0 -888
- package/crates/team-agent/src/leader/start.rs +0 -4501
- package/crates/team-agent/src/leader/takeover.rs +0 -65
- package/crates/team-agent/src/leader/tests/basics.rs +0 -337
- package/crates/team-agent/src/leader/tests/byte_findings.rs +0 -355
- package/crates/team-agent/src/leader/tests/claim_leader_mailbox_flush_contract.rs +0 -544
- package/crates/team-agent/src/leader/tests/historical_inventory_upgrade_gate_red.rs +0 -624
- package/crates/team-agent/src/leader/tests/identity.rs +0 -465
- package/crates/team-agent/src/leader/tests/identity_session_names.rs +0 -42
- package/crates/team-agent/src/leader/tests/idle.rs +0 -113
- package/crates/team-agent/src/leader/tests/lease_api.rs +0 -520
- package/crates/team-agent/src/leader/tests/lease_claim.rs +0 -629
- package/crates/team-agent/src/leader/tests/mod.rs +0 -106
- package/crates/team-agent/src/leader/tests/prompt_contract.rs +0 -268
- package/crates/team-agent/src/leader/tests/rediscover.rs +0 -403
- package/crates/team-agent/src/leader/tests/rehomed_env.rs +0 -83
- package/crates/team-agent/src/leader/tests/team_in_team_state_scope_red.rs +0 -1273
- package/crates/team-agent/src/leader/tests/terminal_and_replay_vs_rerender_invariants_red.rs +0 -417
- package/crates/team-agent/src/leader/tests/wake_start_owner.rs +0 -168
- package/crates/team-agent/src/leader/types.rs +0 -534
- package/crates/team-agent/src/lib.rs +0 -147
- package/crates/team-agent/src/lifecycle/helpers.rs +0 -212
- package/crates/team-agent/src/lifecycle/launch/add_agent.rs +0 -881
- package/crates/team-agent/src/lifecycle/launch/add_agent_state.rs +0 -598
- package/crates/team-agent/src/lifecycle/launch/agent_state.rs +0 -332
- package/crates/team-agent/src/lifecycle/launch/clone_agent.rs +0 -204
- package/crates/team-agent/src/lifecycle/launch/fork_agent.rs +0 -107
- package/crates/team-agent/src/lifecycle/launch/fork_entry.rs +0 -144
- package/crates/team-agent/src/lifecycle/launch/identity.rs +0 -674
- package/crates/team-agent/src/lifecycle/launch/layout.rs +0 -350
- package/crates/team-agent/src/lifecycle/launch/leader_context.rs +0 -589
- package/crates/team-agent/src/lifecycle/launch/mcp_config.rs +0 -160
- package/crates/team-agent/src/lifecycle/launch/ownership.rs +0 -105
- package/crates/team-agent/src/lifecycle/launch/pi_mcp.rs +0 -899
- package/crates/team-agent/src/lifecycle/launch/pi_mcp_extension.js +0 -225
- package/crates/team-agent/src/lifecycle/launch/plan.rs +0 -269
- package/crates/team-agent/src/lifecycle/launch/quick_start.rs +0 -5200
- package/crates/team-agent/src/lifecycle/launch/quick_start_transport.rs +0 -368
- package/crates/team-agent/src/lifecycle/launch/readiness.rs +0 -519
- package/crates/team-agent/src/lifecycle/launch/role_source.rs +0 -287
- package/crates/team-agent/src/lifecycle/launch/spawn.rs +0 -473
- package/crates/team-agent/src/lifecycle/launch/spec_state.rs +0 -801
- package/crates/team-agent/src/lifecycle/launch/state_projection.rs +0 -624
- package/crates/team-agent/src/lifecycle/launch/worker_env.rs +0 -558
- package/crates/team-agent/src/lifecycle/launch.rs +0 -557
- package/crates/team-agent/src/lifecycle/lock.rs +0 -350
- package/crates/team-agent/src/lifecycle/mod.rs +0 -79
- package/crates/team-agent/src/lifecycle/pane_input_lock.rs +0 -239
- package/crates/team-agent/src/lifecycle/profile_launch.rs +0 -1067
- package/crates/team-agent/src/lifecycle/profile_smoke.rs +0 -557
- package/crates/team-agent/src/lifecycle/restart/agent.rs +0 -2199
- package/crates/team-agent/src/lifecycle/restart/avatar.rs +0 -1798
- package/crates/team-agent/src/lifecycle/restart/claude_fork.rs +0 -163
- package/crates/team-agent/src/lifecycle/restart/common.rs +0 -2362
- package/crates/team-agent/src/lifecycle/restart/orchestrator.rs +0 -65
- package/crates/team-agent/src/lifecycle/restart/preflight.rs +0 -113
- package/crates/team-agent/src/lifecycle/restart/rebuild.rs +0 -5135
- package/crates/team-agent/src/lifecycle/restart/remove.rs +0 -1504
- package/crates/team-agent/src/lifecycle/restart/selection.rs +0 -559
- package/crates/team-agent/src/lifecycle/restart/team_state.rs +0 -329
- package/crates/team-agent/src/lifecycle/restart.rs +0 -227
- package/crates/team-agent/src/lifecycle/role_config.rs +0 -1230
- package/crates/team-agent/src/lifecycle/tests/acceptance_b_batch_red.rs +0 -1021
- package/crates/team-agent/src/lifecycle/tests/agent_ops.rs +0 -1205
- package/crates/team-agent/src/lifecycle/tests/b0_legacy_snapshot_nonauthority_contract.rs +0 -265
- package/crates/team-agent/src/lifecycle/tests/b0_reader_hideone_audit_contract.rs +0 -490
- package/crates/team-agent/src/lifecycle/tests/behavioral_diff_264_red.rs +0 -656
- package/crates/team-agent/src/lifecycle/tests/claude_compatible_config_red.rs +0 -325
- package/crates/team-agent/src/lifecycle/tests/claude_profile_launch_red.rs +0 -447
- package/crates/team-agent/src/lifecycle/tests/cli_argv_routing_red.rs +0 -1638
- package/crates/team-agent/src/lifecycle/tests/clone_agent_preserves_source_tools.rs +0 -427
- package/crates/team-agent/src/lifecycle/tests/clone_fork_copilot_perms_red.rs +0 -770
- package/crates/team-agent/src/lifecycle/tests/communication_mode_runtime_contract_red.rs +0 -419
- package/crates/team-agent/src/lifecycle/tests/copilot_provider_red.rs +0 -2035
- package/crates/team-agent/src/lifecycle/tests/core.rs +0 -1327
- package/crates/team-agent/src/lifecycle/tests/f032_startup_prompt_best_effort_red.rs +0 -315
- package/crates/team-agent/src/lifecycle/tests/fixtures/pi_list_models_g0.stdout.txt +0 -222
- package/crates/team-agent/src/lifecycle/tests/fixtures/pi_list_models_spaced_ids.stdout.txt +0 -9
- package/crates/team-agent/src/lifecycle/tests/fixtures/pi_native_mcp_host.mjs +0 -72
- package/crates/team-agent/src/lifecycle/tests/gate_fixtures.rs +0 -562
- package/crates/team-agent/src/lifecycle/tests/grok_require_explicit_model_red.rs +0 -110
- package/crates/team-agent/src/lifecycle/tests/harvest2_a_batch_red.rs +0 -512
- package/crates/team-agent/src/lifecycle/tests/host_cotenant_death_p0_contract.rs +0 -921
- package/crates/team-agent/src/lifecycle/tests/issue238_codex_effort_contract.rs +0 -68
- package/crates/team-agent/src/lifecycle/tests/lane_ops.rs +0 -1851
- package/crates/team-agent/src/lifecycle/tests/launch_spawn.rs +0 -3983
- package/crates/team-agent/src/lifecycle/tests/lifecycle_lock.rs +0 -339
- package/crates/team-agent/src/lifecycle/tests/lifecycle_rollback_red.rs +0 -596
- package/crates/team-agent/src/lifecycle/tests/main_preserved.rs +0 -513
- package/crates/team-agent/src/lifecycle/tests/mcp_tool_name_format_red.rs +0 -69
- package/crates/team-agent/src/lifecycle/tests/phase_b_contracts.rs +0 -824
- package/crates/team-agent/src/lifecycle/tests/phase_golden.rs +0 -723
- package/crates/team-agent/src/lifecycle/tests/pi_catalog_red.rs +0 -164
- package/crates/team-agent/src/lifecycle/tests/pi_command_red.rs +0 -200
- package/crates/team-agent/src/lifecycle/tests/pi_compiler_red.rs +0 -210
- package/crates/team-agent/src/lifecycle/tests/pi_dual_entry_red.rs +0 -527
- package/crates/team-agent/src/lifecycle/tests/pi_executable_mcp_red.rs +0 -1062
- package/crates/team-agent/src/lifecycle/tests/pi_fork_session_inheritance_red.rs +0 -1419
- package/crates/team-agent/src/lifecycle/tests/pi_lifecycle_doctor_red.rs +0 -98
- package/crates/team-agent/src/lifecycle/tests/pi_native_mcp_capability_red.rs +0 -550
- package/crates/team-agent/src/lifecycle/tests/pi_session_delivery_red.rs +0 -226
- package/crates/team-agent/src/lifecycle/tests/quick_start_worker_readiness_red.rs +0 -332
- package/crates/team-agent/src/lifecycle/tests/realmachine_clusters_1_6_red.rs +0 -786
- package/crates/team-agent/src/lifecycle/tests/realmachine_residual_g1_g4_red.rs +0 -648
- package/crates/team-agent/src/lifecycle/tests/restart.rs +0 -185
- package/crates/team-agent/src/lifecycle/tests/restart_build_before_destroy_0540_contract.rs +0 -1053
- package/crates/team-agent/src/lifecycle/tests/restart_liveness_red.rs +0 -679
- package/crates/team-agent/src/lifecycle/tests/restart_rebind_hotfix_252_red.rs +0 -1126
- package/crates/team-agent/src/lifecycle/tests/restart_session_capture_red.rs +0 -153
- package/crates/team-agent/src/lifecycle/tests/resume_recover_red.rs +0 -427
- package/crates/team-agent/src/lifecycle/tests/stale_team_saveconflict_contract.rs +0 -431
- package/crates/team-agent/src/lifecycle/tests/startup_latency_contract.rs +0 -1066
- package/crates/team-agent/src/lifecycle/tests/status_credential_redaction_contract.rs +0 -876
- package/crates/team-agent/src/lifecycle/tests/subprep_codex_trust_roundtrip_red.rs +0 -1026
- package/crates/team-agent/src/lifecycle/tests/swallow_batch4_semantics_red.rs +0 -342
- package/crates/team-agent/src/lifecycle/tests/team_in_team_identity_scope_red.rs +0 -543
- package/crates/team-agent/src/lifecycle/tests/team_in_team_sibling_quick_start_red.rs +0 -1103
- package/crates/team-agent/src/lifecycle/tests/team_in_team_state_scope_red.rs +0 -1276
- package/crates/team-agent/src/lifecycle/tests/team_key_retirement_roster_red.rs +0 -145
- package/crates/team-agent/src/lifecycle/tests/team_key_retirement_txn_red.rs +0 -603
- package/crates/team-agent/src/lifecycle/tests/test_isolation_escape_contract.rs +0 -936
- package/crates/team-agent/src/lifecycle/tests/upgrade_compat_0211_red.rs +0 -970
- package/crates/team-agent/src/lifecycle/tests/verify_rs031_window_consistency_red.rs +0 -995
- package/crates/team-agent/src/lifecycle/tests/worker_spawn_env_red.rs +0 -836
- package/crates/team-agent/src/lifecycle/tests.rs +0 -105
- package/crates/team-agent/src/lifecycle/types.rs +0 -877
- package/crates/team-agent/src/lifecycle/worker_command_context.rs +0 -363
- package/crates/team-agent/src/main.rs +0 -52
- package/crates/team-agent/src/mcp_server/helpers.rs +0 -510
- package/crates/team-agent/src/mcp_server/lifecycle_tools/mod.rs +0 -6
- package/crates/team-agent/src/mcp_server/lifecycle_tools/state_status.rs +0 -43
- package/crates/team-agent/src/mcp_server/mod.rs +0 -119
- package/crates/team-agent/src/mcp_server/normalize.rs +0 -485
- package/crates/team-agent/src/mcp_server/readiness.rs +0 -527
- package/crates/team-agent/src/mcp_server/readiness_tests.rs +0 -348
- package/crates/team-agent/src/mcp_server/tests/golden.rs +0 -285
- package/crates/team-agent/src/mcp_server/tests/normalize.rs +0 -247
- package/crates/team-agent/src/mcp_server/tests/scoped.rs +0 -158
- package/crates/team-agent/src/mcp_server/tests/send.rs +0 -353
- package/crates/team-agent/src/mcp_server/tests/tools.rs +0 -341
- package/crates/team-agent/src/mcp_server/tests/wire.rs +0 -254
- package/crates/team-agent/src/mcp_server/tests.rs +0 -43
- package/crates/team-agent/src/mcp_server/tools.rs +0 -889
- package/crates/team-agent/src/mcp_server/types.rs +0 -396
- package/crates/team-agent/src/mcp_server/wire.rs +0 -867
- package/crates/team-agent/src/messaging/activity.rs +0 -461
- package/crates/team-agent/src/messaging/address.rs +0 -87
- package/crates/team-agent/src/messaging/delivery.rs +0 -3727
- package/crates/team-agent/src/messaging/delivery_consumption_tests.rs +0 -234
- package/crates/team-agent/src/messaging/enter_rescue.rs +0 -362
- package/crates/team-agent/src/messaging/enter_rescue_tests.rs +0 -633
- package/crates/team-agent/src/messaging/helpers.rs +0 -336
- package/crates/team-agent/src/messaging/leader_channel.rs +0 -287
- package/crates/team-agent/src/messaging/leader_receiver.rs +0 -786
- package/crates/team-agent/src/messaging/mod.rs +0 -167
- package/crates/team-agent/src/messaging/persist.rs +0 -377
- package/crates/team-agent/src/messaging/presentation.rs +0 -417
- package/crates/team-agent/src/messaging/reminders.rs +0 -316
- package/crates/team-agent/src/messaging/results/collection_tests.rs +0 -719
- package/crates/team-agent/src/messaging/results.rs +0 -1415
- package/crates/team-agent/src/messaging/scheduler.rs +0 -223
- package/crates/team-agent/src/messaging/selftest.rs +0 -315
- package/crates/team-agent/src/messaging/send.rs +0 -981
- package/crates/team-agent/src/messaging/tests/basic.rs +0 -346
- package/crates/team-agent/src/messaging/tests/dup_inject.rs +0 -657
- package/crates/team-agent/src/messaging/tests/e23.rs +0 -49
- package/crates/team-agent/src/messaging/tests/leader_channel.rs +0 -169
- package/crates/team-agent/src/messaging/tests/leader_inject_acceptance.rs +0 -817
- package/crates/team-agent/src/messaging/tests/main_preserved.rs +0 -85
- package/crates/team-agent/src/messaging/tests/mod.rs +0 -306
- package/crates/team-agent/src/messaging/tests/runtime.rs +0 -3549
- package/crates/team-agent/src/messaging/tests/spine.rs +0 -853
- package/crates/team-agent/src/messaging/tests/wave2.rs +0 -381
- package/crates/team-agent/src/messaging/trust.rs +0 -215
- package/crates/team-agent/src/messaging/types.rs +0 -472
- package/crates/team-agent/src/messaging/wait.rs +0 -366
- package/crates/team-agent/src/messaging/watchers.rs +0 -916
- package/crates/team-agent/src/model/enums.rs +0 -356
- package/crates/team-agent/src/model/errors.rs +0 -18
- package/crates/team-agent/src/model/ids.rs +0 -165
- package/crates/team-agent/src/model/mod.rs +0 -29
- package/crates/team-agent/src/model/name_similarity.rs +0 -252
- package/crates/team-agent/src/model/pane_authority_refusal.rs +0 -359
- package/crates/team-agent/src/model/paths.rs +0 -257
- package/crates/team-agent/src/model/routing.rs +0 -387
- package/crates/team-agent/src/model/spec.rs +0 -1184
- package/crates/team-agent/src/model/task_graph.rs +0 -321
- package/crates/team-agent/src/model/testdata/fuzz.golden.yaml +0 -43
- package/crates/team-agent/src/model/testdata/fuzz.yaml +0 -43
- package/crates/team-agent/src/model/testdata/spec_invalid_a.yaml +0 -209
- package/crates/team-agent/src/model/testdata/team.spec.golden.yaml +0 -208
- package/crates/team-agent/src/model/testdata/team.spec.yaml +0 -208
- package/crates/team-agent/src/model/yaml/tests.rs +0 -188
- package/crates/team-agent/src/model/yaml.rs +0 -802
- package/crates/team-agent/src/os_probe.rs +0 -280
- package/crates/team-agent/src/packaging/install.rs +0 -219
- package/crates/team-agent/src/packaging/migrate.rs +0 -38
- package/crates/team-agent/src/packaging/mod.rs +0 -91
- package/crates/team-agent/src/packaging/repair.rs +0 -33
- package/crates/team-agent/src/packaging/tests.rs +0 -822
- package/crates/team-agent/src/packaging/types.rs +0 -417
- package/crates/team-agent/src/platform/argv.rs +0 -331
- package/crates/team-agent/src/platform/errors.rs +0 -104
- package/crates/team-agent/src/platform/file_lock.rs +0 -279
- package/crates/team-agent/src/platform/mod.rs +0 -5
- package/crates/team-agent/src/platform/process.rs +0 -545
- package/crates/team-agent/src/provider/adapter.rs +0 -1150
- package/crates/team-agent/src/provider/adapters/claude.rs +0 -78
- package/crates/team-agent/src/provider/adapters/claude_fork.rs +0 -131
- package/crates/team-agent/src/provider/adapters/copilot.rs +0 -75
- package/crates/team-agent/src/provider/adapters/copilot_fork.rs +0 -307
- package/crates/team-agent/src/provider/adapters/fake.rs +0 -18
- package/crates/team-agent/src/provider/adapters/mod.rs +0 -30
- package/crates/team-agent/src/provider/adapters/pi.rs +0 -106
- package/crates/team-agent/src/provider/approvals/mod.rs +0 -3
- package/crates/team-agent/src/provider/approvals/parsing.rs +0 -586
- package/crates/team-agent/src/provider/approvals/runtime_prompts.rs +0 -205
- package/crates/team-agent/src/provider/argv_route.rs +0 -405
- package/crates/team-agent/src/provider/bypass_flags.rs +0 -116
- package/crates/team-agent/src/provider/classify.rs +0 -708
- package/crates/team-agent/src/provider/command_helpers.rs +0 -45
- package/crates/team-agent/src/provider/faults.rs +0 -165
- package/crates/team-agent/src/provider/helpers.rs +0 -52
- package/crates/team-agent/src/provider/mod.rs +0 -78
- package/crates/team-agent/src/provider/model_catalog.rs +0 -910
- package/crates/team-agent/src/provider/session/capture.rs +0 -2731
- package/crates/team-agent/src/provider/session/context_fork/claude.rs +0 -123
- package/crates/team-agent/src/provider/session/context_fork/outcome.rs +0 -172
- package/crates/team-agent/src/provider/session/context_fork.rs +0 -396
- package/crates/team-agent/src/provider/session/mod.rs +0 -36
- package/crates/team-agent/src/provider/session/resume.rs +0 -269
- package/crates/team-agent/src/provider/session_scan/claude.rs +0 -523
- package/crates/team-agent/src/provider/session_scan/common/tests.rs +0 -316
- package/crates/team-agent/src/provider/session_scan/common.rs +0 -654
- package/crates/team-agent/src/provider/session_scan/copilot.rs +0 -280
- package/crates/team-agent/src/provider/session_scan/pi.rs +0 -185
- package/crates/team-agent/src/provider/session_scan.rs +0 -71
- package/crates/team-agent/src/provider/startup_prompt.rs +0 -757
- package/crates/team-agent/src/provider/submit_now.rs +0 -71
- package/crates/team-agent/src/provider/testdata/copilot-ready-marker.txt +0 -5
- package/crates/team-agent/src/provider/testdata/copilot-trust-prompt.txt +0 -19
- package/crates/team-agent/src/provider/testdata/grok-models-authenticated.stdout +0 -9
- package/crates/team-agent/src/provider/testdata/grok-models-unauthenticated.stdout +0 -7
- package/crates/team-agent/src/provider/tests/adapter.rs +0 -441
- package/crates/team-agent/src/provider/tests/classify.rs +0 -148
- package/crates/team-agent/src/provider/tests/command_helpers.rs +0 -35
- package/crates/team-agent/src/provider/tests/copilot_fork.rs +0 -191
- package/crates/team-agent/src/provider/tests/faults.rs +0 -200
- package/crates/team-agent/src/provider/tests/idle.rs +0 -177
- package/crates/team-agent/src/provider/tests/wire.rs +0 -213
- package/crates/team-agent/src/provider/tests.rs +0 -32
- package/crates/team-agent/src/provider/types.rs +0 -529
- package/crates/team-agent/src/provider/wire.rs +0 -394
- package/crates/team-agent/src/redaction.rs +0 -259
- package/crates/team-agent/src/state/abnormal_watch/tests/mod.rs +0 -248
- package/crates/team-agent/src/state/abnormal_watch.rs +0 -144
- package/crates/team-agent/src/state/identity.rs +0 -914
- package/crates/team-agent/src/state/identity_keys.rs +0 -132
- package/crates/team-agent/src/state/mod.rs +0 -74
- package/crates/team-agent/src/state/owner_gate.rs +0 -482
- package/crates/team-agent/src/state/ownership.rs +0 -563
- package/crates/team-agent/src/state/paths.rs +0 -395
- package/crates/team-agent/src/state/persist.rs +0 -3811
- package/crates/team-agent/src/state/projection.rs +0 -1508
- package/crates/team-agent/src/state/repository/intent.rs +0 -14
- package/crates/team-agent/src/state/repository/tests.rs +0 -184
- package/crates/team-agent/src/state/repository.rs +0 -866
- package/crates/team-agent/src/state/selector.rs +0 -160
- package/crates/team-agent/src/state/testdata/state-rich.canonical.json +0 -133
- package/crates/team-agent/src/test_support/models_timeout_fixture.rs +0 -144
- package/crates/team-agent/src/tmux_backend/enter_rescue.rs +0 -325
- package/crates/team-agent/src/tmux_backend/enter_rescue_tests.rs +0 -432
- package/crates/team-agent/src/tmux_backend/native_input.rs +0 -513
- package/crates/team-agent/src/tmux_backend/native_input_tests.rs +0 -742
- package/crates/team-agent/src/tmux_backend/tests.rs +0 -4103
- package/crates/team-agent/src/tmux_backend.rs +0 -4492
- package/crates/team-agent/src/topology.rs +0 -537
- package/crates/team-agent/src/transport/enter_rescue.rs +0 -166
- package/crates/team-agent/src/transport/native_input.rs +0 -491
- package/crates/team-agent/src/transport/test_support.rs +0 -743
- package/crates/team-agent/src/transport/tests/behavior.rs +0 -326
- package/crates/team-agent/src/transport/tests/mod.rs +0 -194
- package/crates/team-agent/src/transport/tests/wire.rs +0 -574
- package/crates/team-agent/src/transport.rs +0 -1655
- package/crates/team-agent/src/transport_factory.rs +0 -819
|
@@ -1,2309 +0,0 @@
|
|
|
1
|
-
//! ---
|
|
2
|
-
//! purpose: tick 的异常退出检测步骤——按 agent 读有界 rollout 尾,判「最新一条显式 provider 错误是否属于当前 worker 世代」,据此决定通知还是只留审计
|
|
3
|
-
//! contract:
|
|
4
|
-
//! provides:
|
|
5
|
-
//! - name: detect_abnormal_exits
|
|
6
|
-
//! what: 遍历受监视 agent,产出 worker.abnormal_exit 通知或 dead_only 审计事件,并把本轮观测写回 state 的 abnormal_exit_watch
|
|
7
|
-
//! - name: explicit_process_liveness
|
|
8
|
-
//! what: 从 agent JSON 的显式存活字段解析进程存活态,不做任何探测
|
|
9
|
-
//! - name: read_tail_text
|
|
10
|
-
//! what: 有界读文件尾部文本,超长时从尾部 max_bytes 处起读
|
|
11
|
-
//! - name: metadata_mtime_ns
|
|
12
|
-
//! what: 把文件 metadata 的 mtime 归一成纳秒整数,供跨 tick 变化比较
|
|
13
|
-
//! depends:
|
|
14
|
-
//! - super::super::health
|
|
15
|
-
//! - super::super::tick
|
|
16
|
-
//! - super::super::types
|
|
17
|
-
//! - crate::event_log
|
|
18
|
-
//! - crate::messaging
|
|
19
|
-
//! - crate::provider
|
|
20
|
-
//! - crate::transport
|
|
21
|
-
//! - crate::state::projection
|
|
22
|
-
//! boundary:
|
|
23
|
-
//! - 不重启、不清理、不做物理注入:写事件与 state 观测记录之外,异常通知会经 crate::messaging 的 leader funnel 入库为 accepted 消息(物理注入由后续 deliver_pending 完成)
|
|
24
|
-
//! - 进程已死但没有显式错误时不通知,只落 dead_only 审计
|
|
25
|
-
//! - 通知事件里的 queued 只表示消息入库,不表示已送达
|
|
26
|
-
//! - rollout 尾读是有界的(首行可能被截断,消费方只解析最后一条完整 JSONL 记录)
|
|
27
|
-
//! maturity: wired
|
|
28
|
-
//! ---
|
|
29
|
-
//!
|
|
30
|
-
//! unit-11 (Stage 4) — coordinator tick `abnormal` step group.
|
|
31
|
-
//!
|
|
32
|
-
//! Abnormal-exit detection + classification step extracted from
|
|
33
|
-
//! `coordinator/tick.rs` without behavior changes.
|
|
34
|
-
|
|
35
|
-
use std::path::{Path, PathBuf};
|
|
36
|
-
|
|
37
|
-
use serde_json::Value;
|
|
38
|
-
|
|
39
|
-
use crate::event_log::EventLog;
|
|
40
|
-
use crate::provider::wire::{parse_provider, provider_wire};
|
|
41
|
-
use crate::provider::ProcessLiveness;
|
|
42
|
-
|
|
43
|
-
use super::super::health::pid_is_running;
|
|
44
|
-
use super::super::tick::TickError;
|
|
45
|
-
use super::super::types::Pid;
|
|
46
|
-
|
|
47
|
-
///
|
|
48
|
-
/// #236 `worker.abnormal_exit` watcher.
|
|
49
|
-
///
|
|
50
|
-
/// Notify only when the bounded transcript/rollout tail contains a latest explicit
|
|
51
|
-
/// provider error that is fresh for the current worker cohort. A dead process
|
|
52
|
-
/// without an explicit error remains a suppressed `dead_only` audit event; process
|
|
53
|
-
/// liveness is otherwise diagnostic data for this path.
|
|
54
|
-
/// ---
|
|
55
|
-
/// purpose: 跑一轮异常退出检测,把结论落成事件与 state 里的 abnormal_exit_watch 记录
|
|
56
|
-
/// params:
|
|
57
|
-
/// workspace: workspace 根,用于解析 rollout 相对路径
|
|
58
|
-
/// transport: pane 存活探测通道;只读探测,不注入
|
|
59
|
-
/// state: 可变运行时状态;本轮观测(size/mtime/世代/去重键)写回 coordinator.abnormal_exit_watch
|
|
60
|
-
/// event_log: 事件出口
|
|
61
|
-
/// targets: 本 tick 已采到的 pane 快照,避免每 agent 重新列一遍
|
|
62
|
-
/// returns: 检测跑完即 Ok;「有没有异常」不由返回值表达,只体现在事件与 state 里
|
|
63
|
-
/// errors: 事件写入等不可恢复失败时返回 TickError;单个 agent 的 rollout metadata 读不到不算错误,记为 unverifiable 继续
|
|
64
|
-
/// ---
|
|
65
|
-
pub(crate) fn detect_abnormal_exits(
|
|
66
|
-
workspace: &Path,
|
|
67
|
-
transport: &dyn crate::transport::Transport,
|
|
68
|
-
state: &mut Value,
|
|
69
|
-
event_log: &EventLog,
|
|
70
|
-
targets: &[crate::transport::PaneInfo],
|
|
71
|
-
) -> Result<(), TickError> {
|
|
72
|
-
let snapshot = state.clone();
|
|
73
|
-
let team = crate::state::projection::team_state_key(&snapshot);
|
|
74
|
-
let session_name = snapshot.get("session_name").and_then(Value::as_str);
|
|
75
|
-
for agent in abnormal_watch_agents(&snapshot) {
|
|
76
|
-
// Reject untrusted history without skipping a fresh provider/path observation.
|
|
77
|
-
crate::state::abnormal_watch::begin_observation(state, &agent.agent_id);
|
|
78
|
-
// Pane/process liveness is independent of transcript content. A
|
|
79
|
-
// frozen rollout is common after pane death, so probe before the
|
|
80
|
-
// metadata dedupe gate; only the expensive tail scan stays deduped.
|
|
81
|
-
let liveness = agent_process_liveness(&agent, session_name, targets, transport);
|
|
82
|
-
let rollout_path = resolve_agent_rollout_path(workspace, &agent.rollout_path);
|
|
83
|
-
let metadata = match std::fs::metadata(&rollout_path) {
|
|
84
|
-
Ok(metadata) => metadata,
|
|
85
|
-
Err(error) => {
|
|
86
|
-
upsert_abnormal_watch(
|
|
87
|
-
state,
|
|
88
|
-
&agent.agent_id,
|
|
89
|
-
abnormal_watch_payload(
|
|
90
|
-
&agent,
|
|
91
|
-
None,
|
|
92
|
-
None,
|
|
93
|
-
process_check(
|
|
94
|
-
ProcessLiveness::Unverifiable,
|
|
95
|
-
"rollout_metadata_unavailable".to_string(),
|
|
96
|
-
),
|
|
97
|
-
None,
|
|
98
|
-
ErrorRecency::None,
|
|
99
|
-
None,
|
|
100
|
-
Some(error.to_string()),
|
|
101
|
-
),
|
|
102
|
-
);
|
|
103
|
-
continue;
|
|
104
|
-
}
|
|
105
|
-
};
|
|
106
|
-
let size = metadata.len();
|
|
107
|
-
let mtime_ns = metadata_mtime_ns(&metadata);
|
|
108
|
-
// P1 (C-P1-2/3): (size, mtime_ns) pair gate — an unchanged transcript is not
|
|
109
|
-
// read at all (live sample: 332MB whole-file read per agent per 2s tick).
|
|
110
|
-
// ANY field change (including a size shrink / truncate) falls through to the
|
|
111
|
-
// re-read below.
|
|
112
|
-
if let (Some(mtime), Some(stored)) = (
|
|
113
|
-
mtime_ns,
|
|
114
|
-
abnormal_watch_stored_metadata(state, &agent.agent_id),
|
|
115
|
-
) {
|
|
116
|
-
if stored == (size, mtime) {
|
|
117
|
-
refresh_abnormal_watch_liveness(state, &agent.agent_id, &liveness);
|
|
118
|
-
continue;
|
|
119
|
-
}
|
|
120
|
-
}
|
|
121
|
-
// P1 (C-P1-1): bounded tail read — the abnormal decision only consumes
|
|
122
|
-
// this tail window. The provider scan walks backward inside it; window
|
|
123
|
-
// matches Python `_TAIL_BYTES` (131072, idle_takeover_wiring.py:13),
|
|
124
|
-
// never less.
|
|
125
|
-
let text = match read_tail_text(&rollout_path, ABNORMAL_TAIL_BYTES) {
|
|
126
|
-
Ok(text) => text,
|
|
127
|
-
Err(error) => {
|
|
128
|
-
upsert_abnormal_watch(
|
|
129
|
-
state,
|
|
130
|
-
&agent.agent_id,
|
|
131
|
-
abnormal_watch_payload(
|
|
132
|
-
&agent,
|
|
133
|
-
Some(size),
|
|
134
|
-
mtime_ns,
|
|
135
|
-
process_check(
|
|
136
|
-
ProcessLiveness::Unverifiable,
|
|
137
|
-
"rollout_read_failed".to_string(),
|
|
138
|
-
),
|
|
139
|
-
None,
|
|
140
|
-
ErrorRecency::None,
|
|
141
|
-
None,
|
|
142
|
-
Some(error.to_string()),
|
|
143
|
-
),
|
|
144
|
-
);
|
|
145
|
-
continue;
|
|
146
|
-
}
|
|
147
|
-
};
|
|
148
|
-
let fact = crate::provider::latest_explicit_error_fact(agent.provider, &text);
|
|
149
|
-
let error_observation_key = fact
|
|
150
|
-
.as_ref()
|
|
151
|
-
.map(|fact| abnormal_error_observation_key(&agent, fact));
|
|
152
|
-
let error_observation_cohort = fact.as_ref().map(|_| abnormal_error_cohort_key(&agent));
|
|
153
|
-
let error_recency = abnormal_error_recency(
|
|
154
|
-
state,
|
|
155
|
-
&agent,
|
|
156
|
-
error_observation_key.as_deref(),
|
|
157
|
-
error_observation_cohort.as_deref(),
|
|
158
|
-
);
|
|
159
|
-
let decision = abnormal_exit_decision(liveness.state, fact.as_ref(), error_recency);
|
|
160
|
-
let check_key = abnormal_check_key(
|
|
161
|
-
&agent,
|
|
162
|
-
&liveness,
|
|
163
|
-
fact.as_ref(),
|
|
164
|
-
error_recency,
|
|
165
|
-
error_observation_key.as_deref(),
|
|
166
|
-
);
|
|
167
|
-
upsert_abnormal_watch(
|
|
168
|
-
state,
|
|
169
|
-
&agent.agent_id,
|
|
170
|
-
abnormal_watch_payload(
|
|
171
|
-
&agent,
|
|
172
|
-
Some(size),
|
|
173
|
-
mtime_ns,
|
|
174
|
-
liveness.clone(),
|
|
175
|
-
fact.as_ref().map(|f| f.signature.as_str()),
|
|
176
|
-
error_recency,
|
|
177
|
-
error_observation_key.as_deref(),
|
|
178
|
-
None,
|
|
179
|
-
),
|
|
180
|
-
);
|
|
181
|
-
if let (Some(observation_key), Some(cohort_key)) = (
|
|
182
|
-
error_observation_key.as_deref(),
|
|
183
|
-
error_observation_cohort.as_deref(),
|
|
184
|
-
) {
|
|
185
|
-
mark_abnormal_error_observed(state, &agent.agent_id, observation_key, cohort_key);
|
|
186
|
-
}
|
|
187
|
-
if abnormal_last_check_key(state, &agent.agent_id).as_deref() != Some(check_key.as_str()) {
|
|
188
|
-
write_abnormal_check(
|
|
189
|
-
event_log,
|
|
190
|
-
&team,
|
|
191
|
-
&agent,
|
|
192
|
-
&liveness,
|
|
193
|
-
fact.as_ref(),
|
|
194
|
-
decision,
|
|
195
|
-
error_recency,
|
|
196
|
-
size,
|
|
197
|
-
mtime_ns,
|
|
198
|
-
)?;
|
|
199
|
-
mark_abnormal_checked(state, &agent.agent_id, &check_key);
|
|
200
|
-
}
|
|
201
|
-
let fact = match (decision, fact) {
|
|
202
|
-
(AbnormalExitDecision::Notify, Some(fact)) => fact,
|
|
203
|
-
(AbnormalExitDecision::Suppress(reason), _) => {
|
|
204
|
-
let suppress_key = abnormal_suppression_key(&agent, &liveness, reason);
|
|
205
|
-
if abnormal_last_suppressed_key(state, &agent.agent_id).as_deref()
|
|
206
|
-
!= Some(suppress_key.as_str())
|
|
207
|
-
{
|
|
208
|
-
write_abnormal_suppressed(event_log, &team, &agent, &liveness, reason)?;
|
|
209
|
-
mark_abnormal_suppressed(state, &agent.agent_id, &suppress_key);
|
|
210
|
-
}
|
|
211
|
-
continue;
|
|
212
|
-
}
|
|
213
|
-
(AbnormalExitDecision::NoSignal, _) => continue,
|
|
214
|
-
(AbnormalExitDecision::Notify, None) => continue,
|
|
215
|
-
};
|
|
216
|
-
let dedupe_key = abnormal_dedupe_key(&agent, &fact);
|
|
217
|
-
if abnormal_last_notified_key(state, &agent.agent_id).as_deref()
|
|
218
|
-
== Some(dedupe_key.as_str())
|
|
219
|
-
{
|
|
220
|
-
continue;
|
|
221
|
-
}
|
|
222
|
-
let manual_recovery_hint =
|
|
223
|
-
recovery_hint(agent.agent_id.as_str(), agent.provider, workspace, team.as_str());
|
|
224
|
-
let content = format_abnormal_exit_message(
|
|
225
|
-
&team,
|
|
226
|
-
&agent,
|
|
227
|
-
&fact,
|
|
228
|
-
&liveness,
|
|
229
|
-
size,
|
|
230
|
-
&manual_recovery_hint,
|
|
231
|
-
);
|
|
232
|
-
let outcome = crate::messaging::send_to_leader_receiver(
|
|
233
|
-
workspace,
|
|
234
|
-
state,
|
|
235
|
-
"leader",
|
|
236
|
-
&content,
|
|
237
|
-
None,
|
|
238
|
-
&agent.agent_id,
|
|
239
|
-
false,
|
|
240
|
-
Some(&dedupe_key),
|
|
241
|
-
event_log,
|
|
242
|
-
)?;
|
|
243
|
-
let notification_status = if outcome.ok {
|
|
244
|
-
"queued"
|
|
245
|
-
} else if matches!(outcome.status, crate::messaging::DeliveryStatus::Blocked) {
|
|
246
|
-
"rebind_required"
|
|
247
|
-
} else {
|
|
248
|
-
"refused"
|
|
249
|
-
};
|
|
250
|
-
let provider_process_dead = provider_process_dead_fact(&liveness);
|
|
251
|
-
event_log.write(
|
|
252
|
-
"worker.abnormal_exit",
|
|
253
|
-
serde_json::json!({
|
|
254
|
-
"team_id": team.as_str(),
|
|
255
|
-
"agent_id": agent.agent_id.as_str(),
|
|
256
|
-
"provider": provider_wire(agent.provider),
|
|
257
|
-
"path": agent.rollout_path_display.as_str(),
|
|
258
|
-
"dead_process": liveness.state == ProcessLiveness::Dead,
|
|
259
|
-
"process_dead": liveness.state == ProcessLiveness::Dead,
|
|
260
|
-
"provider_process_dead": provider_process_dead,
|
|
261
|
-
"latest_error": true,
|
|
262
|
-
"latest_explicit_error": true,
|
|
263
|
-
"error_recency": error_recency.as_str(),
|
|
264
|
-
"fresh_error": error_recency.is_fresh(),
|
|
265
|
-
"dead_process_and_latest_error": liveness.state == ProcessLiveness::Dead,
|
|
266
|
-
"dead_process_and_latest_explicit_error": liveness.state == ProcessLiveness::Dead,
|
|
267
|
-
"process_dead_and_latest_explicit_error": liveness.state == ProcessLiveness::Dead,
|
|
268
|
-
"provider_process_dead_and_latest_explicit_error": provider_process_dead,
|
|
269
|
-
"signature": fact.signature.as_str(),
|
|
270
|
-
"turn_id": fact.turn_id.as_ref().map(|id| id.as_str()),
|
|
271
|
-
"apiErrorStatus": fact.api_error_status,
|
|
272
|
-
"error": fact.error.as_deref(),
|
|
273
|
-
"requestId": fact.request_id.as_deref(),
|
|
274
|
-
"assistant_uuid": fact.assistant_uuid.as_deref(),
|
|
275
|
-
"size": size,
|
|
276
|
-
"mtime_ns": mtime_ns,
|
|
277
|
-
"process_liveness": process_liveness_wire(liveness.state),
|
|
278
|
-
"pid_status": liveness.detail.as_str(),
|
|
279
|
-
"notification_message_id": outcome.message_id,
|
|
280
|
-
"notification_status": notification_status,
|
|
281
|
-
"notification_channel": outcome.channel,
|
|
282
|
-
}),
|
|
283
|
-
)?;
|
|
284
|
-
mark_abnormal_notified(state, &agent.agent_id, &dedupe_key);
|
|
285
|
-
}
|
|
286
|
-
Ok(())
|
|
287
|
-
}
|
|
288
|
-
|
|
289
|
-
fn provider_process_dead_fact(liveness: &ProcessCheck) -> bool {
|
|
290
|
-
liveness.state == ProcessLiveness::Dead
|
|
291
|
-
&& !liveness.detail.starts_with("pane_dead:")
|
|
292
|
-
&& !liveness.detail.starts_with("window_missing:")
|
|
293
|
-
}
|
|
294
|
-
|
|
295
|
-
fn worker_provider_exited_fact(liveness: &ProcessCheck) -> bool {
|
|
296
|
-
provider_process_dead_fact(liveness) && liveness.detail.starts_with("worker_provider_exited:")
|
|
297
|
-
}
|
|
298
|
-
|
|
299
|
-
fn refresh_abnormal_watch_liveness(state: &mut Value, agent_id: &str, liveness: &ProcessCheck) {
|
|
300
|
-
let Some(watch) = coordinator_child_object(state, "abnormal_exit_watch")
|
|
301
|
-
.and_then(|watch| watch.get_mut(agent_id))
|
|
302
|
-
.and_then(Value::as_object_mut)
|
|
303
|
-
else {
|
|
304
|
-
return;
|
|
305
|
-
};
|
|
306
|
-
let dead_process = liveness.state == ProcessLiveness::Dead;
|
|
307
|
-
let provider_process_dead = provider_process_dead_fact(liveness);
|
|
308
|
-
let latest_explicit_error = watch
|
|
309
|
-
.get("latest_explicit_error")
|
|
310
|
-
.and_then(Value::as_bool)
|
|
311
|
-
.unwrap_or(false);
|
|
312
|
-
// Persist fact transitions, not the tick's observation time; otherwise an
|
|
313
|
-
// unchanged transcript makes every steady tick rewrite state.json.
|
|
314
|
-
let patch = serde_json::json!({
|
|
315
|
-
"last_liveness": process_liveness_wire(liveness.state),
|
|
316
|
-
"last_liveness_detail": liveness.detail.as_str(),
|
|
317
|
-
"dead_process": dead_process,
|
|
318
|
-
"process_dead": dead_process,
|
|
319
|
-
"provider_process_dead": provider_process_dead,
|
|
320
|
-
"worker_provider_exited": worker_provider_exited_fact(liveness),
|
|
321
|
-
"provider_process_dead_and_latest_explicit_error": provider_process_dead && latest_explicit_error,
|
|
322
|
-
});
|
|
323
|
-
if let Some(patch) = patch.as_object() {
|
|
324
|
-
if patch
|
|
325
|
-
.iter()
|
|
326
|
-
.any(|(key, value)| watch.get(key) != Some(value))
|
|
327
|
-
{
|
|
328
|
-
watch.extend(patch.clone());
|
|
329
|
-
}
|
|
330
|
-
}
|
|
331
|
-
}
|
|
332
|
-
|
|
333
|
-
#[derive(Debug, Clone)]
|
|
334
|
-
struct AbnormalWatchAgent {
|
|
335
|
-
agent_id: String,
|
|
336
|
-
provider: crate::model::enums::Provider,
|
|
337
|
-
rollout_path: PathBuf,
|
|
338
|
-
rollout_path_display: String,
|
|
339
|
-
spawn_epoch: Option<u64>,
|
|
340
|
-
spawned_at: Option<String>,
|
|
341
|
-
status: Option<String>,
|
|
342
|
-
process_liveness: Option<ProcessLiveness>,
|
|
343
|
-
window: Option<String>,
|
|
344
|
-
pane_id: Option<String>,
|
|
345
|
-
pid: Option<Pid>,
|
|
346
|
-
current_command: Option<String>,
|
|
347
|
-
}
|
|
348
|
-
|
|
349
|
-
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
350
|
-
struct ProcessCheck {
|
|
351
|
-
state: ProcessLiveness,
|
|
352
|
-
detail: String,
|
|
353
|
-
}
|
|
354
|
-
|
|
355
|
-
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
356
|
-
enum AbnormalExitDecision {
|
|
357
|
-
Notify,
|
|
358
|
-
Suppress(&'static str),
|
|
359
|
-
NoSignal,
|
|
360
|
-
}
|
|
361
|
-
|
|
362
|
-
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
363
|
-
enum ErrorRecency {
|
|
364
|
-
None,
|
|
365
|
-
Stale,
|
|
366
|
-
Fresh,
|
|
367
|
-
}
|
|
368
|
-
|
|
369
|
-
impl ErrorRecency {
|
|
370
|
-
fn as_str(self) -> &'static str {
|
|
371
|
-
match self {
|
|
372
|
-
Self::None => "none",
|
|
373
|
-
Self::Stale => "stale",
|
|
374
|
-
Self::Fresh => "fresh",
|
|
375
|
-
}
|
|
376
|
-
}
|
|
377
|
-
|
|
378
|
-
fn is_fresh(self) -> bool {
|
|
379
|
-
matches!(self, Self::Fresh)
|
|
380
|
-
}
|
|
381
|
-
}
|
|
382
|
-
|
|
383
|
-
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
384
|
-
struct AbnormalExitGate {
|
|
385
|
-
provider_process_dead: bool,
|
|
386
|
-
latest_explicit_error: bool,
|
|
387
|
-
error_recency: ErrorRecency,
|
|
388
|
-
}
|
|
389
|
-
|
|
390
|
-
impl AbnormalExitGate {
|
|
391
|
-
fn new(
|
|
392
|
-
process_liveness: ProcessLiveness,
|
|
393
|
-
latest_explicit_error: bool,
|
|
394
|
-
error_recency: ErrorRecency,
|
|
395
|
-
) -> Self {
|
|
396
|
-
Self {
|
|
397
|
-
provider_process_dead: process_liveness == ProcessLiveness::Dead,
|
|
398
|
-
latest_explicit_error,
|
|
399
|
-
error_recency,
|
|
400
|
-
}
|
|
401
|
-
}
|
|
402
|
-
|
|
403
|
-
fn should_notify_worker_abnormal_exit(self) -> bool {
|
|
404
|
-
should_notify_worker_abnormal_exit(self.latest_explicit_error, self.error_recency)
|
|
405
|
-
}
|
|
406
|
-
|
|
407
|
-
fn suppressed_reason(self) -> Option<&'static str> {
|
|
408
|
-
match (self.provider_process_dead, self.latest_explicit_error) {
|
|
409
|
-
(true, false) => Some("dead_only"),
|
|
410
|
-
_ => None,
|
|
411
|
-
}
|
|
412
|
-
}
|
|
413
|
-
}
|
|
414
|
-
|
|
415
|
-
fn abnormal_exit_decision(
|
|
416
|
-
process_liveness: ProcessLiveness,
|
|
417
|
-
latest_explicit_error: Option<&crate::provider::FaultFact>,
|
|
418
|
-
error_recency: ErrorRecency,
|
|
419
|
-
) -> AbnormalExitDecision {
|
|
420
|
-
let gate = AbnormalExitGate::new(
|
|
421
|
-
process_liveness,
|
|
422
|
-
latest_explicit_error.is_some(),
|
|
423
|
-
error_recency,
|
|
424
|
-
);
|
|
425
|
-
if gate.should_notify_worker_abnormal_exit() {
|
|
426
|
-
return AbnormalExitDecision::Notify;
|
|
427
|
-
}
|
|
428
|
-
match gate.suppressed_reason() {
|
|
429
|
-
Some(reason) => AbnormalExitDecision::Suppress(reason),
|
|
430
|
-
None => AbnormalExitDecision::NoSignal,
|
|
431
|
-
}
|
|
432
|
-
}
|
|
433
|
-
|
|
434
|
-
fn should_notify_worker_abnormal_exit(
|
|
435
|
-
latest_explicit_error: bool,
|
|
436
|
-
error_recency: ErrorRecency,
|
|
437
|
-
) -> bool {
|
|
438
|
-
latest_explicit_error && error_recency.is_fresh()
|
|
439
|
-
}
|
|
440
|
-
|
|
441
|
-
fn resolve_agent_rollout_path(workspace: &Path, path: &Path) -> PathBuf {
|
|
442
|
-
if path.is_absolute() {
|
|
443
|
-
path.to_path_buf()
|
|
444
|
-
} else {
|
|
445
|
-
workspace.join(path)
|
|
446
|
-
}
|
|
447
|
-
}
|
|
448
|
-
|
|
449
|
-
fn abnormal_watch_agents(state: &Value) -> Vec<AbnormalWatchAgent> {
|
|
450
|
-
let Some(agents) = state.get("agents").and_then(Value::as_object) else {
|
|
451
|
-
return Vec::new();
|
|
452
|
-
};
|
|
453
|
-
agents
|
|
454
|
-
.iter()
|
|
455
|
-
.filter_map(|(agent_id, agent)| {
|
|
456
|
-
if matches!(agent.get("status").and_then(Value::as_str), Some("paused")) {
|
|
457
|
-
return None;
|
|
458
|
-
}
|
|
459
|
-
let provider = agent
|
|
460
|
-
.get("provider")
|
|
461
|
-
.and_then(Value::as_str)
|
|
462
|
-
.and_then(parse_provider)?;
|
|
463
|
-
let rollout_path_display = ["rollout_path", "transcript_path", "session_log_path"]
|
|
464
|
-
.into_iter()
|
|
465
|
-
.find_map(|key| agent.get(key).and_then(Value::as_str))
|
|
466
|
-
.filter(|path| !path.is_empty())?
|
|
467
|
-
.to_string();
|
|
468
|
-
Some(AbnormalWatchAgent {
|
|
469
|
-
agent_id: agent_id.clone(),
|
|
470
|
-
provider,
|
|
471
|
-
rollout_path: PathBuf::from(&rollout_path_display),
|
|
472
|
-
rollout_path_display,
|
|
473
|
-
spawn_epoch: agent.get("spawn_epoch").and_then(Value::as_u64),
|
|
474
|
-
spawned_at: agent
|
|
475
|
-
.get("spawned_at")
|
|
476
|
-
.and_then(Value::as_str)
|
|
477
|
-
.map(str::to_string),
|
|
478
|
-
status: agent
|
|
479
|
-
.get("status")
|
|
480
|
-
.and_then(Value::as_str)
|
|
481
|
-
.map(str::to_string),
|
|
482
|
-
process_liveness: explicit_process_liveness(agent),
|
|
483
|
-
window: agent
|
|
484
|
-
.get("window")
|
|
485
|
-
.and_then(Value::as_str)
|
|
486
|
-
.map(str::to_string),
|
|
487
|
-
pane_id: agent
|
|
488
|
-
.get("pane_id")
|
|
489
|
-
.and_then(Value::as_str)
|
|
490
|
-
.map(str::to_string),
|
|
491
|
-
pid: agent_pid(agent),
|
|
492
|
-
current_command: agent
|
|
493
|
-
.get("pane_current_command")
|
|
494
|
-
.or_else(|| agent.get("current_command"))
|
|
495
|
-
.and_then(Value::as_str)
|
|
496
|
-
.map(str::to_string),
|
|
497
|
-
})
|
|
498
|
-
})
|
|
499
|
-
.collect()
|
|
500
|
-
}
|
|
501
|
-
|
|
502
|
-
fn agent_pid(agent: &Value) -> Option<Pid> {
|
|
503
|
-
// 0.5.41 Slice 4 (fault-invisibility-locate.md §5 point 3 / §6.4,
|
|
504
|
-
// 0.5.39 CR D-m): after the 0.5.39 worker shell wrapper landed,
|
|
505
|
-
// `pane_pid` is the long-lived shell — NOT the provider. Treating
|
|
506
|
-
// it as `provider pid` and probing it via `kill(0)` reports "alive"
|
|
507
|
-
// for a pane whose provider CLI already exited, which is exactly
|
|
508
|
-
// the false-BUSY that suppressed `worker_provider_exited`. Only
|
|
509
|
-
// pids we know are the provider itself (`provider_pid` /
|
|
510
|
-
// `process_id` / `pid` written by explicit spawn accounting, or
|
|
511
|
-
// `child_pid` from the transport's SpawnResult) count here.
|
|
512
|
-
["provider_pid", "process_id", "pid", "child_pid"]
|
|
513
|
-
.into_iter()
|
|
514
|
-
.find_map(|key| json_u32(agent.get(key)).map(Pid::new))
|
|
515
|
-
}
|
|
516
|
-
|
|
517
|
-
/// ---
|
|
518
|
-
/// purpose: 只从 agent JSON 里已写明的存活字段读出进程存活态,不做任何主动探测
|
|
519
|
-
/// params:
|
|
520
|
-
/// agent: agent 状态对象;先下钻 provider_process / process 子对象,再看本层的 *_liveness 串字段
|
|
521
|
-
/// returns: 命中显式字段时给出 Alive/Dead/Unverifiable;没有任何显式字段时为 None,交调用方走探测路径
|
|
522
|
-
/// ---
|
|
523
|
-
pub(crate) fn explicit_process_liveness(agent: &Value) -> Option<ProcessLiveness> {
|
|
524
|
-
if let Some(process) = agent
|
|
525
|
-
.get("provider_process")
|
|
526
|
-
.or_else(|| agent.get("process"))
|
|
527
|
-
{
|
|
528
|
-
if let Some(liveness) = explicit_process_liveness(process) {
|
|
529
|
-
return Some(liveness);
|
|
530
|
-
}
|
|
531
|
-
}
|
|
532
|
-
for key in [
|
|
533
|
-
"provider_process_liveness",
|
|
534
|
-
"process_liveness",
|
|
535
|
-
"pane_liveness",
|
|
536
|
-
] {
|
|
537
|
-
match agent.get(key).and_then(Value::as_str) {
|
|
538
|
-
Some("dead") => return Some(ProcessLiveness::Dead),
|
|
539
|
-
Some("alive" | "live") => return Some(ProcessLiveness::Alive),
|
|
540
|
-
Some("unverifiable" | "unknown") => return Some(ProcessLiveness::Unverifiable),
|
|
541
|
-
_ => {}
|
|
542
|
-
}
|
|
543
|
-
}
|
|
544
|
-
for key in [
|
|
545
|
-
"provider_process_alive",
|
|
546
|
-
"process_alive",
|
|
547
|
-
"provider_alive",
|
|
548
|
-
"alive",
|
|
549
|
-
] {
|
|
550
|
-
if let Some(alive) = agent.get(key).and_then(Value::as_bool) {
|
|
551
|
-
return Some(if alive {
|
|
552
|
-
ProcessLiveness::Alive
|
|
553
|
-
} else {
|
|
554
|
-
ProcessLiveness::Dead
|
|
555
|
-
});
|
|
556
|
-
}
|
|
557
|
-
}
|
|
558
|
-
for key in [
|
|
559
|
-
"provider_process_dead",
|
|
560
|
-
"process_dead",
|
|
561
|
-
"provider_dead",
|
|
562
|
-
"dead",
|
|
563
|
-
] {
|
|
564
|
-
if let Some(dead) = agent.get(key).and_then(Value::as_bool) {
|
|
565
|
-
return Some(if dead {
|
|
566
|
-
ProcessLiveness::Dead
|
|
567
|
-
} else {
|
|
568
|
-
ProcessLiveness::Alive
|
|
569
|
-
});
|
|
570
|
-
}
|
|
571
|
-
}
|
|
572
|
-
for key in ["status", "state", "liveness"] {
|
|
573
|
-
match agent.get(key).and_then(Value::as_str) {
|
|
574
|
-
Some("dead" | "exited" | "terminated" | "crashed" | "missing") => {
|
|
575
|
-
return Some(ProcessLiveness::Dead);
|
|
576
|
-
}
|
|
577
|
-
Some("alive" | "live" | "running") => return Some(ProcessLiveness::Alive),
|
|
578
|
-
Some("unverifiable" | "unknown") => return Some(ProcessLiveness::Unverifiable),
|
|
579
|
-
_ => {}
|
|
580
|
-
}
|
|
581
|
-
}
|
|
582
|
-
None
|
|
583
|
-
}
|
|
584
|
-
|
|
585
|
-
fn json_u32(value: Option<&Value>) -> Option<u32> {
|
|
586
|
-
value
|
|
587
|
-
.and_then(|v| {
|
|
588
|
-
v.as_u64()
|
|
589
|
-
.or_else(|| v.as_i64().and_then(|n| u64::try_from(n).ok()))
|
|
590
|
-
})
|
|
591
|
-
.and_then(|n| u32::try_from(n).ok())
|
|
592
|
-
}
|
|
593
|
-
|
|
594
|
-
fn agent_process_liveness(
|
|
595
|
-
agent: &AbnormalWatchAgent,
|
|
596
|
-
session_name: Option<&str>,
|
|
597
|
-
targets: &[crate::transport::PaneInfo],
|
|
598
|
-
transport: &dyn crate::transport::Transport,
|
|
599
|
-
) -> ProcessCheck {
|
|
600
|
-
// 0.5.41 Slice 4 (fault-invisibility-locate.md §5 point 3 / §6.4,
|
|
601
|
-
// 0.5.39 CR D-m): after the worker wrapper landed, `pane_pid` may
|
|
602
|
-
// be the long-lived shell that survived provider exit. Reordered
|
|
603
|
-
// to prefer POSITIVE provider evidence in this order:
|
|
604
|
-
// 1. Explicit provider pid (agent.pid) — set by the transport's
|
|
605
|
-
// SpawnResult.child_pid or explicit provider_pid state field.
|
|
606
|
-
// 2. Explicit liveness field.
|
|
607
|
-
// 3. Explicit dead-status field.
|
|
608
|
-
// 4. Explicit provider current-command (state.pane_current_command).
|
|
609
|
-
// 5. Live-pane current-command matching provider = alive.
|
|
610
|
-
// 6. Live-pane current-command shell + capture-tail worker exit
|
|
611
|
-
// marker = DEAD (positive proof provider exited).
|
|
612
|
-
// 7. Live pane, no positive evidence = Unverifiable (NOT alive,
|
|
613
|
-
// NOT working). Legacy code returned pid_process_check(pane_pid)
|
|
614
|
-
// here — that was the D-m false BUSY.
|
|
615
|
-
// 8. Pane / window liveness falls back on Unverifiable.
|
|
616
|
-
if let Some(pid) = agent.pid {
|
|
617
|
-
return pid_process_check("pid", pid);
|
|
618
|
-
}
|
|
619
|
-
// 0.5.41 Slice 4 (fault-invisibility-locate.md §5 point 3 / §6.4):
|
|
620
|
-
// BEFORE consulting the loose `explicit_process_liveness` path
|
|
621
|
-
// (which happily maps state.status=`running` to Alive), probe the
|
|
622
|
-
// pane for POSITIVE provider evidence via current-command + worker
|
|
623
|
-
// exit marker. Under the 0.5.39 wrapper, `state.status=running` is
|
|
624
|
-
// administrative spawn accounting, not runtime truth — a provider
|
|
625
|
-
// may have exited into the inert shell tail while state still reads
|
|
626
|
-
// running. The marker probe positively proves that case; when it
|
|
627
|
-
// fires we return Dead here instead of the false Alive further
|
|
628
|
-
// down.
|
|
629
|
-
if let Some(target) = matching_agent_target(agent, session_name, targets) {
|
|
630
|
-
if let Some(command) = target.current_command.as_deref() {
|
|
631
|
-
return pane_command_process_check_with_marker(agent, transport, target, command);
|
|
632
|
-
}
|
|
633
|
-
}
|
|
634
|
-
if let Some(command) = agent.current_command.as_deref() {
|
|
635
|
-
return command_process_check_with_marker(agent, transport, command);
|
|
636
|
-
}
|
|
637
|
-
if agent.pane_id.is_some() {
|
|
638
|
-
// Even without pane current_command / matching target, try the
|
|
639
|
-
// marker probe directly — the wrapper's printf leaves the marker
|
|
640
|
-
// in the pane's capture tail whether or not the transport
|
|
641
|
-
// reports pane_current_command.
|
|
642
|
-
if let Some(check) = worker_provider_exit_marker_check(agent, transport) {
|
|
643
|
-
return check;
|
|
644
|
-
}
|
|
645
|
-
}
|
|
646
|
-
if let Some(liveness) = agent.process_liveness {
|
|
647
|
-
return process_check(
|
|
648
|
-
liveness,
|
|
649
|
-
format!("explicit:{}", process_liveness_wire(liveness)),
|
|
650
|
-
);
|
|
651
|
-
}
|
|
652
|
-
if agent.status.as_deref().is_some_and(|status| {
|
|
653
|
-
matches!(
|
|
654
|
-
status,
|
|
655
|
-
"stopped" | "missing" | "error" | "dead" | "exited" | "terminated" | "crashed"
|
|
656
|
-
)
|
|
657
|
-
}) {
|
|
658
|
-
return process_check(
|
|
659
|
-
ProcessLiveness::Dead,
|
|
660
|
-
format!("status:{}", agent.status.as_deref().unwrap_or("unknown")),
|
|
661
|
-
);
|
|
662
|
-
}
|
|
663
|
-
if let Some(target) = matching_agent_target(agent, session_name, targets) {
|
|
664
|
-
// Reached only when the target has no current_command AND no
|
|
665
|
-
// marker was found earlier. Legacy code returned
|
|
666
|
-
// `pid_process_check("pane_pid", pane_pid)` here — that's the
|
|
667
|
-
// wrapper shell under 0.5.39. Now Unverifiable (not-working).
|
|
668
|
-
let _ = target;
|
|
669
|
-
return process_check(
|
|
670
|
-
ProcessLiveness::Unverifiable,
|
|
671
|
-
"pane_present_provider_evidence_unknown".to_string(),
|
|
672
|
-
);
|
|
673
|
-
}
|
|
674
|
-
if let Some(pane_id) = agent.pane_id.as_deref() {
|
|
675
|
-
let pane = crate::transport::PaneId::new(pane_id);
|
|
676
|
-
return match transport.liveness(&pane) {
|
|
677
|
-
Ok(crate::transport::PaneLiveness::Dead) => {
|
|
678
|
-
process_check(ProcessLiveness::Dead, format!("pane_dead:{pane_id}"))
|
|
679
|
-
}
|
|
680
|
-
Ok(crate::transport::PaneLiveness::Live) => process_check(
|
|
681
|
-
ProcessLiveness::Unverifiable,
|
|
682
|
-
format!("pane_live_provider_evidence_unknown:{pane_id}"),
|
|
683
|
-
),
|
|
684
|
-
Ok(crate::transport::PaneLiveness::Unknown) => process_check(
|
|
685
|
-
ProcessLiveness::Unverifiable,
|
|
686
|
-
format!("pane_unknown:{pane_id}"),
|
|
687
|
-
),
|
|
688
|
-
Err(error) => process_check(
|
|
689
|
-
ProcessLiveness::Unverifiable,
|
|
690
|
-
format!("pane_unverifiable:{pane_id}:{error}"),
|
|
691
|
-
),
|
|
692
|
-
};
|
|
693
|
-
}
|
|
694
|
-
let (Some(session), Some(window)) = (session_name, agent.window.as_deref()) else {
|
|
695
|
-
return process_check(
|
|
696
|
-
ProcessLiveness::Unverifiable,
|
|
697
|
-
"missing_session_or_window".to_string(),
|
|
698
|
-
);
|
|
699
|
-
};
|
|
700
|
-
let session = crate::transport::SessionName::new(session);
|
|
701
|
-
match transport.list_windows(&session) {
|
|
702
|
-
Ok(windows) if windows.iter().any(|known| known.as_str() == window) => process_check(
|
|
703
|
-
ProcessLiveness::Unverifiable,
|
|
704
|
-
"window_present_provider_evidence_unknown".to_string(),
|
|
705
|
-
),
|
|
706
|
-
Ok(_) => process_check(ProcessLiveness::Dead, format!("window_missing:{window}")),
|
|
707
|
-
Err(error) => process_check(
|
|
708
|
-
ProcessLiveness::Unverifiable,
|
|
709
|
-
format!("window_unverifiable:{window}:{error}"),
|
|
710
|
-
),
|
|
711
|
-
}
|
|
712
|
-
}
|
|
713
|
-
|
|
714
|
-
fn matching_agent_target<'a>(
|
|
715
|
-
agent: &AbnormalWatchAgent,
|
|
716
|
-
session_name: Option<&str>,
|
|
717
|
-
targets: &'a [crate::transport::PaneInfo],
|
|
718
|
-
) -> Option<&'a crate::transport::PaneInfo> {
|
|
719
|
-
if let Some(pane_id) = agent.pane_id.as_deref() {
|
|
720
|
-
if let Some(target) = targets
|
|
721
|
-
.iter()
|
|
722
|
-
.find(|target| target.pane_id.as_str() == pane_id)
|
|
723
|
-
{
|
|
724
|
-
return Some(target);
|
|
725
|
-
}
|
|
726
|
-
}
|
|
727
|
-
let (Some(session), Some(window)) = (session_name, agent.window.as_deref()) else {
|
|
728
|
-
return None;
|
|
729
|
-
};
|
|
730
|
-
targets.iter().find(|target| {
|
|
731
|
-
target.session.as_str() == session
|
|
732
|
-
&& target
|
|
733
|
-
.window_name
|
|
734
|
-
.as_ref()
|
|
735
|
-
.is_some_and(|known| known.as_str() == window)
|
|
736
|
-
})
|
|
737
|
-
}
|
|
738
|
-
|
|
739
|
-
fn pid_process_check(label: &str, pid: Pid) -> ProcessCheck {
|
|
740
|
-
match pid_is_running(pid) {
|
|
741
|
-
Ok(true) => process_check(ProcessLiveness::Alive, format!("{label}_running:{pid}")),
|
|
742
|
-
Ok(false) => process_check(ProcessLiveness::Dead, format!("{label}_not_running:{pid}")),
|
|
743
|
-
Err(error) => process_check(
|
|
744
|
-
ProcessLiveness::Unverifiable,
|
|
745
|
-
format!("{label}_unverifiable:{pid}:{error}"),
|
|
746
|
-
),
|
|
747
|
-
}
|
|
748
|
-
}
|
|
749
|
-
|
|
750
|
-
fn command_process_check(provider: crate::model::enums::Provider, command: &str) -> ProcessCheck {
|
|
751
|
-
if crate::leader::command_matches_provider(provider, command) {
|
|
752
|
-
process_check(ProcessLiveness::Alive, format!("current_command:{command}"))
|
|
753
|
-
} else {
|
|
754
|
-
process_check(
|
|
755
|
-
ProcessLiveness::Dead,
|
|
756
|
-
format!("provider_not_foreground:{command}"),
|
|
757
|
-
)
|
|
758
|
-
}
|
|
759
|
-
}
|
|
760
|
-
|
|
761
|
-
fn pane_command_process_check(
|
|
762
|
-
provider: crate::model::enums::Provider,
|
|
763
|
-
pane: &crate::transport::PaneInfo,
|
|
764
|
-
command: &str,
|
|
765
|
-
) -> ProcessCheck {
|
|
766
|
-
if crate::leader::attribute_pane_provider(pane)
|
|
767
|
-
.is_some_and(|candidate| crate::leader::provider_matches(candidate, provider))
|
|
768
|
-
{
|
|
769
|
-
process_check(ProcessLiveness::Alive, format!("current_command:{command}"))
|
|
770
|
-
} else {
|
|
771
|
-
process_check(
|
|
772
|
-
ProcessLiveness::Dead,
|
|
773
|
-
format!("provider_not_foreground:{command}"),
|
|
774
|
-
)
|
|
775
|
-
}
|
|
776
|
-
}
|
|
777
|
-
|
|
778
|
-
/// 0.5.41 Slice 4 (fault-invisibility-locate.md §5 point 3 / §6.4):
|
|
779
|
-
/// after the 0.5.39 worker wrapper landed, a pane whose current command
|
|
780
|
-
/// is a shell + capture-tail contains the worker exit marker is POSITIVE
|
|
781
|
-
/// proof the provider exited. Falls back to plain current-command check
|
|
782
|
-
/// when a pane_id is not available for capture (state-only path).
|
|
783
|
-
fn command_process_check_with_marker(
|
|
784
|
-
agent: &AbnormalWatchAgent,
|
|
785
|
-
transport: &dyn crate::transport::Transport,
|
|
786
|
-
command: &str,
|
|
787
|
-
) -> ProcessCheck {
|
|
788
|
-
if crate::leader::command_matches_provider(agent.provider, command) {
|
|
789
|
-
return process_check(ProcessLiveness::Alive, format!("current_command:{command}"));
|
|
790
|
-
}
|
|
791
|
-
if let Some(check) = worker_provider_exit_marker_check(agent, transport) {
|
|
792
|
-
return check;
|
|
793
|
-
}
|
|
794
|
-
process_check(
|
|
795
|
-
ProcessLiveness::Unverifiable,
|
|
796
|
-
format!("provider_evidence_unverifiable:{command}"),
|
|
797
|
-
)
|
|
798
|
-
}
|
|
799
|
-
|
|
800
|
-
fn pane_command_process_check_with_marker(
|
|
801
|
-
agent: &AbnormalWatchAgent,
|
|
802
|
-
transport: &dyn crate::transport::Transport,
|
|
803
|
-
pane: &crate::transport::PaneInfo,
|
|
804
|
-
command: &str,
|
|
805
|
-
) -> ProcessCheck {
|
|
806
|
-
if crate::leader::attribute_pane_provider(pane)
|
|
807
|
-
.is_some_and(|candidate| crate::leader::provider_matches(candidate, agent.provider))
|
|
808
|
-
{
|
|
809
|
-
return process_check(ProcessLiveness::Alive, format!("current_command:{command}"));
|
|
810
|
-
}
|
|
811
|
-
if let Some(check) = worker_provider_exit_marker_check(agent, transport) {
|
|
812
|
-
return check;
|
|
813
|
-
}
|
|
814
|
-
process_check(
|
|
815
|
-
ProcessLiveness::Unverifiable,
|
|
816
|
-
format!("provider_evidence_unverifiable:{command}"),
|
|
817
|
-
)
|
|
818
|
-
}
|
|
819
|
-
|
|
820
|
-
/// 0.5.41 Slice 4: probe the pane's capture-tail for the single-source
|
|
821
|
-
/// worker exit marker (`tmux_backend::worker_provider_exit_marker`).
|
|
822
|
-
/// Returns `Some(Dead)` when the marker is found (positive proof
|
|
823
|
-
/// provider exited), or `None` when we can't run the capture (no
|
|
824
|
-
/// pane_id in agent), so callers keep their default decision. Absence
|
|
825
|
-
/// of marker in a valid capture is intentionally NOT `alive` (locate
|
|
826
|
-
/// §8 risk: capture tail can scroll past the marker) — callers still
|
|
827
|
-
/// return `Dead` for "provider not foreground" because current command
|
|
828
|
-
/// already disproved provider liveness at that call site.
|
|
829
|
-
fn worker_provider_exit_marker_check(
|
|
830
|
-
agent: &AbnormalWatchAgent,
|
|
831
|
-
transport: &dyn crate::transport::Transport,
|
|
832
|
-
) -> Option<ProcessCheck> {
|
|
833
|
-
let pane_id_str = agent.pane_id.as_deref()?;
|
|
834
|
-
let pane = crate::transport::PaneId::new(pane_id_str);
|
|
835
|
-
let target = crate::transport::Target::Pane(pane);
|
|
836
|
-
let provider_label = crate::provider::wire::command_name(agent.provider);
|
|
837
|
-
let marker = crate::tmux_backend::worker_provider_exit_marker(provider_label);
|
|
838
|
-
let cap = transport
|
|
839
|
-
.capture(&target, crate::transport::CaptureRange::Tail(200))
|
|
840
|
-
.ok()?;
|
|
841
|
-
if cap.text.contains(&marker) {
|
|
842
|
-
// The wrapper writes its exit code immediately after this marker. A
|
|
843
|
-
// truncated capture can leave the marker without a complete token;
|
|
844
|
-
// keep that case explicit rather than guessing success (or zero).
|
|
845
|
-
// Scope: this detail only covers exits that execute the wrapper's
|
|
846
|
-
// marker printf. SIGKILL, direct pane termination, and host power loss
|
|
847
|
-
// produce no marker; that absence is silent, not `rc=unknown`.
|
|
848
|
-
let rc = cap
|
|
849
|
-
.text
|
|
850
|
-
.split_once(&marker)
|
|
851
|
-
.and_then(|(_, tail)| {
|
|
852
|
-
let token = tail.split_whitespace().next()?;
|
|
853
|
-
let end = tail.find(token)? + token.len();
|
|
854
|
-
tail.as_bytes()
|
|
855
|
-
.get(end)
|
|
856
|
-
.is_some_and(u8::is_ascii_whitespace)
|
|
857
|
-
.then_some(token)
|
|
858
|
-
})
|
|
859
|
-
.and_then(|token| token.parse::<i32>().ok())
|
|
860
|
-
.map(|value| value.to_string())
|
|
861
|
-
.unwrap_or_else(|| "unknown".to_string());
|
|
862
|
-
Some(process_check(
|
|
863
|
-
ProcessLiveness::Dead,
|
|
864
|
-
format!("worker_provider_exited:{pane_id_str}:rc={rc}"),
|
|
865
|
-
))
|
|
866
|
-
} else {
|
|
867
|
-
None
|
|
868
|
-
}
|
|
869
|
-
}
|
|
870
|
-
|
|
871
|
-
fn process_check(state: ProcessLiveness, detail: String) -> ProcessCheck {
|
|
872
|
-
ProcessCheck { state, detail }
|
|
873
|
-
}
|
|
874
|
-
|
|
875
|
-
fn process_liveness_wire(state: ProcessLiveness) -> &'static str {
|
|
876
|
-
match state {
|
|
877
|
-
ProcessLiveness::Alive => "alive",
|
|
878
|
-
ProcessLiveness::Dead => "dead",
|
|
879
|
-
ProcessLiveness::Unverifiable => "unverifiable",
|
|
880
|
-
}
|
|
881
|
-
}
|
|
882
|
-
|
|
883
|
-
/// ---
|
|
884
|
-
/// purpose: 把文件修改时间归一成自 UNIX 纪元起的纳秒整数,供跨 tick 比较「rollout 有没有变过」
|
|
885
|
-
/// params:
|
|
886
|
-
/// metadata: 目标文件的 metadata
|
|
887
|
-
/// returns: 纳秒时间戳;取不到 mtime 或早于纪元时为 None。乘加均为 saturating,不会回绕
|
|
888
|
-
/// ---
|
|
889
|
-
pub(crate) fn metadata_mtime_ns(metadata: &std::fs::Metadata) -> Option<u64> {
|
|
890
|
-
let duration = metadata
|
|
891
|
-
.modified()
|
|
892
|
-
.ok()?
|
|
893
|
-
.duration_since(std::time::UNIX_EPOCH)
|
|
894
|
-
.ok()?;
|
|
895
|
-
Some(
|
|
896
|
-
duration
|
|
897
|
-
.as_secs()
|
|
898
|
-
.saturating_mul(1_000_000_000)
|
|
899
|
-
.saturating_add(u64::from(duration.subsec_nanos())),
|
|
900
|
-
)
|
|
901
|
-
}
|
|
902
|
-
|
|
903
|
-
fn abnormal_watch_payload(
|
|
904
|
-
agent: &AbnormalWatchAgent,
|
|
905
|
-
size: Option<u64>,
|
|
906
|
-
mtime_ns: Option<u64>,
|
|
907
|
-
liveness: ProcessCheck,
|
|
908
|
-
signature: Option<&str>,
|
|
909
|
-
error_recency: ErrorRecency,
|
|
910
|
-
error_observation_key: Option<&str>,
|
|
911
|
-
error: Option<String>,
|
|
912
|
-
) -> Value {
|
|
913
|
-
let liveness_wire = process_liveness_wire(liveness.state);
|
|
914
|
-
let dead_process = liveness.state == ProcessLiveness::Dead;
|
|
915
|
-
let latest_explicit_error = signature.is_some();
|
|
916
|
-
let gate = AbnormalExitGate::new(liveness.state, latest_explicit_error, error_recency);
|
|
917
|
-
let notify = gate.should_notify_worker_abnormal_exit();
|
|
918
|
-
let suppressed_reason = gate.suppressed_reason();
|
|
919
|
-
// 0.5.41 Slice 4 (fault-invisibility-locate.md §6.4): typed
|
|
920
|
-
// classification for status/render — `worker_provider_exited` is
|
|
921
|
-
// true only when Dead was proven via the capture-tail worker exit
|
|
922
|
-
// marker (detail prefix `worker_provider_exited:`). status_port's
|
|
923
|
-
// RuntimeFreshness collector reads this field to downgrade the
|
|
924
|
-
// corresponding agent row.
|
|
925
|
-
let provider_process_dead = provider_process_dead_fact(&liveness);
|
|
926
|
-
let worker_provider_exited = worker_provider_exited_fact(&liveness);
|
|
927
|
-
serde_json::json!({
|
|
928
|
-
"path": agent.rollout_path_display.as_str(),
|
|
929
|
-
"provider": provider_wire(agent.provider),
|
|
930
|
-
"mtime_ns": mtime_ns,
|
|
931
|
-
"size": size,
|
|
932
|
-
"last_offset": size,
|
|
933
|
-
"last_signature": signature,
|
|
934
|
-
"last_liveness": liveness_wire,
|
|
935
|
-
"last_liveness_detail": liveness.detail,
|
|
936
|
-
"dead_process": dead_process,
|
|
937
|
-
"process_dead": dead_process,
|
|
938
|
-
"provider_process_dead": provider_process_dead,
|
|
939
|
-
"worker_provider_exited": worker_provider_exited,
|
|
940
|
-
"latest_error": latest_explicit_error,
|
|
941
|
-
"latest_explicit_error": latest_explicit_error,
|
|
942
|
-
"error_recency": error_recency.as_str(),
|
|
943
|
-
"fresh_error": error_recency.is_fresh(),
|
|
944
|
-
"error_observation_key": error_observation_key,
|
|
945
|
-
"dead_process_and_latest_error": dead_process && latest_explicit_error,
|
|
946
|
-
"dead_process_and_latest_explicit_error": dead_process && latest_explicit_error,
|
|
947
|
-
"process_dead_and_latest_explicit_error": dead_process && latest_explicit_error,
|
|
948
|
-
"provider_process_dead_and_latest_explicit_error": provider_process_dead && latest_explicit_error,
|
|
949
|
-
"suppressed_reason": suppressed_reason,
|
|
950
|
-
"notification": notify,
|
|
951
|
-
"last_error": error,
|
|
952
|
-
})
|
|
953
|
-
}
|
|
954
|
-
|
|
955
|
-
fn upsert_abnormal_watch(state: &mut Value, agent_id: &str, mut payload: Value) {
|
|
956
|
-
if let Some(identity) = crate::state::abnormal_watch::observation_identity(state, agent_id) {
|
|
957
|
-
payload["watch_identity"] = identity;
|
|
958
|
-
}
|
|
959
|
-
let preserved = [
|
|
960
|
-
"last_notified_key",
|
|
961
|
-
"last_notified_at",
|
|
962
|
-
"last_suppressed_key",
|
|
963
|
-
"last_suppressed_at",
|
|
964
|
-
"last_check_key",
|
|
965
|
-
"last_check_at",
|
|
966
|
-
"last_error_observation_key",
|
|
967
|
-
"last_error_observation_cohort",
|
|
968
|
-
"last_error_observed_at",
|
|
969
|
-
]
|
|
970
|
-
.into_iter()
|
|
971
|
-
.filter_map(|key| abnormal_watch_field(state, agent_id, key).map(|value| (key, value)))
|
|
972
|
-
.collect::<Vec<_>>();
|
|
973
|
-
if let Some(watch) = coordinator_child_object(state, "abnormal_exit_watch") {
|
|
974
|
-
if let Some(payload_obj) = payload.as_object_mut() {
|
|
975
|
-
for (key, value) in preserved {
|
|
976
|
-
payload_obj.insert(key.to_string(), value);
|
|
977
|
-
}
|
|
978
|
-
}
|
|
979
|
-
watch.insert(agent_id.to_string(), payload);
|
|
980
|
-
}
|
|
981
|
-
}
|
|
982
|
-
|
|
983
|
-
fn coordinator_child_object<'a>(
|
|
984
|
-
state: &'a mut Value,
|
|
985
|
-
key: &str,
|
|
986
|
-
) -> Option<&'a mut serde_json::Map<String, Value>> {
|
|
987
|
-
if !state.is_object() {
|
|
988
|
-
*state = serde_json::json!({});
|
|
989
|
-
}
|
|
990
|
-
let state_obj = state.as_object_mut()?;
|
|
991
|
-
let coordinator = state_obj
|
|
992
|
-
.entry("coordinator".to_string())
|
|
993
|
-
.or_insert_with(|| serde_json::json!({}));
|
|
994
|
-
if !coordinator.is_object() {
|
|
995
|
-
*coordinator = serde_json::json!({});
|
|
996
|
-
}
|
|
997
|
-
let coord_obj = coordinator.as_object_mut()?;
|
|
998
|
-
let child = coord_obj
|
|
999
|
-
.entry(key.to_string())
|
|
1000
|
-
.or_insert_with(|| serde_json::json!({}));
|
|
1001
|
-
if !child.is_object() {
|
|
1002
|
-
*child = serde_json::json!({});
|
|
1003
|
-
}
|
|
1004
|
-
child.as_object_mut()
|
|
1005
|
-
}
|
|
1006
|
-
|
|
1007
|
-
fn abnormal_last_notified_key(state: &Value, agent_id: &str) -> Option<String> {
|
|
1008
|
-
abnormal_watch_str(state, agent_id, "last_notified_key")
|
|
1009
|
-
}
|
|
1010
|
-
|
|
1011
|
-
fn abnormal_last_suppressed_key(state: &Value, agent_id: &str) -> Option<String> {
|
|
1012
|
-
abnormal_watch_str(state, agent_id, "last_suppressed_key")
|
|
1013
|
-
}
|
|
1014
|
-
|
|
1015
|
-
fn abnormal_last_check_key(state: &Value, agent_id: &str) -> Option<String> {
|
|
1016
|
-
abnormal_watch_str(state, agent_id, "last_check_key")
|
|
1017
|
-
}
|
|
1018
|
-
|
|
1019
|
-
fn abnormal_last_error_observation_key(state: &Value, agent_id: &str) -> Option<String> {
|
|
1020
|
-
abnormal_watch_str(state, agent_id, "last_error_observation_key")
|
|
1021
|
-
}
|
|
1022
|
-
|
|
1023
|
-
fn abnormal_last_error_observation_cohort(state: &Value, agent_id: &str) -> Option<String> {
|
|
1024
|
-
abnormal_watch_str(state, agent_id, "last_error_observation_cohort")
|
|
1025
|
-
}
|
|
1026
|
-
|
|
1027
|
-
/// P1: Python `_TAIL_BYTES` parity (idle_takeover_wiring.py:13) — RS must not read less.
|
|
1028
|
-
const ABNORMAL_TAIL_BYTES: u64 = 131_072;
|
|
1029
|
-
|
|
1030
|
-
///
|
|
1031
|
-
/// P1: bounded tail read; a partial first line is harmless (the consumer only parses
|
|
1032
|
-
/// the latest complete JSONL record) and lossy UTF-8 keeps a mid-codepoint seek safe.
|
|
1033
|
-
/// ---
|
|
1034
|
-
/// purpose: 有界读取文件尾部文本,避免整份 rollout 进内存
|
|
1035
|
-
/// params:
|
|
1036
|
-
/// path: 目标文件
|
|
1037
|
-
/// max_bytes: 最多回读的字节数;文件更短时整份读出
|
|
1038
|
-
/// returns: 尾部文本。按字节 seek,首行可能被截断,且用有损 UTF-8 解码,切在码点中间也安全
|
|
1039
|
-
/// errors: 打开、取长度或读取失败时返回底层 io::Error
|
|
1040
|
-
/// ---
|
|
1041
|
-
pub(crate) fn read_tail_text(path: &Path, max_bytes: u64) -> std::io::Result<String> {
|
|
1042
|
-
use std::io::{Read, Seek, SeekFrom};
|
|
1043
|
-
let mut file = std::fs::File::open(path)?;
|
|
1044
|
-
let len = file.metadata()?.len();
|
|
1045
|
-
if len > max_bytes {
|
|
1046
|
-
file.seek(SeekFrom::Start(len - max_bytes))?;
|
|
1047
|
-
}
|
|
1048
|
-
let mut bytes = Vec::new();
|
|
1049
|
-
file.read_to_end(&mut bytes)?;
|
|
1050
|
-
Ok(String::from_utf8_lossy(&bytes).into_owned())
|
|
1051
|
-
}
|
|
1052
|
-
|
|
1053
|
-
/// P1: the previous tick's `(size, mtime_ns)` pair from the abnormal watch payload.
|
|
1054
|
-
fn abnormal_watch_stored_metadata(state: &Value, agent_id: &str) -> Option<(u64, u64)> {
|
|
1055
|
-
let watch = state
|
|
1056
|
-
.get("coordinator")?
|
|
1057
|
-
.get("abnormal_exit_watch")?
|
|
1058
|
-
.get(agent_id)?;
|
|
1059
|
-
Some((
|
|
1060
|
-
watch.get("size")?.as_u64()?,
|
|
1061
|
-
watch.get("mtime_ns")?.as_u64()?,
|
|
1062
|
-
))
|
|
1063
|
-
}
|
|
1064
|
-
|
|
1065
|
-
fn abnormal_watch_str(state: &Value, agent_id: &str, field: &str) -> Option<String> {
|
|
1066
|
-
state
|
|
1067
|
-
.get("coordinator")
|
|
1068
|
-
.and_then(|v| v.get("abnormal_exit_watch"))
|
|
1069
|
-
.and_then(|v| v.get(agent_id))
|
|
1070
|
-
.and_then(|v| v.get(field))
|
|
1071
|
-
.and_then(Value::as_str)
|
|
1072
|
-
.map(str::to_string)
|
|
1073
|
-
}
|
|
1074
|
-
|
|
1075
|
-
fn abnormal_watch_field(state: &Value, agent_id: &str, field: &str) -> Option<Value> {
|
|
1076
|
-
state
|
|
1077
|
-
.get("coordinator")
|
|
1078
|
-
.and_then(|v| v.get("abnormal_exit_watch"))
|
|
1079
|
-
.and_then(|v| v.get(agent_id))
|
|
1080
|
-
.and_then(|v| v.get(field))
|
|
1081
|
-
.cloned()
|
|
1082
|
-
}
|
|
1083
|
-
|
|
1084
|
-
fn mark_abnormal_notified(state: &mut Value, agent_id: &str, key: &str) {
|
|
1085
|
-
if let Some(watch) = coordinator_child_object(state, "abnormal_exit_watch") {
|
|
1086
|
-
let entry = watch
|
|
1087
|
-
.entry(agent_id.to_string())
|
|
1088
|
-
.or_insert_with(|| serde_json::json!({}));
|
|
1089
|
-
if !entry.is_object() {
|
|
1090
|
-
*entry = serde_json::json!({});
|
|
1091
|
-
}
|
|
1092
|
-
if let Some(obj) = entry.as_object_mut() {
|
|
1093
|
-
obj.insert("last_notified_key".to_string(), serde_json::json!(key));
|
|
1094
|
-
obj.insert(
|
|
1095
|
-
"last_notified_at".to_string(),
|
|
1096
|
-
serde_json::json!(chrono::Utc::now().to_rfc3339()),
|
|
1097
|
-
);
|
|
1098
|
-
}
|
|
1099
|
-
}
|
|
1100
|
-
}
|
|
1101
|
-
|
|
1102
|
-
fn mark_abnormal_suppressed(state: &mut Value, agent_id: &str, key: &str) {
|
|
1103
|
-
if let Some(watch) = coordinator_child_object(state, "abnormal_exit_watch") {
|
|
1104
|
-
let entry = watch
|
|
1105
|
-
.entry(agent_id.to_string())
|
|
1106
|
-
.or_insert_with(|| serde_json::json!({}));
|
|
1107
|
-
if !entry.is_object() {
|
|
1108
|
-
*entry = serde_json::json!({});
|
|
1109
|
-
}
|
|
1110
|
-
if let Some(obj) = entry.as_object_mut() {
|
|
1111
|
-
obj.insert("last_suppressed_key".to_string(), serde_json::json!(key));
|
|
1112
|
-
obj.insert(
|
|
1113
|
-
"last_suppressed_at".to_string(),
|
|
1114
|
-
serde_json::json!(chrono::Utc::now().to_rfc3339()),
|
|
1115
|
-
);
|
|
1116
|
-
}
|
|
1117
|
-
}
|
|
1118
|
-
}
|
|
1119
|
-
|
|
1120
|
-
fn mark_abnormal_checked(state: &mut Value, agent_id: &str, key: &str) {
|
|
1121
|
-
if let Some(watch) = coordinator_child_object(state, "abnormal_exit_watch") {
|
|
1122
|
-
let entry = watch
|
|
1123
|
-
.entry(agent_id.to_string())
|
|
1124
|
-
.or_insert_with(|| serde_json::json!({}));
|
|
1125
|
-
if !entry.is_object() {
|
|
1126
|
-
*entry = serde_json::json!({});
|
|
1127
|
-
}
|
|
1128
|
-
if let Some(obj) = entry.as_object_mut() {
|
|
1129
|
-
obj.insert("last_check_key".to_string(), serde_json::json!(key));
|
|
1130
|
-
obj.insert(
|
|
1131
|
-
"last_check_at".to_string(),
|
|
1132
|
-
serde_json::json!(chrono::Utc::now().to_rfc3339()),
|
|
1133
|
-
);
|
|
1134
|
-
}
|
|
1135
|
-
}
|
|
1136
|
-
}
|
|
1137
|
-
|
|
1138
|
-
fn mark_abnormal_error_observed(
|
|
1139
|
-
state: &mut Value,
|
|
1140
|
-
agent_id: &str,
|
|
1141
|
-
observation_key: &str,
|
|
1142
|
-
cohort_key: &str,
|
|
1143
|
-
) {
|
|
1144
|
-
if let Some(watch) = coordinator_child_object(state, "abnormal_exit_watch") {
|
|
1145
|
-
let entry = watch
|
|
1146
|
-
.entry(agent_id.to_string())
|
|
1147
|
-
.or_insert_with(|| serde_json::json!({}));
|
|
1148
|
-
if !entry.is_object() {
|
|
1149
|
-
*entry = serde_json::json!({});
|
|
1150
|
-
}
|
|
1151
|
-
if let Some(obj) = entry.as_object_mut() {
|
|
1152
|
-
obj.insert(
|
|
1153
|
-
"last_error_observation_key".to_string(),
|
|
1154
|
-
serde_json::json!(observation_key),
|
|
1155
|
-
);
|
|
1156
|
-
obj.insert(
|
|
1157
|
-
"last_error_observation_cohort".to_string(),
|
|
1158
|
-
serde_json::json!(cohort_key),
|
|
1159
|
-
);
|
|
1160
|
-
obj.insert(
|
|
1161
|
-
"last_error_observed_at".to_string(),
|
|
1162
|
-
serde_json::json!(chrono::Utc::now().to_rfc3339()),
|
|
1163
|
-
);
|
|
1164
|
-
}
|
|
1165
|
-
}
|
|
1166
|
-
}
|
|
1167
|
-
|
|
1168
|
-
fn write_abnormal_check(
|
|
1169
|
-
event_log: &EventLog,
|
|
1170
|
-
team: &str,
|
|
1171
|
-
agent: &AbnormalWatchAgent,
|
|
1172
|
-
liveness: &ProcessCheck,
|
|
1173
|
-
fact: Option<&crate::provider::FaultFact>,
|
|
1174
|
-
decision: AbnormalExitDecision,
|
|
1175
|
-
error_recency: ErrorRecency,
|
|
1176
|
-
size: u64,
|
|
1177
|
-
mtime_ns: Option<u64>,
|
|
1178
|
-
) -> Result<(), TickError> {
|
|
1179
|
-
let dead_process = liveness.state == ProcessLiveness::Dead;
|
|
1180
|
-
let provider_process_dead = provider_process_dead_fact(liveness);
|
|
1181
|
-
let latest_explicit_error = fact.is_some();
|
|
1182
|
-
event_log.write(
|
|
1183
|
-
"worker.abnormal_exit.check",
|
|
1184
|
-
serde_json::json!({
|
|
1185
|
-
"team_id": team,
|
|
1186
|
-
"agent_id": agent.agent_id.as_str(),
|
|
1187
|
-
"provider": provider_wire(agent.provider),
|
|
1188
|
-
"path": agent.rollout_path_display.as_str(),
|
|
1189
|
-
"size": size,
|
|
1190
|
-
"last_offset": size,
|
|
1191
|
-
"mtime_ns": mtime_ns,
|
|
1192
|
-
"dead_process": dead_process,
|
|
1193
|
-
"process_dead": dead_process,
|
|
1194
|
-
"provider_process_dead": provider_process_dead,
|
|
1195
|
-
"latest_error": latest_explicit_error,
|
|
1196
|
-
"latest_explicit_error": latest_explicit_error,
|
|
1197
|
-
"error_recency": error_recency.as_str(),
|
|
1198
|
-
"fresh_error": error_recency.is_fresh(),
|
|
1199
|
-
"dead_process_and_latest_error": dead_process && latest_explicit_error,
|
|
1200
|
-
"dead_process_and_latest_explicit_error": dead_process && latest_explicit_error,
|
|
1201
|
-
"process_dead_and_latest_explicit_error": dead_process && latest_explicit_error,
|
|
1202
|
-
"provider_process_dead_and_latest_explicit_error": provider_process_dead && latest_explicit_error,
|
|
1203
|
-
"notification": matches!(decision, AbnormalExitDecision::Notify),
|
|
1204
|
-
"suppressed_reason": match decision {
|
|
1205
|
-
AbnormalExitDecision::Suppress(reason) => Some(reason),
|
|
1206
|
-
AbnormalExitDecision::Notify | AbnormalExitDecision::NoSignal => None,
|
|
1207
|
-
},
|
|
1208
|
-
"signature": fact.map(|fact| fact.signature.as_str()),
|
|
1209
|
-
"turn_id": fact.and_then(|fact| fact.turn_id.as_ref().map(|id| id.as_str())),
|
|
1210
|
-
"apiErrorStatus": fact.and_then(|fact| fact.api_error_status),
|
|
1211
|
-
"error": fact.and_then(|fact| fact.error.as_deref()),
|
|
1212
|
-
"requestId": fact.and_then(|fact| fact.request_id.as_deref()),
|
|
1213
|
-
"assistant_uuid": fact.and_then(|fact| fact.assistant_uuid.as_deref()),
|
|
1214
|
-
"process_liveness": process_liveness_wire(liveness.state),
|
|
1215
|
-
"pid_status": liveness.detail.as_str(),
|
|
1216
|
-
}),
|
|
1217
|
-
)?;
|
|
1218
|
-
Ok(())
|
|
1219
|
-
}
|
|
1220
|
-
|
|
1221
|
-
fn write_abnormal_suppressed(
|
|
1222
|
-
event_log: &EventLog,
|
|
1223
|
-
team: &str,
|
|
1224
|
-
agent: &AbnormalWatchAgent,
|
|
1225
|
-
liveness: &ProcessCheck,
|
|
1226
|
-
reason: &str,
|
|
1227
|
-
) -> Result<(), TickError> {
|
|
1228
|
-
let provider_process_dead = provider_process_dead_fact(liveness);
|
|
1229
|
-
event_log.write(
|
|
1230
|
-
"abnormal_exit.single_signal_suppressed",
|
|
1231
|
-
serde_json::json!({
|
|
1232
|
-
"team_id": team,
|
|
1233
|
-
"agent_id": agent.agent_id.as_str(),
|
|
1234
|
-
"provider": provider_wire(agent.provider),
|
|
1235
|
-
"path": agent.rollout_path_display.as_str(),
|
|
1236
|
-
"reason": reason,
|
|
1237
|
-
"notification": false,
|
|
1238
|
-
"dead_process": liveness.state == ProcessLiveness::Dead,
|
|
1239
|
-
"process_dead": liveness.state == ProcessLiveness::Dead,
|
|
1240
|
-
"provider_process_dead": provider_process_dead,
|
|
1241
|
-
"latest_error": false,
|
|
1242
|
-
"latest_explicit_error": false,
|
|
1243
|
-
"error_recency": ErrorRecency::None.as_str(),
|
|
1244
|
-
"fresh_error": false,
|
|
1245
|
-
"dead_process_and_latest_error": false,
|
|
1246
|
-
"dead_process_and_latest_explicit_error": false,
|
|
1247
|
-
"process_dead_and_latest_explicit_error": false,
|
|
1248
|
-
"provider_process_dead_and_latest_explicit_error": false,
|
|
1249
|
-
"process_liveness": process_liveness_wire(liveness.state),
|
|
1250
|
-
"pid_status": liveness.detail.as_str(),
|
|
1251
|
-
}),
|
|
1252
|
-
)?;
|
|
1253
|
-
Ok(())
|
|
1254
|
-
}
|
|
1255
|
-
|
|
1256
|
-
fn abnormal_dedupe_key(agent: &AbnormalWatchAgent, fact: &crate::provider::FaultFact) -> String {
|
|
1257
|
-
let bucket = fact
|
|
1258
|
-
.turn_id
|
|
1259
|
-
.as_ref()
|
|
1260
|
-
.map(|id| id.as_str().to_string())
|
|
1261
|
-
.or_else(|| abnormal_error_fact_identity(fact))
|
|
1262
|
-
.unwrap_or_else(|| "no_error_identity".to_string());
|
|
1263
|
-
format!(
|
|
1264
|
-
"worker.abnormal_exit:{}:{}:{}:{}",
|
|
1265
|
-
agent.agent_id,
|
|
1266
|
-
agent.rollout_path_display,
|
|
1267
|
-
fact.signature.as_str(),
|
|
1268
|
-
bucket
|
|
1269
|
-
)
|
|
1270
|
-
}
|
|
1271
|
-
|
|
1272
|
-
fn abnormal_error_cohort_key(agent: &AbnormalWatchAgent) -> String {
|
|
1273
|
-
let cohort = agent
|
|
1274
|
-
.spawn_epoch
|
|
1275
|
-
.map(|epoch| format!("spawn_epoch:{epoch}"))
|
|
1276
|
-
.or_else(|| {
|
|
1277
|
-
agent
|
|
1278
|
-
.spawned_at
|
|
1279
|
-
.as_deref()
|
|
1280
|
-
.map(|spawned_at| format!("spawned_at:{spawned_at}"))
|
|
1281
|
-
})
|
|
1282
|
-
.unwrap_or_else(|| "legacy".to_string());
|
|
1283
|
-
format!(
|
|
1284
|
-
"worker.abnormal_exit.cohort:{}:{}:{}",
|
|
1285
|
-
agent.agent_id, agent.rollout_path_display, cohort
|
|
1286
|
-
)
|
|
1287
|
-
}
|
|
1288
|
-
|
|
1289
|
-
fn abnormal_error_observation_key(
|
|
1290
|
-
agent: &AbnormalWatchAgent,
|
|
1291
|
-
fact: &crate::provider::FaultFact,
|
|
1292
|
-
) -> String {
|
|
1293
|
-
let bucket =
|
|
1294
|
-
abnormal_error_fact_identity(fact).unwrap_or_else(|| "no_error_identity".to_string());
|
|
1295
|
-
format!(
|
|
1296
|
-
"worker.abnormal_exit.error:{}:{}:{}:{}",
|
|
1297
|
-
agent.agent_id,
|
|
1298
|
-
agent.rollout_path_display,
|
|
1299
|
-
fact.signature.as_str(),
|
|
1300
|
-
bucket
|
|
1301
|
-
)
|
|
1302
|
-
}
|
|
1303
|
-
|
|
1304
|
-
fn abnormal_error_fact_identity(fact: &crate::provider::FaultFact) -> Option<String> {
|
|
1305
|
-
fact.assistant_uuid
|
|
1306
|
-
.as_deref()
|
|
1307
|
-
.or(fact.request_id.as_deref())
|
|
1308
|
-
.or_else(|| fact.turn_id.as_ref().map(|id| id.as_str()))
|
|
1309
|
-
.map(str::to_string)
|
|
1310
|
-
}
|
|
1311
|
-
|
|
1312
|
-
fn abnormal_error_recency(
|
|
1313
|
-
state: &Value,
|
|
1314
|
-
agent: &AbnormalWatchAgent,
|
|
1315
|
-
observation_key: Option<&str>,
|
|
1316
|
-
cohort_key: Option<&str>,
|
|
1317
|
-
) -> ErrorRecency {
|
|
1318
|
-
let (Some(observation_key), Some(cohort_key)) = (observation_key, cohort_key) else {
|
|
1319
|
-
return ErrorRecency::None;
|
|
1320
|
-
};
|
|
1321
|
-
let previous_key = abnormal_last_error_observation_key(state, &agent.agent_id);
|
|
1322
|
-
let previous_cohort = abnormal_last_error_observation_cohort(state, &agent.agent_id);
|
|
1323
|
-
match (previous_key.as_deref(), previous_cohort.as_deref()) {
|
|
1324
|
-
(Some(key), Some(cohort)) if cohort == cohort_key && key != observation_key => {
|
|
1325
|
-
ErrorRecency::Fresh
|
|
1326
|
-
}
|
|
1327
|
-
_ => ErrorRecency::Stale,
|
|
1328
|
-
}
|
|
1329
|
-
}
|
|
1330
|
-
|
|
1331
|
-
fn abnormal_suppression_key(
|
|
1332
|
-
agent: &AbnormalWatchAgent,
|
|
1333
|
-
liveness: &ProcessCheck,
|
|
1334
|
-
reason: &str,
|
|
1335
|
-
) -> String {
|
|
1336
|
-
format!(
|
|
1337
|
-
"abnormal_exit.single_signal_suppressed:{}:{}:{}:{}",
|
|
1338
|
-
agent.agent_id,
|
|
1339
|
-
agent.rollout_path_display,
|
|
1340
|
-
reason,
|
|
1341
|
-
process_liveness_wire(liveness.state)
|
|
1342
|
-
)
|
|
1343
|
-
}
|
|
1344
|
-
|
|
1345
|
-
fn abnormal_check_key(
|
|
1346
|
-
agent: &AbnormalWatchAgent,
|
|
1347
|
-
liveness: &ProcessCheck,
|
|
1348
|
-
fact: Option<&crate::provider::FaultFact>,
|
|
1349
|
-
error_recency: ErrorRecency,
|
|
1350
|
-
error_observation_key: Option<&str>,
|
|
1351
|
-
) -> String {
|
|
1352
|
-
format!(
|
|
1353
|
-
"worker.abnormal_exit.check:{}:{}:{}:{}:{}:{}",
|
|
1354
|
-
agent.agent_id,
|
|
1355
|
-
agent.rollout_path_display,
|
|
1356
|
-
process_liveness_wire(liveness.state),
|
|
1357
|
-
fact.map(|fact| fact.signature.as_str()).unwrap_or("-"),
|
|
1358
|
-
error_recency.as_str(),
|
|
1359
|
-
error_observation_key.unwrap_or("-")
|
|
1360
|
-
)
|
|
1361
|
-
}
|
|
1362
|
-
|
|
1363
|
-
fn format_abnormal_exit_message(
|
|
1364
|
-
team: &str,
|
|
1365
|
-
agent: &AbnormalWatchAgent,
|
|
1366
|
-
fact: &crate::provider::FaultFact,
|
|
1367
|
-
liveness: &ProcessCheck,
|
|
1368
|
-
size: u64,
|
|
1369
|
-
manual_recovery_hint: &str,
|
|
1370
|
-
) -> String {
|
|
1371
|
-
let turn_id = fact.turn_id.as_ref().map(|id| id.as_str()).unwrap_or("-");
|
|
1372
|
-
format!(
|
|
1373
|
-
"Team Agent detected a provider abnormal exit.\n\n\
|
|
1374
|
-
event: worker.abnormal_exit\n\
|
|
1375
|
-
team: {team}\n\
|
|
1376
|
-
node: {node}\n\
|
|
1377
|
-
provider: {provider}\n\
|
|
1378
|
-
signature: {signature}\n\
|
|
1379
|
-
turn_id: {turn_id}\n\
|
|
1380
|
-
transcript: {path}\n\
|
|
1381
|
-
last_offset: {size}\n\
|
|
1382
|
-
pid_status: {pid_status}\n\n\
|
|
1383
|
-
{manual_recovery_hint}",
|
|
1384
|
-
node = agent.agent_id.as_str(),
|
|
1385
|
-
provider = provider_wire(agent.provider),
|
|
1386
|
-
signature = fact.signature.as_str(),
|
|
1387
|
-
path = agent.rollout_path_display.as_str(),
|
|
1388
|
-
pid_status = liveness.detail.as_str(),
|
|
1389
|
-
)
|
|
1390
|
-
}
|
|
1391
|
-
|
|
1392
|
-
/// A pre-upgrade legacy Claude Code, Codex, Grok or Pi seat has no runtime, so
|
|
1393
|
-
/// `start-agent` would be refused; it is stopped/removed and added again.
|
|
1394
|
-
fn recovery_hint(
|
|
1395
|
-
agent_id: &str,
|
|
1396
|
-
provider: crate::model::enums::Provider,
|
|
1397
|
-
workspace: &Path,
|
|
1398
|
-
team: &str,
|
|
1399
|
-
) -> String {
|
|
1400
|
-
let wire = provider_wire(provider);
|
|
1401
|
-
if crate::provider::wire::is_retired_provider(wire) {
|
|
1402
|
-
return format!(
|
|
1403
|
-
"No automatic restart was performed.\n{}",
|
|
1404
|
-
crate::provider::wire::retired_provider_error(wire)
|
|
1405
|
-
);
|
|
1406
|
-
}
|
|
1407
|
-
format!(
|
|
1408
|
-
"No automatic restart was performed.\nTo recover manually, run: {}",
|
|
1409
|
-
recovery_manual_command(agent_id, workspace, team)
|
|
1410
|
-
)
|
|
1411
|
-
}
|
|
1412
|
-
|
|
1413
|
-
fn recovery_manual_command(agent_id: &str, workspace: &Path, team: &str) -> String {
|
|
1414
|
-
let workspace_str = workspace.to_string_lossy();
|
|
1415
|
-
let shell_workspace = workspace_str.replace('\'', "'\\''");
|
|
1416
|
-
format!(
|
|
1417
|
-
"team-agent start-agent {agent_id} --workspace '{shell_workspace}' --team {team} --force --json"
|
|
1418
|
-
)
|
|
1419
|
-
}
|
|
1420
|
-
|
|
1421
|
-
#[cfg(test)]
|
|
1422
|
-
#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
|
1423
|
-
mod tests {
|
|
1424
|
-
use super::*;
|
|
1425
|
-
use crate::coordinator::tick::Coordinator;
|
|
1426
|
-
use crate::coordinator::types::{ProviderRegistry, WorkspacePath};
|
|
1427
|
-
use std::io::Write as _;
|
|
1428
|
-
|
|
1429
|
-
struct NormalRegistry;
|
|
1430
|
-
|
|
1431
|
-
impl ProviderRegistry for NormalRegistry {
|
|
1432
|
-
fn adapter_for(
|
|
1433
|
-
&self,
|
|
1434
|
-
provider: crate::provider::Provider,
|
|
1435
|
-
) -> Box<dyn crate::provider::ProviderAdapter> {
|
|
1436
|
-
crate::provider::get_adapter(provider)
|
|
1437
|
-
}
|
|
1438
|
-
}
|
|
1439
|
-
#[test]
|
|
1440
|
-
fn resource188_foreign_metadata_and_dedupe_cannot_skip_new_baseline() {
|
|
1441
|
-
let dir = temp_abnormal_dir("188-foreign-watch");
|
|
1442
|
-
let rollout = dir.join("rollout-w1.jsonl");
|
|
1443
|
-
let error = "{\"method\":\"turn/completed\",\"params\":{\"turn\":{\"id\":\"old\",\"status\":\"failed\"}}}\n";
|
|
1444
|
-
std::fs::write(&rollout, error).unwrap();
|
|
1445
|
-
seed_abnormal_state(&dir, &rollout, "alive", 1);
|
|
1446
|
-
let mut state = crate::state::persist::load_runtime_state(&dir).unwrap();
|
|
1447
|
-
let metadata = std::fs::metadata(&rollout).unwrap();
|
|
1448
|
-
let mut foreign = crate::state::abnormal_watch::observation_identity(&state, "w1").unwrap();
|
|
1449
|
-
foreign["team_key"] = serde_json::json!("another-team");
|
|
1450
|
-
state["coordinator"] = serde_json::json!({"abnormal_exit_watch": {"w1": {
|
|
1451
|
-
"watch_identity": foreign,
|
|
1452
|
-
"size": metadata.len(), "mtime_ns": metadata_mtime_ns(&metadata),
|
|
1453
|
-
"last_notified_key": "foreign", "last_check_key": "foreign",
|
|
1454
|
-
"last_error_observation_key": "foreign", "last_error_observation_cohort": "foreign"
|
|
1455
|
-
}}});
|
|
1456
|
-
let log = EventLog::new(&dir);
|
|
1457
|
-
let transport = crate::transport::test_support::OfflineTransport::new();
|
|
1458
|
-
detect_abnormal_exits(&dir, &transport, &mut state, &log, &[]).unwrap();
|
|
1459
|
-
let baseline = state["coordinator"]["abnormal_exit_watch"]["w1"].clone();
|
|
1460
|
-
assert_eq!(baseline["error_recency"], "stale");
|
|
1461
|
-
assert!(baseline.get("last_notified_key").is_none());
|
|
1462
|
-
assert!(crate::state::abnormal_watch::matches_observation(
|
|
1463
|
-
&state, "w1", &baseline
|
|
1464
|
-
));
|
|
1465
|
-
let events = log.tail(0).unwrap();
|
|
1466
|
-
detect_abnormal_exits(&dir, &transport, &mut state, &log, &[]).unwrap();
|
|
1467
|
-
assert_eq!(
|
|
1468
|
-
log.tail(0).unwrap(),
|
|
1469
|
-
events,
|
|
1470
|
-
"unchanged baseline must not re-alarm"
|
|
1471
|
-
);
|
|
1472
|
-
assert!(find_event(&events, "worker.abnormal_exit").is_none());
|
|
1473
|
-
}
|
|
1474
|
-
|
|
1475
|
-
#[test]
|
|
1476
|
-
fn resource188_legacy_missing_cohort_still_observes_fresh_errors_and_dedupes() {
|
|
1477
|
-
let dir = temp_abnormal_dir("188-legacy-cohort");
|
|
1478
|
-
let rollout = dir.join("rollout-w1.jsonl");
|
|
1479
|
-
let error = "{\"method\":\"turn/completed\",\"params\":{\"turn\":{\"id\":\"old\",\"status\":\"failed\"}}}\n";
|
|
1480
|
-
std::fs::write(&rollout, error).unwrap();
|
|
1481
|
-
seed_abnormal_state(&dir, &rollout, "alive", 1);
|
|
1482
|
-
let mut state = crate::state::persist::load_runtime_state(&dir).unwrap();
|
|
1483
|
-
state["agents"]["w1"]
|
|
1484
|
-
.as_object_mut()
|
|
1485
|
-
.unwrap()
|
|
1486
|
-
.remove("spawn_epoch");
|
|
1487
|
-
let log = EventLog::new(&dir);
|
|
1488
|
-
let transport = crate::transport::test_support::OfflineTransport::new();
|
|
1489
|
-
detect_abnormal_exits(&dir, &transport, &mut state, &log, &[]).unwrap();
|
|
1490
|
-
assert_eq!(
|
|
1491
|
-
state["coordinator"]["abnormal_exit_watch"]["w1"]["error_recency"],
|
|
1492
|
-
"stale"
|
|
1493
|
-
);
|
|
1494
|
-
assert!(find_event(&log.tail(0).unwrap(), "worker.abnormal_exit").is_none());
|
|
1495
|
-
std::fs::OpenOptions::new()
|
|
1496
|
-
.append(true)
|
|
1497
|
-
.open(&rollout)
|
|
1498
|
-
.unwrap()
|
|
1499
|
-
.write_all(error.replace("old", "new-error").as_bytes())
|
|
1500
|
-
.unwrap();
|
|
1501
|
-
detect_abnormal_exits(&dir, &transport, &mut state, &log, &[]).unwrap();
|
|
1502
|
-
let events = log.tail(0).unwrap();
|
|
1503
|
-
assert_eq!(
|
|
1504
|
-
find_event(&events, "worker.abnormal_exit").unwrap()["error_recency"],
|
|
1505
|
-
"fresh"
|
|
1506
|
-
);
|
|
1507
|
-
let notified =
|
|
1508
|
-
state["coordinator"]["abnormal_exit_watch"]["w1"]["last_notified_key"].clone();
|
|
1509
|
-
assert!(notified.is_string());
|
|
1510
|
-
detect_abnormal_exits(&dir, &transport, &mut state, &log, &[]).unwrap();
|
|
1511
|
-
assert_eq!(log.tail(0).unwrap(), events);
|
|
1512
|
-
assert_eq!(
|
|
1513
|
-
state["coordinator"]["abnormal_exit_watch"]["w1"]["last_notified_key"],
|
|
1514
|
-
notified
|
|
1515
|
-
);
|
|
1516
|
-
state["agents"]["w1"]["spawn_epoch"] = serde_json::json!(2);
|
|
1517
|
-
detect_abnormal_exits(&dir, &transport, &mut state, &log, &[]).unwrap();
|
|
1518
|
-
let watch = &state["coordinator"]["abnormal_exit_watch"]["w1"];
|
|
1519
|
-
assert_eq!(
|
|
1520
|
-
watch["error_recency"], "stale",
|
|
1521
|
-
"a known generation must rebaseline unchanged bytes"
|
|
1522
|
-
);
|
|
1523
|
-
assert!(watch.get("last_notified_key").is_none());
|
|
1524
|
-
assert_eq!(
|
|
1525
|
-
log.tail(0)
|
|
1526
|
-
.unwrap()
|
|
1527
|
-
.iter()
|
|
1528
|
-
.filter(|event| event["event"] == "worker.abnormal_exit")
|
|
1529
|
-
.count(),
|
|
1530
|
-
1
|
|
1531
|
-
);
|
|
1532
|
-
}
|
|
1533
|
-
|
|
1534
|
-
#[test]
|
|
1535
|
-
fn resource188_unknown_team_still_records_fresh_liveness_without_reusing_cache() {
|
|
1536
|
-
let dir = temp_abnormal_dir("188-unknown-team");
|
|
1537
|
-
let rollout = dir.join("rollout-w1.jsonl");
|
|
1538
|
-
std::fs::write(&rollout, "").unwrap();
|
|
1539
|
-
seed_abnormal_state(&dir, &rollout, "dead", 1);
|
|
1540
|
-
let mut state = crate::state::persist::load_runtime_state(&dir).unwrap();
|
|
1541
|
-
state.as_object_mut().unwrap().remove("active_team_key");
|
|
1542
|
-
let log = EventLog::new(&dir);
|
|
1543
|
-
let transport = crate::transport::test_support::OfflineTransport::new();
|
|
1544
|
-
detect_abnormal_exits(&dir, &transport, &mut state, &log, &[]).unwrap();
|
|
1545
|
-
assert_eq!(
|
|
1546
|
-
state["coordinator"]["abnormal_exit_watch"]["w1"]["last_liveness"],
|
|
1547
|
-
"dead"
|
|
1548
|
-
);
|
|
1549
|
-
assert!(state["coordinator"]["abnormal_exit_watch"]["w1"]
|
|
1550
|
-
.get("watch_identity")
|
|
1551
|
-
.is_none());
|
|
1552
|
-
}
|
|
1553
|
-
|
|
1554
|
-
#[test]
|
|
1555
|
-
fn abnormal_stale_error_baselines_then_fresh_alive_error_notifies() {
|
|
1556
|
-
let dir = temp_abnormal_dir("alive-fresh");
|
|
1557
|
-
let rollout = dir.join("rollout-w1.jsonl");
|
|
1558
|
-
std::fs::write(
|
|
1559
|
-
&rollout,
|
|
1560
|
-
"{\"method\":\"turn/completed\",\"params\":{\"turn\":{\"id\":\"t1\",\"status\":\"failed\"}}}\n",
|
|
1561
|
-
)
|
|
1562
|
-
.unwrap();
|
|
1563
|
-
seed_abnormal_state(&dir, &rollout, "alive", 1);
|
|
1564
|
-
let coordinator = abnormal_test_coordinator(&dir);
|
|
1565
|
-
|
|
1566
|
-
coordinator.tick().unwrap();
|
|
1567
|
-
|
|
1568
|
-
let first_events = read_test_events(&dir);
|
|
1569
|
-
assert!(
|
|
1570
|
-
find_event(&first_events, "worker.abnormal_exit").is_none(),
|
|
1571
|
-
"first observed explicit error is stale baseline and must not notify; events={first_events:?}"
|
|
1572
|
-
);
|
|
1573
|
-
let stale_check =
|
|
1574
|
-
find_event(&first_events, "worker.abnormal_exit.check").expect("stale check event");
|
|
1575
|
-
assert_eq!(stale_check["error_recency"], serde_json::json!("stale"));
|
|
1576
|
-
assert_eq!(stale_check["fresh_error"], serde_json::json!(false));
|
|
1577
|
-
assert_eq!(stale_check["notification"], serde_json::json!(false));
|
|
1578
|
-
assert!(
|
|
1579
|
-
find_event(&first_events, "abnormal_exit.single_signal_suppressed").is_none(),
|
|
1580
|
-
"stale/error-only observations must not emit single-signal suppression"
|
|
1581
|
-
);
|
|
1582
|
-
let watch = abnormal_test_watch(&dir);
|
|
1583
|
-
assert_eq!(watch["error_recency"], serde_json::json!("stale"));
|
|
1584
|
-
assert_eq!(watch["fresh_error"], serde_json::json!(false));
|
|
1585
|
-
assert!(
|
|
1586
|
-
watch["last_error_observation_key"].as_str().is_some(),
|
|
1587
|
-
"first explicit error must persist the stale baseline; watch={watch}"
|
|
1588
|
-
);
|
|
1589
|
-
|
|
1590
|
-
std::fs::OpenOptions::new()
|
|
1591
|
-
.append(true)
|
|
1592
|
-
.open(&rollout)
|
|
1593
|
-
.unwrap()
|
|
1594
|
-
.write_all(
|
|
1595
|
-
b"{\"method\":\"turn/completed\",\"params\":{\"turn\":{\"id\":\"t2\",\"status\":\"failed\"}}}\n",
|
|
1596
|
-
)
|
|
1597
|
-
.unwrap();
|
|
1598
|
-
coordinator.tick().unwrap();
|
|
1599
|
-
|
|
1600
|
-
let events = read_test_events(&dir);
|
|
1601
|
-
let abnormal =
|
|
1602
|
-
find_event(&events, "worker.abnormal_exit").expect("fresh alive error notification");
|
|
1603
|
-
assert_eq!(abnormal["provider_process_dead"], serde_json::json!(false));
|
|
1604
|
-
assert_eq!(abnormal["latest_explicit_error"], serde_json::json!(true));
|
|
1605
|
-
assert_eq!(abnormal["error_recency"], serde_json::json!("fresh"));
|
|
1606
|
-
assert_eq!(abnormal["fresh_error"], serde_json::json!(true));
|
|
1607
|
-
assert_eq!(
|
|
1608
|
-
abnormal["provider_process_dead_and_latest_explicit_error"],
|
|
1609
|
-
serde_json::json!(false)
|
|
1610
|
-
);
|
|
1611
|
-
assert_eq!(abnormal["notification_status"], serde_json::json!("queued"));
|
|
1612
|
-
}
|
|
1613
|
-
|
|
1614
|
-
#[test]
|
|
1615
|
-
fn abnormal_claude_assistant_api_error_events_include_structured_details() {
|
|
1616
|
-
let dir = temp_abnormal_dir("claude-api-details");
|
|
1617
|
-
let rollout = dir.join("rollout-w1.jsonl");
|
|
1618
|
-
std::fs::write(
|
|
1619
|
-
&rollout,
|
|
1620
|
-
claude_assistant_api_error_line(
|
|
1621
|
-
"assistant-400",
|
|
1622
|
-
"parent-400",
|
|
1623
|
-
"session-400",
|
|
1624
|
-
400,
|
|
1625
|
-
"unknown",
|
|
1626
|
-
None,
|
|
1627
|
-
),
|
|
1628
|
-
)
|
|
1629
|
-
.unwrap();
|
|
1630
|
-
seed_abnormal_state_with_provider(&dir, &rollout, "alive", 1, "claude_code");
|
|
1631
|
-
let coordinator = abnormal_test_coordinator(&dir);
|
|
1632
|
-
|
|
1633
|
-
coordinator.tick().unwrap();
|
|
1634
|
-
|
|
1635
|
-
let first_events = read_test_events(&dir);
|
|
1636
|
-
let stale_check =
|
|
1637
|
-
find_event(&first_events, "worker.abnormal_exit.check").expect("stale check event");
|
|
1638
|
-
assert_eq!(stale_check["error_recency"], serde_json::json!("stale"));
|
|
1639
|
-
assert_eq!(stale_check["notification"], serde_json::json!(false));
|
|
1640
|
-
assert_eq!(stale_check["apiErrorStatus"], serde_json::json!(400));
|
|
1641
|
-
assert_eq!(stale_check["error"], serde_json::json!("unknown"));
|
|
1642
|
-
assert_eq!(stale_check["requestId"], serde_json::json!(null));
|
|
1643
|
-
assert_eq!(
|
|
1644
|
-
stale_check["assistant_uuid"],
|
|
1645
|
-
serde_json::json!("assistant-400")
|
|
1646
|
-
);
|
|
1647
|
-
|
|
1648
|
-
std::fs::OpenOptions::new()
|
|
1649
|
-
.append(true)
|
|
1650
|
-
.open(&rollout)
|
|
1651
|
-
.unwrap()
|
|
1652
|
-
.write_all(
|
|
1653
|
-
claude_assistant_api_error_line(
|
|
1654
|
-
"assistant-404",
|
|
1655
|
-
"parent-404",
|
|
1656
|
-
"session-404",
|
|
1657
|
-
404,
|
|
1658
|
-
"model_not_found",
|
|
1659
|
-
Some("req_011CceNfWj2aPY5gtCdakULt"),
|
|
1660
|
-
)
|
|
1661
|
-
.as_bytes(),
|
|
1662
|
-
)
|
|
1663
|
-
.unwrap();
|
|
1664
|
-
coordinator.tick().unwrap();
|
|
1665
|
-
|
|
1666
|
-
let events = read_test_events(&dir);
|
|
1667
|
-
let fresh_check =
|
|
1668
|
-
find_event(&events, "worker.abnormal_exit.check").expect("fresh check event");
|
|
1669
|
-
assert_eq!(fresh_check["error_recency"], serde_json::json!("fresh"));
|
|
1670
|
-
assert_eq!(fresh_check["notification"], serde_json::json!(true));
|
|
1671
|
-
assert_eq!(fresh_check["apiErrorStatus"], serde_json::json!(404));
|
|
1672
|
-
assert_eq!(fresh_check["error"], serde_json::json!("model_not_found"));
|
|
1673
|
-
assert_eq!(
|
|
1674
|
-
fresh_check["requestId"],
|
|
1675
|
-
serde_json::json!("req_011CceNfWj2aPY5gtCdakULt")
|
|
1676
|
-
);
|
|
1677
|
-
assert_eq!(
|
|
1678
|
-
fresh_check["assistant_uuid"],
|
|
1679
|
-
serde_json::json!("assistant-404")
|
|
1680
|
-
);
|
|
1681
|
-
|
|
1682
|
-
let abnormal =
|
|
1683
|
-
find_event(&events, "worker.abnormal_exit").expect("fresh alive error notification");
|
|
1684
|
-
assert_eq!(abnormal["signature"], serde_json::json!("api_error"));
|
|
1685
|
-
assert_eq!(abnormal["turn_id"], serde_json::json!("assistant-404"));
|
|
1686
|
-
assert_eq!(abnormal["apiErrorStatus"], serde_json::json!(404));
|
|
1687
|
-
assert_eq!(abnormal["error"], serde_json::json!("model_not_found"));
|
|
1688
|
-
assert_eq!(
|
|
1689
|
-
abnormal["requestId"],
|
|
1690
|
-
serde_json::json!("req_011CceNfWj2aPY5gtCdakULt")
|
|
1691
|
-
);
|
|
1692
|
-
assert_eq!(
|
|
1693
|
-
abnormal["assistant_uuid"],
|
|
1694
|
-
serde_json::json!("assistant-404")
|
|
1695
|
-
);
|
|
1696
|
-
assert_eq!(abnormal["notification_status"], serde_json::json!("queued"));
|
|
1697
|
-
}
|
|
1698
|
-
|
|
1699
|
-
#[test]
|
|
1700
|
-
fn abnormal_claude_assistant_api_error_followed_by_bookkeeping_still_detected() {
|
|
1701
|
-
let dir = temp_abnormal_dir("claude-api-bookkeeping");
|
|
1702
|
-
let rollout = dir.join("rollout-w1.jsonl");
|
|
1703
|
-
let session_id = "97ec1070-f19b-49ed-b60f-3cb158e92053";
|
|
1704
|
-
std::fs::write(
|
|
1705
|
-
&rollout,
|
|
1706
|
-
format!(
|
|
1707
|
-
"{}{}",
|
|
1708
|
-
claude_assistant_api_error_line(
|
|
1709
|
-
"94e88d55-aac7-46bc-85cd-b7fcfc8a9ef6",
|
|
1710
|
-
"245fce6f-2427-4b05-af01-8619df64afab",
|
|
1711
|
-
session_id,
|
|
1712
|
-
404,
|
|
1713
|
-
"model_not_found",
|
|
1714
|
-
Some("req_011CceUVQ94dxahgwAHf8sdS"),
|
|
1715
|
-
),
|
|
1716
|
-
claude_turn_duration_line(
|
|
1717
|
-
"6c96328b-8c01-46c0-b68e-a096a89c904f",
|
|
1718
|
-
"94e88d55-aac7-46bc-85cd-b7fcfc8a9ef6",
|
|
1719
|
-
session_id,
|
|
1720
|
-
),
|
|
1721
|
-
),
|
|
1722
|
-
)
|
|
1723
|
-
.unwrap();
|
|
1724
|
-
seed_abnormal_state_with_provider(&dir, &rollout, "alive", 1, "claude_code");
|
|
1725
|
-
let coordinator = abnormal_test_coordinator(&dir);
|
|
1726
|
-
|
|
1727
|
-
coordinator.tick().unwrap();
|
|
1728
|
-
|
|
1729
|
-
let first_events = read_test_events(&dir);
|
|
1730
|
-
let first_check =
|
|
1731
|
-
find_event(&first_events, "worker.abnormal_exit.check").expect("stale check event");
|
|
1732
|
-
assert_eq!(
|
|
1733
|
-
first_check["latest_explicit_error"],
|
|
1734
|
-
serde_json::json!(true)
|
|
1735
|
-
);
|
|
1736
|
-
assert_eq!(first_check["error_recency"], serde_json::json!("stale"));
|
|
1737
|
-
assert_eq!(first_check["notification"], serde_json::json!(false));
|
|
1738
|
-
assert_eq!(first_check["apiErrorStatus"], serde_json::json!(404));
|
|
1739
|
-
assert_eq!(first_check["error"], serde_json::json!("model_not_found"));
|
|
1740
|
-
assert_eq!(
|
|
1741
|
-
first_check["requestId"],
|
|
1742
|
-
serde_json::json!("req_011CceUVQ94dxahgwAHf8sdS")
|
|
1743
|
-
);
|
|
1744
|
-
assert_eq!(
|
|
1745
|
-
first_check["assistant_uuid"],
|
|
1746
|
-
serde_json::json!("94e88d55-aac7-46bc-85cd-b7fcfc8a9ef6")
|
|
1747
|
-
);
|
|
1748
|
-
assert!(
|
|
1749
|
-
find_event(&first_events, "worker.abnormal_exit").is_none(),
|
|
1750
|
-
"first observed error baselines stale even when followed by turn_duration"
|
|
1751
|
-
);
|
|
1752
|
-
let first_observation_key = abnormal_test_watch(&dir)["last_error_observation_key"]
|
|
1753
|
-
.as_str()
|
|
1754
|
-
.expect("stale baseline persists observation key")
|
|
1755
|
-
.to_string();
|
|
1756
|
-
|
|
1757
|
-
std::fs::OpenOptions::new()
|
|
1758
|
-
.append(true)
|
|
1759
|
-
.open(&rollout)
|
|
1760
|
-
.unwrap()
|
|
1761
|
-
.write_all(
|
|
1762
|
-
format!(
|
|
1763
|
-
"{}{}{}",
|
|
1764
|
-
claude_file_history_snapshot_line(session_id),
|
|
1765
|
-
claude_last_prompt_line(session_id),
|
|
1766
|
-
claude_mode_line(session_id),
|
|
1767
|
-
)
|
|
1768
|
-
.as_bytes(),
|
|
1769
|
-
)
|
|
1770
|
-
.unwrap();
|
|
1771
|
-
coordinator.tick().unwrap();
|
|
1772
|
-
|
|
1773
|
-
let after_bookkeeping_events = read_test_events(&dir);
|
|
1774
|
-
let bookkeeping_check = find_event(&after_bookkeeping_events, "worker.abnormal_exit.check")
|
|
1775
|
-
.expect("bookkeeping check event");
|
|
1776
|
-
assert_eq!(
|
|
1777
|
-
bookkeeping_check["error_recency"],
|
|
1778
|
-
serde_json::json!("stale")
|
|
1779
|
-
);
|
|
1780
|
-
assert_eq!(bookkeeping_check["notification"], serde_json::json!(false));
|
|
1781
|
-
assert_eq!(
|
|
1782
|
-
bookkeeping_check["assistant_uuid"],
|
|
1783
|
-
serde_json::json!("94e88d55-aac7-46bc-85cd-b7fcfc8a9ef6")
|
|
1784
|
-
);
|
|
1785
|
-
assert_eq!(
|
|
1786
|
-
abnormal_test_watch(&dir)["last_error_observation_key"],
|
|
1787
|
-
serde_json::json!(first_observation_key)
|
|
1788
|
-
);
|
|
1789
|
-
assert!(
|
|
1790
|
-
find_event(&after_bookkeeping_events, "worker.abnormal_exit").is_none(),
|
|
1791
|
-
"bookkeeping-only growth after the same error must not notify"
|
|
1792
|
-
);
|
|
1793
|
-
|
|
1794
|
-
std::fs::OpenOptions::new()
|
|
1795
|
-
.append(true)
|
|
1796
|
-
.open(&rollout)
|
|
1797
|
-
.unwrap()
|
|
1798
|
-
.write_all(
|
|
1799
|
-
format!(
|
|
1800
|
-
"{}{}",
|
|
1801
|
-
claude_assistant_api_error_line(
|
|
1802
|
-
"11b8790e-718b-4910-bf6a-62d19135a93b",
|
|
1803
|
-
"ccea3b0f-23e7-48ed-aa57-d59e321becfb",
|
|
1804
|
-
session_id,
|
|
1805
|
-
404,
|
|
1806
|
-
"model_not_found",
|
|
1807
|
-
Some("req_011CceUhRwYfdmLPbv9nMmqX"),
|
|
1808
|
-
),
|
|
1809
|
-
claude_turn_duration_line(
|
|
1810
|
-
"2d382eff-783b-48ca-83c1-c93d3dc8b068",
|
|
1811
|
-
"11b8790e-718b-4910-bf6a-62d19135a93b",
|
|
1812
|
-
session_id,
|
|
1813
|
-
),
|
|
1814
|
-
)
|
|
1815
|
-
.as_bytes(),
|
|
1816
|
-
)
|
|
1817
|
-
.unwrap();
|
|
1818
|
-
coordinator.tick().unwrap();
|
|
1819
|
-
|
|
1820
|
-
let events = read_test_events(&dir);
|
|
1821
|
-
let fresh_check =
|
|
1822
|
-
find_event(&events, "worker.abnormal_exit.check").expect("fresh check event");
|
|
1823
|
-
assert_eq!(fresh_check["error_recency"], serde_json::json!("fresh"));
|
|
1824
|
-
assert_eq!(fresh_check["notification"], serde_json::json!(true));
|
|
1825
|
-
assert_eq!(
|
|
1826
|
-
fresh_check["assistant_uuid"],
|
|
1827
|
-
serde_json::json!("11b8790e-718b-4910-bf6a-62d19135a93b")
|
|
1828
|
-
);
|
|
1829
|
-
let abnormal =
|
|
1830
|
-
find_event(&events, "worker.abnormal_exit").expect("fresh API error notification");
|
|
1831
|
-
assert_eq!(
|
|
1832
|
-
abnormal["turn_id"],
|
|
1833
|
-
serde_json::json!("11b8790e-718b-4910-bf6a-62d19135a93b")
|
|
1834
|
-
);
|
|
1835
|
-
assert_eq!(
|
|
1836
|
-
abnormal["requestId"],
|
|
1837
|
-
serde_json::json!("req_011CceUhRwYfdmLPbv9nMmqX")
|
|
1838
|
-
);
|
|
1839
|
-
assert_eq!(abnormal["notification_status"], serde_json::json!("queued"));
|
|
1840
|
-
}
|
|
1841
|
-
|
|
1842
|
-
#[test]
|
|
1843
|
-
fn codex_failed_turn_then_completed_turn_no_fresh_notification() {
|
|
1844
|
-
let dir = temp_abnormal_dir("codex-failed-then-completed");
|
|
1845
|
-
let rollout = dir.join("rollout-w1.jsonl");
|
|
1846
|
-
std::fs::write(
|
|
1847
|
-
&rollout,
|
|
1848
|
-
"{\"method\":\"turn/completed\",\"params\":{\"turn\":{\"id\":\"t1\",\"status\":\"failed\"}}}\n",
|
|
1849
|
-
)
|
|
1850
|
-
.unwrap();
|
|
1851
|
-
seed_abnormal_state(&dir, &rollout, "alive", 1);
|
|
1852
|
-
let coordinator = abnormal_test_coordinator(&dir);
|
|
1853
|
-
|
|
1854
|
-
coordinator.tick().unwrap();
|
|
1855
|
-
|
|
1856
|
-
let first_events = read_test_events(&dir);
|
|
1857
|
-
let first_check =
|
|
1858
|
-
find_event(&first_events, "worker.abnormal_exit.check").expect("stale check event");
|
|
1859
|
-
assert_eq!(first_check["turn_id"], serde_json::json!("t1"));
|
|
1860
|
-
assert_eq!(first_check["error_recency"], serde_json::json!("stale"));
|
|
1861
|
-
assert_eq!(first_check["notification"], serde_json::json!(false));
|
|
1862
|
-
assert!(
|
|
1863
|
-
find_event(&first_events, "worker.abnormal_exit").is_none(),
|
|
1864
|
-
"first failed turn is only a stale baseline"
|
|
1865
|
-
);
|
|
1866
|
-
|
|
1867
|
-
std::fs::OpenOptions::new()
|
|
1868
|
-
.append(true)
|
|
1869
|
-
.open(&rollout)
|
|
1870
|
-
.unwrap()
|
|
1871
|
-
.write_all(
|
|
1872
|
-
b"{\"method\":\"turn/completed\",\"params\":{\"turn\":{\"id\":\"t-complete\",\"status\":\"completed\"}}}\n",
|
|
1873
|
-
)
|
|
1874
|
-
.unwrap();
|
|
1875
|
-
coordinator.tick().unwrap();
|
|
1876
|
-
|
|
1877
|
-
let after_completed_events = read_test_events(&dir);
|
|
1878
|
-
let completed_check = find_event(&after_completed_events, "worker.abnormal_exit.check")
|
|
1879
|
-
.expect("completed-after-failure check event");
|
|
1880
|
-
assert_eq!(completed_check["turn_id"], serde_json::json!("t1"));
|
|
1881
|
-
assert_eq!(completed_check["error_recency"], serde_json::json!("stale"));
|
|
1882
|
-
assert_eq!(completed_check["fresh_error"], serde_json::json!(false));
|
|
1883
|
-
assert_eq!(completed_check["notification"], serde_json::json!(false));
|
|
1884
|
-
assert!(
|
|
1885
|
-
find_event(&after_completed_events, "worker.abnormal_exit").is_none(),
|
|
1886
|
-
"completed turn after the same old failed turn must not emit a fresh notification"
|
|
1887
|
-
);
|
|
1888
|
-
|
|
1889
|
-
std::fs::OpenOptions::new()
|
|
1890
|
-
.append(true)
|
|
1891
|
-
.open(&rollout)
|
|
1892
|
-
.unwrap()
|
|
1893
|
-
.write_all(
|
|
1894
|
-
b"{\"method\":\"turn/completed\",\"params\":{\"turn\":{\"id\":\"t2\",\"status\":\"failed\"}}}\n",
|
|
1895
|
-
)
|
|
1896
|
-
.unwrap();
|
|
1897
|
-
coordinator.tick().unwrap();
|
|
1898
|
-
|
|
1899
|
-
let events = read_test_events(&dir);
|
|
1900
|
-
let fresh_check =
|
|
1901
|
-
find_event(&events, "worker.abnormal_exit.check").expect("new failed turn check event");
|
|
1902
|
-
assert_eq!(fresh_check["turn_id"], serde_json::json!("t2"));
|
|
1903
|
-
assert_eq!(fresh_check["error_recency"], serde_json::json!("fresh"));
|
|
1904
|
-
assert_eq!(fresh_check["notification"], serde_json::json!(true));
|
|
1905
|
-
let abnormal =
|
|
1906
|
-
find_event(&events, "worker.abnormal_exit").expect("fresh Codex failure notification");
|
|
1907
|
-
assert_eq!(abnormal["turn_id"], serde_json::json!("t2"));
|
|
1908
|
-
assert_eq!(abnormal["notification_status"], serde_json::json!("queued"));
|
|
1909
|
-
}
|
|
1910
|
-
|
|
1911
|
-
#[test]
|
|
1912
|
-
fn abnormal_dead_process_without_explicit_error_stays_dead_only() {
|
|
1913
|
-
let dir = temp_abnormal_dir("dead-only");
|
|
1914
|
-
let rollout = dir.join("rollout-w1.jsonl");
|
|
1915
|
-
std::fs::write(
|
|
1916
|
-
&rollout,
|
|
1917
|
-
"{\"method\":\"turn/completed\",\"params\":{\"turn\":{\"id\":\"t1\",\"status\":\"completed\"}}}\n",
|
|
1918
|
-
)
|
|
1919
|
-
.unwrap();
|
|
1920
|
-
seed_abnormal_state(&dir, &rollout, "dead", 1);
|
|
1921
|
-
let coordinator = abnormal_test_coordinator(&dir);
|
|
1922
|
-
|
|
1923
|
-
coordinator.tick().unwrap();
|
|
1924
|
-
|
|
1925
|
-
let events = read_test_events(&dir);
|
|
1926
|
-
assert!(
|
|
1927
|
-
find_event(&events, "worker.abnormal_exit").is_none(),
|
|
1928
|
-
"dead process without explicit error must not notify; events={events:?}"
|
|
1929
|
-
);
|
|
1930
|
-
let suppressed =
|
|
1931
|
-
find_event(&events, "abnormal_exit.single_signal_suppressed").expect("dead_only");
|
|
1932
|
-
assert_eq!(suppressed["reason"], serde_json::json!("dead_only"));
|
|
1933
|
-
assert_eq!(
|
|
1934
|
-
suppressed["latest_explicit_error"],
|
|
1935
|
-
serde_json::json!(false)
|
|
1936
|
-
);
|
|
1937
|
-
assert_eq!(suppressed["error_recency"], serde_json::json!("none"));
|
|
1938
|
-
assert_eq!(suppressed["fresh_error"], serde_json::json!(false));
|
|
1939
|
-
}
|
|
1940
|
-
|
|
1941
|
-
#[test]
|
|
1942
|
-
fn abnormal_unchanged_transcript_keeps_provider_death_out_of_pane_projection() {
|
|
1943
|
-
let dir = temp_abnormal_dir("unchanged-provider-dead");
|
|
1944
|
-
let rollout = dir.join("rollout-w1.jsonl");
|
|
1945
|
-
std::fs::write(
|
|
1946
|
-
&rollout,
|
|
1947
|
-
"{\"method\":\"turn/completed\",\"params\":{\"turn\":{\"id\":\"t1\",\"status\":\"completed\"}}}\n",
|
|
1948
|
-
)
|
|
1949
|
-
.unwrap();
|
|
1950
|
-
seed_abnormal_state(&dir, &rollout, "alive", 1);
|
|
1951
|
-
let coordinator = abnormal_test_coordinator(&dir);
|
|
1952
|
-
coordinator.tick().unwrap();
|
|
1953
|
-
|
|
1954
|
-
let mut state = crate::state::persist::load_runtime_state(&dir).unwrap();
|
|
1955
|
-
state["agents"]["w1"]["process_liveness"] = serde_json::json!("dead");
|
|
1956
|
-
crate::state::persist::save_runtime_state(&dir, &state).unwrap();
|
|
1957
|
-
coordinator.tick().unwrap();
|
|
1958
|
-
|
|
1959
|
-
let state = crate::state::persist::load_runtime_state(&dir).unwrap();
|
|
1960
|
-
let agent = &state["agents"]["w1"];
|
|
1961
|
-
assert!(
|
|
1962
|
-
agent.get("provider_process_dead").is_none(),
|
|
1963
|
-
"provider death belongs to abnormal watch, not the pane-dead seat projection: {agent}"
|
|
1964
|
-
);
|
|
1965
|
-
assert!(
|
|
1966
|
-
agent.get("stale_reason").is_none(),
|
|
1967
|
-
"provider death must not be mislabeled pane_dead: {agent}"
|
|
1968
|
-
);
|
|
1969
|
-
let watch = &state["coordinator"]["abnormal_exit_watch"]["w1"];
|
|
1970
|
-
assert_eq!(watch["provider_process_dead"], serde_json::json!(true));
|
|
1971
|
-
assert_eq!(watch["worker_provider_exited"], serde_json::json!(false));
|
|
1972
|
-
assert_eq!(
|
|
1973
|
-
watch["last_liveness_detail"],
|
|
1974
|
-
serde_json::json!("explicit:dead")
|
|
1975
|
-
);
|
|
1976
|
-
}
|
|
1977
|
-
|
|
1978
|
-
#[test]
|
|
1979
|
-
fn abnormal_pane_absence_is_not_provider_process_death() {
|
|
1980
|
-
let agent = test_abnormal_agent("/tmp/rollout.jsonl", Some(1), None);
|
|
1981
|
-
for detail in ["pane_dead:%1", "window_missing:w1"] {
|
|
1982
|
-
let payload = abnormal_watch_payload(
|
|
1983
|
-
&agent,
|
|
1984
|
-
Some(1),
|
|
1985
|
-
Some(2),
|
|
1986
|
-
process_check(ProcessLiveness::Dead, detail.to_string()),
|
|
1987
|
-
None,
|
|
1988
|
-
ErrorRecency::None,
|
|
1989
|
-
None,
|
|
1990
|
-
None,
|
|
1991
|
-
);
|
|
1992
|
-
assert_eq!(
|
|
1993
|
-
payload["provider_process_dead"],
|
|
1994
|
-
serde_json::json!(false),
|
|
1995
|
-
"pane absence and provider death are distinct facts: {payload}"
|
|
1996
|
-
);
|
|
1997
|
-
assert_eq!(payload["worker_provider_exited"], serde_json::json!(false));
|
|
1998
|
-
}
|
|
1999
|
-
}
|
|
2000
|
-
|
|
2001
|
-
#[test]
|
|
2002
|
-
fn abnormal_recency_treats_cohort_change_as_stale() {
|
|
2003
|
-
let agent = test_abnormal_agent("/tmp/rollout.jsonl", Some(2), None);
|
|
2004
|
-
let fact = crate::provider::latest_explicit_error_fact(
|
|
2005
|
-
crate::provider::Provider::Codex,
|
|
2006
|
-
"{\"method\":\"turn/completed\",\"params\":{\"turn\":{\"id\":\"t1\",\"status\":\"failed\"}}}\n",
|
|
2007
|
-
)
|
|
2008
|
-
.unwrap();
|
|
2009
|
-
let observation_key = abnormal_error_observation_key(&agent, &fact);
|
|
2010
|
-
let state = serde_json::json!({
|
|
2011
|
-
"coordinator": {
|
|
2012
|
-
"abnormal_exit_watch": {
|
|
2013
|
-
"w1": {
|
|
2014
|
-
"last_error_observation_key": observation_key,
|
|
2015
|
-
"last_error_observation_cohort": "worker.abnormal_exit.cohort:w1:/tmp/rollout.jsonl:spawn_epoch:1"
|
|
2016
|
-
}
|
|
2017
|
-
}
|
|
2018
|
-
}
|
|
2019
|
-
});
|
|
2020
|
-
let cohort_key = abnormal_error_cohort_key(&agent);
|
|
2021
|
-
|
|
2022
|
-
let recency = abnormal_error_recency(
|
|
2023
|
-
&state,
|
|
2024
|
-
&agent,
|
|
2025
|
-
Some(observation_key.as_str()),
|
|
2026
|
-
Some(cohort_key.as_str()),
|
|
2027
|
-
);
|
|
2028
|
-
|
|
2029
|
-
assert_eq!(
|
|
2030
|
-
recency,
|
|
2031
|
-
ErrorRecency::Stale,
|
|
2032
|
-
"cohort change baselines the observed error instead of treating it as fresh"
|
|
2033
|
-
);
|
|
2034
|
-
}
|
|
2035
|
-
|
|
2036
|
-
#[test]
|
|
2037
|
-
fn worker_exit_marker_detail_includes_exit_code() {
|
|
2038
|
-
let pane_id = crate::transport::PaneId::new("%marker-rc");
|
|
2039
|
-
let marker = crate::tmux_backend::worker_provider_exit_marker("codex");
|
|
2040
|
-
let transport = crate::transport::test_support::OfflineTransport::new()
|
|
2041
|
-
.with_capture_for_pane(pane_id.as_str(), format!("\n{marker} 23\n"));
|
|
2042
|
-
let mut agent = test_abnormal_agent("/tmp/rollout.jsonl", Some(1), None);
|
|
2043
|
-
agent.pane_id = Some(pane_id.as_str().to_string());
|
|
2044
|
-
|
|
2045
|
-
let check = worker_provider_exit_marker_check(&agent, &transport).expect("marker hit");
|
|
2046
|
-
|
|
2047
|
-
assert_eq!(check.state, ProcessLiveness::Dead);
|
|
2048
|
-
assert_eq!(check.detail, "worker_provider_exited:%marker-rc:rc=23");
|
|
2049
|
-
}
|
|
2050
|
-
|
|
2051
|
-
#[test]
|
|
2052
|
-
fn worker_exit_marker_detail_uses_unknown_for_truncated_exit_code() {
|
|
2053
|
-
let pane_id = crate::transport::PaneId::new("%marker-truncated");
|
|
2054
|
-
let marker = crate::tmux_backend::worker_provider_exit_marker("codex");
|
|
2055
|
-
let transport = crate::transport::test_support::OfflineTransport::new()
|
|
2056
|
-
.with_capture_for_pane(pane_id.as_str(), format!("\n{marker} "));
|
|
2057
|
-
let mut agent = test_abnormal_agent("/tmp/rollout.jsonl", Some(1), None);
|
|
2058
|
-
agent.pane_id = Some(pane_id.as_str().to_string());
|
|
2059
|
-
|
|
2060
|
-
let check = worker_provider_exit_marker_check(&agent, &transport).expect("marker hit");
|
|
2061
|
-
|
|
2062
|
-
assert_eq!(check.state, ProcessLiveness::Dead);
|
|
2063
|
-
assert_eq!(
|
|
2064
|
-
check.detail,
|
|
2065
|
-
"worker_provider_exited:%marker-truncated:rc=unknown"
|
|
2066
|
-
);
|
|
2067
|
-
}
|
|
2068
|
-
|
|
2069
|
-
#[test]
|
|
2070
|
-
fn abnormal_dead_fresh_error_event_reports_actual_dead_booleans() {
|
|
2071
|
-
let dir = temp_abnormal_dir("dead-fresh");
|
|
2072
|
-
let rollout = dir.join("rollout-w1.jsonl");
|
|
2073
|
-
std::fs::write(
|
|
2074
|
-
&rollout,
|
|
2075
|
-
"{\"method\":\"turn/completed\",\"params\":{\"turn\":{\"id\":\"t1\",\"status\":\"failed\"}}}\n",
|
|
2076
|
-
)
|
|
2077
|
-
.unwrap();
|
|
2078
|
-
seed_abnormal_state(&dir, &rollout, "dead", 1);
|
|
2079
|
-
let coordinator = abnormal_test_coordinator(&dir);
|
|
2080
|
-
coordinator.tick().unwrap();
|
|
2081
|
-
std::fs::OpenOptions::new()
|
|
2082
|
-
.append(true)
|
|
2083
|
-
.open(&rollout)
|
|
2084
|
-
.unwrap()
|
|
2085
|
-
.write_all(
|
|
2086
|
-
b"{\"method\":\"turn/completed\",\"params\":{\"turn\":{\"id\":\"t2\",\"status\":\"failed\"}}}\n",
|
|
2087
|
-
)
|
|
2088
|
-
.unwrap();
|
|
2089
|
-
|
|
2090
|
-
coordinator.tick().unwrap();
|
|
2091
|
-
|
|
2092
|
-
let events = read_test_events(&dir);
|
|
2093
|
-
let abnormal =
|
|
2094
|
-
find_event(&events, "worker.abnormal_exit").expect("fresh dead error notification");
|
|
2095
|
-
assert_eq!(abnormal["provider_process_dead"], serde_json::json!(true));
|
|
2096
|
-
assert_eq!(abnormal["latest_explicit_error"], serde_json::json!(true));
|
|
2097
|
-
assert_eq!(abnormal["error_recency"], serde_json::json!("fresh"));
|
|
2098
|
-
assert_eq!(
|
|
2099
|
-
abnormal["provider_process_dead_and_latest_explicit_error"],
|
|
2100
|
-
serde_json::json!(true)
|
|
2101
|
-
);
|
|
2102
|
-
}
|
|
2103
|
-
|
|
2104
|
-
fn abnormal_test_coordinator(dir: &std::path::Path) -> Coordinator {
|
|
2105
|
-
Coordinator::for_test(
|
|
2106
|
-
WorkspacePath::new(dir.to_path_buf()),
|
|
2107
|
-
Box::new(NormalRegistry),
|
|
2108
|
-
Box::new(
|
|
2109
|
-
crate::transport::test_support::OfflineTransport::new().with_session_present(true),
|
|
2110
|
-
),
|
|
2111
|
-
None,
|
|
2112
|
-
None,
|
|
2113
|
-
)
|
|
2114
|
-
}
|
|
2115
|
-
|
|
2116
|
-
fn seed_abnormal_state(
|
|
2117
|
-
dir: &std::path::Path,
|
|
2118
|
-
rollout: &std::path::Path,
|
|
2119
|
-
liveness: &str,
|
|
2120
|
-
spawn_epoch: u64,
|
|
2121
|
-
) {
|
|
2122
|
-
seed_abnormal_state_with_provider(dir, rollout, liveness, spawn_epoch, "codex");
|
|
2123
|
-
}
|
|
2124
|
-
|
|
2125
|
-
fn seed_abnormal_state_with_provider(
|
|
2126
|
-
dir: &std::path::Path,
|
|
2127
|
-
rollout: &std::path::Path,
|
|
2128
|
-
liveness: &str,
|
|
2129
|
-
spawn_epoch: u64,
|
|
2130
|
-
provider: &str,
|
|
2131
|
-
) {
|
|
2132
|
-
crate::state::persist::save_runtime_state(
|
|
2133
|
-
dir,
|
|
2134
|
-
&serde_json::json!({
|
|
2135
|
-
"active_team_key": "team",
|
|
2136
|
-
"agents": {
|
|
2137
|
-
"w1": {
|
|
2138
|
-
"provider": provider,
|
|
2139
|
-
"status": "running",
|
|
2140
|
-
"agent_id": "w1",
|
|
2141
|
-
"session_id": "session-w1",
|
|
2142
|
-
"rollout_path": rollout.to_string_lossy(),
|
|
2143
|
-
"spawn_cwd": dir.to_string_lossy(),
|
|
2144
|
-
"spawn_epoch": spawn_epoch,
|
|
2145
|
-
"process_liveness": liveness
|
|
2146
|
-
}
|
|
2147
|
-
}
|
|
2148
|
-
}),
|
|
2149
|
-
)
|
|
2150
|
-
.unwrap();
|
|
2151
|
-
}
|
|
2152
|
-
|
|
2153
|
-
fn claude_assistant_api_error_line(
|
|
2154
|
-
uuid: &str,
|
|
2155
|
-
parent_uuid: &str,
|
|
2156
|
-
session_id: &str,
|
|
2157
|
-
status: i64,
|
|
2158
|
-
error: &str,
|
|
2159
|
-
request_id: Option<&str>,
|
|
2160
|
-
) -> String {
|
|
2161
|
-
let mut record = serde_json::json!({
|
|
2162
|
-
"type": "assistant",
|
|
2163
|
-
"parentUuid": parent_uuid,
|
|
2164
|
-
"uuid": uuid,
|
|
2165
|
-
"message": {
|
|
2166
|
-
"role": "assistant",
|
|
2167
|
-
"content": [{"type": "text", "text": "API Error"}]
|
|
2168
|
-
},
|
|
2169
|
-
"error": error,
|
|
2170
|
-
"isApiErrorMessage": true,
|
|
2171
|
-
"apiErrorStatus": status,
|
|
2172
|
-
"sessionId": session_id,
|
|
2173
|
-
"version": "2.1.181"
|
|
2174
|
-
});
|
|
2175
|
-
if let Some(request_id) = request_id {
|
|
2176
|
-
record
|
|
2177
|
-
.as_object_mut()
|
|
2178
|
-
.unwrap()
|
|
2179
|
-
.insert("requestId".to_string(), serde_json::json!(request_id));
|
|
2180
|
-
}
|
|
2181
|
-
format!("{record}\n")
|
|
2182
|
-
}
|
|
2183
|
-
|
|
2184
|
-
fn claude_turn_duration_line(uuid: &str, parent_uuid: &str, session_id: &str) -> String {
|
|
2185
|
-
format!(
|
|
2186
|
-
"{}\n",
|
|
2187
|
-
serde_json::json!({
|
|
2188
|
-
"type": "system",
|
|
2189
|
-
"subtype": "turn_duration",
|
|
2190
|
-
"uuid": uuid,
|
|
2191
|
-
"parentUuid": parent_uuid,
|
|
2192
|
-
"sessionId": session_id
|
|
2193
|
-
})
|
|
2194
|
-
)
|
|
2195
|
-
}
|
|
2196
|
-
|
|
2197
|
-
fn claude_file_history_snapshot_line(session_id: &str) -> String {
|
|
2198
|
-
format!(
|
|
2199
|
-
"{}\n",
|
|
2200
|
-
serde_json::json!({
|
|
2201
|
-
"type": "file-history-snapshot",
|
|
2202
|
-
"sessionId": session_id
|
|
2203
|
-
})
|
|
2204
|
-
)
|
|
2205
|
-
}
|
|
2206
|
-
|
|
2207
|
-
fn claude_last_prompt_line(session_id: &str) -> String {
|
|
2208
|
-
format!(
|
|
2209
|
-
"{}\n",
|
|
2210
|
-
serde_json::json!({
|
|
2211
|
-
"type": "last-prompt",
|
|
2212
|
-
"sessionId": session_id,
|
|
2213
|
-
"prompt": "sanitized"
|
|
2214
|
-
})
|
|
2215
|
-
)
|
|
2216
|
-
}
|
|
2217
|
-
|
|
2218
|
-
fn claude_mode_line(session_id: &str) -> String {
|
|
2219
|
-
format!(
|
|
2220
|
-
"{}\n",
|
|
2221
|
-
serde_json::json!({
|
|
2222
|
-
"type": "mode",
|
|
2223
|
-
"sessionId": session_id,
|
|
2224
|
-
"mode": "default"
|
|
2225
|
-
})
|
|
2226
|
-
)
|
|
2227
|
-
}
|
|
2228
|
-
|
|
2229
|
-
fn abnormal_test_watch(dir: &std::path::Path) -> Value {
|
|
2230
|
-
crate::state::persist::load_runtime_state(dir)
|
|
2231
|
-
.unwrap()
|
|
2232
|
-
.pointer("/coordinator/abnormal_exit_watch/w1")
|
|
2233
|
-
.cloned()
|
|
2234
|
-
.unwrap()
|
|
2235
|
-
}
|
|
2236
|
-
|
|
2237
|
-
fn read_test_events(dir: &std::path::Path) -> Vec<Value> {
|
|
2238
|
-
let events_path = crate::model::paths::logs_dir(dir).join("events.jsonl");
|
|
2239
|
-
std::fs::read_to_string(events_path)
|
|
2240
|
-
.unwrap_or_default()
|
|
2241
|
-
.lines()
|
|
2242
|
-
.filter_map(|line| serde_json::from_str(line).ok())
|
|
2243
|
-
.collect()
|
|
2244
|
-
}
|
|
2245
|
-
|
|
2246
|
-
fn find_event(events: &[Value], name: &str) -> Option<Value> {
|
|
2247
|
-
events
|
|
2248
|
-
.iter()
|
|
2249
|
-
.rev()
|
|
2250
|
-
.find(|event| event.get("event").and_then(Value::as_str) == Some(name))
|
|
2251
|
-
.cloned()
|
|
2252
|
-
}
|
|
2253
|
-
|
|
2254
|
-
fn temp_abnormal_dir(tag: &str) -> std::path::PathBuf {
|
|
2255
|
-
let dir = std::env::temp_dir().join(format!(
|
|
2256
|
-
"team-agent-abnormal-{tag}-{}-{}",
|
|
2257
|
-
std::process::id(),
|
|
2258
|
-
std::time::SystemTime::now()
|
|
2259
|
-
.duration_since(std::time::UNIX_EPOCH)
|
|
2260
|
-
.unwrap()
|
|
2261
|
-
.as_nanos()
|
|
2262
|
-
));
|
|
2263
|
-
std::fs::create_dir_all(&dir).unwrap();
|
|
2264
|
-
dir
|
|
2265
|
-
}
|
|
2266
|
-
|
|
2267
|
-
fn test_abnormal_agent(
|
|
2268
|
-
rollout_path: &str,
|
|
2269
|
-
spawn_epoch: Option<u64>,
|
|
2270
|
-
spawned_at: Option<&str>,
|
|
2271
|
-
) -> AbnormalWatchAgent {
|
|
2272
|
-
AbnormalWatchAgent {
|
|
2273
|
-
agent_id: "w1".to_string(),
|
|
2274
|
-
provider: crate::provider::Provider::Codex,
|
|
2275
|
-
rollout_path: PathBuf::from(rollout_path),
|
|
2276
|
-
rollout_path_display: rollout_path.to_string(),
|
|
2277
|
-
spawn_epoch,
|
|
2278
|
-
spawned_at: spawned_at.map(str::to_string),
|
|
2279
|
-
status: Some("running".to_string()),
|
|
2280
|
-
process_liveness: Some(ProcessLiveness::Alive),
|
|
2281
|
-
window: None,
|
|
2282
|
-
pane_id: None,
|
|
2283
|
-
pid: None,
|
|
2284
|
-
current_command: None,
|
|
2285
|
-
}
|
|
2286
|
-
}
|
|
2287
|
-
|
|
2288
|
-
#[test]
|
|
2289
|
-
fn retired_legacy_rows_get_a_remove_and_re_add_hint_not_a_refused_start_agent() {
|
|
2290
|
-
use crate::model::enums::Provider;
|
|
2291
|
-
let workspace = std::path::Path::new("/tmp/ws");
|
|
2292
|
-
let hint = recovery_hint("w1", Provider::Codex, workspace, "research");
|
|
2293
|
-
assert!(hint.starts_with("No automatic restart was performed.\n"), "{hint}");
|
|
2294
|
-
assert!(
|
|
2295
|
-
hint.contains("seat has no runtime in this release; stop or remove it, then add it again"),
|
|
2296
|
-
"{hint}"
|
|
2297
|
-
);
|
|
2298
|
-
assert!(!hint.contains("start-agent"), "{hint}");
|
|
2299
|
-
assert!(hint.ends_with("(provider: codex)"), "{hint}");
|
|
2300
|
-
// `grok` rows are a runtime alias of grok_build, so they restart like any live provider.
|
|
2301
|
-
for provider in [Provider::Grok, Provider::Copilot] {
|
|
2302
|
-
let hint = recovery_hint("w1", provider, workspace, "research");
|
|
2303
|
-
assert!(
|
|
2304
|
-
hint.ends_with("To recover manually, run: team-agent start-agent w1 --workspace '/tmp/ws' --team research --force --json"),
|
|
2305
|
-
"{hint}"
|
|
2306
|
-
);
|
|
2307
|
-
}
|
|
2308
|
-
}
|
|
2309
|
-
}
|