@team-agent/installer 0.5.127 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (421) hide show
  1. package/README.md +15 -4
  2. package/README.zh.md +1 -1
  3. package/package.json +4 -8
  4. package/skills/team-agent/SKILL.md +41 -26
  5. package/skills/team-agent/command-coverage.json +170 -184
  6. package/skills/team-agent/docs/reference/team-agent-operator.md +66 -32
  7. package/skills/team-agent/references/recovery-runbook.md +1 -1
  8. package/skills/team-agent/references/team-agent-operator.md +66 -32
  9. package/templates/team_state.md +1 -1
  10. package/Cargo.lock +0 -1063
  11. package/Cargo.toml +0 -36
  12. package/crates/team-agent/Cargo.toml +0 -69
  13. package/crates/team-agent/src/app_server_test_support.rs +0 -258
  14. package/crates/team-agent/src/cli/adapters.rs +0 -1298
  15. package/crates/team-agent/src/cli/arguments.rs +0 -401
  16. package/crates/team-agent/src/cli/attach_app_server_leader.rs +0 -98
  17. package/crates/team-agent/src/cli/diagnose.rs +0 -1675
  18. package/crates/team-agent/src/cli/emit.rs +0 -2898
  19. package/crates/team-agent/src/cli/grok_slot.rs +0 -384
  20. package/crates/team-agent/src/cli/helpers.rs +0 -91
  21. package/crates/team-agent/src/cli/intent/runtime.rs +0 -1189
  22. package/crates/team-agent/src/cli/intent/runtime_tests.rs +0 -821
  23. package/crates/team-agent/src/cli/intent/tests.rs +0 -691
  24. package/crates/team-agent/src/cli/intent.rs +0 -664
  25. package/crates/team-agent/src/cli/leader.rs +0 -300
  26. package/crates/team-agent/src/cli/leader_prompt.rs +0 -261
  27. package/crates/team-agent/src/cli/leaders.rs +0 -418
  28. package/crates/team-agent/src/cli/mod.rs +0 -6528
  29. package/crates/team-agent/src/cli/models.rs +0 -449
  30. package/crates/team-agent/src/cli/named_address.rs +0 -1651
  31. package/crates/team-agent/src/cli/profile.rs +0 -342
  32. package/crates/team-agent/src/cli/route.rs +0 -212
  33. package/crates/team-agent/src/cli/send/coordinator.rs +0 -164
  34. package/crates/team-agent/src/cli/send/mailbox.rs +0 -145
  35. package/crates/team-agent/src/cli/send/persist.rs +0 -192
  36. package/crates/team-agent/src/cli/send/presentation.rs +0 -370
  37. package/crates/team-agent/src/cli/send/resolve.rs +0 -573
  38. package/crates/team-agent/src/cli/send.rs +0 -201
  39. package/crates/team-agent/src/cli/shutdown_socket.rs +0 -288
  40. package/crates/team-agent/src/cli/spec.rs +0 -159
  41. package/crates/team-agent/src/cli/status.rs +0 -280
  42. package/crates/team-agent/src/cli/status_port/agents.rs +0 -358
  43. package/crates/team-agent/src/cli/status_port/approvals.rs +0 -79
  44. package/crates/team-agent/src/cli/status_port/brief.rs +0 -2043
  45. package/crates/team-agent/src/cli/status_port/compact.rs +0 -240
  46. package/crates/team-agent/src/cli/status_port/format.rs +0 -146
  47. package/crates/team-agent/src/cli/status_port/inbox.rs +0 -62
  48. package/crates/team-agent/src/cli/status_port/runtime.rs +0 -197
  49. package/crates/team-agent/src/cli/status_port/snapshot.rs +0 -208
  50. package/crates/team-agent/src/cli/status_port/store.rs +0 -416
  51. package/crates/team-agent/src/cli/status_port/tests.rs +0 -90
  52. package/crates/team-agent/src/cli/status_port.rs +0 -92
  53. package/crates/team-agent/src/cli/tests/base.rs +0 -885
  54. package/crates/team-agent/src/cli/tests/divergence.rs +0 -518
  55. package/crates/team-agent/src/cli/tests/lane_c.rs +0 -453
  56. package/crates/team-agent/src/cli/tests/leader_watch.rs +0 -372
  57. package/crates/team-agent/src/cli/tests/main_preserved.rs +0 -430
  58. package/crates/team-agent/src/cli/tests/missing_subcommands.rs +0 -336
  59. package/crates/team-agent/src/cli/tests/mod.rs +0 -137
  60. package/crates/team-agent/src/cli/tests/named_address.rs +0 -760
  61. package/crates/team-agent/src/cli/tests/parser_safety.rs +0 -383
  62. package/crates/team-agent/src/cli/tests/run_delegation.rs +0 -481
  63. package/crates/team-agent/src/cli/tests/shutdown_kill_plan.rs +0 -2179
  64. package/crates/team-agent/src/cli/tests/status_brief.rs +0 -1514
  65. package/crates/team-agent/src/cli/tests/status_send.rs +0 -1091
  66. package/crates/team-agent/src/cli/tests/verb_install_skill.rs +0 -93
  67. package/crates/team-agent/src/cli/tests/verb_profile.rs +0 -392
  68. package/crates/team-agent/src/cli/triage.rs +0 -234
  69. package/crates/team-agent/src/cli/types.rs +0 -591
  70. package/crates/team-agent/src/codex_app_server.rs +0 -546
  71. package/crates/team-agent/src/communication_mode/mod.rs +0 -56
  72. package/crates/team-agent/src/compiler/tests.rs +0 -842
  73. package/crates/team-agent/src/compiler.rs +0 -691
  74. package/crates/team-agent/src/conpty/backend.rs +0 -829
  75. package/crates/team-agent/src/conpty/mod.rs +0 -33
  76. package/crates/team-agent/src/contract_runtime/backend.rs +0 -1610
  77. package/crates/team-agent/src/contract_runtime/commands.rs +0 -671
  78. package/crates/team-agent/src/contract_runtime/config.rs +0 -354
  79. package/crates/team-agent/src/contract_runtime/config_tests.rs +0 -56
  80. package/crates/team-agent/src/contract_runtime/discovery.rs +0 -438
  81. package/crates/team-agent/src/contract_runtime/fork.rs +0 -368
  82. package/crates/team-agent/src/contract_runtime/forward.rs +0 -288
  83. package/crates/team-agent/src/contract_runtime/framework.rs +0 -350
  84. package/crates/team-agent/src/contract_runtime/mcp.rs +0 -399
  85. package/crates/team-agent/src/contract_runtime/mod.rs +0 -17
  86. package/crates/team-agent/src/contract_runtime/prompt.rs +0 -216
  87. package/crates/team-agent/src/contract_runtime/pump.rs +0 -271
  88. package/crates/team-agent/src/contract_runtime/registry.rs +0 -589
  89. package/crates/team-agent/src/contract_runtime/workspace.rs +0 -228
  90. package/crates/team-agent/src/coordinator/backoff.rs +0 -741
  91. package/crates/team-agent/src/coordinator/conpty_shim.rs +0 -805
  92. package/crates/team-agent/src/coordinator/health.rs +0 -1185
  93. package/crates/team-agent/src/coordinator/mod.rs +0 -119
  94. package/crates/team-agent/src/coordinator/orphan.rs +0 -110
  95. package/crates/team-agent/src/coordinator/runtime_detectors.rs +0 -798
  96. package/crates/team-agent/src/coordinator/runtime_observation.rs +0 -84
  97. package/crates/team-agent/src/coordinator/steps/abnormal.rs +0 -2309
  98. package/crates/team-agent/src/coordinator/steps/delivery.rs +0 -15
  99. package/crates/team-agent/src/coordinator/steps/health_sync.rs +0 -15
  100. package/crates/team-agent/src/coordinator/steps/mod.rs +0 -126
  101. package/crates/team-agent/src/coordinator/steps/persist.rs +0 -15
  102. package/crates/team-agent/src/coordinator/steps/runtime_prompts.rs +0 -15
  103. package/crates/team-agent/src/coordinator/steps/session_gate.rs +0 -16
  104. package/crates/team-agent/src/coordinator/tests/a0_lostupdate.rs +0 -158
  105. package/crates/team-agent/src/coordinator/tests/abnormal.rs +0 -312
  106. package/crates/team-agent/src/coordinator/tests/api_error_recovery.rs +0 -338
  107. package/crates/team-agent/src/coordinator/tests/basics.rs +0 -211
  108. package/crates/team-agent/src/coordinator/tests/daemon.rs +0 -514
  109. package/crates/team-agent/src/coordinator/tests/energy.rs +0 -549
  110. package/crates/team-agent/src/coordinator/tests/health_sync.rs +0 -461
  111. package/crates/team-agent/src/coordinator/tests/main_preserved.rs +0 -186
  112. package/crates/team-agent/src/coordinator/tests/mod.rs +0 -295
  113. package/crates/team-agent/src/coordinator/tests/s3_bounded.rs +0 -207
  114. package/crates/team-agent/src/coordinator/tests/spine.rs +0 -308
  115. package/crates/team-agent/src/coordinator/tests/takeover.rs +0 -286
  116. package/crates/team-agent/src/coordinator/tests/tick_core.rs +0 -237
  117. package/crates/team-agent/src/coordinator/tick.rs +0 -3112
  118. package/crates/team-agent/src/coordinator/types.rs +0 -540
  119. package/crates/team-agent/src/db/agent_health_capture.rs +0 -126
  120. package/crates/team-agent/src/db/message_store.rs +0 -1585
  121. package/crates/team-agent/src/db/migration.rs +0 -984
  122. package/crates/team-agent/src/db/mod.rs +0 -26
  123. package/crates/team-agent/src/db/schema.rs +0 -663
  124. package/crates/team-agent/src/diagnose/comms.rs +0 -242
  125. package/crates/team-agent/src/diagnose/mod.rs +0 -44
  126. package/crates/team-agent/src/diagnose/orphans.rs +0 -889
  127. package/crates/team-agent/src/event_log.rs +0 -502
  128. package/crates/team-agent/src/fake_worker.rs +0 -303
  129. package/crates/team-agent/src/kill_audit.rs +0 -106
  130. package/crates/team-agent/src/layout/manager.rs +0 -248
  131. package/crates/team-agent/src/layout/mod.rs +0 -36
  132. package/crates/team-agent/src/layout/overlay.rs +0 -134
  133. package/crates/team-agent/src/layout/placement.rs +0 -52
  134. package/crates/team-agent/src/layout/recovery.rs +0 -106
  135. package/crates/team-agent/src/layout/runtime_sessions.rs +0 -317
  136. package/crates/team-agent/src/layout/sessions.rs +0 -294
  137. package/crates/team-agent/src/layout/tmux_endpoint.rs +0 -165
  138. package/crates/team-agent/src/layout/worker_env.rs +0 -721
  139. package/crates/team-agent/src/layout/worker_window_helpers.rs +0 -37
  140. package/crates/team-agent/src/leader/helpers.rs +0 -178
  141. package/crates/team-agent/src/leader/incident.rs +0 -6
  142. package/crates/team-agent/src/leader/lease.rs +0 -3356
  143. package/crates/team-agent/src/leader/mod.rs +0 -104
  144. package/crates/team-agent/src/leader/owner_bind.rs +0 -400
  145. package/crates/team-agent/src/leader/prompt.rs +0 -905
  146. package/crates/team-agent/src/leader/provider_attribution.rs +0 -247
  147. package/crates/team-agent/src/leader/rediscover/tests.rs +0 -689
  148. package/crates/team-agent/src/leader/rediscover.rs +0 -1192
  149. package/crates/team-agent/src/leader/registry.rs +0 -883
  150. package/crates/team-agent/src/leader/start.rs +0 -4500
  151. package/crates/team-agent/src/leader/takeover.rs +0 -65
  152. package/crates/team-agent/src/leader/tests/basics.rs +0 -337
  153. package/crates/team-agent/src/leader/tests/byte_findings.rs +0 -355
  154. package/crates/team-agent/src/leader/tests/claim_leader_mailbox_flush_contract.rs +0 -544
  155. package/crates/team-agent/src/leader/tests/historical_inventory_upgrade_gate_red.rs +0 -624
  156. package/crates/team-agent/src/leader/tests/identity.rs +0 -465
  157. package/crates/team-agent/src/leader/tests/identity_session_names.rs +0 -42
  158. package/crates/team-agent/src/leader/tests/idle.rs +0 -113
  159. package/crates/team-agent/src/leader/tests/lease_api.rs +0 -520
  160. package/crates/team-agent/src/leader/tests/lease_claim.rs +0 -629
  161. package/crates/team-agent/src/leader/tests/mod.rs +0 -106
  162. package/crates/team-agent/src/leader/tests/prompt_contract.rs +0 -267
  163. package/crates/team-agent/src/leader/tests/rediscover.rs +0 -403
  164. package/crates/team-agent/src/leader/tests/rehomed_env.rs +0 -83
  165. package/crates/team-agent/src/leader/tests/team_in_team_state_scope_red.rs +0 -1273
  166. package/crates/team-agent/src/leader/tests/terminal_and_replay_vs_rerender_invariants_red.rs +0 -417
  167. package/crates/team-agent/src/leader/tests/wake_start_owner.rs +0 -168
  168. package/crates/team-agent/src/leader/types.rs +0 -534
  169. package/crates/team-agent/src/lib.rs +0 -145
  170. package/crates/team-agent/src/lifecycle/helpers.rs +0 -212
  171. package/crates/team-agent/src/lifecycle/launch/add_agent.rs +0 -848
  172. package/crates/team-agent/src/lifecycle/launch/add_agent_state.rs +0 -393
  173. package/crates/team-agent/src/lifecycle/launch/agent_state.rs +0 -332
  174. package/crates/team-agent/src/lifecycle/launch/clone_agent.rs +0 -178
  175. package/crates/team-agent/src/lifecycle/launch/fork_agent.rs +0 -82
  176. package/crates/team-agent/src/lifecycle/launch/fork_entry.rs +0 -63
  177. package/crates/team-agent/src/lifecycle/launch/identity.rs +0 -674
  178. package/crates/team-agent/src/lifecycle/launch/layout.rs +0 -350
  179. package/crates/team-agent/src/lifecycle/launch/leader_context.rs +0 -589
  180. package/crates/team-agent/src/lifecycle/launch/mcp_config.rs +0 -152
  181. package/crates/team-agent/src/lifecycle/launch/ownership.rs +0 -105
  182. package/crates/team-agent/src/lifecycle/launch/pi_mcp.rs +0 -629
  183. package/crates/team-agent/src/lifecycle/launch/pi_mcp_extension.js +0 -226
  184. package/crates/team-agent/src/lifecycle/launch/plan.rs +0 -269
  185. package/crates/team-agent/src/lifecycle/launch/quick_start.rs +0 -5176
  186. package/crates/team-agent/src/lifecycle/launch/quick_start_transport.rs +0 -368
  187. package/crates/team-agent/src/lifecycle/launch/readiness.rs +0 -519
  188. package/crates/team-agent/src/lifecycle/launch/role_source.rs +0 -287
  189. package/crates/team-agent/src/lifecycle/launch/spawn.rs +0 -450
  190. package/crates/team-agent/src/lifecycle/launch/spec_state.rs +0 -801
  191. package/crates/team-agent/src/lifecycle/launch/state_projection.rs +0 -624
  192. package/crates/team-agent/src/lifecycle/launch/worker_env.rs +0 -558
  193. package/crates/team-agent/src/lifecycle/launch.rs +0 -534
  194. package/crates/team-agent/src/lifecycle/lock.rs +0 -350
  195. package/crates/team-agent/src/lifecycle/mod.rs +0 -79
  196. package/crates/team-agent/src/lifecycle/pane_input_lock.rs +0 -239
  197. package/crates/team-agent/src/lifecycle/profile_launch.rs +0 -757
  198. package/crates/team-agent/src/lifecycle/profile_smoke.rs +0 -557
  199. package/crates/team-agent/src/lifecycle/restart/agent.rs +0 -2198
  200. package/crates/team-agent/src/lifecycle/restart/common.rs +0 -2052
  201. package/crates/team-agent/src/lifecycle/restart/orchestrator.rs +0 -65
  202. package/crates/team-agent/src/lifecycle/restart/preflight.rs +0 -113
  203. package/crates/team-agent/src/lifecycle/restart/rebuild.rs +0 -5135
  204. package/crates/team-agent/src/lifecycle/restart/remove.rs +0 -1504
  205. package/crates/team-agent/src/lifecycle/restart/selection.rs +0 -559
  206. package/crates/team-agent/src/lifecycle/restart/team_state.rs +0 -329
  207. package/crates/team-agent/src/lifecycle/restart.rs +0 -223
  208. package/crates/team-agent/src/lifecycle/role_config.rs +0 -1205
  209. package/crates/team-agent/src/lifecycle/tests/acceptance_b_batch_red.rs +0 -1021
  210. package/crates/team-agent/src/lifecycle/tests/agent_ops.rs +0 -1205
  211. package/crates/team-agent/src/lifecycle/tests/b0_legacy_snapshot_nonauthority_contract.rs +0 -265
  212. package/crates/team-agent/src/lifecycle/tests/b0_reader_hideone_audit_contract.rs +0 -490
  213. package/crates/team-agent/src/lifecycle/tests/behavioral_diff_264_red.rs +0 -656
  214. package/crates/team-agent/src/lifecycle/tests/claude_profile_launch_red.rs +0 -135
  215. package/crates/team-agent/src/lifecycle/tests/cli_argv_routing_red.rs +0 -1415
  216. package/crates/team-agent/src/lifecycle/tests/clone_agent_preserves_source_tools.rs +0 -427
  217. package/crates/team-agent/src/lifecycle/tests/clone_fork_copilot_perms_red.rs +0 -396
  218. package/crates/team-agent/src/lifecycle/tests/communication_mode_runtime_contract_red.rs +0 -418
  219. package/crates/team-agent/src/lifecycle/tests/copilot_provider_red.rs +0 -2039
  220. package/crates/team-agent/src/lifecycle/tests/core.rs +0 -1298
  221. package/crates/team-agent/src/lifecycle/tests/f032_startup_prompt_best_effort_red.rs +0 -315
  222. package/crates/team-agent/src/lifecycle/tests/fixtures/pi_list_models_g0.stdout.txt +0 -222
  223. package/crates/team-agent/src/lifecycle/tests/fixtures/pi_list_models_spaced_ids.stdout.txt +0 -9
  224. package/crates/team-agent/src/lifecycle/tests/fixtures/pi_native_mcp_host.mjs +0 -71
  225. package/crates/team-agent/src/lifecycle/tests/gate_fixtures.rs +0 -562
  226. package/crates/team-agent/src/lifecycle/tests/grok_require_explicit_model_red.rs +0 -110
  227. package/crates/team-agent/src/lifecycle/tests/harvest2_a_batch_red.rs +0 -472
  228. package/crates/team-agent/src/lifecycle/tests/host_cotenant_death_p0_contract.rs +0 -921
  229. package/crates/team-agent/src/lifecycle/tests/issue238_codex_effort_contract.rs +0 -68
  230. package/crates/team-agent/src/lifecycle/tests/lane_ops.rs +0 -1851
  231. package/crates/team-agent/src/lifecycle/tests/launch_spawn.rs +0 -3981
  232. package/crates/team-agent/src/lifecycle/tests/lifecycle_lock.rs +0 -339
  233. package/crates/team-agent/src/lifecycle/tests/lifecycle_rollback_red.rs +0 -596
  234. package/crates/team-agent/src/lifecycle/tests/main_preserved.rs +0 -513
  235. package/crates/team-agent/src/lifecycle/tests/mcp_tool_name_format_red.rs +0 -69
  236. package/crates/team-agent/src/lifecycle/tests/phase_b_contracts.rs +0 -824
  237. package/crates/team-agent/src/lifecycle/tests/phase_golden.rs +0 -723
  238. package/crates/team-agent/src/lifecycle/tests/pi_catalog_red.rs +0 -152
  239. package/crates/team-agent/src/lifecycle/tests/pi_command_red.rs +0 -182
  240. package/crates/team-agent/src/lifecycle/tests/pi_dual_entry_red.rs +0 -374
  241. package/crates/team-agent/src/lifecycle/tests/pi_executable_mcp_red.rs +0 -546
  242. package/crates/team-agent/src/lifecycle/tests/pi_lifecycle_doctor_red.rs +0 -31
  243. package/crates/team-agent/src/lifecycle/tests/pi_native_mcp_capability_red.rs +0 -528
  244. package/crates/team-agent/src/lifecycle/tests/pi_session_delivery_red.rs +0 -134
  245. package/crates/team-agent/src/lifecycle/tests/quick_start_worker_readiness_red.rs +0 -333
  246. package/crates/team-agent/src/lifecycle/tests/realmachine_clusters_1_6_red.rs +0 -786
  247. package/crates/team-agent/src/lifecycle/tests/realmachine_residual_g1_g4_red.rs +0 -648
  248. package/crates/team-agent/src/lifecycle/tests/restart.rs +0 -185
  249. package/crates/team-agent/src/lifecycle/tests/restart_build_before_destroy_0540_contract.rs +0 -1053
  250. package/crates/team-agent/src/lifecycle/tests/restart_liveness_red.rs +0 -679
  251. package/crates/team-agent/src/lifecycle/tests/restart_rebind_hotfix_252_red.rs +0 -1126
  252. package/crates/team-agent/src/lifecycle/tests/restart_session_capture_red.rs +0 -153
  253. package/crates/team-agent/src/lifecycle/tests/resume_recover_red.rs +0 -427
  254. package/crates/team-agent/src/lifecycle/tests/stale_team_saveconflict_contract.rs +0 -431
  255. package/crates/team-agent/src/lifecycle/tests/startup_latency_contract.rs +0 -1066
  256. package/crates/team-agent/src/lifecycle/tests/status_credential_redaction_contract.rs +0 -876
  257. package/crates/team-agent/src/lifecycle/tests/subprep_codex_trust_roundtrip_red.rs +0 -1026
  258. package/crates/team-agent/src/lifecycle/tests/swallow_batch4_semantics_red.rs +0 -90
  259. package/crates/team-agent/src/lifecycle/tests/team_in_team_identity_scope_red.rs +0 -543
  260. package/crates/team-agent/src/lifecycle/tests/team_in_team_sibling_quick_start_red.rs +0 -1103
  261. package/crates/team-agent/src/lifecycle/tests/team_in_team_state_scope_red.rs +0 -1276
  262. package/crates/team-agent/src/lifecycle/tests/team_key_retirement_roster_red.rs +0 -145
  263. package/crates/team-agent/src/lifecycle/tests/team_key_retirement_txn_red.rs +0 -603
  264. package/crates/team-agent/src/lifecycle/tests/test_isolation_escape_contract.rs +0 -936
  265. package/crates/team-agent/src/lifecycle/tests/upgrade_compat_0211_red.rs +0 -970
  266. package/crates/team-agent/src/lifecycle/tests/verify_rs031_window_consistency_red.rs +0 -996
  267. package/crates/team-agent/src/lifecycle/tests/worker_spawn_env_red.rs +0 -827
  268. package/crates/team-agent/src/lifecycle/tests.rs +0 -102
  269. package/crates/team-agent/src/lifecycle/types.rs +0 -861
  270. package/crates/team-agent/src/lifecycle/worker_command_context.rs +0 -370
  271. package/crates/team-agent/src/main.rs +0 -52
  272. package/crates/team-agent/src/mcp_server/helpers.rs +0 -510
  273. package/crates/team-agent/src/mcp_server/lifecycle_tools/mod.rs +0 -6
  274. package/crates/team-agent/src/mcp_server/lifecycle_tools/state_status.rs +0 -43
  275. package/crates/team-agent/src/mcp_server/mod.rs +0 -119
  276. package/crates/team-agent/src/mcp_server/normalize.rs +0 -485
  277. package/crates/team-agent/src/mcp_server/readiness.rs +0 -546
  278. package/crates/team-agent/src/mcp_server/readiness_tests.rs +0 -369
  279. package/crates/team-agent/src/mcp_server/tests/golden.rs +0 -285
  280. package/crates/team-agent/src/mcp_server/tests/normalize.rs +0 -247
  281. package/crates/team-agent/src/mcp_server/tests/scoped.rs +0 -168
  282. package/crates/team-agent/src/mcp_server/tests/send.rs +0 -388
  283. package/crates/team-agent/src/mcp_server/tests/tools.rs +0 -343
  284. package/crates/team-agent/src/mcp_server/tests/wire.rs +0 -254
  285. package/crates/team-agent/src/mcp_server/tests.rs +0 -43
  286. package/crates/team-agent/src/mcp_server/tools.rs +0 -1015
  287. package/crates/team-agent/src/mcp_server/types.rs +0 -396
  288. package/crates/team-agent/src/mcp_server/wire.rs +0 -867
  289. package/crates/team-agent/src/messaging/activity.rs +0 -461
  290. package/crates/team-agent/src/messaging/address.rs +0 -87
  291. package/crates/team-agent/src/messaging/delivery.rs +0 -3785
  292. package/crates/team-agent/src/messaging/delivery_consumption_tests.rs +0 -234
  293. package/crates/team-agent/src/messaging/enter_rescue.rs +0 -362
  294. package/crates/team-agent/src/messaging/enter_rescue_tests.rs +0 -633
  295. package/crates/team-agent/src/messaging/helpers.rs +0 -354
  296. package/crates/team-agent/src/messaging/leader_channel.rs +0 -287
  297. package/crates/team-agent/src/messaging/leader_receiver.rs +0 -786
  298. package/crates/team-agent/src/messaging/mod.rs +0 -166
  299. package/crates/team-agent/src/messaging/persist.rs +0 -377
  300. package/crates/team-agent/src/messaging/presentation.rs +0 -417
  301. package/crates/team-agent/src/messaging/results/collection_tests.rs +0 -719
  302. package/crates/team-agent/src/messaging/results.rs +0 -1415
  303. package/crates/team-agent/src/messaging/scheduler.rs +0 -217
  304. package/crates/team-agent/src/messaging/selftest.rs +0 -315
  305. package/crates/team-agent/src/messaging/send.rs +0 -981
  306. package/crates/team-agent/src/messaging/tests/basic.rs +0 -346
  307. package/crates/team-agent/src/messaging/tests/dup_inject.rs +0 -551
  308. package/crates/team-agent/src/messaging/tests/e23.rs +0 -49
  309. package/crates/team-agent/src/messaging/tests/leader_channel.rs +0 -169
  310. package/crates/team-agent/src/messaging/tests/leader_inject_acceptance.rs +0 -817
  311. package/crates/team-agent/src/messaging/tests/main_preserved.rs +0 -85
  312. package/crates/team-agent/src/messaging/tests/mod.rs +0 -306
  313. package/crates/team-agent/src/messaging/tests/runtime.rs +0 -3655
  314. package/crates/team-agent/src/messaging/tests/spine.rs +0 -837
  315. package/crates/team-agent/src/messaging/tests/wave2.rs +0 -381
  316. package/crates/team-agent/src/messaging/trust.rs +0 -215
  317. package/crates/team-agent/src/messaging/types.rs +0 -471
  318. package/crates/team-agent/src/messaging/wait.rs +0 -366
  319. package/crates/team-agent/src/messaging/watchers.rs +0 -916
  320. package/crates/team-agent/src/model/enums.rs +0 -356
  321. package/crates/team-agent/src/model/errors.rs +0 -18
  322. package/crates/team-agent/src/model/ids.rs +0 -165
  323. package/crates/team-agent/src/model/mod.rs +0 -29
  324. package/crates/team-agent/src/model/name_similarity.rs +0 -252
  325. package/crates/team-agent/src/model/pane_authority_refusal.rs +0 -359
  326. package/crates/team-agent/src/model/paths.rs +0 -257
  327. package/crates/team-agent/src/model/routing.rs +0 -387
  328. package/crates/team-agent/src/model/spec.rs +0 -1184
  329. package/crates/team-agent/src/model/task_graph.rs +0 -321
  330. package/crates/team-agent/src/model/testdata/fuzz.golden.yaml +0 -43
  331. package/crates/team-agent/src/model/testdata/fuzz.yaml +0 -43
  332. package/crates/team-agent/src/model/testdata/spec_invalid_a.yaml +0 -209
  333. package/crates/team-agent/src/model/testdata/team.spec.golden.yaml +0 -208
  334. package/crates/team-agent/src/model/testdata/team.spec.yaml +0 -208
  335. package/crates/team-agent/src/model/yaml/tests.rs +0 -188
  336. package/crates/team-agent/src/model/yaml.rs +0 -802
  337. package/crates/team-agent/src/os_probe.rs +0 -280
  338. package/crates/team-agent/src/packaging/install.rs +0 -219
  339. package/crates/team-agent/src/packaging/migrate.rs +0 -38
  340. package/crates/team-agent/src/packaging/mod.rs +0 -91
  341. package/crates/team-agent/src/packaging/repair.rs +0 -33
  342. package/crates/team-agent/src/packaging/tests.rs +0 -822
  343. package/crates/team-agent/src/packaging/types.rs +0 -417
  344. package/crates/team-agent/src/platform/argv.rs +0 -331
  345. package/crates/team-agent/src/platform/errors.rs +0 -104
  346. package/crates/team-agent/src/platform/file_lock.rs +0 -279
  347. package/crates/team-agent/src/platform/mod.rs +0 -5
  348. package/crates/team-agent/src/platform/process.rs +0 -545
  349. package/crates/team-agent/src/provider/adapter.rs +0 -898
  350. package/crates/team-agent/src/provider/adapters/copilot.rs +0 -75
  351. package/crates/team-agent/src/provider/adapters/copilot_fork.rs +0 -307
  352. package/crates/team-agent/src/provider/adapters/fake.rs +0 -18
  353. package/crates/team-agent/src/provider/adapters/mod.rs +0 -28
  354. package/crates/team-agent/src/provider/approvals/mod.rs +0 -3
  355. package/crates/team-agent/src/provider/approvals/parsing.rs +0 -586
  356. package/crates/team-agent/src/provider/approvals/runtime_prompts.rs +0 -205
  357. package/crates/team-agent/src/provider/argv_route.rs +0 -405
  358. package/crates/team-agent/src/provider/bypass_flags.rs +0 -116
  359. package/crates/team-agent/src/provider/classify.rs +0 -773
  360. package/crates/team-agent/src/provider/command_helpers.rs +0 -45
  361. package/crates/team-agent/src/provider/faults.rs +0 -165
  362. package/crates/team-agent/src/provider/helpers.rs +0 -52
  363. package/crates/team-agent/src/provider/mod.rs +0 -78
  364. package/crates/team-agent/src/provider/model_catalog.rs +0 -722
  365. package/crates/team-agent/src/provider/session/capture.rs +0 -2631
  366. package/crates/team-agent/src/provider/session/context_fork/outcome.rs +0 -168
  367. package/crates/team-agent/src/provider/session/context_fork.rs +0 -264
  368. package/crates/team-agent/src/provider/session/mod.rs +0 -36
  369. package/crates/team-agent/src/provider/session/resume.rs +0 -269
  370. package/crates/team-agent/src/provider/session_scan/common/tests.rs +0 -316
  371. package/crates/team-agent/src/provider/session_scan/common.rs +0 -618
  372. package/crates/team-agent/src/provider/session_scan/copilot.rs +0 -280
  373. package/crates/team-agent/src/provider/session_scan/pi.rs +0 -152
  374. package/crates/team-agent/src/provider/session_scan.rs +0 -61
  375. package/crates/team-agent/src/provider/startup_prompt.rs +0 -757
  376. package/crates/team-agent/src/provider/submit_now.rs +0 -71
  377. package/crates/team-agent/src/provider/testdata/copilot-ready-marker.txt +0 -5
  378. package/crates/team-agent/src/provider/testdata/copilot-trust-prompt.txt +0 -19
  379. package/crates/team-agent/src/provider/testdata/grok-models-authenticated.stdout +0 -9
  380. package/crates/team-agent/src/provider/testdata/grok-models-unauthenticated.stdout +0 -7
  381. package/crates/team-agent/src/provider/tests/adapter.rs +0 -354
  382. package/crates/team-agent/src/provider/tests/classify.rs +0 -201
  383. package/crates/team-agent/src/provider/tests/command_helpers.rs +0 -26
  384. package/crates/team-agent/src/provider/tests/copilot_fork.rs +0 -191
  385. package/crates/team-agent/src/provider/tests/faults.rs +0 -200
  386. package/crates/team-agent/src/provider/tests/idle.rs +0 -140
  387. package/crates/team-agent/src/provider/tests/wire.rs +0 -213
  388. package/crates/team-agent/src/provider/tests.rs +0 -32
  389. package/crates/team-agent/src/provider/types.rs +0 -529
  390. package/crates/team-agent/src/provider/wire.rs +0 -406
  391. package/crates/team-agent/src/redaction.rs +0 -259
  392. package/crates/team-agent/src/state/abnormal_watch/tests/mod.rs +0 -248
  393. package/crates/team-agent/src/state/abnormal_watch.rs +0 -144
  394. package/crates/team-agent/src/state/identity.rs +0 -914
  395. package/crates/team-agent/src/state/identity_keys.rs +0 -132
  396. package/crates/team-agent/src/state/mod.rs +0 -74
  397. package/crates/team-agent/src/state/owner_gate.rs +0 -482
  398. package/crates/team-agent/src/state/ownership.rs +0 -563
  399. package/crates/team-agent/src/state/paths.rs +0 -395
  400. package/crates/team-agent/src/state/persist.rs +0 -3811
  401. package/crates/team-agent/src/state/projection.rs +0 -1508
  402. package/crates/team-agent/src/state/repository/intent.rs +0 -14
  403. package/crates/team-agent/src/state/repository/tests.rs +0 -184
  404. package/crates/team-agent/src/state/repository.rs +0 -866
  405. package/crates/team-agent/src/state/selector.rs +0 -160
  406. package/crates/team-agent/src/state/testdata/state-rich.canonical.json +0 -133
  407. package/crates/team-agent/src/tmux_backend/enter_rescue.rs +0 -325
  408. package/crates/team-agent/src/tmux_backend/enter_rescue_tests.rs +0 -432
  409. package/crates/team-agent/src/tmux_backend/native_input.rs +0 -513
  410. package/crates/team-agent/src/tmux_backend/native_input_tests.rs +0 -742
  411. package/crates/team-agent/src/tmux_backend/tests.rs +0 -4254
  412. package/crates/team-agent/src/tmux_backend.rs +0 -4527
  413. package/crates/team-agent/src/topology.rs +0 -537
  414. package/crates/team-agent/src/transport/enter_rescue.rs +0 -166
  415. package/crates/team-agent/src/transport/native_input.rs +0 -491
  416. package/crates/team-agent/src/transport/test_support.rs +0 -743
  417. package/crates/team-agent/src/transport/tests/behavior.rs +0 -326
  418. package/crates/team-agent/src/transport/tests/mod.rs +0 -194
  419. package/crates/team-agent/src/transport/tests/wire.rs +0 -574
  420. package/crates/team-agent/src/transport.rs +0 -1655
  421. package/crates/team-agent/src/transport_factory.rs +0 -819
@@ -1,2309 +0,0 @@
1
- //! ---
2
- //! purpose: tick 的异常退出检测步骤——按 agent 读有界 rollout 尾,判「最新一条显式 provider 错误是否属于当前 worker 世代」,据此决定通知还是只留审计
3
- //! contract:
4
- //! provides:
5
- //! - name: detect_abnormal_exits
6
- //! what: 遍历受监视 agent,产出 worker.abnormal_exit 通知或 dead_only 审计事件,并把本轮观测写回 state 的 abnormal_exit_watch
7
- //! - name: explicit_process_liveness
8
- //! what: 从 agent JSON 的显式存活字段解析进程存活态,不做任何探测
9
- //! - name: read_tail_text
10
- //! what: 有界读文件尾部文本,超长时从尾部 max_bytes 处起读
11
- //! - name: metadata_mtime_ns
12
- //! what: 把文件 metadata 的 mtime 归一成纳秒整数,供跨 tick 变化比较
13
- //! depends:
14
- //! - super::super::health
15
- //! - super::super::tick
16
- //! - super::super::types
17
- //! - crate::event_log
18
- //! - crate::messaging
19
- //! - crate::provider
20
- //! - crate::transport
21
- //! - crate::state::projection
22
- //! boundary:
23
- //! - 不重启、不清理、不做物理注入:写事件与 state 观测记录之外,异常通知会经 crate::messaging 的 leader funnel 入库为 accepted 消息(物理注入由后续 deliver_pending 完成)
24
- //! - 进程已死但没有显式错误时不通知,只落 dead_only 审计
25
- //! - 通知事件里的 queued 只表示消息入库,不表示已送达
26
- //! - rollout 尾读是有界的(首行可能被截断,消费方只解析最后一条完整 JSONL 记录)
27
- //! maturity: wired
28
- //! ---
29
- //!
30
- //! unit-11 (Stage 4) — coordinator tick `abnormal` step group.
31
- //!
32
- //! Abnormal-exit detection + classification step extracted from
33
- //! `coordinator/tick.rs` without behavior changes.
34
-
35
- use std::path::{Path, PathBuf};
36
-
37
- use serde_json::Value;
38
-
39
- use crate::event_log::EventLog;
40
- use crate::provider::wire::{parse_provider, provider_wire};
41
- use crate::provider::ProcessLiveness;
42
-
43
- use super::super::health::pid_is_running;
44
- use super::super::tick::TickError;
45
- use super::super::types::Pid;
46
-
47
- ///
48
- /// #236 `worker.abnormal_exit` watcher.
49
- ///
50
- /// Notify only when the bounded transcript/rollout tail contains a latest explicit
51
- /// provider error that is fresh for the current worker cohort. A dead process
52
- /// without an explicit error remains a suppressed `dead_only` audit event; process
53
- /// liveness is otherwise diagnostic data for this path.
54
- /// ---
55
- /// purpose: 跑一轮异常退出检测,把结论落成事件与 state 里的 abnormal_exit_watch 记录
56
- /// params:
57
- /// workspace: workspace 根,用于解析 rollout 相对路径
58
- /// transport: pane 存活探测通道;只读探测,不注入
59
- /// state: 可变运行时状态;本轮观测(size/mtime/世代/去重键)写回 coordinator.abnormal_exit_watch
60
- /// event_log: 事件出口
61
- /// targets: 本 tick 已采到的 pane 快照,避免每 agent 重新列一遍
62
- /// returns: 检测跑完即 Ok;「有没有异常」不由返回值表达,只体现在事件与 state 里
63
- /// errors: 事件写入等不可恢复失败时返回 TickError;单个 agent 的 rollout metadata 读不到不算错误,记为 unverifiable 继续
64
- /// ---
65
- pub(crate) fn detect_abnormal_exits(
66
- workspace: &Path,
67
- transport: &dyn crate::transport::Transport,
68
- state: &mut Value,
69
- event_log: &EventLog,
70
- targets: &[crate::transport::PaneInfo],
71
- ) -> Result<(), TickError> {
72
- let snapshot = state.clone();
73
- let team = crate::state::projection::team_state_key(&snapshot);
74
- let session_name = snapshot.get("session_name").and_then(Value::as_str);
75
- for agent in abnormal_watch_agents(&snapshot) {
76
- // Reject untrusted history without skipping a fresh provider/path observation.
77
- crate::state::abnormal_watch::begin_observation(state, &agent.agent_id);
78
- // Pane/process liveness is independent of transcript content. A
79
- // frozen rollout is common after pane death, so probe before the
80
- // metadata dedupe gate; only the expensive tail scan stays deduped.
81
- let liveness = agent_process_liveness(&agent, session_name, targets, transport);
82
- let rollout_path = resolve_agent_rollout_path(workspace, &agent.rollout_path);
83
- let metadata = match std::fs::metadata(&rollout_path) {
84
- Ok(metadata) => metadata,
85
- Err(error) => {
86
- upsert_abnormal_watch(
87
- state,
88
- &agent.agent_id,
89
- abnormal_watch_payload(
90
- &agent,
91
- None,
92
- None,
93
- process_check(
94
- ProcessLiveness::Unverifiable,
95
- "rollout_metadata_unavailable".to_string(),
96
- ),
97
- None,
98
- ErrorRecency::None,
99
- None,
100
- Some(error.to_string()),
101
- ),
102
- );
103
- continue;
104
- }
105
- };
106
- let size = metadata.len();
107
- let mtime_ns = metadata_mtime_ns(&metadata);
108
- // P1 (C-P1-2/3): (size, mtime_ns) pair gate — an unchanged transcript is not
109
- // read at all (live sample: 332MB whole-file read per agent per 2s tick).
110
- // ANY field change (including a size shrink / truncate) falls through to the
111
- // re-read below.
112
- if let (Some(mtime), Some(stored)) = (
113
- mtime_ns,
114
- abnormal_watch_stored_metadata(state, &agent.agent_id),
115
- ) {
116
- if stored == (size, mtime) {
117
- refresh_abnormal_watch_liveness(state, &agent.agent_id, &liveness);
118
- continue;
119
- }
120
- }
121
- // P1 (C-P1-1): bounded tail read — the abnormal decision only consumes
122
- // this tail window. The provider scan walks backward inside it; window
123
- // matches Python `_TAIL_BYTES` (131072, idle_takeover_wiring.py:13),
124
- // never less.
125
- let text = match read_tail_text(&rollout_path, ABNORMAL_TAIL_BYTES) {
126
- Ok(text) => text,
127
- Err(error) => {
128
- upsert_abnormal_watch(
129
- state,
130
- &agent.agent_id,
131
- abnormal_watch_payload(
132
- &agent,
133
- Some(size),
134
- mtime_ns,
135
- process_check(
136
- ProcessLiveness::Unverifiable,
137
- "rollout_read_failed".to_string(),
138
- ),
139
- None,
140
- ErrorRecency::None,
141
- None,
142
- Some(error.to_string()),
143
- ),
144
- );
145
- continue;
146
- }
147
- };
148
- let fact = crate::provider::latest_explicit_error_fact(agent.provider, &text);
149
- let error_observation_key = fact
150
- .as_ref()
151
- .map(|fact| abnormal_error_observation_key(&agent, fact));
152
- let error_observation_cohort = fact.as_ref().map(|_| abnormal_error_cohort_key(&agent));
153
- let error_recency = abnormal_error_recency(
154
- state,
155
- &agent,
156
- error_observation_key.as_deref(),
157
- error_observation_cohort.as_deref(),
158
- );
159
- let decision = abnormal_exit_decision(liveness.state, fact.as_ref(), error_recency);
160
- let check_key = abnormal_check_key(
161
- &agent,
162
- &liveness,
163
- fact.as_ref(),
164
- error_recency,
165
- error_observation_key.as_deref(),
166
- );
167
- upsert_abnormal_watch(
168
- state,
169
- &agent.agent_id,
170
- abnormal_watch_payload(
171
- &agent,
172
- Some(size),
173
- mtime_ns,
174
- liveness.clone(),
175
- fact.as_ref().map(|f| f.signature.as_str()),
176
- error_recency,
177
- error_observation_key.as_deref(),
178
- None,
179
- ),
180
- );
181
- if let (Some(observation_key), Some(cohort_key)) = (
182
- error_observation_key.as_deref(),
183
- error_observation_cohort.as_deref(),
184
- ) {
185
- mark_abnormal_error_observed(state, &agent.agent_id, observation_key, cohort_key);
186
- }
187
- if abnormal_last_check_key(state, &agent.agent_id).as_deref() != Some(check_key.as_str()) {
188
- write_abnormal_check(
189
- event_log,
190
- &team,
191
- &agent,
192
- &liveness,
193
- fact.as_ref(),
194
- decision,
195
- error_recency,
196
- size,
197
- mtime_ns,
198
- )?;
199
- mark_abnormal_checked(state, &agent.agent_id, &check_key);
200
- }
201
- let fact = match (decision, fact) {
202
- (AbnormalExitDecision::Notify, Some(fact)) => fact,
203
- (AbnormalExitDecision::Suppress(reason), _) => {
204
- let suppress_key = abnormal_suppression_key(&agent, &liveness, reason);
205
- if abnormal_last_suppressed_key(state, &agent.agent_id).as_deref()
206
- != Some(suppress_key.as_str())
207
- {
208
- write_abnormal_suppressed(event_log, &team, &agent, &liveness, reason)?;
209
- mark_abnormal_suppressed(state, &agent.agent_id, &suppress_key);
210
- }
211
- continue;
212
- }
213
- (AbnormalExitDecision::NoSignal, _) => continue,
214
- (AbnormalExitDecision::Notify, None) => continue,
215
- };
216
- let dedupe_key = abnormal_dedupe_key(&agent, &fact);
217
- if abnormal_last_notified_key(state, &agent.agent_id).as_deref()
218
- == Some(dedupe_key.as_str())
219
- {
220
- continue;
221
- }
222
- let manual_recovery_hint =
223
- recovery_hint(agent.agent_id.as_str(), agent.provider, workspace, team.as_str());
224
- let content = format_abnormal_exit_message(
225
- &team,
226
- &agent,
227
- &fact,
228
- &liveness,
229
- size,
230
- &manual_recovery_hint,
231
- );
232
- let outcome = crate::messaging::send_to_leader_receiver(
233
- workspace,
234
- state,
235
- "leader",
236
- &content,
237
- None,
238
- &agent.agent_id,
239
- false,
240
- Some(&dedupe_key),
241
- event_log,
242
- )?;
243
- let notification_status = if outcome.ok {
244
- "queued"
245
- } else if matches!(outcome.status, crate::messaging::DeliveryStatus::Blocked) {
246
- "rebind_required"
247
- } else {
248
- "refused"
249
- };
250
- let provider_process_dead = provider_process_dead_fact(&liveness);
251
- event_log.write(
252
- "worker.abnormal_exit",
253
- serde_json::json!({
254
- "team_id": team.as_str(),
255
- "agent_id": agent.agent_id.as_str(),
256
- "provider": provider_wire(agent.provider),
257
- "path": agent.rollout_path_display.as_str(),
258
- "dead_process": liveness.state == ProcessLiveness::Dead,
259
- "process_dead": liveness.state == ProcessLiveness::Dead,
260
- "provider_process_dead": provider_process_dead,
261
- "latest_error": true,
262
- "latest_explicit_error": true,
263
- "error_recency": error_recency.as_str(),
264
- "fresh_error": error_recency.is_fresh(),
265
- "dead_process_and_latest_error": liveness.state == ProcessLiveness::Dead,
266
- "dead_process_and_latest_explicit_error": liveness.state == ProcessLiveness::Dead,
267
- "process_dead_and_latest_explicit_error": liveness.state == ProcessLiveness::Dead,
268
- "provider_process_dead_and_latest_explicit_error": provider_process_dead,
269
- "signature": fact.signature.as_str(),
270
- "turn_id": fact.turn_id.as_ref().map(|id| id.as_str()),
271
- "apiErrorStatus": fact.api_error_status,
272
- "error": fact.error.as_deref(),
273
- "requestId": fact.request_id.as_deref(),
274
- "assistant_uuid": fact.assistant_uuid.as_deref(),
275
- "size": size,
276
- "mtime_ns": mtime_ns,
277
- "process_liveness": process_liveness_wire(liveness.state),
278
- "pid_status": liveness.detail.as_str(),
279
- "notification_message_id": outcome.message_id,
280
- "notification_status": notification_status,
281
- "notification_channel": outcome.channel,
282
- }),
283
- )?;
284
- mark_abnormal_notified(state, &agent.agent_id, &dedupe_key);
285
- }
286
- Ok(())
287
- }
288
-
289
- fn provider_process_dead_fact(liveness: &ProcessCheck) -> bool {
290
- liveness.state == ProcessLiveness::Dead
291
- && !liveness.detail.starts_with("pane_dead:")
292
- && !liveness.detail.starts_with("window_missing:")
293
- }
294
-
295
- fn worker_provider_exited_fact(liveness: &ProcessCheck) -> bool {
296
- provider_process_dead_fact(liveness) && liveness.detail.starts_with("worker_provider_exited:")
297
- }
298
-
299
- fn refresh_abnormal_watch_liveness(state: &mut Value, agent_id: &str, liveness: &ProcessCheck) {
300
- let Some(watch) = coordinator_child_object(state, "abnormal_exit_watch")
301
- .and_then(|watch| watch.get_mut(agent_id))
302
- .and_then(Value::as_object_mut)
303
- else {
304
- return;
305
- };
306
- let dead_process = liveness.state == ProcessLiveness::Dead;
307
- let provider_process_dead = provider_process_dead_fact(liveness);
308
- let latest_explicit_error = watch
309
- .get("latest_explicit_error")
310
- .and_then(Value::as_bool)
311
- .unwrap_or(false);
312
- // Persist fact transitions, not the tick's observation time; otherwise an
313
- // unchanged transcript makes every steady tick rewrite state.json.
314
- let patch = serde_json::json!({
315
- "last_liveness": process_liveness_wire(liveness.state),
316
- "last_liveness_detail": liveness.detail.as_str(),
317
- "dead_process": dead_process,
318
- "process_dead": dead_process,
319
- "provider_process_dead": provider_process_dead,
320
- "worker_provider_exited": worker_provider_exited_fact(liveness),
321
- "provider_process_dead_and_latest_explicit_error": provider_process_dead && latest_explicit_error,
322
- });
323
- if let Some(patch) = patch.as_object() {
324
- if patch
325
- .iter()
326
- .any(|(key, value)| watch.get(key) != Some(value))
327
- {
328
- watch.extend(patch.clone());
329
- }
330
- }
331
- }
332
-
333
- #[derive(Debug, Clone)]
334
- struct AbnormalWatchAgent {
335
- agent_id: String,
336
- provider: crate::model::enums::Provider,
337
- rollout_path: PathBuf,
338
- rollout_path_display: String,
339
- spawn_epoch: Option<u64>,
340
- spawned_at: Option<String>,
341
- status: Option<String>,
342
- process_liveness: Option<ProcessLiveness>,
343
- window: Option<String>,
344
- pane_id: Option<String>,
345
- pid: Option<Pid>,
346
- current_command: Option<String>,
347
- }
348
-
349
- #[derive(Debug, Clone, PartialEq, Eq)]
350
- struct ProcessCheck {
351
- state: ProcessLiveness,
352
- detail: String,
353
- }
354
-
355
- #[derive(Debug, Clone, Copy, PartialEq, Eq)]
356
- enum AbnormalExitDecision {
357
- Notify,
358
- Suppress(&'static str),
359
- NoSignal,
360
- }
361
-
362
- #[derive(Debug, Clone, Copy, PartialEq, Eq)]
363
- enum ErrorRecency {
364
- None,
365
- Stale,
366
- Fresh,
367
- }
368
-
369
- impl ErrorRecency {
370
- fn as_str(self) -> &'static str {
371
- match self {
372
- Self::None => "none",
373
- Self::Stale => "stale",
374
- Self::Fresh => "fresh",
375
- }
376
- }
377
-
378
- fn is_fresh(self) -> bool {
379
- matches!(self, Self::Fresh)
380
- }
381
- }
382
-
383
- #[derive(Debug, Clone, Copy, PartialEq, Eq)]
384
- struct AbnormalExitGate {
385
- provider_process_dead: bool,
386
- latest_explicit_error: bool,
387
- error_recency: ErrorRecency,
388
- }
389
-
390
- impl AbnormalExitGate {
391
- fn new(
392
- process_liveness: ProcessLiveness,
393
- latest_explicit_error: bool,
394
- error_recency: ErrorRecency,
395
- ) -> Self {
396
- Self {
397
- provider_process_dead: process_liveness == ProcessLiveness::Dead,
398
- latest_explicit_error,
399
- error_recency,
400
- }
401
- }
402
-
403
- fn should_notify_worker_abnormal_exit(self) -> bool {
404
- should_notify_worker_abnormal_exit(self.latest_explicit_error, self.error_recency)
405
- }
406
-
407
- fn suppressed_reason(self) -> Option<&'static str> {
408
- match (self.provider_process_dead, self.latest_explicit_error) {
409
- (true, false) => Some("dead_only"),
410
- _ => None,
411
- }
412
- }
413
- }
414
-
415
- fn abnormal_exit_decision(
416
- process_liveness: ProcessLiveness,
417
- latest_explicit_error: Option<&crate::provider::FaultFact>,
418
- error_recency: ErrorRecency,
419
- ) -> AbnormalExitDecision {
420
- let gate = AbnormalExitGate::new(
421
- process_liveness,
422
- latest_explicit_error.is_some(),
423
- error_recency,
424
- );
425
- if gate.should_notify_worker_abnormal_exit() {
426
- return AbnormalExitDecision::Notify;
427
- }
428
- match gate.suppressed_reason() {
429
- Some(reason) => AbnormalExitDecision::Suppress(reason),
430
- None => AbnormalExitDecision::NoSignal,
431
- }
432
- }
433
-
434
- fn should_notify_worker_abnormal_exit(
435
- latest_explicit_error: bool,
436
- error_recency: ErrorRecency,
437
- ) -> bool {
438
- latest_explicit_error && error_recency.is_fresh()
439
- }
440
-
441
- fn resolve_agent_rollout_path(workspace: &Path, path: &Path) -> PathBuf {
442
- if path.is_absolute() {
443
- path.to_path_buf()
444
- } else {
445
- workspace.join(path)
446
- }
447
- }
448
-
449
- fn abnormal_watch_agents(state: &Value) -> Vec<AbnormalWatchAgent> {
450
- let Some(agents) = state.get("agents").and_then(Value::as_object) else {
451
- return Vec::new();
452
- };
453
- agents
454
- .iter()
455
- .filter_map(|(agent_id, agent)| {
456
- if matches!(agent.get("status").and_then(Value::as_str), Some("paused")) {
457
- return None;
458
- }
459
- let provider = agent
460
- .get("provider")
461
- .and_then(Value::as_str)
462
- .and_then(parse_provider)?;
463
- let rollout_path_display = ["rollout_path", "transcript_path", "session_log_path"]
464
- .into_iter()
465
- .find_map(|key| agent.get(key).and_then(Value::as_str))
466
- .filter(|path| !path.is_empty())?
467
- .to_string();
468
- Some(AbnormalWatchAgent {
469
- agent_id: agent_id.clone(),
470
- provider,
471
- rollout_path: PathBuf::from(&rollout_path_display),
472
- rollout_path_display,
473
- spawn_epoch: agent.get("spawn_epoch").and_then(Value::as_u64),
474
- spawned_at: agent
475
- .get("spawned_at")
476
- .and_then(Value::as_str)
477
- .map(str::to_string),
478
- status: agent
479
- .get("status")
480
- .and_then(Value::as_str)
481
- .map(str::to_string),
482
- process_liveness: explicit_process_liveness(agent),
483
- window: agent
484
- .get("window")
485
- .and_then(Value::as_str)
486
- .map(str::to_string),
487
- pane_id: agent
488
- .get("pane_id")
489
- .and_then(Value::as_str)
490
- .map(str::to_string),
491
- pid: agent_pid(agent),
492
- current_command: agent
493
- .get("pane_current_command")
494
- .or_else(|| agent.get("current_command"))
495
- .and_then(Value::as_str)
496
- .map(str::to_string),
497
- })
498
- })
499
- .collect()
500
- }
501
-
502
- fn agent_pid(agent: &Value) -> Option<Pid> {
503
- // 0.5.41 Slice 4 (fault-invisibility-locate.md §5 point 3 / §6.4,
504
- // 0.5.39 CR D-m): after the 0.5.39 worker shell wrapper landed,
505
- // `pane_pid` is the long-lived shell — NOT the provider. Treating
506
- // it as `provider pid` and probing it via `kill(0)` reports "alive"
507
- // for a pane whose provider CLI already exited, which is exactly
508
- // the false-BUSY that suppressed `worker_provider_exited`. Only
509
- // pids we know are the provider itself (`provider_pid` /
510
- // `process_id` / `pid` written by explicit spawn accounting, or
511
- // `child_pid` from the transport's SpawnResult) count here.
512
- ["provider_pid", "process_id", "pid", "child_pid"]
513
- .into_iter()
514
- .find_map(|key| json_u32(agent.get(key)).map(Pid::new))
515
- }
516
-
517
- /// ---
518
- /// purpose: 只从 agent JSON 里已写明的存活字段读出进程存活态,不做任何主动探测
519
- /// params:
520
- /// agent: agent 状态对象;先下钻 provider_process / process 子对象,再看本层的 *_liveness 串字段
521
- /// returns: 命中显式字段时给出 Alive/Dead/Unverifiable;没有任何显式字段时为 None,交调用方走探测路径
522
- /// ---
523
- pub(crate) fn explicit_process_liveness(agent: &Value) -> Option<ProcessLiveness> {
524
- if let Some(process) = agent
525
- .get("provider_process")
526
- .or_else(|| agent.get("process"))
527
- {
528
- if let Some(liveness) = explicit_process_liveness(process) {
529
- return Some(liveness);
530
- }
531
- }
532
- for key in [
533
- "provider_process_liveness",
534
- "process_liveness",
535
- "pane_liveness",
536
- ] {
537
- match agent.get(key).and_then(Value::as_str) {
538
- Some("dead") => return Some(ProcessLiveness::Dead),
539
- Some("alive" | "live") => return Some(ProcessLiveness::Alive),
540
- Some("unverifiable" | "unknown") => return Some(ProcessLiveness::Unverifiable),
541
- _ => {}
542
- }
543
- }
544
- for key in [
545
- "provider_process_alive",
546
- "process_alive",
547
- "provider_alive",
548
- "alive",
549
- ] {
550
- if let Some(alive) = agent.get(key).and_then(Value::as_bool) {
551
- return Some(if alive {
552
- ProcessLiveness::Alive
553
- } else {
554
- ProcessLiveness::Dead
555
- });
556
- }
557
- }
558
- for key in [
559
- "provider_process_dead",
560
- "process_dead",
561
- "provider_dead",
562
- "dead",
563
- ] {
564
- if let Some(dead) = agent.get(key).and_then(Value::as_bool) {
565
- return Some(if dead {
566
- ProcessLiveness::Dead
567
- } else {
568
- ProcessLiveness::Alive
569
- });
570
- }
571
- }
572
- for key in ["status", "state", "liveness"] {
573
- match agent.get(key).and_then(Value::as_str) {
574
- Some("dead" | "exited" | "terminated" | "crashed" | "missing") => {
575
- return Some(ProcessLiveness::Dead);
576
- }
577
- Some("alive" | "live" | "running") => return Some(ProcessLiveness::Alive),
578
- Some("unverifiable" | "unknown") => return Some(ProcessLiveness::Unverifiable),
579
- _ => {}
580
- }
581
- }
582
- None
583
- }
584
-
585
- fn json_u32(value: Option<&Value>) -> Option<u32> {
586
- value
587
- .and_then(|v| {
588
- v.as_u64()
589
- .or_else(|| v.as_i64().and_then(|n| u64::try_from(n).ok()))
590
- })
591
- .and_then(|n| u32::try_from(n).ok())
592
- }
593
-
594
- fn agent_process_liveness(
595
- agent: &AbnormalWatchAgent,
596
- session_name: Option<&str>,
597
- targets: &[crate::transport::PaneInfo],
598
- transport: &dyn crate::transport::Transport,
599
- ) -> ProcessCheck {
600
- // 0.5.41 Slice 4 (fault-invisibility-locate.md §5 point 3 / §6.4,
601
- // 0.5.39 CR D-m): after the worker wrapper landed, `pane_pid` may
602
- // be the long-lived shell that survived provider exit. Reordered
603
- // to prefer POSITIVE provider evidence in this order:
604
- // 1. Explicit provider pid (agent.pid) — set by the transport's
605
- // SpawnResult.child_pid or explicit provider_pid state field.
606
- // 2. Explicit liveness field.
607
- // 3. Explicit dead-status field.
608
- // 4. Explicit provider current-command (state.pane_current_command).
609
- // 5. Live-pane current-command matching provider = alive.
610
- // 6. Live-pane current-command shell + capture-tail worker exit
611
- // marker = DEAD (positive proof provider exited).
612
- // 7. Live pane, no positive evidence = Unverifiable (NOT alive,
613
- // NOT working). Legacy code returned pid_process_check(pane_pid)
614
- // here — that was the D-m false BUSY.
615
- // 8. Pane / window liveness falls back on Unverifiable.
616
- if let Some(pid) = agent.pid {
617
- return pid_process_check("pid", pid);
618
- }
619
- // 0.5.41 Slice 4 (fault-invisibility-locate.md §5 point 3 / §6.4):
620
- // BEFORE consulting the loose `explicit_process_liveness` path
621
- // (which happily maps state.status=`running` to Alive), probe the
622
- // pane for POSITIVE provider evidence via current-command + worker
623
- // exit marker. Under the 0.5.39 wrapper, `state.status=running` is
624
- // administrative spawn accounting, not runtime truth — a provider
625
- // may have exited into the inert shell tail while state still reads
626
- // running. The marker probe positively proves that case; when it
627
- // fires we return Dead here instead of the false Alive further
628
- // down.
629
- if let Some(target) = matching_agent_target(agent, session_name, targets) {
630
- if let Some(command) = target.current_command.as_deref() {
631
- return pane_command_process_check_with_marker(agent, transport, target, command);
632
- }
633
- }
634
- if let Some(command) = agent.current_command.as_deref() {
635
- return command_process_check_with_marker(agent, transport, command);
636
- }
637
- if agent.pane_id.is_some() {
638
- // Even without pane current_command / matching target, try the
639
- // marker probe directly — the wrapper's printf leaves the marker
640
- // in the pane's capture tail whether or not the transport
641
- // reports pane_current_command.
642
- if let Some(check) = worker_provider_exit_marker_check(agent, transport) {
643
- return check;
644
- }
645
- }
646
- if let Some(liveness) = agent.process_liveness {
647
- return process_check(
648
- liveness,
649
- format!("explicit:{}", process_liveness_wire(liveness)),
650
- );
651
- }
652
- if agent.status.as_deref().is_some_and(|status| {
653
- matches!(
654
- status,
655
- "stopped" | "missing" | "error" | "dead" | "exited" | "terminated" | "crashed"
656
- )
657
- }) {
658
- return process_check(
659
- ProcessLiveness::Dead,
660
- format!("status:{}", agent.status.as_deref().unwrap_or("unknown")),
661
- );
662
- }
663
- if let Some(target) = matching_agent_target(agent, session_name, targets) {
664
- // Reached only when the target has no current_command AND no
665
- // marker was found earlier. Legacy code returned
666
- // `pid_process_check("pane_pid", pane_pid)` here — that's the
667
- // wrapper shell under 0.5.39. Now Unverifiable (not-working).
668
- let _ = target;
669
- return process_check(
670
- ProcessLiveness::Unverifiable,
671
- "pane_present_provider_evidence_unknown".to_string(),
672
- );
673
- }
674
- if let Some(pane_id) = agent.pane_id.as_deref() {
675
- let pane = crate::transport::PaneId::new(pane_id);
676
- return match transport.liveness(&pane) {
677
- Ok(crate::transport::PaneLiveness::Dead) => {
678
- process_check(ProcessLiveness::Dead, format!("pane_dead:{pane_id}"))
679
- }
680
- Ok(crate::transport::PaneLiveness::Live) => process_check(
681
- ProcessLiveness::Unverifiable,
682
- format!("pane_live_provider_evidence_unknown:{pane_id}"),
683
- ),
684
- Ok(crate::transport::PaneLiveness::Unknown) => process_check(
685
- ProcessLiveness::Unverifiable,
686
- format!("pane_unknown:{pane_id}"),
687
- ),
688
- Err(error) => process_check(
689
- ProcessLiveness::Unverifiable,
690
- format!("pane_unverifiable:{pane_id}:{error}"),
691
- ),
692
- };
693
- }
694
- let (Some(session), Some(window)) = (session_name, agent.window.as_deref()) else {
695
- return process_check(
696
- ProcessLiveness::Unverifiable,
697
- "missing_session_or_window".to_string(),
698
- );
699
- };
700
- let session = crate::transport::SessionName::new(session);
701
- match transport.list_windows(&session) {
702
- Ok(windows) if windows.iter().any(|known| known.as_str() == window) => process_check(
703
- ProcessLiveness::Unverifiable,
704
- "window_present_provider_evidence_unknown".to_string(),
705
- ),
706
- Ok(_) => process_check(ProcessLiveness::Dead, format!("window_missing:{window}")),
707
- Err(error) => process_check(
708
- ProcessLiveness::Unverifiable,
709
- format!("window_unverifiable:{window}:{error}"),
710
- ),
711
- }
712
- }
713
-
714
- fn matching_agent_target<'a>(
715
- agent: &AbnormalWatchAgent,
716
- session_name: Option<&str>,
717
- targets: &'a [crate::transport::PaneInfo],
718
- ) -> Option<&'a crate::transport::PaneInfo> {
719
- if let Some(pane_id) = agent.pane_id.as_deref() {
720
- if let Some(target) = targets
721
- .iter()
722
- .find(|target| target.pane_id.as_str() == pane_id)
723
- {
724
- return Some(target);
725
- }
726
- }
727
- let (Some(session), Some(window)) = (session_name, agent.window.as_deref()) else {
728
- return None;
729
- };
730
- targets.iter().find(|target| {
731
- target.session.as_str() == session
732
- && target
733
- .window_name
734
- .as_ref()
735
- .is_some_and(|known| known.as_str() == window)
736
- })
737
- }
738
-
739
- fn pid_process_check(label: &str, pid: Pid) -> ProcessCheck {
740
- match pid_is_running(pid) {
741
- Ok(true) => process_check(ProcessLiveness::Alive, format!("{label}_running:{pid}")),
742
- Ok(false) => process_check(ProcessLiveness::Dead, format!("{label}_not_running:{pid}")),
743
- Err(error) => process_check(
744
- ProcessLiveness::Unverifiable,
745
- format!("{label}_unverifiable:{pid}:{error}"),
746
- ),
747
- }
748
- }
749
-
750
- fn command_process_check(provider: crate::model::enums::Provider, command: &str) -> ProcessCheck {
751
- if crate::leader::command_matches_provider(provider, command) {
752
- process_check(ProcessLiveness::Alive, format!("current_command:{command}"))
753
- } else {
754
- process_check(
755
- ProcessLiveness::Dead,
756
- format!("provider_not_foreground:{command}"),
757
- )
758
- }
759
- }
760
-
761
- fn pane_command_process_check(
762
- provider: crate::model::enums::Provider,
763
- pane: &crate::transport::PaneInfo,
764
- command: &str,
765
- ) -> ProcessCheck {
766
- if crate::leader::attribute_pane_provider(pane)
767
- .is_some_and(|candidate| crate::leader::provider_matches(candidate, provider))
768
- {
769
- process_check(ProcessLiveness::Alive, format!("current_command:{command}"))
770
- } else {
771
- process_check(
772
- ProcessLiveness::Dead,
773
- format!("provider_not_foreground:{command}"),
774
- )
775
- }
776
- }
777
-
778
- /// 0.5.41 Slice 4 (fault-invisibility-locate.md §5 point 3 / §6.4):
779
- /// after the 0.5.39 worker wrapper landed, a pane whose current command
780
- /// is a shell + capture-tail contains the worker exit marker is POSITIVE
781
- /// proof the provider exited. Falls back to plain current-command check
782
- /// when a pane_id is not available for capture (state-only path).
783
- fn command_process_check_with_marker(
784
- agent: &AbnormalWatchAgent,
785
- transport: &dyn crate::transport::Transport,
786
- command: &str,
787
- ) -> ProcessCheck {
788
- if crate::leader::command_matches_provider(agent.provider, command) {
789
- return process_check(ProcessLiveness::Alive, format!("current_command:{command}"));
790
- }
791
- if let Some(check) = worker_provider_exit_marker_check(agent, transport) {
792
- return check;
793
- }
794
- process_check(
795
- ProcessLiveness::Unverifiable,
796
- format!("provider_evidence_unverifiable:{command}"),
797
- )
798
- }
799
-
800
- fn pane_command_process_check_with_marker(
801
- agent: &AbnormalWatchAgent,
802
- transport: &dyn crate::transport::Transport,
803
- pane: &crate::transport::PaneInfo,
804
- command: &str,
805
- ) -> ProcessCheck {
806
- if crate::leader::attribute_pane_provider(pane)
807
- .is_some_and(|candidate| crate::leader::provider_matches(candidate, agent.provider))
808
- {
809
- return process_check(ProcessLiveness::Alive, format!("current_command:{command}"));
810
- }
811
- if let Some(check) = worker_provider_exit_marker_check(agent, transport) {
812
- return check;
813
- }
814
- process_check(
815
- ProcessLiveness::Unverifiable,
816
- format!("provider_evidence_unverifiable:{command}"),
817
- )
818
- }
819
-
820
- /// 0.5.41 Slice 4: probe the pane's capture-tail for the single-source
821
- /// worker exit marker (`tmux_backend::worker_provider_exit_marker`).
822
- /// Returns `Some(Dead)` when the marker is found (positive proof
823
- /// provider exited), or `None` when we can't run the capture (no
824
- /// pane_id in agent), so callers keep their default decision. Absence
825
- /// of marker in a valid capture is intentionally NOT `alive` (locate
826
- /// §8 risk: capture tail can scroll past the marker) — callers still
827
- /// return `Dead` for "provider not foreground" because current command
828
- /// already disproved provider liveness at that call site.
829
- fn worker_provider_exit_marker_check(
830
- agent: &AbnormalWatchAgent,
831
- transport: &dyn crate::transport::Transport,
832
- ) -> Option<ProcessCheck> {
833
- let pane_id_str = agent.pane_id.as_deref()?;
834
- let pane = crate::transport::PaneId::new(pane_id_str);
835
- let target = crate::transport::Target::Pane(pane);
836
- let provider_label = crate::provider::wire::command_name(agent.provider);
837
- let marker = crate::tmux_backend::worker_provider_exit_marker(provider_label);
838
- let cap = transport
839
- .capture(&target, crate::transport::CaptureRange::Tail(200))
840
- .ok()?;
841
- if cap.text.contains(&marker) {
842
- // The wrapper writes its exit code immediately after this marker. A
843
- // truncated capture can leave the marker without a complete token;
844
- // keep that case explicit rather than guessing success (or zero).
845
- // Scope: this detail only covers exits that execute the wrapper's
846
- // marker printf. SIGKILL, direct pane termination, and host power loss
847
- // produce no marker; that absence is silent, not `rc=unknown`.
848
- let rc = cap
849
- .text
850
- .split_once(&marker)
851
- .and_then(|(_, tail)| {
852
- let token = tail.split_whitespace().next()?;
853
- let end = tail.find(token)? + token.len();
854
- tail.as_bytes()
855
- .get(end)
856
- .is_some_and(u8::is_ascii_whitespace)
857
- .then_some(token)
858
- })
859
- .and_then(|token| token.parse::<i32>().ok())
860
- .map(|value| value.to_string())
861
- .unwrap_or_else(|| "unknown".to_string());
862
- Some(process_check(
863
- ProcessLiveness::Dead,
864
- format!("worker_provider_exited:{pane_id_str}:rc={rc}"),
865
- ))
866
- } else {
867
- None
868
- }
869
- }
870
-
871
- fn process_check(state: ProcessLiveness, detail: String) -> ProcessCheck {
872
- ProcessCheck { state, detail }
873
- }
874
-
875
- fn process_liveness_wire(state: ProcessLiveness) -> &'static str {
876
- match state {
877
- ProcessLiveness::Alive => "alive",
878
- ProcessLiveness::Dead => "dead",
879
- ProcessLiveness::Unverifiable => "unverifiable",
880
- }
881
- }
882
-
883
- /// ---
884
- /// purpose: 把文件修改时间归一成自 UNIX 纪元起的纳秒整数,供跨 tick 比较「rollout 有没有变过」
885
- /// params:
886
- /// metadata: 目标文件的 metadata
887
- /// returns: 纳秒时间戳;取不到 mtime 或早于纪元时为 None。乘加均为 saturating,不会回绕
888
- /// ---
889
- pub(crate) fn metadata_mtime_ns(metadata: &std::fs::Metadata) -> Option<u64> {
890
- let duration = metadata
891
- .modified()
892
- .ok()?
893
- .duration_since(std::time::UNIX_EPOCH)
894
- .ok()?;
895
- Some(
896
- duration
897
- .as_secs()
898
- .saturating_mul(1_000_000_000)
899
- .saturating_add(u64::from(duration.subsec_nanos())),
900
- )
901
- }
902
-
903
- fn abnormal_watch_payload(
904
- agent: &AbnormalWatchAgent,
905
- size: Option<u64>,
906
- mtime_ns: Option<u64>,
907
- liveness: ProcessCheck,
908
- signature: Option<&str>,
909
- error_recency: ErrorRecency,
910
- error_observation_key: Option<&str>,
911
- error: Option<String>,
912
- ) -> Value {
913
- let liveness_wire = process_liveness_wire(liveness.state);
914
- let dead_process = liveness.state == ProcessLiveness::Dead;
915
- let latest_explicit_error = signature.is_some();
916
- let gate = AbnormalExitGate::new(liveness.state, latest_explicit_error, error_recency);
917
- let notify = gate.should_notify_worker_abnormal_exit();
918
- let suppressed_reason = gate.suppressed_reason();
919
- // 0.5.41 Slice 4 (fault-invisibility-locate.md §6.4): typed
920
- // classification for status/render — `worker_provider_exited` is
921
- // true only when Dead was proven via the capture-tail worker exit
922
- // marker (detail prefix `worker_provider_exited:`). status_port's
923
- // RuntimeFreshness collector reads this field to downgrade the
924
- // corresponding agent row.
925
- let provider_process_dead = provider_process_dead_fact(&liveness);
926
- let worker_provider_exited = worker_provider_exited_fact(&liveness);
927
- serde_json::json!({
928
- "path": agent.rollout_path_display.as_str(),
929
- "provider": provider_wire(agent.provider),
930
- "mtime_ns": mtime_ns,
931
- "size": size,
932
- "last_offset": size,
933
- "last_signature": signature,
934
- "last_liveness": liveness_wire,
935
- "last_liveness_detail": liveness.detail,
936
- "dead_process": dead_process,
937
- "process_dead": dead_process,
938
- "provider_process_dead": provider_process_dead,
939
- "worker_provider_exited": worker_provider_exited,
940
- "latest_error": latest_explicit_error,
941
- "latest_explicit_error": latest_explicit_error,
942
- "error_recency": error_recency.as_str(),
943
- "fresh_error": error_recency.is_fresh(),
944
- "error_observation_key": error_observation_key,
945
- "dead_process_and_latest_error": dead_process && latest_explicit_error,
946
- "dead_process_and_latest_explicit_error": dead_process && latest_explicit_error,
947
- "process_dead_and_latest_explicit_error": dead_process && latest_explicit_error,
948
- "provider_process_dead_and_latest_explicit_error": provider_process_dead && latest_explicit_error,
949
- "suppressed_reason": suppressed_reason,
950
- "notification": notify,
951
- "last_error": error,
952
- })
953
- }
954
-
955
- fn upsert_abnormal_watch(state: &mut Value, agent_id: &str, mut payload: Value) {
956
- if let Some(identity) = crate::state::abnormal_watch::observation_identity(state, agent_id) {
957
- payload["watch_identity"] = identity;
958
- }
959
- let preserved = [
960
- "last_notified_key",
961
- "last_notified_at",
962
- "last_suppressed_key",
963
- "last_suppressed_at",
964
- "last_check_key",
965
- "last_check_at",
966
- "last_error_observation_key",
967
- "last_error_observation_cohort",
968
- "last_error_observed_at",
969
- ]
970
- .into_iter()
971
- .filter_map(|key| abnormal_watch_field(state, agent_id, key).map(|value| (key, value)))
972
- .collect::<Vec<_>>();
973
- if let Some(watch) = coordinator_child_object(state, "abnormal_exit_watch") {
974
- if let Some(payload_obj) = payload.as_object_mut() {
975
- for (key, value) in preserved {
976
- payload_obj.insert(key.to_string(), value);
977
- }
978
- }
979
- watch.insert(agent_id.to_string(), payload);
980
- }
981
- }
982
-
983
- fn coordinator_child_object<'a>(
984
- state: &'a mut Value,
985
- key: &str,
986
- ) -> Option<&'a mut serde_json::Map<String, Value>> {
987
- if !state.is_object() {
988
- *state = serde_json::json!({});
989
- }
990
- let state_obj = state.as_object_mut()?;
991
- let coordinator = state_obj
992
- .entry("coordinator".to_string())
993
- .or_insert_with(|| serde_json::json!({}));
994
- if !coordinator.is_object() {
995
- *coordinator = serde_json::json!({});
996
- }
997
- let coord_obj = coordinator.as_object_mut()?;
998
- let child = coord_obj
999
- .entry(key.to_string())
1000
- .or_insert_with(|| serde_json::json!({}));
1001
- if !child.is_object() {
1002
- *child = serde_json::json!({});
1003
- }
1004
- child.as_object_mut()
1005
- }
1006
-
1007
- fn abnormal_last_notified_key(state: &Value, agent_id: &str) -> Option<String> {
1008
- abnormal_watch_str(state, agent_id, "last_notified_key")
1009
- }
1010
-
1011
- fn abnormal_last_suppressed_key(state: &Value, agent_id: &str) -> Option<String> {
1012
- abnormal_watch_str(state, agent_id, "last_suppressed_key")
1013
- }
1014
-
1015
- fn abnormal_last_check_key(state: &Value, agent_id: &str) -> Option<String> {
1016
- abnormal_watch_str(state, agent_id, "last_check_key")
1017
- }
1018
-
1019
- fn abnormal_last_error_observation_key(state: &Value, agent_id: &str) -> Option<String> {
1020
- abnormal_watch_str(state, agent_id, "last_error_observation_key")
1021
- }
1022
-
1023
- fn abnormal_last_error_observation_cohort(state: &Value, agent_id: &str) -> Option<String> {
1024
- abnormal_watch_str(state, agent_id, "last_error_observation_cohort")
1025
- }
1026
-
1027
- /// P1: Python `_TAIL_BYTES` parity (idle_takeover_wiring.py:13) — RS must not read less.
1028
- const ABNORMAL_TAIL_BYTES: u64 = 131_072;
1029
-
1030
- ///
1031
- /// P1: bounded tail read; a partial first line is harmless (the consumer only parses
1032
- /// the latest complete JSONL record) and lossy UTF-8 keeps a mid-codepoint seek safe.
1033
- /// ---
1034
- /// purpose: 有界读取文件尾部文本,避免整份 rollout 进内存
1035
- /// params:
1036
- /// path: 目标文件
1037
- /// max_bytes: 最多回读的字节数;文件更短时整份读出
1038
- /// returns: 尾部文本。按字节 seek,首行可能被截断,且用有损 UTF-8 解码,切在码点中间也安全
1039
- /// errors: 打开、取长度或读取失败时返回底层 io::Error
1040
- /// ---
1041
- pub(crate) fn read_tail_text(path: &Path, max_bytes: u64) -> std::io::Result<String> {
1042
- use std::io::{Read, Seek, SeekFrom};
1043
- let mut file = std::fs::File::open(path)?;
1044
- let len = file.metadata()?.len();
1045
- if len > max_bytes {
1046
- file.seek(SeekFrom::Start(len - max_bytes))?;
1047
- }
1048
- let mut bytes = Vec::new();
1049
- file.read_to_end(&mut bytes)?;
1050
- Ok(String::from_utf8_lossy(&bytes).into_owned())
1051
- }
1052
-
1053
- /// P1: the previous tick's `(size, mtime_ns)` pair from the abnormal watch payload.
1054
- fn abnormal_watch_stored_metadata(state: &Value, agent_id: &str) -> Option<(u64, u64)> {
1055
- let watch = state
1056
- .get("coordinator")?
1057
- .get("abnormal_exit_watch")?
1058
- .get(agent_id)?;
1059
- Some((
1060
- watch.get("size")?.as_u64()?,
1061
- watch.get("mtime_ns")?.as_u64()?,
1062
- ))
1063
- }
1064
-
1065
- fn abnormal_watch_str(state: &Value, agent_id: &str, field: &str) -> Option<String> {
1066
- state
1067
- .get("coordinator")
1068
- .and_then(|v| v.get("abnormal_exit_watch"))
1069
- .and_then(|v| v.get(agent_id))
1070
- .and_then(|v| v.get(field))
1071
- .and_then(Value::as_str)
1072
- .map(str::to_string)
1073
- }
1074
-
1075
- fn abnormal_watch_field(state: &Value, agent_id: &str, field: &str) -> Option<Value> {
1076
- state
1077
- .get("coordinator")
1078
- .and_then(|v| v.get("abnormal_exit_watch"))
1079
- .and_then(|v| v.get(agent_id))
1080
- .and_then(|v| v.get(field))
1081
- .cloned()
1082
- }
1083
-
1084
- fn mark_abnormal_notified(state: &mut Value, agent_id: &str, key: &str) {
1085
- if let Some(watch) = coordinator_child_object(state, "abnormal_exit_watch") {
1086
- let entry = watch
1087
- .entry(agent_id.to_string())
1088
- .or_insert_with(|| serde_json::json!({}));
1089
- if !entry.is_object() {
1090
- *entry = serde_json::json!({});
1091
- }
1092
- if let Some(obj) = entry.as_object_mut() {
1093
- obj.insert("last_notified_key".to_string(), serde_json::json!(key));
1094
- obj.insert(
1095
- "last_notified_at".to_string(),
1096
- serde_json::json!(chrono::Utc::now().to_rfc3339()),
1097
- );
1098
- }
1099
- }
1100
- }
1101
-
1102
- fn mark_abnormal_suppressed(state: &mut Value, agent_id: &str, key: &str) {
1103
- if let Some(watch) = coordinator_child_object(state, "abnormal_exit_watch") {
1104
- let entry = watch
1105
- .entry(agent_id.to_string())
1106
- .or_insert_with(|| serde_json::json!({}));
1107
- if !entry.is_object() {
1108
- *entry = serde_json::json!({});
1109
- }
1110
- if let Some(obj) = entry.as_object_mut() {
1111
- obj.insert("last_suppressed_key".to_string(), serde_json::json!(key));
1112
- obj.insert(
1113
- "last_suppressed_at".to_string(),
1114
- serde_json::json!(chrono::Utc::now().to_rfc3339()),
1115
- );
1116
- }
1117
- }
1118
- }
1119
-
1120
- fn mark_abnormal_checked(state: &mut Value, agent_id: &str, key: &str) {
1121
- if let Some(watch) = coordinator_child_object(state, "abnormal_exit_watch") {
1122
- let entry = watch
1123
- .entry(agent_id.to_string())
1124
- .or_insert_with(|| serde_json::json!({}));
1125
- if !entry.is_object() {
1126
- *entry = serde_json::json!({});
1127
- }
1128
- if let Some(obj) = entry.as_object_mut() {
1129
- obj.insert("last_check_key".to_string(), serde_json::json!(key));
1130
- obj.insert(
1131
- "last_check_at".to_string(),
1132
- serde_json::json!(chrono::Utc::now().to_rfc3339()),
1133
- );
1134
- }
1135
- }
1136
- }
1137
-
1138
- fn mark_abnormal_error_observed(
1139
- state: &mut Value,
1140
- agent_id: &str,
1141
- observation_key: &str,
1142
- cohort_key: &str,
1143
- ) {
1144
- if let Some(watch) = coordinator_child_object(state, "abnormal_exit_watch") {
1145
- let entry = watch
1146
- .entry(agent_id.to_string())
1147
- .or_insert_with(|| serde_json::json!({}));
1148
- if !entry.is_object() {
1149
- *entry = serde_json::json!({});
1150
- }
1151
- if let Some(obj) = entry.as_object_mut() {
1152
- obj.insert(
1153
- "last_error_observation_key".to_string(),
1154
- serde_json::json!(observation_key),
1155
- );
1156
- obj.insert(
1157
- "last_error_observation_cohort".to_string(),
1158
- serde_json::json!(cohort_key),
1159
- );
1160
- obj.insert(
1161
- "last_error_observed_at".to_string(),
1162
- serde_json::json!(chrono::Utc::now().to_rfc3339()),
1163
- );
1164
- }
1165
- }
1166
- }
1167
-
1168
- fn write_abnormal_check(
1169
- event_log: &EventLog,
1170
- team: &str,
1171
- agent: &AbnormalWatchAgent,
1172
- liveness: &ProcessCheck,
1173
- fact: Option<&crate::provider::FaultFact>,
1174
- decision: AbnormalExitDecision,
1175
- error_recency: ErrorRecency,
1176
- size: u64,
1177
- mtime_ns: Option<u64>,
1178
- ) -> Result<(), TickError> {
1179
- let dead_process = liveness.state == ProcessLiveness::Dead;
1180
- let provider_process_dead = provider_process_dead_fact(liveness);
1181
- let latest_explicit_error = fact.is_some();
1182
- event_log.write(
1183
- "worker.abnormal_exit.check",
1184
- serde_json::json!({
1185
- "team_id": team,
1186
- "agent_id": agent.agent_id.as_str(),
1187
- "provider": provider_wire(agent.provider),
1188
- "path": agent.rollout_path_display.as_str(),
1189
- "size": size,
1190
- "last_offset": size,
1191
- "mtime_ns": mtime_ns,
1192
- "dead_process": dead_process,
1193
- "process_dead": dead_process,
1194
- "provider_process_dead": provider_process_dead,
1195
- "latest_error": latest_explicit_error,
1196
- "latest_explicit_error": latest_explicit_error,
1197
- "error_recency": error_recency.as_str(),
1198
- "fresh_error": error_recency.is_fresh(),
1199
- "dead_process_and_latest_error": dead_process && latest_explicit_error,
1200
- "dead_process_and_latest_explicit_error": dead_process && latest_explicit_error,
1201
- "process_dead_and_latest_explicit_error": dead_process && latest_explicit_error,
1202
- "provider_process_dead_and_latest_explicit_error": provider_process_dead && latest_explicit_error,
1203
- "notification": matches!(decision, AbnormalExitDecision::Notify),
1204
- "suppressed_reason": match decision {
1205
- AbnormalExitDecision::Suppress(reason) => Some(reason),
1206
- AbnormalExitDecision::Notify | AbnormalExitDecision::NoSignal => None,
1207
- },
1208
- "signature": fact.map(|fact| fact.signature.as_str()),
1209
- "turn_id": fact.and_then(|fact| fact.turn_id.as_ref().map(|id| id.as_str())),
1210
- "apiErrorStatus": fact.and_then(|fact| fact.api_error_status),
1211
- "error": fact.and_then(|fact| fact.error.as_deref()),
1212
- "requestId": fact.and_then(|fact| fact.request_id.as_deref()),
1213
- "assistant_uuid": fact.and_then(|fact| fact.assistant_uuid.as_deref()),
1214
- "process_liveness": process_liveness_wire(liveness.state),
1215
- "pid_status": liveness.detail.as_str(),
1216
- }),
1217
- )?;
1218
- Ok(())
1219
- }
1220
-
1221
- fn write_abnormal_suppressed(
1222
- event_log: &EventLog,
1223
- team: &str,
1224
- agent: &AbnormalWatchAgent,
1225
- liveness: &ProcessCheck,
1226
- reason: &str,
1227
- ) -> Result<(), TickError> {
1228
- let provider_process_dead = provider_process_dead_fact(liveness);
1229
- event_log.write(
1230
- "abnormal_exit.single_signal_suppressed",
1231
- serde_json::json!({
1232
- "team_id": team,
1233
- "agent_id": agent.agent_id.as_str(),
1234
- "provider": provider_wire(agent.provider),
1235
- "path": agent.rollout_path_display.as_str(),
1236
- "reason": reason,
1237
- "notification": false,
1238
- "dead_process": liveness.state == ProcessLiveness::Dead,
1239
- "process_dead": liveness.state == ProcessLiveness::Dead,
1240
- "provider_process_dead": provider_process_dead,
1241
- "latest_error": false,
1242
- "latest_explicit_error": false,
1243
- "error_recency": ErrorRecency::None.as_str(),
1244
- "fresh_error": false,
1245
- "dead_process_and_latest_error": false,
1246
- "dead_process_and_latest_explicit_error": false,
1247
- "process_dead_and_latest_explicit_error": false,
1248
- "provider_process_dead_and_latest_explicit_error": false,
1249
- "process_liveness": process_liveness_wire(liveness.state),
1250
- "pid_status": liveness.detail.as_str(),
1251
- }),
1252
- )?;
1253
- Ok(())
1254
- }
1255
-
1256
- fn abnormal_dedupe_key(agent: &AbnormalWatchAgent, fact: &crate::provider::FaultFact) -> String {
1257
- let bucket = fact
1258
- .turn_id
1259
- .as_ref()
1260
- .map(|id| id.as_str().to_string())
1261
- .or_else(|| abnormal_error_fact_identity(fact))
1262
- .unwrap_or_else(|| "no_error_identity".to_string());
1263
- format!(
1264
- "worker.abnormal_exit:{}:{}:{}:{}",
1265
- agent.agent_id,
1266
- agent.rollout_path_display,
1267
- fact.signature.as_str(),
1268
- bucket
1269
- )
1270
- }
1271
-
1272
- fn abnormal_error_cohort_key(agent: &AbnormalWatchAgent) -> String {
1273
- let cohort = agent
1274
- .spawn_epoch
1275
- .map(|epoch| format!("spawn_epoch:{epoch}"))
1276
- .or_else(|| {
1277
- agent
1278
- .spawned_at
1279
- .as_deref()
1280
- .map(|spawned_at| format!("spawned_at:{spawned_at}"))
1281
- })
1282
- .unwrap_or_else(|| "legacy".to_string());
1283
- format!(
1284
- "worker.abnormal_exit.cohort:{}:{}:{}",
1285
- agent.agent_id, agent.rollout_path_display, cohort
1286
- )
1287
- }
1288
-
1289
- fn abnormal_error_observation_key(
1290
- agent: &AbnormalWatchAgent,
1291
- fact: &crate::provider::FaultFact,
1292
- ) -> String {
1293
- let bucket =
1294
- abnormal_error_fact_identity(fact).unwrap_or_else(|| "no_error_identity".to_string());
1295
- format!(
1296
- "worker.abnormal_exit.error:{}:{}:{}:{}",
1297
- agent.agent_id,
1298
- agent.rollout_path_display,
1299
- fact.signature.as_str(),
1300
- bucket
1301
- )
1302
- }
1303
-
1304
- fn abnormal_error_fact_identity(fact: &crate::provider::FaultFact) -> Option<String> {
1305
- fact.assistant_uuid
1306
- .as_deref()
1307
- .or(fact.request_id.as_deref())
1308
- .or_else(|| fact.turn_id.as_ref().map(|id| id.as_str()))
1309
- .map(str::to_string)
1310
- }
1311
-
1312
- fn abnormal_error_recency(
1313
- state: &Value,
1314
- agent: &AbnormalWatchAgent,
1315
- observation_key: Option<&str>,
1316
- cohort_key: Option<&str>,
1317
- ) -> ErrorRecency {
1318
- let (Some(observation_key), Some(cohort_key)) = (observation_key, cohort_key) else {
1319
- return ErrorRecency::None;
1320
- };
1321
- let previous_key = abnormal_last_error_observation_key(state, &agent.agent_id);
1322
- let previous_cohort = abnormal_last_error_observation_cohort(state, &agent.agent_id);
1323
- match (previous_key.as_deref(), previous_cohort.as_deref()) {
1324
- (Some(key), Some(cohort)) if cohort == cohort_key && key != observation_key => {
1325
- ErrorRecency::Fresh
1326
- }
1327
- _ => ErrorRecency::Stale,
1328
- }
1329
- }
1330
-
1331
- fn abnormal_suppression_key(
1332
- agent: &AbnormalWatchAgent,
1333
- liveness: &ProcessCheck,
1334
- reason: &str,
1335
- ) -> String {
1336
- format!(
1337
- "abnormal_exit.single_signal_suppressed:{}:{}:{}:{}",
1338
- agent.agent_id,
1339
- agent.rollout_path_display,
1340
- reason,
1341
- process_liveness_wire(liveness.state)
1342
- )
1343
- }
1344
-
1345
- fn abnormal_check_key(
1346
- agent: &AbnormalWatchAgent,
1347
- liveness: &ProcessCheck,
1348
- fact: Option<&crate::provider::FaultFact>,
1349
- error_recency: ErrorRecency,
1350
- error_observation_key: Option<&str>,
1351
- ) -> String {
1352
- format!(
1353
- "worker.abnormal_exit.check:{}:{}:{}:{}:{}:{}",
1354
- agent.agent_id,
1355
- agent.rollout_path_display,
1356
- process_liveness_wire(liveness.state),
1357
- fact.map(|fact| fact.signature.as_str()).unwrap_or("-"),
1358
- error_recency.as_str(),
1359
- error_observation_key.unwrap_or("-")
1360
- )
1361
- }
1362
-
1363
- fn format_abnormal_exit_message(
1364
- team: &str,
1365
- agent: &AbnormalWatchAgent,
1366
- fact: &crate::provider::FaultFact,
1367
- liveness: &ProcessCheck,
1368
- size: u64,
1369
- manual_recovery_hint: &str,
1370
- ) -> String {
1371
- let turn_id = fact.turn_id.as_ref().map(|id| id.as_str()).unwrap_or("-");
1372
- format!(
1373
- "Team Agent detected a provider abnormal exit.\n\n\
1374
- event: worker.abnormal_exit\n\
1375
- team: {team}\n\
1376
- node: {node}\n\
1377
- provider: {provider}\n\
1378
- signature: {signature}\n\
1379
- turn_id: {turn_id}\n\
1380
- transcript: {path}\n\
1381
- last_offset: {size}\n\
1382
- pid_status: {pid_status}\n\n\
1383
- {manual_recovery_hint}",
1384
- node = agent.agent_id.as_str(),
1385
- provider = provider_wire(agent.provider),
1386
- signature = fact.signature.as_str(),
1387
- path = agent.rollout_path_display.as_str(),
1388
- pid_status = liveness.detail.as_str(),
1389
- )
1390
- }
1391
-
1392
- /// A pre-upgrade legacy Claude Code, Codex, Grok or Pi seat has no runtime, so
1393
- /// `start-agent` would be refused; it is stopped/removed and added again.
1394
- fn recovery_hint(
1395
- agent_id: &str,
1396
- provider: crate::model::enums::Provider,
1397
- workspace: &Path,
1398
- team: &str,
1399
- ) -> String {
1400
- let wire = provider_wire(provider);
1401
- if crate::provider::wire::is_retired_provider(wire) {
1402
- return format!(
1403
- "No automatic restart was performed.\n{}",
1404
- crate::provider::wire::retired_provider_error(wire)
1405
- );
1406
- }
1407
- format!(
1408
- "No automatic restart was performed.\nTo recover manually, run: {}",
1409
- recovery_manual_command(agent_id, workspace, team)
1410
- )
1411
- }
1412
-
1413
- fn recovery_manual_command(agent_id: &str, workspace: &Path, team: &str) -> String {
1414
- let workspace_str = workspace.to_string_lossy();
1415
- let shell_workspace = workspace_str.replace('\'', "'\\''");
1416
- format!(
1417
- "team-agent start-agent {agent_id} --workspace '{shell_workspace}' --team {team} --force --json"
1418
- )
1419
- }
1420
-
1421
- #[cfg(test)]
1422
- #[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
1423
- mod tests {
1424
- use super::*;
1425
- use crate::coordinator::tick::Coordinator;
1426
- use crate::coordinator::types::{ProviderRegistry, WorkspacePath};
1427
- use std::io::Write as _;
1428
-
1429
- struct NormalRegistry;
1430
-
1431
- impl ProviderRegistry for NormalRegistry {
1432
- fn adapter_for(
1433
- &self,
1434
- provider: crate::provider::Provider,
1435
- ) -> Box<dyn crate::provider::ProviderAdapter> {
1436
- crate::provider::get_adapter(provider)
1437
- }
1438
- }
1439
- #[test]
1440
- fn resource188_foreign_metadata_and_dedupe_cannot_skip_new_baseline() {
1441
- let dir = temp_abnormal_dir("188-foreign-watch");
1442
- let rollout = dir.join("rollout-w1.jsonl");
1443
- let error = "{\"method\":\"turn/completed\",\"params\":{\"turn\":{\"id\":\"old\",\"status\":\"failed\"}}}\n";
1444
- std::fs::write(&rollout, error).unwrap();
1445
- seed_abnormal_state(&dir, &rollout, "alive", 1);
1446
- let mut state = crate::state::persist::load_runtime_state(&dir).unwrap();
1447
- let metadata = std::fs::metadata(&rollout).unwrap();
1448
- let mut foreign = crate::state::abnormal_watch::observation_identity(&state, "w1").unwrap();
1449
- foreign["team_key"] = serde_json::json!("another-team");
1450
- state["coordinator"] = serde_json::json!({"abnormal_exit_watch": {"w1": {
1451
- "watch_identity": foreign,
1452
- "size": metadata.len(), "mtime_ns": metadata_mtime_ns(&metadata),
1453
- "last_notified_key": "foreign", "last_check_key": "foreign",
1454
- "last_error_observation_key": "foreign", "last_error_observation_cohort": "foreign"
1455
- }}});
1456
- let log = EventLog::new(&dir);
1457
- let transport = crate::transport::test_support::OfflineTransport::new();
1458
- detect_abnormal_exits(&dir, &transport, &mut state, &log, &[]).unwrap();
1459
- let baseline = state["coordinator"]["abnormal_exit_watch"]["w1"].clone();
1460
- assert_eq!(baseline["error_recency"], "stale");
1461
- assert!(baseline.get("last_notified_key").is_none());
1462
- assert!(crate::state::abnormal_watch::matches_observation(
1463
- &state, "w1", &baseline
1464
- ));
1465
- let events = log.tail(0).unwrap();
1466
- detect_abnormal_exits(&dir, &transport, &mut state, &log, &[]).unwrap();
1467
- assert_eq!(
1468
- log.tail(0).unwrap(),
1469
- events,
1470
- "unchanged baseline must not re-alarm"
1471
- );
1472
- assert!(find_event(&events, "worker.abnormal_exit").is_none());
1473
- }
1474
-
1475
- #[test]
1476
- fn resource188_legacy_missing_cohort_still_observes_fresh_errors_and_dedupes() {
1477
- let dir = temp_abnormal_dir("188-legacy-cohort");
1478
- let rollout = dir.join("rollout-w1.jsonl");
1479
- let error = "{\"method\":\"turn/completed\",\"params\":{\"turn\":{\"id\":\"old\",\"status\":\"failed\"}}}\n";
1480
- std::fs::write(&rollout, error).unwrap();
1481
- seed_abnormal_state(&dir, &rollout, "alive", 1);
1482
- let mut state = crate::state::persist::load_runtime_state(&dir).unwrap();
1483
- state["agents"]["w1"]
1484
- .as_object_mut()
1485
- .unwrap()
1486
- .remove("spawn_epoch");
1487
- let log = EventLog::new(&dir);
1488
- let transport = crate::transport::test_support::OfflineTransport::new();
1489
- detect_abnormal_exits(&dir, &transport, &mut state, &log, &[]).unwrap();
1490
- assert_eq!(
1491
- state["coordinator"]["abnormal_exit_watch"]["w1"]["error_recency"],
1492
- "stale"
1493
- );
1494
- assert!(find_event(&log.tail(0).unwrap(), "worker.abnormal_exit").is_none());
1495
- std::fs::OpenOptions::new()
1496
- .append(true)
1497
- .open(&rollout)
1498
- .unwrap()
1499
- .write_all(error.replace("old", "new-error").as_bytes())
1500
- .unwrap();
1501
- detect_abnormal_exits(&dir, &transport, &mut state, &log, &[]).unwrap();
1502
- let events = log.tail(0).unwrap();
1503
- assert_eq!(
1504
- find_event(&events, "worker.abnormal_exit").unwrap()["error_recency"],
1505
- "fresh"
1506
- );
1507
- let notified =
1508
- state["coordinator"]["abnormal_exit_watch"]["w1"]["last_notified_key"].clone();
1509
- assert!(notified.is_string());
1510
- detect_abnormal_exits(&dir, &transport, &mut state, &log, &[]).unwrap();
1511
- assert_eq!(log.tail(0).unwrap(), events);
1512
- assert_eq!(
1513
- state["coordinator"]["abnormal_exit_watch"]["w1"]["last_notified_key"],
1514
- notified
1515
- );
1516
- state["agents"]["w1"]["spawn_epoch"] = serde_json::json!(2);
1517
- detect_abnormal_exits(&dir, &transport, &mut state, &log, &[]).unwrap();
1518
- let watch = &state["coordinator"]["abnormal_exit_watch"]["w1"];
1519
- assert_eq!(
1520
- watch["error_recency"], "stale",
1521
- "a known generation must rebaseline unchanged bytes"
1522
- );
1523
- assert!(watch.get("last_notified_key").is_none());
1524
- assert_eq!(
1525
- log.tail(0)
1526
- .unwrap()
1527
- .iter()
1528
- .filter(|event| event["event"] == "worker.abnormal_exit")
1529
- .count(),
1530
- 1
1531
- );
1532
- }
1533
-
1534
- #[test]
1535
- fn resource188_unknown_team_still_records_fresh_liveness_without_reusing_cache() {
1536
- let dir = temp_abnormal_dir("188-unknown-team");
1537
- let rollout = dir.join("rollout-w1.jsonl");
1538
- std::fs::write(&rollout, "").unwrap();
1539
- seed_abnormal_state(&dir, &rollout, "dead", 1);
1540
- let mut state = crate::state::persist::load_runtime_state(&dir).unwrap();
1541
- state.as_object_mut().unwrap().remove("active_team_key");
1542
- let log = EventLog::new(&dir);
1543
- let transport = crate::transport::test_support::OfflineTransport::new();
1544
- detect_abnormal_exits(&dir, &transport, &mut state, &log, &[]).unwrap();
1545
- assert_eq!(
1546
- state["coordinator"]["abnormal_exit_watch"]["w1"]["last_liveness"],
1547
- "dead"
1548
- );
1549
- assert!(state["coordinator"]["abnormal_exit_watch"]["w1"]
1550
- .get("watch_identity")
1551
- .is_none());
1552
- }
1553
-
1554
- #[test]
1555
- fn abnormal_stale_error_baselines_then_fresh_alive_error_notifies() {
1556
- let dir = temp_abnormal_dir("alive-fresh");
1557
- let rollout = dir.join("rollout-w1.jsonl");
1558
- std::fs::write(
1559
- &rollout,
1560
- "{\"method\":\"turn/completed\",\"params\":{\"turn\":{\"id\":\"t1\",\"status\":\"failed\"}}}\n",
1561
- )
1562
- .unwrap();
1563
- seed_abnormal_state(&dir, &rollout, "alive", 1);
1564
- let coordinator = abnormal_test_coordinator(&dir);
1565
-
1566
- coordinator.tick().unwrap();
1567
-
1568
- let first_events = read_test_events(&dir);
1569
- assert!(
1570
- find_event(&first_events, "worker.abnormal_exit").is_none(),
1571
- "first observed explicit error is stale baseline and must not notify; events={first_events:?}"
1572
- );
1573
- let stale_check =
1574
- find_event(&first_events, "worker.abnormal_exit.check").expect("stale check event");
1575
- assert_eq!(stale_check["error_recency"], serde_json::json!("stale"));
1576
- assert_eq!(stale_check["fresh_error"], serde_json::json!(false));
1577
- assert_eq!(stale_check["notification"], serde_json::json!(false));
1578
- assert!(
1579
- find_event(&first_events, "abnormal_exit.single_signal_suppressed").is_none(),
1580
- "stale/error-only observations must not emit single-signal suppression"
1581
- );
1582
- let watch = abnormal_test_watch(&dir);
1583
- assert_eq!(watch["error_recency"], serde_json::json!("stale"));
1584
- assert_eq!(watch["fresh_error"], serde_json::json!(false));
1585
- assert!(
1586
- watch["last_error_observation_key"].as_str().is_some(),
1587
- "first explicit error must persist the stale baseline; watch={watch}"
1588
- );
1589
-
1590
- std::fs::OpenOptions::new()
1591
- .append(true)
1592
- .open(&rollout)
1593
- .unwrap()
1594
- .write_all(
1595
- b"{\"method\":\"turn/completed\",\"params\":{\"turn\":{\"id\":\"t2\",\"status\":\"failed\"}}}\n",
1596
- )
1597
- .unwrap();
1598
- coordinator.tick().unwrap();
1599
-
1600
- let events = read_test_events(&dir);
1601
- let abnormal =
1602
- find_event(&events, "worker.abnormal_exit").expect("fresh alive error notification");
1603
- assert_eq!(abnormal["provider_process_dead"], serde_json::json!(false));
1604
- assert_eq!(abnormal["latest_explicit_error"], serde_json::json!(true));
1605
- assert_eq!(abnormal["error_recency"], serde_json::json!("fresh"));
1606
- assert_eq!(abnormal["fresh_error"], serde_json::json!(true));
1607
- assert_eq!(
1608
- abnormal["provider_process_dead_and_latest_explicit_error"],
1609
- serde_json::json!(false)
1610
- );
1611
- assert_eq!(abnormal["notification_status"], serde_json::json!("queued"));
1612
- }
1613
-
1614
- #[test]
1615
- fn abnormal_claude_assistant_api_error_events_include_structured_details() {
1616
- let dir = temp_abnormal_dir("claude-api-details");
1617
- let rollout = dir.join("rollout-w1.jsonl");
1618
- std::fs::write(
1619
- &rollout,
1620
- claude_assistant_api_error_line(
1621
- "assistant-400",
1622
- "parent-400",
1623
- "session-400",
1624
- 400,
1625
- "unknown",
1626
- None,
1627
- ),
1628
- )
1629
- .unwrap();
1630
- seed_abnormal_state_with_provider(&dir, &rollout, "alive", 1, "claude_code");
1631
- let coordinator = abnormal_test_coordinator(&dir);
1632
-
1633
- coordinator.tick().unwrap();
1634
-
1635
- let first_events = read_test_events(&dir);
1636
- let stale_check =
1637
- find_event(&first_events, "worker.abnormal_exit.check").expect("stale check event");
1638
- assert_eq!(stale_check["error_recency"], serde_json::json!("stale"));
1639
- assert_eq!(stale_check["notification"], serde_json::json!(false));
1640
- assert_eq!(stale_check["apiErrorStatus"], serde_json::json!(400));
1641
- assert_eq!(stale_check["error"], serde_json::json!("unknown"));
1642
- assert_eq!(stale_check["requestId"], serde_json::json!(null));
1643
- assert_eq!(
1644
- stale_check["assistant_uuid"],
1645
- serde_json::json!("assistant-400")
1646
- );
1647
-
1648
- std::fs::OpenOptions::new()
1649
- .append(true)
1650
- .open(&rollout)
1651
- .unwrap()
1652
- .write_all(
1653
- claude_assistant_api_error_line(
1654
- "assistant-404",
1655
- "parent-404",
1656
- "session-404",
1657
- 404,
1658
- "model_not_found",
1659
- Some("req_011CceNfWj2aPY5gtCdakULt"),
1660
- )
1661
- .as_bytes(),
1662
- )
1663
- .unwrap();
1664
- coordinator.tick().unwrap();
1665
-
1666
- let events = read_test_events(&dir);
1667
- let fresh_check =
1668
- find_event(&events, "worker.abnormal_exit.check").expect("fresh check event");
1669
- assert_eq!(fresh_check["error_recency"], serde_json::json!("fresh"));
1670
- assert_eq!(fresh_check["notification"], serde_json::json!(true));
1671
- assert_eq!(fresh_check["apiErrorStatus"], serde_json::json!(404));
1672
- assert_eq!(fresh_check["error"], serde_json::json!("model_not_found"));
1673
- assert_eq!(
1674
- fresh_check["requestId"],
1675
- serde_json::json!("req_011CceNfWj2aPY5gtCdakULt")
1676
- );
1677
- assert_eq!(
1678
- fresh_check["assistant_uuid"],
1679
- serde_json::json!("assistant-404")
1680
- );
1681
-
1682
- let abnormal =
1683
- find_event(&events, "worker.abnormal_exit").expect("fresh alive error notification");
1684
- assert_eq!(abnormal["signature"], serde_json::json!("api_error"));
1685
- assert_eq!(abnormal["turn_id"], serde_json::json!("assistant-404"));
1686
- assert_eq!(abnormal["apiErrorStatus"], serde_json::json!(404));
1687
- assert_eq!(abnormal["error"], serde_json::json!("model_not_found"));
1688
- assert_eq!(
1689
- abnormal["requestId"],
1690
- serde_json::json!("req_011CceNfWj2aPY5gtCdakULt")
1691
- );
1692
- assert_eq!(
1693
- abnormal["assistant_uuid"],
1694
- serde_json::json!("assistant-404")
1695
- );
1696
- assert_eq!(abnormal["notification_status"], serde_json::json!("queued"));
1697
- }
1698
-
1699
- #[test]
1700
- fn abnormal_claude_assistant_api_error_followed_by_bookkeeping_still_detected() {
1701
- let dir = temp_abnormal_dir("claude-api-bookkeeping");
1702
- let rollout = dir.join("rollout-w1.jsonl");
1703
- let session_id = "97ec1070-f19b-49ed-b60f-3cb158e92053";
1704
- std::fs::write(
1705
- &rollout,
1706
- format!(
1707
- "{}{}",
1708
- claude_assistant_api_error_line(
1709
- "94e88d55-aac7-46bc-85cd-b7fcfc8a9ef6",
1710
- "245fce6f-2427-4b05-af01-8619df64afab",
1711
- session_id,
1712
- 404,
1713
- "model_not_found",
1714
- Some("req_011CceUVQ94dxahgwAHf8sdS"),
1715
- ),
1716
- claude_turn_duration_line(
1717
- "6c96328b-8c01-46c0-b68e-a096a89c904f",
1718
- "94e88d55-aac7-46bc-85cd-b7fcfc8a9ef6",
1719
- session_id,
1720
- ),
1721
- ),
1722
- )
1723
- .unwrap();
1724
- seed_abnormal_state_with_provider(&dir, &rollout, "alive", 1, "claude_code");
1725
- let coordinator = abnormal_test_coordinator(&dir);
1726
-
1727
- coordinator.tick().unwrap();
1728
-
1729
- let first_events = read_test_events(&dir);
1730
- let first_check =
1731
- find_event(&first_events, "worker.abnormal_exit.check").expect("stale check event");
1732
- assert_eq!(
1733
- first_check["latest_explicit_error"],
1734
- serde_json::json!(true)
1735
- );
1736
- assert_eq!(first_check["error_recency"], serde_json::json!("stale"));
1737
- assert_eq!(first_check["notification"], serde_json::json!(false));
1738
- assert_eq!(first_check["apiErrorStatus"], serde_json::json!(404));
1739
- assert_eq!(first_check["error"], serde_json::json!("model_not_found"));
1740
- assert_eq!(
1741
- first_check["requestId"],
1742
- serde_json::json!("req_011CceUVQ94dxahgwAHf8sdS")
1743
- );
1744
- assert_eq!(
1745
- first_check["assistant_uuid"],
1746
- serde_json::json!("94e88d55-aac7-46bc-85cd-b7fcfc8a9ef6")
1747
- );
1748
- assert!(
1749
- find_event(&first_events, "worker.abnormal_exit").is_none(),
1750
- "first observed error baselines stale even when followed by turn_duration"
1751
- );
1752
- let first_observation_key = abnormal_test_watch(&dir)["last_error_observation_key"]
1753
- .as_str()
1754
- .expect("stale baseline persists observation key")
1755
- .to_string();
1756
-
1757
- std::fs::OpenOptions::new()
1758
- .append(true)
1759
- .open(&rollout)
1760
- .unwrap()
1761
- .write_all(
1762
- format!(
1763
- "{}{}{}",
1764
- claude_file_history_snapshot_line(session_id),
1765
- claude_last_prompt_line(session_id),
1766
- claude_mode_line(session_id),
1767
- )
1768
- .as_bytes(),
1769
- )
1770
- .unwrap();
1771
- coordinator.tick().unwrap();
1772
-
1773
- let after_bookkeeping_events = read_test_events(&dir);
1774
- let bookkeeping_check = find_event(&after_bookkeeping_events, "worker.abnormal_exit.check")
1775
- .expect("bookkeeping check event");
1776
- assert_eq!(
1777
- bookkeeping_check["error_recency"],
1778
- serde_json::json!("stale")
1779
- );
1780
- assert_eq!(bookkeeping_check["notification"], serde_json::json!(false));
1781
- assert_eq!(
1782
- bookkeeping_check["assistant_uuid"],
1783
- serde_json::json!("94e88d55-aac7-46bc-85cd-b7fcfc8a9ef6")
1784
- );
1785
- assert_eq!(
1786
- abnormal_test_watch(&dir)["last_error_observation_key"],
1787
- serde_json::json!(first_observation_key)
1788
- );
1789
- assert!(
1790
- find_event(&after_bookkeeping_events, "worker.abnormal_exit").is_none(),
1791
- "bookkeeping-only growth after the same error must not notify"
1792
- );
1793
-
1794
- std::fs::OpenOptions::new()
1795
- .append(true)
1796
- .open(&rollout)
1797
- .unwrap()
1798
- .write_all(
1799
- format!(
1800
- "{}{}",
1801
- claude_assistant_api_error_line(
1802
- "11b8790e-718b-4910-bf6a-62d19135a93b",
1803
- "ccea3b0f-23e7-48ed-aa57-d59e321becfb",
1804
- session_id,
1805
- 404,
1806
- "model_not_found",
1807
- Some("req_011CceUhRwYfdmLPbv9nMmqX"),
1808
- ),
1809
- claude_turn_duration_line(
1810
- "2d382eff-783b-48ca-83c1-c93d3dc8b068",
1811
- "11b8790e-718b-4910-bf6a-62d19135a93b",
1812
- session_id,
1813
- ),
1814
- )
1815
- .as_bytes(),
1816
- )
1817
- .unwrap();
1818
- coordinator.tick().unwrap();
1819
-
1820
- let events = read_test_events(&dir);
1821
- let fresh_check =
1822
- find_event(&events, "worker.abnormal_exit.check").expect("fresh check event");
1823
- assert_eq!(fresh_check["error_recency"], serde_json::json!("fresh"));
1824
- assert_eq!(fresh_check["notification"], serde_json::json!(true));
1825
- assert_eq!(
1826
- fresh_check["assistant_uuid"],
1827
- serde_json::json!("11b8790e-718b-4910-bf6a-62d19135a93b")
1828
- );
1829
- let abnormal =
1830
- find_event(&events, "worker.abnormal_exit").expect("fresh API error notification");
1831
- assert_eq!(
1832
- abnormal["turn_id"],
1833
- serde_json::json!("11b8790e-718b-4910-bf6a-62d19135a93b")
1834
- );
1835
- assert_eq!(
1836
- abnormal["requestId"],
1837
- serde_json::json!("req_011CceUhRwYfdmLPbv9nMmqX")
1838
- );
1839
- assert_eq!(abnormal["notification_status"], serde_json::json!("queued"));
1840
- }
1841
-
1842
- #[test]
1843
- fn codex_failed_turn_then_completed_turn_no_fresh_notification() {
1844
- let dir = temp_abnormal_dir("codex-failed-then-completed");
1845
- let rollout = dir.join("rollout-w1.jsonl");
1846
- std::fs::write(
1847
- &rollout,
1848
- "{\"method\":\"turn/completed\",\"params\":{\"turn\":{\"id\":\"t1\",\"status\":\"failed\"}}}\n",
1849
- )
1850
- .unwrap();
1851
- seed_abnormal_state(&dir, &rollout, "alive", 1);
1852
- let coordinator = abnormal_test_coordinator(&dir);
1853
-
1854
- coordinator.tick().unwrap();
1855
-
1856
- let first_events = read_test_events(&dir);
1857
- let first_check =
1858
- find_event(&first_events, "worker.abnormal_exit.check").expect("stale check event");
1859
- assert_eq!(first_check["turn_id"], serde_json::json!("t1"));
1860
- assert_eq!(first_check["error_recency"], serde_json::json!("stale"));
1861
- assert_eq!(first_check["notification"], serde_json::json!(false));
1862
- assert!(
1863
- find_event(&first_events, "worker.abnormal_exit").is_none(),
1864
- "first failed turn is only a stale baseline"
1865
- );
1866
-
1867
- std::fs::OpenOptions::new()
1868
- .append(true)
1869
- .open(&rollout)
1870
- .unwrap()
1871
- .write_all(
1872
- b"{\"method\":\"turn/completed\",\"params\":{\"turn\":{\"id\":\"t-complete\",\"status\":\"completed\"}}}\n",
1873
- )
1874
- .unwrap();
1875
- coordinator.tick().unwrap();
1876
-
1877
- let after_completed_events = read_test_events(&dir);
1878
- let completed_check = find_event(&after_completed_events, "worker.abnormal_exit.check")
1879
- .expect("completed-after-failure check event");
1880
- assert_eq!(completed_check["turn_id"], serde_json::json!("t1"));
1881
- assert_eq!(completed_check["error_recency"], serde_json::json!("stale"));
1882
- assert_eq!(completed_check["fresh_error"], serde_json::json!(false));
1883
- assert_eq!(completed_check["notification"], serde_json::json!(false));
1884
- assert!(
1885
- find_event(&after_completed_events, "worker.abnormal_exit").is_none(),
1886
- "completed turn after the same old failed turn must not emit a fresh notification"
1887
- );
1888
-
1889
- std::fs::OpenOptions::new()
1890
- .append(true)
1891
- .open(&rollout)
1892
- .unwrap()
1893
- .write_all(
1894
- b"{\"method\":\"turn/completed\",\"params\":{\"turn\":{\"id\":\"t2\",\"status\":\"failed\"}}}\n",
1895
- )
1896
- .unwrap();
1897
- coordinator.tick().unwrap();
1898
-
1899
- let events = read_test_events(&dir);
1900
- let fresh_check =
1901
- find_event(&events, "worker.abnormal_exit.check").expect("new failed turn check event");
1902
- assert_eq!(fresh_check["turn_id"], serde_json::json!("t2"));
1903
- assert_eq!(fresh_check["error_recency"], serde_json::json!("fresh"));
1904
- assert_eq!(fresh_check["notification"], serde_json::json!(true));
1905
- let abnormal =
1906
- find_event(&events, "worker.abnormal_exit").expect("fresh Codex failure notification");
1907
- assert_eq!(abnormal["turn_id"], serde_json::json!("t2"));
1908
- assert_eq!(abnormal["notification_status"], serde_json::json!("queued"));
1909
- }
1910
-
1911
- #[test]
1912
- fn abnormal_dead_process_without_explicit_error_stays_dead_only() {
1913
- let dir = temp_abnormal_dir("dead-only");
1914
- let rollout = dir.join("rollout-w1.jsonl");
1915
- std::fs::write(
1916
- &rollout,
1917
- "{\"method\":\"turn/completed\",\"params\":{\"turn\":{\"id\":\"t1\",\"status\":\"completed\"}}}\n",
1918
- )
1919
- .unwrap();
1920
- seed_abnormal_state(&dir, &rollout, "dead", 1);
1921
- let coordinator = abnormal_test_coordinator(&dir);
1922
-
1923
- coordinator.tick().unwrap();
1924
-
1925
- let events = read_test_events(&dir);
1926
- assert!(
1927
- find_event(&events, "worker.abnormal_exit").is_none(),
1928
- "dead process without explicit error must not notify; events={events:?}"
1929
- );
1930
- let suppressed =
1931
- find_event(&events, "abnormal_exit.single_signal_suppressed").expect("dead_only");
1932
- assert_eq!(suppressed["reason"], serde_json::json!("dead_only"));
1933
- assert_eq!(
1934
- suppressed["latest_explicit_error"],
1935
- serde_json::json!(false)
1936
- );
1937
- assert_eq!(suppressed["error_recency"], serde_json::json!("none"));
1938
- assert_eq!(suppressed["fresh_error"], serde_json::json!(false));
1939
- }
1940
-
1941
- #[test]
1942
- fn abnormal_unchanged_transcript_keeps_provider_death_out_of_pane_projection() {
1943
- let dir = temp_abnormal_dir("unchanged-provider-dead");
1944
- let rollout = dir.join("rollout-w1.jsonl");
1945
- std::fs::write(
1946
- &rollout,
1947
- "{\"method\":\"turn/completed\",\"params\":{\"turn\":{\"id\":\"t1\",\"status\":\"completed\"}}}\n",
1948
- )
1949
- .unwrap();
1950
- seed_abnormal_state(&dir, &rollout, "alive", 1);
1951
- let coordinator = abnormal_test_coordinator(&dir);
1952
- coordinator.tick().unwrap();
1953
-
1954
- let mut state = crate::state::persist::load_runtime_state(&dir).unwrap();
1955
- state["agents"]["w1"]["process_liveness"] = serde_json::json!("dead");
1956
- crate::state::persist::save_runtime_state(&dir, &state).unwrap();
1957
- coordinator.tick().unwrap();
1958
-
1959
- let state = crate::state::persist::load_runtime_state(&dir).unwrap();
1960
- let agent = &state["agents"]["w1"];
1961
- assert!(
1962
- agent.get("provider_process_dead").is_none(),
1963
- "provider death belongs to abnormal watch, not the pane-dead seat projection: {agent}"
1964
- );
1965
- assert!(
1966
- agent.get("stale_reason").is_none(),
1967
- "provider death must not be mislabeled pane_dead: {agent}"
1968
- );
1969
- let watch = &state["coordinator"]["abnormal_exit_watch"]["w1"];
1970
- assert_eq!(watch["provider_process_dead"], serde_json::json!(true));
1971
- assert_eq!(watch["worker_provider_exited"], serde_json::json!(false));
1972
- assert_eq!(
1973
- watch["last_liveness_detail"],
1974
- serde_json::json!("explicit:dead")
1975
- );
1976
- }
1977
-
1978
- #[test]
1979
- fn abnormal_pane_absence_is_not_provider_process_death() {
1980
- let agent = test_abnormal_agent("/tmp/rollout.jsonl", Some(1), None);
1981
- for detail in ["pane_dead:%1", "window_missing:w1"] {
1982
- let payload = abnormal_watch_payload(
1983
- &agent,
1984
- Some(1),
1985
- Some(2),
1986
- process_check(ProcessLiveness::Dead, detail.to_string()),
1987
- None,
1988
- ErrorRecency::None,
1989
- None,
1990
- None,
1991
- );
1992
- assert_eq!(
1993
- payload["provider_process_dead"],
1994
- serde_json::json!(false),
1995
- "pane absence and provider death are distinct facts: {payload}"
1996
- );
1997
- assert_eq!(payload["worker_provider_exited"], serde_json::json!(false));
1998
- }
1999
- }
2000
-
2001
- #[test]
2002
- fn abnormal_recency_treats_cohort_change_as_stale() {
2003
- let agent = test_abnormal_agent("/tmp/rollout.jsonl", Some(2), None);
2004
- let fact = crate::provider::latest_explicit_error_fact(
2005
- crate::provider::Provider::Codex,
2006
- "{\"method\":\"turn/completed\",\"params\":{\"turn\":{\"id\":\"t1\",\"status\":\"failed\"}}}\n",
2007
- )
2008
- .unwrap();
2009
- let observation_key = abnormal_error_observation_key(&agent, &fact);
2010
- let state = serde_json::json!({
2011
- "coordinator": {
2012
- "abnormal_exit_watch": {
2013
- "w1": {
2014
- "last_error_observation_key": observation_key,
2015
- "last_error_observation_cohort": "worker.abnormal_exit.cohort:w1:/tmp/rollout.jsonl:spawn_epoch:1"
2016
- }
2017
- }
2018
- }
2019
- });
2020
- let cohort_key = abnormal_error_cohort_key(&agent);
2021
-
2022
- let recency = abnormal_error_recency(
2023
- &state,
2024
- &agent,
2025
- Some(observation_key.as_str()),
2026
- Some(cohort_key.as_str()),
2027
- );
2028
-
2029
- assert_eq!(
2030
- recency,
2031
- ErrorRecency::Stale,
2032
- "cohort change baselines the observed error instead of treating it as fresh"
2033
- );
2034
- }
2035
-
2036
- #[test]
2037
- fn worker_exit_marker_detail_includes_exit_code() {
2038
- let pane_id = crate::transport::PaneId::new("%marker-rc");
2039
- let marker = crate::tmux_backend::worker_provider_exit_marker("codex");
2040
- let transport = crate::transport::test_support::OfflineTransport::new()
2041
- .with_capture_for_pane(pane_id.as_str(), format!("\n{marker} 23\n"));
2042
- let mut agent = test_abnormal_agent("/tmp/rollout.jsonl", Some(1), None);
2043
- agent.pane_id = Some(pane_id.as_str().to_string());
2044
-
2045
- let check = worker_provider_exit_marker_check(&agent, &transport).expect("marker hit");
2046
-
2047
- assert_eq!(check.state, ProcessLiveness::Dead);
2048
- assert_eq!(check.detail, "worker_provider_exited:%marker-rc:rc=23");
2049
- }
2050
-
2051
- #[test]
2052
- fn worker_exit_marker_detail_uses_unknown_for_truncated_exit_code() {
2053
- let pane_id = crate::transport::PaneId::new("%marker-truncated");
2054
- let marker = crate::tmux_backend::worker_provider_exit_marker("codex");
2055
- let transport = crate::transport::test_support::OfflineTransport::new()
2056
- .with_capture_for_pane(pane_id.as_str(), format!("\n{marker} "));
2057
- let mut agent = test_abnormal_agent("/tmp/rollout.jsonl", Some(1), None);
2058
- agent.pane_id = Some(pane_id.as_str().to_string());
2059
-
2060
- let check = worker_provider_exit_marker_check(&agent, &transport).expect("marker hit");
2061
-
2062
- assert_eq!(check.state, ProcessLiveness::Dead);
2063
- assert_eq!(
2064
- check.detail,
2065
- "worker_provider_exited:%marker-truncated:rc=unknown"
2066
- );
2067
- }
2068
-
2069
- #[test]
2070
- fn abnormal_dead_fresh_error_event_reports_actual_dead_booleans() {
2071
- let dir = temp_abnormal_dir("dead-fresh");
2072
- let rollout = dir.join("rollout-w1.jsonl");
2073
- std::fs::write(
2074
- &rollout,
2075
- "{\"method\":\"turn/completed\",\"params\":{\"turn\":{\"id\":\"t1\",\"status\":\"failed\"}}}\n",
2076
- )
2077
- .unwrap();
2078
- seed_abnormal_state(&dir, &rollout, "dead", 1);
2079
- let coordinator = abnormal_test_coordinator(&dir);
2080
- coordinator.tick().unwrap();
2081
- std::fs::OpenOptions::new()
2082
- .append(true)
2083
- .open(&rollout)
2084
- .unwrap()
2085
- .write_all(
2086
- b"{\"method\":\"turn/completed\",\"params\":{\"turn\":{\"id\":\"t2\",\"status\":\"failed\"}}}\n",
2087
- )
2088
- .unwrap();
2089
-
2090
- coordinator.tick().unwrap();
2091
-
2092
- let events = read_test_events(&dir);
2093
- let abnormal =
2094
- find_event(&events, "worker.abnormal_exit").expect("fresh dead error notification");
2095
- assert_eq!(abnormal["provider_process_dead"], serde_json::json!(true));
2096
- assert_eq!(abnormal["latest_explicit_error"], serde_json::json!(true));
2097
- assert_eq!(abnormal["error_recency"], serde_json::json!("fresh"));
2098
- assert_eq!(
2099
- abnormal["provider_process_dead_and_latest_explicit_error"],
2100
- serde_json::json!(true)
2101
- );
2102
- }
2103
-
2104
- fn abnormal_test_coordinator(dir: &std::path::Path) -> Coordinator {
2105
- Coordinator::for_test(
2106
- WorkspacePath::new(dir.to_path_buf()),
2107
- Box::new(NormalRegistry),
2108
- Box::new(
2109
- crate::transport::test_support::OfflineTransport::new().with_session_present(true),
2110
- ),
2111
- None,
2112
- None,
2113
- )
2114
- }
2115
-
2116
- fn seed_abnormal_state(
2117
- dir: &std::path::Path,
2118
- rollout: &std::path::Path,
2119
- liveness: &str,
2120
- spawn_epoch: u64,
2121
- ) {
2122
- seed_abnormal_state_with_provider(dir, rollout, liveness, spawn_epoch, "codex");
2123
- }
2124
-
2125
- fn seed_abnormal_state_with_provider(
2126
- dir: &std::path::Path,
2127
- rollout: &std::path::Path,
2128
- liveness: &str,
2129
- spawn_epoch: u64,
2130
- provider: &str,
2131
- ) {
2132
- crate::state::persist::save_runtime_state(
2133
- dir,
2134
- &serde_json::json!({
2135
- "active_team_key": "team",
2136
- "agents": {
2137
- "w1": {
2138
- "provider": provider,
2139
- "status": "running",
2140
- "agent_id": "w1",
2141
- "session_id": "session-w1",
2142
- "rollout_path": rollout.to_string_lossy(),
2143
- "spawn_cwd": dir.to_string_lossy(),
2144
- "spawn_epoch": spawn_epoch,
2145
- "process_liveness": liveness
2146
- }
2147
- }
2148
- }),
2149
- )
2150
- .unwrap();
2151
- }
2152
-
2153
- fn claude_assistant_api_error_line(
2154
- uuid: &str,
2155
- parent_uuid: &str,
2156
- session_id: &str,
2157
- status: i64,
2158
- error: &str,
2159
- request_id: Option<&str>,
2160
- ) -> String {
2161
- let mut record = serde_json::json!({
2162
- "type": "assistant",
2163
- "parentUuid": parent_uuid,
2164
- "uuid": uuid,
2165
- "message": {
2166
- "role": "assistant",
2167
- "content": [{"type": "text", "text": "API Error"}]
2168
- },
2169
- "error": error,
2170
- "isApiErrorMessage": true,
2171
- "apiErrorStatus": status,
2172
- "sessionId": session_id,
2173
- "version": "2.1.181"
2174
- });
2175
- if let Some(request_id) = request_id {
2176
- record
2177
- .as_object_mut()
2178
- .unwrap()
2179
- .insert("requestId".to_string(), serde_json::json!(request_id));
2180
- }
2181
- format!("{record}\n")
2182
- }
2183
-
2184
- fn claude_turn_duration_line(uuid: &str, parent_uuid: &str, session_id: &str) -> String {
2185
- format!(
2186
- "{}\n",
2187
- serde_json::json!({
2188
- "type": "system",
2189
- "subtype": "turn_duration",
2190
- "uuid": uuid,
2191
- "parentUuid": parent_uuid,
2192
- "sessionId": session_id
2193
- })
2194
- )
2195
- }
2196
-
2197
- fn claude_file_history_snapshot_line(session_id: &str) -> String {
2198
- format!(
2199
- "{}\n",
2200
- serde_json::json!({
2201
- "type": "file-history-snapshot",
2202
- "sessionId": session_id
2203
- })
2204
- )
2205
- }
2206
-
2207
- fn claude_last_prompt_line(session_id: &str) -> String {
2208
- format!(
2209
- "{}\n",
2210
- serde_json::json!({
2211
- "type": "last-prompt",
2212
- "sessionId": session_id,
2213
- "prompt": "sanitized"
2214
- })
2215
- )
2216
- }
2217
-
2218
- fn claude_mode_line(session_id: &str) -> String {
2219
- format!(
2220
- "{}\n",
2221
- serde_json::json!({
2222
- "type": "mode",
2223
- "sessionId": session_id,
2224
- "mode": "default"
2225
- })
2226
- )
2227
- }
2228
-
2229
- fn abnormal_test_watch(dir: &std::path::Path) -> Value {
2230
- crate::state::persist::load_runtime_state(dir)
2231
- .unwrap()
2232
- .pointer("/coordinator/abnormal_exit_watch/w1")
2233
- .cloned()
2234
- .unwrap()
2235
- }
2236
-
2237
- fn read_test_events(dir: &std::path::Path) -> Vec<Value> {
2238
- let events_path = crate::model::paths::logs_dir(dir).join("events.jsonl");
2239
- std::fs::read_to_string(events_path)
2240
- .unwrap_or_default()
2241
- .lines()
2242
- .filter_map(|line| serde_json::from_str(line).ok())
2243
- .collect()
2244
- }
2245
-
2246
- fn find_event(events: &[Value], name: &str) -> Option<Value> {
2247
- events
2248
- .iter()
2249
- .rev()
2250
- .find(|event| event.get("event").and_then(Value::as_str) == Some(name))
2251
- .cloned()
2252
- }
2253
-
2254
- fn temp_abnormal_dir(tag: &str) -> std::path::PathBuf {
2255
- let dir = std::env::temp_dir().join(format!(
2256
- "team-agent-abnormal-{tag}-{}-{}",
2257
- std::process::id(),
2258
- std::time::SystemTime::now()
2259
- .duration_since(std::time::UNIX_EPOCH)
2260
- .unwrap()
2261
- .as_nanos()
2262
- ));
2263
- std::fs::create_dir_all(&dir).unwrap();
2264
- dir
2265
- }
2266
-
2267
- fn test_abnormal_agent(
2268
- rollout_path: &str,
2269
- spawn_epoch: Option<u64>,
2270
- spawned_at: Option<&str>,
2271
- ) -> AbnormalWatchAgent {
2272
- AbnormalWatchAgent {
2273
- agent_id: "w1".to_string(),
2274
- provider: crate::provider::Provider::Codex,
2275
- rollout_path: PathBuf::from(rollout_path),
2276
- rollout_path_display: rollout_path.to_string(),
2277
- spawn_epoch,
2278
- spawned_at: spawned_at.map(str::to_string),
2279
- status: Some("running".to_string()),
2280
- process_liveness: Some(ProcessLiveness::Alive),
2281
- window: None,
2282
- pane_id: None,
2283
- pid: None,
2284
- current_command: None,
2285
- }
2286
- }
2287
-
2288
- #[test]
2289
- fn retired_legacy_rows_get_a_remove_and_re_add_hint_not_a_refused_start_agent() {
2290
- use crate::model::enums::Provider;
2291
- let workspace = std::path::Path::new("/tmp/ws");
2292
- for provider in [Provider::Claude, Provider::ClaudeCode, Provider::Codex, Provider::Grok, Provider::Pi] {
2293
- let hint = recovery_hint("w1", provider, workspace, "research");
2294
- assert!(hint.starts_with("No automatic restart was performed.\n"), "{hint}");
2295
- assert!(
2296
- hint.contains("seat has no runtime in this release; stop or remove it, then add it again"),
2297
- "{hint}"
2298
- );
2299
- assert!(!hint.contains("start-agent"), "{hint}");
2300
- }
2301
- assert!(recovery_hint("w1", Provider::Codex, workspace, "research").ends_with("(provider: codex)"));
2302
- assert!(recovery_hint("w1", Provider::Grok, workspace, "research").ends_with("(provider: grok)"));
2303
- let hint = recovery_hint("w1", Provider::Copilot, workspace, "research");
2304
- assert!(
2305
- hint.ends_with("To recover manually, run: team-agent start-agent w1 --workspace '/tmp/ws' --team research --force --json"),
2306
- "{hint}"
2307
- );
2308
- }
2309
- }