@team-agent/installer 0.5.128 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (436) hide show
  1. package/README.md +4 -4
  2. package/README.zh.md +1 -1
  3. package/package.json +4 -8
  4. package/skills/team-agent/SKILL.md +37 -20
  5. package/skills/team-agent/command-coverage.json +169 -183
  6. package/skills/team-agent/docs/reference/team-agent-operator.md +64 -30
  7. package/skills/team-agent/references/recovery-runbook.md +1 -1
  8. package/skills/team-agent/references/team-agent-operator.md +64 -30
  9. package/templates/team_state.md +1 -1
  10. package/Cargo.lock +0 -1063
  11. package/Cargo.toml +0 -36
  12. package/crates/team-agent/Cargo.toml +0 -69
  13. package/crates/team-agent/src/app_server_test_support.rs +0 -258
  14. package/crates/team-agent/src/cli/adapters.rs +0 -1288
  15. package/crates/team-agent/src/cli/arguments.rs +0 -405
  16. package/crates/team-agent/src/cli/attach_app_server_leader.rs +0 -98
  17. package/crates/team-agent/src/cli/diagnose.rs +0 -1721
  18. package/crates/team-agent/src/cli/emit.rs +0 -2930
  19. package/crates/team-agent/src/cli/grok_slot.rs +0 -384
  20. package/crates/team-agent/src/cli/helpers.rs +0 -91
  21. package/crates/team-agent/src/cli/intent/runtime.rs +0 -1216
  22. package/crates/team-agent/src/cli/intent/runtime_tests.rs +0 -912
  23. package/crates/team-agent/src/cli/intent/tests.rs +0 -689
  24. package/crates/team-agent/src/cli/intent.rs +0 -664
  25. package/crates/team-agent/src/cli/leader.rs +0 -300
  26. package/crates/team-agent/src/cli/leader_prompt.rs +0 -261
  27. package/crates/team-agent/src/cli/leaders.rs +0 -418
  28. package/crates/team-agent/src/cli/mod.rs +0 -6545
  29. package/crates/team-agent/src/cli/models.rs +0 -1206
  30. package/crates/team-agent/src/cli/named_address.rs +0 -1651
  31. package/crates/team-agent/src/cli/profile.rs +0 -342
  32. package/crates/team-agent/src/cli/route.rs +0 -212
  33. package/crates/team-agent/src/cli/schedule.rs +0 -175
  34. package/crates/team-agent/src/cli/send/coordinator.rs +0 -164
  35. package/crates/team-agent/src/cli/send/mailbox.rs +0 -145
  36. package/crates/team-agent/src/cli/send/persist.rs +0 -156
  37. package/crates/team-agent/src/cli/send/presentation.rs +0 -370
  38. package/crates/team-agent/src/cli/send/resolve.rs +0 -573
  39. package/crates/team-agent/src/cli/send.rs +0 -199
  40. package/crates/team-agent/src/cli/shutdown_socket.rs +0 -288
  41. package/crates/team-agent/src/cli/spec.rs +0 -165
  42. package/crates/team-agent/src/cli/status.rs +0 -280
  43. package/crates/team-agent/src/cli/status_port/agents.rs +0 -358
  44. package/crates/team-agent/src/cli/status_port/approvals.rs +0 -79
  45. package/crates/team-agent/src/cli/status_port/brief.rs +0 -1981
  46. package/crates/team-agent/src/cli/status_port/compact.rs +0 -240
  47. package/crates/team-agent/src/cli/status_port/format.rs +0 -146
  48. package/crates/team-agent/src/cli/status_port/inbox.rs +0 -62
  49. package/crates/team-agent/src/cli/status_port/runtime.rs +0 -197
  50. package/crates/team-agent/src/cli/status_port/snapshot.rs +0 -208
  51. package/crates/team-agent/src/cli/status_port/store.rs +0 -416
  52. package/crates/team-agent/src/cli/status_port/tests.rs +0 -90
  53. package/crates/team-agent/src/cli/status_port.rs +0 -92
  54. package/crates/team-agent/src/cli/tests/base.rs +0 -885
  55. package/crates/team-agent/src/cli/tests/divergence.rs +0 -518
  56. package/crates/team-agent/src/cli/tests/lane_c.rs +0 -453
  57. package/crates/team-agent/src/cli/tests/leader_watch.rs +0 -322
  58. package/crates/team-agent/src/cli/tests/main_preserved.rs +0 -430
  59. package/crates/team-agent/src/cli/tests/missing_subcommands.rs +0 -336
  60. package/crates/team-agent/src/cli/tests/mod.rs +0 -137
  61. package/crates/team-agent/src/cli/tests/named_address.rs +0 -761
  62. package/crates/team-agent/src/cli/tests/parser_safety.rs +0 -383
  63. package/crates/team-agent/src/cli/tests/run_delegation.rs +0 -484
  64. package/crates/team-agent/src/cli/tests/shutdown_kill_plan.rs +0 -2179
  65. package/crates/team-agent/src/cli/tests/status_brief.rs +0 -1514
  66. package/crates/team-agent/src/cli/tests/status_send.rs +0 -888
  67. package/crates/team-agent/src/cli/tests/verb_install_skill.rs +0 -93
  68. package/crates/team-agent/src/cli/tests/verb_profile.rs +0 -392
  69. package/crates/team-agent/src/cli/triage.rs +0 -234
  70. package/crates/team-agent/src/cli/types.rs +0 -591
  71. package/crates/team-agent/src/codex_app_server.rs +0 -546
  72. package/crates/team-agent/src/communication_mode/mod.rs +0 -56
  73. package/crates/team-agent/src/compiler/tests.rs +0 -827
  74. package/crates/team-agent/src/compiler.rs +0 -890
  75. package/crates/team-agent/src/conpty/backend.rs +0 -829
  76. package/crates/team-agent/src/conpty/mod.rs +0 -33
  77. package/crates/team-agent/src/contract_runtime/backend.rs +0 -1744
  78. package/crates/team-agent/src/contract_runtime/commands.rs +0 -671
  79. package/crates/team-agent/src/contract_runtime/config.rs +0 -364
  80. package/crates/team-agent/src/contract_runtime/config_tests.rs +0 -76
  81. package/crates/team-agent/src/contract_runtime/discovery.rs +0 -449
  82. package/crates/team-agent/src/contract_runtime/fork.rs +0 -361
  83. package/crates/team-agent/src/contract_runtime/forward.rs +0 -288
  84. package/crates/team-agent/src/contract_runtime/framework.rs +0 -367
  85. package/crates/team-agent/src/contract_runtime/mcp.rs +0 -394
  86. package/crates/team-agent/src/contract_runtime/mod.rs +0 -17
  87. package/crates/team-agent/src/contract_runtime/prompt.rs +0 -216
  88. package/crates/team-agent/src/contract_runtime/pump.rs +0 -271
  89. package/crates/team-agent/src/contract_runtime/registry.rs +0 -553
  90. package/crates/team-agent/src/contract_runtime/workspace.rs +0 -228
  91. package/crates/team-agent/src/coordinator/backoff.rs +0 -741
  92. package/crates/team-agent/src/coordinator/conpty_shim.rs +0 -805
  93. package/crates/team-agent/src/coordinator/health.rs +0 -1185
  94. package/crates/team-agent/src/coordinator/mod.rs +0 -119
  95. package/crates/team-agent/src/coordinator/orphan.rs +0 -110
  96. package/crates/team-agent/src/coordinator/runtime_detectors.rs +0 -798
  97. package/crates/team-agent/src/coordinator/runtime_observation.rs +0 -84
  98. package/crates/team-agent/src/coordinator/steps/abnormal.rs +0 -2309
  99. package/crates/team-agent/src/coordinator/steps/delivery.rs +0 -15
  100. package/crates/team-agent/src/coordinator/steps/health_sync.rs +0 -15
  101. package/crates/team-agent/src/coordinator/steps/mod.rs +0 -126
  102. package/crates/team-agent/src/coordinator/steps/persist.rs +0 -15
  103. package/crates/team-agent/src/coordinator/steps/runtime_prompts.rs +0 -15
  104. package/crates/team-agent/src/coordinator/steps/session_gate.rs +0 -16
  105. package/crates/team-agent/src/coordinator/tests/a0_lostupdate.rs +0 -158
  106. package/crates/team-agent/src/coordinator/tests/abnormal.rs +0 -312
  107. package/crates/team-agent/src/coordinator/tests/api_error_recovery.rs +0 -300
  108. package/crates/team-agent/src/coordinator/tests/basics.rs +0 -211
  109. package/crates/team-agent/src/coordinator/tests/daemon.rs +0 -514
  110. package/crates/team-agent/src/coordinator/tests/energy.rs +0 -549
  111. package/crates/team-agent/src/coordinator/tests/health_sync.rs +0 -503
  112. package/crates/team-agent/src/coordinator/tests/main_preserved.rs +0 -186
  113. package/crates/team-agent/src/coordinator/tests/mod.rs +0 -295
  114. package/crates/team-agent/src/coordinator/tests/s3_bounded.rs +0 -207
  115. package/crates/team-agent/src/coordinator/tests/spine.rs +0 -308
  116. package/crates/team-agent/src/coordinator/tests/takeover.rs +0 -286
  117. package/crates/team-agent/src/coordinator/tests/tick_core.rs +0 -237
  118. package/crates/team-agent/src/coordinator/tick.rs +0 -3024
  119. package/crates/team-agent/src/coordinator/types.rs +0 -540
  120. package/crates/team-agent/src/db/agent_health_capture.rs +0 -126
  121. package/crates/team-agent/src/db/message_store.rs +0 -1527
  122. package/crates/team-agent/src/db/migration.rs +0 -984
  123. package/crates/team-agent/src/db/mod.rs +0 -26
  124. package/crates/team-agent/src/db/schema.rs +0 -663
  125. package/crates/team-agent/src/diagnose/comms.rs +0 -242
  126. package/crates/team-agent/src/diagnose/mod.rs +0 -44
  127. package/crates/team-agent/src/diagnose/orphans.rs +0 -889
  128. package/crates/team-agent/src/direct_delivery.rs +0 -368
  129. package/crates/team-agent/src/event_log.rs +0 -502
  130. package/crates/team-agent/src/fake_worker.rs +0 -303
  131. package/crates/team-agent/src/inbox_store.rs +0 -169
  132. package/crates/team-agent/src/kill_audit.rs +0 -106
  133. package/crates/team-agent/src/layout/manager.rs +0 -248
  134. package/crates/team-agent/src/layout/mod.rs +0 -36
  135. package/crates/team-agent/src/layout/overlay.rs +0 -134
  136. package/crates/team-agent/src/layout/placement.rs +0 -52
  137. package/crates/team-agent/src/layout/recovery.rs +0 -106
  138. package/crates/team-agent/src/layout/runtime_sessions.rs +0 -317
  139. package/crates/team-agent/src/layout/sessions.rs +0 -294
  140. package/crates/team-agent/src/layout/tmux_endpoint.rs +0 -165
  141. package/crates/team-agent/src/layout/worker_env.rs +0 -721
  142. package/crates/team-agent/src/layout/worker_window_helpers.rs +0 -37
  143. package/crates/team-agent/src/leader/helpers.rs +0 -178
  144. package/crates/team-agent/src/leader/incident.rs +0 -6
  145. package/crates/team-agent/src/leader/lease.rs +0 -3356
  146. package/crates/team-agent/src/leader/mod.rs +0 -104
  147. package/crates/team-agent/src/leader/owner_bind.rs +0 -400
  148. package/crates/team-agent/src/leader/prompt.rs +0 -905
  149. package/crates/team-agent/src/leader/provider_attribution.rs +0 -247
  150. package/crates/team-agent/src/leader/rediscover/tests.rs +0 -689
  151. package/crates/team-agent/src/leader/rediscover.rs +0 -1192
  152. package/crates/team-agent/src/leader/registry.rs +0 -888
  153. package/crates/team-agent/src/leader/start.rs +0 -4501
  154. package/crates/team-agent/src/leader/takeover.rs +0 -65
  155. package/crates/team-agent/src/leader/tests/basics.rs +0 -337
  156. package/crates/team-agent/src/leader/tests/byte_findings.rs +0 -355
  157. package/crates/team-agent/src/leader/tests/claim_leader_mailbox_flush_contract.rs +0 -544
  158. package/crates/team-agent/src/leader/tests/historical_inventory_upgrade_gate_red.rs +0 -624
  159. package/crates/team-agent/src/leader/tests/identity.rs +0 -465
  160. package/crates/team-agent/src/leader/tests/identity_session_names.rs +0 -42
  161. package/crates/team-agent/src/leader/tests/idle.rs +0 -113
  162. package/crates/team-agent/src/leader/tests/lease_api.rs +0 -520
  163. package/crates/team-agent/src/leader/tests/lease_claim.rs +0 -629
  164. package/crates/team-agent/src/leader/tests/mod.rs +0 -106
  165. package/crates/team-agent/src/leader/tests/prompt_contract.rs +0 -268
  166. package/crates/team-agent/src/leader/tests/rediscover.rs +0 -403
  167. package/crates/team-agent/src/leader/tests/rehomed_env.rs +0 -83
  168. package/crates/team-agent/src/leader/tests/team_in_team_state_scope_red.rs +0 -1273
  169. package/crates/team-agent/src/leader/tests/terminal_and_replay_vs_rerender_invariants_red.rs +0 -417
  170. package/crates/team-agent/src/leader/tests/wake_start_owner.rs +0 -168
  171. package/crates/team-agent/src/leader/types.rs +0 -534
  172. package/crates/team-agent/src/lib.rs +0 -147
  173. package/crates/team-agent/src/lifecycle/helpers.rs +0 -212
  174. package/crates/team-agent/src/lifecycle/launch/add_agent.rs +0 -881
  175. package/crates/team-agent/src/lifecycle/launch/add_agent_state.rs +0 -598
  176. package/crates/team-agent/src/lifecycle/launch/agent_state.rs +0 -332
  177. package/crates/team-agent/src/lifecycle/launch/clone_agent.rs +0 -204
  178. package/crates/team-agent/src/lifecycle/launch/fork_agent.rs +0 -107
  179. package/crates/team-agent/src/lifecycle/launch/fork_entry.rs +0 -144
  180. package/crates/team-agent/src/lifecycle/launch/identity.rs +0 -674
  181. package/crates/team-agent/src/lifecycle/launch/layout.rs +0 -350
  182. package/crates/team-agent/src/lifecycle/launch/leader_context.rs +0 -589
  183. package/crates/team-agent/src/lifecycle/launch/mcp_config.rs +0 -160
  184. package/crates/team-agent/src/lifecycle/launch/ownership.rs +0 -105
  185. package/crates/team-agent/src/lifecycle/launch/pi_mcp.rs +0 -899
  186. package/crates/team-agent/src/lifecycle/launch/pi_mcp_extension.js +0 -225
  187. package/crates/team-agent/src/lifecycle/launch/plan.rs +0 -269
  188. package/crates/team-agent/src/lifecycle/launch/quick_start.rs +0 -5200
  189. package/crates/team-agent/src/lifecycle/launch/quick_start_transport.rs +0 -368
  190. package/crates/team-agent/src/lifecycle/launch/readiness.rs +0 -519
  191. package/crates/team-agent/src/lifecycle/launch/role_source.rs +0 -287
  192. package/crates/team-agent/src/lifecycle/launch/spawn.rs +0 -473
  193. package/crates/team-agent/src/lifecycle/launch/spec_state.rs +0 -801
  194. package/crates/team-agent/src/lifecycle/launch/state_projection.rs +0 -624
  195. package/crates/team-agent/src/lifecycle/launch/worker_env.rs +0 -558
  196. package/crates/team-agent/src/lifecycle/launch.rs +0 -557
  197. package/crates/team-agent/src/lifecycle/lock.rs +0 -350
  198. package/crates/team-agent/src/lifecycle/mod.rs +0 -79
  199. package/crates/team-agent/src/lifecycle/pane_input_lock.rs +0 -239
  200. package/crates/team-agent/src/lifecycle/profile_launch.rs +0 -1067
  201. package/crates/team-agent/src/lifecycle/profile_smoke.rs +0 -557
  202. package/crates/team-agent/src/lifecycle/restart/agent.rs +0 -2199
  203. package/crates/team-agent/src/lifecycle/restart/avatar.rs +0 -1798
  204. package/crates/team-agent/src/lifecycle/restart/claude_fork.rs +0 -163
  205. package/crates/team-agent/src/lifecycle/restart/common.rs +0 -2362
  206. package/crates/team-agent/src/lifecycle/restart/orchestrator.rs +0 -65
  207. package/crates/team-agent/src/lifecycle/restart/preflight.rs +0 -113
  208. package/crates/team-agent/src/lifecycle/restart/rebuild.rs +0 -5135
  209. package/crates/team-agent/src/lifecycle/restart/remove.rs +0 -1504
  210. package/crates/team-agent/src/lifecycle/restart/selection.rs +0 -559
  211. package/crates/team-agent/src/lifecycle/restart/team_state.rs +0 -329
  212. package/crates/team-agent/src/lifecycle/restart.rs +0 -227
  213. package/crates/team-agent/src/lifecycle/role_config.rs +0 -1230
  214. package/crates/team-agent/src/lifecycle/tests/acceptance_b_batch_red.rs +0 -1021
  215. package/crates/team-agent/src/lifecycle/tests/agent_ops.rs +0 -1205
  216. package/crates/team-agent/src/lifecycle/tests/b0_legacy_snapshot_nonauthority_contract.rs +0 -265
  217. package/crates/team-agent/src/lifecycle/tests/b0_reader_hideone_audit_contract.rs +0 -490
  218. package/crates/team-agent/src/lifecycle/tests/behavioral_diff_264_red.rs +0 -656
  219. package/crates/team-agent/src/lifecycle/tests/claude_compatible_config_red.rs +0 -325
  220. package/crates/team-agent/src/lifecycle/tests/claude_profile_launch_red.rs +0 -447
  221. package/crates/team-agent/src/lifecycle/tests/cli_argv_routing_red.rs +0 -1638
  222. package/crates/team-agent/src/lifecycle/tests/clone_agent_preserves_source_tools.rs +0 -427
  223. package/crates/team-agent/src/lifecycle/tests/clone_fork_copilot_perms_red.rs +0 -770
  224. package/crates/team-agent/src/lifecycle/tests/communication_mode_runtime_contract_red.rs +0 -419
  225. package/crates/team-agent/src/lifecycle/tests/copilot_provider_red.rs +0 -2035
  226. package/crates/team-agent/src/lifecycle/tests/core.rs +0 -1327
  227. package/crates/team-agent/src/lifecycle/tests/f032_startup_prompt_best_effort_red.rs +0 -315
  228. package/crates/team-agent/src/lifecycle/tests/fixtures/pi_list_models_g0.stdout.txt +0 -222
  229. package/crates/team-agent/src/lifecycle/tests/fixtures/pi_list_models_spaced_ids.stdout.txt +0 -9
  230. package/crates/team-agent/src/lifecycle/tests/fixtures/pi_native_mcp_host.mjs +0 -72
  231. package/crates/team-agent/src/lifecycle/tests/gate_fixtures.rs +0 -562
  232. package/crates/team-agent/src/lifecycle/tests/grok_require_explicit_model_red.rs +0 -110
  233. package/crates/team-agent/src/lifecycle/tests/harvest2_a_batch_red.rs +0 -512
  234. package/crates/team-agent/src/lifecycle/tests/host_cotenant_death_p0_contract.rs +0 -921
  235. package/crates/team-agent/src/lifecycle/tests/issue238_codex_effort_contract.rs +0 -68
  236. package/crates/team-agent/src/lifecycle/tests/lane_ops.rs +0 -1851
  237. package/crates/team-agent/src/lifecycle/tests/launch_spawn.rs +0 -3983
  238. package/crates/team-agent/src/lifecycle/tests/lifecycle_lock.rs +0 -339
  239. package/crates/team-agent/src/lifecycle/tests/lifecycle_rollback_red.rs +0 -596
  240. package/crates/team-agent/src/lifecycle/tests/main_preserved.rs +0 -513
  241. package/crates/team-agent/src/lifecycle/tests/mcp_tool_name_format_red.rs +0 -69
  242. package/crates/team-agent/src/lifecycle/tests/phase_b_contracts.rs +0 -824
  243. package/crates/team-agent/src/lifecycle/tests/phase_golden.rs +0 -723
  244. package/crates/team-agent/src/lifecycle/tests/pi_catalog_red.rs +0 -164
  245. package/crates/team-agent/src/lifecycle/tests/pi_command_red.rs +0 -200
  246. package/crates/team-agent/src/lifecycle/tests/pi_compiler_red.rs +0 -210
  247. package/crates/team-agent/src/lifecycle/tests/pi_dual_entry_red.rs +0 -527
  248. package/crates/team-agent/src/lifecycle/tests/pi_executable_mcp_red.rs +0 -1062
  249. package/crates/team-agent/src/lifecycle/tests/pi_fork_session_inheritance_red.rs +0 -1419
  250. package/crates/team-agent/src/lifecycle/tests/pi_lifecycle_doctor_red.rs +0 -98
  251. package/crates/team-agent/src/lifecycle/tests/pi_native_mcp_capability_red.rs +0 -550
  252. package/crates/team-agent/src/lifecycle/tests/pi_session_delivery_red.rs +0 -226
  253. package/crates/team-agent/src/lifecycle/tests/quick_start_worker_readiness_red.rs +0 -332
  254. package/crates/team-agent/src/lifecycle/tests/realmachine_clusters_1_6_red.rs +0 -786
  255. package/crates/team-agent/src/lifecycle/tests/realmachine_residual_g1_g4_red.rs +0 -648
  256. package/crates/team-agent/src/lifecycle/tests/restart.rs +0 -185
  257. package/crates/team-agent/src/lifecycle/tests/restart_build_before_destroy_0540_contract.rs +0 -1053
  258. package/crates/team-agent/src/lifecycle/tests/restart_liveness_red.rs +0 -679
  259. package/crates/team-agent/src/lifecycle/tests/restart_rebind_hotfix_252_red.rs +0 -1126
  260. package/crates/team-agent/src/lifecycle/tests/restart_session_capture_red.rs +0 -153
  261. package/crates/team-agent/src/lifecycle/tests/resume_recover_red.rs +0 -427
  262. package/crates/team-agent/src/lifecycle/tests/stale_team_saveconflict_contract.rs +0 -431
  263. package/crates/team-agent/src/lifecycle/tests/startup_latency_contract.rs +0 -1066
  264. package/crates/team-agent/src/lifecycle/tests/status_credential_redaction_contract.rs +0 -876
  265. package/crates/team-agent/src/lifecycle/tests/subprep_codex_trust_roundtrip_red.rs +0 -1026
  266. package/crates/team-agent/src/lifecycle/tests/swallow_batch4_semantics_red.rs +0 -342
  267. package/crates/team-agent/src/lifecycle/tests/team_in_team_identity_scope_red.rs +0 -543
  268. package/crates/team-agent/src/lifecycle/tests/team_in_team_sibling_quick_start_red.rs +0 -1103
  269. package/crates/team-agent/src/lifecycle/tests/team_in_team_state_scope_red.rs +0 -1276
  270. package/crates/team-agent/src/lifecycle/tests/team_key_retirement_roster_red.rs +0 -145
  271. package/crates/team-agent/src/lifecycle/tests/team_key_retirement_txn_red.rs +0 -603
  272. package/crates/team-agent/src/lifecycle/tests/test_isolation_escape_contract.rs +0 -936
  273. package/crates/team-agent/src/lifecycle/tests/upgrade_compat_0211_red.rs +0 -970
  274. package/crates/team-agent/src/lifecycle/tests/verify_rs031_window_consistency_red.rs +0 -995
  275. package/crates/team-agent/src/lifecycle/tests/worker_spawn_env_red.rs +0 -836
  276. package/crates/team-agent/src/lifecycle/tests.rs +0 -105
  277. package/crates/team-agent/src/lifecycle/types.rs +0 -877
  278. package/crates/team-agent/src/lifecycle/worker_command_context.rs +0 -363
  279. package/crates/team-agent/src/main.rs +0 -52
  280. package/crates/team-agent/src/mcp_server/helpers.rs +0 -510
  281. package/crates/team-agent/src/mcp_server/lifecycle_tools/mod.rs +0 -6
  282. package/crates/team-agent/src/mcp_server/lifecycle_tools/state_status.rs +0 -43
  283. package/crates/team-agent/src/mcp_server/mod.rs +0 -119
  284. package/crates/team-agent/src/mcp_server/normalize.rs +0 -485
  285. package/crates/team-agent/src/mcp_server/readiness.rs +0 -527
  286. package/crates/team-agent/src/mcp_server/readiness_tests.rs +0 -348
  287. package/crates/team-agent/src/mcp_server/tests/golden.rs +0 -285
  288. package/crates/team-agent/src/mcp_server/tests/normalize.rs +0 -247
  289. package/crates/team-agent/src/mcp_server/tests/scoped.rs +0 -158
  290. package/crates/team-agent/src/mcp_server/tests/send.rs +0 -353
  291. package/crates/team-agent/src/mcp_server/tests/tools.rs +0 -341
  292. package/crates/team-agent/src/mcp_server/tests/wire.rs +0 -254
  293. package/crates/team-agent/src/mcp_server/tests.rs +0 -43
  294. package/crates/team-agent/src/mcp_server/tools.rs +0 -889
  295. package/crates/team-agent/src/mcp_server/types.rs +0 -396
  296. package/crates/team-agent/src/mcp_server/wire.rs +0 -867
  297. package/crates/team-agent/src/messaging/activity.rs +0 -461
  298. package/crates/team-agent/src/messaging/address.rs +0 -87
  299. package/crates/team-agent/src/messaging/delivery.rs +0 -3727
  300. package/crates/team-agent/src/messaging/delivery_consumption_tests.rs +0 -234
  301. package/crates/team-agent/src/messaging/enter_rescue.rs +0 -362
  302. package/crates/team-agent/src/messaging/enter_rescue_tests.rs +0 -633
  303. package/crates/team-agent/src/messaging/helpers.rs +0 -336
  304. package/crates/team-agent/src/messaging/leader_channel.rs +0 -287
  305. package/crates/team-agent/src/messaging/leader_receiver.rs +0 -786
  306. package/crates/team-agent/src/messaging/mod.rs +0 -167
  307. package/crates/team-agent/src/messaging/persist.rs +0 -377
  308. package/crates/team-agent/src/messaging/presentation.rs +0 -417
  309. package/crates/team-agent/src/messaging/reminders.rs +0 -316
  310. package/crates/team-agent/src/messaging/results/collection_tests.rs +0 -719
  311. package/crates/team-agent/src/messaging/results.rs +0 -1415
  312. package/crates/team-agent/src/messaging/scheduler.rs +0 -223
  313. package/crates/team-agent/src/messaging/selftest.rs +0 -315
  314. package/crates/team-agent/src/messaging/send.rs +0 -981
  315. package/crates/team-agent/src/messaging/tests/basic.rs +0 -346
  316. package/crates/team-agent/src/messaging/tests/dup_inject.rs +0 -657
  317. package/crates/team-agent/src/messaging/tests/e23.rs +0 -49
  318. package/crates/team-agent/src/messaging/tests/leader_channel.rs +0 -169
  319. package/crates/team-agent/src/messaging/tests/leader_inject_acceptance.rs +0 -817
  320. package/crates/team-agent/src/messaging/tests/main_preserved.rs +0 -85
  321. package/crates/team-agent/src/messaging/tests/mod.rs +0 -306
  322. package/crates/team-agent/src/messaging/tests/runtime.rs +0 -3549
  323. package/crates/team-agent/src/messaging/tests/spine.rs +0 -853
  324. package/crates/team-agent/src/messaging/tests/wave2.rs +0 -381
  325. package/crates/team-agent/src/messaging/trust.rs +0 -215
  326. package/crates/team-agent/src/messaging/types.rs +0 -472
  327. package/crates/team-agent/src/messaging/wait.rs +0 -366
  328. package/crates/team-agent/src/messaging/watchers.rs +0 -916
  329. package/crates/team-agent/src/model/enums.rs +0 -356
  330. package/crates/team-agent/src/model/errors.rs +0 -18
  331. package/crates/team-agent/src/model/ids.rs +0 -165
  332. package/crates/team-agent/src/model/mod.rs +0 -29
  333. package/crates/team-agent/src/model/name_similarity.rs +0 -252
  334. package/crates/team-agent/src/model/pane_authority_refusal.rs +0 -359
  335. package/crates/team-agent/src/model/paths.rs +0 -257
  336. package/crates/team-agent/src/model/routing.rs +0 -387
  337. package/crates/team-agent/src/model/spec.rs +0 -1184
  338. package/crates/team-agent/src/model/task_graph.rs +0 -321
  339. package/crates/team-agent/src/model/testdata/fuzz.golden.yaml +0 -43
  340. package/crates/team-agent/src/model/testdata/fuzz.yaml +0 -43
  341. package/crates/team-agent/src/model/testdata/spec_invalid_a.yaml +0 -209
  342. package/crates/team-agent/src/model/testdata/team.spec.golden.yaml +0 -208
  343. package/crates/team-agent/src/model/testdata/team.spec.yaml +0 -208
  344. package/crates/team-agent/src/model/yaml/tests.rs +0 -188
  345. package/crates/team-agent/src/model/yaml.rs +0 -802
  346. package/crates/team-agent/src/os_probe.rs +0 -280
  347. package/crates/team-agent/src/packaging/install.rs +0 -219
  348. package/crates/team-agent/src/packaging/migrate.rs +0 -38
  349. package/crates/team-agent/src/packaging/mod.rs +0 -91
  350. package/crates/team-agent/src/packaging/repair.rs +0 -33
  351. package/crates/team-agent/src/packaging/tests.rs +0 -822
  352. package/crates/team-agent/src/packaging/types.rs +0 -417
  353. package/crates/team-agent/src/platform/argv.rs +0 -331
  354. package/crates/team-agent/src/platform/errors.rs +0 -104
  355. package/crates/team-agent/src/platform/file_lock.rs +0 -279
  356. package/crates/team-agent/src/platform/mod.rs +0 -5
  357. package/crates/team-agent/src/platform/process.rs +0 -545
  358. package/crates/team-agent/src/provider/adapter.rs +0 -1150
  359. package/crates/team-agent/src/provider/adapters/claude.rs +0 -78
  360. package/crates/team-agent/src/provider/adapters/claude_fork.rs +0 -131
  361. package/crates/team-agent/src/provider/adapters/copilot.rs +0 -75
  362. package/crates/team-agent/src/provider/adapters/copilot_fork.rs +0 -307
  363. package/crates/team-agent/src/provider/adapters/fake.rs +0 -18
  364. package/crates/team-agent/src/provider/adapters/mod.rs +0 -30
  365. package/crates/team-agent/src/provider/adapters/pi.rs +0 -106
  366. package/crates/team-agent/src/provider/approvals/mod.rs +0 -3
  367. package/crates/team-agent/src/provider/approvals/parsing.rs +0 -586
  368. package/crates/team-agent/src/provider/approvals/runtime_prompts.rs +0 -205
  369. package/crates/team-agent/src/provider/argv_route.rs +0 -405
  370. package/crates/team-agent/src/provider/bypass_flags.rs +0 -116
  371. package/crates/team-agent/src/provider/classify.rs +0 -708
  372. package/crates/team-agent/src/provider/command_helpers.rs +0 -45
  373. package/crates/team-agent/src/provider/faults.rs +0 -165
  374. package/crates/team-agent/src/provider/helpers.rs +0 -52
  375. package/crates/team-agent/src/provider/mod.rs +0 -78
  376. package/crates/team-agent/src/provider/model_catalog.rs +0 -910
  377. package/crates/team-agent/src/provider/session/capture.rs +0 -2731
  378. package/crates/team-agent/src/provider/session/context_fork/claude.rs +0 -123
  379. package/crates/team-agent/src/provider/session/context_fork/outcome.rs +0 -172
  380. package/crates/team-agent/src/provider/session/context_fork.rs +0 -396
  381. package/crates/team-agent/src/provider/session/mod.rs +0 -36
  382. package/crates/team-agent/src/provider/session/resume.rs +0 -269
  383. package/crates/team-agent/src/provider/session_scan/claude.rs +0 -523
  384. package/crates/team-agent/src/provider/session_scan/common/tests.rs +0 -316
  385. package/crates/team-agent/src/provider/session_scan/common.rs +0 -654
  386. package/crates/team-agent/src/provider/session_scan/copilot.rs +0 -280
  387. package/crates/team-agent/src/provider/session_scan/pi.rs +0 -185
  388. package/crates/team-agent/src/provider/session_scan.rs +0 -71
  389. package/crates/team-agent/src/provider/startup_prompt.rs +0 -757
  390. package/crates/team-agent/src/provider/submit_now.rs +0 -71
  391. package/crates/team-agent/src/provider/testdata/copilot-ready-marker.txt +0 -5
  392. package/crates/team-agent/src/provider/testdata/copilot-trust-prompt.txt +0 -19
  393. package/crates/team-agent/src/provider/testdata/grok-models-authenticated.stdout +0 -9
  394. package/crates/team-agent/src/provider/testdata/grok-models-unauthenticated.stdout +0 -7
  395. package/crates/team-agent/src/provider/tests/adapter.rs +0 -441
  396. package/crates/team-agent/src/provider/tests/classify.rs +0 -148
  397. package/crates/team-agent/src/provider/tests/command_helpers.rs +0 -35
  398. package/crates/team-agent/src/provider/tests/copilot_fork.rs +0 -191
  399. package/crates/team-agent/src/provider/tests/faults.rs +0 -200
  400. package/crates/team-agent/src/provider/tests/idle.rs +0 -177
  401. package/crates/team-agent/src/provider/tests/wire.rs +0 -213
  402. package/crates/team-agent/src/provider/tests.rs +0 -32
  403. package/crates/team-agent/src/provider/types.rs +0 -529
  404. package/crates/team-agent/src/provider/wire.rs +0 -394
  405. package/crates/team-agent/src/redaction.rs +0 -259
  406. package/crates/team-agent/src/state/abnormal_watch/tests/mod.rs +0 -248
  407. package/crates/team-agent/src/state/abnormal_watch.rs +0 -144
  408. package/crates/team-agent/src/state/identity.rs +0 -914
  409. package/crates/team-agent/src/state/identity_keys.rs +0 -132
  410. package/crates/team-agent/src/state/mod.rs +0 -74
  411. package/crates/team-agent/src/state/owner_gate.rs +0 -482
  412. package/crates/team-agent/src/state/ownership.rs +0 -563
  413. package/crates/team-agent/src/state/paths.rs +0 -395
  414. package/crates/team-agent/src/state/persist.rs +0 -3811
  415. package/crates/team-agent/src/state/projection.rs +0 -1508
  416. package/crates/team-agent/src/state/repository/intent.rs +0 -14
  417. package/crates/team-agent/src/state/repository/tests.rs +0 -184
  418. package/crates/team-agent/src/state/repository.rs +0 -866
  419. package/crates/team-agent/src/state/selector.rs +0 -160
  420. package/crates/team-agent/src/state/testdata/state-rich.canonical.json +0 -133
  421. package/crates/team-agent/src/test_support/models_timeout_fixture.rs +0 -144
  422. package/crates/team-agent/src/tmux_backend/enter_rescue.rs +0 -325
  423. package/crates/team-agent/src/tmux_backend/enter_rescue_tests.rs +0 -432
  424. package/crates/team-agent/src/tmux_backend/native_input.rs +0 -513
  425. package/crates/team-agent/src/tmux_backend/native_input_tests.rs +0 -742
  426. package/crates/team-agent/src/tmux_backend/tests.rs +0 -4103
  427. package/crates/team-agent/src/tmux_backend.rs +0 -4492
  428. package/crates/team-agent/src/topology.rs +0 -537
  429. package/crates/team-agent/src/transport/enter_rescue.rs +0 -166
  430. package/crates/team-agent/src/transport/native_input.rs +0 -491
  431. package/crates/team-agent/src/transport/test_support.rs +0 -743
  432. package/crates/team-agent/src/transport/tests/behavior.rs +0 -326
  433. package/crates/team-agent/src/transport/tests/mod.rs +0 -194
  434. package/crates/team-agent/src/transport/tests/wire.rs +0 -574
  435. package/crates/team-agent/src/transport.rs +0 -1655
  436. package/crates/team-agent/src/transport_factory.rs +0 -819
@@ -1,2309 +0,0 @@
1
- //! ---
2
- //! purpose: tick 的异常退出检测步骤——按 agent 读有界 rollout 尾,判「最新一条显式 provider 错误是否属于当前 worker 世代」,据此决定通知还是只留审计
3
- //! contract:
4
- //! provides:
5
- //! - name: detect_abnormal_exits
6
- //! what: 遍历受监视 agent,产出 worker.abnormal_exit 通知或 dead_only 审计事件,并把本轮观测写回 state 的 abnormal_exit_watch
7
- //! - name: explicit_process_liveness
8
- //! what: 从 agent JSON 的显式存活字段解析进程存活态,不做任何探测
9
- //! - name: read_tail_text
10
- //! what: 有界读文件尾部文本,超长时从尾部 max_bytes 处起读
11
- //! - name: metadata_mtime_ns
12
- //! what: 把文件 metadata 的 mtime 归一成纳秒整数,供跨 tick 变化比较
13
- //! depends:
14
- //! - super::super::health
15
- //! - super::super::tick
16
- //! - super::super::types
17
- //! - crate::event_log
18
- //! - crate::messaging
19
- //! - crate::provider
20
- //! - crate::transport
21
- //! - crate::state::projection
22
- //! boundary:
23
- //! - 不重启、不清理、不做物理注入:写事件与 state 观测记录之外,异常通知会经 crate::messaging 的 leader funnel 入库为 accepted 消息(物理注入由后续 deliver_pending 完成)
24
- //! - 进程已死但没有显式错误时不通知,只落 dead_only 审计
25
- //! - 通知事件里的 queued 只表示消息入库,不表示已送达
26
- //! - rollout 尾读是有界的(首行可能被截断,消费方只解析最后一条完整 JSONL 记录)
27
- //! maturity: wired
28
- //! ---
29
- //!
30
- //! unit-11 (Stage 4) — coordinator tick `abnormal` step group.
31
- //!
32
- //! Abnormal-exit detection + classification step extracted from
33
- //! `coordinator/tick.rs` without behavior changes.
34
-
35
- use std::path::{Path, PathBuf};
36
-
37
- use serde_json::Value;
38
-
39
- use crate::event_log::EventLog;
40
- use crate::provider::wire::{parse_provider, provider_wire};
41
- use crate::provider::ProcessLiveness;
42
-
43
- use super::super::health::pid_is_running;
44
- use super::super::tick::TickError;
45
- use super::super::types::Pid;
46
-
47
- ///
48
- /// #236 `worker.abnormal_exit` watcher.
49
- ///
50
- /// Notify only when the bounded transcript/rollout tail contains a latest explicit
51
- /// provider error that is fresh for the current worker cohort. A dead process
52
- /// without an explicit error remains a suppressed `dead_only` audit event; process
53
- /// liveness is otherwise diagnostic data for this path.
54
- /// ---
55
- /// purpose: 跑一轮异常退出检测,把结论落成事件与 state 里的 abnormal_exit_watch 记录
56
- /// params:
57
- /// workspace: workspace 根,用于解析 rollout 相对路径
58
- /// transport: pane 存活探测通道;只读探测,不注入
59
- /// state: 可变运行时状态;本轮观测(size/mtime/世代/去重键)写回 coordinator.abnormal_exit_watch
60
- /// event_log: 事件出口
61
- /// targets: 本 tick 已采到的 pane 快照,避免每 agent 重新列一遍
62
- /// returns: 检测跑完即 Ok;「有没有异常」不由返回值表达,只体现在事件与 state 里
63
- /// errors: 事件写入等不可恢复失败时返回 TickError;单个 agent 的 rollout metadata 读不到不算错误,记为 unverifiable 继续
64
- /// ---
65
- pub(crate) fn detect_abnormal_exits(
66
- workspace: &Path,
67
- transport: &dyn crate::transport::Transport,
68
- state: &mut Value,
69
- event_log: &EventLog,
70
- targets: &[crate::transport::PaneInfo],
71
- ) -> Result<(), TickError> {
72
- let snapshot = state.clone();
73
- let team = crate::state::projection::team_state_key(&snapshot);
74
- let session_name = snapshot.get("session_name").and_then(Value::as_str);
75
- for agent in abnormal_watch_agents(&snapshot) {
76
- // Reject untrusted history without skipping a fresh provider/path observation.
77
- crate::state::abnormal_watch::begin_observation(state, &agent.agent_id);
78
- // Pane/process liveness is independent of transcript content. A
79
- // frozen rollout is common after pane death, so probe before the
80
- // metadata dedupe gate; only the expensive tail scan stays deduped.
81
- let liveness = agent_process_liveness(&agent, session_name, targets, transport);
82
- let rollout_path = resolve_agent_rollout_path(workspace, &agent.rollout_path);
83
- let metadata = match std::fs::metadata(&rollout_path) {
84
- Ok(metadata) => metadata,
85
- Err(error) => {
86
- upsert_abnormal_watch(
87
- state,
88
- &agent.agent_id,
89
- abnormal_watch_payload(
90
- &agent,
91
- None,
92
- None,
93
- process_check(
94
- ProcessLiveness::Unverifiable,
95
- "rollout_metadata_unavailable".to_string(),
96
- ),
97
- None,
98
- ErrorRecency::None,
99
- None,
100
- Some(error.to_string()),
101
- ),
102
- );
103
- continue;
104
- }
105
- };
106
- let size = metadata.len();
107
- let mtime_ns = metadata_mtime_ns(&metadata);
108
- // P1 (C-P1-2/3): (size, mtime_ns) pair gate — an unchanged transcript is not
109
- // read at all (live sample: 332MB whole-file read per agent per 2s tick).
110
- // ANY field change (including a size shrink / truncate) falls through to the
111
- // re-read below.
112
- if let (Some(mtime), Some(stored)) = (
113
- mtime_ns,
114
- abnormal_watch_stored_metadata(state, &agent.agent_id),
115
- ) {
116
- if stored == (size, mtime) {
117
- refresh_abnormal_watch_liveness(state, &agent.agent_id, &liveness);
118
- continue;
119
- }
120
- }
121
- // P1 (C-P1-1): bounded tail read — the abnormal decision only consumes
122
- // this tail window. The provider scan walks backward inside it; window
123
- // matches Python `_TAIL_BYTES` (131072, idle_takeover_wiring.py:13),
124
- // never less.
125
- let text = match read_tail_text(&rollout_path, ABNORMAL_TAIL_BYTES) {
126
- Ok(text) => text,
127
- Err(error) => {
128
- upsert_abnormal_watch(
129
- state,
130
- &agent.agent_id,
131
- abnormal_watch_payload(
132
- &agent,
133
- Some(size),
134
- mtime_ns,
135
- process_check(
136
- ProcessLiveness::Unverifiable,
137
- "rollout_read_failed".to_string(),
138
- ),
139
- None,
140
- ErrorRecency::None,
141
- None,
142
- Some(error.to_string()),
143
- ),
144
- );
145
- continue;
146
- }
147
- };
148
- let fact = crate::provider::latest_explicit_error_fact(agent.provider, &text);
149
- let error_observation_key = fact
150
- .as_ref()
151
- .map(|fact| abnormal_error_observation_key(&agent, fact));
152
- let error_observation_cohort = fact.as_ref().map(|_| abnormal_error_cohort_key(&agent));
153
- let error_recency = abnormal_error_recency(
154
- state,
155
- &agent,
156
- error_observation_key.as_deref(),
157
- error_observation_cohort.as_deref(),
158
- );
159
- let decision = abnormal_exit_decision(liveness.state, fact.as_ref(), error_recency);
160
- let check_key = abnormal_check_key(
161
- &agent,
162
- &liveness,
163
- fact.as_ref(),
164
- error_recency,
165
- error_observation_key.as_deref(),
166
- );
167
- upsert_abnormal_watch(
168
- state,
169
- &agent.agent_id,
170
- abnormal_watch_payload(
171
- &agent,
172
- Some(size),
173
- mtime_ns,
174
- liveness.clone(),
175
- fact.as_ref().map(|f| f.signature.as_str()),
176
- error_recency,
177
- error_observation_key.as_deref(),
178
- None,
179
- ),
180
- );
181
- if let (Some(observation_key), Some(cohort_key)) = (
182
- error_observation_key.as_deref(),
183
- error_observation_cohort.as_deref(),
184
- ) {
185
- mark_abnormal_error_observed(state, &agent.agent_id, observation_key, cohort_key);
186
- }
187
- if abnormal_last_check_key(state, &agent.agent_id).as_deref() != Some(check_key.as_str()) {
188
- write_abnormal_check(
189
- event_log,
190
- &team,
191
- &agent,
192
- &liveness,
193
- fact.as_ref(),
194
- decision,
195
- error_recency,
196
- size,
197
- mtime_ns,
198
- )?;
199
- mark_abnormal_checked(state, &agent.agent_id, &check_key);
200
- }
201
- let fact = match (decision, fact) {
202
- (AbnormalExitDecision::Notify, Some(fact)) => fact,
203
- (AbnormalExitDecision::Suppress(reason), _) => {
204
- let suppress_key = abnormal_suppression_key(&agent, &liveness, reason);
205
- if abnormal_last_suppressed_key(state, &agent.agent_id).as_deref()
206
- != Some(suppress_key.as_str())
207
- {
208
- write_abnormal_suppressed(event_log, &team, &agent, &liveness, reason)?;
209
- mark_abnormal_suppressed(state, &agent.agent_id, &suppress_key);
210
- }
211
- continue;
212
- }
213
- (AbnormalExitDecision::NoSignal, _) => continue,
214
- (AbnormalExitDecision::Notify, None) => continue,
215
- };
216
- let dedupe_key = abnormal_dedupe_key(&agent, &fact);
217
- if abnormal_last_notified_key(state, &agent.agent_id).as_deref()
218
- == Some(dedupe_key.as_str())
219
- {
220
- continue;
221
- }
222
- let manual_recovery_hint =
223
- recovery_hint(agent.agent_id.as_str(), agent.provider, workspace, team.as_str());
224
- let content = format_abnormal_exit_message(
225
- &team,
226
- &agent,
227
- &fact,
228
- &liveness,
229
- size,
230
- &manual_recovery_hint,
231
- );
232
- let outcome = crate::messaging::send_to_leader_receiver(
233
- workspace,
234
- state,
235
- "leader",
236
- &content,
237
- None,
238
- &agent.agent_id,
239
- false,
240
- Some(&dedupe_key),
241
- event_log,
242
- )?;
243
- let notification_status = if outcome.ok {
244
- "queued"
245
- } else if matches!(outcome.status, crate::messaging::DeliveryStatus::Blocked) {
246
- "rebind_required"
247
- } else {
248
- "refused"
249
- };
250
- let provider_process_dead = provider_process_dead_fact(&liveness);
251
- event_log.write(
252
- "worker.abnormal_exit",
253
- serde_json::json!({
254
- "team_id": team.as_str(),
255
- "agent_id": agent.agent_id.as_str(),
256
- "provider": provider_wire(agent.provider),
257
- "path": agent.rollout_path_display.as_str(),
258
- "dead_process": liveness.state == ProcessLiveness::Dead,
259
- "process_dead": liveness.state == ProcessLiveness::Dead,
260
- "provider_process_dead": provider_process_dead,
261
- "latest_error": true,
262
- "latest_explicit_error": true,
263
- "error_recency": error_recency.as_str(),
264
- "fresh_error": error_recency.is_fresh(),
265
- "dead_process_and_latest_error": liveness.state == ProcessLiveness::Dead,
266
- "dead_process_and_latest_explicit_error": liveness.state == ProcessLiveness::Dead,
267
- "process_dead_and_latest_explicit_error": liveness.state == ProcessLiveness::Dead,
268
- "provider_process_dead_and_latest_explicit_error": provider_process_dead,
269
- "signature": fact.signature.as_str(),
270
- "turn_id": fact.turn_id.as_ref().map(|id| id.as_str()),
271
- "apiErrorStatus": fact.api_error_status,
272
- "error": fact.error.as_deref(),
273
- "requestId": fact.request_id.as_deref(),
274
- "assistant_uuid": fact.assistant_uuid.as_deref(),
275
- "size": size,
276
- "mtime_ns": mtime_ns,
277
- "process_liveness": process_liveness_wire(liveness.state),
278
- "pid_status": liveness.detail.as_str(),
279
- "notification_message_id": outcome.message_id,
280
- "notification_status": notification_status,
281
- "notification_channel": outcome.channel,
282
- }),
283
- )?;
284
- mark_abnormal_notified(state, &agent.agent_id, &dedupe_key);
285
- }
286
- Ok(())
287
- }
288
-
289
- fn provider_process_dead_fact(liveness: &ProcessCheck) -> bool {
290
- liveness.state == ProcessLiveness::Dead
291
- && !liveness.detail.starts_with("pane_dead:")
292
- && !liveness.detail.starts_with("window_missing:")
293
- }
294
-
295
- fn worker_provider_exited_fact(liveness: &ProcessCheck) -> bool {
296
- provider_process_dead_fact(liveness) && liveness.detail.starts_with("worker_provider_exited:")
297
- }
298
-
299
- fn refresh_abnormal_watch_liveness(state: &mut Value, agent_id: &str, liveness: &ProcessCheck) {
300
- let Some(watch) = coordinator_child_object(state, "abnormal_exit_watch")
301
- .and_then(|watch| watch.get_mut(agent_id))
302
- .and_then(Value::as_object_mut)
303
- else {
304
- return;
305
- };
306
- let dead_process = liveness.state == ProcessLiveness::Dead;
307
- let provider_process_dead = provider_process_dead_fact(liveness);
308
- let latest_explicit_error = watch
309
- .get("latest_explicit_error")
310
- .and_then(Value::as_bool)
311
- .unwrap_or(false);
312
- // Persist fact transitions, not the tick's observation time; otherwise an
313
- // unchanged transcript makes every steady tick rewrite state.json.
314
- let patch = serde_json::json!({
315
- "last_liveness": process_liveness_wire(liveness.state),
316
- "last_liveness_detail": liveness.detail.as_str(),
317
- "dead_process": dead_process,
318
- "process_dead": dead_process,
319
- "provider_process_dead": provider_process_dead,
320
- "worker_provider_exited": worker_provider_exited_fact(liveness),
321
- "provider_process_dead_and_latest_explicit_error": provider_process_dead && latest_explicit_error,
322
- });
323
- if let Some(patch) = patch.as_object() {
324
- if patch
325
- .iter()
326
- .any(|(key, value)| watch.get(key) != Some(value))
327
- {
328
- watch.extend(patch.clone());
329
- }
330
- }
331
- }
332
-
333
- #[derive(Debug, Clone)]
334
- struct AbnormalWatchAgent {
335
- agent_id: String,
336
- provider: crate::model::enums::Provider,
337
- rollout_path: PathBuf,
338
- rollout_path_display: String,
339
- spawn_epoch: Option<u64>,
340
- spawned_at: Option<String>,
341
- status: Option<String>,
342
- process_liveness: Option<ProcessLiveness>,
343
- window: Option<String>,
344
- pane_id: Option<String>,
345
- pid: Option<Pid>,
346
- current_command: Option<String>,
347
- }
348
-
349
- #[derive(Debug, Clone, PartialEq, Eq)]
350
- struct ProcessCheck {
351
- state: ProcessLiveness,
352
- detail: String,
353
- }
354
-
355
- #[derive(Debug, Clone, Copy, PartialEq, Eq)]
356
- enum AbnormalExitDecision {
357
- Notify,
358
- Suppress(&'static str),
359
- NoSignal,
360
- }
361
-
362
- #[derive(Debug, Clone, Copy, PartialEq, Eq)]
363
- enum ErrorRecency {
364
- None,
365
- Stale,
366
- Fresh,
367
- }
368
-
369
- impl ErrorRecency {
370
- fn as_str(self) -> &'static str {
371
- match self {
372
- Self::None => "none",
373
- Self::Stale => "stale",
374
- Self::Fresh => "fresh",
375
- }
376
- }
377
-
378
- fn is_fresh(self) -> bool {
379
- matches!(self, Self::Fresh)
380
- }
381
- }
382
-
383
- #[derive(Debug, Clone, Copy, PartialEq, Eq)]
384
- struct AbnormalExitGate {
385
- provider_process_dead: bool,
386
- latest_explicit_error: bool,
387
- error_recency: ErrorRecency,
388
- }
389
-
390
- impl AbnormalExitGate {
391
- fn new(
392
- process_liveness: ProcessLiveness,
393
- latest_explicit_error: bool,
394
- error_recency: ErrorRecency,
395
- ) -> Self {
396
- Self {
397
- provider_process_dead: process_liveness == ProcessLiveness::Dead,
398
- latest_explicit_error,
399
- error_recency,
400
- }
401
- }
402
-
403
- fn should_notify_worker_abnormal_exit(self) -> bool {
404
- should_notify_worker_abnormal_exit(self.latest_explicit_error, self.error_recency)
405
- }
406
-
407
- fn suppressed_reason(self) -> Option<&'static str> {
408
- match (self.provider_process_dead, self.latest_explicit_error) {
409
- (true, false) => Some("dead_only"),
410
- _ => None,
411
- }
412
- }
413
- }
414
-
415
- fn abnormal_exit_decision(
416
- process_liveness: ProcessLiveness,
417
- latest_explicit_error: Option<&crate::provider::FaultFact>,
418
- error_recency: ErrorRecency,
419
- ) -> AbnormalExitDecision {
420
- let gate = AbnormalExitGate::new(
421
- process_liveness,
422
- latest_explicit_error.is_some(),
423
- error_recency,
424
- );
425
- if gate.should_notify_worker_abnormal_exit() {
426
- return AbnormalExitDecision::Notify;
427
- }
428
- match gate.suppressed_reason() {
429
- Some(reason) => AbnormalExitDecision::Suppress(reason),
430
- None => AbnormalExitDecision::NoSignal,
431
- }
432
- }
433
-
434
- fn should_notify_worker_abnormal_exit(
435
- latest_explicit_error: bool,
436
- error_recency: ErrorRecency,
437
- ) -> bool {
438
- latest_explicit_error && error_recency.is_fresh()
439
- }
440
-
441
- fn resolve_agent_rollout_path(workspace: &Path, path: &Path) -> PathBuf {
442
- if path.is_absolute() {
443
- path.to_path_buf()
444
- } else {
445
- workspace.join(path)
446
- }
447
- }
448
-
449
- fn abnormal_watch_agents(state: &Value) -> Vec<AbnormalWatchAgent> {
450
- let Some(agents) = state.get("agents").and_then(Value::as_object) else {
451
- return Vec::new();
452
- };
453
- agents
454
- .iter()
455
- .filter_map(|(agent_id, agent)| {
456
- if matches!(agent.get("status").and_then(Value::as_str), Some("paused")) {
457
- return None;
458
- }
459
- let provider = agent
460
- .get("provider")
461
- .and_then(Value::as_str)
462
- .and_then(parse_provider)?;
463
- let rollout_path_display = ["rollout_path", "transcript_path", "session_log_path"]
464
- .into_iter()
465
- .find_map(|key| agent.get(key).and_then(Value::as_str))
466
- .filter(|path| !path.is_empty())?
467
- .to_string();
468
- Some(AbnormalWatchAgent {
469
- agent_id: agent_id.clone(),
470
- provider,
471
- rollout_path: PathBuf::from(&rollout_path_display),
472
- rollout_path_display,
473
- spawn_epoch: agent.get("spawn_epoch").and_then(Value::as_u64),
474
- spawned_at: agent
475
- .get("spawned_at")
476
- .and_then(Value::as_str)
477
- .map(str::to_string),
478
- status: agent
479
- .get("status")
480
- .and_then(Value::as_str)
481
- .map(str::to_string),
482
- process_liveness: explicit_process_liveness(agent),
483
- window: agent
484
- .get("window")
485
- .and_then(Value::as_str)
486
- .map(str::to_string),
487
- pane_id: agent
488
- .get("pane_id")
489
- .and_then(Value::as_str)
490
- .map(str::to_string),
491
- pid: agent_pid(agent),
492
- current_command: agent
493
- .get("pane_current_command")
494
- .or_else(|| agent.get("current_command"))
495
- .and_then(Value::as_str)
496
- .map(str::to_string),
497
- })
498
- })
499
- .collect()
500
- }
501
-
502
- fn agent_pid(agent: &Value) -> Option<Pid> {
503
- // 0.5.41 Slice 4 (fault-invisibility-locate.md §5 point 3 / §6.4,
504
- // 0.5.39 CR D-m): after the 0.5.39 worker shell wrapper landed,
505
- // `pane_pid` is the long-lived shell — NOT the provider. Treating
506
- // it as `provider pid` and probing it via `kill(0)` reports "alive"
507
- // for a pane whose provider CLI already exited, which is exactly
508
- // the false-BUSY that suppressed `worker_provider_exited`. Only
509
- // pids we know are the provider itself (`provider_pid` /
510
- // `process_id` / `pid` written by explicit spawn accounting, or
511
- // `child_pid` from the transport's SpawnResult) count here.
512
- ["provider_pid", "process_id", "pid", "child_pid"]
513
- .into_iter()
514
- .find_map(|key| json_u32(agent.get(key)).map(Pid::new))
515
- }
516
-
517
- /// ---
518
- /// purpose: 只从 agent JSON 里已写明的存活字段读出进程存活态,不做任何主动探测
519
- /// params:
520
- /// agent: agent 状态对象;先下钻 provider_process / process 子对象,再看本层的 *_liveness 串字段
521
- /// returns: 命中显式字段时给出 Alive/Dead/Unverifiable;没有任何显式字段时为 None,交调用方走探测路径
522
- /// ---
523
- pub(crate) fn explicit_process_liveness(agent: &Value) -> Option<ProcessLiveness> {
524
- if let Some(process) = agent
525
- .get("provider_process")
526
- .or_else(|| agent.get("process"))
527
- {
528
- if let Some(liveness) = explicit_process_liveness(process) {
529
- return Some(liveness);
530
- }
531
- }
532
- for key in [
533
- "provider_process_liveness",
534
- "process_liveness",
535
- "pane_liveness",
536
- ] {
537
- match agent.get(key).and_then(Value::as_str) {
538
- Some("dead") => return Some(ProcessLiveness::Dead),
539
- Some("alive" | "live") => return Some(ProcessLiveness::Alive),
540
- Some("unverifiable" | "unknown") => return Some(ProcessLiveness::Unverifiable),
541
- _ => {}
542
- }
543
- }
544
- for key in [
545
- "provider_process_alive",
546
- "process_alive",
547
- "provider_alive",
548
- "alive",
549
- ] {
550
- if let Some(alive) = agent.get(key).and_then(Value::as_bool) {
551
- return Some(if alive {
552
- ProcessLiveness::Alive
553
- } else {
554
- ProcessLiveness::Dead
555
- });
556
- }
557
- }
558
- for key in [
559
- "provider_process_dead",
560
- "process_dead",
561
- "provider_dead",
562
- "dead",
563
- ] {
564
- if let Some(dead) = agent.get(key).and_then(Value::as_bool) {
565
- return Some(if dead {
566
- ProcessLiveness::Dead
567
- } else {
568
- ProcessLiveness::Alive
569
- });
570
- }
571
- }
572
- for key in ["status", "state", "liveness"] {
573
- match agent.get(key).and_then(Value::as_str) {
574
- Some("dead" | "exited" | "terminated" | "crashed" | "missing") => {
575
- return Some(ProcessLiveness::Dead);
576
- }
577
- Some("alive" | "live" | "running") => return Some(ProcessLiveness::Alive),
578
- Some("unverifiable" | "unknown") => return Some(ProcessLiveness::Unverifiable),
579
- _ => {}
580
- }
581
- }
582
- None
583
- }
584
-
585
- fn json_u32(value: Option<&Value>) -> Option<u32> {
586
- value
587
- .and_then(|v| {
588
- v.as_u64()
589
- .or_else(|| v.as_i64().and_then(|n| u64::try_from(n).ok()))
590
- })
591
- .and_then(|n| u32::try_from(n).ok())
592
- }
593
-
594
- fn agent_process_liveness(
595
- agent: &AbnormalWatchAgent,
596
- session_name: Option<&str>,
597
- targets: &[crate::transport::PaneInfo],
598
- transport: &dyn crate::transport::Transport,
599
- ) -> ProcessCheck {
600
- // 0.5.41 Slice 4 (fault-invisibility-locate.md §5 point 3 / §6.4,
601
- // 0.5.39 CR D-m): after the worker wrapper landed, `pane_pid` may
602
- // be the long-lived shell that survived provider exit. Reordered
603
- // to prefer POSITIVE provider evidence in this order:
604
- // 1. Explicit provider pid (agent.pid) — set by the transport's
605
- // SpawnResult.child_pid or explicit provider_pid state field.
606
- // 2. Explicit liveness field.
607
- // 3. Explicit dead-status field.
608
- // 4. Explicit provider current-command (state.pane_current_command).
609
- // 5. Live-pane current-command matching provider = alive.
610
- // 6. Live-pane current-command shell + capture-tail worker exit
611
- // marker = DEAD (positive proof provider exited).
612
- // 7. Live pane, no positive evidence = Unverifiable (NOT alive,
613
- // NOT working). Legacy code returned pid_process_check(pane_pid)
614
- // here — that was the D-m false BUSY.
615
- // 8. Pane / window liveness falls back on Unverifiable.
616
- if let Some(pid) = agent.pid {
617
- return pid_process_check("pid", pid);
618
- }
619
- // 0.5.41 Slice 4 (fault-invisibility-locate.md §5 point 3 / §6.4):
620
- // BEFORE consulting the loose `explicit_process_liveness` path
621
- // (which happily maps state.status=`running` to Alive), probe the
622
- // pane for POSITIVE provider evidence via current-command + worker
623
- // exit marker. Under the 0.5.39 wrapper, `state.status=running` is
624
- // administrative spawn accounting, not runtime truth — a provider
625
- // may have exited into the inert shell tail while state still reads
626
- // running. The marker probe positively proves that case; when it
627
- // fires we return Dead here instead of the false Alive further
628
- // down.
629
- if let Some(target) = matching_agent_target(agent, session_name, targets) {
630
- if let Some(command) = target.current_command.as_deref() {
631
- return pane_command_process_check_with_marker(agent, transport, target, command);
632
- }
633
- }
634
- if let Some(command) = agent.current_command.as_deref() {
635
- return command_process_check_with_marker(agent, transport, command);
636
- }
637
- if agent.pane_id.is_some() {
638
- // Even without pane current_command / matching target, try the
639
- // marker probe directly — the wrapper's printf leaves the marker
640
- // in the pane's capture tail whether or not the transport
641
- // reports pane_current_command.
642
- if let Some(check) = worker_provider_exit_marker_check(agent, transport) {
643
- return check;
644
- }
645
- }
646
- if let Some(liveness) = agent.process_liveness {
647
- return process_check(
648
- liveness,
649
- format!("explicit:{}", process_liveness_wire(liveness)),
650
- );
651
- }
652
- if agent.status.as_deref().is_some_and(|status| {
653
- matches!(
654
- status,
655
- "stopped" | "missing" | "error" | "dead" | "exited" | "terminated" | "crashed"
656
- )
657
- }) {
658
- return process_check(
659
- ProcessLiveness::Dead,
660
- format!("status:{}", agent.status.as_deref().unwrap_or("unknown")),
661
- );
662
- }
663
- if let Some(target) = matching_agent_target(agent, session_name, targets) {
664
- // Reached only when the target has no current_command AND no
665
- // marker was found earlier. Legacy code returned
666
- // `pid_process_check("pane_pid", pane_pid)` here — that's the
667
- // wrapper shell under 0.5.39. Now Unverifiable (not-working).
668
- let _ = target;
669
- return process_check(
670
- ProcessLiveness::Unverifiable,
671
- "pane_present_provider_evidence_unknown".to_string(),
672
- );
673
- }
674
- if let Some(pane_id) = agent.pane_id.as_deref() {
675
- let pane = crate::transport::PaneId::new(pane_id);
676
- return match transport.liveness(&pane) {
677
- Ok(crate::transport::PaneLiveness::Dead) => {
678
- process_check(ProcessLiveness::Dead, format!("pane_dead:{pane_id}"))
679
- }
680
- Ok(crate::transport::PaneLiveness::Live) => process_check(
681
- ProcessLiveness::Unverifiable,
682
- format!("pane_live_provider_evidence_unknown:{pane_id}"),
683
- ),
684
- Ok(crate::transport::PaneLiveness::Unknown) => process_check(
685
- ProcessLiveness::Unverifiable,
686
- format!("pane_unknown:{pane_id}"),
687
- ),
688
- Err(error) => process_check(
689
- ProcessLiveness::Unverifiable,
690
- format!("pane_unverifiable:{pane_id}:{error}"),
691
- ),
692
- };
693
- }
694
- let (Some(session), Some(window)) = (session_name, agent.window.as_deref()) else {
695
- return process_check(
696
- ProcessLiveness::Unverifiable,
697
- "missing_session_or_window".to_string(),
698
- );
699
- };
700
- let session = crate::transport::SessionName::new(session);
701
- match transport.list_windows(&session) {
702
- Ok(windows) if windows.iter().any(|known| known.as_str() == window) => process_check(
703
- ProcessLiveness::Unverifiable,
704
- "window_present_provider_evidence_unknown".to_string(),
705
- ),
706
- Ok(_) => process_check(ProcessLiveness::Dead, format!("window_missing:{window}")),
707
- Err(error) => process_check(
708
- ProcessLiveness::Unverifiable,
709
- format!("window_unverifiable:{window}:{error}"),
710
- ),
711
- }
712
- }
713
-
714
- fn matching_agent_target<'a>(
715
- agent: &AbnormalWatchAgent,
716
- session_name: Option<&str>,
717
- targets: &'a [crate::transport::PaneInfo],
718
- ) -> Option<&'a crate::transport::PaneInfo> {
719
- if let Some(pane_id) = agent.pane_id.as_deref() {
720
- if let Some(target) = targets
721
- .iter()
722
- .find(|target| target.pane_id.as_str() == pane_id)
723
- {
724
- return Some(target);
725
- }
726
- }
727
- let (Some(session), Some(window)) = (session_name, agent.window.as_deref()) else {
728
- return None;
729
- };
730
- targets.iter().find(|target| {
731
- target.session.as_str() == session
732
- && target
733
- .window_name
734
- .as_ref()
735
- .is_some_and(|known| known.as_str() == window)
736
- })
737
- }
738
-
739
- fn pid_process_check(label: &str, pid: Pid) -> ProcessCheck {
740
- match pid_is_running(pid) {
741
- Ok(true) => process_check(ProcessLiveness::Alive, format!("{label}_running:{pid}")),
742
- Ok(false) => process_check(ProcessLiveness::Dead, format!("{label}_not_running:{pid}")),
743
- Err(error) => process_check(
744
- ProcessLiveness::Unverifiable,
745
- format!("{label}_unverifiable:{pid}:{error}"),
746
- ),
747
- }
748
- }
749
-
750
- fn command_process_check(provider: crate::model::enums::Provider, command: &str) -> ProcessCheck {
751
- if crate::leader::command_matches_provider(provider, command) {
752
- process_check(ProcessLiveness::Alive, format!("current_command:{command}"))
753
- } else {
754
- process_check(
755
- ProcessLiveness::Dead,
756
- format!("provider_not_foreground:{command}"),
757
- )
758
- }
759
- }
760
-
761
- fn pane_command_process_check(
762
- provider: crate::model::enums::Provider,
763
- pane: &crate::transport::PaneInfo,
764
- command: &str,
765
- ) -> ProcessCheck {
766
- if crate::leader::attribute_pane_provider(pane)
767
- .is_some_and(|candidate| crate::leader::provider_matches(candidate, provider))
768
- {
769
- process_check(ProcessLiveness::Alive, format!("current_command:{command}"))
770
- } else {
771
- process_check(
772
- ProcessLiveness::Dead,
773
- format!("provider_not_foreground:{command}"),
774
- )
775
- }
776
- }
777
-
778
- /// 0.5.41 Slice 4 (fault-invisibility-locate.md §5 point 3 / §6.4):
779
- /// after the 0.5.39 worker wrapper landed, a pane whose current command
780
- /// is a shell + capture-tail contains the worker exit marker is POSITIVE
781
- /// proof the provider exited. Falls back to plain current-command check
782
- /// when a pane_id is not available for capture (state-only path).
783
- fn command_process_check_with_marker(
784
- agent: &AbnormalWatchAgent,
785
- transport: &dyn crate::transport::Transport,
786
- command: &str,
787
- ) -> ProcessCheck {
788
- if crate::leader::command_matches_provider(agent.provider, command) {
789
- return process_check(ProcessLiveness::Alive, format!("current_command:{command}"));
790
- }
791
- if let Some(check) = worker_provider_exit_marker_check(agent, transport) {
792
- return check;
793
- }
794
- process_check(
795
- ProcessLiveness::Unverifiable,
796
- format!("provider_evidence_unverifiable:{command}"),
797
- )
798
- }
799
-
800
- fn pane_command_process_check_with_marker(
801
- agent: &AbnormalWatchAgent,
802
- transport: &dyn crate::transport::Transport,
803
- pane: &crate::transport::PaneInfo,
804
- command: &str,
805
- ) -> ProcessCheck {
806
- if crate::leader::attribute_pane_provider(pane)
807
- .is_some_and(|candidate| crate::leader::provider_matches(candidate, agent.provider))
808
- {
809
- return process_check(ProcessLiveness::Alive, format!("current_command:{command}"));
810
- }
811
- if let Some(check) = worker_provider_exit_marker_check(agent, transport) {
812
- return check;
813
- }
814
- process_check(
815
- ProcessLiveness::Unverifiable,
816
- format!("provider_evidence_unverifiable:{command}"),
817
- )
818
- }
819
-
820
- /// 0.5.41 Slice 4: probe the pane's capture-tail for the single-source
821
- /// worker exit marker (`tmux_backend::worker_provider_exit_marker`).
822
- /// Returns `Some(Dead)` when the marker is found (positive proof
823
- /// provider exited), or `None` when we can't run the capture (no
824
- /// pane_id in agent), so callers keep their default decision. Absence
825
- /// of marker in a valid capture is intentionally NOT `alive` (locate
826
- /// §8 risk: capture tail can scroll past the marker) — callers still
827
- /// return `Dead` for "provider not foreground" because current command
828
- /// already disproved provider liveness at that call site.
829
- fn worker_provider_exit_marker_check(
830
- agent: &AbnormalWatchAgent,
831
- transport: &dyn crate::transport::Transport,
832
- ) -> Option<ProcessCheck> {
833
- let pane_id_str = agent.pane_id.as_deref()?;
834
- let pane = crate::transport::PaneId::new(pane_id_str);
835
- let target = crate::transport::Target::Pane(pane);
836
- let provider_label = crate::provider::wire::command_name(agent.provider);
837
- let marker = crate::tmux_backend::worker_provider_exit_marker(provider_label);
838
- let cap = transport
839
- .capture(&target, crate::transport::CaptureRange::Tail(200))
840
- .ok()?;
841
- if cap.text.contains(&marker) {
842
- // The wrapper writes its exit code immediately after this marker. A
843
- // truncated capture can leave the marker without a complete token;
844
- // keep that case explicit rather than guessing success (or zero).
845
- // Scope: this detail only covers exits that execute the wrapper's
846
- // marker printf. SIGKILL, direct pane termination, and host power loss
847
- // produce no marker; that absence is silent, not `rc=unknown`.
848
- let rc = cap
849
- .text
850
- .split_once(&marker)
851
- .and_then(|(_, tail)| {
852
- let token = tail.split_whitespace().next()?;
853
- let end = tail.find(token)? + token.len();
854
- tail.as_bytes()
855
- .get(end)
856
- .is_some_and(u8::is_ascii_whitespace)
857
- .then_some(token)
858
- })
859
- .and_then(|token| token.parse::<i32>().ok())
860
- .map(|value| value.to_string())
861
- .unwrap_or_else(|| "unknown".to_string());
862
- Some(process_check(
863
- ProcessLiveness::Dead,
864
- format!("worker_provider_exited:{pane_id_str}:rc={rc}"),
865
- ))
866
- } else {
867
- None
868
- }
869
- }
870
-
871
- fn process_check(state: ProcessLiveness, detail: String) -> ProcessCheck {
872
- ProcessCheck { state, detail }
873
- }
874
-
875
- fn process_liveness_wire(state: ProcessLiveness) -> &'static str {
876
- match state {
877
- ProcessLiveness::Alive => "alive",
878
- ProcessLiveness::Dead => "dead",
879
- ProcessLiveness::Unverifiable => "unverifiable",
880
- }
881
- }
882
-
883
- /// ---
884
- /// purpose: 把文件修改时间归一成自 UNIX 纪元起的纳秒整数,供跨 tick 比较「rollout 有没有变过」
885
- /// params:
886
- /// metadata: 目标文件的 metadata
887
- /// returns: 纳秒时间戳;取不到 mtime 或早于纪元时为 None。乘加均为 saturating,不会回绕
888
- /// ---
889
- pub(crate) fn metadata_mtime_ns(metadata: &std::fs::Metadata) -> Option<u64> {
890
- let duration = metadata
891
- .modified()
892
- .ok()?
893
- .duration_since(std::time::UNIX_EPOCH)
894
- .ok()?;
895
- Some(
896
- duration
897
- .as_secs()
898
- .saturating_mul(1_000_000_000)
899
- .saturating_add(u64::from(duration.subsec_nanos())),
900
- )
901
- }
902
-
903
- fn abnormal_watch_payload(
904
- agent: &AbnormalWatchAgent,
905
- size: Option<u64>,
906
- mtime_ns: Option<u64>,
907
- liveness: ProcessCheck,
908
- signature: Option<&str>,
909
- error_recency: ErrorRecency,
910
- error_observation_key: Option<&str>,
911
- error: Option<String>,
912
- ) -> Value {
913
- let liveness_wire = process_liveness_wire(liveness.state);
914
- let dead_process = liveness.state == ProcessLiveness::Dead;
915
- let latest_explicit_error = signature.is_some();
916
- let gate = AbnormalExitGate::new(liveness.state, latest_explicit_error, error_recency);
917
- let notify = gate.should_notify_worker_abnormal_exit();
918
- let suppressed_reason = gate.suppressed_reason();
919
- // 0.5.41 Slice 4 (fault-invisibility-locate.md §6.4): typed
920
- // classification for status/render — `worker_provider_exited` is
921
- // true only when Dead was proven via the capture-tail worker exit
922
- // marker (detail prefix `worker_provider_exited:`). status_port's
923
- // RuntimeFreshness collector reads this field to downgrade the
924
- // corresponding agent row.
925
- let provider_process_dead = provider_process_dead_fact(&liveness);
926
- let worker_provider_exited = worker_provider_exited_fact(&liveness);
927
- serde_json::json!({
928
- "path": agent.rollout_path_display.as_str(),
929
- "provider": provider_wire(agent.provider),
930
- "mtime_ns": mtime_ns,
931
- "size": size,
932
- "last_offset": size,
933
- "last_signature": signature,
934
- "last_liveness": liveness_wire,
935
- "last_liveness_detail": liveness.detail,
936
- "dead_process": dead_process,
937
- "process_dead": dead_process,
938
- "provider_process_dead": provider_process_dead,
939
- "worker_provider_exited": worker_provider_exited,
940
- "latest_error": latest_explicit_error,
941
- "latest_explicit_error": latest_explicit_error,
942
- "error_recency": error_recency.as_str(),
943
- "fresh_error": error_recency.is_fresh(),
944
- "error_observation_key": error_observation_key,
945
- "dead_process_and_latest_error": dead_process && latest_explicit_error,
946
- "dead_process_and_latest_explicit_error": dead_process && latest_explicit_error,
947
- "process_dead_and_latest_explicit_error": dead_process && latest_explicit_error,
948
- "provider_process_dead_and_latest_explicit_error": provider_process_dead && latest_explicit_error,
949
- "suppressed_reason": suppressed_reason,
950
- "notification": notify,
951
- "last_error": error,
952
- })
953
- }
954
-
955
- fn upsert_abnormal_watch(state: &mut Value, agent_id: &str, mut payload: Value) {
956
- if let Some(identity) = crate::state::abnormal_watch::observation_identity(state, agent_id) {
957
- payload["watch_identity"] = identity;
958
- }
959
- let preserved = [
960
- "last_notified_key",
961
- "last_notified_at",
962
- "last_suppressed_key",
963
- "last_suppressed_at",
964
- "last_check_key",
965
- "last_check_at",
966
- "last_error_observation_key",
967
- "last_error_observation_cohort",
968
- "last_error_observed_at",
969
- ]
970
- .into_iter()
971
- .filter_map(|key| abnormal_watch_field(state, agent_id, key).map(|value| (key, value)))
972
- .collect::<Vec<_>>();
973
- if let Some(watch) = coordinator_child_object(state, "abnormal_exit_watch") {
974
- if let Some(payload_obj) = payload.as_object_mut() {
975
- for (key, value) in preserved {
976
- payload_obj.insert(key.to_string(), value);
977
- }
978
- }
979
- watch.insert(agent_id.to_string(), payload);
980
- }
981
- }
982
-
983
- fn coordinator_child_object<'a>(
984
- state: &'a mut Value,
985
- key: &str,
986
- ) -> Option<&'a mut serde_json::Map<String, Value>> {
987
- if !state.is_object() {
988
- *state = serde_json::json!({});
989
- }
990
- let state_obj = state.as_object_mut()?;
991
- let coordinator = state_obj
992
- .entry("coordinator".to_string())
993
- .or_insert_with(|| serde_json::json!({}));
994
- if !coordinator.is_object() {
995
- *coordinator = serde_json::json!({});
996
- }
997
- let coord_obj = coordinator.as_object_mut()?;
998
- let child = coord_obj
999
- .entry(key.to_string())
1000
- .or_insert_with(|| serde_json::json!({}));
1001
- if !child.is_object() {
1002
- *child = serde_json::json!({});
1003
- }
1004
- child.as_object_mut()
1005
- }
1006
-
1007
- fn abnormal_last_notified_key(state: &Value, agent_id: &str) -> Option<String> {
1008
- abnormal_watch_str(state, agent_id, "last_notified_key")
1009
- }
1010
-
1011
- fn abnormal_last_suppressed_key(state: &Value, agent_id: &str) -> Option<String> {
1012
- abnormal_watch_str(state, agent_id, "last_suppressed_key")
1013
- }
1014
-
1015
- fn abnormal_last_check_key(state: &Value, agent_id: &str) -> Option<String> {
1016
- abnormal_watch_str(state, agent_id, "last_check_key")
1017
- }
1018
-
1019
- fn abnormal_last_error_observation_key(state: &Value, agent_id: &str) -> Option<String> {
1020
- abnormal_watch_str(state, agent_id, "last_error_observation_key")
1021
- }
1022
-
1023
- fn abnormal_last_error_observation_cohort(state: &Value, agent_id: &str) -> Option<String> {
1024
- abnormal_watch_str(state, agent_id, "last_error_observation_cohort")
1025
- }
1026
-
1027
- /// P1: Python `_TAIL_BYTES` parity (idle_takeover_wiring.py:13) — RS must not read less.
1028
- const ABNORMAL_TAIL_BYTES: u64 = 131_072;
1029
-
1030
- ///
1031
- /// P1: bounded tail read; a partial first line is harmless (the consumer only parses
1032
- /// the latest complete JSONL record) and lossy UTF-8 keeps a mid-codepoint seek safe.
1033
- /// ---
1034
- /// purpose: 有界读取文件尾部文本,避免整份 rollout 进内存
1035
- /// params:
1036
- /// path: 目标文件
1037
- /// max_bytes: 最多回读的字节数;文件更短时整份读出
1038
- /// returns: 尾部文本。按字节 seek,首行可能被截断,且用有损 UTF-8 解码,切在码点中间也安全
1039
- /// errors: 打开、取长度或读取失败时返回底层 io::Error
1040
- /// ---
1041
- pub(crate) fn read_tail_text(path: &Path, max_bytes: u64) -> std::io::Result<String> {
1042
- use std::io::{Read, Seek, SeekFrom};
1043
- let mut file = std::fs::File::open(path)?;
1044
- let len = file.metadata()?.len();
1045
- if len > max_bytes {
1046
- file.seek(SeekFrom::Start(len - max_bytes))?;
1047
- }
1048
- let mut bytes = Vec::new();
1049
- file.read_to_end(&mut bytes)?;
1050
- Ok(String::from_utf8_lossy(&bytes).into_owned())
1051
- }
1052
-
1053
- /// P1: the previous tick's `(size, mtime_ns)` pair from the abnormal watch payload.
1054
- fn abnormal_watch_stored_metadata(state: &Value, agent_id: &str) -> Option<(u64, u64)> {
1055
- let watch = state
1056
- .get("coordinator")?
1057
- .get("abnormal_exit_watch")?
1058
- .get(agent_id)?;
1059
- Some((
1060
- watch.get("size")?.as_u64()?,
1061
- watch.get("mtime_ns")?.as_u64()?,
1062
- ))
1063
- }
1064
-
1065
- fn abnormal_watch_str(state: &Value, agent_id: &str, field: &str) -> Option<String> {
1066
- state
1067
- .get("coordinator")
1068
- .and_then(|v| v.get("abnormal_exit_watch"))
1069
- .and_then(|v| v.get(agent_id))
1070
- .and_then(|v| v.get(field))
1071
- .and_then(Value::as_str)
1072
- .map(str::to_string)
1073
- }
1074
-
1075
- fn abnormal_watch_field(state: &Value, agent_id: &str, field: &str) -> Option<Value> {
1076
- state
1077
- .get("coordinator")
1078
- .and_then(|v| v.get("abnormal_exit_watch"))
1079
- .and_then(|v| v.get(agent_id))
1080
- .and_then(|v| v.get(field))
1081
- .cloned()
1082
- }
1083
-
1084
- fn mark_abnormal_notified(state: &mut Value, agent_id: &str, key: &str) {
1085
- if let Some(watch) = coordinator_child_object(state, "abnormal_exit_watch") {
1086
- let entry = watch
1087
- .entry(agent_id.to_string())
1088
- .or_insert_with(|| serde_json::json!({}));
1089
- if !entry.is_object() {
1090
- *entry = serde_json::json!({});
1091
- }
1092
- if let Some(obj) = entry.as_object_mut() {
1093
- obj.insert("last_notified_key".to_string(), serde_json::json!(key));
1094
- obj.insert(
1095
- "last_notified_at".to_string(),
1096
- serde_json::json!(chrono::Utc::now().to_rfc3339()),
1097
- );
1098
- }
1099
- }
1100
- }
1101
-
1102
- fn mark_abnormal_suppressed(state: &mut Value, agent_id: &str, key: &str) {
1103
- if let Some(watch) = coordinator_child_object(state, "abnormal_exit_watch") {
1104
- let entry = watch
1105
- .entry(agent_id.to_string())
1106
- .or_insert_with(|| serde_json::json!({}));
1107
- if !entry.is_object() {
1108
- *entry = serde_json::json!({});
1109
- }
1110
- if let Some(obj) = entry.as_object_mut() {
1111
- obj.insert("last_suppressed_key".to_string(), serde_json::json!(key));
1112
- obj.insert(
1113
- "last_suppressed_at".to_string(),
1114
- serde_json::json!(chrono::Utc::now().to_rfc3339()),
1115
- );
1116
- }
1117
- }
1118
- }
1119
-
1120
- fn mark_abnormal_checked(state: &mut Value, agent_id: &str, key: &str) {
1121
- if let Some(watch) = coordinator_child_object(state, "abnormal_exit_watch") {
1122
- let entry = watch
1123
- .entry(agent_id.to_string())
1124
- .or_insert_with(|| serde_json::json!({}));
1125
- if !entry.is_object() {
1126
- *entry = serde_json::json!({});
1127
- }
1128
- if let Some(obj) = entry.as_object_mut() {
1129
- obj.insert("last_check_key".to_string(), serde_json::json!(key));
1130
- obj.insert(
1131
- "last_check_at".to_string(),
1132
- serde_json::json!(chrono::Utc::now().to_rfc3339()),
1133
- );
1134
- }
1135
- }
1136
- }
1137
-
1138
- fn mark_abnormal_error_observed(
1139
- state: &mut Value,
1140
- agent_id: &str,
1141
- observation_key: &str,
1142
- cohort_key: &str,
1143
- ) {
1144
- if let Some(watch) = coordinator_child_object(state, "abnormal_exit_watch") {
1145
- let entry = watch
1146
- .entry(agent_id.to_string())
1147
- .or_insert_with(|| serde_json::json!({}));
1148
- if !entry.is_object() {
1149
- *entry = serde_json::json!({});
1150
- }
1151
- if let Some(obj) = entry.as_object_mut() {
1152
- obj.insert(
1153
- "last_error_observation_key".to_string(),
1154
- serde_json::json!(observation_key),
1155
- );
1156
- obj.insert(
1157
- "last_error_observation_cohort".to_string(),
1158
- serde_json::json!(cohort_key),
1159
- );
1160
- obj.insert(
1161
- "last_error_observed_at".to_string(),
1162
- serde_json::json!(chrono::Utc::now().to_rfc3339()),
1163
- );
1164
- }
1165
- }
1166
- }
1167
-
1168
- fn write_abnormal_check(
1169
- event_log: &EventLog,
1170
- team: &str,
1171
- agent: &AbnormalWatchAgent,
1172
- liveness: &ProcessCheck,
1173
- fact: Option<&crate::provider::FaultFact>,
1174
- decision: AbnormalExitDecision,
1175
- error_recency: ErrorRecency,
1176
- size: u64,
1177
- mtime_ns: Option<u64>,
1178
- ) -> Result<(), TickError> {
1179
- let dead_process = liveness.state == ProcessLiveness::Dead;
1180
- let provider_process_dead = provider_process_dead_fact(liveness);
1181
- let latest_explicit_error = fact.is_some();
1182
- event_log.write(
1183
- "worker.abnormal_exit.check",
1184
- serde_json::json!({
1185
- "team_id": team,
1186
- "agent_id": agent.agent_id.as_str(),
1187
- "provider": provider_wire(agent.provider),
1188
- "path": agent.rollout_path_display.as_str(),
1189
- "size": size,
1190
- "last_offset": size,
1191
- "mtime_ns": mtime_ns,
1192
- "dead_process": dead_process,
1193
- "process_dead": dead_process,
1194
- "provider_process_dead": provider_process_dead,
1195
- "latest_error": latest_explicit_error,
1196
- "latest_explicit_error": latest_explicit_error,
1197
- "error_recency": error_recency.as_str(),
1198
- "fresh_error": error_recency.is_fresh(),
1199
- "dead_process_and_latest_error": dead_process && latest_explicit_error,
1200
- "dead_process_and_latest_explicit_error": dead_process && latest_explicit_error,
1201
- "process_dead_and_latest_explicit_error": dead_process && latest_explicit_error,
1202
- "provider_process_dead_and_latest_explicit_error": provider_process_dead && latest_explicit_error,
1203
- "notification": matches!(decision, AbnormalExitDecision::Notify),
1204
- "suppressed_reason": match decision {
1205
- AbnormalExitDecision::Suppress(reason) => Some(reason),
1206
- AbnormalExitDecision::Notify | AbnormalExitDecision::NoSignal => None,
1207
- },
1208
- "signature": fact.map(|fact| fact.signature.as_str()),
1209
- "turn_id": fact.and_then(|fact| fact.turn_id.as_ref().map(|id| id.as_str())),
1210
- "apiErrorStatus": fact.and_then(|fact| fact.api_error_status),
1211
- "error": fact.and_then(|fact| fact.error.as_deref()),
1212
- "requestId": fact.and_then(|fact| fact.request_id.as_deref()),
1213
- "assistant_uuid": fact.and_then(|fact| fact.assistant_uuid.as_deref()),
1214
- "process_liveness": process_liveness_wire(liveness.state),
1215
- "pid_status": liveness.detail.as_str(),
1216
- }),
1217
- )?;
1218
- Ok(())
1219
- }
1220
-
1221
- fn write_abnormal_suppressed(
1222
- event_log: &EventLog,
1223
- team: &str,
1224
- agent: &AbnormalWatchAgent,
1225
- liveness: &ProcessCheck,
1226
- reason: &str,
1227
- ) -> Result<(), TickError> {
1228
- let provider_process_dead = provider_process_dead_fact(liveness);
1229
- event_log.write(
1230
- "abnormal_exit.single_signal_suppressed",
1231
- serde_json::json!({
1232
- "team_id": team,
1233
- "agent_id": agent.agent_id.as_str(),
1234
- "provider": provider_wire(agent.provider),
1235
- "path": agent.rollout_path_display.as_str(),
1236
- "reason": reason,
1237
- "notification": false,
1238
- "dead_process": liveness.state == ProcessLiveness::Dead,
1239
- "process_dead": liveness.state == ProcessLiveness::Dead,
1240
- "provider_process_dead": provider_process_dead,
1241
- "latest_error": false,
1242
- "latest_explicit_error": false,
1243
- "error_recency": ErrorRecency::None.as_str(),
1244
- "fresh_error": false,
1245
- "dead_process_and_latest_error": false,
1246
- "dead_process_and_latest_explicit_error": false,
1247
- "process_dead_and_latest_explicit_error": false,
1248
- "provider_process_dead_and_latest_explicit_error": false,
1249
- "process_liveness": process_liveness_wire(liveness.state),
1250
- "pid_status": liveness.detail.as_str(),
1251
- }),
1252
- )?;
1253
- Ok(())
1254
- }
1255
-
1256
- fn abnormal_dedupe_key(agent: &AbnormalWatchAgent, fact: &crate::provider::FaultFact) -> String {
1257
- let bucket = fact
1258
- .turn_id
1259
- .as_ref()
1260
- .map(|id| id.as_str().to_string())
1261
- .or_else(|| abnormal_error_fact_identity(fact))
1262
- .unwrap_or_else(|| "no_error_identity".to_string());
1263
- format!(
1264
- "worker.abnormal_exit:{}:{}:{}:{}",
1265
- agent.agent_id,
1266
- agent.rollout_path_display,
1267
- fact.signature.as_str(),
1268
- bucket
1269
- )
1270
- }
1271
-
1272
- fn abnormal_error_cohort_key(agent: &AbnormalWatchAgent) -> String {
1273
- let cohort = agent
1274
- .spawn_epoch
1275
- .map(|epoch| format!("spawn_epoch:{epoch}"))
1276
- .or_else(|| {
1277
- agent
1278
- .spawned_at
1279
- .as_deref()
1280
- .map(|spawned_at| format!("spawned_at:{spawned_at}"))
1281
- })
1282
- .unwrap_or_else(|| "legacy".to_string());
1283
- format!(
1284
- "worker.abnormal_exit.cohort:{}:{}:{}",
1285
- agent.agent_id, agent.rollout_path_display, cohort
1286
- )
1287
- }
1288
-
1289
- fn abnormal_error_observation_key(
1290
- agent: &AbnormalWatchAgent,
1291
- fact: &crate::provider::FaultFact,
1292
- ) -> String {
1293
- let bucket =
1294
- abnormal_error_fact_identity(fact).unwrap_or_else(|| "no_error_identity".to_string());
1295
- format!(
1296
- "worker.abnormal_exit.error:{}:{}:{}:{}",
1297
- agent.agent_id,
1298
- agent.rollout_path_display,
1299
- fact.signature.as_str(),
1300
- bucket
1301
- )
1302
- }
1303
-
1304
- fn abnormal_error_fact_identity(fact: &crate::provider::FaultFact) -> Option<String> {
1305
- fact.assistant_uuid
1306
- .as_deref()
1307
- .or(fact.request_id.as_deref())
1308
- .or_else(|| fact.turn_id.as_ref().map(|id| id.as_str()))
1309
- .map(str::to_string)
1310
- }
1311
-
1312
- fn abnormal_error_recency(
1313
- state: &Value,
1314
- agent: &AbnormalWatchAgent,
1315
- observation_key: Option<&str>,
1316
- cohort_key: Option<&str>,
1317
- ) -> ErrorRecency {
1318
- let (Some(observation_key), Some(cohort_key)) = (observation_key, cohort_key) else {
1319
- return ErrorRecency::None;
1320
- };
1321
- let previous_key = abnormal_last_error_observation_key(state, &agent.agent_id);
1322
- let previous_cohort = abnormal_last_error_observation_cohort(state, &agent.agent_id);
1323
- match (previous_key.as_deref(), previous_cohort.as_deref()) {
1324
- (Some(key), Some(cohort)) if cohort == cohort_key && key != observation_key => {
1325
- ErrorRecency::Fresh
1326
- }
1327
- _ => ErrorRecency::Stale,
1328
- }
1329
- }
1330
-
1331
- fn abnormal_suppression_key(
1332
- agent: &AbnormalWatchAgent,
1333
- liveness: &ProcessCheck,
1334
- reason: &str,
1335
- ) -> String {
1336
- format!(
1337
- "abnormal_exit.single_signal_suppressed:{}:{}:{}:{}",
1338
- agent.agent_id,
1339
- agent.rollout_path_display,
1340
- reason,
1341
- process_liveness_wire(liveness.state)
1342
- )
1343
- }
1344
-
1345
- fn abnormal_check_key(
1346
- agent: &AbnormalWatchAgent,
1347
- liveness: &ProcessCheck,
1348
- fact: Option<&crate::provider::FaultFact>,
1349
- error_recency: ErrorRecency,
1350
- error_observation_key: Option<&str>,
1351
- ) -> String {
1352
- format!(
1353
- "worker.abnormal_exit.check:{}:{}:{}:{}:{}:{}",
1354
- agent.agent_id,
1355
- agent.rollout_path_display,
1356
- process_liveness_wire(liveness.state),
1357
- fact.map(|fact| fact.signature.as_str()).unwrap_or("-"),
1358
- error_recency.as_str(),
1359
- error_observation_key.unwrap_or("-")
1360
- )
1361
- }
1362
-
1363
- fn format_abnormal_exit_message(
1364
- team: &str,
1365
- agent: &AbnormalWatchAgent,
1366
- fact: &crate::provider::FaultFact,
1367
- liveness: &ProcessCheck,
1368
- size: u64,
1369
- manual_recovery_hint: &str,
1370
- ) -> String {
1371
- let turn_id = fact.turn_id.as_ref().map(|id| id.as_str()).unwrap_or("-");
1372
- format!(
1373
- "Team Agent detected a provider abnormal exit.\n\n\
1374
- event: worker.abnormal_exit\n\
1375
- team: {team}\n\
1376
- node: {node}\n\
1377
- provider: {provider}\n\
1378
- signature: {signature}\n\
1379
- turn_id: {turn_id}\n\
1380
- transcript: {path}\n\
1381
- last_offset: {size}\n\
1382
- pid_status: {pid_status}\n\n\
1383
- {manual_recovery_hint}",
1384
- node = agent.agent_id.as_str(),
1385
- provider = provider_wire(agent.provider),
1386
- signature = fact.signature.as_str(),
1387
- path = agent.rollout_path_display.as_str(),
1388
- pid_status = liveness.detail.as_str(),
1389
- )
1390
- }
1391
-
1392
- /// A pre-upgrade legacy Claude Code, Codex, Grok or Pi seat has no runtime, so
1393
- /// `start-agent` would be refused; it is stopped/removed and added again.
1394
- fn recovery_hint(
1395
- agent_id: &str,
1396
- provider: crate::model::enums::Provider,
1397
- workspace: &Path,
1398
- team: &str,
1399
- ) -> String {
1400
- let wire = provider_wire(provider);
1401
- if crate::provider::wire::is_retired_provider(wire) {
1402
- return format!(
1403
- "No automatic restart was performed.\n{}",
1404
- crate::provider::wire::retired_provider_error(wire)
1405
- );
1406
- }
1407
- format!(
1408
- "No automatic restart was performed.\nTo recover manually, run: {}",
1409
- recovery_manual_command(agent_id, workspace, team)
1410
- )
1411
- }
1412
-
1413
- fn recovery_manual_command(agent_id: &str, workspace: &Path, team: &str) -> String {
1414
- let workspace_str = workspace.to_string_lossy();
1415
- let shell_workspace = workspace_str.replace('\'', "'\\''");
1416
- format!(
1417
- "team-agent start-agent {agent_id} --workspace '{shell_workspace}' --team {team} --force --json"
1418
- )
1419
- }
1420
-
1421
- #[cfg(test)]
1422
- #[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
1423
- mod tests {
1424
- use super::*;
1425
- use crate::coordinator::tick::Coordinator;
1426
- use crate::coordinator::types::{ProviderRegistry, WorkspacePath};
1427
- use std::io::Write as _;
1428
-
1429
- struct NormalRegistry;
1430
-
1431
- impl ProviderRegistry for NormalRegistry {
1432
- fn adapter_for(
1433
- &self,
1434
- provider: crate::provider::Provider,
1435
- ) -> Box<dyn crate::provider::ProviderAdapter> {
1436
- crate::provider::get_adapter(provider)
1437
- }
1438
- }
1439
- #[test]
1440
- fn resource188_foreign_metadata_and_dedupe_cannot_skip_new_baseline() {
1441
- let dir = temp_abnormal_dir("188-foreign-watch");
1442
- let rollout = dir.join("rollout-w1.jsonl");
1443
- let error = "{\"method\":\"turn/completed\",\"params\":{\"turn\":{\"id\":\"old\",\"status\":\"failed\"}}}\n";
1444
- std::fs::write(&rollout, error).unwrap();
1445
- seed_abnormal_state(&dir, &rollout, "alive", 1);
1446
- let mut state = crate::state::persist::load_runtime_state(&dir).unwrap();
1447
- let metadata = std::fs::metadata(&rollout).unwrap();
1448
- let mut foreign = crate::state::abnormal_watch::observation_identity(&state, "w1").unwrap();
1449
- foreign["team_key"] = serde_json::json!("another-team");
1450
- state["coordinator"] = serde_json::json!({"abnormal_exit_watch": {"w1": {
1451
- "watch_identity": foreign,
1452
- "size": metadata.len(), "mtime_ns": metadata_mtime_ns(&metadata),
1453
- "last_notified_key": "foreign", "last_check_key": "foreign",
1454
- "last_error_observation_key": "foreign", "last_error_observation_cohort": "foreign"
1455
- }}});
1456
- let log = EventLog::new(&dir);
1457
- let transport = crate::transport::test_support::OfflineTransport::new();
1458
- detect_abnormal_exits(&dir, &transport, &mut state, &log, &[]).unwrap();
1459
- let baseline = state["coordinator"]["abnormal_exit_watch"]["w1"].clone();
1460
- assert_eq!(baseline["error_recency"], "stale");
1461
- assert!(baseline.get("last_notified_key").is_none());
1462
- assert!(crate::state::abnormal_watch::matches_observation(
1463
- &state, "w1", &baseline
1464
- ));
1465
- let events = log.tail(0).unwrap();
1466
- detect_abnormal_exits(&dir, &transport, &mut state, &log, &[]).unwrap();
1467
- assert_eq!(
1468
- log.tail(0).unwrap(),
1469
- events,
1470
- "unchanged baseline must not re-alarm"
1471
- );
1472
- assert!(find_event(&events, "worker.abnormal_exit").is_none());
1473
- }
1474
-
1475
- #[test]
1476
- fn resource188_legacy_missing_cohort_still_observes_fresh_errors_and_dedupes() {
1477
- let dir = temp_abnormal_dir("188-legacy-cohort");
1478
- let rollout = dir.join("rollout-w1.jsonl");
1479
- let error = "{\"method\":\"turn/completed\",\"params\":{\"turn\":{\"id\":\"old\",\"status\":\"failed\"}}}\n";
1480
- std::fs::write(&rollout, error).unwrap();
1481
- seed_abnormal_state(&dir, &rollout, "alive", 1);
1482
- let mut state = crate::state::persist::load_runtime_state(&dir).unwrap();
1483
- state["agents"]["w1"]
1484
- .as_object_mut()
1485
- .unwrap()
1486
- .remove("spawn_epoch");
1487
- let log = EventLog::new(&dir);
1488
- let transport = crate::transport::test_support::OfflineTransport::new();
1489
- detect_abnormal_exits(&dir, &transport, &mut state, &log, &[]).unwrap();
1490
- assert_eq!(
1491
- state["coordinator"]["abnormal_exit_watch"]["w1"]["error_recency"],
1492
- "stale"
1493
- );
1494
- assert!(find_event(&log.tail(0).unwrap(), "worker.abnormal_exit").is_none());
1495
- std::fs::OpenOptions::new()
1496
- .append(true)
1497
- .open(&rollout)
1498
- .unwrap()
1499
- .write_all(error.replace("old", "new-error").as_bytes())
1500
- .unwrap();
1501
- detect_abnormal_exits(&dir, &transport, &mut state, &log, &[]).unwrap();
1502
- let events = log.tail(0).unwrap();
1503
- assert_eq!(
1504
- find_event(&events, "worker.abnormal_exit").unwrap()["error_recency"],
1505
- "fresh"
1506
- );
1507
- let notified =
1508
- state["coordinator"]["abnormal_exit_watch"]["w1"]["last_notified_key"].clone();
1509
- assert!(notified.is_string());
1510
- detect_abnormal_exits(&dir, &transport, &mut state, &log, &[]).unwrap();
1511
- assert_eq!(log.tail(0).unwrap(), events);
1512
- assert_eq!(
1513
- state["coordinator"]["abnormal_exit_watch"]["w1"]["last_notified_key"],
1514
- notified
1515
- );
1516
- state["agents"]["w1"]["spawn_epoch"] = serde_json::json!(2);
1517
- detect_abnormal_exits(&dir, &transport, &mut state, &log, &[]).unwrap();
1518
- let watch = &state["coordinator"]["abnormal_exit_watch"]["w1"];
1519
- assert_eq!(
1520
- watch["error_recency"], "stale",
1521
- "a known generation must rebaseline unchanged bytes"
1522
- );
1523
- assert!(watch.get("last_notified_key").is_none());
1524
- assert_eq!(
1525
- log.tail(0)
1526
- .unwrap()
1527
- .iter()
1528
- .filter(|event| event["event"] == "worker.abnormal_exit")
1529
- .count(),
1530
- 1
1531
- );
1532
- }
1533
-
1534
- #[test]
1535
- fn resource188_unknown_team_still_records_fresh_liveness_without_reusing_cache() {
1536
- let dir = temp_abnormal_dir("188-unknown-team");
1537
- let rollout = dir.join("rollout-w1.jsonl");
1538
- std::fs::write(&rollout, "").unwrap();
1539
- seed_abnormal_state(&dir, &rollout, "dead", 1);
1540
- let mut state = crate::state::persist::load_runtime_state(&dir).unwrap();
1541
- state.as_object_mut().unwrap().remove("active_team_key");
1542
- let log = EventLog::new(&dir);
1543
- let transport = crate::transport::test_support::OfflineTransport::new();
1544
- detect_abnormal_exits(&dir, &transport, &mut state, &log, &[]).unwrap();
1545
- assert_eq!(
1546
- state["coordinator"]["abnormal_exit_watch"]["w1"]["last_liveness"],
1547
- "dead"
1548
- );
1549
- assert!(state["coordinator"]["abnormal_exit_watch"]["w1"]
1550
- .get("watch_identity")
1551
- .is_none());
1552
- }
1553
-
1554
- #[test]
1555
- fn abnormal_stale_error_baselines_then_fresh_alive_error_notifies() {
1556
- let dir = temp_abnormal_dir("alive-fresh");
1557
- let rollout = dir.join("rollout-w1.jsonl");
1558
- std::fs::write(
1559
- &rollout,
1560
- "{\"method\":\"turn/completed\",\"params\":{\"turn\":{\"id\":\"t1\",\"status\":\"failed\"}}}\n",
1561
- )
1562
- .unwrap();
1563
- seed_abnormal_state(&dir, &rollout, "alive", 1);
1564
- let coordinator = abnormal_test_coordinator(&dir);
1565
-
1566
- coordinator.tick().unwrap();
1567
-
1568
- let first_events = read_test_events(&dir);
1569
- assert!(
1570
- find_event(&first_events, "worker.abnormal_exit").is_none(),
1571
- "first observed explicit error is stale baseline and must not notify; events={first_events:?}"
1572
- );
1573
- let stale_check =
1574
- find_event(&first_events, "worker.abnormal_exit.check").expect("stale check event");
1575
- assert_eq!(stale_check["error_recency"], serde_json::json!("stale"));
1576
- assert_eq!(stale_check["fresh_error"], serde_json::json!(false));
1577
- assert_eq!(stale_check["notification"], serde_json::json!(false));
1578
- assert!(
1579
- find_event(&first_events, "abnormal_exit.single_signal_suppressed").is_none(),
1580
- "stale/error-only observations must not emit single-signal suppression"
1581
- );
1582
- let watch = abnormal_test_watch(&dir);
1583
- assert_eq!(watch["error_recency"], serde_json::json!("stale"));
1584
- assert_eq!(watch["fresh_error"], serde_json::json!(false));
1585
- assert!(
1586
- watch["last_error_observation_key"].as_str().is_some(),
1587
- "first explicit error must persist the stale baseline; watch={watch}"
1588
- );
1589
-
1590
- std::fs::OpenOptions::new()
1591
- .append(true)
1592
- .open(&rollout)
1593
- .unwrap()
1594
- .write_all(
1595
- b"{\"method\":\"turn/completed\",\"params\":{\"turn\":{\"id\":\"t2\",\"status\":\"failed\"}}}\n",
1596
- )
1597
- .unwrap();
1598
- coordinator.tick().unwrap();
1599
-
1600
- let events = read_test_events(&dir);
1601
- let abnormal =
1602
- find_event(&events, "worker.abnormal_exit").expect("fresh alive error notification");
1603
- assert_eq!(abnormal["provider_process_dead"], serde_json::json!(false));
1604
- assert_eq!(abnormal["latest_explicit_error"], serde_json::json!(true));
1605
- assert_eq!(abnormal["error_recency"], serde_json::json!("fresh"));
1606
- assert_eq!(abnormal["fresh_error"], serde_json::json!(true));
1607
- assert_eq!(
1608
- abnormal["provider_process_dead_and_latest_explicit_error"],
1609
- serde_json::json!(false)
1610
- );
1611
- assert_eq!(abnormal["notification_status"], serde_json::json!("queued"));
1612
- }
1613
-
1614
- #[test]
1615
- fn abnormal_claude_assistant_api_error_events_include_structured_details() {
1616
- let dir = temp_abnormal_dir("claude-api-details");
1617
- let rollout = dir.join("rollout-w1.jsonl");
1618
- std::fs::write(
1619
- &rollout,
1620
- claude_assistant_api_error_line(
1621
- "assistant-400",
1622
- "parent-400",
1623
- "session-400",
1624
- 400,
1625
- "unknown",
1626
- None,
1627
- ),
1628
- )
1629
- .unwrap();
1630
- seed_abnormal_state_with_provider(&dir, &rollout, "alive", 1, "claude_code");
1631
- let coordinator = abnormal_test_coordinator(&dir);
1632
-
1633
- coordinator.tick().unwrap();
1634
-
1635
- let first_events = read_test_events(&dir);
1636
- let stale_check =
1637
- find_event(&first_events, "worker.abnormal_exit.check").expect("stale check event");
1638
- assert_eq!(stale_check["error_recency"], serde_json::json!("stale"));
1639
- assert_eq!(stale_check["notification"], serde_json::json!(false));
1640
- assert_eq!(stale_check["apiErrorStatus"], serde_json::json!(400));
1641
- assert_eq!(stale_check["error"], serde_json::json!("unknown"));
1642
- assert_eq!(stale_check["requestId"], serde_json::json!(null));
1643
- assert_eq!(
1644
- stale_check["assistant_uuid"],
1645
- serde_json::json!("assistant-400")
1646
- );
1647
-
1648
- std::fs::OpenOptions::new()
1649
- .append(true)
1650
- .open(&rollout)
1651
- .unwrap()
1652
- .write_all(
1653
- claude_assistant_api_error_line(
1654
- "assistant-404",
1655
- "parent-404",
1656
- "session-404",
1657
- 404,
1658
- "model_not_found",
1659
- Some("req_011CceNfWj2aPY5gtCdakULt"),
1660
- )
1661
- .as_bytes(),
1662
- )
1663
- .unwrap();
1664
- coordinator.tick().unwrap();
1665
-
1666
- let events = read_test_events(&dir);
1667
- let fresh_check =
1668
- find_event(&events, "worker.abnormal_exit.check").expect("fresh check event");
1669
- assert_eq!(fresh_check["error_recency"], serde_json::json!("fresh"));
1670
- assert_eq!(fresh_check["notification"], serde_json::json!(true));
1671
- assert_eq!(fresh_check["apiErrorStatus"], serde_json::json!(404));
1672
- assert_eq!(fresh_check["error"], serde_json::json!("model_not_found"));
1673
- assert_eq!(
1674
- fresh_check["requestId"],
1675
- serde_json::json!("req_011CceNfWj2aPY5gtCdakULt")
1676
- );
1677
- assert_eq!(
1678
- fresh_check["assistant_uuid"],
1679
- serde_json::json!("assistant-404")
1680
- );
1681
-
1682
- let abnormal =
1683
- find_event(&events, "worker.abnormal_exit").expect("fresh alive error notification");
1684
- assert_eq!(abnormal["signature"], serde_json::json!("api_error"));
1685
- assert_eq!(abnormal["turn_id"], serde_json::json!("assistant-404"));
1686
- assert_eq!(abnormal["apiErrorStatus"], serde_json::json!(404));
1687
- assert_eq!(abnormal["error"], serde_json::json!("model_not_found"));
1688
- assert_eq!(
1689
- abnormal["requestId"],
1690
- serde_json::json!("req_011CceNfWj2aPY5gtCdakULt")
1691
- );
1692
- assert_eq!(
1693
- abnormal["assistant_uuid"],
1694
- serde_json::json!("assistant-404")
1695
- );
1696
- assert_eq!(abnormal["notification_status"], serde_json::json!("queued"));
1697
- }
1698
-
1699
- #[test]
1700
- fn abnormal_claude_assistant_api_error_followed_by_bookkeeping_still_detected() {
1701
- let dir = temp_abnormal_dir("claude-api-bookkeeping");
1702
- let rollout = dir.join("rollout-w1.jsonl");
1703
- let session_id = "97ec1070-f19b-49ed-b60f-3cb158e92053";
1704
- std::fs::write(
1705
- &rollout,
1706
- format!(
1707
- "{}{}",
1708
- claude_assistant_api_error_line(
1709
- "94e88d55-aac7-46bc-85cd-b7fcfc8a9ef6",
1710
- "245fce6f-2427-4b05-af01-8619df64afab",
1711
- session_id,
1712
- 404,
1713
- "model_not_found",
1714
- Some("req_011CceUVQ94dxahgwAHf8sdS"),
1715
- ),
1716
- claude_turn_duration_line(
1717
- "6c96328b-8c01-46c0-b68e-a096a89c904f",
1718
- "94e88d55-aac7-46bc-85cd-b7fcfc8a9ef6",
1719
- session_id,
1720
- ),
1721
- ),
1722
- )
1723
- .unwrap();
1724
- seed_abnormal_state_with_provider(&dir, &rollout, "alive", 1, "claude_code");
1725
- let coordinator = abnormal_test_coordinator(&dir);
1726
-
1727
- coordinator.tick().unwrap();
1728
-
1729
- let first_events = read_test_events(&dir);
1730
- let first_check =
1731
- find_event(&first_events, "worker.abnormal_exit.check").expect("stale check event");
1732
- assert_eq!(
1733
- first_check["latest_explicit_error"],
1734
- serde_json::json!(true)
1735
- );
1736
- assert_eq!(first_check["error_recency"], serde_json::json!("stale"));
1737
- assert_eq!(first_check["notification"], serde_json::json!(false));
1738
- assert_eq!(first_check["apiErrorStatus"], serde_json::json!(404));
1739
- assert_eq!(first_check["error"], serde_json::json!("model_not_found"));
1740
- assert_eq!(
1741
- first_check["requestId"],
1742
- serde_json::json!("req_011CceUVQ94dxahgwAHf8sdS")
1743
- );
1744
- assert_eq!(
1745
- first_check["assistant_uuid"],
1746
- serde_json::json!("94e88d55-aac7-46bc-85cd-b7fcfc8a9ef6")
1747
- );
1748
- assert!(
1749
- find_event(&first_events, "worker.abnormal_exit").is_none(),
1750
- "first observed error baselines stale even when followed by turn_duration"
1751
- );
1752
- let first_observation_key = abnormal_test_watch(&dir)["last_error_observation_key"]
1753
- .as_str()
1754
- .expect("stale baseline persists observation key")
1755
- .to_string();
1756
-
1757
- std::fs::OpenOptions::new()
1758
- .append(true)
1759
- .open(&rollout)
1760
- .unwrap()
1761
- .write_all(
1762
- format!(
1763
- "{}{}{}",
1764
- claude_file_history_snapshot_line(session_id),
1765
- claude_last_prompt_line(session_id),
1766
- claude_mode_line(session_id),
1767
- )
1768
- .as_bytes(),
1769
- )
1770
- .unwrap();
1771
- coordinator.tick().unwrap();
1772
-
1773
- let after_bookkeeping_events = read_test_events(&dir);
1774
- let bookkeeping_check = find_event(&after_bookkeeping_events, "worker.abnormal_exit.check")
1775
- .expect("bookkeeping check event");
1776
- assert_eq!(
1777
- bookkeeping_check["error_recency"],
1778
- serde_json::json!("stale")
1779
- );
1780
- assert_eq!(bookkeeping_check["notification"], serde_json::json!(false));
1781
- assert_eq!(
1782
- bookkeeping_check["assistant_uuid"],
1783
- serde_json::json!("94e88d55-aac7-46bc-85cd-b7fcfc8a9ef6")
1784
- );
1785
- assert_eq!(
1786
- abnormal_test_watch(&dir)["last_error_observation_key"],
1787
- serde_json::json!(first_observation_key)
1788
- );
1789
- assert!(
1790
- find_event(&after_bookkeeping_events, "worker.abnormal_exit").is_none(),
1791
- "bookkeeping-only growth after the same error must not notify"
1792
- );
1793
-
1794
- std::fs::OpenOptions::new()
1795
- .append(true)
1796
- .open(&rollout)
1797
- .unwrap()
1798
- .write_all(
1799
- format!(
1800
- "{}{}",
1801
- claude_assistant_api_error_line(
1802
- "11b8790e-718b-4910-bf6a-62d19135a93b",
1803
- "ccea3b0f-23e7-48ed-aa57-d59e321becfb",
1804
- session_id,
1805
- 404,
1806
- "model_not_found",
1807
- Some("req_011CceUhRwYfdmLPbv9nMmqX"),
1808
- ),
1809
- claude_turn_duration_line(
1810
- "2d382eff-783b-48ca-83c1-c93d3dc8b068",
1811
- "11b8790e-718b-4910-bf6a-62d19135a93b",
1812
- session_id,
1813
- ),
1814
- )
1815
- .as_bytes(),
1816
- )
1817
- .unwrap();
1818
- coordinator.tick().unwrap();
1819
-
1820
- let events = read_test_events(&dir);
1821
- let fresh_check =
1822
- find_event(&events, "worker.abnormal_exit.check").expect("fresh check event");
1823
- assert_eq!(fresh_check["error_recency"], serde_json::json!("fresh"));
1824
- assert_eq!(fresh_check["notification"], serde_json::json!(true));
1825
- assert_eq!(
1826
- fresh_check["assistant_uuid"],
1827
- serde_json::json!("11b8790e-718b-4910-bf6a-62d19135a93b")
1828
- );
1829
- let abnormal =
1830
- find_event(&events, "worker.abnormal_exit").expect("fresh API error notification");
1831
- assert_eq!(
1832
- abnormal["turn_id"],
1833
- serde_json::json!("11b8790e-718b-4910-bf6a-62d19135a93b")
1834
- );
1835
- assert_eq!(
1836
- abnormal["requestId"],
1837
- serde_json::json!("req_011CceUhRwYfdmLPbv9nMmqX")
1838
- );
1839
- assert_eq!(abnormal["notification_status"], serde_json::json!("queued"));
1840
- }
1841
-
1842
- #[test]
1843
- fn codex_failed_turn_then_completed_turn_no_fresh_notification() {
1844
- let dir = temp_abnormal_dir("codex-failed-then-completed");
1845
- let rollout = dir.join("rollout-w1.jsonl");
1846
- std::fs::write(
1847
- &rollout,
1848
- "{\"method\":\"turn/completed\",\"params\":{\"turn\":{\"id\":\"t1\",\"status\":\"failed\"}}}\n",
1849
- )
1850
- .unwrap();
1851
- seed_abnormal_state(&dir, &rollout, "alive", 1);
1852
- let coordinator = abnormal_test_coordinator(&dir);
1853
-
1854
- coordinator.tick().unwrap();
1855
-
1856
- let first_events = read_test_events(&dir);
1857
- let first_check =
1858
- find_event(&first_events, "worker.abnormal_exit.check").expect("stale check event");
1859
- assert_eq!(first_check["turn_id"], serde_json::json!("t1"));
1860
- assert_eq!(first_check["error_recency"], serde_json::json!("stale"));
1861
- assert_eq!(first_check["notification"], serde_json::json!(false));
1862
- assert!(
1863
- find_event(&first_events, "worker.abnormal_exit").is_none(),
1864
- "first failed turn is only a stale baseline"
1865
- );
1866
-
1867
- std::fs::OpenOptions::new()
1868
- .append(true)
1869
- .open(&rollout)
1870
- .unwrap()
1871
- .write_all(
1872
- b"{\"method\":\"turn/completed\",\"params\":{\"turn\":{\"id\":\"t-complete\",\"status\":\"completed\"}}}\n",
1873
- )
1874
- .unwrap();
1875
- coordinator.tick().unwrap();
1876
-
1877
- let after_completed_events = read_test_events(&dir);
1878
- let completed_check = find_event(&after_completed_events, "worker.abnormal_exit.check")
1879
- .expect("completed-after-failure check event");
1880
- assert_eq!(completed_check["turn_id"], serde_json::json!("t1"));
1881
- assert_eq!(completed_check["error_recency"], serde_json::json!("stale"));
1882
- assert_eq!(completed_check["fresh_error"], serde_json::json!(false));
1883
- assert_eq!(completed_check["notification"], serde_json::json!(false));
1884
- assert!(
1885
- find_event(&after_completed_events, "worker.abnormal_exit").is_none(),
1886
- "completed turn after the same old failed turn must not emit a fresh notification"
1887
- );
1888
-
1889
- std::fs::OpenOptions::new()
1890
- .append(true)
1891
- .open(&rollout)
1892
- .unwrap()
1893
- .write_all(
1894
- b"{\"method\":\"turn/completed\",\"params\":{\"turn\":{\"id\":\"t2\",\"status\":\"failed\"}}}\n",
1895
- )
1896
- .unwrap();
1897
- coordinator.tick().unwrap();
1898
-
1899
- let events = read_test_events(&dir);
1900
- let fresh_check =
1901
- find_event(&events, "worker.abnormal_exit.check").expect("new failed turn check event");
1902
- assert_eq!(fresh_check["turn_id"], serde_json::json!("t2"));
1903
- assert_eq!(fresh_check["error_recency"], serde_json::json!("fresh"));
1904
- assert_eq!(fresh_check["notification"], serde_json::json!(true));
1905
- let abnormal =
1906
- find_event(&events, "worker.abnormal_exit").expect("fresh Codex failure notification");
1907
- assert_eq!(abnormal["turn_id"], serde_json::json!("t2"));
1908
- assert_eq!(abnormal["notification_status"], serde_json::json!("queued"));
1909
- }
1910
-
1911
- #[test]
1912
- fn abnormal_dead_process_without_explicit_error_stays_dead_only() {
1913
- let dir = temp_abnormal_dir("dead-only");
1914
- let rollout = dir.join("rollout-w1.jsonl");
1915
- std::fs::write(
1916
- &rollout,
1917
- "{\"method\":\"turn/completed\",\"params\":{\"turn\":{\"id\":\"t1\",\"status\":\"completed\"}}}\n",
1918
- )
1919
- .unwrap();
1920
- seed_abnormal_state(&dir, &rollout, "dead", 1);
1921
- let coordinator = abnormal_test_coordinator(&dir);
1922
-
1923
- coordinator.tick().unwrap();
1924
-
1925
- let events = read_test_events(&dir);
1926
- assert!(
1927
- find_event(&events, "worker.abnormal_exit").is_none(),
1928
- "dead process without explicit error must not notify; events={events:?}"
1929
- );
1930
- let suppressed =
1931
- find_event(&events, "abnormal_exit.single_signal_suppressed").expect("dead_only");
1932
- assert_eq!(suppressed["reason"], serde_json::json!("dead_only"));
1933
- assert_eq!(
1934
- suppressed["latest_explicit_error"],
1935
- serde_json::json!(false)
1936
- );
1937
- assert_eq!(suppressed["error_recency"], serde_json::json!("none"));
1938
- assert_eq!(suppressed["fresh_error"], serde_json::json!(false));
1939
- }
1940
-
1941
- #[test]
1942
- fn abnormal_unchanged_transcript_keeps_provider_death_out_of_pane_projection() {
1943
- let dir = temp_abnormal_dir("unchanged-provider-dead");
1944
- let rollout = dir.join("rollout-w1.jsonl");
1945
- std::fs::write(
1946
- &rollout,
1947
- "{\"method\":\"turn/completed\",\"params\":{\"turn\":{\"id\":\"t1\",\"status\":\"completed\"}}}\n",
1948
- )
1949
- .unwrap();
1950
- seed_abnormal_state(&dir, &rollout, "alive", 1);
1951
- let coordinator = abnormal_test_coordinator(&dir);
1952
- coordinator.tick().unwrap();
1953
-
1954
- let mut state = crate::state::persist::load_runtime_state(&dir).unwrap();
1955
- state["agents"]["w1"]["process_liveness"] = serde_json::json!("dead");
1956
- crate::state::persist::save_runtime_state(&dir, &state).unwrap();
1957
- coordinator.tick().unwrap();
1958
-
1959
- let state = crate::state::persist::load_runtime_state(&dir).unwrap();
1960
- let agent = &state["agents"]["w1"];
1961
- assert!(
1962
- agent.get("provider_process_dead").is_none(),
1963
- "provider death belongs to abnormal watch, not the pane-dead seat projection: {agent}"
1964
- );
1965
- assert!(
1966
- agent.get("stale_reason").is_none(),
1967
- "provider death must not be mislabeled pane_dead: {agent}"
1968
- );
1969
- let watch = &state["coordinator"]["abnormal_exit_watch"]["w1"];
1970
- assert_eq!(watch["provider_process_dead"], serde_json::json!(true));
1971
- assert_eq!(watch["worker_provider_exited"], serde_json::json!(false));
1972
- assert_eq!(
1973
- watch["last_liveness_detail"],
1974
- serde_json::json!("explicit:dead")
1975
- );
1976
- }
1977
-
1978
- #[test]
1979
- fn abnormal_pane_absence_is_not_provider_process_death() {
1980
- let agent = test_abnormal_agent("/tmp/rollout.jsonl", Some(1), None);
1981
- for detail in ["pane_dead:%1", "window_missing:w1"] {
1982
- let payload = abnormal_watch_payload(
1983
- &agent,
1984
- Some(1),
1985
- Some(2),
1986
- process_check(ProcessLiveness::Dead, detail.to_string()),
1987
- None,
1988
- ErrorRecency::None,
1989
- None,
1990
- None,
1991
- );
1992
- assert_eq!(
1993
- payload["provider_process_dead"],
1994
- serde_json::json!(false),
1995
- "pane absence and provider death are distinct facts: {payload}"
1996
- );
1997
- assert_eq!(payload["worker_provider_exited"], serde_json::json!(false));
1998
- }
1999
- }
2000
-
2001
- #[test]
2002
- fn abnormal_recency_treats_cohort_change_as_stale() {
2003
- let agent = test_abnormal_agent("/tmp/rollout.jsonl", Some(2), None);
2004
- let fact = crate::provider::latest_explicit_error_fact(
2005
- crate::provider::Provider::Codex,
2006
- "{\"method\":\"turn/completed\",\"params\":{\"turn\":{\"id\":\"t1\",\"status\":\"failed\"}}}\n",
2007
- )
2008
- .unwrap();
2009
- let observation_key = abnormal_error_observation_key(&agent, &fact);
2010
- let state = serde_json::json!({
2011
- "coordinator": {
2012
- "abnormal_exit_watch": {
2013
- "w1": {
2014
- "last_error_observation_key": observation_key,
2015
- "last_error_observation_cohort": "worker.abnormal_exit.cohort:w1:/tmp/rollout.jsonl:spawn_epoch:1"
2016
- }
2017
- }
2018
- }
2019
- });
2020
- let cohort_key = abnormal_error_cohort_key(&agent);
2021
-
2022
- let recency = abnormal_error_recency(
2023
- &state,
2024
- &agent,
2025
- Some(observation_key.as_str()),
2026
- Some(cohort_key.as_str()),
2027
- );
2028
-
2029
- assert_eq!(
2030
- recency,
2031
- ErrorRecency::Stale,
2032
- "cohort change baselines the observed error instead of treating it as fresh"
2033
- );
2034
- }
2035
-
2036
- #[test]
2037
- fn worker_exit_marker_detail_includes_exit_code() {
2038
- let pane_id = crate::transport::PaneId::new("%marker-rc");
2039
- let marker = crate::tmux_backend::worker_provider_exit_marker("codex");
2040
- let transport = crate::transport::test_support::OfflineTransport::new()
2041
- .with_capture_for_pane(pane_id.as_str(), format!("\n{marker} 23\n"));
2042
- let mut agent = test_abnormal_agent("/tmp/rollout.jsonl", Some(1), None);
2043
- agent.pane_id = Some(pane_id.as_str().to_string());
2044
-
2045
- let check = worker_provider_exit_marker_check(&agent, &transport).expect("marker hit");
2046
-
2047
- assert_eq!(check.state, ProcessLiveness::Dead);
2048
- assert_eq!(check.detail, "worker_provider_exited:%marker-rc:rc=23");
2049
- }
2050
-
2051
- #[test]
2052
- fn worker_exit_marker_detail_uses_unknown_for_truncated_exit_code() {
2053
- let pane_id = crate::transport::PaneId::new("%marker-truncated");
2054
- let marker = crate::tmux_backend::worker_provider_exit_marker("codex");
2055
- let transport = crate::transport::test_support::OfflineTransport::new()
2056
- .with_capture_for_pane(pane_id.as_str(), format!("\n{marker} "));
2057
- let mut agent = test_abnormal_agent("/tmp/rollout.jsonl", Some(1), None);
2058
- agent.pane_id = Some(pane_id.as_str().to_string());
2059
-
2060
- let check = worker_provider_exit_marker_check(&agent, &transport).expect("marker hit");
2061
-
2062
- assert_eq!(check.state, ProcessLiveness::Dead);
2063
- assert_eq!(
2064
- check.detail,
2065
- "worker_provider_exited:%marker-truncated:rc=unknown"
2066
- );
2067
- }
2068
-
2069
- #[test]
2070
- fn abnormal_dead_fresh_error_event_reports_actual_dead_booleans() {
2071
- let dir = temp_abnormal_dir("dead-fresh");
2072
- let rollout = dir.join("rollout-w1.jsonl");
2073
- std::fs::write(
2074
- &rollout,
2075
- "{\"method\":\"turn/completed\",\"params\":{\"turn\":{\"id\":\"t1\",\"status\":\"failed\"}}}\n",
2076
- )
2077
- .unwrap();
2078
- seed_abnormal_state(&dir, &rollout, "dead", 1);
2079
- let coordinator = abnormal_test_coordinator(&dir);
2080
- coordinator.tick().unwrap();
2081
- std::fs::OpenOptions::new()
2082
- .append(true)
2083
- .open(&rollout)
2084
- .unwrap()
2085
- .write_all(
2086
- b"{\"method\":\"turn/completed\",\"params\":{\"turn\":{\"id\":\"t2\",\"status\":\"failed\"}}}\n",
2087
- )
2088
- .unwrap();
2089
-
2090
- coordinator.tick().unwrap();
2091
-
2092
- let events = read_test_events(&dir);
2093
- let abnormal =
2094
- find_event(&events, "worker.abnormal_exit").expect("fresh dead error notification");
2095
- assert_eq!(abnormal["provider_process_dead"], serde_json::json!(true));
2096
- assert_eq!(abnormal["latest_explicit_error"], serde_json::json!(true));
2097
- assert_eq!(abnormal["error_recency"], serde_json::json!("fresh"));
2098
- assert_eq!(
2099
- abnormal["provider_process_dead_and_latest_explicit_error"],
2100
- serde_json::json!(true)
2101
- );
2102
- }
2103
-
2104
- fn abnormal_test_coordinator(dir: &std::path::Path) -> Coordinator {
2105
- Coordinator::for_test(
2106
- WorkspacePath::new(dir.to_path_buf()),
2107
- Box::new(NormalRegistry),
2108
- Box::new(
2109
- crate::transport::test_support::OfflineTransport::new().with_session_present(true),
2110
- ),
2111
- None,
2112
- None,
2113
- )
2114
- }
2115
-
2116
- fn seed_abnormal_state(
2117
- dir: &std::path::Path,
2118
- rollout: &std::path::Path,
2119
- liveness: &str,
2120
- spawn_epoch: u64,
2121
- ) {
2122
- seed_abnormal_state_with_provider(dir, rollout, liveness, spawn_epoch, "codex");
2123
- }
2124
-
2125
- fn seed_abnormal_state_with_provider(
2126
- dir: &std::path::Path,
2127
- rollout: &std::path::Path,
2128
- liveness: &str,
2129
- spawn_epoch: u64,
2130
- provider: &str,
2131
- ) {
2132
- crate::state::persist::save_runtime_state(
2133
- dir,
2134
- &serde_json::json!({
2135
- "active_team_key": "team",
2136
- "agents": {
2137
- "w1": {
2138
- "provider": provider,
2139
- "status": "running",
2140
- "agent_id": "w1",
2141
- "session_id": "session-w1",
2142
- "rollout_path": rollout.to_string_lossy(),
2143
- "spawn_cwd": dir.to_string_lossy(),
2144
- "spawn_epoch": spawn_epoch,
2145
- "process_liveness": liveness
2146
- }
2147
- }
2148
- }),
2149
- )
2150
- .unwrap();
2151
- }
2152
-
2153
- fn claude_assistant_api_error_line(
2154
- uuid: &str,
2155
- parent_uuid: &str,
2156
- session_id: &str,
2157
- status: i64,
2158
- error: &str,
2159
- request_id: Option<&str>,
2160
- ) -> String {
2161
- let mut record = serde_json::json!({
2162
- "type": "assistant",
2163
- "parentUuid": parent_uuid,
2164
- "uuid": uuid,
2165
- "message": {
2166
- "role": "assistant",
2167
- "content": [{"type": "text", "text": "API Error"}]
2168
- },
2169
- "error": error,
2170
- "isApiErrorMessage": true,
2171
- "apiErrorStatus": status,
2172
- "sessionId": session_id,
2173
- "version": "2.1.181"
2174
- });
2175
- if let Some(request_id) = request_id {
2176
- record
2177
- .as_object_mut()
2178
- .unwrap()
2179
- .insert("requestId".to_string(), serde_json::json!(request_id));
2180
- }
2181
- format!("{record}\n")
2182
- }
2183
-
2184
- fn claude_turn_duration_line(uuid: &str, parent_uuid: &str, session_id: &str) -> String {
2185
- format!(
2186
- "{}\n",
2187
- serde_json::json!({
2188
- "type": "system",
2189
- "subtype": "turn_duration",
2190
- "uuid": uuid,
2191
- "parentUuid": parent_uuid,
2192
- "sessionId": session_id
2193
- })
2194
- )
2195
- }
2196
-
2197
- fn claude_file_history_snapshot_line(session_id: &str) -> String {
2198
- format!(
2199
- "{}\n",
2200
- serde_json::json!({
2201
- "type": "file-history-snapshot",
2202
- "sessionId": session_id
2203
- })
2204
- )
2205
- }
2206
-
2207
- fn claude_last_prompt_line(session_id: &str) -> String {
2208
- format!(
2209
- "{}\n",
2210
- serde_json::json!({
2211
- "type": "last-prompt",
2212
- "sessionId": session_id,
2213
- "prompt": "sanitized"
2214
- })
2215
- )
2216
- }
2217
-
2218
- fn claude_mode_line(session_id: &str) -> String {
2219
- format!(
2220
- "{}\n",
2221
- serde_json::json!({
2222
- "type": "mode",
2223
- "sessionId": session_id,
2224
- "mode": "default"
2225
- })
2226
- )
2227
- }
2228
-
2229
- fn abnormal_test_watch(dir: &std::path::Path) -> Value {
2230
- crate::state::persist::load_runtime_state(dir)
2231
- .unwrap()
2232
- .pointer("/coordinator/abnormal_exit_watch/w1")
2233
- .cloned()
2234
- .unwrap()
2235
- }
2236
-
2237
- fn read_test_events(dir: &std::path::Path) -> Vec<Value> {
2238
- let events_path = crate::model::paths::logs_dir(dir).join("events.jsonl");
2239
- std::fs::read_to_string(events_path)
2240
- .unwrap_or_default()
2241
- .lines()
2242
- .filter_map(|line| serde_json::from_str(line).ok())
2243
- .collect()
2244
- }
2245
-
2246
- fn find_event(events: &[Value], name: &str) -> Option<Value> {
2247
- events
2248
- .iter()
2249
- .rev()
2250
- .find(|event| event.get("event").and_then(Value::as_str) == Some(name))
2251
- .cloned()
2252
- }
2253
-
2254
- fn temp_abnormal_dir(tag: &str) -> std::path::PathBuf {
2255
- let dir = std::env::temp_dir().join(format!(
2256
- "team-agent-abnormal-{tag}-{}-{}",
2257
- std::process::id(),
2258
- std::time::SystemTime::now()
2259
- .duration_since(std::time::UNIX_EPOCH)
2260
- .unwrap()
2261
- .as_nanos()
2262
- ));
2263
- std::fs::create_dir_all(&dir).unwrap();
2264
- dir
2265
- }
2266
-
2267
- fn test_abnormal_agent(
2268
- rollout_path: &str,
2269
- spawn_epoch: Option<u64>,
2270
- spawned_at: Option<&str>,
2271
- ) -> AbnormalWatchAgent {
2272
- AbnormalWatchAgent {
2273
- agent_id: "w1".to_string(),
2274
- provider: crate::provider::Provider::Codex,
2275
- rollout_path: PathBuf::from(rollout_path),
2276
- rollout_path_display: rollout_path.to_string(),
2277
- spawn_epoch,
2278
- spawned_at: spawned_at.map(str::to_string),
2279
- status: Some("running".to_string()),
2280
- process_liveness: Some(ProcessLiveness::Alive),
2281
- window: None,
2282
- pane_id: None,
2283
- pid: None,
2284
- current_command: None,
2285
- }
2286
- }
2287
-
2288
- #[test]
2289
- fn retired_legacy_rows_get_a_remove_and_re_add_hint_not_a_refused_start_agent() {
2290
- use crate::model::enums::Provider;
2291
- let workspace = std::path::Path::new("/tmp/ws");
2292
- let hint = recovery_hint("w1", Provider::Codex, workspace, "research");
2293
- assert!(hint.starts_with("No automatic restart was performed.\n"), "{hint}");
2294
- assert!(
2295
- hint.contains("seat has no runtime in this release; stop or remove it, then add it again"),
2296
- "{hint}"
2297
- );
2298
- assert!(!hint.contains("start-agent"), "{hint}");
2299
- assert!(hint.ends_with("(provider: codex)"), "{hint}");
2300
- // `grok` rows are a runtime alias of grok_build, so they restart like any live provider.
2301
- for provider in [Provider::Grok, Provider::Copilot] {
2302
- let hint = recovery_hint("w1", provider, workspace, "research");
2303
- assert!(
2304
- hint.ends_with("To recover manually, run: team-agent start-agent w1 --workspace '/tmp/ws' --team research --force --json"),
2305
- "{hint}"
2306
- );
2307
- }
2308
- }
2309
- }