@team-agent/installer 0.5.128 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -4
- package/README.zh.md +1 -1
- package/package.json +4 -8
- package/skills/team-agent/SKILL.md +37 -20
- package/skills/team-agent/command-coverage.json +169 -183
- package/skills/team-agent/docs/reference/team-agent-operator.md +64 -30
- package/skills/team-agent/references/recovery-runbook.md +1 -1
- package/skills/team-agent/references/team-agent-operator.md +64 -30
- package/templates/team_state.md +1 -1
- package/Cargo.lock +0 -1063
- package/Cargo.toml +0 -36
- package/crates/team-agent/Cargo.toml +0 -69
- package/crates/team-agent/src/app_server_test_support.rs +0 -258
- package/crates/team-agent/src/cli/adapters.rs +0 -1288
- package/crates/team-agent/src/cli/arguments.rs +0 -405
- package/crates/team-agent/src/cli/attach_app_server_leader.rs +0 -98
- package/crates/team-agent/src/cli/diagnose.rs +0 -1721
- package/crates/team-agent/src/cli/emit.rs +0 -2930
- package/crates/team-agent/src/cli/grok_slot.rs +0 -384
- package/crates/team-agent/src/cli/helpers.rs +0 -91
- package/crates/team-agent/src/cli/intent/runtime.rs +0 -1216
- package/crates/team-agent/src/cli/intent/runtime_tests.rs +0 -912
- package/crates/team-agent/src/cli/intent/tests.rs +0 -689
- package/crates/team-agent/src/cli/intent.rs +0 -664
- package/crates/team-agent/src/cli/leader.rs +0 -300
- package/crates/team-agent/src/cli/leader_prompt.rs +0 -261
- package/crates/team-agent/src/cli/leaders.rs +0 -418
- package/crates/team-agent/src/cli/mod.rs +0 -6545
- package/crates/team-agent/src/cli/models.rs +0 -1206
- package/crates/team-agent/src/cli/named_address.rs +0 -1651
- package/crates/team-agent/src/cli/profile.rs +0 -342
- package/crates/team-agent/src/cli/route.rs +0 -212
- package/crates/team-agent/src/cli/schedule.rs +0 -175
- package/crates/team-agent/src/cli/send/coordinator.rs +0 -164
- package/crates/team-agent/src/cli/send/mailbox.rs +0 -145
- package/crates/team-agent/src/cli/send/persist.rs +0 -156
- package/crates/team-agent/src/cli/send/presentation.rs +0 -370
- package/crates/team-agent/src/cli/send/resolve.rs +0 -573
- package/crates/team-agent/src/cli/send.rs +0 -199
- package/crates/team-agent/src/cli/shutdown_socket.rs +0 -288
- package/crates/team-agent/src/cli/spec.rs +0 -165
- package/crates/team-agent/src/cli/status.rs +0 -280
- package/crates/team-agent/src/cli/status_port/agents.rs +0 -358
- package/crates/team-agent/src/cli/status_port/approvals.rs +0 -79
- package/crates/team-agent/src/cli/status_port/brief.rs +0 -1981
- package/crates/team-agent/src/cli/status_port/compact.rs +0 -240
- package/crates/team-agent/src/cli/status_port/format.rs +0 -146
- package/crates/team-agent/src/cli/status_port/inbox.rs +0 -62
- package/crates/team-agent/src/cli/status_port/runtime.rs +0 -197
- package/crates/team-agent/src/cli/status_port/snapshot.rs +0 -208
- package/crates/team-agent/src/cli/status_port/store.rs +0 -416
- package/crates/team-agent/src/cli/status_port/tests.rs +0 -90
- package/crates/team-agent/src/cli/status_port.rs +0 -92
- package/crates/team-agent/src/cli/tests/base.rs +0 -885
- package/crates/team-agent/src/cli/tests/divergence.rs +0 -518
- package/crates/team-agent/src/cli/tests/lane_c.rs +0 -453
- package/crates/team-agent/src/cli/tests/leader_watch.rs +0 -322
- package/crates/team-agent/src/cli/tests/main_preserved.rs +0 -430
- package/crates/team-agent/src/cli/tests/missing_subcommands.rs +0 -336
- package/crates/team-agent/src/cli/tests/mod.rs +0 -137
- package/crates/team-agent/src/cli/tests/named_address.rs +0 -761
- package/crates/team-agent/src/cli/tests/parser_safety.rs +0 -383
- package/crates/team-agent/src/cli/tests/run_delegation.rs +0 -484
- package/crates/team-agent/src/cli/tests/shutdown_kill_plan.rs +0 -2179
- package/crates/team-agent/src/cli/tests/status_brief.rs +0 -1514
- package/crates/team-agent/src/cli/tests/status_send.rs +0 -888
- package/crates/team-agent/src/cli/tests/verb_install_skill.rs +0 -93
- package/crates/team-agent/src/cli/tests/verb_profile.rs +0 -392
- package/crates/team-agent/src/cli/triage.rs +0 -234
- package/crates/team-agent/src/cli/types.rs +0 -591
- package/crates/team-agent/src/codex_app_server.rs +0 -546
- package/crates/team-agent/src/communication_mode/mod.rs +0 -56
- package/crates/team-agent/src/compiler/tests.rs +0 -827
- package/crates/team-agent/src/compiler.rs +0 -890
- package/crates/team-agent/src/conpty/backend.rs +0 -829
- package/crates/team-agent/src/conpty/mod.rs +0 -33
- package/crates/team-agent/src/contract_runtime/backend.rs +0 -1744
- package/crates/team-agent/src/contract_runtime/commands.rs +0 -671
- package/crates/team-agent/src/contract_runtime/config.rs +0 -364
- package/crates/team-agent/src/contract_runtime/config_tests.rs +0 -76
- package/crates/team-agent/src/contract_runtime/discovery.rs +0 -449
- package/crates/team-agent/src/contract_runtime/fork.rs +0 -361
- package/crates/team-agent/src/contract_runtime/forward.rs +0 -288
- package/crates/team-agent/src/contract_runtime/framework.rs +0 -367
- package/crates/team-agent/src/contract_runtime/mcp.rs +0 -394
- package/crates/team-agent/src/contract_runtime/mod.rs +0 -17
- package/crates/team-agent/src/contract_runtime/prompt.rs +0 -216
- package/crates/team-agent/src/contract_runtime/pump.rs +0 -271
- package/crates/team-agent/src/contract_runtime/registry.rs +0 -553
- package/crates/team-agent/src/contract_runtime/workspace.rs +0 -228
- package/crates/team-agent/src/coordinator/backoff.rs +0 -741
- package/crates/team-agent/src/coordinator/conpty_shim.rs +0 -805
- package/crates/team-agent/src/coordinator/health.rs +0 -1185
- package/crates/team-agent/src/coordinator/mod.rs +0 -119
- package/crates/team-agent/src/coordinator/orphan.rs +0 -110
- package/crates/team-agent/src/coordinator/runtime_detectors.rs +0 -798
- package/crates/team-agent/src/coordinator/runtime_observation.rs +0 -84
- package/crates/team-agent/src/coordinator/steps/abnormal.rs +0 -2309
- package/crates/team-agent/src/coordinator/steps/delivery.rs +0 -15
- package/crates/team-agent/src/coordinator/steps/health_sync.rs +0 -15
- package/crates/team-agent/src/coordinator/steps/mod.rs +0 -126
- package/crates/team-agent/src/coordinator/steps/persist.rs +0 -15
- package/crates/team-agent/src/coordinator/steps/runtime_prompts.rs +0 -15
- package/crates/team-agent/src/coordinator/steps/session_gate.rs +0 -16
- package/crates/team-agent/src/coordinator/tests/a0_lostupdate.rs +0 -158
- package/crates/team-agent/src/coordinator/tests/abnormal.rs +0 -312
- package/crates/team-agent/src/coordinator/tests/api_error_recovery.rs +0 -300
- package/crates/team-agent/src/coordinator/tests/basics.rs +0 -211
- package/crates/team-agent/src/coordinator/tests/daemon.rs +0 -514
- package/crates/team-agent/src/coordinator/tests/energy.rs +0 -549
- package/crates/team-agent/src/coordinator/tests/health_sync.rs +0 -503
- package/crates/team-agent/src/coordinator/tests/main_preserved.rs +0 -186
- package/crates/team-agent/src/coordinator/tests/mod.rs +0 -295
- package/crates/team-agent/src/coordinator/tests/s3_bounded.rs +0 -207
- package/crates/team-agent/src/coordinator/tests/spine.rs +0 -308
- package/crates/team-agent/src/coordinator/tests/takeover.rs +0 -286
- package/crates/team-agent/src/coordinator/tests/tick_core.rs +0 -237
- package/crates/team-agent/src/coordinator/tick.rs +0 -3024
- package/crates/team-agent/src/coordinator/types.rs +0 -540
- package/crates/team-agent/src/db/agent_health_capture.rs +0 -126
- package/crates/team-agent/src/db/message_store.rs +0 -1527
- package/crates/team-agent/src/db/migration.rs +0 -984
- package/crates/team-agent/src/db/mod.rs +0 -26
- package/crates/team-agent/src/db/schema.rs +0 -663
- package/crates/team-agent/src/diagnose/comms.rs +0 -242
- package/crates/team-agent/src/diagnose/mod.rs +0 -44
- package/crates/team-agent/src/diagnose/orphans.rs +0 -889
- package/crates/team-agent/src/direct_delivery.rs +0 -368
- package/crates/team-agent/src/event_log.rs +0 -502
- package/crates/team-agent/src/fake_worker.rs +0 -303
- package/crates/team-agent/src/inbox_store.rs +0 -169
- package/crates/team-agent/src/kill_audit.rs +0 -106
- package/crates/team-agent/src/layout/manager.rs +0 -248
- package/crates/team-agent/src/layout/mod.rs +0 -36
- package/crates/team-agent/src/layout/overlay.rs +0 -134
- package/crates/team-agent/src/layout/placement.rs +0 -52
- package/crates/team-agent/src/layout/recovery.rs +0 -106
- package/crates/team-agent/src/layout/runtime_sessions.rs +0 -317
- package/crates/team-agent/src/layout/sessions.rs +0 -294
- package/crates/team-agent/src/layout/tmux_endpoint.rs +0 -165
- package/crates/team-agent/src/layout/worker_env.rs +0 -721
- package/crates/team-agent/src/layout/worker_window_helpers.rs +0 -37
- package/crates/team-agent/src/leader/helpers.rs +0 -178
- package/crates/team-agent/src/leader/incident.rs +0 -6
- package/crates/team-agent/src/leader/lease.rs +0 -3356
- package/crates/team-agent/src/leader/mod.rs +0 -104
- package/crates/team-agent/src/leader/owner_bind.rs +0 -400
- package/crates/team-agent/src/leader/prompt.rs +0 -905
- package/crates/team-agent/src/leader/provider_attribution.rs +0 -247
- package/crates/team-agent/src/leader/rediscover/tests.rs +0 -689
- package/crates/team-agent/src/leader/rediscover.rs +0 -1192
- package/crates/team-agent/src/leader/registry.rs +0 -888
- package/crates/team-agent/src/leader/start.rs +0 -4501
- package/crates/team-agent/src/leader/takeover.rs +0 -65
- package/crates/team-agent/src/leader/tests/basics.rs +0 -337
- package/crates/team-agent/src/leader/tests/byte_findings.rs +0 -355
- package/crates/team-agent/src/leader/tests/claim_leader_mailbox_flush_contract.rs +0 -544
- package/crates/team-agent/src/leader/tests/historical_inventory_upgrade_gate_red.rs +0 -624
- package/crates/team-agent/src/leader/tests/identity.rs +0 -465
- package/crates/team-agent/src/leader/tests/identity_session_names.rs +0 -42
- package/crates/team-agent/src/leader/tests/idle.rs +0 -113
- package/crates/team-agent/src/leader/tests/lease_api.rs +0 -520
- package/crates/team-agent/src/leader/tests/lease_claim.rs +0 -629
- package/crates/team-agent/src/leader/tests/mod.rs +0 -106
- package/crates/team-agent/src/leader/tests/prompt_contract.rs +0 -268
- package/crates/team-agent/src/leader/tests/rediscover.rs +0 -403
- package/crates/team-agent/src/leader/tests/rehomed_env.rs +0 -83
- package/crates/team-agent/src/leader/tests/team_in_team_state_scope_red.rs +0 -1273
- package/crates/team-agent/src/leader/tests/terminal_and_replay_vs_rerender_invariants_red.rs +0 -417
- package/crates/team-agent/src/leader/tests/wake_start_owner.rs +0 -168
- package/crates/team-agent/src/leader/types.rs +0 -534
- package/crates/team-agent/src/lib.rs +0 -147
- package/crates/team-agent/src/lifecycle/helpers.rs +0 -212
- package/crates/team-agent/src/lifecycle/launch/add_agent.rs +0 -881
- package/crates/team-agent/src/lifecycle/launch/add_agent_state.rs +0 -598
- package/crates/team-agent/src/lifecycle/launch/agent_state.rs +0 -332
- package/crates/team-agent/src/lifecycle/launch/clone_agent.rs +0 -204
- package/crates/team-agent/src/lifecycle/launch/fork_agent.rs +0 -107
- package/crates/team-agent/src/lifecycle/launch/fork_entry.rs +0 -144
- package/crates/team-agent/src/lifecycle/launch/identity.rs +0 -674
- package/crates/team-agent/src/lifecycle/launch/layout.rs +0 -350
- package/crates/team-agent/src/lifecycle/launch/leader_context.rs +0 -589
- package/crates/team-agent/src/lifecycle/launch/mcp_config.rs +0 -160
- package/crates/team-agent/src/lifecycle/launch/ownership.rs +0 -105
- package/crates/team-agent/src/lifecycle/launch/pi_mcp.rs +0 -899
- package/crates/team-agent/src/lifecycle/launch/pi_mcp_extension.js +0 -225
- package/crates/team-agent/src/lifecycle/launch/plan.rs +0 -269
- package/crates/team-agent/src/lifecycle/launch/quick_start.rs +0 -5200
- package/crates/team-agent/src/lifecycle/launch/quick_start_transport.rs +0 -368
- package/crates/team-agent/src/lifecycle/launch/readiness.rs +0 -519
- package/crates/team-agent/src/lifecycle/launch/role_source.rs +0 -287
- package/crates/team-agent/src/lifecycle/launch/spawn.rs +0 -473
- package/crates/team-agent/src/lifecycle/launch/spec_state.rs +0 -801
- package/crates/team-agent/src/lifecycle/launch/state_projection.rs +0 -624
- package/crates/team-agent/src/lifecycle/launch/worker_env.rs +0 -558
- package/crates/team-agent/src/lifecycle/launch.rs +0 -557
- package/crates/team-agent/src/lifecycle/lock.rs +0 -350
- package/crates/team-agent/src/lifecycle/mod.rs +0 -79
- package/crates/team-agent/src/lifecycle/pane_input_lock.rs +0 -239
- package/crates/team-agent/src/lifecycle/profile_launch.rs +0 -1067
- package/crates/team-agent/src/lifecycle/profile_smoke.rs +0 -557
- package/crates/team-agent/src/lifecycle/restart/agent.rs +0 -2199
- package/crates/team-agent/src/lifecycle/restart/avatar.rs +0 -1798
- package/crates/team-agent/src/lifecycle/restart/claude_fork.rs +0 -163
- package/crates/team-agent/src/lifecycle/restart/common.rs +0 -2362
- package/crates/team-agent/src/lifecycle/restart/orchestrator.rs +0 -65
- package/crates/team-agent/src/lifecycle/restart/preflight.rs +0 -113
- package/crates/team-agent/src/lifecycle/restart/rebuild.rs +0 -5135
- package/crates/team-agent/src/lifecycle/restart/remove.rs +0 -1504
- package/crates/team-agent/src/lifecycle/restart/selection.rs +0 -559
- package/crates/team-agent/src/lifecycle/restart/team_state.rs +0 -329
- package/crates/team-agent/src/lifecycle/restart.rs +0 -227
- package/crates/team-agent/src/lifecycle/role_config.rs +0 -1230
- package/crates/team-agent/src/lifecycle/tests/acceptance_b_batch_red.rs +0 -1021
- package/crates/team-agent/src/lifecycle/tests/agent_ops.rs +0 -1205
- package/crates/team-agent/src/lifecycle/tests/b0_legacy_snapshot_nonauthority_contract.rs +0 -265
- package/crates/team-agent/src/lifecycle/tests/b0_reader_hideone_audit_contract.rs +0 -490
- package/crates/team-agent/src/lifecycle/tests/behavioral_diff_264_red.rs +0 -656
- package/crates/team-agent/src/lifecycle/tests/claude_compatible_config_red.rs +0 -325
- package/crates/team-agent/src/lifecycle/tests/claude_profile_launch_red.rs +0 -447
- package/crates/team-agent/src/lifecycle/tests/cli_argv_routing_red.rs +0 -1638
- package/crates/team-agent/src/lifecycle/tests/clone_agent_preserves_source_tools.rs +0 -427
- package/crates/team-agent/src/lifecycle/tests/clone_fork_copilot_perms_red.rs +0 -770
- package/crates/team-agent/src/lifecycle/tests/communication_mode_runtime_contract_red.rs +0 -419
- package/crates/team-agent/src/lifecycle/tests/copilot_provider_red.rs +0 -2035
- package/crates/team-agent/src/lifecycle/tests/core.rs +0 -1327
- package/crates/team-agent/src/lifecycle/tests/f032_startup_prompt_best_effort_red.rs +0 -315
- package/crates/team-agent/src/lifecycle/tests/fixtures/pi_list_models_g0.stdout.txt +0 -222
- package/crates/team-agent/src/lifecycle/tests/fixtures/pi_list_models_spaced_ids.stdout.txt +0 -9
- package/crates/team-agent/src/lifecycle/tests/fixtures/pi_native_mcp_host.mjs +0 -72
- package/crates/team-agent/src/lifecycle/tests/gate_fixtures.rs +0 -562
- package/crates/team-agent/src/lifecycle/tests/grok_require_explicit_model_red.rs +0 -110
- package/crates/team-agent/src/lifecycle/tests/harvest2_a_batch_red.rs +0 -512
- package/crates/team-agent/src/lifecycle/tests/host_cotenant_death_p0_contract.rs +0 -921
- package/crates/team-agent/src/lifecycle/tests/issue238_codex_effort_contract.rs +0 -68
- package/crates/team-agent/src/lifecycle/tests/lane_ops.rs +0 -1851
- package/crates/team-agent/src/lifecycle/tests/launch_spawn.rs +0 -3983
- package/crates/team-agent/src/lifecycle/tests/lifecycle_lock.rs +0 -339
- package/crates/team-agent/src/lifecycle/tests/lifecycle_rollback_red.rs +0 -596
- package/crates/team-agent/src/lifecycle/tests/main_preserved.rs +0 -513
- package/crates/team-agent/src/lifecycle/tests/mcp_tool_name_format_red.rs +0 -69
- package/crates/team-agent/src/lifecycle/tests/phase_b_contracts.rs +0 -824
- package/crates/team-agent/src/lifecycle/tests/phase_golden.rs +0 -723
- package/crates/team-agent/src/lifecycle/tests/pi_catalog_red.rs +0 -164
- package/crates/team-agent/src/lifecycle/tests/pi_command_red.rs +0 -200
- package/crates/team-agent/src/lifecycle/tests/pi_compiler_red.rs +0 -210
- package/crates/team-agent/src/lifecycle/tests/pi_dual_entry_red.rs +0 -527
- package/crates/team-agent/src/lifecycle/tests/pi_executable_mcp_red.rs +0 -1062
- package/crates/team-agent/src/lifecycle/tests/pi_fork_session_inheritance_red.rs +0 -1419
- package/crates/team-agent/src/lifecycle/tests/pi_lifecycle_doctor_red.rs +0 -98
- package/crates/team-agent/src/lifecycle/tests/pi_native_mcp_capability_red.rs +0 -550
- package/crates/team-agent/src/lifecycle/tests/pi_session_delivery_red.rs +0 -226
- package/crates/team-agent/src/lifecycle/tests/quick_start_worker_readiness_red.rs +0 -332
- package/crates/team-agent/src/lifecycle/tests/realmachine_clusters_1_6_red.rs +0 -786
- package/crates/team-agent/src/lifecycle/tests/realmachine_residual_g1_g4_red.rs +0 -648
- package/crates/team-agent/src/lifecycle/tests/restart.rs +0 -185
- package/crates/team-agent/src/lifecycle/tests/restart_build_before_destroy_0540_contract.rs +0 -1053
- package/crates/team-agent/src/lifecycle/tests/restart_liveness_red.rs +0 -679
- package/crates/team-agent/src/lifecycle/tests/restart_rebind_hotfix_252_red.rs +0 -1126
- package/crates/team-agent/src/lifecycle/tests/restart_session_capture_red.rs +0 -153
- package/crates/team-agent/src/lifecycle/tests/resume_recover_red.rs +0 -427
- package/crates/team-agent/src/lifecycle/tests/stale_team_saveconflict_contract.rs +0 -431
- package/crates/team-agent/src/lifecycle/tests/startup_latency_contract.rs +0 -1066
- package/crates/team-agent/src/lifecycle/tests/status_credential_redaction_contract.rs +0 -876
- package/crates/team-agent/src/lifecycle/tests/subprep_codex_trust_roundtrip_red.rs +0 -1026
- package/crates/team-agent/src/lifecycle/tests/swallow_batch4_semantics_red.rs +0 -342
- package/crates/team-agent/src/lifecycle/tests/team_in_team_identity_scope_red.rs +0 -543
- package/crates/team-agent/src/lifecycle/tests/team_in_team_sibling_quick_start_red.rs +0 -1103
- package/crates/team-agent/src/lifecycle/tests/team_in_team_state_scope_red.rs +0 -1276
- package/crates/team-agent/src/lifecycle/tests/team_key_retirement_roster_red.rs +0 -145
- package/crates/team-agent/src/lifecycle/tests/team_key_retirement_txn_red.rs +0 -603
- package/crates/team-agent/src/lifecycle/tests/test_isolation_escape_contract.rs +0 -936
- package/crates/team-agent/src/lifecycle/tests/upgrade_compat_0211_red.rs +0 -970
- package/crates/team-agent/src/lifecycle/tests/verify_rs031_window_consistency_red.rs +0 -995
- package/crates/team-agent/src/lifecycle/tests/worker_spawn_env_red.rs +0 -836
- package/crates/team-agent/src/lifecycle/tests.rs +0 -105
- package/crates/team-agent/src/lifecycle/types.rs +0 -877
- package/crates/team-agent/src/lifecycle/worker_command_context.rs +0 -363
- package/crates/team-agent/src/main.rs +0 -52
- package/crates/team-agent/src/mcp_server/helpers.rs +0 -510
- package/crates/team-agent/src/mcp_server/lifecycle_tools/mod.rs +0 -6
- package/crates/team-agent/src/mcp_server/lifecycle_tools/state_status.rs +0 -43
- package/crates/team-agent/src/mcp_server/mod.rs +0 -119
- package/crates/team-agent/src/mcp_server/normalize.rs +0 -485
- package/crates/team-agent/src/mcp_server/readiness.rs +0 -527
- package/crates/team-agent/src/mcp_server/readiness_tests.rs +0 -348
- package/crates/team-agent/src/mcp_server/tests/golden.rs +0 -285
- package/crates/team-agent/src/mcp_server/tests/normalize.rs +0 -247
- package/crates/team-agent/src/mcp_server/tests/scoped.rs +0 -158
- package/crates/team-agent/src/mcp_server/tests/send.rs +0 -353
- package/crates/team-agent/src/mcp_server/tests/tools.rs +0 -341
- package/crates/team-agent/src/mcp_server/tests/wire.rs +0 -254
- package/crates/team-agent/src/mcp_server/tests.rs +0 -43
- package/crates/team-agent/src/mcp_server/tools.rs +0 -889
- package/crates/team-agent/src/mcp_server/types.rs +0 -396
- package/crates/team-agent/src/mcp_server/wire.rs +0 -867
- package/crates/team-agent/src/messaging/activity.rs +0 -461
- package/crates/team-agent/src/messaging/address.rs +0 -87
- package/crates/team-agent/src/messaging/delivery.rs +0 -3727
- package/crates/team-agent/src/messaging/delivery_consumption_tests.rs +0 -234
- package/crates/team-agent/src/messaging/enter_rescue.rs +0 -362
- package/crates/team-agent/src/messaging/enter_rescue_tests.rs +0 -633
- package/crates/team-agent/src/messaging/helpers.rs +0 -336
- package/crates/team-agent/src/messaging/leader_channel.rs +0 -287
- package/crates/team-agent/src/messaging/leader_receiver.rs +0 -786
- package/crates/team-agent/src/messaging/mod.rs +0 -167
- package/crates/team-agent/src/messaging/persist.rs +0 -377
- package/crates/team-agent/src/messaging/presentation.rs +0 -417
- package/crates/team-agent/src/messaging/reminders.rs +0 -316
- package/crates/team-agent/src/messaging/results/collection_tests.rs +0 -719
- package/crates/team-agent/src/messaging/results.rs +0 -1415
- package/crates/team-agent/src/messaging/scheduler.rs +0 -223
- package/crates/team-agent/src/messaging/selftest.rs +0 -315
- package/crates/team-agent/src/messaging/send.rs +0 -981
- package/crates/team-agent/src/messaging/tests/basic.rs +0 -346
- package/crates/team-agent/src/messaging/tests/dup_inject.rs +0 -657
- package/crates/team-agent/src/messaging/tests/e23.rs +0 -49
- package/crates/team-agent/src/messaging/tests/leader_channel.rs +0 -169
- package/crates/team-agent/src/messaging/tests/leader_inject_acceptance.rs +0 -817
- package/crates/team-agent/src/messaging/tests/main_preserved.rs +0 -85
- package/crates/team-agent/src/messaging/tests/mod.rs +0 -306
- package/crates/team-agent/src/messaging/tests/runtime.rs +0 -3549
- package/crates/team-agent/src/messaging/tests/spine.rs +0 -853
- package/crates/team-agent/src/messaging/tests/wave2.rs +0 -381
- package/crates/team-agent/src/messaging/trust.rs +0 -215
- package/crates/team-agent/src/messaging/types.rs +0 -472
- package/crates/team-agent/src/messaging/wait.rs +0 -366
- package/crates/team-agent/src/messaging/watchers.rs +0 -916
- package/crates/team-agent/src/model/enums.rs +0 -356
- package/crates/team-agent/src/model/errors.rs +0 -18
- package/crates/team-agent/src/model/ids.rs +0 -165
- package/crates/team-agent/src/model/mod.rs +0 -29
- package/crates/team-agent/src/model/name_similarity.rs +0 -252
- package/crates/team-agent/src/model/pane_authority_refusal.rs +0 -359
- package/crates/team-agent/src/model/paths.rs +0 -257
- package/crates/team-agent/src/model/routing.rs +0 -387
- package/crates/team-agent/src/model/spec.rs +0 -1184
- package/crates/team-agent/src/model/task_graph.rs +0 -321
- package/crates/team-agent/src/model/testdata/fuzz.golden.yaml +0 -43
- package/crates/team-agent/src/model/testdata/fuzz.yaml +0 -43
- package/crates/team-agent/src/model/testdata/spec_invalid_a.yaml +0 -209
- package/crates/team-agent/src/model/testdata/team.spec.golden.yaml +0 -208
- package/crates/team-agent/src/model/testdata/team.spec.yaml +0 -208
- package/crates/team-agent/src/model/yaml/tests.rs +0 -188
- package/crates/team-agent/src/model/yaml.rs +0 -802
- package/crates/team-agent/src/os_probe.rs +0 -280
- package/crates/team-agent/src/packaging/install.rs +0 -219
- package/crates/team-agent/src/packaging/migrate.rs +0 -38
- package/crates/team-agent/src/packaging/mod.rs +0 -91
- package/crates/team-agent/src/packaging/repair.rs +0 -33
- package/crates/team-agent/src/packaging/tests.rs +0 -822
- package/crates/team-agent/src/packaging/types.rs +0 -417
- package/crates/team-agent/src/platform/argv.rs +0 -331
- package/crates/team-agent/src/platform/errors.rs +0 -104
- package/crates/team-agent/src/platform/file_lock.rs +0 -279
- package/crates/team-agent/src/platform/mod.rs +0 -5
- package/crates/team-agent/src/platform/process.rs +0 -545
- package/crates/team-agent/src/provider/adapter.rs +0 -1150
- package/crates/team-agent/src/provider/adapters/claude.rs +0 -78
- package/crates/team-agent/src/provider/adapters/claude_fork.rs +0 -131
- package/crates/team-agent/src/provider/adapters/copilot.rs +0 -75
- package/crates/team-agent/src/provider/adapters/copilot_fork.rs +0 -307
- package/crates/team-agent/src/provider/adapters/fake.rs +0 -18
- package/crates/team-agent/src/provider/adapters/mod.rs +0 -30
- package/crates/team-agent/src/provider/adapters/pi.rs +0 -106
- package/crates/team-agent/src/provider/approvals/mod.rs +0 -3
- package/crates/team-agent/src/provider/approvals/parsing.rs +0 -586
- package/crates/team-agent/src/provider/approvals/runtime_prompts.rs +0 -205
- package/crates/team-agent/src/provider/argv_route.rs +0 -405
- package/crates/team-agent/src/provider/bypass_flags.rs +0 -116
- package/crates/team-agent/src/provider/classify.rs +0 -708
- package/crates/team-agent/src/provider/command_helpers.rs +0 -45
- package/crates/team-agent/src/provider/faults.rs +0 -165
- package/crates/team-agent/src/provider/helpers.rs +0 -52
- package/crates/team-agent/src/provider/mod.rs +0 -78
- package/crates/team-agent/src/provider/model_catalog.rs +0 -910
- package/crates/team-agent/src/provider/session/capture.rs +0 -2731
- package/crates/team-agent/src/provider/session/context_fork/claude.rs +0 -123
- package/crates/team-agent/src/provider/session/context_fork/outcome.rs +0 -172
- package/crates/team-agent/src/provider/session/context_fork.rs +0 -396
- package/crates/team-agent/src/provider/session/mod.rs +0 -36
- package/crates/team-agent/src/provider/session/resume.rs +0 -269
- package/crates/team-agent/src/provider/session_scan/claude.rs +0 -523
- package/crates/team-agent/src/provider/session_scan/common/tests.rs +0 -316
- package/crates/team-agent/src/provider/session_scan/common.rs +0 -654
- package/crates/team-agent/src/provider/session_scan/copilot.rs +0 -280
- package/crates/team-agent/src/provider/session_scan/pi.rs +0 -185
- package/crates/team-agent/src/provider/session_scan.rs +0 -71
- package/crates/team-agent/src/provider/startup_prompt.rs +0 -757
- package/crates/team-agent/src/provider/submit_now.rs +0 -71
- package/crates/team-agent/src/provider/testdata/copilot-ready-marker.txt +0 -5
- package/crates/team-agent/src/provider/testdata/copilot-trust-prompt.txt +0 -19
- package/crates/team-agent/src/provider/testdata/grok-models-authenticated.stdout +0 -9
- package/crates/team-agent/src/provider/testdata/grok-models-unauthenticated.stdout +0 -7
- package/crates/team-agent/src/provider/tests/adapter.rs +0 -441
- package/crates/team-agent/src/provider/tests/classify.rs +0 -148
- package/crates/team-agent/src/provider/tests/command_helpers.rs +0 -35
- package/crates/team-agent/src/provider/tests/copilot_fork.rs +0 -191
- package/crates/team-agent/src/provider/tests/faults.rs +0 -200
- package/crates/team-agent/src/provider/tests/idle.rs +0 -177
- package/crates/team-agent/src/provider/tests/wire.rs +0 -213
- package/crates/team-agent/src/provider/tests.rs +0 -32
- package/crates/team-agent/src/provider/types.rs +0 -529
- package/crates/team-agent/src/provider/wire.rs +0 -394
- package/crates/team-agent/src/redaction.rs +0 -259
- package/crates/team-agent/src/state/abnormal_watch/tests/mod.rs +0 -248
- package/crates/team-agent/src/state/abnormal_watch.rs +0 -144
- package/crates/team-agent/src/state/identity.rs +0 -914
- package/crates/team-agent/src/state/identity_keys.rs +0 -132
- package/crates/team-agent/src/state/mod.rs +0 -74
- package/crates/team-agent/src/state/owner_gate.rs +0 -482
- package/crates/team-agent/src/state/ownership.rs +0 -563
- package/crates/team-agent/src/state/paths.rs +0 -395
- package/crates/team-agent/src/state/persist.rs +0 -3811
- package/crates/team-agent/src/state/projection.rs +0 -1508
- package/crates/team-agent/src/state/repository/intent.rs +0 -14
- package/crates/team-agent/src/state/repository/tests.rs +0 -184
- package/crates/team-agent/src/state/repository.rs +0 -866
- package/crates/team-agent/src/state/selector.rs +0 -160
- package/crates/team-agent/src/state/testdata/state-rich.canonical.json +0 -133
- package/crates/team-agent/src/test_support/models_timeout_fixture.rs +0 -144
- package/crates/team-agent/src/tmux_backend/enter_rescue.rs +0 -325
- package/crates/team-agent/src/tmux_backend/enter_rescue_tests.rs +0 -432
- package/crates/team-agent/src/tmux_backend/native_input.rs +0 -513
- package/crates/team-agent/src/tmux_backend/native_input_tests.rs +0 -742
- package/crates/team-agent/src/tmux_backend/tests.rs +0 -4103
- package/crates/team-agent/src/tmux_backend.rs +0 -4492
- package/crates/team-agent/src/topology.rs +0 -537
- package/crates/team-agent/src/transport/enter_rescue.rs +0 -166
- package/crates/team-agent/src/transport/native_input.rs +0 -491
- package/crates/team-agent/src/transport/test_support.rs +0 -743
- package/crates/team-agent/src/transport/tests/behavior.rs +0 -326
- package/crates/team-agent/src/transport/tests/mod.rs +0 -194
- package/crates/team-agent/src/transport/tests/wire.rs +0 -574
- package/crates/team-agent/src/transport.rs +0 -1655
- package/crates/team-agent/src/transport_factory.rs +0 -819
|
@@ -1,3024 +0,0 @@
|
|
|
1
|
-
//! ---
|
|
2
|
-
//! purpose: coordinator 的核心——Coordinator 本体与单次 tick 的固定顺序编排,外加心跳 sidecar 与 worker 运行态解析
|
|
3
|
-
//! contract:
|
|
4
|
-
//! provides:
|
|
5
|
-
//! - name: tick
|
|
6
|
-
//! what: 跑一轮固定顺序的编排:会话门 → 捕获缺失 session → 启动/审批提示 → 健康对账 → 投递与到期事件 → 只读探测 → 原子持久化 → 回收结果
|
|
7
|
-
//! - name: write_coordinator_heartbeat
|
|
8
|
-
//! what: 写 runtime/coordinator_tick.json 心跳 sidecar(阶段、状态、boot id、迭代计数)
|
|
9
|
-
//! - name: read_coordinator_heartbeat
|
|
10
|
-
//! what: 只读取出最近一次心跳,供 status/diagnose 消费
|
|
11
|
-
//! - name: resolve_worker_runtime_state_with_fg_pgrp
|
|
12
|
-
//! what: 由 agent JSON 与活动分类结果解析 worker 运行态,缺信号时判 Unknown 而非 Idle
|
|
13
|
-
//! depends:
|
|
14
|
-
//! - super::health
|
|
15
|
-
//! - super::types
|
|
16
|
-
//! - super::steps
|
|
17
|
-
//! - super::steps::abnormal
|
|
18
|
-
//! - super::runtime_observation
|
|
19
|
-
//! - crate::state
|
|
20
|
-
//! - crate::message_store
|
|
21
|
-
//! - crate::messaging
|
|
22
|
-
//! - crate::leader
|
|
23
|
-
//! - crate::provider
|
|
24
|
-
//! - crate::transport
|
|
25
|
-
//! - crate::event_log
|
|
26
|
-
//! - crate::os_probe
|
|
27
|
-
//! boundary:
|
|
28
|
-
//! - 不实现投递、到期事件与结果回收的本体,那些在 crate::messaging,本模块只按序调用
|
|
29
|
-
//! - 无 pending obligation 时不注入任何探索性 prompt;无既定义务时唯一会投递的是 should_ping 成立时的一条中性提示。例外:屏上出现审批提示且策略允许时,runtime_approval 会经 transport.send_keys 注入应答按键(不是 prompt,是对已观测提示的应答)
|
|
30
|
-
//! - 监测步骤失败降级并继续,TickReport.ok=true 不代表每一步都成功——各步失败只在事件里
|
|
31
|
-
//! - 持久化失败走 degraded 报告而不是 panic,也不是 Err
|
|
32
|
-
//! - 不直接依赖 provider client crate,provider 一律经注入的 ProviderAdapter
|
|
33
|
-
//! maturity: wired
|
|
34
|
-
//! ---
|
|
35
|
-
//!
|
|
36
|
-
//! Coordinator core:daemon lifecycle 宿主 + 单次 tick 编排(19 步固定顺序)+ health/start/stop。
|
|
37
|
-
|
|
38
|
-
use std::collections::BTreeMap;
|
|
39
|
-
use std::path::{Path, PathBuf};
|
|
40
|
-
|
|
41
|
-
use serde_json::Value;
|
|
42
|
-
use thiserror::Error;
|
|
43
|
-
|
|
44
|
-
use crate::event_log::EventLog;
|
|
45
|
-
use crate::leader::{TakeoverReminderResult, TurnClassification, TurnStateClassifier};
|
|
46
|
-
use crate::provider::{
|
|
47
|
-
approval_choice_keys, awaiting_human_confirm_fact, awaiting_human_confirm_reason,
|
|
48
|
-
choose_internal_mcp_approval_choice, extract_approval_prompt, runtime_approval_decision,
|
|
49
|
-
ProcessLiveness, RuntimeApprovalDecision, TurnState,
|
|
50
|
-
};
|
|
51
|
-
|
|
52
|
-
use super::health::{
|
|
53
|
-
coordinator_log_path, coordinator_meta_path, coordinator_metadata_ok, coordinator_pid_path,
|
|
54
|
-
pid_is_running, read_coordinator_metadata, write_coordinator_metadata,
|
|
55
|
-
};
|
|
56
|
-
use super::runtime_observation::{self, CapturedRuntimeFact};
|
|
57
|
-
use super::steps::TickStepGroup;
|
|
58
|
-
use super::types::{
|
|
59
|
-
AgentId, CoordinatorHealthStatus, HealthReport, MetadataSource, Pid, ProviderRegistry,
|
|
60
|
-
SchemaHealth, StartError, StartOutcome, StartReport, StopError, StopOutcome, StopReport,
|
|
61
|
-
TickStopReason, WorkspacePath,
|
|
62
|
-
};
|
|
63
|
-
use super::types::{
|
|
64
|
-
CollectedResult, CompactionResult, DeadlockAlert, DeliveredMessage, FiredScheduledEvent,
|
|
65
|
-
IdleAlert, LeaderApiError, SessionDriftResult,
|
|
66
|
-
};
|
|
67
|
-
|
|
68
|
-
const STARTUP_PROMPT_GRACE_SECS: i64 = 120;
|
|
69
|
-
const RUNTIME_APPROVAL_INITIAL_BACKOFF_SECS: i64 = 30;
|
|
70
|
-
const RUNTIME_APPROVAL_MAX_BACKOFF_SECS: i64 = 300;
|
|
71
|
-
const IDLE_HEALTH_CAPTURE_INTERVAL_SECS: i64 = 60;
|
|
72
|
-
pub(crate) const HEARTBEAT_STATUS_PANIC: &str = "panic";
|
|
73
|
-
|
|
74
|
-
// ===========================================================================
|
|
75
|
-
// TickReport / TickError(§10:tick(..) -> Result<TickReport, TickError>)
|
|
76
|
-
// ===========================================================================
|
|
77
|
-
|
|
78
|
-
/// 单次 tick 报告(`lifecycle.py:373-385` 成功 / `:349-363` degraded)。
|
|
79
|
-
/// degraded 用 `ok:false, reason: Some(PersistenceDegraded)`(card 表)。
|
|
80
|
-
/// `stop:true` 触发主循环退出(tmux_session_missing)。
|
|
81
|
-
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
82
|
-
pub struct TickReport {
|
|
83
|
-
/// `ok`(`lifecycle.py:374`)。
|
|
84
|
-
pub ok: bool,
|
|
85
|
-
/// `stop`(`lifecycle.py:279/375`)—— true 触发主循环 break。
|
|
86
|
-
pub stop: bool,
|
|
87
|
-
/// 非 ok 时的原因(`lifecycle.py:279,353`)。
|
|
88
|
-
pub reason: Option<TickStopReason>,
|
|
89
|
-
/// bug-084:tick-end save 是否成功落盘(`lifecycle.py:354`)。`None` ⇔ 未走到 save(早退)。
|
|
90
|
-
pub persisted: Option<bool>,
|
|
91
|
-
/// `_deliver_pending_messages` 投递条数/句柄(`lifecycle.py:285`)——cross-dep step 11。
|
|
92
|
-
pub delivered: Vec<DeliveredMessage>,
|
|
93
|
-
/// `_fire_due_scheduled_events` 触发的 scheduled(`lifecycle.py:286`)——cross-dep step 11。
|
|
94
|
-
pub scheduled: Vec<FiredScheduledEvent>,
|
|
95
|
-
/// `_detect_stuck_agents` 卡住的 agent(`lifecycle.py:287`)——cross-dep step 11。
|
|
96
|
-
pub stuck: Vec<AgentId>,
|
|
97
|
-
/// idle take-over 提醒(`lifecycle.py:303-308`)——should_ping 时一条。
|
|
98
|
-
pub idle_alerts: Vec<IdleAlert>,
|
|
99
|
-
/// `detect_cross_worker_deadlocks`(`lifecycle.py:309`)——cross-dep step 11。
|
|
100
|
-
pub deadlock_alerts: Vec<DeadlockAlert>,
|
|
101
|
-
/// `detect_compaction_degradation` 结果(`lifecycle.py:310-330`,仅 codex)——cross-dep step 11。
|
|
102
|
-
pub compaction: Vec<CompactionResult>,
|
|
103
|
-
/// `detect_session_drift` 结果(`lifecycle.py:331-343`,仅 codex)——cross-dep step 11。
|
|
104
|
-
pub session_drift: Vec<SessionDriftResult>,
|
|
105
|
-
/// `detect_leader_api_errors`(`lifecycle.py:344`)——cross-dep step 11。
|
|
106
|
-
pub api_errors: Vec<LeaderApiError>,
|
|
107
|
-
/// Reserved result projection slot; report_result now finalizes directly.
|
|
108
|
-
pub results: Vec<CollectedResult>,
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
/// tick 失败错误(§10:daemon-path 返 Result)。bug-084:`save_runtime_state` 失败**不**走这里
|
|
112
|
-
/// (那是 degraded `TickReport`,主循环不 catch 它);本 enum 是 tick 编排其余环节(load state /
|
|
113
|
-
/// store 构造 / 原子调用)的硬失败,主循环 catch 后退避(`__main__.py:62`)。
|
|
114
|
-
#[derive(Debug, Error)]
|
|
115
|
-
pub enum TickError {
|
|
116
|
-
/// `load_runtime_state` 失败(state.json 损坏 / 锁)。
|
|
117
|
-
#[error("load runtime state failed: {0}")]
|
|
118
|
-
StateLoad(#[from] crate::state::StateError),
|
|
119
|
-
/// `MessageStore(workspace)` 构造失败(`lifecycle.py:275`)。
|
|
120
|
-
#[error("message store: {0}")]
|
|
121
|
-
MessageStore(#[from] crate::message_store::MessageStoreError),
|
|
122
|
-
/// EventLog 写失败。
|
|
123
|
-
#[error("event log: {0}")]
|
|
124
|
-
EventLog(#[from] crate::event_log::EventLogError),
|
|
125
|
-
/// transport 探测失败(tmux session 存活查询等)。
|
|
126
|
-
#[error("transport: {0}")]
|
|
127
|
-
Transport(#[from] crate::transport::TransportError),
|
|
128
|
-
/// provider trait 调用失败(startup/runtime prompt handlers, classifiers)。
|
|
129
|
-
#[error("provider: {0}")]
|
|
130
|
-
Provider(#[from] crate::provider::ProviderError),
|
|
131
|
-
/// messaging subsystem failure(delivery/scheduler/result watchers).
|
|
132
|
-
#[error("messaging: {0}")]
|
|
133
|
-
Messaging(#[from] crate::messaging::MessagingError),
|
|
134
|
-
/// coordinator.tick panic caught by the daemon loop.
|
|
135
|
-
#[error("panic: {0}")]
|
|
136
|
-
Panic(String),
|
|
137
|
-
}
|
|
138
|
-
|
|
139
|
-
/// Issue 2 (Round 3b gate review §6): when the runtime state has
|
|
140
|
-
/// `active_team_key` AND `teams.<key>` is a populated object, return the
|
|
141
|
-
/// team-scoped projection so the coordinator's tick reads `session_name` /
|
|
142
|
-
/// `agents` / `leader_receiver` from the team's nested entry rather than
|
|
143
|
-
/// the top-level (often stale) view. When the projection cannot be derived,
|
|
144
|
-
/// returns `None` and the tick falls back to the raw state — preserving
|
|
145
|
-
/// behavior for legacy single-team workspaces and tests that don't seed
|
|
146
|
-
/// `teams.<key>`. Sibling teams under `state.teams.*` are NOT touched.
|
|
147
|
-
fn coordinator_team_scoped_state(
|
|
148
|
-
workspace: &std::path::Path,
|
|
149
|
-
raw_state: &Value,
|
|
150
|
-
daemon_team_key: Option<&str>,
|
|
151
|
-
) -> Option<Value> {
|
|
152
|
-
let teams = raw_state.get("teams").and_then(Value::as_object)?;
|
|
153
|
-
let active = raw_state
|
|
154
|
-
.get("active_team_key")
|
|
155
|
-
.and_then(Value::as_str)
|
|
156
|
-
.filter(|s| !s.is_empty());
|
|
157
|
-
let selected = daemon_team_key
|
|
158
|
-
.filter(|key| !key.is_empty() && teams.contains_key(*key))
|
|
159
|
-
.or_else(|| active.filter(|key| teams.contains_key(*key)))?;
|
|
160
|
-
crate::state::projection::select_runtime_state(workspace, Some(selected)).ok()
|
|
161
|
-
}
|
|
162
|
-
|
|
163
|
-
fn live_sibling_team_state(
|
|
164
|
-
state: &Value,
|
|
165
|
-
missing_session: &str,
|
|
166
|
-
transport: &dyn crate::transport::Transport,
|
|
167
|
-
) -> Result<Option<(String, Value)>, crate::transport::TransportError> {
|
|
168
|
-
let Some(teams) = state.get("teams").and_then(Value::as_object) else {
|
|
169
|
-
return Ok(None);
|
|
170
|
-
};
|
|
171
|
-
for (team_key, team_state) in teams {
|
|
172
|
-
let status = team_state
|
|
173
|
-
.get("status")
|
|
174
|
-
.and_then(Value::as_str)
|
|
175
|
-
.map(|status| status.to_ascii_lowercase());
|
|
176
|
-
if status.is_some_and(|status| status != "alive" && status != "running") {
|
|
177
|
-
continue;
|
|
178
|
-
}
|
|
179
|
-
let Some(session_name) = team_state
|
|
180
|
-
.get("session_name")
|
|
181
|
-
.and_then(Value::as_str)
|
|
182
|
-
.filter(|session| !session.is_empty() && *session != missing_session)
|
|
183
|
-
else {
|
|
184
|
-
continue;
|
|
185
|
-
};
|
|
186
|
-
let session = crate::transport::SessionName::new(session_name);
|
|
187
|
-
if transport.has_session(&session)? {
|
|
188
|
-
return Ok(Some((
|
|
189
|
-
team_key.clone(),
|
|
190
|
-
crate::state::projection::project_top_level_view(state, team_key),
|
|
191
|
-
)));
|
|
192
|
-
}
|
|
193
|
-
}
|
|
194
|
-
Ok(None)
|
|
195
|
-
}
|
|
196
|
-
|
|
197
|
-
// ===========================================================================
|
|
198
|
-
// Coordinator struct(daemon lifecycle + tick orchestration)
|
|
199
|
-
// ===========================================================================
|
|
200
|
-
|
|
201
|
-
/// tick 末原子 save 失败注入钩(bug-084)。生产装配为 `None`(走真实 `save_runtime_state`);
|
|
202
|
-
/// 测试装配一个返回 `Err` 的闭包,在不触碰真实磁盘的前提下强制 save 失败,断言 degraded
|
|
203
|
-
/// `TickReport` 而非 panic/Err。porter 在 `tick` 的「ATOMIC save」包裹点先查它再落真实 save。
|
|
204
|
-
pub type SaveHook =
|
|
205
|
-
Box<dyn Fn(&WorkspacePath, &Value) -> Result<(), crate::state::StateError> + Send + Sync>;
|
|
206
|
-
|
|
207
|
-
/// tick 链式副作用 ORDER 记录器(测试探针)。porter 在 `tick` 的每个原子调用点 push 一个
|
|
208
|
-
/// 稳定步骤名;测试断言固定序列。生产装配为 `None`(零开销,porter 用 `if let Some(rec)` 守卫)。
|
|
209
|
-
pub type OrderRecorder = std::sync::Arc<std::sync::Mutex<Vec<&'static str>>>;
|
|
210
|
-
|
|
211
|
-
/// per-workspace coordinator。daemon 主循环 + 单次 tick 编排的宿主。
|
|
212
|
-
///
|
|
213
|
-
/// provider 调用一律经注入的 `ProviderAdapter` trait object(MUST-NOT-13:**绝不**依赖任何
|
|
214
|
-
/// provider client crate;测试 mock 断言调用计数 = 0)。transport 探测经注入的 `Transport` trait。
|
|
215
|
-
pub struct Coordinator {
|
|
216
|
-
workspace: WorkspacePath,
|
|
217
|
-
/// provider adapter 解析器(`get_provider_registry` 等价;经 trait 注入,可 mock)。
|
|
218
|
-
#[allow(dead_code)]
|
|
219
|
-
provider_registry: Box<dyn ProviderRegistry>,
|
|
220
|
-
/// transport 控制面(tmux session 存活探测等;经 trait 注入,可 mock)。
|
|
221
|
-
#[allow(dead_code)]
|
|
222
|
-
transport: Box<dyn crate::transport::Transport>,
|
|
223
|
-
/// Daemon CLI-selected team key. When present, tick projection uses this
|
|
224
|
-
/// instead of a stale root active_team_key.
|
|
225
|
-
#[allow(dead_code)]
|
|
226
|
-
daemon_team_key: Option<String>,
|
|
227
|
-
/// bug-084 save 注入钩。`None` ⇔ 真实 `state::save_runtime_state`。
|
|
228
|
-
#[allow(dead_code)]
|
|
229
|
-
save_hook: Option<SaveHook>,
|
|
230
|
-
#[cfg(test)]
|
|
231
|
-
before_save_hook: Option<SaveHook>,
|
|
232
|
-
/// tick 副作用 ORDER 探针。`None` ⇔ 不记录(生产)。
|
|
233
|
-
#[allow(dead_code)]
|
|
234
|
-
order_recorder: Option<OrderRecorder>,
|
|
235
|
-
}
|
|
236
|
-
|
|
237
|
-
impl Coordinator {
|
|
238
|
-
/// 构造(注入 provider registry + transport)。spawn 出的 daemon 在 `run` 前装配它。
|
|
239
|
-
/// ---
|
|
240
|
-
/// purpose: 装配一个 Coordinator——注入 provider registry 与 transport,两者都是 trait object 以便替换与 mock
|
|
241
|
-
/// params:
|
|
242
|
-
/// workspace: 本实例服务的 workspace 根
|
|
243
|
-
/// provider_registry: provider adapter 解析器;provider 一律经它取,绝不直连 provider client crate
|
|
244
|
-
/// transport: 会话/pane 控制面,用于存活探测与注入
|
|
245
|
-
/// returns: 未绑定 team_key、无 save 钩子、无顺序探针的实例
|
|
246
|
-
/// ---
|
|
247
|
-
pub fn new(
|
|
248
|
-
workspace: WorkspacePath,
|
|
249
|
-
provider_registry: Box<dyn ProviderRegistry>,
|
|
250
|
-
transport: Box<dyn crate::transport::Transport>,
|
|
251
|
-
) -> Self {
|
|
252
|
-
Self {
|
|
253
|
-
workspace,
|
|
254
|
-
provider_registry,
|
|
255
|
-
transport,
|
|
256
|
-
daemon_team_key: None,
|
|
257
|
-
save_hook: None,
|
|
258
|
-
#[cfg(test)]
|
|
259
|
-
before_save_hook: None,
|
|
260
|
-
order_recorder: None,
|
|
261
|
-
}
|
|
262
|
-
}
|
|
263
|
-
|
|
264
|
-
/// ---
|
|
265
|
-
/// purpose: 绑定 daemon 侧选定的 team_key,让 tick 的状态投影用它而不是可能过期的根 active_team_key
|
|
266
|
-
/// params:
|
|
267
|
-
/// team_key: 目标 team;None 或空串都视为未绑定,tick 会回落到 state 推导
|
|
268
|
-
/// returns: 绑定后的自身
|
|
269
|
-
/// ---
|
|
270
|
-
pub(crate) fn with_team_key(mut self, team_key: Option<String>) -> Self {
|
|
271
|
-
self.daemon_team_key = team_key.filter(|key| !key.is_empty());
|
|
272
|
-
self
|
|
273
|
-
}
|
|
274
|
-
|
|
275
|
-
/// 测试装配:直接构出 `Coordinator`(不经 `new` 的 `unimplemented!()`),注入 mock
|
|
276
|
-
/// transport + mock provider registry + 可选 save 注入钩 + ORDER 探针。**纯 test-support
|
|
277
|
-
/// 脚手架**(真实 impl,非 `unimplemented!()`):它只装配字段,不执行任何 daemon 逻辑;
|
|
278
|
-
/// tick/health/start/stop 仍是 `unimplemented!()` 生产体,因此调它们的契约仍 RED。
|
|
279
|
-
/// ---
|
|
280
|
-
/// purpose: 测试装配口——直接构出实例并注入 save 钩子与步骤顺序探针
|
|
281
|
-
/// params:
|
|
282
|
-
/// workspace: workspace 根
|
|
283
|
-
/// provider_registry: 可 mock 的 provider adapter 解析器
|
|
284
|
-
/// transport: 可 mock 的传输控制面
|
|
285
|
-
/// save_hook: 替换真实持久化,用于制造持久化失败
|
|
286
|
-
/// order_recorder: 记录 tick 内各步骤名,用于断言固定顺序
|
|
287
|
-
/// returns: 只装配字段的实例,本函数自身不执行任何 daemon 逻辑
|
|
288
|
-
/// cfg: test
|
|
289
|
-
/// ---
|
|
290
|
-
#[cfg(test)]
|
|
291
|
-
pub(crate) fn for_test(
|
|
292
|
-
workspace: WorkspacePath,
|
|
293
|
-
provider_registry: Box<dyn ProviderRegistry>,
|
|
294
|
-
transport: Box<dyn crate::transport::Transport>,
|
|
295
|
-
save_hook: Option<SaveHook>,
|
|
296
|
-
order_recorder: Option<OrderRecorder>,
|
|
297
|
-
) -> Self {
|
|
298
|
-
Self {
|
|
299
|
-
workspace,
|
|
300
|
-
provider_registry,
|
|
301
|
-
transport,
|
|
302
|
-
daemon_team_key: None,
|
|
303
|
-
save_hook,
|
|
304
|
-
before_save_hook: None,
|
|
305
|
-
order_recorder,
|
|
306
|
-
}
|
|
307
|
-
}
|
|
308
|
-
|
|
309
|
-
/// A test barrier before the real repository save; never replaces the save.
|
|
310
|
-
#[cfg(test)]
|
|
311
|
-
pub(crate) fn with_before_save_hook(mut self, hook: SaveHook) -> Self {
|
|
312
|
-
self.before_save_hook = Some(hook);
|
|
313
|
-
self
|
|
314
|
-
}
|
|
315
|
-
|
|
316
|
-
// ── tick 编排(lifecycle.py:250-385)──────────────────────────────────────
|
|
317
|
-
|
|
318
|
-
/// 单次 tick(`coordinator_tick`,`lifecycle.py:250`)。固定顺序串 step 8-11 原子:
|
|
319
|
-
/// load state → tmux session 存活门(missing → stop:true)→ capture missing sessions →
|
|
320
|
-
/// refresh runtime statuses → provider startup/runtime prompts → sync health →
|
|
321
|
-
/// deliver pending → fire scheduled → detect stuck → idle/takeover ping(should_ping 时一条)→
|
|
322
|
-
/// deadlock/compaction/drift/api-error 只读探测 → **原子 save state(bug-084 唯一包裹点)** →
|
|
323
|
-
/// collect results → prune dedupe log。
|
|
324
|
-
///
|
|
325
|
-
/// §10:daemon-path 返 `Result<TickReport, TickError>`。bug-084:save 失败返
|
|
326
|
-
/// degraded `Ok(TickReport{ok:false, reason:PersistenceDegraded, persisted:Some(false)})`
|
|
327
|
-
/// (**不**走 `Err`,主循环不 catch degraded,只 catch `Err` 退避)。
|
|
328
|
-
/// §84:无 pending obligation + event 时**绝不**注入探索性 prompt。
|
|
329
|
-
///
|
|
330
|
-
/// PORTER:在 ATOMIC save 包裹点先查 `self.save_hook`(`Some` → 用它代替真实
|
|
331
|
-
/// `state::save_runtime_state`,bug-084 测试注入失败);在每个 step8-11 原子调用点
|
|
332
|
-
/// `if let Some(rec) = &self.order_recorder { rec.lock()...push(STEP_NAME) }`(tick
|
|
333
|
-
/// 副作用 ORDER 测试断言固定序列)。生产两者均 `None`,零开销。
|
|
334
|
-
/// ---
|
|
335
|
-
/// purpose: 跑一轮 tick——按固定顺序把各步骤串起来,只投递既定义务,绝不凭空注入
|
|
336
|
-
/// returns: TickReport。传输会话不在了 → stop=true 且 reason=TmuxSessionMissing,主循环据此退出;持久化失败 → ok=false、reason=PersistenceDegraded、persisted=Some(false),但仍是 Ok;正常 → ok=true 并带本轮投递、到期事件、回收结果等清单。注意 ok=true 只说明主干走完,监测类步骤失败已被降级吞掉,真相只在事件日志里
|
|
337
|
-
/// errors: 加载状态、打开 message store、传输探测或事件写入这些主干操作失败时返回 TickError,由主循环 catch 后退避
|
|
338
|
-
/// ---
|
|
339
|
-
pub fn tick(&self) -> Result<TickReport, TickError> {
|
|
340
|
-
self.record_step(TickStepGroup::SessionGate, "load_state");
|
|
341
|
-
let raw_state = crate::state::persist::load_runtime_state(self.workspace.as_path())?;
|
|
342
|
-
// Issue 2 (Round 3b gate review §6): when the runtime carries
|
|
343
|
-
// `active_team_key` AND `teams.<key>` exists, project the team-scoped
|
|
344
|
-
// view (session_name / agents / leader_receiver come from the team's
|
|
345
|
-
// nested object) instead of the raw top-level state. Otherwise the
|
|
346
|
-
// coordinator would probe the wrong tmux session (e.g. stale
|
|
347
|
-
// `session_name=team-tmp` while the real team session is
|
|
348
|
-
// `team-prerelease-040-round3b`) and emit `coordinator.session_missing`
|
|
349
|
-
// even though the right session is alive. Fall back to raw state when
|
|
350
|
-
// no team scope can be derived (legacy single-team workspaces).
|
|
351
|
-
let mut state = coordinator_team_scoped_state(
|
|
352
|
-
self.workspace.as_path(),
|
|
353
|
-
&raw_state,
|
|
354
|
-
self.daemon_team_key.as_deref(),
|
|
355
|
-
)
|
|
356
|
-
.unwrap_or(raw_state);
|
|
357
|
-
let before_observation = state.clone();
|
|
358
|
-
let store = crate::message_store::MessageStore::open(self.workspace.as_path())?;
|
|
359
|
-
let event_log = EventLog::new(self.workspace.as_path());
|
|
360
|
-
increment_coordinator_tick_iteration_count(&self.workspace);
|
|
361
|
-
|
|
362
|
-
#[cfg(unix)]
|
|
363
|
-
if let Err(error) =
|
|
364
|
-
crate::contract_runtime::pump::tick(self.workspace.as_path(), &mut state)
|
|
365
|
-
{
|
|
366
|
-
let _ = event_log.write(
|
|
367
|
-
"contract.tick.failed",
|
|
368
|
-
serde_json::json!({"error":error.to_string()}),
|
|
369
|
-
);
|
|
370
|
-
}
|
|
371
|
-
#[cfg(unix)]
|
|
372
|
-
let contract_only = crate::contract_runtime::framework::only_contract(&state);
|
|
373
|
-
#[cfg(not(unix))]
|
|
374
|
-
let contract_only = false;
|
|
375
|
-
self.record_step(TickStepGroup::SessionGate, "tmux_session_gate");
|
|
376
|
-
if let Some(session_name) = state
|
|
377
|
-
.get("session_name")
|
|
378
|
-
.and_then(Value::as_str)
|
|
379
|
-
.filter(|s| !s.is_empty())
|
|
380
|
-
.map(str::to_owned)
|
|
381
|
-
{
|
|
382
|
-
let session = crate::transport::SessionName::new(&session_name);
|
|
383
|
-
if !contract_only && !self.transport.has_session(&session)? {
|
|
384
|
-
event_log.write(
|
|
385
|
-
"coordinator.session_missing",
|
|
386
|
-
serde_json::json!({"session": session_name}),
|
|
387
|
-
)?;
|
|
388
|
-
if let Some((sibling_team, sibling_state)) =
|
|
389
|
-
live_sibling_team_state(&state, &session_name, self.transport.as_ref())?
|
|
390
|
-
{
|
|
391
|
-
event_log.write(
|
|
392
|
-
"coordinator.session_missing_sibling_alive",
|
|
393
|
-
serde_json::json!({
|
|
394
|
-
"session": session_name,
|
|
395
|
-
"sibling_team": sibling_team,
|
|
396
|
-
"action": "continue_with_sibling_team",
|
|
397
|
-
}),
|
|
398
|
-
)?;
|
|
399
|
-
state = sibling_state;
|
|
400
|
-
} else {
|
|
401
|
-
notify_session_missing(
|
|
402
|
-
self.workspace.as_path(),
|
|
403
|
-
&state,
|
|
404
|
-
&event_log,
|
|
405
|
-
&session_name,
|
|
406
|
-
)?;
|
|
407
|
-
return Ok(empty_tick_report(
|
|
408
|
-
false,
|
|
409
|
-
true,
|
|
410
|
-
Some(TickStopReason::TmuxSessionMissing),
|
|
411
|
-
None,
|
|
412
|
-
));
|
|
413
|
-
}
|
|
414
|
-
}
|
|
415
|
-
}
|
|
416
|
-
|
|
417
|
-
self.record_step(TickStepGroup::SessionGate, "capture_missing");
|
|
418
|
-
if let Err(error) = self.capture_missing_sessions(&mut state, &event_log) {
|
|
419
|
-
let _ = event_log.write(
|
|
420
|
-
"coordinator.tick.capture_missing_failed",
|
|
421
|
-
serde_json::json!({"error": error.to_string()}),
|
|
422
|
-
);
|
|
423
|
-
}
|
|
424
|
-
|
|
425
|
-
// Slice 1 energy gate: one pane snapshot per tick feeds probe eligibility,
|
|
426
|
-
// health sync, and abnormal-exit detection. Missing panes are filtered
|
|
427
|
-
// before any capture-pane call.
|
|
428
|
-
let pane_snapshot = self.transport.list_targets().unwrap_or_default();
|
|
429
|
-
let window_snapshot = state
|
|
430
|
-
.get("session_name")
|
|
431
|
-
.and_then(Value::as_str)
|
|
432
|
-
.map(crate::transport::SessionName::new)
|
|
433
|
-
.and_then(|session| self.transport.list_windows(&session).ok())
|
|
434
|
-
.unwrap_or_default();
|
|
435
|
-
let has_work_obligation = tick_has_work_obligation(&store);
|
|
436
|
-
|
|
437
|
-
self.record_step(TickStepGroup::HealthSync, "refresh_statuses");
|
|
438
|
-
// TODO(spine slice 2b): split lightweight runtime status refresh from health sync.
|
|
439
|
-
|
|
440
|
-
self.record_step(TickStepGroup::RuntimePrompts, "startup_prompts");
|
|
441
|
-
self.handle_startup_prompts(&mut state, &event_log, &pane_snapshot, &window_snapshot);
|
|
442
|
-
|
|
443
|
-
// #229 step2-retry: once an agent's `startup_prompts` flipped to `handled`
|
|
444
|
-
// (this tick OR earlier), `queued_until_trust` messages for that recipient
|
|
445
|
-
// become deliverable. Reset them to `accepted` so the existing
|
|
446
|
-
// `deliver_pending` step below picks them up on THIS tick. Reuses the
|
|
447
|
-
// delivery pipeline; no new injector. Best-effort logging on inner errors.
|
|
448
|
-
if let Err(error) =
|
|
449
|
-
self.requeue_trust_retries_for_handled_agents(&state, &store, &event_log)
|
|
450
|
-
{
|
|
451
|
-
let _ = event_log.write(
|
|
452
|
-
"messaging.trust_retry_requeue_failed",
|
|
453
|
-
serde_json::json!({"error": error.to_string()}),
|
|
454
|
-
);
|
|
455
|
-
}
|
|
456
|
-
|
|
457
|
-
// B-4 / 036b N36 三路可用 — 监测步(runtime_prompts / sync_health /
|
|
458
|
-
// detect_abnormal_exits)失败必须降级+continue,**不能**用 `?` 中断 tick,
|
|
459
|
-
// 否则 deliver_pending(下行投递主干)够不到,消息卡 accepted。
|
|
460
|
-
// bug-084 哲学 + A-6 同族:每步独立 try,失败写 `coordinator.tick.<step>_failed`
|
|
461
|
-
// 事件后继续走下一步;tick 本身仍返 Ok。
|
|
462
|
-
self.record_step(TickStepGroup::RuntimePrompts, "runtime_prompts");
|
|
463
|
-
if let Err(error) = self.handle_runtime_approval_prompts(
|
|
464
|
-
&mut state,
|
|
465
|
-
&event_log,
|
|
466
|
-
&pane_snapshot,
|
|
467
|
-
&window_snapshot,
|
|
468
|
-
) {
|
|
469
|
-
let _ = event_log.write(
|
|
470
|
-
"coordinator.tick.runtime_prompts_failed",
|
|
471
|
-
serde_json::json!({"error": error.to_string()}),
|
|
472
|
-
);
|
|
473
|
-
}
|
|
474
|
-
|
|
475
|
-
self.record_step(TickStepGroup::HealthSync, "sync_health");
|
|
476
|
-
// P5 (C-P5-1, N3): ONE pane snapshot per tick, shared by sync_health and the
|
|
477
|
-
// abnormal-exit pass (same-tick reuse only — the snapshot does not outlive
|
|
478
|
-
// this tick; every tick re-reads).
|
|
479
|
-
let captures_by_agent = match self.sync_agent_health(
|
|
480
|
-
&mut state,
|
|
481
|
-
&store,
|
|
482
|
-
&event_log,
|
|
483
|
-
&pane_snapshot,
|
|
484
|
-
&window_snapshot,
|
|
485
|
-
has_work_obligation,
|
|
486
|
-
) {
|
|
487
|
-
Ok(captures) => captures,
|
|
488
|
-
Err(error) => {
|
|
489
|
-
let _ = event_log.write(
|
|
490
|
-
"coordinator.tick.sync_health_failed",
|
|
491
|
-
serde_json::json!({"error": error.to_string()}),
|
|
492
|
-
);
|
|
493
|
-
BTreeMap::new()
|
|
494
|
-
}
|
|
495
|
-
};
|
|
496
|
-
if let Err(error) = crate::coordinator::steps::abnormal::detect_abnormal_exits(
|
|
497
|
-
self.workspace.as_path(),
|
|
498
|
-
self.transport.as_ref(),
|
|
499
|
-
&mut state,
|
|
500
|
-
&event_log,
|
|
501
|
-
&pane_snapshot,
|
|
502
|
-
) {
|
|
503
|
-
let _ = event_log.write(
|
|
504
|
-
"coordinator.tick.detect_abnormal_failed",
|
|
505
|
-
serde_json::json!({"error": error.to_string()}),
|
|
506
|
-
);
|
|
507
|
-
}
|
|
508
|
-
|
|
509
|
-
self.record_step(TickStepGroup::Delivery, "deliver_pending");
|
|
510
|
-
let delivered = crate::messaging::deliver_pending_messages(
|
|
511
|
-
self.workspace.as_path(),
|
|
512
|
-
&state,
|
|
513
|
-
self.transport.as_ref(),
|
|
514
|
-
&event_log,
|
|
515
|
-
)?
|
|
516
|
-
.into_iter()
|
|
517
|
-
.map(|message_id| DeliveredMessage { message_id })
|
|
518
|
-
.collect::<Vec<_>>();
|
|
519
|
-
|
|
520
|
-
self.record_step(TickStepGroup::Delivery, "fire_scheduled");
|
|
521
|
-
let scheduled = crate::messaging::fire_due_scheduled_events(
|
|
522
|
-
self.workspace.as_path(),
|
|
523
|
-
&store,
|
|
524
|
-
self.transport.as_ref(),
|
|
525
|
-
&event_log,
|
|
526
|
-
)?
|
|
527
|
-
.into_iter()
|
|
528
|
-
.map(|id| FiredScheduledEvent { id })
|
|
529
|
-
.collect::<Vec<_>>();
|
|
530
|
-
|
|
531
|
-
// #236 nag_removal (N35): the time/state-inferred idle/stuck/deadlock nag
|
|
532
|
-
// generators are no longer wired in. Step labels stay (tick ORDER lock) but
|
|
533
|
-
// each body is a strict "produce no nag output" — empty `stuck`, empty
|
|
534
|
-
// `idle_alerts`, empty `deadlock_alerts`. Delivery primitives
|
|
535
|
-
// (deliver_pending / fire_scheduled / collect_results) above and below this
|
|
536
|
-
// block continue to flow unchanged. `_state` / `_store` here are intentionally
|
|
537
|
-
// unused (the lookups they powered were nag inputs only).
|
|
538
|
-
self.record_step(TickStepGroup::Abnormal, "detect_stuck");
|
|
539
|
-
let stuck: Vec<AgentId> = Vec::new();
|
|
540
|
-
self.record_step(TickStepGroup::Abnormal, "record_unknown_idle");
|
|
541
|
-
self.record_step(TickStepGroup::Abnormal, "evaluate_takeover");
|
|
542
|
-
let idle_alerts: Vec<IdleAlert> = Vec::new();
|
|
543
|
-
self.record_step(TickStepGroup::Abnormal, "detect_deadlocks");
|
|
544
|
-
let deadlock_alerts: Vec<DeadlockAlert> = Vec::new();
|
|
545
|
-
let _ = &store;
|
|
546
|
-
|
|
547
|
-
self.record_step(TickStepGroup::Abnormal, "detect_compaction");
|
|
548
|
-
self.record_step(TickStepGroup::Abnormal, "detect_drift");
|
|
549
|
-
self.record_step(TickStepGroup::Abnormal, "detect_api_errors");
|
|
550
|
-
let leader_capture = self.capture_leader_receiver(&state);
|
|
551
|
-
let observations = runtime_observation::observe(
|
|
552
|
-
self.workspace.as_path(),
|
|
553
|
-
&mut state,
|
|
554
|
-
captures_by_agent,
|
|
555
|
-
leader_capture,
|
|
556
|
-
);
|
|
557
|
-
let mut collections = TickCollections {
|
|
558
|
-
delivered,
|
|
559
|
-
scheduled,
|
|
560
|
-
stuck,
|
|
561
|
-
idle_alerts,
|
|
562
|
-
deadlock_alerts,
|
|
563
|
-
compaction: observations.compaction,
|
|
564
|
-
session_drift: observations.session_drift,
|
|
565
|
-
api_errors: observations.api_errors,
|
|
566
|
-
results: Vec::new(),
|
|
567
|
-
};
|
|
568
|
-
|
|
569
|
-
self.record_step(TickStepGroup::Persist, "atomic_save");
|
|
570
|
-
#[cfg(test)]
|
|
571
|
-
if let Some(hook) = &self.before_save_hook {
|
|
572
|
-
hook(&self.workspace, &state)?;
|
|
573
|
-
}
|
|
574
|
-
let saved = match &self.save_hook {
|
|
575
|
-
Some(hook) => hook(&self.workspace, &state),
|
|
576
|
-
None => {
|
|
577
|
-
// Keep the sampled topology guard, but commit only this tick's
|
|
578
|
-
// observations onto latest so task/note writers cannot be lost.
|
|
579
|
-
let team_key = crate::state::projection::team_state_key(&state);
|
|
580
|
-
crate::state::repository::StateRepository::new(self.workspace.as_path())
|
|
581
|
-
.commit_observations(
|
|
582
|
-
crate::state::repository::StateWriteIntent::CoordinatorTick {
|
|
583
|
-
team_key: team_key.as_str(),
|
|
584
|
-
},
|
|
585
|
-
&before_observation,
|
|
586
|
-
&state,
|
|
587
|
-
)
|
|
588
|
-
}
|
|
589
|
-
};
|
|
590
|
-
if saved.is_err() {
|
|
591
|
-
return Ok(base_tick_report(
|
|
592
|
-
false,
|
|
593
|
-
false,
|
|
594
|
-
Some(TickStopReason::PersistenceDegraded),
|
|
595
|
-
Some(false),
|
|
596
|
-
collections,
|
|
597
|
-
));
|
|
598
|
-
}
|
|
599
|
-
self.record_step(TickStepGroup::Persist, "prune_dedupe_log");
|
|
600
|
-
Ok(base_tick_report(true, false, None, Some(true), collections))
|
|
601
|
-
}
|
|
602
|
-
|
|
603
|
-
// #236 nag_removal (N35): the framework-synthesized idle/stuck/deadlock nag
|
|
604
|
-
// generators (record_unknown_idle_nodes / evaluate_takeover / build_idle_nodes)
|
|
605
|
-
// were removed by design. Delivery primitives still flow through the rest of
|
|
606
|
-
// the tick body unchanged.
|
|
607
|
-
|
|
608
|
-
fn capture_missing_sessions(
|
|
609
|
-
&self,
|
|
610
|
-
state: &mut Value,
|
|
611
|
-
event_log: &EventLog,
|
|
612
|
-
) -> Result<(), TickError> {
|
|
613
|
-
let report = crate::session_capture::capture_missing_provider_sessions_once(
|
|
614
|
-
state,
|
|
615
|
-
&mut |provider| self.provider_registry.adapter_for(provider),
|
|
616
|
-
true,
|
|
617
|
-
0,
|
|
618
|
-
)?;
|
|
619
|
-
// RM-039-STAT-001 third-round fix (architect verdict 2026-06-22):
|
|
620
|
-
// when capture_missing_sessions assigns a new rollout_path to an
|
|
621
|
-
// agent, clear that agent's `coordinator_idle_capture_next_at` so
|
|
622
|
-
// the JSONL classifier is not gated by stale warm-idle suppression
|
|
623
|
-
// on the SAME tick that the rollout becomes readable. Without this,
|
|
624
|
-
// an agent that was IDLE before attribution would carry the old
|
|
625
|
-
// suppression timestamp into sync_agent_health and only the
|
|
626
|
-
// expensive pane Tail(40) path would observe its working state.
|
|
627
|
-
if !report.assigned.is_empty() {
|
|
628
|
-
if let Some(agents) = state
|
|
629
|
-
.get_mut("agents")
|
|
630
|
-
.and_then(serde_json::Value::as_object_mut)
|
|
631
|
-
{
|
|
632
|
-
for assigned_agent_id in &report.assigned {
|
|
633
|
-
if let Some(agent_obj) = agents
|
|
634
|
-
.get_mut(assigned_agent_id)
|
|
635
|
-
.and_then(serde_json::Value::as_object_mut)
|
|
636
|
-
{
|
|
637
|
-
agent_obj.remove("coordinator_idle_capture_next_at");
|
|
638
|
-
}
|
|
639
|
-
}
|
|
640
|
-
}
|
|
641
|
-
}
|
|
642
|
-
for failure in report.capture_failures {
|
|
643
|
-
event_log.write(
|
|
644
|
-
"coordinator.tick.capture_missing_failed",
|
|
645
|
-
serde_json::json!({
|
|
646
|
-
"agent_id": failure.agent_id,
|
|
647
|
-
"error": failure.error,
|
|
648
|
-
}),
|
|
649
|
-
)?;
|
|
650
|
-
}
|
|
651
|
-
// Bug 2 (0.3.32): emit `session.captured` for every newly-assigned
|
|
652
|
-
// capture so event-log repair (recover_resume_session_from_events)
|
|
653
|
-
// can replay durable session truth even if state was lost. Architect
|
|
654
|
-
// §4 fix #5.
|
|
655
|
-
for agent_id in &report.assigned {
|
|
656
|
-
let agent = state.get("agents").and_then(|a| a.get(agent_id.as_str()));
|
|
657
|
-
if let Some(agent) = agent {
|
|
658
|
-
event_log.write(
|
|
659
|
-
"session.captured",
|
|
660
|
-
serde_json::json!({
|
|
661
|
-
"agent_id": agent_id,
|
|
662
|
-
"provider": agent.get("provider").and_then(Value::as_str),
|
|
663
|
-
"session_id": agent.get("session_id").and_then(Value::as_str),
|
|
664
|
-
"rollout_path": agent.get("rollout_path").and_then(Value::as_str),
|
|
665
|
-
"captured_via": agent.get("captured_via").and_then(Value::as_str),
|
|
666
|
-
"attribution_confidence": agent.get("attribution_confidence").and_then(Value::as_str),
|
|
667
|
-
"spawn_cwd": agent.get("spawn_cwd").and_then(Value::as_str),
|
|
668
|
-
"spawned_at": agent.get("spawned_at").and_then(Value::as_str),
|
|
669
|
-
}),
|
|
670
|
-
)?;
|
|
671
|
-
}
|
|
672
|
-
}
|
|
673
|
-
// Bug 2 (0.3.32): enrich `attribution_ambiguous` event with diagnostic
|
|
674
|
-
// payload — provider, spawned_at, candidate_count, and reason code.
|
|
675
|
-
// Pre-fix the event carried only agent_id + spawn_cwd, leaving
|
|
676
|
-
// operators unable to tell whether the failure was zero candidates,
|
|
677
|
-
// multiple same-cwd candidates, stale pre-spawn candidates, or
|
|
678
|
-
// expected-id miss. Architect §4 fix #4.
|
|
679
|
-
// 0.4.6 Stage 3: emit throttled `provider.session.transcript_missing`
|
|
680
|
-
// events for agents that transitioned into the transcript_missing
|
|
681
|
-
// capture_state on this pass. The capture pass only flags a
|
|
682
|
-
// transition (prev_state != next_state), so the event fires at
|
|
683
|
-
// most once per (agent_id, spawn_epoch) per state change.
|
|
684
|
-
for missing in &report.transcript_missing {
|
|
685
|
-
let agent = state
|
|
686
|
-
.get("agents")
|
|
687
|
-
.and_then(|a| a.get(missing.agent_id.as_str()));
|
|
688
|
-
let pane_id = agent.and_then(|a| a.get("pane_id")).and_then(Value::as_str);
|
|
689
|
-
let pane_pid = agent
|
|
690
|
-
.and_then(|a| a.get("pane_pid"))
|
|
691
|
-
.and_then(Value::as_u64);
|
|
692
|
-
let provider = agent
|
|
693
|
-
.and_then(|a| a.get("provider"))
|
|
694
|
-
.and_then(Value::as_str);
|
|
695
|
-
let session_id_in_argv = agent
|
|
696
|
-
.and_then(|a| a.get("session_id_in_argv"))
|
|
697
|
-
.and_then(Value::as_str);
|
|
698
|
-
let first_send_at = agent
|
|
699
|
-
.and_then(|a| a.get("first_send_at"))
|
|
700
|
-
.and_then(Value::as_str);
|
|
701
|
-
let last_result_at = agent
|
|
702
|
-
.and_then(|a| a.get("last_result_at"))
|
|
703
|
-
.and_then(Value::as_str);
|
|
704
|
-
let last_pane_output_at = agent
|
|
705
|
-
.and_then(|a| a.get("last_pane_output_at"))
|
|
706
|
-
.and_then(Value::as_str);
|
|
707
|
-
event_log.write(
|
|
708
|
-
"provider.session.transcript_missing",
|
|
709
|
-
serde_json::json!({
|
|
710
|
-
"agent_id": missing.agent_id,
|
|
711
|
-
"provider": provider,
|
|
712
|
-
"pane_id": pane_id,
|
|
713
|
-
"pane_pid": pane_pid,
|
|
714
|
-
"spawn_epoch": missing.spawn_epoch,
|
|
715
|
-
"expected_session_id": missing.expected_session_id,
|
|
716
|
-
"session_id_in_argv": session_id_in_argv,
|
|
717
|
-
"spawn_cwd": missing.spawn_cwd,
|
|
718
|
-
"candidate_count": missing.candidate_count,
|
|
719
|
-
"first_send_at": first_send_at,
|
|
720
|
-
"last_result_at": last_result_at,
|
|
721
|
-
"last_pane_output_at": last_pane_output_at,
|
|
722
|
-
}),
|
|
723
|
-
)?;
|
|
724
|
-
}
|
|
725
|
-
for mismatch in &report.identity_mismatches {
|
|
726
|
-
event_log.write(
|
|
727
|
-
"provider.session.identity_mismatch",
|
|
728
|
-
serde_json::json!({
|
|
729
|
-
"agent_id": mismatch.agent_id,
|
|
730
|
-
"expected_agent_id": mismatch.expected_agent_id,
|
|
731
|
-
"embedded_agent_id": mismatch.embedded_agent_id,
|
|
732
|
-
"session_id": mismatch.session_id,
|
|
733
|
-
"rollout_path": mismatch.rollout_path,
|
|
734
|
-
"spawn_cwd": mismatch.spawn_cwd,
|
|
735
|
-
}),
|
|
736
|
-
)?;
|
|
737
|
-
}
|
|
738
|
-
for ambiguous in report.ambiguous {
|
|
739
|
-
let candidate_count = report
|
|
740
|
-
.candidate_count_by_agent
|
|
741
|
-
.get(&ambiguous.agent_id)
|
|
742
|
-
.copied()
|
|
743
|
-
.unwrap_or(0);
|
|
744
|
-
let agent = state
|
|
745
|
-
.get("agents")
|
|
746
|
-
.and_then(|a| a.get(ambiguous.agent_id.as_str()));
|
|
747
|
-
let provider = agent
|
|
748
|
-
.and_then(|a| a.get("provider"))
|
|
749
|
-
.and_then(Value::as_str);
|
|
750
|
-
let spawned_at = agent
|
|
751
|
-
.and_then(|a| a.get("spawned_at"))
|
|
752
|
-
.and_then(Value::as_str);
|
|
753
|
-
// Bounded reason codes (architect §4 fix #4 enumeration):
|
|
754
|
-
// "zero_candidates" — no candidate after capture scan
|
|
755
|
-
// "multiple_post_spawn_candidates" — >1 candidates, none uniquely safe
|
|
756
|
-
// "claimed_collision" — only candidate already claimed by sibling
|
|
757
|
-
let reason = if candidate_count == 0 {
|
|
758
|
-
"zero_candidates"
|
|
759
|
-
} else if candidate_count > 1 {
|
|
760
|
-
"multiple_post_spawn_candidates"
|
|
761
|
-
} else {
|
|
762
|
-
"claimed_collision"
|
|
763
|
-
};
|
|
764
|
-
event_log.write(
|
|
765
|
-
"provider.session.attribution_ambiguous",
|
|
766
|
-
serde_json::json!({
|
|
767
|
-
"agent_id": ambiguous.agent_id,
|
|
768
|
-
"spawn_cwd": ambiguous.spawn_cwd,
|
|
769
|
-
"provider": provider,
|
|
770
|
-
"spawned_at": spawned_at,
|
|
771
|
-
"candidate_count": candidate_count,
|
|
772
|
-
"reason": reason,
|
|
773
|
-
}),
|
|
774
|
-
)?;
|
|
775
|
-
}
|
|
776
|
-
Ok(())
|
|
777
|
-
}
|
|
778
|
-
|
|
779
|
-
fn sync_agent_health(
|
|
780
|
-
&self,
|
|
781
|
-
state: &mut Value,
|
|
782
|
-
store: &crate::message_store::MessageStore,
|
|
783
|
-
event_log: &EventLog,
|
|
784
|
-
pane_infos: &[crate::transport::PaneInfo],
|
|
785
|
-
window_snapshot: &[crate::transport::WindowName],
|
|
786
|
-
has_work_obligation: bool,
|
|
787
|
-
) -> Result<BTreeMap<AgentId, CapturedRuntimeFact>, TickError> {
|
|
788
|
-
let mut captures = BTreeMap::new();
|
|
789
|
-
let snapshot = state.clone();
|
|
790
|
-
let team = crate::state::projection::team_state_key(&snapshot);
|
|
791
|
-
let team_key = Some(crate::model::ids::TeamKey::new(team.clone()));
|
|
792
|
-
let session_name = state
|
|
793
|
-
.get("session_name")
|
|
794
|
-
.and_then(Value::as_str)
|
|
795
|
-
.map(str::to_string);
|
|
796
|
-
// B-4 / 036b N36 三路可用 — sync_health 内 per-agent capture 失败本就降级
|
|
797
|
-
// (写 coordinator.agent_capture_failed 后 continue),不打断 deliver_pending
|
|
798
|
-
// 主干。但 contract 要求一条【tick 级】可观测的 step-failed 信号 —
|
|
799
|
-
// sync_health 失败一旦发生就在末尾 emit `coordinator.tick.sync_health_failed`
|
|
800
|
-
// (含 "tick" + "_failed" 双串),避免 silent。
|
|
801
|
-
let mut had_capture_failure = false;
|
|
802
|
-
// P5 (C-P5-2): one list-windows per SESSION per tick — memoized across the
|
|
803
|
-
// agent loop instead of one fork per agent.
|
|
804
|
-
let mut windows_by_session: BTreeMap<
|
|
805
|
-
String,
|
|
806
|
-
Result<Vec<crate::transport::WindowName>, String>,
|
|
807
|
-
> = BTreeMap::new();
|
|
808
|
-
if let Some(session_name) = session_name.as_deref() {
|
|
809
|
-
windows_by_session.insert(session_name.to_string(), Ok(window_snapshot.to_vec()));
|
|
810
|
-
}
|
|
811
|
-
let Some(agents) = state.get_mut("agents").and_then(Value::as_object_mut) else {
|
|
812
|
-
return Ok(captures);
|
|
813
|
-
};
|
|
814
|
-
for (agent_id, agent) in agents {
|
|
815
|
-
if !agent_probe_base_eligible(agent) {
|
|
816
|
-
continue;
|
|
817
|
-
}
|
|
818
|
-
// RM-039-STAT-001 third-round fix (architect verdict 2026-06-22):
|
|
819
|
-
// try the provider JSONL classifier FIRST, BEFORE any pane-target
|
|
820
|
-
// / window checks and BEFORE warm_idle_capture_suppressed. The
|
|
821
|
-
// historical chain ran JSONL only inside the pane-capture branch,
|
|
822
|
-
// so a stale pane-fallback `idle_prompt` could land first
|
|
823
|
-
// (before session attribution completed), then
|
|
824
|
-
// `warm_idle_capture_suppressed` would gate subsequent ticks for
|
|
825
|
-
// IDLE_HEALTH_CAPTURE_INTERVAL_SECS even after JSONL gained a
|
|
826
|
-
// `task_started` record + the worker process was alive. Now: if
|
|
827
|
-
// the JSONL classifier returns a definite activity (not Unknown),
|
|
828
|
-
// we write activity + agent_health here and bypass the warm-idle
|
|
829
|
-
// suppression for the rest of this iteration. Pane capture still
|
|
830
|
-
// runs so the CapturedRuntimeFact map stays populated for
|
|
831
|
-
// downstream runtime detectors; only the activity classify
|
|
832
|
-
// happens once.
|
|
833
|
-
let jsonl_first = jsonl_activity_for_agent(agent);
|
|
834
|
-
if let Some(activity) = jsonl_first.as_ref() {
|
|
835
|
-
remember_idle_capture_schedule(agent, activity);
|
|
836
|
-
write_activity(agent, activity, false);
|
|
837
|
-
let last_output_at_now = agent
|
|
838
|
-
.get("last_output_at")
|
|
839
|
-
.and_then(Value::as_str)
|
|
840
|
-
.map(str::to_string);
|
|
841
|
-
write_agent_health(
|
|
842
|
-
store,
|
|
843
|
-
&team,
|
|
844
|
-
agent_id,
|
|
845
|
-
agent,
|
|
846
|
-
activity,
|
|
847
|
-
last_output_at_now.as_deref(),
|
|
848
|
-
)?;
|
|
849
|
-
}
|
|
850
|
-
let Some((session, window, target)) =
|
|
851
|
-
capture_window_target(agent, session_name.as_deref())
|
|
852
|
-
else {
|
|
853
|
-
continue;
|
|
854
|
-
};
|
|
855
|
-
let windows = match windows_by_session
|
|
856
|
-
.entry(session.as_str().to_string())
|
|
857
|
-
.or_insert_with(|| {
|
|
858
|
-
self.transport
|
|
859
|
-
.list_windows(&session)
|
|
860
|
-
.map_err(|error| error.to_string())
|
|
861
|
-
}) {
|
|
862
|
-
Ok(windows) => windows.clone(),
|
|
863
|
-
Err(error) => {
|
|
864
|
-
had_capture_failure = true;
|
|
865
|
-
event_log.write(
|
|
866
|
-
"coordinator.agent_capture_failed",
|
|
867
|
-
serde_json::json!({
|
|
868
|
-
"agent_id": agent_id,
|
|
869
|
-
"target": format!("{target:?}"),
|
|
870
|
-
"error": error.clone(),
|
|
871
|
-
}),
|
|
872
|
-
)?;
|
|
873
|
-
continue;
|
|
874
|
-
}
|
|
875
|
-
};
|
|
876
|
-
if !windows.iter().any(|known| known == &window) {
|
|
877
|
-
// Missing is proof only inside a non-empty same-session
|
|
878
|
-
// inventory (a live peer exists). An empty snapshot is an
|
|
879
|
-
// unavailable transport, not proof every worker died.
|
|
880
|
-
if windows.is_empty() {
|
|
881
|
-
continue;
|
|
882
|
-
}
|
|
883
|
-
if let Some(agent_obj) = agent.as_object_mut() {
|
|
884
|
-
agent_obj.insert("status".to_string(), serde_json::json!("stopped"));
|
|
885
|
-
agent_obj.insert("worker_state".to_string(), serde_json::json!("DEAD"));
|
|
886
|
-
agent_obj.insert("stale".to_string(), serde_json::json!(true));
|
|
887
|
-
agent_obj.insert("stale_reason".to_string(), serde_json::json!("pane_dead"));
|
|
888
|
-
}
|
|
889
|
-
let conn = crate::db::schema::open_db(store.db_path()).map_err(|error| {
|
|
890
|
-
TickError::MessageStore(crate::message_store::MessageStoreError::Db(error))
|
|
891
|
-
})?;
|
|
892
|
-
conn.execute(
|
|
893
|
-
"insert into agent_health(owner_team_id, agent_id, status, updated_at) \
|
|
894
|
-
values (?1, ?2, 'DEAD', ?3) \
|
|
895
|
-
on conflict(owner_team_id, agent_id) do update set \
|
|
896
|
-
status='DEAD', updated_at=excluded.updated_at",
|
|
897
|
-
rusqlite::params![
|
|
898
|
-
team_key.as_ref().map(|key| key.as_str()),
|
|
899
|
-
agent_id,
|
|
900
|
-
chrono::Utc::now().to_rfc3339()
|
|
901
|
-
],
|
|
902
|
-
)
|
|
903
|
-
.map_err(|error| {
|
|
904
|
-
TickError::MessageStore(crate::message_store::MessageStoreError::Sqlite(error))
|
|
905
|
-
})?;
|
|
906
|
-
let _ = event_log.write(
|
|
907
|
-
"coordinator.agent_pane_dead",
|
|
908
|
-
serde_json::json!({
|
|
909
|
-
"agent_id": agent_id,
|
|
910
|
-
"target": format!("{target:?}"),
|
|
911
|
-
"stale_reason": "pane_dead",
|
|
912
|
-
}),
|
|
913
|
-
);
|
|
914
|
-
continue;
|
|
915
|
-
}
|
|
916
|
-
// Warm-idle suppression still gates pane fallback ONLY. When
|
|
917
|
-
// JSONL above produced a definite activity, we already wrote it,
|
|
918
|
-
// so warm-idle gating no longer matters for the activity path —
|
|
919
|
-
// but we still want CapturedRuntimeFact populated for downstream
|
|
920
|
-
// detectors, so fall through into the pane capture block below
|
|
921
|
-
// (skipping the pane-classify activity write at the bottom).
|
|
922
|
-
if jsonl_first.is_none() && warm_idle_capture_suppressed(agent, has_work_obligation) {
|
|
923
|
-
continue;
|
|
924
|
-
}
|
|
925
|
-
let captured = match self
|
|
926
|
-
.transport
|
|
927
|
-
.capture(&target, crate::transport::CaptureRange::Tail(40))
|
|
928
|
-
{
|
|
929
|
-
Ok(captured) => captured,
|
|
930
|
-
Err(error) => {
|
|
931
|
-
had_capture_failure = true;
|
|
932
|
-
event_log.write(
|
|
933
|
-
"coordinator.agent_capture_failed",
|
|
934
|
-
serde_json::json!({
|
|
935
|
-
"agent_id": agent_id,
|
|
936
|
-
"target": format!("{target:?}"),
|
|
937
|
-
"error": error.to_string(),
|
|
938
|
-
}),
|
|
939
|
-
)?;
|
|
940
|
-
continue;
|
|
941
|
-
}
|
|
942
|
-
};
|
|
943
|
-
let pane_in_mode = agent
|
|
944
|
-
.get("pane_in_mode")
|
|
945
|
-
.and_then(Value::as_bool)
|
|
946
|
-
.unwrap_or(false);
|
|
947
|
-
let current_command = agent
|
|
948
|
-
.get("pane_current_command")
|
|
949
|
-
.or_else(|| agent.get("current_command"))
|
|
950
|
-
.and_then(Value::as_str)
|
|
951
|
-
.map(str::to_string);
|
|
952
|
-
// Python approvals/status.py:68-73 — last_output_at advances ONLY when the
|
|
953
|
-
// scrollback sha256 digest changed (last_output_hash gate), and it is
|
|
954
|
-
// refreshed BEFORE classification (the classifier sees the updated value).
|
|
955
|
-
// A non-empty but UNCHANGED capture must not dirty the state every tick
|
|
956
|
-
// (P3 umbrella: steady second tick is a zero state write).
|
|
957
|
-
//
|
|
958
|
-
// 0.5.32 follow-up (`.team/artifacts/0532-r1-real-fail-triage.md` §6):
|
|
959
|
-
// fresh-cohort startup-output guard. After the restart clear helper wipes
|
|
960
|
-
// `activity`/`worker_state`/`last_output_at`/`last_output_hash`, the first
|
|
961
|
-
// non-empty pane capture (fake worker's READY line, provider startup
|
|
962
|
-
// banner) is trivially different from the missing digest. If we let it
|
|
963
|
-
// set `last_output_at=now`, classify_agent_activity falls through to
|
|
964
|
-
// `recent_provider_output => Working` even though no structural signal
|
|
965
|
-
// exists — that's the R1 real-machine false busy.
|
|
966
|
-
//
|
|
967
|
-
// Narrow gate: fresh cohort (no activity/worker_state/last_output_at
|
|
968
|
-
// /last_output_hash/current turn/task fields), no work obligation
|
|
969
|
-
// arming a real send, and no structural prompt/working signal in the
|
|
970
|
-
// capture → still update `last_output_hash` (dedup) but do NOT write
|
|
971
|
-
// `last_output_at`. The classifier then returns Uncertain=UNKNOWN,
|
|
972
|
-
// which is the honest post-clear observation. Preserves recent_provider_output
|
|
973
|
-
// for non-startup normal output (later capture has prior last_output_at
|
|
974
|
-
// or last_output_hash, so `fresh_cohort` false → gate skipped).
|
|
975
|
-
let fresh_cohort = agent_is_fresh_cohort(agent);
|
|
976
|
-
let has_structural_signal = latest_pane_signal_is_structural(&captured.text);
|
|
977
|
-
let output_advanced =
|
|
978
|
-
!captured.text.is_empty() && scrollback_digest_advanced(agent, &captured.text);
|
|
979
|
-
let suppress_last_output_at =
|
|
980
|
-
output_advanced && fresh_cohort && !has_work_obligation && !has_structural_signal;
|
|
981
|
-
if output_advanced && !suppress_last_output_at {
|
|
982
|
-
if let Some(agent_obj) = agent.as_object_mut() {
|
|
983
|
-
agent_obj.insert(
|
|
984
|
-
"last_output_at".to_string(),
|
|
985
|
-
serde_json::json!(chrono::Utc::now().to_rfc3339()),
|
|
986
|
-
);
|
|
987
|
-
}
|
|
988
|
-
}
|
|
989
|
-
let last_output_at_now = agent
|
|
990
|
-
.get("last_output_at")
|
|
991
|
-
.and_then(Value::as_str)
|
|
992
|
-
.map(str::to_string);
|
|
993
|
-
// E47 (0.3.24 P0, idle/busy 假阳): the provider JSONL classifier
|
|
994
|
-
// already ran at the top of this iteration. If it returned a
|
|
995
|
-
// definite fact (Some), activity + agent_health were written
|
|
996
|
-
// there; we MUST NOT classify-and-overwrite here, otherwise the
|
|
997
|
-
// pane fallback would flip the JSONL truth back to a stale
|
|
998
|
-
// idle_prompt. Only run the pane fallback when JSONL returned
|
|
999
|
-
// None (no readable rollout, TurnState::Unknown, or unparseable
|
|
1000
|
-
// log). RM-039-STAT-001 third-round fix (architect verdict
|
|
1001
|
-
// 2026-06-22).
|
|
1002
|
-
let last_output_at = last_output_at_now;
|
|
1003
|
-
if jsonl_first.is_none() {
|
|
1004
|
-
let activity = crate::messaging::classify_agent_activity(
|
|
1005
|
-
&snapshot,
|
|
1006
|
-
&captured.text,
|
|
1007
|
-
pane_in_mode,
|
|
1008
|
-
current_command.as_deref(),
|
|
1009
|
-
last_output_at.as_deref(),
|
|
1010
|
-
);
|
|
1011
|
-
remember_idle_capture_schedule(agent, &activity);
|
|
1012
|
-
write_activity(agent, &activity, false);
|
|
1013
|
-
write_agent_health(
|
|
1014
|
-
store,
|
|
1015
|
-
&team,
|
|
1016
|
-
agent_id,
|
|
1017
|
-
agent,
|
|
1018
|
-
&activity,
|
|
1019
|
-
last_output_at.as_deref(),
|
|
1020
|
-
)?;
|
|
1021
|
-
}
|
|
1022
|
-
let pane_info = matching_capture_pane_info(agent, &session, &window, pane_infos);
|
|
1023
|
-
let pane_id = pane_info
|
|
1024
|
-
.as_ref()
|
|
1025
|
-
.map(|info| info.pane_id.clone())
|
|
1026
|
-
.or_else(|| agent_pane_id(agent));
|
|
1027
|
-
let rollout_path = agent_rollout_path(agent).map(crate::provider::RolloutPath::new);
|
|
1028
|
-
captures.insert(
|
|
1029
|
-
AgentId::new(agent_id.clone()),
|
|
1030
|
-
CapturedRuntimeFact {
|
|
1031
|
-
team_key: team_key.clone(),
|
|
1032
|
-
agent_id: AgentId::new(agent_id.clone()),
|
|
1033
|
-
provider: agent
|
|
1034
|
-
.get("provider")
|
|
1035
|
-
.and_then(Value::as_str)
|
|
1036
|
-
.and_then(parse_provider),
|
|
1037
|
-
session_name: Some(session),
|
|
1038
|
-
window: Some(window),
|
|
1039
|
-
pane_id,
|
|
1040
|
-
scrollback_tail: captured.text,
|
|
1041
|
-
pane_info,
|
|
1042
|
-
agent_state_snapshot: agent.clone(),
|
|
1043
|
-
stored_session_id: agent
|
|
1044
|
-
.get("session_id")
|
|
1045
|
-
.and_then(Value::as_str)
|
|
1046
|
-
.map(str::to_string),
|
|
1047
|
-
last_output_at,
|
|
1048
|
-
rollout_path,
|
|
1049
|
-
process_liveness:
|
|
1050
|
-
crate::coordinator::steps::abnormal::explicit_process_liveness(agent),
|
|
1051
|
-
},
|
|
1052
|
-
);
|
|
1053
|
-
}
|
|
1054
|
-
// B-4 step-level signal:若本 tick 有任一 capture 失败,emit
|
|
1055
|
-
// `coordinator.tick.sync_health_failed`(含 "tick" + "_failed")让 contract
|
|
1056
|
-
// 可观测,deliver_pending 主干不受影响。
|
|
1057
|
-
if had_capture_failure {
|
|
1058
|
-
let _ = event_log.write(
|
|
1059
|
-
"coordinator.tick.sync_health_failed",
|
|
1060
|
-
serde_json::json!({"step": "sync_health", "degraded": true}),
|
|
1061
|
-
);
|
|
1062
|
-
}
|
|
1063
|
-
Ok(captures)
|
|
1064
|
-
}
|
|
1065
|
-
|
|
1066
|
-
fn capture_leader_receiver(
|
|
1067
|
-
&self,
|
|
1068
|
-
state: &Value,
|
|
1069
|
-
) -> Option<runtime_observation::LeaderCaptureFact> {
|
|
1070
|
-
let receiver = state.get("leader_receiver")?.clone();
|
|
1071
|
-
let pane_id = receiver
|
|
1072
|
-
.get("pane_id")
|
|
1073
|
-
.and_then(Value::as_str)
|
|
1074
|
-
.filter(|pane_id| !pane_id.is_empty())
|
|
1075
|
-
.map(crate::transport::PaneId::new)?;
|
|
1076
|
-
let captured = self
|
|
1077
|
-
.transport
|
|
1078
|
-
.capture(
|
|
1079
|
-
&crate::transport::Target::Pane(pane_id.clone()),
|
|
1080
|
-
crate::transport::CaptureRange::Tail(40),
|
|
1081
|
-
)
|
|
1082
|
-
.ok()?;
|
|
1083
|
-
Some(runtime_observation::LeaderCaptureFact {
|
|
1084
|
-
team_key: Some(crate::model::ids::TeamKey::new(
|
|
1085
|
-
crate::state::projection::team_state_key(state),
|
|
1086
|
-
)),
|
|
1087
|
-
leader_receiver: Some(receiver),
|
|
1088
|
-
pane_id: Some(pane_id),
|
|
1089
|
-
scrollback_tail: captured.text,
|
|
1090
|
-
})
|
|
1091
|
-
}
|
|
1092
|
-
|
|
1093
|
-
fn handle_startup_prompts(
|
|
1094
|
-
&self,
|
|
1095
|
-
state: &mut Value,
|
|
1096
|
-
event_log: &EventLog,
|
|
1097
|
-
pane_infos: &[crate::transport::PaneInfo],
|
|
1098
|
-
windows: &[crate::transport::WindowName],
|
|
1099
|
-
) {
|
|
1100
|
-
let session_name = state
|
|
1101
|
-
.get("session_name")
|
|
1102
|
-
.and_then(Value::as_str)
|
|
1103
|
-
.map(str::to_string);
|
|
1104
|
-
let Some(agents) = state.get_mut("agents").and_then(Value::as_object_mut) else {
|
|
1105
|
-
return;
|
|
1106
|
-
};
|
|
1107
|
-
for (agent_id, agent) in agents {
|
|
1108
|
-
if !agent_probe_base_eligible(agent) {
|
|
1109
|
-
continue;
|
|
1110
|
-
}
|
|
1111
|
-
// #229 step1-idem: once trust is auto-answered, the row carries
|
|
1112
|
-
// `startup_prompts = "handled"` (or "complete"). Both are terminal for
|
|
1113
|
-
// this tick loop — repeated ticks must not re-classify, re-send Enter,
|
|
1114
|
-
// or re-emit `startup_prompt_handled`. Treating "handled" the same as
|
|
1115
|
-
// "complete" makes the observable artifact exactly-once across ticks.
|
|
1116
|
-
if agent
|
|
1117
|
-
.get("startup_prompts")
|
|
1118
|
-
.and_then(Value::as_str)
|
|
1119
|
-
.is_some_and(|status| matches!(status, "handled" | "complete"))
|
|
1120
|
-
{
|
|
1121
|
-
continue;
|
|
1122
|
-
}
|
|
1123
|
-
let Some(provider) = agent
|
|
1124
|
-
.get("provider")
|
|
1125
|
-
.and_then(Value::as_str)
|
|
1126
|
-
.and_then(parse_provider)
|
|
1127
|
-
else {
|
|
1128
|
-
continue;
|
|
1129
|
-
};
|
|
1130
|
-
let Some((session, window, target)) =
|
|
1131
|
-
capture_window_target(agent, session_name.as_deref())
|
|
1132
|
-
else {
|
|
1133
|
-
continue;
|
|
1134
|
-
};
|
|
1135
|
-
if !agent_window_present(agent, &session, &window, pane_infos, windows) {
|
|
1136
|
-
continue;
|
|
1137
|
-
}
|
|
1138
|
-
clear_startup_probe_disable_if_epoch_changed(agent);
|
|
1139
|
-
if startup_probe_disabled_for_epoch(agent) {
|
|
1140
|
-
continue;
|
|
1141
|
-
}
|
|
1142
|
-
if !startup_probe_within_grace(agent) {
|
|
1143
|
-
disable_startup_probe_for_epoch(agent);
|
|
1144
|
-
continue;
|
|
1145
|
-
}
|
|
1146
|
-
let adapter = self.provider_registry.adapter_for(provider);
|
|
1147
|
-
let outcome =
|
|
1148
|
-
adapter.handle_startup_prompts_outcome(self.transport.as_ref(), &target, 1, 0.0);
|
|
1149
|
-
// swallow batch 2 ② (A1): an unobservable pane is a surfaced failure, not a
|
|
1150
|
-
// silent "no prompts" — the agent's startup_prompts state stays un-handled.
|
|
1151
|
-
if let Some(error) = &outcome.capture_error {
|
|
1152
|
-
let _ = event_log.write(
|
|
1153
|
-
"provider.startup_prompt_failed",
|
|
1154
|
-
serde_json::json!({
|
|
1155
|
-
"agent_id": agent_id,
|
|
1156
|
-
"target": format!("{target:?}"),
|
|
1157
|
-
"error": error,
|
|
1158
|
-
}),
|
|
1159
|
-
);
|
|
1160
|
-
}
|
|
1161
|
-
let handled = outcome.handled;
|
|
1162
|
-
if handled.is_empty() {
|
|
1163
|
-
continue;
|
|
1164
|
-
}
|
|
1165
|
-
let handled_payload = serde_json::Value::Array(
|
|
1166
|
-
handled
|
|
1167
|
-
.into_iter()
|
|
1168
|
-
.map(|prompt| {
|
|
1169
|
-
serde_json::json!({
|
|
1170
|
-
"prompt": prompt.prompt,
|
|
1171
|
-
"action": prompt.action,
|
|
1172
|
-
})
|
|
1173
|
-
})
|
|
1174
|
-
.collect(),
|
|
1175
|
-
);
|
|
1176
|
-
// #229 step1 observability: emit `startup_prompt_handled` so the trust
|
|
1177
|
-
// answer is observable in events.jsonl (was silent state-write only).
|
|
1178
|
-
// Best-effort — state write below is the source of truth.
|
|
1179
|
-
let _ = event_log.write(
|
|
1180
|
-
"startup_prompt_handled",
|
|
1181
|
-
serde_json::json!({
|
|
1182
|
-
"agent_id": agent_id,
|
|
1183
|
-
"provider": provider,
|
|
1184
|
-
"handled": handled_payload.clone(),
|
|
1185
|
-
}),
|
|
1186
|
-
);
|
|
1187
|
-
let Some(agent_obj) = agent.as_object_mut() else {
|
|
1188
|
-
continue;
|
|
1189
|
-
};
|
|
1190
|
-
agent_obj.insert("startup_prompts".to_string(), serde_json::json!("handled"));
|
|
1191
|
-
agent_obj.insert(
|
|
1192
|
-
"startup_prompt_status".to_string(),
|
|
1193
|
-
serde_json::json!("handled"),
|
|
1194
|
-
);
|
|
1195
|
-
agent_obj.remove("startup_prompt_probe_disabled_at");
|
|
1196
|
-
agent_obj.insert("startup_prompt_handled".to_string(), handled_payload);
|
|
1197
|
-
}
|
|
1198
|
-
}
|
|
1199
|
-
|
|
1200
|
-
/// #229 step2-retry: after `handle_startup_prompts` flips an agent's status to
|
|
1201
|
-
/// `handled`/`complete`, scan `messages` for `queued_until_trust` rows targeting
|
|
1202
|
-
/// that recipient and flip them back to `accepted` so this same tick's
|
|
1203
|
-
/// `deliver_pending` replays them. Same row, same message_id, same pipeline.
|
|
1204
|
-
fn requeue_trust_retries_for_handled_agents(
|
|
1205
|
-
&self,
|
|
1206
|
-
state: &Value,
|
|
1207
|
-
store: &crate::message_store::MessageStore,
|
|
1208
|
-
event_log: &EventLog,
|
|
1209
|
-
) -> Result<(), crate::message_store::MessageStoreError> {
|
|
1210
|
-
let Some(agents) = state.get("agents").and_then(Value::as_object) else {
|
|
1211
|
-
return Ok(());
|
|
1212
|
-
};
|
|
1213
|
-
let handled_recipients: Vec<&str> = agents
|
|
1214
|
-
.iter()
|
|
1215
|
-
.filter(|(_, agent)| {
|
|
1216
|
-
agent
|
|
1217
|
-
.get("startup_prompts")
|
|
1218
|
-
.and_then(Value::as_str)
|
|
1219
|
-
.is_some_and(|status| matches!(status, "handled" | "complete"))
|
|
1220
|
-
})
|
|
1221
|
-
.map(|(id, _)| id.as_str())
|
|
1222
|
-
.collect();
|
|
1223
|
-
if handled_recipients.is_empty() {
|
|
1224
|
-
return Ok(());
|
|
1225
|
-
}
|
|
1226
|
-
let conn = crate::db::schema::open_db(store.db_path())?;
|
|
1227
|
-
let mut stmt = conn.prepare(
|
|
1228
|
-
"select message_id, recipient from messages where status = 'queued_until_trust'",
|
|
1229
|
-
)?;
|
|
1230
|
-
let rows = stmt
|
|
1231
|
-
.query_map([], |row| {
|
|
1232
|
-
Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
|
|
1233
|
-
})?
|
|
1234
|
-
.collect::<Result<Vec<(String, String)>, _>>()?;
|
|
1235
|
-
for (message_id, recipient) in rows {
|
|
1236
|
-
if !handled_recipients.iter().any(|r| *r == recipient.as_str()) {
|
|
1237
|
-
continue;
|
|
1238
|
-
}
|
|
1239
|
-
store.mark(&message_id, "accepted", None)?;
|
|
1240
|
-
let _ = event_log.write(
|
|
1241
|
-
"messaging.trust_retry_requeued",
|
|
1242
|
-
serde_json::json!({
|
|
1243
|
-
"message_id": message_id,
|
|
1244
|
-
"recipient": recipient,
|
|
1245
|
-
"reason": "startup_prompt_handled",
|
|
1246
|
-
}),
|
|
1247
|
-
);
|
|
1248
|
-
}
|
|
1249
|
-
Ok(())
|
|
1250
|
-
}
|
|
1251
|
-
|
|
1252
|
-
fn handle_runtime_approval_prompts(
|
|
1253
|
-
&self,
|
|
1254
|
-
state: &mut Value,
|
|
1255
|
-
event_log: &EventLog,
|
|
1256
|
-
pane_infos: &[crate::transport::PaneInfo],
|
|
1257
|
-
windows: &[crate::transport::WindowName],
|
|
1258
|
-
) -> Result<(), TickError> {
|
|
1259
|
-
let snapshot = state.clone();
|
|
1260
|
-
let team = crate::state::projection::team_state_key(&snapshot);
|
|
1261
|
-
let session_name = snapshot
|
|
1262
|
-
.get("session_name")
|
|
1263
|
-
.and_then(Value::as_str)
|
|
1264
|
-
.map(str::to_string);
|
|
1265
|
-
let mut dedup_updates = Vec::new();
|
|
1266
|
-
{
|
|
1267
|
-
let Some(agents) = state.get_mut("agents").and_then(Value::as_object_mut) else {
|
|
1268
|
-
return Ok(());
|
|
1269
|
-
};
|
|
1270
|
-
for (agent_id, agent) in agents {
|
|
1271
|
-
if !agent_probe_base_eligible(agent) {
|
|
1272
|
-
clear_awaiting_human_confirm(agent);
|
|
1273
|
-
continue;
|
|
1274
|
-
}
|
|
1275
|
-
let approval_policy = runtime_approval_policy_from_agent(agent);
|
|
1276
|
-
let auto_answer_allowed = approval_policy.auto_answer_allowed();
|
|
1277
|
-
let Some(target) = runtime_approval_target(agent, session_name.as_deref()) else {
|
|
1278
|
-
clear_awaiting_human_confirm(agent);
|
|
1279
|
-
dedup_updates.push(AwaitingDedupUpdate::Clear {
|
|
1280
|
-
team: team.clone(),
|
|
1281
|
-
agent_id: agent_id.to_string(),
|
|
1282
|
-
});
|
|
1283
|
-
continue;
|
|
1284
|
-
};
|
|
1285
|
-
let target_present = capture_window_target(agent, session_name.as_deref())
|
|
1286
|
-
.map_or_else(
|
|
1287
|
-
|| runtime_approval_target_present(&target, pane_infos, windows),
|
|
1288
|
-
|(session, window, _)| {
|
|
1289
|
-
agent_window_present(agent, &session, &window, pane_infos, windows)
|
|
1290
|
-
},
|
|
1291
|
-
);
|
|
1292
|
-
if !target_present {
|
|
1293
|
-
continue;
|
|
1294
|
-
}
|
|
1295
|
-
if runtime_approval_backoff_active(agent) {
|
|
1296
|
-
continue;
|
|
1297
|
-
}
|
|
1298
|
-
let captured = match self
|
|
1299
|
-
.transport
|
|
1300
|
-
.capture(&target, crate::transport::CaptureRange::Tail(80))
|
|
1301
|
-
{
|
|
1302
|
-
Ok(captured) => captured,
|
|
1303
|
-
Err(error) => {
|
|
1304
|
-
event_log.write(
|
|
1305
|
-
"runtime_approval.capture_failed",
|
|
1306
|
-
serde_json::json!({
|
|
1307
|
-
"agent_id": agent_id,
|
|
1308
|
-
"target": format!("{target:?}"),
|
|
1309
|
-
"error": error.to_string(),
|
|
1310
|
-
}),
|
|
1311
|
-
)?;
|
|
1312
|
-
remember_runtime_approval_backoff(agent);
|
|
1313
|
-
continue;
|
|
1314
|
-
}
|
|
1315
|
-
};
|
|
1316
|
-
let Some(prompt) = extract_approval_prompt(agent_id, &captured.text) else {
|
|
1317
|
-
clear_awaiting_human_confirm(agent);
|
|
1318
|
-
dedup_updates.push(AwaitingDedupUpdate::Clear {
|
|
1319
|
-
team: team.clone(),
|
|
1320
|
-
agent_id: agent_id.to_string(),
|
|
1321
|
-
});
|
|
1322
|
-
remember_runtime_approval_backoff(agent);
|
|
1323
|
-
continue;
|
|
1324
|
-
};
|
|
1325
|
-
clear_runtime_approval_backoff(agent);
|
|
1326
|
-
match runtime_approval_decision(&prompt, auto_answer_allowed) {
|
|
1327
|
-
RuntimeApprovalDecision::AutoApprove => {
|
|
1328
|
-
clear_awaiting_human_confirm(agent);
|
|
1329
|
-
dedup_updates.push(AwaitingDedupUpdate::Clear {
|
|
1330
|
-
team: team.clone(),
|
|
1331
|
-
agent_id: agent_id.to_string(),
|
|
1332
|
-
});
|
|
1333
|
-
let choice = choose_internal_mcp_approval_choice(&prompt);
|
|
1334
|
-
let keys = approval_choice_keys(&prompt, &captured.text, &choice)
|
|
1335
|
-
.into_iter()
|
|
1336
|
-
.filter_map(runtime_approval_key)
|
|
1337
|
-
.collect::<Vec<_>>();
|
|
1338
|
-
// A-6 / Python approvals/runtime_prompts.py:21-43: prompts are handled
|
|
1339
|
-
// per-agent with run_cmd(check=False) — one agent's tmux failure must
|
|
1340
|
-
// not abort the whole tick for the rest.
|
|
1341
|
-
if let Err(error) = self.transport.send_keys(&target, &keys) {
|
|
1342
|
-
event_log.write(
|
|
1343
|
-
"runtime_approval.send_keys_failed",
|
|
1344
|
-
serde_json::json!({
|
|
1345
|
-
"agent_id": agent_id,
|
|
1346
|
-
"target": format!("{target:?}"),
|
|
1347
|
-
"tool": prompt.tool,
|
|
1348
|
-
"error": error.to_string(),
|
|
1349
|
-
}),
|
|
1350
|
-
)?;
|
|
1351
|
-
continue;
|
|
1352
|
-
}
|
|
1353
|
-
let after = self
|
|
1354
|
-
.transport
|
|
1355
|
-
.capture(&target, crate::transport::CaptureRange::Tail(80))
|
|
1356
|
-
.ok()
|
|
1357
|
-
.and_then(|capture| extract_approval_prompt(agent_id, &capture.text));
|
|
1358
|
-
let cleared = after.as_ref().is_none_or(|after| {
|
|
1359
|
-
after.prompt != prompt.prompt || after.tool != prompt.tool
|
|
1360
|
-
});
|
|
1361
|
-
event_log.write(
|
|
1362
|
-
"runtime_approval.auto_approved",
|
|
1363
|
-
serde_json::json!({
|
|
1364
|
-
"agent_id": agent_id,
|
|
1365
|
-
"server": prompt.server.as_deref(),
|
|
1366
|
-
"tool": prompt.tool.as_deref(),
|
|
1367
|
-
"choice": choice,
|
|
1368
|
-
"cleared": cleared,
|
|
1369
|
-
"policy_source": approval_policy.source,
|
|
1370
|
-
"inherited": approval_policy.inherited,
|
|
1371
|
-
"explicit_yes_confirmed": approval_policy.explicit_yes_confirmed,
|
|
1372
|
-
"worker_capability_above_leader": approval_policy.worker_capability_above_leader,
|
|
1373
|
-
}),
|
|
1374
|
-
)?;
|
|
1375
|
-
event_log.write(
|
|
1376
|
-
"mcp.tool.auto_approved",
|
|
1377
|
-
serde_json::json!({
|
|
1378
|
-
"agent_id": agent_id,
|
|
1379
|
-
"server": prompt.server.as_deref(),
|
|
1380
|
-
"tool": prompt.tool.as_deref(),
|
|
1381
|
-
"choice": choice,
|
|
1382
|
-
"cleared": cleared,
|
|
1383
|
-
"inherit_reason": approval_policy.inherit_reason(),
|
|
1384
|
-
"bypass_source": approval_policy.source,
|
|
1385
|
-
"provider": approval_policy.provider,
|
|
1386
|
-
"flag": approval_policy.flag,
|
|
1387
|
-
"inherited": approval_policy.inherited,
|
|
1388
|
-
"explicit_yes_confirmed": approval_policy.explicit_yes_confirmed,
|
|
1389
|
-
"worker_capability_above_leader": approval_policy.worker_capability_above_leader,
|
|
1390
|
-
}),
|
|
1391
|
-
)?;
|
|
1392
|
-
}
|
|
1393
|
-
RuntimeApprovalDecision::AwaitingHumanConfirm => {
|
|
1394
|
-
let Some(reason) =
|
|
1395
|
-
awaiting_human_confirm_reason(&prompt, auto_answer_allowed)
|
|
1396
|
-
else {
|
|
1397
|
-
continue;
|
|
1398
|
-
};
|
|
1399
|
-
let fact = awaiting_human_confirm_fact(&team, agent_id, &prompt, reason);
|
|
1400
|
-
let previous = agent
|
|
1401
|
-
.get("awaiting_human_confirm")
|
|
1402
|
-
.and_then(|v| v.get("fingerprint"))
|
|
1403
|
-
.and_then(Value::as_str);
|
|
1404
|
-
if previous == Some(fact.fingerprint.as_str())
|
|
1405
|
-
|| state_awaiting_human_confirm_fingerprint(&snapshot, &team, agent_id)
|
|
1406
|
-
.as_deref()
|
|
1407
|
-
== Some(fact.fingerprint.as_str())
|
|
1408
|
-
{
|
|
1409
|
-
remember_awaiting_human_confirm(agent, &fact);
|
|
1410
|
-
continue;
|
|
1411
|
-
}
|
|
1412
|
-
let notification = awaiting_human_confirm_payload(agent, &fact);
|
|
1413
|
-
let content = notification.to_string();
|
|
1414
|
-
let _ = crate::messaging::send_to_leader_receiver(
|
|
1415
|
-
self.workspace.as_path(),
|
|
1416
|
-
&snapshot,
|
|
1417
|
-
"leader",
|
|
1418
|
-
&content,
|
|
1419
|
-
None,
|
|
1420
|
-
agent_id,
|
|
1421
|
-
false,
|
|
1422
|
-
Some(&fact.dedupe_key),
|
|
1423
|
-
event_log,
|
|
1424
|
-
)?;
|
|
1425
|
-
event_log.write("worker.awaiting_human_confirm", notification)?;
|
|
1426
|
-
remember_awaiting_human_confirm(agent, &fact);
|
|
1427
|
-
dedup_updates.push(AwaitingDedupUpdate::Remember(fact.clone()));
|
|
1428
|
-
match reason {
|
|
1429
|
-
"tool_not_allowlisted" => {
|
|
1430
|
-
event_log.write(
|
|
1431
|
-
"runtime_approval.tool_not_allowlisted",
|
|
1432
|
-
serde_json::json!({
|
|
1433
|
-
"agent_id": agent_id,
|
|
1434
|
-
"tool": prompt.tool,
|
|
1435
|
-
"kind": prompt.kind,
|
|
1436
|
-
"prompt": prompt.prompt,
|
|
1437
|
-
}),
|
|
1438
|
-
)?;
|
|
1439
|
-
}
|
|
1440
|
-
"leader_restricted" | "leader_safety_restricted" => {
|
|
1441
|
-
event_log.write(
|
|
1442
|
-
"runtime_approval.blocked_by_leader_safety",
|
|
1443
|
-
serde_json::json!({
|
|
1444
|
-
"agent_id": agent_id,
|
|
1445
|
-
"tool": prompt.tool,
|
|
1446
|
-
"command": prompt.command,
|
|
1447
|
-
"kind": prompt.kind,
|
|
1448
|
-
"prompt": prompt.prompt,
|
|
1449
|
-
}),
|
|
1450
|
-
)?;
|
|
1451
|
-
}
|
|
1452
|
-
"command_approval_requires_human" => {
|
|
1453
|
-
event_log.write(
|
|
1454
|
-
"runtime_approval.command_approval_requires_human",
|
|
1455
|
-
serde_json::json!({
|
|
1456
|
-
"agent_id": agent_id,
|
|
1457
|
-
"tool": prompt.tool,
|
|
1458
|
-
"command": prompt.command,
|
|
1459
|
-
"kind": prompt.kind,
|
|
1460
|
-
"prompt": prompt.prompt,
|
|
1461
|
-
}),
|
|
1462
|
-
)?;
|
|
1463
|
-
}
|
|
1464
|
-
_ => {}
|
|
1465
|
-
}
|
|
1466
|
-
}
|
|
1467
|
-
RuntimeApprovalDecision::Ignore => {
|
|
1468
|
-
clear_awaiting_human_confirm(agent);
|
|
1469
|
-
dedup_updates.push(AwaitingDedupUpdate::Clear {
|
|
1470
|
-
team: team.clone(),
|
|
1471
|
-
agent_id: agent_id.to_string(),
|
|
1472
|
-
});
|
|
1473
|
-
}
|
|
1474
|
-
}
|
|
1475
|
-
}
|
|
1476
|
-
}
|
|
1477
|
-
for update in dedup_updates {
|
|
1478
|
-
match update {
|
|
1479
|
-
AwaitingDedupUpdate::Remember(fact) => {
|
|
1480
|
-
remember_state_awaiting_human_confirm(state, &fact)
|
|
1481
|
-
}
|
|
1482
|
-
AwaitingDedupUpdate::Clear { team, agent_id } => {
|
|
1483
|
-
clear_state_awaiting_human_confirm(state, &team, &agent_id)
|
|
1484
|
-
}
|
|
1485
|
-
}
|
|
1486
|
-
}
|
|
1487
|
-
Ok(())
|
|
1488
|
-
}
|
|
1489
|
-
|
|
1490
|
-
// ── health / start / stop(lifecycle.py:26-247)───────────────────────────
|
|
1491
|
-
|
|
1492
|
-
/// `coordinator_health`(`lifecycle.py:26`)。pid + meta + schema 三合一健康。
|
|
1493
|
-
/// doctor / start 前置调它。`ok = running ∧ metadata_ok ∧ schema_ok`。
|
|
1494
|
-
/// ---
|
|
1495
|
-
/// purpose: 取本 workspace 的 coordinator 健康报告
|
|
1496
|
-
/// returns: 与 health::coordinator_health 完全一致的报告
|
|
1497
|
-
/// errors: 当前实现恒为 Ok;返回类型保留 Result 以便未来健康判定引入可失败路径
|
|
1498
|
-
/// ---
|
|
1499
|
-
pub fn health(&self) -> Result<HealthReport, TickError> {
|
|
1500
|
-
Ok(super::health::coordinator_health(&self.workspace))
|
|
1501
|
-
}
|
|
1502
|
-
|
|
1503
|
-
/// `start_coordinator`(`lifecycle.py:49`)。幂等启动:已健康 no-op;metadata 不兼容先 stop 再起;
|
|
1504
|
-
/// schema 不兼容拒启给 hint;否则 spawn 自身二进制子命令(`team-agent coordinator --workspace ..`,
|
|
1505
|
-
/// Python 是 `python -m team_agent.coordinator`,`lifecycle.py:108`)。
|
|
1506
|
-
/// **schema 兼容门**:三元任一不匹配 → restart_incompatible,**不可静默继续**(card §89)。
|
|
1507
|
-
/// ---
|
|
1508
|
-
/// purpose: 幂等启动本 workspace 的 daemon 子进程
|
|
1509
|
-
/// returns: 与 health::start_coordinator 完全一致的报告;「拒启」体现为 ok=false 而非 Err
|
|
1510
|
-
/// errors: 建目录、开日志、spawn 或写 pid/metadata 失败时返回 StartError
|
|
1511
|
-
/// ---
|
|
1512
|
-
pub fn start(&self) -> Result<StartReport, StartError> {
|
|
1513
|
-
super::health::start_coordinator(&self.workspace)
|
|
1514
|
-
}
|
|
1515
|
-
|
|
1516
|
-
/// `stop_coordinator`(`lifecycle.py:229`)。SIGTERM + 清 pid/meta。pid 非整数 → 清文件返回。
|
|
1517
|
-
/// ---
|
|
1518
|
-
/// purpose: 清掉本 workspace 的 coordinator pid/meta 文件
|
|
1519
|
-
/// returns: pid 文件不存在 → Missing;文件在且内容可解析 → Stopped 并带该 pid;文件在但内容非法 → InvalidPidRemoved
|
|
1520
|
-
/// errors: 删文件失败时返回 StopError
|
|
1521
|
-
/// ---
|
|
1522
|
-
/// 注意:本方法只删文件,**不向 daemon 发任何信号**——与 health::stop_coordinator
|
|
1523
|
-
/// 的「真终止进程」语义不同。调它之后 daemon 仍会继续 tick,而健康探测会报 Missing。
|
|
1524
|
-
pub fn stop(&self) -> Result<StopReport, StopError> {
|
|
1525
|
-
let pid_path = coordinator_pid_path(&self.workspace);
|
|
1526
|
-
if !pid_path.exists() {
|
|
1527
|
-
return Ok(StopReport {
|
|
1528
|
-
ok: true,
|
|
1529
|
-
status: StopOutcome::Missing,
|
|
1530
|
-
pid: None,
|
|
1531
|
-
});
|
|
1532
|
-
}
|
|
1533
|
-
let pid = read_pid_file(&pid_path);
|
|
1534
|
-
remove_file_if_exists(&pid_path)?;
|
|
1535
|
-
remove_file_if_exists(&coordinator_meta_path(&self.workspace))?;
|
|
1536
|
-
match pid {
|
|
1537
|
-
Some(pid) => Ok(StopReport {
|
|
1538
|
-
ok: true,
|
|
1539
|
-
status: StopOutcome::Stopped,
|
|
1540
|
-
pid: Some(pid),
|
|
1541
|
-
}),
|
|
1542
|
-
None => Ok(StopReport {
|
|
1543
|
-
ok: true,
|
|
1544
|
-
status: StopOutcome::InvalidPidRemoved,
|
|
1545
|
-
pid: None,
|
|
1546
|
-
}),
|
|
1547
|
-
}
|
|
1548
|
-
}
|
|
1549
|
-
|
|
1550
|
-
/// `message_store_schema_health`(`lifecycle.py:197`)。DB 列兼容门:区分 pre-init 必需列缺失
|
|
1551
|
-
/// (拒启)vs migratable 列缺失(可迁移)。`doctor --fix-schema` 用其 action hint。
|
|
1552
|
-
/// ---
|
|
1553
|
-
/// purpose: 查本队 message store 的 schema 兼容门
|
|
1554
|
-
/// returns: 与 health::message_store_schema_health 一致;打不开库时 ok=false 并带原文与修复 hint
|
|
1555
|
-
/// ---
|
|
1556
|
-
pub fn schema_health(&self) -> SchemaHealth {
|
|
1557
|
-
// A-8: the gate must inspect the REAL team.db (Python lifecycle.py:197+
|
|
1558
|
-
// message_store_schema_health); a hardcoded ok:true left the card §89
|
|
1559
|
-
// restart_incompatible door permanently dead.
|
|
1560
|
-
super::health::message_store_schema_health(&self.workspace)
|
|
1561
|
-
}
|
|
1562
|
-
|
|
1563
|
-
fn record_step(&self, group: TickStepGroup, step: &'static str) {
|
|
1564
|
-
debug_assert!(TickStepGroup::ordered().contains(&group));
|
|
1565
|
-
if let Some(recorder) = &self.order_recorder {
|
|
1566
|
-
if let Ok(mut guard) = recorder.lock() {
|
|
1567
|
-
guard.push(step);
|
|
1568
|
-
}
|
|
1569
|
-
}
|
|
1570
|
-
}
|
|
1571
|
-
}
|
|
1572
|
-
|
|
1573
|
-
fn base_tick_report(
|
|
1574
|
-
ok: bool,
|
|
1575
|
-
stop: bool,
|
|
1576
|
-
reason: Option<TickStopReason>,
|
|
1577
|
-
persisted: Option<bool>,
|
|
1578
|
-
collections: TickCollections,
|
|
1579
|
-
) -> TickReport {
|
|
1580
|
-
TickReport {
|
|
1581
|
-
ok,
|
|
1582
|
-
stop,
|
|
1583
|
-
reason,
|
|
1584
|
-
persisted,
|
|
1585
|
-
delivered: collections.delivered,
|
|
1586
|
-
scheduled: collections.scheduled,
|
|
1587
|
-
stuck: collections.stuck,
|
|
1588
|
-
idle_alerts: collections.idle_alerts,
|
|
1589
|
-
deadlock_alerts: collections.deadlock_alerts,
|
|
1590
|
-
compaction: collections.compaction,
|
|
1591
|
-
session_drift: collections.session_drift,
|
|
1592
|
-
api_errors: collections.api_errors,
|
|
1593
|
-
results: collections.results,
|
|
1594
|
-
}
|
|
1595
|
-
}
|
|
1596
|
-
|
|
1597
|
-
#[derive(Default)]
|
|
1598
|
-
struct TickCollections {
|
|
1599
|
-
delivered: Vec<DeliveredMessage>,
|
|
1600
|
-
scheduled: Vec<FiredScheduledEvent>,
|
|
1601
|
-
stuck: Vec<AgentId>,
|
|
1602
|
-
idle_alerts: Vec<IdleAlert>,
|
|
1603
|
-
deadlock_alerts: Vec<DeadlockAlert>,
|
|
1604
|
-
compaction: Vec<CompactionResult>,
|
|
1605
|
-
session_drift: Vec<SessionDriftResult>,
|
|
1606
|
-
api_errors: Vec<LeaderApiError>,
|
|
1607
|
-
results: Vec<CollectedResult>,
|
|
1608
|
-
}
|
|
1609
|
-
|
|
1610
|
-
fn empty_tick_report(
|
|
1611
|
-
ok: bool,
|
|
1612
|
-
stop: bool,
|
|
1613
|
-
reason: Option<TickStopReason>,
|
|
1614
|
-
persisted: Option<bool>,
|
|
1615
|
-
) -> TickReport {
|
|
1616
|
-
base_tick_report(ok, stop, reason, persisted, TickCollections::default())
|
|
1617
|
-
}
|
|
1618
|
-
|
|
1619
|
-
struct ProviderTurnClassifier;
|
|
1620
|
-
|
|
1621
|
-
impl TurnStateClassifier for ProviderTurnClassifier {
|
|
1622
|
-
fn classify(
|
|
1623
|
-
&self,
|
|
1624
|
-
provider: crate::provider::Provider,
|
|
1625
|
-
session_log_text: &str,
|
|
1626
|
-
) -> Result<TurnClassification, crate::leader::LeaderError> {
|
|
1627
|
-
let result = crate::provider::classify(
|
|
1628
|
-
provider,
|
|
1629
|
-
session_log_text,
|
|
1630
|
-
ProcessLiveness::Unverifiable,
|
|
1631
|
-
0.0,
|
|
1632
|
-
)
|
|
1633
|
-
.map_err(|e| crate::leader::LeaderError::Validation(e.to_string()))?;
|
|
1634
|
-
Ok(TurnClassification {
|
|
1635
|
-
state: result.state,
|
|
1636
|
-
turn_id: result.turn_id.map(|id| id.as_str().to_string()),
|
|
1637
|
-
annotations: result.annotations,
|
|
1638
|
-
reason: Some(result.reason),
|
|
1639
|
-
})
|
|
1640
|
-
}
|
|
1641
|
-
}
|
|
1642
|
-
|
|
1643
|
-
/// P3 (C-P3-1, N1): the tick counter is a transient diagnostic, NOT source-of-truth
|
|
1644
|
-
/// state — keeping it in state.json made EVERY tick dirty and defeated both save
|
|
1645
|
-
/// short-circuits. It lives in its own metadata file; old state files still carrying
|
|
1646
|
-
/// `coordinator.coordinator_tick_iteration_count` load fine (read-compat, C-P3-3) —
|
|
1647
|
-
/// new versions simply stop writing it.
|
|
1648
|
-
fn increment_coordinator_tick_iteration_count(workspace: &WorkspacePath) {
|
|
1649
|
-
let _ = write_coordinator_heartbeat(
|
|
1650
|
-
workspace,
|
|
1651
|
-
Pid::new(std::process::id()),
|
|
1652
|
-
None,
|
|
1653
|
-
"tick_running",
|
|
1654
|
-
Some("running"),
|
|
1655
|
-
None,
|
|
1656
|
-
true,
|
|
1657
|
-
);
|
|
1658
|
-
}
|
|
1659
|
-
|
|
1660
|
-
/// ---
|
|
1661
|
-
/// purpose: 写心跳 sidecar runtime/coordinator_tick.json,让外部不必进程内也能看到 daemon 走到哪一步
|
|
1662
|
-
/// params:
|
|
1663
|
-
/// workspace: workspace 根
|
|
1664
|
-
/// pid: 当前 daemon pid
|
|
1665
|
-
/// boot_id: daemon 世代 id;None 时沿用文件里已有值,都没有则用 coord_unknown_<pid>
|
|
1666
|
-
/// last_phase: 本次写入代表的阶段名;值为 tick_running 时刷新 last_tick_started_at,否则刷新 last_tick_finished_at
|
|
1667
|
-
/// last_tick_status: 上一轮 tick 的状态串
|
|
1668
|
-
/// last_error: 上一轮的错误串
|
|
1669
|
-
/// increment_count: 是否把迭代计数加一
|
|
1670
|
-
/// returns: 写成功返回 ();先写临时文件再 rename,读方不会看到半份
|
|
1671
|
-
/// errors: 建目录、写临时文件或 rename 失败时返回 io::Error
|
|
1672
|
-
/// ---
|
|
1673
|
-
/// ⚠ 字段 `host_boot_time` 写的是**本次写入时刻的墙钟**,不是主机启动时间——
|
|
1674
|
-
/// 每次心跳都会变。主机启动身份在 `host_boot_id`,按名消费 host_boot_time 会拿到假事实。
|
|
1675
|
-
pub(crate) fn write_coordinator_heartbeat(
|
|
1676
|
-
workspace: &WorkspacePath,
|
|
1677
|
-
pid: Pid,
|
|
1678
|
-
boot_id: Option<&str>,
|
|
1679
|
-
last_phase: &str,
|
|
1680
|
-
last_tick_status: Option<&str>,
|
|
1681
|
-
last_error: Option<&str>,
|
|
1682
|
-
increment_count: bool,
|
|
1683
|
-
) -> std::io::Result<()> {
|
|
1684
|
-
let path = coordinator_heartbeat_path(workspace);
|
|
1685
|
-
let value = read_coordinator_heartbeat_value(&path);
|
|
1686
|
-
let next_count = value
|
|
1687
|
-
.get("coordinator_tick_iteration_count")
|
|
1688
|
-
.and_then(Value::as_u64)
|
|
1689
|
-
.unwrap_or(0)
|
|
1690
|
-
.saturating_add(u64::from(increment_count));
|
|
1691
|
-
let now = chrono::Utc::now().to_rfc3339();
|
|
1692
|
-
let identity = crate::coordinator::current_coordinator_binary_identity();
|
|
1693
|
-
let boot_id = boot_id
|
|
1694
|
-
.map(ToString::to_string)
|
|
1695
|
-
.or_else(|| {
|
|
1696
|
-
value
|
|
1697
|
-
.get("boot_id")
|
|
1698
|
-
.and_then(Value::as_str)
|
|
1699
|
-
.map(ToString::to_string)
|
|
1700
|
-
})
|
|
1701
|
-
.unwrap_or_else(|| format!("coord_unknown_{}", pid.get()));
|
|
1702
|
-
// 0.5.41 Slice 1 (fault-invisibility-locate.md §5/§6.1): stamp the
|
|
1703
|
-
// host boot identity onto every heartbeat write so downstream
|
|
1704
|
-
// status/diagnose can detect "all pane/pid/session bindings predate
|
|
1705
|
-
// the current host boot" without inventing a new sidecar. Existing
|
|
1706
|
-
// `boot_id` stays as the daemon-generation id.
|
|
1707
|
-
let host_boot_id = probe_host_boot_id().unwrap_or_else(|| "unknown".to_string());
|
|
1708
|
-
let mut next = serde_json::json!({
|
|
1709
|
-
"coordinator_tick_iteration_count": next_count,
|
|
1710
|
-
"pid": pid.get(),
|
|
1711
|
-
"boot_id": boot_id,
|
|
1712
|
-
"host_boot_id": host_boot_id,
|
|
1713
|
-
"host_boot_time": now.clone(),
|
|
1714
|
-
"binary_path": identity.binary_path,
|
|
1715
|
-
"binary_version": identity.binary_version,
|
|
1716
|
-
"last_phase": last_phase,
|
|
1717
|
-
"last_tick_status": last_tick_status,
|
|
1718
|
-
"last_error": last_error,
|
|
1719
|
-
"updated_at": now,
|
|
1720
|
-
});
|
|
1721
|
-
if last_phase == "tick_running" {
|
|
1722
|
-
next["last_tick_started_at"] = serde_json::Value::String(now.clone());
|
|
1723
|
-
if let Some(finished) = value.get("last_tick_finished_at").cloned() {
|
|
1724
|
-
next["last_tick_finished_at"] = finished;
|
|
1725
|
-
}
|
|
1726
|
-
} else {
|
|
1727
|
-
if let Some(started) = value.get("last_tick_started_at").cloned() {
|
|
1728
|
-
next["last_tick_started_at"] = started;
|
|
1729
|
-
}
|
|
1730
|
-
next["last_tick_finished_at"] = serde_json::Value::String(chrono::Utc::now().to_rfc3339());
|
|
1731
|
-
}
|
|
1732
|
-
write_coordinator_heartbeat_value(&path, &next)
|
|
1733
|
-
}
|
|
1734
|
-
|
|
1735
|
-
fn coordinator_heartbeat_path(workspace: &WorkspacePath) -> PathBuf {
|
|
1736
|
-
crate::model::paths::runtime_dir(workspace.as_path()).join("coordinator_tick.json")
|
|
1737
|
-
}
|
|
1738
|
-
|
|
1739
|
-
/// 0.5.41 Slice 1 (fault-invisibility-locate.md §6.1): single-helper
|
|
1740
|
-
/// probe for the current host boot identity. Test overrides via
|
|
1741
|
-
/// `TEAM_AGENT_TEST_HOST_BOOT_ID` (deterministic RED path). On real
|
|
1742
|
-
/// hosts, reads platform files: Linux `/proc/sys/kernel/random/boot_id`
|
|
1743
|
-
/// (128-bit UUID stable across the boot), macOS `sysctl kern.boottime`
|
|
1744
|
-
/// falls back to a formatted timestamp. Returns None when the platform
|
|
1745
|
-
/// signal cannot be read — the caller MUST NOT guess; downstream
|
|
1746
|
-
/// treats a missing/mismatched host boot separately from a matched one.
|
|
1747
|
-
/// ---
|
|
1748
|
-
/// purpose: 探测当前主机的启动身份,用来识别「所有 pane/pid/会话绑定都早于本次开机」
|
|
1749
|
-
/// returns: Linux 读 boot_id 得 linux-<uuid>;macOS 读 kern.boottime 得 macos-<...>;测试可用 TEAM_AGENT_TEST_HOST_BOOT_ID 覆盖。平台信号读不到时为 None——调用方必须把「读不到」与「不匹配」分开处理,不许猜
|
|
1750
|
-
/// ---
|
|
1751
|
-
pub(crate) fn probe_host_boot_id() -> Option<String> {
|
|
1752
|
-
if let Ok(override_id) = std::env::var("TEAM_AGENT_TEST_HOST_BOOT_ID") {
|
|
1753
|
-
if !override_id.is_empty() {
|
|
1754
|
-
return Some(override_id);
|
|
1755
|
-
}
|
|
1756
|
-
}
|
|
1757
|
-
// Linux: stable UUID for the current boot.
|
|
1758
|
-
if let Ok(raw) = std::fs::read_to_string("/proc/sys/kernel/random/boot_id") {
|
|
1759
|
-
let trimmed = raw.trim();
|
|
1760
|
-
if !trimmed.is_empty() {
|
|
1761
|
-
return Some(format!("linux-{trimmed}"));
|
|
1762
|
-
}
|
|
1763
|
-
}
|
|
1764
|
-
// macOS/BSD: kern.boottime via sysctl. Cheap and root-free.
|
|
1765
|
-
#[cfg(target_os = "macos")]
|
|
1766
|
-
{
|
|
1767
|
-
if let Ok(output) = std::process::Command::new("sysctl")
|
|
1768
|
-
.args(["-n", "kern.boottime"])
|
|
1769
|
-
.output()
|
|
1770
|
-
{
|
|
1771
|
-
if output.status.success() {
|
|
1772
|
-
let text = String::from_utf8_lossy(&output.stdout);
|
|
1773
|
-
let trimmed = text.trim();
|
|
1774
|
-
if !trimmed.is_empty() {
|
|
1775
|
-
return Some(format!("macos-{trimmed}"));
|
|
1776
|
-
}
|
|
1777
|
-
}
|
|
1778
|
-
}
|
|
1779
|
-
}
|
|
1780
|
-
None
|
|
1781
|
-
}
|
|
1782
|
-
|
|
1783
|
-
/// 0.5.41 Slice 1: expose the last-written heartbeat sidecar so
|
|
1784
|
-
/// `cli::diagnose` / `cli::status_port` can consume it read-only. Any
|
|
1785
|
-
/// error (missing file, malformed JSON) returns None — callers treat
|
|
1786
|
-
/// that as "unknown, fall back to existing facts" per locate §8 risk
|
|
1787
|
-
/// note, they must never guess a stale/fresh verdict from absence.
|
|
1788
|
-
/// ---
|
|
1789
|
-
/// purpose: 只读取出最近一次写入的心跳 sidecar
|
|
1790
|
-
/// params:
|
|
1791
|
-
/// workspace: workspace 根
|
|
1792
|
-
/// returns: 解析成功才有值;文件缺失或 JSON 损坏一律 None。调用方须把 None 当作「未知」回落到既有事实,不得据此断定新鲜或陈旧
|
|
1793
|
-
/// ---
|
|
1794
|
-
pub fn read_coordinator_heartbeat(workspace: &WorkspacePath) -> Option<Value> {
|
|
1795
|
-
let path = coordinator_heartbeat_path(workspace);
|
|
1796
|
-
let text = std::fs::read_to_string(path).ok()?;
|
|
1797
|
-
serde_json::from_str(&text).ok()
|
|
1798
|
-
}
|
|
1799
|
-
|
|
1800
|
-
fn read_coordinator_heartbeat_value(path: &Path) -> Value {
|
|
1801
|
-
std::fs::read_to_string(path)
|
|
1802
|
-
.ok()
|
|
1803
|
-
.and_then(|text| serde_json::from_str::<Value>(&text).ok())
|
|
1804
|
-
.unwrap_or_else(|| serde_json::json!({}))
|
|
1805
|
-
}
|
|
1806
|
-
|
|
1807
|
-
fn write_coordinator_heartbeat_value(path: &Path, value: &Value) -> std::io::Result<()> {
|
|
1808
|
-
if let Some(parent) = path.parent() {
|
|
1809
|
-
std::fs::create_dir_all(parent)?;
|
|
1810
|
-
}
|
|
1811
|
-
let tmp = path.with_extension("json.tmp");
|
|
1812
|
-
std::fs::write(&tmp, value.to_string())?;
|
|
1813
|
-
std::fs::rename(tmp, path)
|
|
1814
|
-
}
|
|
1815
|
-
|
|
1816
|
-
fn idle_node_value(node: &crate::leader::IdleNode) -> Value {
|
|
1817
|
-
serde_json::json!({
|
|
1818
|
-
"node_id": node.node_id,
|
|
1819
|
-
"role": match node.role {
|
|
1820
|
-
crate::leader::NodeRole::Worker => "worker",
|
|
1821
|
-
crate::leader::NodeRole::Leader => "leader",
|
|
1822
|
-
},
|
|
1823
|
-
"state": turn_state_wire(node.state),
|
|
1824
|
-
})
|
|
1825
|
-
}
|
|
1826
|
-
|
|
1827
|
-
fn turn_state_wire(state: TurnState) -> &'static str {
|
|
1828
|
-
match state {
|
|
1829
|
-
TurnState::Idle => "idle",
|
|
1830
|
-
TurnState::Working => "working",
|
|
1831
|
-
TurnState::IdleInterrupted => "idle_interrupted",
|
|
1832
|
-
TurnState::BlockedOnHuman => "blocked_on_human",
|
|
1833
|
-
TurnState::Abnormal => "abnormal",
|
|
1834
|
-
TurnState::Unknown => "unknown",
|
|
1835
|
-
}
|
|
1836
|
-
}
|
|
1837
|
-
|
|
1838
|
-
fn monotonic_seconds() -> f64 {
|
|
1839
|
-
match std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH) {
|
|
1840
|
-
Ok(duration) => duration.as_secs_f64(),
|
|
1841
|
-
Err(_) => 0.0,
|
|
1842
|
-
}
|
|
1843
|
-
}
|
|
1844
|
-
|
|
1845
|
-
use crate::provider::wire::parse_provider;
|
|
1846
|
-
|
|
1847
|
-
fn capture_window_target(
|
|
1848
|
-
agent: &Value,
|
|
1849
|
-
session_name: Option<&str>,
|
|
1850
|
-
) -> Option<(
|
|
1851
|
-
crate::transport::SessionName,
|
|
1852
|
-
crate::transport::WindowName,
|
|
1853
|
-
crate::transport::Target,
|
|
1854
|
-
)> {
|
|
1855
|
-
let window = agent
|
|
1856
|
-
.get("window")
|
|
1857
|
-
.and_then(Value::as_str)
|
|
1858
|
-
.filter(|s| !s.is_empty())?;
|
|
1859
|
-
let session = session_name.filter(|s| !s.is_empty())?;
|
|
1860
|
-
let session = crate::transport::SessionName::new(session);
|
|
1861
|
-
let window = crate::transport::WindowName::new(window);
|
|
1862
|
-
Some((
|
|
1863
|
-
session.clone(),
|
|
1864
|
-
window.clone(),
|
|
1865
|
-
crate::transport::Target::SessionWindow { session, window },
|
|
1866
|
-
))
|
|
1867
|
-
}
|
|
1868
|
-
|
|
1869
|
-
fn tick_has_work_obligation(store: &crate::message_store::MessageStore) -> bool {
|
|
1870
|
-
let Ok(conn) = crate::db::schema::open_db(store.db_path()) else {
|
|
1871
|
-
return true;
|
|
1872
|
-
};
|
|
1873
|
-
let pending: i64 = conn.query_row(
|
|
1874
|
-
"select count(*) from messages where status in ('pending', 'accepted', 'target_resolved')",
|
|
1875
|
-
[],
|
|
1876
|
-
|row| row.get(0),
|
|
1877
|
-
).unwrap_or(1);
|
|
1878
|
-
if pending > 0 {
|
|
1879
|
-
return true;
|
|
1880
|
-
}
|
|
1881
|
-
let now = chrono::Utc::now().to_rfc3339();
|
|
1882
|
-
let due: i64 = conn
|
|
1883
|
-
.query_row(
|
|
1884
|
-
"select count(*) from scheduled_events where status = 'pending' and due_at <= ?1",
|
|
1885
|
-
[now],
|
|
1886
|
-
|row| row.get(0),
|
|
1887
|
-
)
|
|
1888
|
-
.unwrap_or(1);
|
|
1889
|
-
due > 0
|
|
1890
|
-
}
|
|
1891
|
-
|
|
1892
|
-
fn agent_probe_base_eligible(agent: &Value) -> bool {
|
|
1893
|
-
#[cfg(unix)]
|
|
1894
|
-
if crate::contract_runtime::framework::is_contract(agent) {
|
|
1895
|
-
return false;
|
|
1896
|
-
}
|
|
1897
|
-
let status = agent.get("status").and_then(Value::as_str);
|
|
1898
|
-
!matches!(
|
|
1899
|
-
status,
|
|
1900
|
-
Some("missing" | "stopped" | "dead" | "exited" | "terminated" | "removed" | "failed")
|
|
1901
|
-
)
|
|
1902
|
-
}
|
|
1903
|
-
|
|
1904
|
-
fn agent_window_present(
|
|
1905
|
-
agent: &Value,
|
|
1906
|
-
session: &crate::transport::SessionName,
|
|
1907
|
-
window: &crate::transport::WindowName,
|
|
1908
|
-
pane_infos: &[crate::transport::PaneInfo],
|
|
1909
|
-
windows: &[crate::transport::WindowName],
|
|
1910
|
-
) -> bool {
|
|
1911
|
-
if let Some(pane_id) = agent_pane_id(agent) {
|
|
1912
|
-
if pane_infos.iter().any(|info| info.pane_id == pane_id) {
|
|
1913
|
-
return true;
|
|
1914
|
-
}
|
|
1915
|
-
}
|
|
1916
|
-
if pane_infos.iter().any(|info| {
|
|
1917
|
-
&info.session == session
|
|
1918
|
-
&& info
|
|
1919
|
-
.window_name
|
|
1920
|
-
.as_ref()
|
|
1921
|
-
.is_some_and(|known_window| known_window == window)
|
|
1922
|
-
}) {
|
|
1923
|
-
return true;
|
|
1924
|
-
}
|
|
1925
|
-
if !pane_infos.is_empty() {
|
|
1926
|
-
return false;
|
|
1927
|
-
}
|
|
1928
|
-
windows.is_empty() || windows.iter().any(|known| known == window)
|
|
1929
|
-
}
|
|
1930
|
-
|
|
1931
|
-
fn runtime_approval_target_present(
|
|
1932
|
-
target: &crate::transport::Target,
|
|
1933
|
-
pane_infos: &[crate::transport::PaneInfo],
|
|
1934
|
-
windows: &[crate::transport::WindowName],
|
|
1935
|
-
) -> bool {
|
|
1936
|
-
match target {
|
|
1937
|
-
crate::transport::Target::Pane(pane) => {
|
|
1938
|
-
if pane_infos.iter().any(|info| &info.pane_id == pane) {
|
|
1939
|
-
return true;
|
|
1940
|
-
}
|
|
1941
|
-
pane_infos.is_empty()
|
|
1942
|
-
}
|
|
1943
|
-
crate::transport::Target::SessionWindow { session, window } => {
|
|
1944
|
-
if pane_infos.iter().any(|info| {
|
|
1945
|
-
&info.session == session
|
|
1946
|
-
&& info
|
|
1947
|
-
.window_name
|
|
1948
|
-
.as_ref()
|
|
1949
|
-
.is_some_and(|known_window| known_window == window)
|
|
1950
|
-
}) {
|
|
1951
|
-
return true;
|
|
1952
|
-
}
|
|
1953
|
-
if !pane_infos.is_empty() {
|
|
1954
|
-
return false;
|
|
1955
|
-
}
|
|
1956
|
-
windows.is_empty() || windows.iter().any(|known| known == window)
|
|
1957
|
-
}
|
|
1958
|
-
}
|
|
1959
|
-
}
|
|
1960
|
-
|
|
1961
|
-
fn agent_process_epoch(agent: &Value) -> String {
|
|
1962
|
-
if let Some(pid) = agent.get("pane_pid").and_then(Value::as_u64) {
|
|
1963
|
-
return format!("pane_pid:{pid}");
|
|
1964
|
-
}
|
|
1965
|
-
if let Some(spawned_at) = agent.get("spawned_at").and_then(Value::as_str) {
|
|
1966
|
-
return format!("spawned_at:{spawned_at}");
|
|
1967
|
-
}
|
|
1968
|
-
if let Some(pane_id) = agent.get("pane_id").and_then(Value::as_str) {
|
|
1969
|
-
return format!("pane:{pane_id}");
|
|
1970
|
-
}
|
|
1971
|
-
agent
|
|
1972
|
-
.get("window")
|
|
1973
|
-
.and_then(Value::as_str)
|
|
1974
|
-
.map(|window| format!("window:{window}"))
|
|
1975
|
-
.unwrap_or_else(|| "unknown".to_string())
|
|
1976
|
-
}
|
|
1977
|
-
|
|
1978
|
-
fn startup_probe_disabled_for_epoch(agent: &Value) -> bool {
|
|
1979
|
-
let epoch = agent_process_epoch(agent);
|
|
1980
|
-
agent.get("startup_prompt_status").and_then(Value::as_str) == Some("disabled_for_epoch")
|
|
1981
|
-
&& agent
|
|
1982
|
-
.get("startup_prompt_probe_epoch")
|
|
1983
|
-
.and_then(Value::as_str)
|
|
1984
|
-
== Some(epoch.as_str())
|
|
1985
|
-
}
|
|
1986
|
-
|
|
1987
|
-
fn clear_startup_probe_disable_if_epoch_changed(agent: &mut Value) {
|
|
1988
|
-
if agent.get("startup_prompt_status").and_then(Value::as_str) != Some("disabled_for_epoch") {
|
|
1989
|
-
return;
|
|
1990
|
-
}
|
|
1991
|
-
let epoch = agent_process_epoch(agent);
|
|
1992
|
-
if agent
|
|
1993
|
-
.get("startup_prompt_probe_epoch")
|
|
1994
|
-
.and_then(Value::as_str)
|
|
1995
|
-
== Some(epoch.as_str())
|
|
1996
|
-
{
|
|
1997
|
-
return;
|
|
1998
|
-
}
|
|
1999
|
-
if let Some(agent) = agent.as_object_mut() {
|
|
2000
|
-
agent.remove("startup_prompt_status");
|
|
2001
|
-
agent.remove("startup_prompts");
|
|
2002
|
-
agent.remove("startup_prompt_probe_disabled_at");
|
|
2003
|
-
}
|
|
2004
|
-
}
|
|
2005
|
-
|
|
2006
|
-
fn startup_probe_within_grace(agent: &Value) -> bool {
|
|
2007
|
-
let Some(spawned_at) = agent.get("spawned_at").and_then(Value::as_str) else {
|
|
2008
|
-
return true;
|
|
2009
|
-
};
|
|
2010
|
-
let Some(spawned_at) = parse_rfc3339_utc(spawned_at) else {
|
|
2011
|
-
return true;
|
|
2012
|
-
};
|
|
2013
|
-
chrono::Utc::now()
|
|
2014
|
-
.signed_duration_since(spawned_at)
|
|
2015
|
-
.num_seconds()
|
|
2016
|
-
<= STARTUP_PROMPT_GRACE_SECS
|
|
2017
|
-
}
|
|
2018
|
-
|
|
2019
|
-
fn disable_startup_probe_for_epoch(agent: &mut Value) {
|
|
2020
|
-
let epoch = agent_process_epoch(agent);
|
|
2021
|
-
if let Some(agent) = agent.as_object_mut() {
|
|
2022
|
-
agent.insert(
|
|
2023
|
-
"startup_prompt_status".to_string(),
|
|
2024
|
-
serde_json::json!("disabled_for_epoch"),
|
|
2025
|
-
);
|
|
2026
|
-
agent.insert(
|
|
2027
|
-
"startup_prompts".to_string(),
|
|
2028
|
-
serde_json::json!("disabled_for_epoch"),
|
|
2029
|
-
);
|
|
2030
|
-
agent.insert(
|
|
2031
|
-
"startup_prompt_probe_epoch".to_string(),
|
|
2032
|
-
serde_json::json!(epoch),
|
|
2033
|
-
);
|
|
2034
|
-
agent.insert(
|
|
2035
|
-
"startup_prompt_probe_disabled_at".to_string(),
|
|
2036
|
-
serde_json::json!(chrono::Utc::now().to_rfc3339()),
|
|
2037
|
-
);
|
|
2038
|
-
}
|
|
2039
|
-
}
|
|
2040
|
-
|
|
2041
|
-
fn runtime_approval_backoff_active(agent: &Value) -> bool {
|
|
2042
|
-
let Some(next) = agent
|
|
2043
|
-
.pointer("/runtime_approval_probe/next_probe_at")
|
|
2044
|
-
.and_then(Value::as_str)
|
|
2045
|
-
.and_then(parse_rfc3339_utc)
|
|
2046
|
-
else {
|
|
2047
|
-
return false;
|
|
2048
|
-
};
|
|
2049
|
-
next > chrono::Utc::now()
|
|
2050
|
-
}
|
|
2051
|
-
|
|
2052
|
-
fn remember_runtime_approval_backoff(agent: &mut Value) {
|
|
2053
|
-
let previous = agent
|
|
2054
|
-
.pointer("/runtime_approval_probe/backoff_secs")
|
|
2055
|
-
.and_then(Value::as_i64)
|
|
2056
|
-
.unwrap_or(0);
|
|
2057
|
-
let backoff = if previous <= 0 {
|
|
2058
|
-
RUNTIME_APPROVAL_INITIAL_BACKOFF_SECS
|
|
2059
|
-
} else {
|
|
2060
|
-
previous
|
|
2061
|
-
.saturating_mul(2)
|
|
2062
|
-
.min(RUNTIME_APPROVAL_MAX_BACKOFF_SECS)
|
|
2063
|
-
};
|
|
2064
|
-
let next = chrono::Utc::now() + chrono::Duration::seconds(backoff);
|
|
2065
|
-
if let Some(agent) = agent.as_object_mut() {
|
|
2066
|
-
agent.insert(
|
|
2067
|
-
"runtime_approval_probe".to_string(),
|
|
2068
|
-
serde_json::json!({
|
|
2069
|
-
"backoff_secs": backoff,
|
|
2070
|
-
"next_probe_at": next.to_rfc3339(),
|
|
2071
|
-
}),
|
|
2072
|
-
);
|
|
2073
|
-
}
|
|
2074
|
-
}
|
|
2075
|
-
|
|
2076
|
-
fn clear_runtime_approval_backoff(agent: &mut Value) {
|
|
2077
|
-
if let Some(agent) = agent.as_object_mut() {
|
|
2078
|
-
agent.remove("runtime_approval_probe");
|
|
2079
|
-
}
|
|
2080
|
-
}
|
|
2081
|
-
|
|
2082
|
-
fn warm_idle_capture_suppressed(agent: &Value, has_work_obligation: bool) -> bool {
|
|
2083
|
-
if has_work_obligation {
|
|
2084
|
-
return false;
|
|
2085
|
-
}
|
|
2086
|
-
let status = agent
|
|
2087
|
-
.pointer("/activity/status")
|
|
2088
|
-
.and_then(Value::as_str)
|
|
2089
|
-
.or_else(|| agent.get("status").and_then(Value::as_str));
|
|
2090
|
-
if status != Some("idle") {
|
|
2091
|
-
return false;
|
|
2092
|
-
}
|
|
2093
|
-
if runtime_approval_backoff_active(agent) {
|
|
2094
|
-
return true;
|
|
2095
|
-
}
|
|
2096
|
-
agent
|
|
2097
|
-
.get("coordinator_idle_capture_next_at")
|
|
2098
|
-
.and_then(Value::as_str)
|
|
2099
|
-
.and_then(parse_rfc3339_utc)
|
|
2100
|
-
.is_some_and(|next| next > chrono::Utc::now())
|
|
2101
|
-
}
|
|
2102
|
-
|
|
2103
|
-
fn remember_idle_capture_schedule(agent: &mut Value, activity: &crate::messaging::AgentActivity) {
|
|
2104
|
-
if activity.status != crate::messaging::ActivityStatus::Idle {
|
|
2105
|
-
return;
|
|
2106
|
-
}
|
|
2107
|
-
if let Some(agent) = agent.as_object_mut() {
|
|
2108
|
-
let next =
|
|
2109
|
-
chrono::Utc::now() + chrono::Duration::seconds(IDLE_HEALTH_CAPTURE_INTERVAL_SECS);
|
|
2110
|
-
agent.insert(
|
|
2111
|
-
"coordinator_idle_capture_next_at".to_string(),
|
|
2112
|
-
serde_json::json!(next.to_rfc3339()),
|
|
2113
|
-
);
|
|
2114
|
-
}
|
|
2115
|
-
}
|
|
2116
|
-
|
|
2117
|
-
fn parse_rfc3339_utc(raw: &str) -> Option<chrono::DateTime<chrono::Utc>> {
|
|
2118
|
-
chrono::DateTime::parse_from_rfc3339(raw)
|
|
2119
|
-
.ok()
|
|
2120
|
-
.map(|value| value.with_timezone(&chrono::Utc))
|
|
2121
|
-
}
|
|
2122
|
-
|
|
2123
|
-
/// 0.5.32 (`.team/artifacts/restart-resumed-stale-activity-locate.md` §5):
|
|
2124
|
-
/// return true when the rollout file's modification time is strictly older
|
|
2125
|
-
/// than the agent's current `spawned_at`. Used by `jsonl_activity_for_agent`
|
|
2126
|
-
/// to refuse classifying a pre-spawn transcript tail into fresh-cohort
|
|
2127
|
-
/// activity. Missing `spawned_at`, missing mtime, or unparseable timestamp
|
|
2128
|
-
/// → false (do not block classification; the caller keeps the pre-0.5.32
|
|
2129
|
-
/// behavior).
|
|
2130
|
-
///
|
|
2131
|
-
/// 0.5.33 boundary tightening (leader 派单 msg_42dbad7f2938 / earlier cr
|
|
2132
|
-
/// batch older-or-equal 预警): use strict `<` not `<=`. Fast machines can
|
|
2133
|
-
/// produce a fresh post-spawn rollout whose mtime shares the second-resolution
|
|
2134
|
-
/// `spawned_at`; treating "equal" as pre-spawn would wrongly reject the
|
|
2135
|
-
/// first post-spawn transcript record. Genuinely pre-spawn transcripts have
|
|
2136
|
-
/// mtime strictly before `spawned_at` and remain rejected. The equal case
|
|
2137
|
-
/// falls back to the classifier + pane fallback, which is the ambiguous-
|
|
2138
|
-
/// but-safe path.
|
|
2139
|
-
fn jsonl_older_than_spawn_boundary(agent: &Value, metadata: &std::fs::Metadata) -> bool {
|
|
2140
|
-
let Some(spawned_at) = agent
|
|
2141
|
-
.get("spawned_at")
|
|
2142
|
-
.and_then(Value::as_str)
|
|
2143
|
-
.and_then(parse_rfc3339_utc)
|
|
2144
|
-
else {
|
|
2145
|
-
return false;
|
|
2146
|
-
};
|
|
2147
|
-
let Ok(mtime) = metadata.modified() else {
|
|
2148
|
-
return false;
|
|
2149
|
-
};
|
|
2150
|
-
let Ok(mtime_since_epoch) = mtime.duration_since(std::time::UNIX_EPOCH) else {
|
|
2151
|
-
return false;
|
|
2152
|
-
};
|
|
2153
|
-
let mtime_utc =
|
|
2154
|
-
chrono::DateTime::<chrono::Utc>::from(std::time::UNIX_EPOCH + mtime_since_epoch);
|
|
2155
|
-
mtime_utc < spawned_at
|
|
2156
|
-
}
|
|
2157
|
-
|
|
2158
|
-
fn matching_capture_pane_info(
|
|
2159
|
-
agent: &Value,
|
|
2160
|
-
session: &crate::transport::SessionName,
|
|
2161
|
-
window: &crate::transport::WindowName,
|
|
2162
|
-
pane_infos: &[crate::transport::PaneInfo],
|
|
2163
|
-
) -> Option<crate::transport::PaneInfo> {
|
|
2164
|
-
if let Some(pane_id) = agent_pane_id(agent) {
|
|
2165
|
-
if let Some(info) = pane_infos.iter().find(|info| info.pane_id == pane_id) {
|
|
2166
|
-
return Some(info.clone());
|
|
2167
|
-
}
|
|
2168
|
-
}
|
|
2169
|
-
pane_infos
|
|
2170
|
-
.iter()
|
|
2171
|
-
.find(|info| {
|
|
2172
|
-
&info.session == session
|
|
2173
|
-
&& info
|
|
2174
|
-
.window_name
|
|
2175
|
-
.as_ref()
|
|
2176
|
-
.is_some_and(|known_window| known_window == window)
|
|
2177
|
-
})
|
|
2178
|
-
.cloned()
|
|
2179
|
-
}
|
|
2180
|
-
|
|
2181
|
-
fn agent_pane_id(agent: &Value) -> Option<crate::transport::PaneId> {
|
|
2182
|
-
agent
|
|
2183
|
-
.get("pane_id")
|
|
2184
|
-
.and_then(Value::as_str)
|
|
2185
|
-
.filter(|pane_id| !pane_id.is_empty())
|
|
2186
|
-
.map(crate::transport::PaneId::new)
|
|
2187
|
-
}
|
|
2188
|
-
|
|
2189
|
-
fn agent_rollout_path(agent: &Value) -> Option<PathBuf> {
|
|
2190
|
-
["rollout_path", "transcript_path", "session_log_path"]
|
|
2191
|
-
.into_iter()
|
|
2192
|
-
.find_map(|key| agent.get(key).and_then(Value::as_str))
|
|
2193
|
-
.filter(|path| !path.is_empty())
|
|
2194
|
-
.map(PathBuf::from)
|
|
2195
|
-
}
|
|
2196
|
-
|
|
2197
|
-
/// Memory-growth fix (architect §5): bounded tail cap for JSONL activity reads.
|
|
2198
|
-
/// Matches `ABNORMAL_TAIL_BYTES` (131_072 bytes) — the abnormal-exit path
|
|
2199
|
-
/// already proved this size is sufficient to capture the latest lifecycle
|
|
2200
|
-
/// records across all providers (claude / codex / copilot).
|
|
2201
|
-
const JSONL_ACTIVITY_TAIL_BYTES: u64 = 131_072;
|
|
2202
|
-
|
|
2203
|
-
/// Memory-growth fix (architect §5): per-process `(path, size, mtime_ns) →
|
|
2204
|
-
/// activity` cache. When a rollout file hasn't changed since the previous
|
|
2205
|
-
/// tick, we skip the file read AND the classification entirely. This is the
|
|
2206
|
-
/// dominant savings: a 538MB Claude transcript that updates every few seconds
|
|
2207
|
-
/// is touched only when its size or mtime actually moves. Stored values are
|
|
2208
|
-
/// small (Option<AgentActivity> = enum + short rationale string); we never
|
|
2209
|
-
/// cache the transcript text or parsed JSON.
|
|
2210
|
-
struct JsonlActivityCacheEntry {
|
|
2211
|
-
size: u64,
|
|
2212
|
-
mtime_ns: u64,
|
|
2213
|
-
activity: Option<crate::messaging::AgentActivity>,
|
|
2214
|
-
}
|
|
2215
|
-
|
|
2216
|
-
fn jsonl_activity_cache(
|
|
2217
|
-
) -> &'static std::sync::Mutex<std::collections::HashMap<PathBuf, JsonlActivityCacheEntry>> {
|
|
2218
|
-
static CACHE: std::sync::OnceLock<
|
|
2219
|
-
std::sync::Mutex<std::collections::HashMap<PathBuf, JsonlActivityCacheEntry>>,
|
|
2220
|
-
> = std::sync::OnceLock::new();
|
|
2221
|
-
CACHE.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
|
|
2222
|
-
}
|
|
2223
|
-
|
|
2224
|
-
/// E47 (0.3.24 P0, idle/busy 假阳): consult the authoritative provider JSONL
|
|
2225
|
-
/// classifier and map to neutral `AgentActivity`. Returns `None` when the
|
|
2226
|
-
/// classifier reports `TurnState::Unknown` (unreadable JSONL / no lifecycle
|
|
2227
|
-
/// fact yet) so the caller falls back to the TUI scan — this honours the
|
|
2228
|
-
/// IRON LAW (activity.rs:3 / bug-071/077/085): no-signal = Uncertain (not
|
|
2229
|
-
/// silently coerced to Idle); but here Unknown means "JSONL gave no signal",
|
|
2230
|
-
/// so we hand off to the TUI scanner which has its OWN no-signal → Uncertain
|
|
2231
|
-
/// path. Copilot/Gemini/Fake providers (which don't have JSONL — classify.rs
|
|
2232
|
-
/// returns Unknown for them) thus keep using TUI scanning unchanged.
|
|
2233
|
-
///
|
|
2234
|
-
/// Memory-growth fix (architect analysis 2026-06-23): bounded tail read +
|
|
2235
|
-
/// metadata cache. Pre-fix `std::fs::read_to_string` on a 538MB Claude
|
|
2236
|
-
/// transcript every 5s caused 200MB+ coordinator RSS plateaus from allocator
|
|
2237
|
-
/// fragmentation. Now bounded to 128KiB tail and skipped entirely when
|
|
2238
|
-
/// (size, mtime_ns) is unchanged.
|
|
2239
|
-
fn jsonl_activity_for_agent(agent: &Value) -> Option<crate::messaging::AgentActivity> {
|
|
2240
|
-
let rollout_path = agent_rollout_path(agent)?;
|
|
2241
|
-
let provider = agent
|
|
2242
|
-
.get("provider")
|
|
2243
|
-
.and_then(Value::as_str)
|
|
2244
|
-
.and_then(parse_provider)?;
|
|
2245
|
-
|
|
2246
|
-
// Metadata check + cache lookup. Cache hit when the rollout file has
|
|
2247
|
-
// not changed since the previous tick: return the cached classification
|
|
2248
|
-
// without re-reading the file. Truncation (size shrink with stable mtime)
|
|
2249
|
-
// still forces re-read because size is part of the cache key.
|
|
2250
|
-
let metadata = std::fs::metadata(&rollout_path).ok()?;
|
|
2251
|
-
let size = metadata.len();
|
|
2252
|
-
let mtime_ns = crate::coordinator::steps::abnormal::metadata_mtime_ns(&metadata)?;
|
|
2253
|
-
// 0.5.32 (`.team/artifacts/restart-resumed-stale-activity-locate.md` §5):
|
|
2254
|
-
// freshness gate — a rollout file whose last modification is older than
|
|
2255
|
-
// the agent's current `spawned_at` cannot describe post-spawn behavior.
|
|
2256
|
-
// Refusing to classify (return None) here keeps stale pre-restart
|
|
2257
|
-
// Working tails from re-poisoning a fresh cohort's activity/worker_state;
|
|
2258
|
-
// pane fallback in the caller is still free to produce a fresh
|
|
2259
|
-
// classification when a live pane observation exists.
|
|
2260
|
-
if jsonl_older_than_spawn_boundary(agent, &metadata) {
|
|
2261
|
-
return None;
|
|
2262
|
-
}
|
|
2263
|
-
if let Ok(cache) = jsonl_activity_cache().lock() {
|
|
2264
|
-
if let Some(entry) = cache.get(&rollout_path) {
|
|
2265
|
-
if entry.size == size && entry.mtime_ns == mtime_ns {
|
|
2266
|
-
return entry.activity.clone();
|
|
2267
|
-
}
|
|
2268
|
-
}
|
|
2269
|
-
}
|
|
2270
|
-
|
|
2271
|
-
// Cache miss: bounded tail read + classify. The classifier only needs the
|
|
2272
|
-
// latest lifecycle records to determine idle/working state; the
|
|
2273
|
-
// abnormal-exit path uses the same 128KiB tail and is sufficient for
|
|
2274
|
-
// claude / codex / copilot lifecycle markers.
|
|
2275
|
-
let log_text = crate::coordinator::steps::abnormal::read_tail_text(
|
|
2276
|
-
&rollout_path,
|
|
2277
|
-
JSONL_ACTIVITY_TAIL_BYTES,
|
|
2278
|
-
)
|
|
2279
|
-
.ok()?;
|
|
2280
|
-
let process = crate::coordinator::steps::abnormal::explicit_process_liveness(agent)
|
|
2281
|
-
.unwrap_or(ProcessLiveness::Unverifiable);
|
|
2282
|
-
let activity = crate::provider::classify(provider, &log_text, process, 0.0)
|
|
2283
|
-
.ok()
|
|
2284
|
-
.and_then(|result| {
|
|
2285
|
-
use crate::messaging::{ActivityStatus, AgentActivity};
|
|
2286
|
-
use crate::provider::types::TurnState;
|
|
2287
|
-
let status = match result.state {
|
|
2288
|
-
TurnState::Idle => ActivityStatus::Idle,
|
|
2289
|
-
TurnState::IdleInterrupted => ActivityStatus::Idle,
|
|
2290
|
-
TurnState::Working => ActivityStatus::Working,
|
|
2291
|
-
TurnState::BlockedOnHuman | TurnState::Abnormal => ActivityStatus::Uncertain,
|
|
2292
|
-
TurnState::Unknown => return None,
|
|
2293
|
-
};
|
|
2294
|
-
Some(AgentActivity {
|
|
2295
|
-
status,
|
|
2296
|
-
confidence: 0.95,
|
|
2297
|
-
rationale: format!("provider_jsonl:{}", result.reason),
|
|
2298
|
-
})
|
|
2299
|
-
});
|
|
2300
|
-
|
|
2301
|
-
// Store the classification (including None / Unknown) so the next tick
|
|
2302
|
-
// can short-circuit when the file is unchanged.
|
|
2303
|
-
if let Ok(mut cache) = jsonl_activity_cache().lock() {
|
|
2304
|
-
cache.insert(
|
|
2305
|
-
rollout_path,
|
|
2306
|
-
JsonlActivityCacheEntry {
|
|
2307
|
-
size,
|
|
2308
|
-
mtime_ns,
|
|
2309
|
-
activity: activity.clone(),
|
|
2310
|
-
},
|
|
2311
|
-
);
|
|
2312
|
-
}
|
|
2313
|
-
activity
|
|
2314
|
-
}
|
|
2315
|
-
|
|
2316
|
-
fn runtime_approval_target(
|
|
2317
|
-
agent: &Value,
|
|
2318
|
-
session_name: Option<&str>,
|
|
2319
|
-
) -> Option<crate::transport::Target> {
|
|
2320
|
-
if let Some(pane_id) = agent
|
|
2321
|
-
.get("pane_id")
|
|
2322
|
-
.and_then(Value::as_str)
|
|
2323
|
-
.filter(|pane_id| !pane_id.is_empty())
|
|
2324
|
-
{
|
|
2325
|
-
return Some(crate::transport::Target::Pane(
|
|
2326
|
-
crate::transport::PaneId::new(pane_id),
|
|
2327
|
-
));
|
|
2328
|
-
}
|
|
2329
|
-
capture_window_target(agent, session_name).map(|(_, _, target)| target)
|
|
2330
|
-
}
|
|
2331
|
-
|
|
2332
|
-
fn runtime_approval_key(raw: String) -> Option<crate::transport::Key> {
|
|
2333
|
-
match raw.as_str() {
|
|
2334
|
-
"Enter" => Some(crate::transport::Key::Enter),
|
|
2335
|
-
"Up" => Some(crate::transport::Key::Up),
|
|
2336
|
-
"Down" => Some(crate::transport::Key::Down),
|
|
2337
|
-
"Left" => Some(crate::transport::Key::Left),
|
|
2338
|
-
"Right" => Some(crate::transport::Key::Right),
|
|
2339
|
-
other => {
|
|
2340
|
-
let mut chars = other.chars();
|
|
2341
|
-
let ch = chars.next()?;
|
|
2342
|
-
if chars.next().is_none() {
|
|
2343
|
-
Some(crate::transport::Key::Char(ch))
|
|
2344
|
-
} else {
|
|
2345
|
-
None
|
|
2346
|
-
}
|
|
2347
|
-
}
|
|
2348
|
-
}
|
|
2349
|
-
}
|
|
2350
|
-
|
|
2351
|
-
#[derive(Debug, Clone)]
|
|
2352
|
-
struct RuntimeApprovalPolicy {
|
|
2353
|
-
enabled: bool,
|
|
2354
|
-
source: String,
|
|
2355
|
-
inherited: bool,
|
|
2356
|
-
explicit_yes_confirmed: bool,
|
|
2357
|
-
provider: Option<String>,
|
|
2358
|
-
flag: Option<String>,
|
|
2359
|
-
worker_capability_above_leader: bool,
|
|
2360
|
-
}
|
|
2361
|
-
|
|
2362
|
-
impl RuntimeApprovalPolicy {
|
|
2363
|
-
fn auto_answer_allowed(&self) -> bool {
|
|
2364
|
-
if !self.enabled {
|
|
2365
|
-
return false;
|
|
2366
|
-
}
|
|
2367
|
-
let source_allows = match self.source.as_str() {
|
|
2368
|
-
"leader_process" => self.inherited,
|
|
2369
|
-
"runtime_config" => self.explicit_yes_confirmed,
|
|
2370
|
-
_ => false,
|
|
2371
|
-
};
|
|
2372
|
-
source_allows
|
|
2373
|
-
&& (!self.worker_capability_above_leader
|
|
2374
|
-
|| (self.source == "runtime_config" && self.explicit_yes_confirmed))
|
|
2375
|
-
}
|
|
2376
|
-
|
|
2377
|
-
fn inherit_reason(&self) -> &'static str {
|
|
2378
|
-
match self.source.as_str() {
|
|
2379
|
-
"leader_process" if self.inherited => "leader_bypass",
|
|
2380
|
-
"runtime_config" if self.explicit_yes_confirmed => "runtime_config_explicit_yes",
|
|
2381
|
-
_ => "none",
|
|
2382
|
-
}
|
|
2383
|
-
}
|
|
2384
|
-
}
|
|
2385
|
-
|
|
2386
|
-
fn runtime_approval_policy_from_agent(agent: &Value) -> RuntimeApprovalPolicy {
|
|
2387
|
-
let policy = agent
|
|
2388
|
-
.get("effective_approval_policy")
|
|
2389
|
-
.and_then(Value::as_object);
|
|
2390
|
-
RuntimeApprovalPolicy {
|
|
2391
|
-
enabled: policy
|
|
2392
|
-
.and_then(|p| p.get("enabled"))
|
|
2393
|
-
.and_then(Value::as_bool)
|
|
2394
|
-
.unwrap_or(false),
|
|
2395
|
-
source: policy
|
|
2396
|
-
.and_then(|p| p.get("source"))
|
|
2397
|
-
.and_then(Value::as_str)
|
|
2398
|
-
.unwrap_or("disabled")
|
|
2399
|
-
.to_string(),
|
|
2400
|
-
inherited: policy
|
|
2401
|
-
.and_then(|p| p.get("inherited"))
|
|
2402
|
-
.and_then(Value::as_bool)
|
|
2403
|
-
.unwrap_or(false),
|
|
2404
|
-
explicit_yes_confirmed: policy
|
|
2405
|
-
.and_then(|p| p.get("explicit_yes_confirmed"))
|
|
2406
|
-
.and_then(Value::as_bool)
|
|
2407
|
-
.unwrap_or(false),
|
|
2408
|
-
provider: policy
|
|
2409
|
-
.and_then(|p| p.get("provider"))
|
|
2410
|
-
.and_then(Value::as_str)
|
|
2411
|
-
.map(str::to_string),
|
|
2412
|
-
flag: policy
|
|
2413
|
-
.and_then(|p| p.get("flag"))
|
|
2414
|
-
.and_then(Value::as_str)
|
|
2415
|
-
.map(str::to_string),
|
|
2416
|
-
worker_capability_above_leader: policy
|
|
2417
|
-
.and_then(|p| p.get("worker_capability_above_leader"))
|
|
2418
|
-
.and_then(Value::as_bool)
|
|
2419
|
-
.unwrap_or(false),
|
|
2420
|
-
}
|
|
2421
|
-
}
|
|
2422
|
-
|
|
2423
|
-
fn awaiting_human_confirm_payload(
|
|
2424
|
-
agent: &Value,
|
|
2425
|
-
fact: &crate::provider::AwaitingHumanConfirmFact,
|
|
2426
|
-
) -> Value {
|
|
2427
|
-
let mut payload = fact.to_event_payload();
|
|
2428
|
-
let excerpt = fact
|
|
2429
|
-
.prompt
|
|
2430
|
-
.lines()
|
|
2431
|
-
.next()
|
|
2432
|
-
.unwrap_or("")
|
|
2433
|
-
.chars()
|
|
2434
|
-
.take(240)
|
|
2435
|
-
.collect::<String>();
|
|
2436
|
-
if let Some(obj) = payload.as_object_mut() {
|
|
2437
|
-
obj.insert("team_id".to_string(), serde_json::json!(fact.team));
|
|
2438
|
-
obj.insert("owner_team_id".to_string(), serde_json::json!(fact.team));
|
|
2439
|
-
if let Some(provider) = agent.get("provider").and_then(Value::as_str) {
|
|
2440
|
-
obj.insert("provider".to_string(), serde_json::json!(provider));
|
|
2441
|
-
}
|
|
2442
|
-
if let Some(pane_id) = agent.get("pane_id").and_then(Value::as_str) {
|
|
2443
|
-
obj.insert("pane_id".to_string(), serde_json::json!(pane_id));
|
|
2444
|
-
}
|
|
2445
|
-
obj.insert("excerpt".to_string(), serde_json::json!(excerpt));
|
|
2446
|
-
}
|
|
2447
|
-
payload
|
|
2448
|
-
}
|
|
2449
|
-
|
|
2450
|
-
enum AwaitingDedupUpdate {
|
|
2451
|
-
Remember(crate::provider::AwaitingHumanConfirmFact),
|
|
2452
|
-
Clear { team: String, agent_id: String },
|
|
2453
|
-
}
|
|
2454
|
-
|
|
2455
|
-
fn state_awaiting_human_confirm_fingerprint(
|
|
2456
|
-
state: &Value,
|
|
2457
|
-
team: &str,
|
|
2458
|
-
agent_id: &str,
|
|
2459
|
-
) -> Option<String> {
|
|
2460
|
-
state
|
|
2461
|
-
.get("coordinator")
|
|
2462
|
-
.and_then(|coordinator| {
|
|
2463
|
-
coordinator
|
|
2464
|
-
.get("awaiting_human_confirm_seen")
|
|
2465
|
-
.or_else(|| coordinator.get("awaiting_human_confirm"))
|
|
2466
|
-
})
|
|
2467
|
-
.and_then(|by_team| by_team.get(team))
|
|
2468
|
-
.and_then(|by_agent| by_agent.get(agent_id))
|
|
2469
|
-
.and_then(|record| record.get("fingerprint"))
|
|
2470
|
-
.and_then(Value::as_str)
|
|
2471
|
-
.map(str::to_string)
|
|
2472
|
-
}
|
|
2473
|
-
|
|
2474
|
-
fn remember_state_awaiting_human_confirm(
|
|
2475
|
-
state: &mut Value,
|
|
2476
|
-
fact: &crate::provider::AwaitingHumanConfirmFact,
|
|
2477
|
-
) {
|
|
2478
|
-
let Some(state_obj) = state.as_object_mut() else {
|
|
2479
|
-
return;
|
|
2480
|
-
};
|
|
2481
|
-
let coordinator = state_obj
|
|
2482
|
-
.entry("coordinator".to_string())
|
|
2483
|
-
.or_insert_with(|| serde_json::json!({}));
|
|
2484
|
-
if !coordinator.is_object() {
|
|
2485
|
-
*coordinator = serde_json::json!({});
|
|
2486
|
-
}
|
|
2487
|
-
let Some(coord_obj) = coordinator.as_object_mut() else {
|
|
2488
|
-
return;
|
|
2489
|
-
};
|
|
2490
|
-
let awaiting = coord_obj
|
|
2491
|
-
.entry("awaiting_human_confirm_seen".to_string())
|
|
2492
|
-
.or_insert_with(|| serde_json::json!({}));
|
|
2493
|
-
if !awaiting.is_object() {
|
|
2494
|
-
*awaiting = serde_json::json!({});
|
|
2495
|
-
}
|
|
2496
|
-
let Some(awaiting_obj) = awaiting.as_object_mut() else {
|
|
2497
|
-
return;
|
|
2498
|
-
};
|
|
2499
|
-
let team_entry = awaiting_obj
|
|
2500
|
-
.entry(fact.team.clone())
|
|
2501
|
-
.or_insert_with(|| serde_json::json!({}));
|
|
2502
|
-
if !team_entry.is_object() {
|
|
2503
|
-
*team_entry = serde_json::json!({});
|
|
2504
|
-
}
|
|
2505
|
-
if let Some(team_obj) = team_entry.as_object_mut() {
|
|
2506
|
-
team_obj.insert(
|
|
2507
|
-
fact.agent_id.clone(),
|
|
2508
|
-
serde_json::json!({
|
|
2509
|
-
"team": fact.team,
|
|
2510
|
-
"team_id": fact.team,
|
|
2511
|
-
"owner_team_id": fact.team,
|
|
2512
|
-
"agent_id": fact.agent_id,
|
|
2513
|
-
"fingerprint": fact.fingerprint,
|
|
2514
|
-
"dedupe_key": fact.dedupe_key,
|
|
2515
|
-
"prompt_kind": fact.prompt_kind,
|
|
2516
|
-
"reason": fact.reason,
|
|
2517
|
-
}),
|
|
2518
|
-
);
|
|
2519
|
-
}
|
|
2520
|
-
}
|
|
2521
|
-
|
|
2522
|
-
fn clear_state_awaiting_human_confirm(state: &mut Value, team: &str, agent_id: &str) {
|
|
2523
|
-
let Some(awaiting_obj) = state
|
|
2524
|
-
.get_mut("coordinator")
|
|
2525
|
-
.and_then(|coordinator| coordinator.get_mut("awaiting_human_confirm_seen"))
|
|
2526
|
-
.and_then(Value::as_object_mut)
|
|
2527
|
-
else {
|
|
2528
|
-
return;
|
|
2529
|
-
};
|
|
2530
|
-
let remove_team = if let Some(team_value) = awaiting_obj.get_mut(team) {
|
|
2531
|
-
if let Some(team_obj) = team_value.as_object_mut() {
|
|
2532
|
-
team_obj.remove(agent_id);
|
|
2533
|
-
team_obj.is_empty()
|
|
2534
|
-
} else {
|
|
2535
|
-
true
|
|
2536
|
-
}
|
|
2537
|
-
} else {
|
|
2538
|
-
false
|
|
2539
|
-
};
|
|
2540
|
-
if remove_team {
|
|
2541
|
-
awaiting_obj.remove(team);
|
|
2542
|
-
}
|
|
2543
|
-
}
|
|
2544
|
-
|
|
2545
|
-
fn remember_awaiting_human_confirm(
|
|
2546
|
-
agent: &mut Value,
|
|
2547
|
-
fact: &crate::provider::AwaitingHumanConfirmFact,
|
|
2548
|
-
) {
|
|
2549
|
-
if let Some(agent_obj) = agent.as_object_mut() {
|
|
2550
|
-
agent_obj.insert(
|
|
2551
|
-
"awaiting_human_confirm".to_string(),
|
|
2552
|
-
serde_json::json!({
|
|
2553
|
-
"team": fact.team,
|
|
2554
|
-
"team_id": fact.team,
|
|
2555
|
-
"owner_team_id": fact.team,
|
|
2556
|
-
"agent_id": fact.agent_id,
|
|
2557
|
-
"fingerprint": fact.fingerprint,
|
|
2558
|
-
"dedupe_key": fact.dedupe_key,
|
|
2559
|
-
"prompt_kind": fact.prompt_kind,
|
|
2560
|
-
"reason": fact.reason,
|
|
2561
|
-
}),
|
|
2562
|
-
);
|
|
2563
|
-
}
|
|
2564
|
-
}
|
|
2565
|
-
|
|
2566
|
-
fn clear_awaiting_human_confirm(agent: &mut Value) {
|
|
2567
|
-
if let Some(agent_obj) = agent.as_object_mut() {
|
|
2568
|
-
agent_obj.remove("awaiting_human_confirm");
|
|
2569
|
-
}
|
|
2570
|
-
}
|
|
2571
|
-
|
|
2572
|
-
/// 0.5.32 follow-up (`.team/artifacts/0532-r1-real-fail-triage.md` §6/§7):
|
|
2573
|
-
/// "fresh cohort" = the state a restart / start-agent clear helper leaves
|
|
2574
|
-
/// behind, before any post-spawn tick has repopulated observations.
|
|
2575
|
-
/// Requires `spawned_at` present so the gate fires only for real
|
|
2576
|
-
/// restart/start cohorts; coordinator-only fixtures without a spawn
|
|
2577
|
-
/// boundary keep pre-0.5.32 pane fallback behavior (spine2 sync_health
|
|
2578
|
-
/// tests seed bare provider/window/pane_id and expect `last_output_at`
|
|
2579
|
-
/// to be recorded on the first delta).
|
|
2580
|
-
fn agent_is_fresh_cohort(agent: &Value) -> bool {
|
|
2581
|
-
let spawned_at_present = agent
|
|
2582
|
-
.get("spawned_at")
|
|
2583
|
-
.and_then(Value::as_str)
|
|
2584
|
-
.is_some_and(|s| !s.is_empty());
|
|
2585
|
-
if !spawned_at_present {
|
|
2586
|
-
return false;
|
|
2587
|
-
}
|
|
2588
|
-
const CLEARED_FIELDS: &[&str] = &[
|
|
2589
|
-
"activity",
|
|
2590
|
-
"worker_state",
|
|
2591
|
-
"last_output_at",
|
|
2592
|
-
"last_output_hash",
|
|
2593
|
-
"current_turn_message_id",
|
|
2594
|
-
"current_task_id",
|
|
2595
|
-
"task_id",
|
|
2596
|
-
];
|
|
2597
|
-
CLEARED_FIELDS.iter().all(|field| {
|
|
2598
|
-
agent
|
|
2599
|
-
.get(*field)
|
|
2600
|
-
.map(|value| value.is_null())
|
|
2601
|
-
.unwrap_or(true)
|
|
2602
|
-
})
|
|
2603
|
-
}
|
|
2604
|
-
|
|
2605
|
-
/// 0.5.32 follow-up: is there a structural pane signal in `text`?
|
|
2606
|
-
/// True when `latest_prompt_signal` recognizes anything (idle prompt char,
|
|
2607
|
-
/// working spinner/keyword, or fake READY/WORKING marker). When true, the
|
|
2608
|
-
/// classifier will produce a definite non-`recent_provider_output` activity
|
|
2609
|
-
/// on its own, so the freshness gate must not suppress `last_output_at`.
|
|
2610
|
-
fn latest_pane_signal_is_structural(text: &str) -> bool {
|
|
2611
|
-
crate::messaging::helpers::latest_prompt_signal(text).is_some()
|
|
2612
|
-
}
|
|
2613
|
-
|
|
2614
|
-
/// Python approvals/status.py:68-72 — sha256 the scrollback, compare to the stored
|
|
2615
|
-
/// `last_output_hash`; only a CHANGED digest counts as advanced output (and stores
|
|
2616
|
-
/// the new digest).
|
|
2617
|
-
fn scrollback_digest_advanced(agent: &mut Value, text: &str) -> bool {
|
|
2618
|
-
use sha2::Digest;
|
|
2619
|
-
let mut hasher = sha2::Sha256::new();
|
|
2620
|
-
hasher.update(text.as_bytes());
|
|
2621
|
-
let digest = format!("{:x}", hasher.finalize());
|
|
2622
|
-
let unchanged = agent
|
|
2623
|
-
.get("last_output_hash")
|
|
2624
|
-
.and_then(Value::as_str)
|
|
2625
|
-
.is_some_and(|stored| stored == digest);
|
|
2626
|
-
if unchanged {
|
|
2627
|
-
return false;
|
|
2628
|
-
}
|
|
2629
|
-
if let Some(obj) = agent.as_object_mut() {
|
|
2630
|
-
obj.insert("last_output_hash".to_string(), serde_json::json!(digest));
|
|
2631
|
-
}
|
|
2632
|
-
true
|
|
2633
|
-
}
|
|
2634
|
-
|
|
2635
|
-
fn write_activity(
|
|
2636
|
-
agent: &mut Value,
|
|
2637
|
-
activity: &crate::messaging::AgentActivity,
|
|
2638
|
-
output_advanced: bool,
|
|
2639
|
-
) -> Option<String> {
|
|
2640
|
-
let previous_last_output = agent
|
|
2641
|
-
.get("last_output_at")
|
|
2642
|
-
.and_then(Value::as_str)
|
|
2643
|
-
.map(str::to_string);
|
|
2644
|
-
// 0.4.x Phase 1: resolve the 5-state worker runtime state alongside the
|
|
2645
|
-
// legacy `activity` write. CR R3: activity field is preserved as the
|
|
2646
|
-
// deprecated surface; worker_state is the new canonical surface.
|
|
2647
|
-
let worker_state = resolve_worker_runtime_state_with_fg_pgrp(agent, Some(activity));
|
|
2648
|
-
let Some(agent_obj) = agent.as_object_mut() else {
|
|
2649
|
-
return previous_last_output;
|
|
2650
|
-
};
|
|
2651
|
-
let status = activity_status_wire(activity.status);
|
|
2652
|
-
agent_obj.insert(
|
|
2653
|
-
"activity".to_string(),
|
|
2654
|
-
serde_json::json!({
|
|
2655
|
-
"status": status,
|
|
2656
|
-
"confidence": activity.confidence,
|
|
2657
|
-
"rationale": activity.rationale,
|
|
2658
|
-
}),
|
|
2659
|
-
);
|
|
2660
|
-
agent_obj.insert(
|
|
2661
|
-
"worker_state".to_string(),
|
|
2662
|
-
serde_json::json!(worker_state.as_wire()),
|
|
2663
|
-
);
|
|
2664
|
-
if output_advanced {
|
|
2665
|
-
let last_output_at = chrono::Utc::now().to_rfc3339();
|
|
2666
|
-
agent_obj.insert(
|
|
2667
|
-
"last_output_at".to_string(),
|
|
2668
|
-
serde_json::json!(last_output_at.clone()),
|
|
2669
|
-
);
|
|
2670
|
-
return Some(last_output_at);
|
|
2671
|
-
}
|
|
2672
|
-
previous_last_output
|
|
2673
|
-
}
|
|
2674
|
-
|
|
2675
|
-
/// 0.4.x Phase 1 worker-runtime-state resolver.
|
|
2676
|
-
///
|
|
2677
|
-
/// Pure resolution from (agent JSON, optional activity classifier output).
|
|
2678
|
-
/// Reads `agent.pane_id`, `agent.pane_pid`, `agent.awaiting_human_confirm`,
|
|
2679
|
-
/// and the trust/startup approval flags. Uses
|
|
2680
|
-
/// `os_probe::pane_foreground_and_root_pgrp` to detect a child process
|
|
2681
|
-
/// occupying the terminal foreground.
|
|
2682
|
-
///
|
|
2683
|
-
/// Precedence (matches Phase 1 plan §4):
|
|
2684
|
-
/// 1. Dead — pane_pid exists but `getpgid` returns no process,
|
|
2685
|
-
/// OR foreground probe explicitly reports the PID gone.
|
|
2686
|
-
/// 2. Blocked — agent.awaiting_human_confirm / startup-trust flags.
|
|
2687
|
-
/// 3. Busy — fg_pgrp != root_pgrp (child occupies terminal) OR
|
|
2688
|
-
/// JSONL classifier reports Working.
|
|
2689
|
-
/// 4. ProbablyIdle — JSONL classifier reports Idle AND no fg-pgrp
|
|
2690
|
-
/// conflict (or probe unavailable + activity idle).
|
|
2691
|
-
/// 5. Unknown — missing pane_pid, probe error, or no decisive
|
|
2692
|
-
/// signal. Iron law: never silently Idle.
|
|
2693
|
-
/// ---
|
|
2694
|
-
/// purpose: 由 agent 状态与活动分类结果解析出 worker 的运行态
|
|
2695
|
-
/// params:
|
|
2696
|
-
/// agent: agent 状态对象;读 pane_pid、awaiting_human_confirm 与审批/信任标志
|
|
2697
|
-
/// activity: JSONL 活动分类结果;None 表示本轮没有分类信号
|
|
2698
|
-
/// returns: 按优先级判定——先 Blocked(等人确认/等信任提示),再用前台进程组探测判 Busy,再由活动分类给 Busy/ProbablyIdle;任何一步缺信号都落到 Unknown。铁律是绝不在信息不足时静默判 Idle
|
|
2699
|
-
/// ---
|
|
2700
|
-
pub(crate) fn resolve_worker_runtime_state_with_fg_pgrp(
|
|
2701
|
-
agent: &Value,
|
|
2702
|
-
activity: Option<&crate::messaging::AgentActivity>,
|
|
2703
|
-
) -> crate::messaging::WorkerRuntimeState {
|
|
2704
|
-
use crate::messaging::{ActivityStatus, WorkerRuntimeState};
|
|
2705
|
-
|
|
2706
|
-
// §4.2 Blocked: trust prompt / awaiting human confirm.
|
|
2707
|
-
let blocked = agent
|
|
2708
|
-
.get("awaiting_human_confirm")
|
|
2709
|
-
.and_then(Value::as_bool)
|
|
2710
|
-
.unwrap_or(false)
|
|
2711
|
-
|| agent
|
|
2712
|
-
.get("approval")
|
|
2713
|
-
.and_then(Value::as_str)
|
|
2714
|
-
.is_some_and(|s| s == "pending" || s == "awaiting_trust_prompt")
|
|
2715
|
-
|| agent
|
|
2716
|
-
.get("approval_status")
|
|
2717
|
-
.and_then(Value::as_str)
|
|
2718
|
-
.is_some_and(|s| s == "pending" || s == "awaiting_trust_prompt");
|
|
2719
|
-
if blocked {
|
|
2720
|
-
return WorkerRuntimeState::Blocked;
|
|
2721
|
-
}
|
|
2722
|
-
|
|
2723
|
-
// §4.3 Busy via fg-pgrp probe (when pane_pid is available).
|
|
2724
|
-
let pane_pid = agent
|
|
2725
|
-
.get("pane_pid")
|
|
2726
|
-
.and_then(Value::as_u64)
|
|
2727
|
-
.map(|p| p as u32);
|
|
2728
|
-
if let Some(pid) = pane_pid {
|
|
2729
|
-
match crate::os_probe::pane_foreground_and_root_pgrp(pid) {
|
|
2730
|
-
Ok(Some((tpgid, pgid))) => {
|
|
2731
|
-
if tpgid != pgid {
|
|
2732
|
-
return WorkerRuntimeState::Busy;
|
|
2733
|
-
}
|
|
2734
|
-
// Same pgrp — pane process owns foreground. Defer to
|
|
2735
|
-
// JSONL/activity for finer state.
|
|
2736
|
-
}
|
|
2737
|
-
Ok(None) => {
|
|
2738
|
-
// Probe could not read: missing PID, no controlling
|
|
2739
|
-
// terminal, ps unavailable. Fall through to activity
|
|
2740
|
-
// classifier; if that's also missing/uncertain, the
|
|
2741
|
-
// function returns Unknown.
|
|
2742
|
-
}
|
|
2743
|
-
Err(_) => {
|
|
2744
|
-
// Subprocess error — degrade to Unknown unless activity
|
|
2745
|
-
// is decisive.
|
|
2746
|
-
}
|
|
2747
|
-
}
|
|
2748
|
-
}
|
|
2749
|
-
|
|
2750
|
-
// §4.4/4.5 Activity-derived classification.
|
|
2751
|
-
if let Some(activity) = activity {
|
|
2752
|
-
return match activity.status {
|
|
2753
|
-
ActivityStatus::Working => WorkerRuntimeState::Busy,
|
|
2754
|
-
ActivityStatus::Idle => WorkerRuntimeState::ProbablyIdle,
|
|
2755
|
-
ActivityStatus::Stuck | ActivityStatus::Uncertain => WorkerRuntimeState::Unknown,
|
|
2756
|
-
};
|
|
2757
|
-
}
|
|
2758
|
-
|
|
2759
|
-
WorkerRuntimeState::Unknown
|
|
2760
|
-
}
|
|
2761
|
-
|
|
2762
|
-
fn activity_status_wire(status: crate::messaging::ActivityStatus) -> &'static str {
|
|
2763
|
-
match status {
|
|
2764
|
-
crate::messaging::ActivityStatus::Idle => "idle",
|
|
2765
|
-
crate::messaging::ActivityStatus::Working => "working",
|
|
2766
|
-
crate::messaging::ActivityStatus::Stuck => "stuck",
|
|
2767
|
-
crate::messaging::ActivityStatus::Uncertain => "uncertain",
|
|
2768
|
-
}
|
|
2769
|
-
}
|
|
2770
|
-
|
|
2771
|
-
fn agent_health_status_wire(status: crate::messaging::ActivityStatus) -> &'static str {
|
|
2772
|
-
match status {
|
|
2773
|
-
crate::messaging::ActivityStatus::Idle => "IDLE",
|
|
2774
|
-
crate::messaging::ActivityStatus::Working => "WORKING",
|
|
2775
|
-
crate::messaging::ActivityStatus::Stuck => "STUCK",
|
|
2776
|
-
crate::messaging::ActivityStatus::Uncertain => "UNKNOWN",
|
|
2777
|
-
}
|
|
2778
|
-
}
|
|
2779
|
-
|
|
2780
|
-
/// 0.4.x Phase 1: read the worker_state wire string that `write_activity`
|
|
2781
|
-
/// persisted alongside the legacy activity. Returns None when no worker_state
|
|
2782
|
-
/// has been written yet (older state row pre-upgrade).
|
|
2783
|
-
/// ---
|
|
2784
|
-
/// purpose: 读回 write_activity 与活动一起持久化的 worker_state 线字符串
|
|
2785
|
-
/// params:
|
|
2786
|
-
/// agent: agent 状态对象
|
|
2787
|
-
/// returns: 已写过 worker_state 时解析出运行态;升级前的旧状态行没有该字段,返回 None
|
|
2788
|
-
/// ---
|
|
2789
|
-
/// 现状:本仓内没有任何调用点(仅定义),与其他 pub(crate) 项不同——按机器事实记录,
|
|
2790
|
-
/// 不据此推断它「已接线」。
|
|
2791
|
-
pub(crate) fn agent_worker_state(agent: &Value) -> Option<crate::messaging::WorkerRuntimeState> {
|
|
2792
|
-
agent
|
|
2793
|
-
.get("worker_state")
|
|
2794
|
-
.and_then(Value::as_str)
|
|
2795
|
-
.map(crate::messaging::WorkerRuntimeState::parse_wire)
|
|
2796
|
-
}
|
|
2797
|
-
|
|
2798
|
-
fn write_agent_health(
|
|
2799
|
-
store: &crate::message_store::MessageStore,
|
|
2800
|
-
team: &str,
|
|
2801
|
-
agent_id: &str,
|
|
2802
|
-
agent: &Value,
|
|
2803
|
-
activity: &crate::messaging::AgentActivity,
|
|
2804
|
-
last_output_at: Option<&str>,
|
|
2805
|
-
) -> Result<(), crate::messaging::MessagingError> {
|
|
2806
|
-
let conn = crate::db::schema::open_db(store.db_path())?;
|
|
2807
|
-
let status = agent_health_status_wire(activity.status);
|
|
2808
|
-
let updated_at = chrono::Utc::now().to_rfc3339();
|
|
2809
|
-
let context_usage_pct = agent
|
|
2810
|
-
.get("context_usage_pct")
|
|
2811
|
-
.or_else(|| agent.get("context_usage_percent"))
|
|
2812
|
-
.and_then(Value::as_i64);
|
|
2813
|
-
// Phase-DX E2: read the renamed `current_turn_message_id` (leader→worker turn
|
|
2814
|
-
// proxy, written by delivery::arm_turn_open) with fallbacks to the legacy field
|
|
2815
|
-
// names for backwards state compatibility. The SQL column stays `current_task_id`
|
|
2816
|
-
// (agent_health schema) — a rename would require a DB migration, which Phase-DX
|
|
2817
|
-
// forbids.
|
|
2818
|
-
let current_task_id = agent
|
|
2819
|
-
.get("current_turn_message_id")
|
|
2820
|
-
.or_else(|| agent.get("current_task_id"))
|
|
2821
|
-
.or_else(|| agent.get("task_id"))
|
|
2822
|
-
.and_then(Value::as_str);
|
|
2823
|
-
conn.execute(
|
|
2824
|
-
"insert into agent_health(
|
|
2825
|
-
owner_team_id, agent_id, status, last_output_at, context_usage_pct, current_task_id, updated_at
|
|
2826
|
-
) values (?1, ?2, ?3, ?4, ?5, ?6, ?7)
|
|
2827
|
-
on conflict(owner_team_id, agent_id) do update set
|
|
2828
|
-
status = excluded.status,
|
|
2829
|
-
last_output_at = coalesce(excluded.last_output_at, agent_health.last_output_at),
|
|
2830
|
-
context_usage_pct = excluded.context_usage_pct,
|
|
2831
|
-
current_task_id = excluded.current_task_id,
|
|
2832
|
-
updated_at = excluded.updated_at",
|
|
2833
|
-
rusqlite::params![
|
|
2834
|
-
team,
|
|
2835
|
-
agent_id,
|
|
2836
|
-
status,
|
|
2837
|
-
last_output_at,
|
|
2838
|
-
context_usage_pct,
|
|
2839
|
-
current_task_id,
|
|
2840
|
-
updated_at,
|
|
2841
|
-
],
|
|
2842
|
-
)?;
|
|
2843
|
-
Ok(())
|
|
2844
|
-
}
|
|
2845
|
-
|
|
2846
|
-
fn read_pid_file(path: &Path) -> Option<Pid> {
|
|
2847
|
-
let text = std::fs::read_to_string(path).ok()?;
|
|
2848
|
-
let pid = text.trim().parse::<u32>().ok()?;
|
|
2849
|
-
Some(Pid(pid))
|
|
2850
|
-
}
|
|
2851
|
-
|
|
2852
|
-
fn remove_file_if_exists(path: &Path) -> Result<(), std::io::Error> {
|
|
2853
|
-
match std::fs::remove_file(path) {
|
|
2854
|
-
Ok(()) => Ok(()),
|
|
2855
|
-
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
|
|
2856
|
-
Err(e) => Err(e),
|
|
2857
|
-
}
|
|
2858
|
-
}
|
|
2859
|
-
|
|
2860
|
-
fn notify_session_missing(
|
|
2861
|
-
workspace: &Path,
|
|
2862
|
-
state: &Value,
|
|
2863
|
-
event_log: &EventLog,
|
|
2864
|
-
session_name: &str,
|
|
2865
|
-
) -> Result<(), TickError> {
|
|
2866
|
-
let content = format!(
|
|
2867
|
-
"coordinator.session_missing\nerror: tmux session {session_name} is missing; coordinator is stopping\naction: restart the team or recover the missing tmux session\nlog: .team/logs/events.jsonl"
|
|
2868
|
-
);
|
|
2869
|
-
let dedupe_key = format!("coordinator.session_missing:{session_name}");
|
|
2870
|
-
match crate::messaging::send_to_leader_receiver(
|
|
2871
|
-
workspace,
|
|
2872
|
-
state,
|
|
2873
|
-
"leader",
|
|
2874
|
-
&content,
|
|
2875
|
-
None,
|
|
2876
|
-
"coordinator",
|
|
2877
|
-
false,
|
|
2878
|
-
Some(&dedupe_key),
|
|
2879
|
-
event_log,
|
|
2880
|
-
) {
|
|
2881
|
-
Ok(outcome) => {
|
|
2882
|
-
event_log.write(
|
|
2883
|
-
"coordinator.session_missing_alert",
|
|
2884
|
-
serde_json::json!({
|
|
2885
|
-
"session": session_name,
|
|
2886
|
-
"leader_notification_status": crate::messaging::helpers::status_wire(outcome.status),
|
|
2887
|
-
"message_id": outcome.message_id,
|
|
2888
|
-
}),
|
|
2889
|
-
)?;
|
|
2890
|
-
}
|
|
2891
|
-
Err(error) => {
|
|
2892
|
-
event_log.write(
|
|
2893
|
-
"coordinator.session_missing_alert_failed",
|
|
2894
|
-
serde_json::json!({
|
|
2895
|
-
"session": session_name,
|
|
2896
|
-
"error": error.to_string(),
|
|
2897
|
-
"action": "inspect .team/logs/events.jsonl and restart the team",
|
|
2898
|
-
}),
|
|
2899
|
-
)?;
|
|
2900
|
-
}
|
|
2901
|
-
}
|
|
2902
|
-
Ok(())
|
|
2903
|
-
}
|
|
2904
|
-
|
|
2905
|
-
#[cfg(test)]
|
|
2906
|
-
#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
|
2907
|
-
mod u1_tests {
|
|
2908
|
-
use super::*;
|
|
2909
|
-
use std::io::Write as _;
|
|
2910
|
-
|
|
2911
|
-
struct CaptureFailureRegistry;
|
|
2912
|
-
|
|
2913
|
-
impl ProviderRegistry for CaptureFailureRegistry {
|
|
2914
|
-
fn adapter_for(
|
|
2915
|
-
&self,
|
|
2916
|
-
provider: crate::provider::Provider,
|
|
2917
|
-
) -> Box<dyn crate::provider::ProviderAdapter> {
|
|
2918
|
-
Box::new(
|
|
2919
|
-
crate::session_capture::test_support::CaptureCandidatesAdapter::new(
|
|
2920
|
-
provider,
|
|
2921
|
-
Some("w1"),
|
|
2922
|
-
"capture exploded",
|
|
2923
|
-
),
|
|
2924
|
-
)
|
|
2925
|
-
}
|
|
2926
|
-
}
|
|
2927
|
-
|
|
2928
|
-
#[test]
|
|
2929
|
-
fn tick_logs_capture_missing_failure_and_continues() {
|
|
2930
|
-
let dir = std::env::temp_dir().join(format!(
|
|
2931
|
-
"team-agent-u1-capture-missing-{}-{}",
|
|
2932
|
-
std::process::id(),
|
|
2933
|
-
std::time::SystemTime::now()
|
|
2934
|
-
.duration_since(std::time::UNIX_EPOCH)
|
|
2935
|
-
.unwrap()
|
|
2936
|
-
.as_nanos()
|
|
2937
|
-
));
|
|
2938
|
-
std::fs::create_dir_all(&dir).unwrap();
|
|
2939
|
-
crate::state::persist::save_runtime_state(
|
|
2940
|
-
&dir,
|
|
2941
|
-
&serde_json::json!({
|
|
2942
|
-
"agents": {
|
|
2943
|
-
"w1": {
|
|
2944
|
-
"provider": "codex",
|
|
2945
|
-
"status": "running",
|
|
2946
|
-
"spawn_cwd": dir.to_string_lossy()
|
|
2947
|
-
}
|
|
2948
|
-
}
|
|
2949
|
-
}),
|
|
2950
|
-
)
|
|
2951
|
-
.unwrap();
|
|
2952
|
-
let coordinator = Coordinator::for_test(
|
|
2953
|
-
WorkspacePath::new(dir.clone()),
|
|
2954
|
-
Box::new(CaptureFailureRegistry),
|
|
2955
|
-
Box::new(crate::transport::test_support::OfflineTransport::new()),
|
|
2956
|
-
None,
|
|
2957
|
-
None,
|
|
2958
|
-
);
|
|
2959
|
-
|
|
2960
|
-
let report = coordinator
|
|
2961
|
-
.tick()
|
|
2962
|
-
.expect("capture_missing failure must be logged and not abort the tick");
|
|
2963
|
-
|
|
2964
|
-
assert!(report.ok, "tick should continue to a successful report");
|
|
2965
|
-
let events_path = crate::model::paths::logs_dir(&dir).join("events.jsonl");
|
|
2966
|
-
let events = std::fs::read_to_string(events_path).unwrap();
|
|
2967
|
-
let has_capture_failure = events.lines().any(|line| {
|
|
2968
|
-
serde_json::from_str::<Value>(line)
|
|
2969
|
-
.ok()
|
|
2970
|
-
.and_then(|event| {
|
|
2971
|
-
event
|
|
2972
|
-
.get("event")
|
|
2973
|
-
.and_then(Value::as_str)
|
|
2974
|
-
.map(str::to_string)
|
|
2975
|
-
})
|
|
2976
|
-
.as_deref()
|
|
2977
|
-
== Some("coordinator.tick.capture_missing_failed")
|
|
2978
|
-
});
|
|
2979
|
-
assert!(
|
|
2980
|
-
has_capture_failure,
|
|
2981
|
-
"capture_missing failure must be visible in events.jsonl; got {events}"
|
|
2982
|
-
);
|
|
2983
|
-
}
|
|
2984
|
-
|
|
2985
|
-
#[test]
|
|
2986
|
-
fn daemon_team_key_projection_beats_stale_root_active_team_key() {
|
|
2987
|
-
let dir = std::env::temp_dir().join(format!(
|
|
2988
|
-
"team-agent-daemon-team-key-{}-{}",
|
|
2989
|
-
std::process::id(),
|
|
2990
|
-
std::time::SystemTime::now()
|
|
2991
|
-
.duration_since(std::time::UNIX_EPOCH)
|
|
2992
|
-
.unwrap()
|
|
2993
|
-
.as_nanos()
|
|
2994
|
-
));
|
|
2995
|
-
std::fs::create_dir_all(&dir).unwrap();
|
|
2996
|
-
let raw = serde_json::json!({
|
|
2997
|
-
"active_team_key": "stale-b",
|
|
2998
|
-
"session_name": "team-stale-b",
|
|
2999
|
-
"teams": {
|
|
3000
|
-
"fresh-a": {
|
|
3001
|
-
"active_team_key": "fresh-a",
|
|
3002
|
-
"team_key": "fresh-a",
|
|
3003
|
-
"session_name": "team-fresh-a",
|
|
3004
|
-
"agents": {}
|
|
3005
|
-
},
|
|
3006
|
-
"stale-b": {
|
|
3007
|
-
"active_team_key": "stale-b",
|
|
3008
|
-
"team_key": "stale-b",
|
|
3009
|
-
"session_name": "team-stale-b",
|
|
3010
|
-
"agents": {}
|
|
3011
|
-
}
|
|
3012
|
-
}
|
|
3013
|
-
});
|
|
3014
|
-
crate::state::persist::save_runtime_state(&dir, &raw).unwrap();
|
|
3015
|
-
|
|
3016
|
-
let selected =
|
|
3017
|
-
coordinator_team_scoped_state(&dir, &raw, Some("fresh-a")).expect("selected team");
|
|
3018
|
-
|
|
3019
|
-
assert_eq!(
|
|
3020
|
-
selected.get("session_name").and_then(Value::as_str),
|
|
3021
|
-
Some("team-fresh-a")
|
|
3022
|
-
);
|
|
3023
|
-
}
|
|
3024
|
-
}
|