agent-control-plane 0.4.9 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/README.md +72 -9
  2. package/npm/bin/agent-control-plane.js +1 -1
  3. package/package.json +39 -33
  4. package/tools/bin/debug-session.sh +106 -0
  5. package/tools/bin/flow-runtime-doctor-linux.sh +136 -0
  6. package/tools/bin/flow-runtime-doctor.sh +5 -1
  7. package/tools/bin/install-project-systemd.sh +255 -0
  8. package/tools/bin/project-runtimectl.sh +45 -0
  9. package/tools/bin/project-systemd-bootstrap.sh +74 -0
  10. package/tools/bin/uninstall-project-systemd.sh +87 -0
  11. package/tools/dashboard/app.js +198 -5
  12. package/tools/dashboard/issue_queue_state.py +101 -0
  13. package/tools/dashboard/server.py +123 -1
  14. package/tools/dashboard/styles.css +526 -455
  15. package/tools/bin/agent-cleanup-worktree +0 -247
  16. package/tools/bin/agent-github-update-labels +0 -105
  17. package/tools/bin/agent-init-worktree +0 -216
  18. package/tools/bin/agent-project-archive-run +0 -52
  19. package/tools/bin/agent-project-capture-worker +0 -46
  20. package/tools/bin/agent-project-catch-up-issue-pr-links +0 -118
  21. package/tools/bin/agent-project-catch-up-merged-prs +0 -195
  22. package/tools/bin/agent-project-catch-up-scheduled-issue-retries +0 -123
  23. package/tools/bin/agent-project-cleanup-session +0 -513
  24. package/tools/bin/agent-project-detached-launch +0 -127
  25. package/tools/bin/agent-project-heartbeat-loop +0 -1029
  26. package/tools/bin/agent-project-open-issue-worktree +0 -89
  27. package/tools/bin/agent-project-open-pr-worktree +0 -80
  28. package/tools/bin/agent-project-publish-issue-pr +0 -468
  29. package/tools/bin/agent-project-reconcile-issue-session +0 -1409
  30. package/tools/bin/agent-project-reconcile-pr-session +0 -1288
  31. package/tools/bin/agent-project-retry-state +0 -158
  32. package/tools/bin/agent-project-run-claude-session +0 -805
  33. package/tools/bin/agent-project-run-codex-resilient +0 -963
  34. package/tools/bin/agent-project-run-codex-session +0 -435
  35. package/tools/bin/agent-project-run-kilo-session +0 -369
  36. package/tools/bin/agent-project-run-ollama-session +0 -658
  37. package/tools/bin/agent-project-run-openclaw-session +0 -1309
  38. package/tools/bin/agent-project-run-opencode-session +0 -377
  39. package/tools/bin/agent-project-run-pi-session +0 -479
  40. package/tools/bin/agent-project-sync-anchor-repo +0 -139
  41. package/tools/bin/agent-project-sync-source-repo-main +0 -163
  42. package/tools/bin/agent-project-worker-status +0 -188
  43. package/tools/bin/branch-verification-guard.sh +0 -364
  44. package/tools/bin/capture-worker.sh +0 -18
  45. package/tools/bin/cleanup-worktree.sh +0 -52
  46. package/tools/bin/codex-quota +0 -31
  47. package/tools/bin/create-follow-up-issue.sh +0 -114
  48. package/tools/bin/dashboard-launchd-bootstrap.sh +0 -50
  49. package/tools/bin/issue-publish-localization-guard.sh +0 -142
  50. package/tools/bin/issue-publish-scope-guard.sh +0 -242
  51. package/tools/bin/issue-requires-local-workspace-install.sh +0 -31
  52. package/tools/bin/issue-resource-class.sh +0 -12
  53. package/tools/bin/kick-scheduler.sh +0 -75
  54. package/tools/bin/label-follow-up-issues.sh +0 -14
  55. package/tools/bin/new-pr-worktree.sh +0 -50
  56. package/tools/bin/new-worktree.sh +0 -49
  57. package/tools/bin/pr-risk.sh +0 -12
  58. package/tools/bin/prepare-worktree.sh +0 -142
  59. package/tools/bin/provider-cooldown-state.sh +0 -204
  60. package/tools/bin/publish-issue-worker.sh +0 -31
  61. package/tools/bin/reconcile-bootstrap-lib.sh +0 -113
  62. package/tools/bin/reconcile-issue-worker.sh +0 -34
  63. package/tools/bin/reconcile-pr-worker.sh +0 -34
  64. package/tools/bin/record-verification.sh +0 -71
  65. package/tools/bin/render-flow-config.sh +0 -98
  66. package/tools/bin/resident-issue-controller-lib.sh +0 -448
  67. package/tools/bin/retry-state.sh +0 -31
  68. package/tools/bin/reuse-issue-worktree.sh +0 -121
  69. package/tools/bin/run-codex-bypass.sh +0 -3
  70. package/tools/bin/run-codex-safe.sh +0 -3
  71. package/tools/bin/run-codex-task.sh +0 -280
  72. package/tools/bin/serve-dashboard.sh +0 -5
  73. package/tools/bin/start-issue-worker.sh +0 -943
  74. package/tools/bin/start-pr-fix-worker.sh +0 -528
  75. package/tools/bin/start-pr-merge-repair-worker.sh +0 -8
  76. package/tools/bin/start-pr-review-worker.sh +0 -261
  77. package/tools/bin/start-resident-issue-loop.sh +0 -499
  78. package/tools/bin/update-github-labels.sh +0 -14
  79. package/tools/bin/worker-status.sh +0 -19
  80. package/tools/bin/workflow-catalog.sh +0 -77
@@ -1,1029 +0,0 @@
1
- #!/usr/bin/env bash
2
- set -euo pipefail
3
-
4
- usage() {
5
- cat <<'EOF'
6
- Usage:
7
- agent-project-heartbeat-loop --repo-slug <owner/repo> --runs-root <path> --state-root <path> --issue-prefix <prefix> --pr-prefix <prefix> --hook-file <path> [options]
8
-
9
- Shared scheduler loop for project adapters. The shared engine owns:
10
- - completed worker reconciliation
11
- - retry/cooldown gating
12
- - concurrency accounting
13
- - issue/PR launch ordering
14
- - operational summary output
15
-
16
- Options:
17
- --memory-dir <path> Optional memory log root
18
- --max-concurrent-workers <n> Default 5
19
- --max-concurrent-heavy-workers <n> Default 1
20
- --max-concurrent-pr-workers <n> Default 5
21
- --max-recurring-issue-workers <n> Default 1
22
- --max-concurrent-scheduled-issue-workers <n> Default 0 (disabled)
23
- --max-concurrent-scheduled-heavy-workers <n> Default 1
24
- --max-concurrent-blocked-recovery-issue-workers <n> Default 0 (disabled)
25
- --blocked-recovery-cooldown-seconds <n> Default 0 (disabled)
26
- --max-open-agent-prs-for-recurring <n> Default 0 (disabled)
27
- --max-launches-per-pass <n> Default matches max-concurrent-workers
28
- --heavy-running-label <name> Default HEAVY_ISSUE
29
- --heavy-deferred-key <name> Default HEAVY_DEFERRED
30
- --heavy-deferred-message <text> Default heavy-queue message
31
- --help Show this help
32
- EOF
33
- }
34
-
35
- shared_agent_home="${SHARED_AGENT_HOME:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}"
36
- repo_slug=""
37
- runs_root=""
38
- state_root=""
39
- memory_dir=""
40
- issue_prefix=""
41
- pr_prefix=""
42
- hook_file=""
43
- max_concurrent_workers="5"
44
- max_concurrent_heavy_workers="1"
45
- max_concurrent_pr_workers="5"
46
- max_recurring_issue_workers="1"
47
- max_concurrent_scheduled_issue_workers="0"
48
- max_concurrent_scheduled_heavy_workers="1"
49
- max_concurrent_blocked_recovery_issue_workers="0"
50
- blocked_recovery_cooldown_seconds="0"
51
- max_open_agent_prs_for_recurring="0"
52
- max_launches_per_pass="$max_concurrent_workers"
53
- heavy_running_label="HEAVY_ISSUE"
54
- heavy_deferred_key="HEAVY_DEFERRED"
55
- heavy_deferred_message="Heavy issues remain queued until the heavy slot is free."
56
-
57
- while [[ $# -gt 0 ]]; do
58
- case "$1" in
59
- --repo-slug) repo_slug="${2:-}"; shift 2 ;;
60
- --runs-root) runs_root="${2:-}"; shift 2 ;;
61
- --state-root) state_root="${2:-}"; shift 2 ;;
62
- --memory-dir) memory_dir="${2:-}"; shift 2 ;;
63
- --issue-prefix) issue_prefix="${2:-}"; shift 2 ;;
64
- --pr-prefix) pr_prefix="${2:-}"; shift 2 ;;
65
- --hook-file) hook_file="${2:-}"; shift 2 ;;
66
- --max-concurrent-workers) max_concurrent_workers="${2:-}"; shift 2 ;;
67
- --max-concurrent-heavy-workers) max_concurrent_heavy_workers="${2:-}"; shift 2 ;;
68
- --max-concurrent-pr-workers) max_concurrent_pr_workers="${2:-}"; shift 2 ;;
69
- --max-recurring-issue-workers) max_recurring_issue_workers="${2:-}"; shift 2 ;;
70
- --max-concurrent-scheduled-issue-workers) max_concurrent_scheduled_issue_workers="${2:-}"; shift 2 ;;
71
- --max-concurrent-scheduled-heavy-workers) max_concurrent_scheduled_heavy_workers="${2:-}"; shift 2 ;;
72
- --max-concurrent-blocked-recovery-issue-workers) max_concurrent_blocked_recovery_issue_workers="${2:-}"; shift 2 ;;
73
- --blocked-recovery-cooldown-seconds) blocked_recovery_cooldown_seconds="${2:-}"; shift 2 ;;
74
- --max-open-agent-prs-for-recurring) max_open_agent_prs_for_recurring="${2:-}"; shift 2 ;;
75
- --max-launches-per-pass) max_launches_per_pass="${2:-}"; shift 2 ;;
76
- --heavy-running-label) heavy_running_label="${2:-}"; shift 2 ;;
77
- --heavy-deferred-key) heavy_deferred_key="${2:-}"; shift 2 ;;
78
- --heavy-deferred-message) heavy_deferred_message="${2:-}"; shift 2 ;;
79
- --help|-h) usage; exit 0 ;;
80
- *) echo "Unknown argument: $1" >&2; usage >&2; exit 1 ;;
81
- esac
82
- done
83
-
84
- if [[ -z "$repo_slug" || -z "$runs_root" || -z "$state_root" || -z "$issue_prefix" || -z "$pr_prefix" || -z "$hook_file" ]]; then
85
- usage >&2
86
- exit 1
87
- fi
88
-
89
- mkdir -p "$runs_root"
90
- memory_file=""
91
- tmux_sessions_cache=""
92
- tmux_sessions_cache_loaded="no"
93
- auth_wait_workers_cache=""
94
- auth_wait_workers_cache_loaded="no"
95
- all_running_workers_cache=""
96
- all_running_workers_cache_loaded="no"
97
- running_issue_workers_cache=""
98
- running_issue_workers_cache_loaded="no"
99
- running_pr_workers_cache=""
100
- running_pr_workers_cache_loaded="no"
101
- completed_workers_cache=""
102
- completed_workers_cache_loaded="no"
103
- ready_issue_ids_cache=""
104
- ready_issue_ids_cache_loaded="no"
105
- open_agent_pr_ids_cache=""
106
- open_agent_pr_ids_cache_loaded="no"
107
- running_issue_ids_cache=""
108
- running_issue_ids_cache_loaded="no"
109
- exclusive_issue_ids_cache=""
110
- exclusive_issue_ids_cache_loaded="no"
111
- exclusive_pr_ids_cache=""
112
- exclusive_pr_ids_cache_loaded="no"
113
- blocked_recovery_issue_ids_cache=""
114
- blocked_recovery_issue_ids_cache_loaded="no"
115
- ordered_ready_issue_ids_cache=""
116
- ordered_ready_issue_ids_cache_loaded="no"
117
- due_scheduled_issue_ids_cache=""
118
- due_scheduled_issue_ids_cache_loaded="no"
119
- due_blocked_recovery_issue_ids_cache=""
120
- due_blocked_recovery_issue_ids_cache_loaded="no"
121
- issue_attr_cache_dir=""
122
- pr_attr_cache_dir=""
123
- pr_risk_cache_dir=""
124
- pr_risk_runtime_cache_dir="${state_root}/pr-risk-cache"
125
- pr_risk_runtime_cache_ttl_seconds="${ACP_PR_RISK_CACHE_TTL_SECONDS:-${F_LOSNING_PR_RISK_CACHE_TTL_SECONDS:-300}}"
126
- recurring_rotation_dir="${state_root}/recurring"
127
- recurring_rotation_file="${recurring_rotation_dir}/last-launched-issue-id"
128
- scheduled_state_dir="${state_root}/scheduled-issues"
129
- blocked_recovery_state_dir="${state_root}/blocked-recovery-issues"
130
- pending_launch_dir="${state_root}/pending-launches"
131
- if [[ -n "$memory_dir" ]]; then
132
- mkdir -p "$memory_dir"
133
- memory_file="${memory_dir}/$(date +%F).md"
134
- touch "$memory_file"
135
- fi
136
- mkdir -p "$recurring_rotation_dir"
137
- mkdir -p "$scheduled_state_dir"
138
- mkdir -p "$blocked_recovery_state_dir"
139
- mkdir -p "$pending_launch_dir"
140
- mkdir -p "$pr_risk_runtime_cache_dir"
141
-
142
- record_memory() {
143
- local message="${1:?message required}"
144
- [[ -n "$memory_file" ]] || return 0
145
- printf -- "- %s %s\n" "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" "$message" >>"$memory_file"
146
- }
147
-
148
- print_block() {
149
- local header="${1:?header required}"
150
- local body="${2:-}"
151
- printf '%s\n' "$header"
152
- if [[ -n "$body" ]]; then
153
- printf '%s\n' "$body"
154
- fi
155
- }
156
-
157
- log_phase() {
158
- printf 'HEARTBEAT_LOOP_PHASE=%s\n' "${1:?phase required}"
159
- }
160
-
161
- if [[ ! -f "$hook_file" ]]; then
162
- echo "missing hook file: $hook_file" >&2
163
- exit 1
164
- fi
165
- # shellcheck source=/dev/null
166
- source "$hook_file"
167
-
168
- resident_lib="${shared_agent_home}/skills/openclaw/agent-control-plane/tools/bin/flow-resident-worker-lib.sh"
169
- if [[ -f "${resident_lib}" ]]; then
170
- # shellcheck source=/dev/null
171
- source "${resident_lib}"
172
- fi
173
-
174
- if declare -F flow_resident_issue_reap_stale_state >/dev/null 2>&1; then
175
- flow_resident_issue_reap_stale_state >/dev/null 2>&1 || true
176
- fi
177
-
178
- required_hooks=(
179
- heartbeat_list_ready_issue_ids
180
- heartbeat_list_open_agent_pr_ids
181
- heartbeat_issue_is_heavy
182
- heartbeat_issue_is_recurring
183
- heartbeat_sync_pr_labels
184
- heartbeat_pr_risk_json
185
- heartbeat_mark_issue_running
186
- heartbeat_issue_launch_failed
187
- heartbeat_start_issue_worker
188
- heartbeat_mark_pr_running
189
- heartbeat_clear_pr_running
190
- heartbeat_start_pr_merge_repair_worker
191
- heartbeat_start_pr_review_worker
192
- heartbeat_start_pr_fix_worker
193
- heartbeat_start_pr_ci_refresh
194
- heartbeat_reconcile_issue
195
- heartbeat_reconcile_pr
196
- )
197
-
198
- for hook_name in "${required_hooks[@]}"; do
199
- if ! declare -F "$hook_name" >/dev/null 2>&1; then
200
- echo "missing required heartbeat hook: $hook_name" >&2
201
- exit 1
202
- fi
203
- done
204
-
205
- if ! declare -F heartbeat_list_exclusive_issue_ids >/dev/null 2>&1; then
206
- heartbeat_list_exclusive_issue_ids() { :; }
207
- fi
208
-
209
- if ! declare -F heartbeat_list_exclusive_pr_ids >/dev/null 2>&1; then
210
- heartbeat_list_exclusive_pr_ids() { :; }
211
- fi
212
-
213
- if ! declare -F heartbeat_issue_is_exclusive >/dev/null 2>&1; then
214
- heartbeat_issue_is_exclusive() { printf 'no\n'; }
215
- fi
216
-
217
- if ! declare -F heartbeat_pr_is_exclusive >/dev/null 2>&1; then
218
- heartbeat_pr_is_exclusive() { printf 'no\n'; }
219
- fi
220
-
221
- if ! declare -F heartbeat_list_running_issue_ids >/dev/null 2>&1; then
222
- heartbeat_list_running_issue_ids() { :; }
223
- fi
224
-
225
- if ! declare -F heartbeat_list_blocked_recovery_issue_ids >/dev/null 2>&1; then
226
- heartbeat_list_blocked_recovery_issue_ids() { :; }
227
- fi
228
-
229
- if ! declare -F heartbeat_sync_issue_labels >/dev/null 2>&1; then
230
- heartbeat_sync_issue_labels() { :; }
231
- fi
232
-
233
-
234
- # --- Source modular libraries ---
235
-
236
- heartbeat_lib_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
237
- # shellcheck source=/dev/null
238
- source "${heartbeat_lib_dir}/heartbeat-loop-worker-lib.sh"
239
- # shellcheck source=/dev/null
240
- source "${heartbeat_lib_dir}/heartbeat-loop-cache-lib.sh"
241
- # shellcheck source=/dev/null
242
- source "${heartbeat_lib_dir}/heartbeat-loop-counting-lib.sh"
243
- # shellcheck source=/dev/null
244
- source "${heartbeat_lib_dir}/heartbeat-loop-scheduling-lib.sh"
245
- # shellcheck source=/dev/null
246
- source "${heartbeat_lib_dir}/heartbeat-loop-pr-strategy-lib.sh"
247
-
248
- # --- Launch staging & rollback ---
249
-
250
- launch_in_progress_kind=""
251
- launch_in_progress_id=""
252
- launch_in_progress_issue_is_heavy="no"
253
- launch_in_progress_cleanup_enabled="no"
254
- reserved_pr_launch_ids=""
255
-
256
- clear_launch_in_progress() {
257
- launch_in_progress_kind=""
258
- launch_in_progress_id=""
259
- launch_in_progress_issue_is_heavy="no"
260
- launch_in_progress_cleanup_enabled="no"
261
- }
262
-
263
- pr_launch_reserved() {
264
- local pr_number="${1:?pr number required}"
265
- [[ -n "$reserved_pr_launch_ids" ]] || return 1
266
- grep -Fxq "$pr_number" <<<"$reserved_pr_launch_ids"
267
- }
268
-
269
- reserve_pr_launch() {
270
- local pr_number="${1:?pr number required}"
271
- if pr_launch_reserved "$pr_number"; then
272
- return 0
273
- fi
274
- reserved_pr_launch_ids+="${pr_number}"$'\n'
275
- }
276
-
277
- cleanup_scheduler_caches() {
278
- tmux_sessions_cache=""
279
- tmux_sessions_cache_loaded="no"
280
- all_running_workers_cache=""
281
- all_running_workers_cache_loaded="no"
282
- running_issue_workers_cache=""
283
- running_issue_workers_cache_loaded="no"
284
- running_pr_workers_cache=""
285
- running_pr_workers_cache_loaded="no"
286
- completed_workers_cache=""
287
- completed_workers_cache_loaded="no"
288
- ready_issue_ids_cache=""
289
- ready_issue_ids_cache_loaded="no"
290
- open_agent_pr_ids_cache=""
291
- open_agent_pr_ids_cache_loaded="no"
292
- running_issue_ids_cache=""
293
- running_issue_ids_cache_loaded="no"
294
- exclusive_issue_ids_cache=""
295
- exclusive_issue_ids_cache_loaded="no"
296
- exclusive_pr_ids_cache=""
297
- exclusive_pr_ids_cache_loaded="no"
298
- blocked_recovery_issue_ids_cache=""
299
- blocked_recovery_issue_ids_cache_loaded="no"
300
- ordered_ready_issue_ids_cache=""
301
- ordered_ready_issue_ids_cache_loaded="no"
302
- due_scheduled_issue_ids_cache=""
303
- due_scheduled_issue_ids_cache_loaded="no"
304
- due_blocked_recovery_issue_ids_cache=""
305
- due_blocked_recovery_issue_ids_cache_loaded="no"
306
- if [[ -n "${issue_attr_cache_dir:-}" && -d "${issue_attr_cache_dir}" ]]; then
307
- rm -rf "${issue_attr_cache_dir}" || true
308
- fi
309
- if [[ -n "${pr_attr_cache_dir:-}" && -d "${pr_attr_cache_dir}" ]]; then
310
- rm -rf "${pr_attr_cache_dir}" || true
311
- fi
312
- if [[ -n "${pr_risk_cache_dir:-}" && -d "${pr_risk_cache_dir}" ]]; then
313
- rm -rf "${pr_risk_cache_dir}" || true
314
- fi
315
- if declare -F heartbeat_invalidate_snapshot_cache >/dev/null 2>&1; then
316
- heartbeat_invalidate_snapshot_cache
317
- fi
318
- }
319
-
320
- stage_issue_launch() {
321
- local issue_id="${1:?issue id required}"
322
- local is_heavy="${2:-no}"
323
- launch_in_progress_kind="issue"
324
- launch_in_progress_id="$issue_id"
325
- launch_in_progress_issue_is_heavy="$is_heavy"
326
- launch_in_progress_cleanup_enabled="yes"
327
- heartbeat_mark_issue_running "$issue_id" "$is_heavy"
328
- }
329
-
330
- stage_pr_launch() {
331
- local pr_number="${1:?pr number required}"
332
- reserve_pr_launch "$pr_number"
333
- launch_in_progress_kind="pr"
334
- launch_in_progress_id="$pr_number"
335
- launch_in_progress_issue_is_heavy="no"
336
- launch_in_progress_cleanup_enabled="yes"
337
- heartbeat_mark_pr_running "$pr_number"
338
- }
339
-
340
- rollback_launch_in_progress() {
341
- # Always cleanup caches on exit to avoid leaks
342
- cleanup_scheduler_caches
343
- # Only rollback labels if a launch was in progress and failed
344
- if [[ "${launch_in_progress_cleanup_enabled:-no}" == "yes" ]]; then
345
- case "${launch_in_progress_kind:-}" in
346
- issue)
347
- if [[ -n "${launch_in_progress_id:-}" ]]; then
348
- heartbeat_issue_launch_failed "${launch_in_progress_id}" >/dev/null 2>&1 || true
349
- fi
350
- ;;
351
- pr)
352
- if [[ -n "${launch_in_progress_id:-}" ]]; then
353
- heartbeat_clear_pr_running "${launch_in_progress_id}" >/dev/null 2>&1 || true
354
- fi
355
- ;;
356
- esac
357
- fi
358
- clear_launch_in_progress
359
- }
360
-
361
- trap rollback_launch_in_progress EXIT INT TERM
362
-
363
- # --- PR launch dispatcher ---
364
-
365
- launch_pr_candidate_json() {
366
- local pr_candidate_json="${1:?pr candidate json required}"
367
- local pr_number pr_lane launch_out
368
-
369
- pr_number="$(jq -r '.number' <<<"$pr_candidate_json")"
370
- pr_lane="$(jq -r '.agentLane' <<<"$pr_candidate_json")"
371
- stage_pr_launch "$pr_number"
372
-
373
- case "$pr_lane" in
374
- double-check-1|double-check-2|automerge)
375
- if ! launch_out="$(heartbeat_start_pr_review_worker "$pr_number" 2>&1)"; then
376
- heartbeat_clear_pr_running "$pr_number" || true
377
- clear_launch_in_progress
378
- record_memory "failed to launch PR review worker for #${pr_number}"
379
- print_block "LAUNCH_FAILED_PR=${pr_number}" "$launch_out"
380
- exit 1
381
- fi
382
- record_memory "launched PR review worker for #${pr_number}"
383
- ;;
384
- merge-repair)
385
- if ! launch_out="$(heartbeat_start_pr_merge_repair_worker "$pr_number" 2>&1)"; then
386
- heartbeat_clear_pr_running "$pr_number" || true
387
- clear_launch_in_progress
388
- record_memory "failed to launch PR merge-repair worker for #${pr_number}"
389
- print_block "LAUNCH_FAILED_PR=${pr_number}" "$launch_out"
390
- exit 1
391
- fi
392
- record_memory "launched PR merge-repair worker for #${pr_number}"
393
- ;;
394
- ci-refresh)
395
- if ! launch_out="$(heartbeat_start_pr_ci_refresh "$pr_number" 2>&1)"; then
396
- heartbeat_clear_pr_running "$pr_number" || true
397
- clear_launch_in_progress
398
- record_memory "failed to trigger PR ci-refresh for #${pr_number}"
399
- print_block "LAUNCH_FAILED_PR=${pr_number}" "$launch_out"
400
- exit 1
401
- fi
402
- heartbeat_clear_pr_running "$pr_number" || true
403
- record_memory "triggered PR ci-refresh for #${pr_number}"
404
- ;;
405
- fix)
406
- if ! launch_out="$(heartbeat_start_pr_fix_worker "$pr_number" 2>&1)"; then
407
- heartbeat_clear_pr_running "$pr_number" || true
408
- clear_launch_in_progress
409
- record_memory "failed to launch PR fix worker for #${pr_number}"
410
- print_block "LAUNCH_FAILED_PR=${pr_number}" "$launch_out"
411
- exit 1
412
- fi
413
- record_memory "launched PR fix worker for #${pr_number}"
414
- ;;
415
- *)
416
- launch_out="Unsupported PR lane: ${pr_lane}"
417
- heartbeat_clear_pr_running "$pr_number" || true
418
- clear_launch_in_progress
419
- print_block "LAUNCH_FAILED_PR=${pr_number}" "$launch_out"
420
- exit 1
421
- ;;
422
- esac
423
-
424
- clear_launch_in_progress
425
- print_block "LAUNCHED_PR=${pr_number}" "$(printf 'LANE=%s\n%s' "$pr_lane" "$launch_out")"
426
- if [[ "$pr_lane" != "ci-refresh" ]]; then
427
- running_total_count=$((running_total_count + 1))
428
- running_pr_count=$((running_pr_count + 1))
429
- fi
430
- launched_pr_count=$((launched_pr_count + 1))
431
- if (( launch_budget_remaining > 0 )); then
432
- launch_budget_remaining=$((launch_budget_remaining - 1))
433
- fi
434
- }
435
-
436
- # ============================================================================
437
- # MAIN EXECUTION
438
- # ============================================================================
439
-
440
- log_phase "reconcile-completed-workers:start"
441
- ensure_completed_workers_cache
442
- while IFS= read -r completed_session; do
443
- [[ -n "$completed_session" ]] || continue
444
- case "$completed_session" in
445
- "${issue_prefix}"*)
446
- if reconcile_out="$(heartbeat_reconcile_issue "$completed_session" 2>&1)"; then
447
- record_memory "reconciled issue worker ${completed_session}"
448
- print_block "RECONCILED_SESSION=${completed_session}" "$reconcile_out"
449
- else
450
- record_memory "failed to reconcile issue worker ${completed_session}"
451
- print_block "RECONCILE_FAILED_SESSION=${completed_session}" "$reconcile_out"
452
- fi
453
- ;;
454
- "${pr_prefix}"*)
455
- if reconcile_out="$(heartbeat_reconcile_pr "$completed_session" 2>&1)"; then
456
- record_memory "reconciled PR worker ${completed_session}"
457
- print_block "RECONCILED_SESSION=${completed_session}" "$reconcile_out"
458
- else
459
- completed_pr_number="${completed_session#${pr_prefix}}"
460
- if [[ -n "$completed_pr_number" ]]; then
461
- heartbeat_clear_pr_running "$completed_pr_number" >/dev/null || true
462
- heartbeat_sync_pr_labels "$completed_pr_number" >/dev/null || true
463
- fi
464
- record_memory "failed to reconcile PR worker ${completed_session}"
465
- print_block "RECONCILE_FAILED_SESSION=${completed_session}" "$reconcile_out"
466
- fi
467
- ;;
468
- *)
469
- echo "unknown completed worker session: ${completed_session}" >&2
470
- exit 1
471
- ;;
472
- esac
473
- done <<<"$completed_workers_cache"
474
- log_phase "reconcile-completed-workers:end"
475
-
476
- log_phase "sync-open-agent-issues:start"
477
- sync_open_agent_issues
478
- log_phase "sync-open-agent-issues:end"
479
- log_phase "sync-open-agent-prs:start"
480
- sync_open_agent_prs
481
- log_phase "sync-open-agent-prs:end"
482
-
483
- log_phase "snapshot-running-counts:start"
484
- running_workers_now="$(all_running_workers)"
485
- auth_wait_workers_now="$(auth_wait_workers)"
486
- running_issue_workers_now="$(running_issue_workers)"
487
- running_pr_workers_now="$(running_pr_workers)"
488
- pending_issue_count="$(pending_issue_launch_count)"
489
- pending_pr_count="$(pending_pr_launch_count)"
490
- pending_heavy_issue_count="$(pending_heavy_issue_launch_count)"
491
- pending_scheduled_issue_count="$(pending_scheduled_issue_launch_count)"
492
- pending_scheduled_heavy_issue_count="$(pending_scheduled_heavy_issue_launch_count)"
493
- pending_recurring_issue_count="$(pending_recurring_issue_launch_count)"
494
- pending_blocked_recovery_issue_count="$(pending_blocked_recovery_issue_launch_count)"
495
- pending_exclusive_issue_count="$(pending_exclusive_issue_launch_count)"
496
- pending_exclusive_pr_count="$(pending_exclusive_pr_launch_count)"
497
- auth_wait_worker_count="$(worker_count "$auth_wait_workers_now")"
498
- running_issue_count="$(worker_count "$running_issue_workers_now")"
499
- running_pr_count="$(worker_count "$running_pr_workers_now")"
500
- running_total_count=$((running_issue_count + running_pr_count))
501
- running_issue_count=$((running_issue_count + pending_issue_count))
502
- running_pr_count=$((running_pr_count + pending_pr_count))
503
- running_total_count=$((running_total_count + pending_issue_count + pending_pr_count))
504
- running_heavy_issue_count="$(running_heavy_issue_workers)"
505
- running_non_recurring_issue_count="$(running_non_recurring_issue_workers)"
506
- running_recurring_issue_count="$(running_recurring_issue_workers)"
507
- running_scheduled_issue_count="$(running_scheduled_issue_workers)"
508
- running_scheduled_heavy_issue_count="$(running_scheduled_heavy_issue_workers)"
509
- running_blocked_recovery_issue_count="$(running_blocked_recovery_issue_workers)"
510
- running_exclusive_issue_count="$(running_exclusive_issue_workers)"
511
- running_exclusive_pr_count="$(running_exclusive_pr_workers)"
512
- running_heavy_issue_count=$((running_heavy_issue_count + pending_heavy_issue_count))
513
- running_scheduled_issue_count=$((running_scheduled_issue_count + pending_scheduled_issue_count))
514
- running_scheduled_heavy_issue_count=$((running_scheduled_heavy_issue_count + pending_scheduled_heavy_issue_count))
515
- running_recurring_issue_count=$((running_recurring_issue_count + pending_recurring_issue_count))
516
- running_blocked_recovery_issue_count=$((running_blocked_recovery_issue_count + pending_blocked_recovery_issue_count))
517
- running_non_recurring_issue_count=$((running_non_recurring_issue_count + pending_issue_count - pending_recurring_issue_count - pending_scheduled_issue_count))
518
- running_exclusive_issue_count=$((running_exclusive_issue_count + pending_exclusive_issue_count))
519
- running_exclusive_pr_count=$((running_exclusive_pr_count + pending_exclusive_pr_count))
520
- ensure_open_agent_pr_ids_cache
521
- open_agent_pr_count="$(worker_count "$open_agent_pr_ids_cache")"
522
- ready_non_recurring_count="$(ready_non_recurring_issue_count)"
523
- log_phase "snapshot-running-counts:end"
524
-
525
- exclusive_lock_mode="no"
526
- exclusive_lock_kind=""
527
- exclusive_lock_item=""
528
- exclusive_waiting_reason=""
529
- ensure_exclusive_pr_ids_cache
530
- ensure_exclusive_issue_ids_cache
531
- exclusive_pr_item_id="${exclusive_pr_ids_cache%%$'\n'*}"
532
- exclusive_issue_item_id="${exclusive_issue_ids_cache%%$'\n'*}"
533
- [[ "$exclusive_pr_item_id" == "$exclusive_pr_ids_cache" && -z "$exclusive_pr_ids_cache" ]] && exclusive_pr_item_id=""
534
- [[ "$exclusive_issue_item_id" == "$exclusive_issue_ids_cache" && -z "$exclusive_issue_ids_cache" ]] && exclusive_issue_item_id=""
535
- exclusive_pr_risk_json=""
536
-
537
- if (( running_exclusive_issue_count + running_exclusive_pr_count > 0 )); then
538
- exclusive_lock_mode="running"
539
- elif [[ -n "$exclusive_pr_item_id" ]]; then
540
- exclusive_lock_mode="pending"
541
- exclusive_lock_kind="pr"
542
- exclusive_lock_item="$exclusive_pr_item_id"
543
- exclusive_pr_risk_json="$(cached_pr_risk_json "$exclusive_pr_item_id")"
544
- elif [[ -n "$exclusive_issue_item_id" ]]; then
545
- exclusive_lock_mode="pending"
546
- exclusive_lock_kind="issue"
547
- exclusive_lock_item="$exclusive_issue_item_id"
548
- fi
549
-
550
- launched_issue_count=0
551
- launched_pr_count=0
552
- launch_budget_remaining="$max_launches_per_pass"
553
- heavy_deferred_count=0
554
- retry_deferred_issue_count=0
555
- provider_launch_suppressed="no"
556
- provider_cooldown_backend=""
557
- provider_cooldown_model=""
558
- provider_cooldown_until=""
559
- provider_cooldown_reason=""
560
- issue_capacity_limit="$max_concurrent_workers"
561
- reserved_pr_slots=0
562
- pr_backlog_eligible_count=0
563
- priority_review_backlog_count=0
564
- priority_review_launches_remaining="${ACP_MAX_DOUBLE_CHECK_FAST_LAUNCHES:-${F_LOSNING_MAX_DOUBLE_CHECK_FAST_LAUNCHES:-3}}"
565
-
566
- if provider_cooldown_out="$(provider_cooldown_state 2>/dev/null || true)"; then
567
- provider_cooldown_ready="$(awk -F= '/^READY=/{print $2}' <<<"$provider_cooldown_out")"
568
- provider_cooldown_backend="$(awk -F= '/^BACKEND=/{print $2}' <<<"$provider_cooldown_out")"
569
- provider_cooldown_model="$(awk -F= '/^MODEL=/{print $2}' <<<"$provider_cooldown_out")"
570
- provider_cooldown_until="$(awk -F= '/^NEXT_ATTEMPT_AT=/{print $2}' <<<"$provider_cooldown_out")"
571
- provider_cooldown_reason="$(awk -F= '/^LAST_REASON=/{print $2}' <<<"$provider_cooldown_out")"
572
- if [[ "${provider_cooldown_ready}" == "no" ]]; then
573
- provider_launch_suppressed="yes"
574
- launch_budget_remaining=0
575
- fi
576
- fi
577
-
578
-
579
- if [[ "$exclusive_lock_mode" == "pending" && "$exclusive_lock_kind" == "pr" ]]; then
580
- pr_number="$exclusive_lock_item"
581
- pr_lane="$(jq -r '.agentLane' <<<"$exclusive_pr_risk_json")"
582
- if pending_pr_launch_active "$pr_number"; then
583
- exclusive_waiting_reason="launch-pending"
584
- elif [[ "$pr_lane" == "human-review" ]]; then
585
- # Do not lock exclusive for PRs requiring human review; treat as normal PR
586
- exclusive_waiting_reason=""
587
- exclusive_lock_mode="no"
588
- exclusive_lock_kind=""
589
- exclusive_lock_item=""
590
- elif [[ "$pr_lane" != "double-check-1" && "$pr_lane" != "double-check-2" && "$pr_lane" != "automerge" && "$pr_lane" != "merge-repair" && "$pr_lane" != "ci-refresh" && "$pr_lane" != "fix" ]]; then
591
- exclusive_waiting_reason="lane-${pr_lane}"
592
- elif [[ "$provider_launch_suppressed" == "yes" ]]; then
593
- exclusive_waiting_reason="provider-cooldown"
594
- elif ! retry_ready pr "$pr_number"; then
595
- exclusive_waiting_reason="retry-cooldown"
596
- elif (( running_pr_count >= max_concurrent_pr_workers )); then
597
- exclusive_waiting_reason="pr-capacity-full"
598
- elif (( running_total_count >= max_concurrent_workers )); then
599
- exclusive_waiting_reason="capacity-full"
600
- else
601
- stage_pr_launch "$pr_number"
602
- case "$pr_lane" in
603
- double-check-1|double-check-2|automerge)
604
- if ! launch_out="$(heartbeat_start_pr_review_worker "$pr_number" 2>&1)"; then
605
- heartbeat_clear_pr_running "$pr_number" || true
606
- clear_launch_in_progress
607
- record_memory "failed to launch exclusive PR review worker for #${pr_number}"
608
- print_block "LAUNCH_FAILED_PR=${pr_number}" "$launch_out"
609
- exit 1
610
- fi
611
- record_memory "launched exclusive PR review worker for #${pr_number}"
612
- ;;
613
- merge-repair)
614
- if ! launch_out="$(heartbeat_start_pr_merge_repair_worker "$pr_number" 2>&1)"; then
615
- heartbeat_clear_pr_running "$pr_number" || true
616
- clear_launch_in_progress
617
- record_memory "failed to launch exclusive PR merge-repair worker for #${pr_number}"
618
- print_block "LAUNCH_FAILED_PR=${pr_number}" "$launch_out"
619
- exit 1
620
- fi
621
- record_memory "launched exclusive PR merge-repair worker for #${pr_number}"
622
- ;;
623
- ci-refresh)
624
- if ! launch_out="$(heartbeat_start_pr_ci_refresh "$pr_number" 2>&1)"; then
625
- heartbeat_clear_pr_running "$pr_number" || true
626
- clear_launch_in_progress
627
- record_memory "failed to trigger exclusive PR ci-refresh for #${pr_number}"
628
- print_block "LAUNCH_FAILED_PR=${pr_number}" "$launch_out"
629
- exit 1
630
- fi
631
- heartbeat_clear_pr_running "$pr_number" || true
632
- record_memory "triggered exclusive PR ci-refresh for #${pr_number}"
633
- ;;
634
- fix)
635
- if ! launch_out="$(heartbeat_start_pr_fix_worker "$pr_number" 2>&1)"; then
636
- heartbeat_clear_pr_running "$pr_number" || true
637
- clear_launch_in_progress
638
- record_memory "failed to launch exclusive PR fix worker for #${pr_number}"
639
- print_block "LAUNCH_FAILED_PR=${pr_number}" "$launch_out"
640
- exit 1
641
- fi
642
- record_memory "launched exclusive PR fix worker for #${pr_number}"
643
- ;;
644
- esac
645
- clear_launch_in_progress
646
- print_block "LAUNCHED_PR=${pr_number}" "$(printf 'LANE=%s\nEXCLUSIVE=yes\n%s' "$pr_lane" "$launch_out")"
647
- if [[ "$pr_lane" != "ci-refresh" ]]; then
648
- running_total_count=$((running_total_count + 1))
649
- running_pr_count=$((running_pr_count + 1))
650
- running_exclusive_pr_count=$((running_exclusive_pr_count + 1))
651
- fi
652
- launched_pr_count=$((launched_pr_count + 1))
653
- if (( launch_budget_remaining > 0 )); then
654
- launch_budget_remaining=$((launch_budget_remaining - 1))
655
- fi
656
- exclusive_lock_mode="running"
657
- exclusive_waiting_reason=""
658
- fi
659
- elif [[ "$exclusive_lock_mode" == "pending" && "$exclusive_lock_kind" == "issue" ]]; then
660
- issue_id="$exclusive_lock_item"
661
- is_heavy="$(cached_issue_attr heavy "$issue_id")"
662
- is_recurring="$(cached_issue_attr recurring "$issue_id")"
663
- if pending_issue_launch_active "$issue_id"; then
664
- exclusive_waiting_reason="launch-pending"
665
- elif [[ "$provider_launch_suppressed" == "yes" ]]; then
666
- exclusive_waiting_reason="provider-cooldown"
667
- elif ! retry_ready issue "$issue_id"; then
668
- exclusive_waiting_reason="retry-cooldown"
669
- elif (( running_total_count >= max_concurrent_workers )); then
670
- exclusive_waiting_reason="capacity-full"
671
- elif [[ "$is_heavy" == "yes" ]] && (( running_heavy_issue_count >= max_concurrent_heavy_workers )); then
672
- exclusive_waiting_reason="heavy-slot-busy"
673
- else
674
- stage_issue_launch "$issue_id" "$is_heavy"
675
- if ! launch_out="$(heartbeat_start_issue_worker "$issue_id" 2>&1)"; then
676
- heartbeat_issue_launch_failed "$issue_id" "$is_heavy" || true
677
- clear_launch_in_progress
678
- record_memory "failed to launch exclusive issue worker for #${issue_id}"
679
- print_block "LAUNCH_FAILED_ISSUE=${issue_id}" "$launch_out"
680
- exit 1
681
- fi
682
-
683
- clear_launch_in_progress
684
- record_memory "launched exclusive issue worker for #${issue_id}"
685
- print_block "LAUNCHED_ISSUE=${issue_id}" "$(printf 'EXCLUSIVE=yes\n%s' "$launch_out")"
686
- launched_issue_count=$((launched_issue_count + 1))
687
- if (( launch_budget_remaining > 0 )); then
688
- launch_budget_remaining=$((launch_budget_remaining - 1))
689
- fi
690
- running_total_count=$((running_total_count + 1))
691
- running_issue_count=$((running_issue_count + 1))
692
- running_exclusive_issue_count=$((running_exclusive_issue_count + 1))
693
- if [[ "$is_heavy" == "yes" ]]; then
694
- running_heavy_issue_count=$((running_heavy_issue_count + 1))
695
- fi
696
- if [[ "$is_recurring" == "yes" ]]; then
697
- running_recurring_issue_count=$((running_recurring_issue_count + 1))
698
- record_recurring_issue_launch "$issue_id"
699
- else
700
- running_non_recurring_issue_count=$((running_non_recurring_issue_count + 1))
701
- fi
702
- exclusive_lock_mode="running"
703
- exclusive_waiting_reason=""
704
- fi
705
- fi
706
-
707
- if [[ "$exclusive_lock_mode" == "no" ]]; then
708
- launch_budget_remaining="${launch_budget_remaining:-$max_launches_per_pass}"
709
- priority_review_launches_remaining="${priority_review_launches_remaining:-${ACP_MAX_DOUBLE_CHECK_FAST_LAUNCHES:-${F_LOSNING_MAX_DOUBLE_CHECK_FAST_LAUNCHES:-3}}}"
710
- log_phase "launch-planning:start"
711
- log_phase "launch-planning:scheduled-issue-loop:start"
712
- ensure_due_scheduled_issue_ids_cache
713
- while IFS= read -r issue_id; do
714
- [[ -n "$issue_id" ]] || continue
715
- if (( launch_budget_remaining <= 0 )); then
716
- break
717
- fi
718
- if pending_issue_launch_active "$issue_id"; then
719
- continue
720
- fi
721
- if (( running_scheduled_issue_count >= max_concurrent_scheduled_issue_workers )); then
722
- break
723
- fi
724
-
725
- is_heavy="$(cached_issue_attr heavy "$issue_id")"
726
- if [[ "$is_heavy" == "yes" ]] && (( running_scheduled_heavy_issue_count >= max_concurrent_scheduled_heavy_workers )); then
727
- heavy_deferred_count=$((heavy_deferred_count + 1))
728
- continue
729
- fi
730
-
731
- stage_issue_launch "$issue_id" "$is_heavy"
732
- if ! launch_out="$(heartbeat_start_issue_worker "$issue_id" 2>&1)"; then
733
- heartbeat_issue_launch_failed "$issue_id" "$is_heavy" || true
734
- clear_launch_in_progress
735
- record_memory "failed to launch scheduled issue worker for #${issue_id}"
736
- print_block "LAUNCH_FAILED_SCHEDULED_ISSUE=${issue_id}" "$launch_out"
737
- exit 1
738
- fi
739
-
740
- clear_launch_in_progress
741
- record_scheduled_issue_launch "$issue_id"
742
- record_memory "launched scheduled issue worker for #${issue_id}"
743
- print_block "LAUNCHED_SCHEDULED_ISSUE=${issue_id}" "$(printf 'SCHEDULED=yes\n%s' "$launch_out")"
744
- launched_issue_count=$((launched_issue_count + 1))
745
- if (( launch_budget_remaining > 0 )); then
746
- launch_budget_remaining=$((launch_budget_remaining - 1))
747
- fi
748
- running_total_count=$((running_total_count + 1))
749
- running_issue_count=$((running_issue_count + 1))
750
- running_scheduled_issue_count=$((running_scheduled_issue_count + 1))
751
- if [[ "$is_heavy" == "yes" ]]; then
752
- running_heavy_issue_count=$((running_heavy_issue_count + 1))
753
- running_scheduled_heavy_issue_count=$((running_scheduled_heavy_issue_count + 1))
754
- fi
755
- done <<<"$due_scheduled_issue_ids_cache"
756
- log_phase "launch-planning:scheduled-issue-loop:end"
757
- # Fast-path independent review lanes so high-risk PRs do not wait behind
758
- # issue work or lower-priority PR lanes in short heartbeat windows.
759
- log_phase "launch-planning:priority-review:start"
760
- while (( priority_review_launches_remaining > 0 )) && (( launch_budget_remaining > 0 )) && (( running_pr_count < max_concurrent_pr_workers )) && (( running_total_count < max_concurrent_workers )); do
761
- pr_candidate_json="$(next_priority_review_pr_candidate_json || true)"
762
- [[ -n "$pr_candidate_json" ]] || break
763
- launch_pr_candidate_json "$pr_candidate_json"
764
- priority_review_launches_remaining=$((priority_review_launches_remaining - 1))
765
- done
766
- log_phase "launch-planning:priority-review:end"
767
- log_phase "launch-planning:first-pr:start"
768
- if (( launch_budget_remaining > 0 )) && (( running_pr_count < max_concurrent_pr_workers )) && (( running_total_count < max_concurrent_workers )); then
769
- pr_candidate_json="$(next_pr_candidate_json || true)"
770
- if [[ -n "$pr_candidate_json" ]]; then
771
- # Launch a PR before issue work so short heartbeat windows do not starve PR lanes.
772
- launch_pr_candidate_json "$pr_candidate_json"
773
- fi
774
- fi
775
- log_phase "launch-planning:first-pr:end"
776
-
777
- log_phase "launch-planning:backlog-counts:start"
778
- pr_backlog_eligible_count="$(eligible_pr_backlog_count)"
779
- priority_review_backlog_count="$(priority_review_backlog_count)"
780
- if (( pr_backlog_eligible_count > 0 )); then
781
- reserved_pr_slots=4
782
- if (( priority_review_backlog_count > reserved_pr_slots )); then
783
- reserved_pr_slots="$priority_review_backlog_count"
784
- fi
785
- if (( reserved_pr_slots > pr_backlog_eligible_count )); then
786
- reserved_pr_slots="$pr_backlog_eligible_count"
787
- fi
788
- if (( reserved_pr_slots > max_concurrent_pr_workers - running_pr_count )); then
789
- reserved_pr_slots=$((max_concurrent_pr_workers - running_pr_count))
790
- fi
791
- if (( reserved_pr_slots > max_concurrent_workers - running_total_count )); then
792
- reserved_pr_slots=$((max_concurrent_workers - running_total_count))
793
- fi
794
- if (( reserved_pr_slots < 0 )); then
795
- reserved_pr_slots=0
796
- fi
797
- fi
798
- issue_capacity_limit=$((max_concurrent_workers - reserved_pr_slots))
799
- if (( issue_capacity_limit < running_total_count )); then
800
- issue_capacity_limit="$running_total_count"
801
- fi
802
- log_phase "launch-planning:backlog-counts:end"
803
-
804
- log_phase "launch-planning:blocked-recovery-loop:start"
805
- ensure_due_blocked_recovery_issue_ids_cache
806
- while IFS= read -r issue_id; do
807
- [[ -n "$issue_id" ]] || continue
808
- if pending_issue_launch_active "$issue_id"; then
809
- continue
810
- fi
811
- if (( max_concurrent_blocked_recovery_issue_workers <= 0 )); then
812
- break
813
- fi
814
- if (( launch_budget_remaining <= 0 )); then
815
- break
816
- fi
817
- if (( running_total_count >= max_concurrent_workers )); then
818
- break
819
- fi
820
- if (( running_blocked_recovery_issue_count >= max_concurrent_blocked_recovery_issue_workers )); then
821
- break
822
- fi
823
- if ! retry_ready issue "$issue_id"; then
824
- retry_deferred_issue_count=$((retry_deferred_issue_count + 1))
825
- continue
826
- fi
827
-
828
- is_heavy="$(cached_issue_attr heavy "$issue_id")"
829
- is_recurring="$(cached_issue_attr recurring "$issue_id")"
830
- if [[ "$is_recurring" == "yes" ]]; then
831
- if (( running_recurring_issue_count >= max_recurring_issue_workers )); then
832
- continue
833
- fi
834
- if (( max_open_agent_prs_for_recurring > 0 )) && (( open_agent_pr_count >= max_open_agent_prs_for_recurring )); then
835
- continue
836
- fi
837
- fi
838
- if [[ "$is_heavy" == "yes" ]] && (( running_heavy_issue_count >= max_concurrent_heavy_workers )); then
839
- heavy_deferred_count=$((heavy_deferred_count + 1))
840
- continue
841
- fi
842
-
843
- stage_issue_launch "$issue_id" "$is_heavy"
844
- if ! launch_out="$(heartbeat_start_issue_worker "$issue_id" 2>&1)"; then
845
- heartbeat_issue_launch_failed "$issue_id" "$is_heavy" || true
846
- clear_launch_in_progress
847
- clear_blocked_recovery_issue_state "$issue_id"
848
- record_memory "failed to launch blocked-recovery issue worker for #${issue_id}"
849
- print_block "LAUNCH_FAILED_BLOCKED_RECOVERY_ISSUE=${issue_id}" "$launch_out"
850
- exit 1
851
- fi
852
-
853
- record_blocked_recovery_issue_launch "$issue_id"
854
- clear_launch_in_progress
855
- record_memory "launched blocked-recovery issue worker for #${issue_id}"
856
- print_block "LAUNCHED_BLOCKED_RECOVERY_ISSUE=${issue_id}" "$(printf 'BLOCKED_RECOVERY=yes\n%s' "$launch_out")"
857
- launched_issue_count=$((launched_issue_count + 1))
858
- if (( launch_budget_remaining > 0 )); then
859
- launch_budget_remaining=$((launch_budget_remaining - 1))
860
- fi
861
- running_total_count=$((running_total_count + 1))
862
- running_issue_count=$((running_issue_count + 1))
863
- running_blocked_recovery_issue_count=$((running_blocked_recovery_issue_count + 1))
864
- if [[ "$is_heavy" == "yes" ]]; then
865
- running_heavy_issue_count=$((running_heavy_issue_count + 1))
866
- fi
867
- if [[ "$is_recurring" == "yes" ]]; then
868
- running_recurring_issue_count=$((running_recurring_issue_count + 1))
869
- record_recurring_issue_launch "$issue_id"
870
- else
871
- running_non_recurring_issue_count=$((running_non_recurring_issue_count + 1))
872
- fi
873
- done <<<"$due_blocked_recovery_issue_ids_cache"
874
- log_phase "launch-planning:blocked-recovery-loop:end"
875
-
876
- log_phase "launch-planning:issue-loop:start"
877
- ensure_ordered_ready_issue_ids_cache
878
- while IFS= read -r issue_id; do
879
- [[ -n "$issue_id" ]] || continue
880
- if pending_issue_launch_active "$issue_id"; then
881
- continue
882
- fi
883
- if (( launch_budget_remaining <= 0 )); then
884
- break
885
- fi
886
- if (( running_total_count >= issue_capacity_limit )); then
887
- break
888
- fi
889
- if ! retry_ready issue "$issue_id"; then
890
- retry_deferred_issue_count=$((retry_deferred_issue_count + 1))
891
- continue
892
- fi
893
-
894
- if [[ "$(cached_issue_attr scheduled "$issue_id")" == "yes" ]]; then
895
- continue
896
- fi
897
-
898
- is_heavy="$(cached_issue_attr heavy "$issue_id")"
899
- is_recurring="$(cached_issue_attr recurring "$issue_id")"
900
- if [[ "$is_recurring" == "yes" ]]; then
901
- if (( running_recurring_issue_count >= max_recurring_issue_workers )); then
902
- continue
903
- fi
904
- if (( max_open_agent_prs_for_recurring > 0 )) && (( open_agent_pr_count >= max_open_agent_prs_for_recurring )); then
905
- continue
906
- fi
907
- fi
908
- if [[ "$is_heavy" == "yes" ]] && (( running_heavy_issue_count >= max_concurrent_heavy_workers )); then
909
- heavy_deferred_count=$((heavy_deferred_count + 1))
910
- continue
911
- fi
912
-
913
- stage_issue_launch "$issue_id" "$is_heavy"
914
- if ! launch_out="$(heartbeat_start_issue_worker "$issue_id" 2>&1)"; then
915
- heartbeat_issue_launch_failed "$issue_id" "$is_heavy" || true
916
- clear_launch_in_progress
917
- record_memory "failed to launch issue worker for #${issue_id}"
918
- print_block "LAUNCH_FAILED_ISSUE=${issue_id}" "$launch_out"
919
- exit 1
920
- fi
921
-
922
- clear_launch_in_progress
923
- record_memory "launched issue worker for #${issue_id}"
924
- print_block "LAUNCHED_ISSUE=${issue_id}" "$launch_out"
925
- launched_issue_count=$((launched_issue_count + 1))
926
- if (( launch_budget_remaining > 0 )); then
927
- launch_budget_remaining=$((launch_budget_remaining - 1))
928
- fi
929
- running_total_count=$((running_total_count + 1))
930
- running_issue_count=$((running_issue_count + 1))
931
- if [[ "$is_heavy" == "yes" ]]; then
932
- running_heavy_issue_count=$((running_heavy_issue_count + 1))
933
- fi
934
- if [[ "$is_recurring" == "yes" ]]; then
935
- running_recurring_issue_count=$((running_recurring_issue_count + 1))
936
- record_recurring_issue_launch "$issue_id"
937
- else
938
- running_non_recurring_issue_count=$((running_non_recurring_issue_count + 1))
939
- fi
940
- done <<<"$ordered_ready_issue_ids_cache"
941
- log_phase "launch-planning:issue-loop:end"
942
-
943
- log_phase "launch-planning:pr-fill:start"
944
- while (( launch_budget_remaining > 0 )) && (( running_pr_count < max_concurrent_pr_workers )) && (( running_total_count < max_concurrent_workers )); do
945
- pr_candidate_json="$(next_pr_candidate_json || true)"
946
- [[ -n "$pr_candidate_json" ]] || break
947
- launch_pr_candidate_json "$pr_candidate_json"
948
- done
949
- log_phase "launch-planning:pr-fill:end"
950
- log_phase "launch-planning:end"
951
- fi
952
-
953
- log_phase "post-launch-summary:start"
954
- human_review_ids="$(human_review_pr_ids)"
955
- human_review_count="$(worker_count "$human_review_ids")"
956
- retry_deferred_pr_count=0
957
- ensure_open_agent_pr_ids_cache
958
- while IFS= read -r pr_number; do
959
- [[ -n "$pr_number" ]] || continue
960
- if tmux has-session -t "${pr_prefix}${pr_number}" 2>/dev/null; then
961
- continue
962
- fi
963
- risk_json="$(cached_pr_risk_json "$pr_number")"
964
- lane="$(jq -r '.agentLane' <<<"$risk_json")"
965
- if [[ "$lane" != "fix" && "$lane" != "merge-repair" && "$lane" != "ci-refresh" && "$lane" != "automerge" && "$lane" != "double-check-1" && "$lane" != "double-check-2" ]]; then
966
- continue
967
- fi
968
- if ! retry_ready pr "$pr_number"; then
969
- retry_deferred_pr_count=$((retry_deferred_pr_count + 1))
970
- fi
971
- done <<<"$open_agent_pr_ids_cache"
972
- log_phase "post-launch-summary:end"
973
-
974
- if (( launched_issue_count > 0 || running_total_count > 0 || auth_wait_worker_count > 0 )); then
975
- print_block "RUNNING_COUNTS" "$(printf 'TOTAL=%s\nISSUE=%s\nPR=%s\nAUTH_WAITING_WORKERS=%s\nSCHEDULED_ISSUE=%s\nBLOCKED_RECOVERY_ISSUE=%s\nLAUNCHED_ISSUES=%s\nLAUNCHED_PRS=%s\nLAUNCH_BUDGET_REMAINING=%s\nPR_BACKLOG_ELIGIBLE=%s\nPR_RESERVED_SLOTS=%s\nISSUE_CAPACITY_LIMIT=%s\n%s=%s' "$running_total_count" "$running_issue_count" "$running_pr_count" "$auth_wait_worker_count" "$running_scheduled_issue_count" "$running_blocked_recovery_issue_count" "$launched_issue_count" "$launched_pr_count" "$launch_budget_remaining" "$pr_backlog_eligible_count" "$reserved_pr_slots" "$issue_capacity_limit" "$heavy_running_label" "$running_heavy_issue_count")"
976
- ensure_all_running_workers_cache
977
- while IFS= read -r running_session; do
978
- [[ -n "$running_session" ]] || continue
979
- status_out="$(
980
- "${shared_agent_home}/tools/bin/agent-project-worker-status" \
981
- --runs-root "$runs_root" \
982
- --session "$running_session"
983
- )"
984
- print_block "RUNNING_SESSION=${running_session}" "$status_out"
985
- done <<<"$all_running_workers_cache"
986
- if [[ "$exclusive_lock_mode" != "no" ]]; then
987
- print_block "EXCLUSIVE_QUEUE_LOCK" "$(printf 'STATE=%s\nKIND=%s\nITEM_ID=%s\nREASON=%s' "$exclusive_lock_mode" "${exclusive_lock_kind:-}" "${exclusive_lock_item:-}" "${exclusive_waiting_reason:-active}")"
988
- fi
989
- if (( human_review_count > 0 )); then
990
- print_block "PENDING_HUMAN_REVIEW=${human_review_count}" "$(printf 'High-risk PRs are green and intentionally idle pending human review.\nPRS=%s' "$(printf '%s\n' "$human_review_ids" | paste -sd, -)")"
991
- fi
992
- if (( heavy_deferred_count > 0 )); then
993
- print_block "${heavy_deferred_key}=${heavy_deferred_count}" "$heavy_deferred_message"
994
- fi
995
- if (( retry_deferred_issue_count > 0 || retry_deferred_pr_count > 0 )); then
996
- print_block "RETRY_DEFERRED" "$(printf 'ISSUES=%s\nPRS=%s' "$retry_deferred_issue_count" "$retry_deferred_pr_count")"
997
- fi
998
- if [[ "$provider_launch_suppressed" == "yes" ]]; then
999
- print_block "PROVIDER_COOLDOWN" "$(printf 'BACKEND=%s\nMODEL=%s\nNEXT_ATTEMPT_AT=%s\nLAST_REASON=%s' "${provider_cooldown_backend:-}" "${provider_cooldown_model:-}" "${provider_cooldown_until:-}" "${provider_cooldown_reason:-provider-quota-limit}")"
1000
- fi
1001
- exit 0
1002
- fi
1003
-
1004
- if [[ "$exclusive_lock_mode" != "no" ]]; then
1005
- print_block "EXCLUSIVE_QUEUE_LOCK" "$(printf 'STATE=%s\nKIND=%s\nITEM_ID=%s\nREASON=%s' "$exclusive_lock_mode" "${exclusive_lock_kind:-}" "${exclusive_lock_item:-}" "${exclusive_waiting_reason:-active}")"
1006
- exit 0
1007
- fi
1008
-
1009
- if (( human_review_count > 0 )); then
1010
- print_block "PENDING_HUMAN_REVIEW=${human_review_count}" "$(printf 'High-risk PRs are green and intentionally idle pending human review.\nPRS=%s' "$(printf '%s\n' "$human_review_ids" | paste -sd, -)")"
1011
- exit 0
1012
- fi
1013
-
1014
- if (( heavy_deferred_count > 0 )); then
1015
- print_block "${heavy_deferred_key}=${heavy_deferred_count}" "$heavy_deferred_message"
1016
- exit 0
1017
- fi
1018
-
1019
- if (( retry_deferred_issue_count > 0 || retry_deferred_pr_count > 0 )); then
1020
- print_block "RETRY_DEFERRED" "$(printf 'ISSUES=%s\nPRS=%s' "$retry_deferred_issue_count" "$retry_deferred_pr_count")"
1021
- exit 0
1022
- fi
1023
-
1024
- if [[ "$provider_launch_suppressed" == "yes" ]]; then
1025
- print_block "PROVIDER_COOLDOWN" "$(printf 'BACKEND=%s\nMODEL=%s\nNEXT_ATTEMPT_AT=%s\nLAST_REASON=%s' "${provider_cooldown_backend:-}" "${provider_cooldown_model:-}" "${provider_cooldown_until:-}" "${provider_cooldown_reason:-provider-quota-limit}")"
1026
- exit 0
1027
- fi
1028
-
1029
- echo "HEARTBEAT_OK"