agent-control-plane 0.1.16 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/README.md +93 -14
  2. package/bin/pr-risk.sh +28 -6
  3. package/hooks/heartbeat-hooks.sh +62 -22
  4. package/npm/bin/agent-control-plane.js +360 -10
  5. package/package.json +6 -3
  6. package/references/architecture.md +8 -0
  7. package/references/control-plane-map.md +6 -2
  8. package/references/release-checklist.md +0 -2
  9. package/tools/bin/agent-github-update-labels +6 -1
  10. package/tools/bin/agent-project-catch-up-issue-pr-links +118 -0
  11. package/tools/bin/agent-project-catch-up-merged-prs +78 -21
  12. package/tools/bin/agent-project-catch-up-scheduled-issue-retries +123 -0
  13. package/tools/bin/agent-project-cleanup-session +132 -4
  14. package/tools/bin/agent-project-heartbeat-loop +116 -1461
  15. package/tools/bin/agent-project-reconcile-issue-session +90 -117
  16. package/tools/bin/agent-project-reconcile-pr-session +76 -111
  17. package/tools/bin/agent-project-run-claude-session +12 -2
  18. package/tools/bin/agent-project-run-codex-resilient +86 -9
  19. package/tools/bin/agent-project-run-codex-session +16 -5
  20. package/tools/bin/agent-project-run-kilo-session +356 -14
  21. package/tools/bin/agent-project-run-ollama-session +658 -0
  22. package/tools/bin/agent-project-run-openclaw-session +37 -25
  23. package/tools/bin/agent-project-run-opencode-session +364 -14
  24. package/tools/bin/agent-project-run-pi-session +479 -0
  25. package/tools/bin/agent-project-worker-status +11 -8
  26. package/tools/bin/cleanup-worktree.sh +6 -1
  27. package/tools/bin/flow-config-lib.sh +196 -3
  28. package/tools/bin/flow-resident-worker-lib.sh +120 -2
  29. package/tools/bin/flow-shell-lib.sh +29 -2
  30. package/tools/bin/heartbeat-loop-cache-lib.sh +164 -0
  31. package/tools/bin/heartbeat-loop-counting-lib.sh +306 -0
  32. package/tools/bin/heartbeat-loop-pr-strategy-lib.sh +199 -0
  33. package/tools/bin/heartbeat-loop-scheduling-lib.sh +506 -0
  34. package/tools/bin/heartbeat-loop-worker-lib.sh +319 -0
  35. package/tools/bin/heartbeat-recovery-preflight.sh +13 -1
  36. package/tools/bin/heartbeat-safe-auto.sh +119 -20
  37. package/tools/bin/install-project-launchd.sh +19 -2
  38. package/tools/bin/prepare-worktree.sh +4 -4
  39. package/tools/bin/profile-activate.sh +2 -2
  40. package/tools/bin/profile-adopt.sh +2 -2
  41. package/tools/bin/project-init.sh +1 -1
  42. package/tools/bin/project-launchd-bootstrap.sh +11 -8
  43. package/tools/bin/project-runtimectl.sh +90 -7
  44. package/tools/bin/provider-cooldown-state.sh +14 -14
  45. package/tools/bin/reconcile-bootstrap-lib.sh +113 -0
  46. package/tools/bin/render-flow-config.sh +30 -33
  47. package/tools/bin/resident-issue-controller-lib.sh +448 -0
  48. package/tools/bin/resident-issue-queue-status.py +35 -0
  49. package/tools/bin/run-codex-task.sh +53 -4
  50. package/tools/bin/scaffold-profile.sh +18 -3
  51. package/tools/bin/start-issue-worker.sh +1 -1
  52. package/tools/bin/start-pr-fix-worker.sh +30 -0
  53. package/tools/bin/start-pr-review-worker.sh +31 -0
  54. package/tools/bin/start-resident-issue-loop.sh +27 -438
  55. package/tools/bin/sync-agent-repo.sh +2 -2
  56. package/tools/bin/sync-dependency-baseline.sh +3 -3
  57. package/tools/bin/sync-shared-agent-home.sh +4 -1
  58. package/tools/dashboard/app.js +7 -0
  59. package/tools/dashboard/dashboard_snapshot.py +13 -29
  60. package/tools/templates/pr-fix-template.md +3 -7
  61. package/tools/templates/pr-merge-repair-template.md +3 -7
  62. package/tools/templates/pr-review-template.md +2 -1
  63. package/SKILL.md +0 -149
@@ -158,269 +158,6 @@ log_phase() {
158
158
  printf 'HEARTBEAT_LOOP_PHASE=%s\n' "${1:?phase required}"
159
159
  }
160
160
 
161
- all_tmux_sessions() {
162
- ensure_tmux_sessions_cache
163
- printf '%s\n' "$tmux_sessions_cache"
164
- }
165
-
166
- session_matches_prefix() {
167
- local session="${1:?session required}"
168
- [[ "$session" == "${issue_prefix}"* || "$session" == "${pr_prefix}"* ]]
169
- }
170
-
171
- session_runner_state() {
172
- local session="${1:?session required}"
173
- local runner_state_file="${runs_root}/${session}/runner.env"
174
- if [[ ! -f "$runner_state_file" ]]; then
175
- return 1
176
- fi
177
- awk -F= '/^RUNNER_STATE=/{print $2; exit}' "$runner_state_file"
178
- }
179
-
180
- session_is_auth_waiting() {
181
- local session="${1:?session required}"
182
- local runner_state=""
183
- runner_state="$(session_runner_state "$session" || true)"
184
- [[ "$runner_state" == "waiting-auth-refresh" || "$runner_state" == "switching-account" ]]
185
- }
186
-
187
- all_running_workers() {
188
- ensure_all_running_workers_cache
189
- printf '%s\n' "$all_running_workers_cache"
190
- }
191
-
192
- running_issue_workers() {
193
- ensure_running_issue_workers_cache
194
- printf '%s\n' "$running_issue_workers_cache"
195
- }
196
-
197
- pending_launch_pid() {
198
- local kind="${1:?kind required}"
199
- local item_id="${2:?item id required}"
200
- local pending_file pid
201
-
202
- pending_file="${pending_launch_dir}/${kind}-${item_id}.pid"
203
- if [[ ! -f "$pending_file" ]]; then
204
- return 1
205
- fi
206
-
207
- pid="$(tr -d '[:space:]' <"$pending_file" 2>/dev/null || true)"
208
- if [[ -z "$pid" ]]; then
209
- rm -f "$pending_file"
210
- return 1
211
- fi
212
-
213
- if kill -0 "$pid" 2>/dev/null; then
214
- printf '%s\n' "$pid"
215
- return 0
216
- fi
217
-
218
- rm -f "$pending_file"
219
- return 1
220
- }
221
-
222
- pending_issue_launch_active() {
223
- local issue_id="${1:?issue id required}"
224
- if tmux has-session -t "${issue_prefix}${issue_id}" 2>/dev/null; then
225
- rm -f "${pending_launch_dir}/issue-${issue_id}.pid" 2>/dev/null || true
226
- return 1
227
- fi
228
- pending_launch_pid issue "$issue_id" >/dev/null
229
- }
230
-
231
- resident_issue_controller_file() {
232
- local issue_id="${1:?issue id required}"
233
- printf '%s/resident-workers/issues/%s/controller.env\n' "${state_root}" "${issue_id}"
234
- }
235
-
236
- resident_issue_controller_state() {
237
- local issue_id="${1:?issue id required}"
238
- local controller_file state=""
239
-
240
- controller_file="$(resident_issue_controller_file "$issue_id")"
241
- [[ -f "${controller_file}" ]] || return 1
242
-
243
- state="$(awk -F= '/^CONTROLLER_STATE=/{print $2; exit}' "${controller_file}" 2>/dev/null | tr -d '"' || true)"
244
- [[ -n "${state}" ]] || return 1
245
- printf '%s\n' "${state}"
246
- }
247
-
248
- pending_issue_launch_counts_toward_capacity() {
249
- local issue_id="${1:?issue id required}"
250
- local controller_state=""
251
-
252
- if ! pending_issue_launch_active "${issue_id}"; then
253
- return 1
254
- fi
255
-
256
- controller_state="$(resident_issue_controller_state "${issue_id}" || true)"
257
- if [[ -n "${controller_state}" ]]; then
258
- case "${controller_state}" in
259
- idle|sleeping|waiting-due|waiting-open-pr|waiting-provider)
260
- return 1
261
- ;;
262
- esac
263
- fi
264
-
265
- return 0
266
- }
267
-
268
- pending_pr_launch_active() {
269
- local pr_id="${1:?pr id required}"
270
- if tmux has-session -t "${pr_prefix}${pr_id}" 2>/dev/null; then
271
- rm -f "${pending_launch_dir}/pr-${pr_id}.pid" 2>/dev/null || true
272
- return 1
273
- fi
274
- pending_launch_pid pr "$pr_id" >/dev/null
275
- }
276
-
277
- running_pr_workers() {
278
- ensure_running_pr_workers_cache
279
- printf '%s\n' "$running_pr_workers_cache"
280
- }
281
-
282
- ensure_tmux_sessions_cache() {
283
- if [[ "$tmux_sessions_cache_loaded" != "yes" ]]; then
284
- tmux_sessions_cache="$(tmux list-sessions -F '#S' 2>/dev/null || true)"
285
- tmux_sessions_cache_loaded="yes"
286
- fi
287
- }
288
-
289
- ensure_all_running_workers_cache() {
290
- local session
291
- if [[ "$all_running_workers_cache_loaded" == "yes" ]]; then
292
- return 0
293
- fi
294
- ensure_tmux_sessions_cache
295
- all_running_workers_cache=""
296
- while IFS= read -r session; do
297
- [[ -n "$session" ]] || continue
298
- if session_matches_prefix "$session"; then
299
- all_running_workers_cache+="${session}"$'\n'
300
- fi
301
- done <<<"$tmux_sessions_cache"
302
- all_running_workers_cache="${all_running_workers_cache%$'\n'}"
303
- all_running_workers_cache_loaded="yes"
304
- }
305
-
306
- auth_wait_workers() {
307
- ensure_auth_wait_workers_cache
308
- printf '%s\n' "$auth_wait_workers_cache"
309
- }
310
-
311
- ensure_auth_wait_workers_cache() {
312
- local session
313
- if [[ "$auth_wait_workers_cache_loaded" == "yes" ]]; then
314
- return 0
315
- fi
316
- ensure_tmux_sessions_cache
317
- auth_wait_workers_cache=""
318
- while IFS= read -r session; do
319
- [[ -n "$session" ]] || continue
320
- session_matches_prefix "$session" || continue
321
- if session_is_auth_waiting "$session"; then
322
- auth_wait_workers_cache+="${session}"$'\n'
323
- fi
324
- done <<<"$tmux_sessions_cache"
325
- auth_wait_workers_cache="${auth_wait_workers_cache%$'\n'}"
326
- auth_wait_workers_cache_loaded="yes"
327
- }
328
-
329
- ensure_running_issue_workers_cache() {
330
- local session
331
- if [[ "$running_issue_workers_cache_loaded" == "yes" ]]; then
332
- return 0
333
- fi
334
- ensure_tmux_sessions_cache
335
- running_issue_workers_cache=""
336
- while IFS= read -r session; do
337
- [[ -n "$session" ]] || continue
338
- if [[ "$session" == "${issue_prefix}"* ]]; then
339
- if session_is_auth_waiting "$session"; then
340
- continue
341
- fi
342
- running_issue_workers_cache+="${session}"$'\n'
343
- fi
344
- done <<<"$tmux_sessions_cache"
345
- running_issue_workers_cache="${running_issue_workers_cache%$'\n'}"
346
- running_issue_workers_cache_loaded="yes"
347
- }
348
-
349
- ensure_running_pr_workers_cache() {
350
- local session
351
- if [[ "$running_pr_workers_cache_loaded" == "yes" ]]; then
352
- return 0
353
- fi
354
- ensure_tmux_sessions_cache
355
- running_pr_workers_cache=""
356
- while IFS= read -r session; do
357
- [[ -n "$session" ]] || continue
358
- if [[ "$session" == "${pr_prefix}"* ]]; then
359
- if session_is_auth_waiting "$session"; then
360
- continue
361
- fi
362
- running_pr_workers_cache+="${session}"$'\n'
363
- fi
364
- done <<<"$tmux_sessions_cache"
365
- running_pr_workers_cache="${running_pr_workers_cache%$'\n'}"
366
- running_pr_workers_cache_loaded="yes"
367
- }
368
-
369
- worker_count() {
370
- local workers="${1:-}"
371
- if [[ -z "$workers" ]]; then
372
- printf '0\n'
373
- return
374
- fi
375
- printf '%s\n' "$workers" | sed '/^$/d' | wc -l | tr -d ' '
376
- }
377
-
378
- retry_ready() {
379
- local kind="${1:?kind required}"
380
- local item_id="${2:?item id required}"
381
- local retry_out ready
382
-
383
- retry_out="$(
384
- "${shared_agent_home}/tools/bin/agent-project-retry-state" \
385
- --state-root "$state_root" \
386
- --kind "$kind" \
387
- --item-id "$item_id" \
388
- --action get
389
- )"
390
- ready="$(awk -F= '/^READY=/{print $2}' <<<"$retry_out")"
391
- [[ "$ready" == "yes" ]]
392
- }
393
-
394
- provider_cooldown_state() {
395
- "${shared_agent_home}/tools/bin/provider-cooldown-state.sh" get
396
- }
397
-
398
- issue_id_from_session() {
399
- local session="${1:?session required}"
400
- local issue_id=""
401
- if [[ "$session" == "${issue_prefix}"* ]]; then
402
- issue_id="${session#${issue_prefix}}"
403
- fi
404
- if [[ "$issue_id" =~ ^[0-9]+$ ]]; then
405
- printf '%s\n' "$issue_id"
406
- return 0
407
- fi
408
- return 1
409
- }
410
-
411
- pr_id_from_session() {
412
- local session="${1:?session required}"
413
- local pr_id=""
414
- if [[ "$session" == "${pr_prefix}"* ]]; then
415
- pr_id="${session#${pr_prefix}}"
416
- fi
417
- if [[ "$pr_id" =~ ^[0-9]+$ ]]; then
418
- printf '%s\n' "$pr_id"
419
- return 0
420
- fi
421
- return 1
422
- }
423
-
424
161
  if [[ ! -f "$hook_file" ]]; then
425
162
  echo "missing hook file: $hook_file" >&2
426
163
  exit 1
@@ -493,6 +230,23 @@ if ! declare -F heartbeat_sync_issue_labels >/dev/null 2>&1; then
493
230
  heartbeat_sync_issue_labels() { :; }
494
231
  fi
495
232
 
233
+
234
+ # --- Source modular libraries ---
235
+
236
+ heartbeat_lib_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
237
+ # shellcheck source=/dev/null
238
+ source "${heartbeat_lib_dir}/heartbeat-loop-worker-lib.sh"
239
+ # shellcheck source=/dev/null
240
+ source "${heartbeat_lib_dir}/heartbeat-loop-cache-lib.sh"
241
+ # shellcheck source=/dev/null
242
+ source "${heartbeat_lib_dir}/heartbeat-loop-counting-lib.sh"
243
+ # shellcheck source=/dev/null
244
+ source "${heartbeat_lib_dir}/heartbeat-loop-scheduling-lib.sh"
245
+ # shellcheck source=/dev/null
246
+ source "${heartbeat_lib_dir}/heartbeat-loop-pr-strategy-lib.sh"
247
+
248
+ # --- Launch staging & rollback ---
249
+
496
250
  launch_in_progress_kind=""
497
251
  launch_in_progress_id=""
498
252
  launch_in_progress_issue_is_heavy="no"
@@ -558,6 +312,9 @@ cleanup_scheduler_caches() {
558
312
  if [[ -n "${pr_risk_cache_dir:-}" && -d "${pr_risk_cache_dir}" ]]; then
559
313
  rm -rf "${pr_risk_cache_dir}" || true
560
314
  fi
315
+ if declare -F heartbeat_invalidate_snapshot_cache >/dev/null 2>&1; then
316
+ heartbeat_invalidate_snapshot_cache
317
+ fi
561
318
  }
562
319
 
563
320
  stage_issue_launch() {
@@ -603,1150 +360,118 @@ rollback_launch_in_progress() {
603
360
 
604
361
  trap rollback_launch_in_progress EXIT INT TERM
605
362
 
606
- cache_prefix() {
607
- local raw_prefix="${issue_prefix:-${pr_prefix:-agent-control-plane}}"
608
- local sanitized=""
609
-
610
- sanitized="$(printf '%s' "${raw_prefix}" | tr '/[:space:]' '-' | tr -cd '[:alnum:]_.-')"
611
- if [[ -z "${sanitized}" ]]; then
612
- sanitized="agent-control-plane"
613
- fi
614
-
615
- printf '%s\n' "${sanitized}"
616
- }
617
-
618
- ensure_issue_attr_cache_dir() {
619
- if [[ -z "${issue_attr_cache_dir:-}" || ! -d "${issue_attr_cache_dir:-}" ]]; then
620
- issue_attr_cache_dir="$(mktemp -d "${TMPDIR:-/tmp}/$(cache_prefix)-issue-attrs.XXXXXX")"
621
- fi
622
- }
623
-
624
- ensure_pr_attr_cache_dir() {
625
- if [[ -z "${pr_attr_cache_dir:-}" || ! -d "${pr_attr_cache_dir:-}" ]]; then
626
- pr_attr_cache_dir="$(mktemp -d "${TMPDIR:-/tmp}/$(cache_prefix)-pr-attrs.XXXXXX")"
627
- fi
628
- }
629
-
630
- ensure_pr_risk_cache_dir() {
631
- if [[ -z "${pr_risk_cache_dir:-}" || ! -d "${pr_risk_cache_dir:-}" ]]; then
632
- pr_risk_cache_dir="$(mktemp -d "${TMPDIR:-/tmp}/$(cache_prefix)-pr-risk.XXXXXX")"
633
- fi
634
- }
635
-
636
- pr_risk_runtime_cache_fresh() {
637
- local cache_file="${1:?cache file required}"
638
- local modified_at now age
639
- [[ -f "$cache_file" ]] || return 1
640
- modified_at="$(stat -f '%m' "$cache_file" 2>/dev/null || true)"
641
- [[ "$modified_at" =~ ^[0-9]+$ ]] || return 1
642
- now="$(date +%s)"
643
- age=$((now - modified_at))
644
- (( age >= 0 && age <= pr_risk_runtime_cache_ttl_seconds ))
645
- }
363
+ # --- PR launch dispatcher ---
646
364
 
647
- cached_issue_attr() {
648
- local attr_name="${1:?attr name required}"
649
- local issue_id="${2:?issue id required}"
650
- local cache_file attr_value
365
+ launch_pr_candidate_json() {
366
+ local pr_candidate_json="${1:?pr candidate json required}"
367
+ local pr_number pr_lane launch_out
651
368
 
652
- ensure_issue_attr_cache_dir
653
- cache_file="${issue_attr_cache_dir}/${issue_id}.${attr_name}"
654
- if [[ -f "${cache_file}" ]]; then
655
- cat "${cache_file}"
656
- return 0
657
- fi
369
+ pr_number="$(jq -r '.number' <<<"$pr_candidate_json")"
370
+ pr_lane="$(jq -r '.agentLane' <<<"$pr_candidate_json")"
371
+ stage_pr_launch "$pr_number"
658
372
 
659
- case "${attr_name}" in
660
- heavy)
661
- attr_value="$(heartbeat_issue_is_heavy "${issue_id}")"
662
- ;;
663
- recurring)
664
- attr_value="$(heartbeat_issue_is_recurring "${issue_id}")"
373
+ case "$pr_lane" in
374
+ double-check-1|double-check-2|automerge)
375
+ if ! launch_out="$(heartbeat_start_pr_review_worker "$pr_number" 2>&1)"; then
376
+ heartbeat_clear_pr_running "$pr_number" || true
377
+ clear_launch_in_progress
378
+ record_memory "failed to launch PR review worker for #${pr_number}"
379
+ print_block "LAUNCH_FAILED_PR=${pr_number}" "$launch_out"
380
+ exit 1
381
+ fi
382
+ record_memory "launched PR review worker for #${pr_number}"
665
383
  ;;
666
- scheduled)
667
- attr_value="$(heartbeat_issue_is_scheduled "${issue_id}")"
384
+ merge-repair)
385
+ if ! launch_out="$(heartbeat_start_pr_merge_repair_worker "$pr_number" 2>&1)"; then
386
+ heartbeat_clear_pr_running "$pr_number" || true
387
+ clear_launch_in_progress
388
+ record_memory "failed to launch PR merge-repair worker for #${pr_number}"
389
+ print_block "LAUNCH_FAILED_PR=${pr_number}" "$launch_out"
390
+ exit 1
391
+ fi
392
+ record_memory "launched PR merge-repair worker for #${pr_number}"
668
393
  ;;
669
- schedule_interval_seconds)
670
- attr_value="$(heartbeat_issue_schedule_interval_seconds "${issue_id}")"
394
+ ci-refresh)
395
+ if ! launch_out="$(heartbeat_start_pr_ci_refresh "$pr_number" 2>&1)"; then
396
+ heartbeat_clear_pr_running "$pr_number" || true
397
+ clear_launch_in_progress
398
+ record_memory "failed to trigger PR ci-refresh for #${pr_number}"
399
+ print_block "LAUNCH_FAILED_PR=${pr_number}" "$launch_out"
400
+ exit 1
401
+ fi
402
+ heartbeat_clear_pr_running "$pr_number" || true
403
+ record_memory "triggered PR ci-refresh for #${pr_number}"
671
404
  ;;
672
- exclusive)
673
- attr_value="$(heartbeat_issue_is_exclusive "${issue_id}")"
405
+ fix)
406
+ if ! launch_out="$(heartbeat_start_pr_fix_worker "$pr_number" 2>&1)"; then
407
+ heartbeat_clear_pr_running "$pr_number" || true
408
+ clear_launch_in_progress
409
+ record_memory "failed to launch PR fix worker for #${pr_number}"
410
+ print_block "LAUNCH_FAILED_PR=${pr_number}" "$launch_out"
411
+ exit 1
412
+ fi
413
+ record_memory "launched PR fix worker for #${pr_number}"
674
414
  ;;
675
415
  *)
676
- echo "unsupported issue cache attr: ${attr_name}" >&2
677
- return 1
416
+ launch_out="Unsupported PR lane: ${pr_lane}"
417
+ heartbeat_clear_pr_running "$pr_number" || true
418
+ clear_launch_in_progress
419
+ print_block "LAUNCH_FAILED_PR=${pr_number}" "$launch_out"
420
+ exit 1
678
421
  ;;
679
422
  esac
680
423
 
681
- printf '%s\n' "${attr_value}" >"${cache_file}"
682
- printf '%s\n' "${attr_value}"
683
- }
684
-
685
- cached_pr_is_exclusive() {
686
- local pr_number="${1:?pr number required}"
687
- local cache_file attr_value
688
-
689
- ensure_pr_attr_cache_dir
690
- cache_file="${pr_attr_cache_dir}/${pr_number}.exclusive"
691
- if [[ -f "${cache_file}" ]]; then
692
- cat "${cache_file}"
693
- return 0
694
- fi
695
-
696
- attr_value="$(heartbeat_pr_is_exclusive "${pr_number}")"
697
- printf '%s\n' "${attr_value}" >"${cache_file}"
698
- printf '%s\n' "${attr_value}"
699
- }
700
-
701
- cached_pr_risk_json() {
702
- local pr_number="${1:?pr number required}"
703
- local cache_file runtime_cache_file risk_json
704
-
705
- ensure_pr_risk_cache_dir
706
- cache_file="${pr_risk_cache_dir}/${pr_number}.json"
707
- runtime_cache_file="${pr_risk_runtime_cache_dir}/${pr_number}.json"
708
- if [[ -f "${cache_file}" ]]; then
709
- cat "${cache_file}"
710
- return 0
424
+ clear_launch_in_progress
425
+ print_block "LAUNCHED_PR=${pr_number}" "$(printf 'LANE=%s\n%s' "$pr_lane" "$launch_out")"
426
+ if [[ "$pr_lane" != "ci-refresh" ]]; then
427
+ running_total_count=$((running_total_count + 1))
428
+ running_pr_count=$((running_pr_count + 1))
711
429
  fi
712
-
713
- if pr_risk_runtime_cache_fresh "${runtime_cache_file}"; then
714
- cp "${runtime_cache_file}" "${cache_file}"
715
- cat "${cache_file}"
716
- return 0
430
+ launched_pr_count=$((launched_pr_count + 1))
431
+ if (( launch_budget_remaining > 0 )); then
432
+ launch_budget_remaining=$((launch_budget_remaining - 1))
717
433
  fi
718
-
719
- risk_json="$(heartbeat_pr_risk_json "${pr_number}")"
720
- printf '%s\n' "${risk_json}" >"${cache_file}"
721
- printf '%s\n' "${risk_json}" >"${runtime_cache_file}"
722
- printf '%s\n' "${risk_json}"
723
434
  }
724
435
 
725
- running_heavy_issue_workers() {
726
- local session issue_id is_heavy count=0
727
- ensure_running_issue_workers_cache
728
- while IFS= read -r session; do
729
- [[ -n "$session" ]] || continue
730
- issue_id="$(issue_id_from_session "$session" || true)"
731
- [[ -n "$issue_id" ]] || continue
732
- is_heavy="$(cached_issue_attr heavy "$issue_id")"
733
- if [[ "$is_heavy" == "yes" ]]; then
734
- count=$((count + 1))
735
- fi
736
- done <<<"$running_issue_workers_cache"
737
- printf '%s\n' "$count"
738
- }
739
-
740
- pending_issue_launch_count() {
741
- local pending_file issue_id count=0
742
- for pending_file in "${pending_launch_dir}"/issue-*.pid; do
743
- [[ -f "$pending_file" ]] || continue
744
- issue_id="${pending_file##*/issue-}"
745
- issue_id="${issue_id%.pid}"
746
- [[ -n "$issue_id" ]] || continue
747
- if tmux has-session -t "${issue_prefix}${issue_id}" 2>/dev/null; then
748
- continue
749
- fi
750
- if pending_issue_launch_counts_toward_capacity "$issue_id"; then
751
- count=$((count + 1))
752
- fi
753
- done
754
- printf '%s\n' "$count"
755
- }
436
+ # ============================================================================
437
+ # MAIN EXECUTION
438
+ # ============================================================================
756
439
 
757
- pending_pr_launch_count() {
758
- local pending_file pr_id count=0
759
- for pending_file in "${pending_launch_dir}"/pr-*.pid; do
760
- [[ -f "$pending_file" ]] || continue
761
- pr_id="${pending_file##*/pr-}"
762
- pr_id="${pr_id%.pid}"
763
- [[ -n "$pr_id" ]] || continue
764
- if tmux has-session -t "${pr_prefix}${pr_id}" 2>/dev/null; then
765
- continue
766
- fi
767
- if pending_pr_launch_active "$pr_id"; then
768
- count=$((count + 1))
769
- fi
770
- done
771
- printf '%s\n' "$count"
772
- }
773
-
774
- pending_heavy_issue_launch_count() {
775
- local pending_file issue_id count=0
776
- for pending_file in "${pending_launch_dir}"/issue-*.pid; do
777
- [[ -f "$pending_file" ]] || continue
778
- issue_id="${pending_file##*/issue-}"
779
- issue_id="${issue_id%.pid}"
780
- [[ -n "$issue_id" ]] || continue
781
- if tmux has-session -t "${issue_prefix}${issue_id}" 2>/dev/null; then
782
- continue
783
- fi
784
- if pending_issue_launch_counts_toward_capacity "$issue_id" && [[ "$(cached_issue_attr heavy "$issue_id")" == "yes" ]]; then
785
- count=$((count + 1))
786
- fi
787
- done
788
- printf '%s\n' "$count"
789
- }
790
-
791
- pending_scheduled_issue_launch_count() {
792
- local pending_file issue_id count=0
793
- for pending_file in "${pending_launch_dir}"/issue-*.pid; do
794
- [[ -f "$pending_file" ]] || continue
795
- issue_id="${pending_file##*/issue-}"
796
- issue_id="${issue_id%.pid}"
797
- [[ -n "$issue_id" ]] || continue
798
- if tmux has-session -t "${issue_prefix}${issue_id}" 2>/dev/null; then
799
- continue
800
- fi
801
- if pending_issue_launch_counts_toward_capacity "$issue_id" && [[ "$(cached_issue_attr scheduled "$issue_id")" == "yes" ]]; then
802
- count=$((count + 1))
803
- fi
804
- done
805
- printf '%s\n' "$count"
806
- }
807
-
808
- pending_scheduled_heavy_issue_launch_count() {
809
- local pending_file issue_id count=0
810
- for pending_file in "${pending_launch_dir}"/issue-*.pid; do
811
- [[ -f "$pending_file" ]] || continue
812
- issue_id="${pending_file##*/issue-}"
813
- issue_id="${issue_id%.pid}"
814
- [[ -n "$issue_id" ]] || continue
815
- if tmux has-session -t "${issue_prefix}${issue_id}" 2>/dev/null; then
816
- continue
817
- fi
818
- if pending_issue_launch_counts_toward_capacity "$issue_id" \
819
- && [[ "$(cached_issue_attr scheduled "$issue_id")" == "yes" ]] \
820
- && [[ "$(cached_issue_attr heavy "$issue_id")" == "yes" ]]; then
821
- count=$((count + 1))
822
- fi
823
- done
824
- printf '%s\n' "$count"
825
- }
826
-
827
- pending_recurring_issue_launch_count() {
828
- local pending_file issue_id count=0
829
- for pending_file in "${pending_launch_dir}"/issue-*.pid; do
830
- [[ -f "$pending_file" ]] || continue
831
- issue_id="${pending_file##*/issue-}"
832
- issue_id="${issue_id%.pid}"
833
- [[ -n "$issue_id" ]] || continue
834
- if tmux has-session -t "${issue_prefix}${issue_id}" 2>/dev/null; then
835
- continue
836
- fi
837
- if pending_issue_launch_counts_toward_capacity "$issue_id" \
838
- && [[ "$(cached_issue_attr scheduled "$issue_id")" != "yes" ]] \
839
- && [[ "$(cached_issue_attr recurring "$issue_id")" == "yes" ]]; then
840
- count=$((count + 1))
841
- fi
842
- done
843
- printf '%s\n' "$count"
844
- }
845
-
846
- pending_blocked_recovery_issue_launch_count() {
847
- local pending_file issue_id count=0
848
- for pending_file in "${pending_launch_dir}"/issue-*.pid; do
849
- [[ -f "$pending_file" ]] || continue
850
- issue_id="${pending_file##*/issue-}"
851
- issue_id="${issue_id%.pid}"
852
- [[ -n "$issue_id" ]] || continue
853
- if tmux has-session -t "${issue_prefix}${issue_id}" 2>/dev/null; then
854
- continue
855
- fi
856
- if pending_issue_launch_counts_toward_capacity "$issue_id" && blocked_recovery_issue_has_state "$issue_id"; then
857
- count=$((count + 1))
858
- fi
859
- done
860
- printf '%s\n' "$count"
861
- }
862
-
863
- pending_exclusive_issue_launch_count() {
864
- local pending_file issue_id count=0
865
- for pending_file in "${pending_launch_dir}"/issue-*.pid; do
866
- [[ -f "$pending_file" ]] || continue
867
- issue_id="${pending_file##*/issue-}"
868
- issue_id="${issue_id%.pid}"
869
- [[ -n "$issue_id" ]] || continue
870
- if tmux has-session -t "${issue_prefix}${issue_id}" 2>/dev/null; then
871
- continue
872
- fi
873
- if pending_issue_launch_counts_toward_capacity "$issue_id" && [[ "$(cached_issue_attr exclusive "$issue_id")" == "yes" ]]; then
874
- count=$((count + 1))
875
- fi
876
- done
877
- printf '%s\n' "$count"
878
- }
879
-
880
- pending_exclusive_pr_launch_count() {
881
- local pending_file pr_id count=0
882
- for pending_file in "${pending_launch_dir}"/pr-*.pid; do
883
- [[ -f "$pending_file" ]] || continue
884
- pr_id="${pending_file##*/pr-}"
885
- pr_id="${pr_id%.pid}"
886
- [[ -n "$pr_id" ]] || continue
887
- if tmux has-session -t "${pr_prefix}${pr_id}" 2>/dev/null; then
888
- continue
889
- fi
890
- if pending_pr_launch_active "$pr_id" && [[ "$(cached_pr_is_exclusive "$pr_id")" == "yes" ]]; then
891
- count=$((count + 1))
892
- fi
893
- done
894
- printf '%s\n' "$count"
895
- }
896
-
897
- running_non_recurring_issue_workers() {
898
- local session issue_id is_recurring is_scheduled count=0
899
- ensure_running_issue_workers_cache
900
- while IFS= read -r session; do
901
- [[ -n "$session" ]] || continue
902
- issue_id="$(issue_id_from_session "$session" || true)"
903
- [[ -n "$issue_id" ]] || continue
904
- is_scheduled="$(cached_issue_attr scheduled "$issue_id")"
905
- if [[ "$is_scheduled" == "yes" ]]; then
906
- continue
907
- fi
908
- is_recurring="$(cached_issue_attr recurring "$issue_id")"
909
- if [[ "$is_recurring" != "yes" ]]; then
910
- count=$((count + 1))
911
- fi
912
- done <<<"$running_issue_workers_cache"
913
- printf '%s\n' "$count"
914
- }
915
-
916
- running_recurring_issue_workers() {
917
- local session issue_id is_recurring is_scheduled count=0
918
- ensure_running_issue_workers_cache
919
- while IFS= read -r session; do
920
- [[ -n "$session" ]] || continue
921
- issue_id="$(issue_id_from_session "$session" || true)"
922
- [[ -n "$issue_id" ]] || continue
923
- is_scheduled="$(cached_issue_attr scheduled "$issue_id")"
924
- if [[ "$is_scheduled" == "yes" ]]; then
925
- continue
926
- fi
927
- is_recurring="$(cached_issue_attr recurring "$issue_id")"
928
- if [[ "$is_recurring" == "yes" ]]; then
929
- count=$((count + 1))
930
- fi
931
- done <<<"$running_issue_workers_cache"
932
- # Also count pending recurring launches that are still in progress
933
- # (prevents infinite respawning when workers die before creating tmux sessions)
934
- count=$((count + $(pending_recurring_issue_launch_count)))
935
- printf '%s\n' "$count"
936
- }
937
-
938
- running_blocked_recovery_issue_workers() {
939
- local session issue_id count=0
940
- ensure_running_issue_workers_cache
941
- while IFS= read -r session; do
942
- [[ -n "$session" ]] || continue
943
- issue_id="$(issue_id_from_session "$session" || true)"
944
- [[ -n "$issue_id" ]] || continue
945
- if blocked_recovery_issue_has_state "$issue_id"; then
946
- count=$((count + 1))
947
- fi
948
- done <<<"$running_issue_workers_cache"
949
- printf '%s\n' "$count"
950
- }
951
-
952
- running_exclusive_issue_workers() {
953
- local session issue_id is_exclusive count=0
954
- ensure_running_issue_workers_cache
955
- while IFS= read -r session; do
956
- [[ -n "$session" ]] || continue
957
- issue_id="$(issue_id_from_session "$session" || true)"
958
- [[ -n "$issue_id" ]] || continue
959
- is_exclusive="$(cached_issue_attr exclusive "$issue_id")"
960
- if [[ "$is_exclusive" == "yes" ]]; then
961
- count=$((count + 1))
962
- fi
963
- done <<<"$running_issue_workers_cache"
964
- printf '%s\n' "$count"
965
- }
966
-
967
- running_exclusive_pr_workers() {
968
- local session pr_id is_exclusive count=0
969
- ensure_running_pr_workers_cache
970
- while IFS= read -r session; do
971
- [[ -n "$session" ]] || continue
972
- pr_id="$(pr_id_from_session "$session" || true)"
973
- [[ -n "$pr_id" ]] || continue
974
- is_exclusive="$(cached_pr_is_exclusive "$pr_id")"
975
- if [[ "$is_exclusive" == "yes" ]]; then
976
- count=$((count + 1))
977
- fi
978
- done <<<"$running_pr_workers_cache"
979
- printf '%s\n' "$count"
980
- }
981
-
982
- running_scheduled_issue_workers() {
983
- local session issue_id is_scheduled count=0
984
- ensure_running_issue_workers_cache
985
- while IFS= read -r session; do
986
- [[ -n "$session" ]] || continue
987
- issue_id="$(issue_id_from_session "$session" || true)"
988
- [[ -n "$issue_id" ]] || continue
989
- is_scheduled="$(cached_issue_attr scheduled "$issue_id")"
990
- if [[ "$is_scheduled" == "yes" ]]; then
991
- count=$((count + 1))
992
- fi
993
- done <<<"$running_issue_workers_cache"
994
- printf '%s\n' "$count"
995
- }
996
-
997
- running_scheduled_heavy_issue_workers() {
998
- local session issue_id is_scheduled is_heavy count=0
999
- ensure_running_issue_workers_cache
1000
- while IFS= read -r session; do
1001
- [[ -n "$session" ]] || continue
1002
- issue_id="$(issue_id_from_session "$session" || true)"
1003
- [[ -n "$issue_id" ]] || continue
1004
- is_scheduled="$(cached_issue_attr scheduled "$issue_id")"
1005
- is_heavy="$(cached_issue_attr heavy "$issue_id")"
1006
- if [[ "$is_scheduled" == "yes" && "$is_heavy" == "yes" ]]; then
1007
- count=$((count + 1))
1008
- fi
1009
- done <<<"$running_issue_workers_cache"
1010
- printf '%s\n' "$count"
1011
- }
1012
-
1013
- ready_non_recurring_issue_count() {
1014
- local issue_id is_recurring count=0
1015
- ensure_ready_issue_ids_cache
1016
- while IFS= read -r issue_id; do
1017
- [[ -n "$issue_id" ]] || continue
1018
- if [[ "$(cached_issue_attr scheduled "$issue_id")" == "yes" ]]; then
1019
- continue
1020
- fi
1021
- is_recurring="$(cached_issue_attr recurring "$issue_id")"
1022
- if [[ "$is_recurring" != "yes" ]]; then
1023
- count=$((count + 1))
1024
- fi
1025
- done <<<"$ready_issue_ids_cache"
1026
- printf '%s\n' "$count"
1027
- }
1028
-
1029
- blocked_recovery_issue_ids() {
1030
- ensure_blocked_recovery_issue_ids_cache
1031
- printf '%s\n' "$blocked_recovery_issue_ids_cache"
1032
- }
1033
-
1034
- ordered_ready_issue_ids() {
1035
- ensure_ordered_ready_issue_ids_cache
1036
- printf '%s\n' "$ordered_ready_issue_ids_cache"
1037
- }
1038
-
1039
- due_scheduled_issue_ids() {
1040
- ensure_due_scheduled_issue_ids_cache
1041
- printf '%s\n' "$due_scheduled_issue_ids_cache"
1042
- }
1043
-
1044
- due_blocked_recovery_issue_ids() {
1045
- ensure_due_blocked_recovery_issue_ids_cache
1046
- printf '%s\n' "$due_blocked_recovery_issue_ids_cache"
1047
- }
1048
-
1049
- ensure_due_scheduled_issue_ids_cache() {
1050
- if [[ "$due_scheduled_issue_ids_cache_loaded" != "yes" ]]; then
1051
- due_scheduled_issue_ids_cache="$(build_due_scheduled_issue_ids_cache)"
1052
- due_scheduled_issue_ids_cache_loaded="yes"
1053
- fi
1054
- }
1055
-
1056
- ensure_due_blocked_recovery_issue_ids_cache() {
1057
- if [[ "$due_blocked_recovery_issue_ids_cache_loaded" != "yes" ]]; then
1058
- due_blocked_recovery_issue_ids_cache="$(build_due_blocked_recovery_issue_ids_cache)"
1059
- due_blocked_recovery_issue_ids_cache_loaded="yes"
1060
- fi
1061
- }
1062
-
1063
- build_due_scheduled_issue_ids_cache() {
1064
- local issue_id now_epoch due_epoch
1065
- now_epoch="$(date +%s)"
1066
- ensure_ready_issue_ids_cache
1067
- while IFS= read -r issue_id; do
1068
- [[ -n "$issue_id" ]] || continue
1069
- if [[ "$(cached_issue_attr scheduled "$issue_id")" != "yes" ]]; then
1070
- continue
1071
- fi
1072
- if ! scheduled_issue_is_due "$issue_id"; then
1073
- continue
1074
- fi
1075
- due_epoch="$(scheduled_issue_due_epoch "$issue_id")"
1076
- if ! [[ "${due_epoch:-}" =~ ^[0-9]+$ ]]; then
1077
- due_epoch=0
1078
- fi
1079
- printf '%s\t%s\n' "$due_epoch" "$issue_id"
1080
- done <<<"$ready_issue_ids_cache" | sort -n -k1,1 -k2,2n | cut -f2
1081
- }
1082
-
1083
- build_due_blocked_recovery_issue_ids_cache() {
1084
- local issue_id due_epoch
1085
- if (( max_concurrent_blocked_recovery_issue_workers <= 0 )); then
1086
- return 0
1087
- fi
1088
-
1089
- ensure_blocked_recovery_issue_ids_cache
1090
- while IFS= read -r issue_id; do
1091
- [[ -n "$issue_id" ]] || continue
1092
- if ! blocked_recovery_issue_is_due "$issue_id"; then
1093
- continue
1094
- fi
1095
- due_epoch="$(blocked_recovery_issue_due_epoch "$issue_id")"
1096
- if ! [[ "${due_epoch:-}" =~ ^[0-9]+$ ]]; then
1097
- due_epoch=0
1098
- fi
1099
- printf '%s\t%s\n' "$due_epoch" "$issue_id"
1100
- done <<<"$blocked_recovery_issue_ids_cache" | sort -n -k1,1 -k2,2n | cut -f2
1101
- }
1102
-
1103
- build_ordered_ready_issue_ids_cache() {
1104
- local issue_id is_recurring last_recurring_issue seen_last="no"
1105
- local -a recurring_ids=()
1106
- ensure_ready_issue_ids_cache
1107
- while IFS= read -r issue_id; do
1108
- [[ -n "$issue_id" ]] || continue
1109
- if [[ "$(cached_issue_attr scheduled "$issue_id")" == "yes" ]]; then
1110
- continue
1111
- fi
1112
- is_recurring="$(cached_issue_attr recurring "$issue_id")"
1113
- if [[ "$is_recurring" != "yes" ]]; then
1114
- printf '%s\n' "$issue_id"
1115
- else
1116
- recurring_ids+=("$issue_id")
1117
- fi
1118
- done <<<"$ready_issue_ids_cache"
1119
-
1120
- if (( ${#recurring_ids[@]} == 0 )); then
1121
- return 0
1122
- fi
1123
-
1124
- last_recurring_issue="$(last_launched_recurring_issue_id || true)"
1125
- if [[ -n "$last_recurring_issue" ]]; then
1126
- local emitted_after_last=0
1127
- for issue_id in "${recurring_ids[@]}"; do
1128
- if [[ "$seen_last" == "yes" ]]; then
1129
- printf '%s\n' "$issue_id"
1130
- emitted_after_last=$((emitted_after_last + 1))
1131
- fi
1132
- if [[ "$issue_id" == "$last_recurring_issue" ]]; then
1133
- seen_last="yes"
1134
- fi
1135
- done
1136
- fi
1137
-
1138
- for issue_id in "${recurring_ids[@]}"; do
1139
- # Stop the wrap-around once we reach the last-launched issue, but only
1140
- # when the first loop already emitted at least one issue after it.
1141
- # When there is exactly one recurring issue (or the last-launched issue
1142
- # is the final element), emitted_after_last is 0, so we must still
1143
- # include it here to avoid producing an empty list.
1144
- if [[ -n "$last_recurring_issue" && "$seen_last" == "yes" && "$issue_id" == "$last_recurring_issue" && "$emitted_after_last" -gt 0 ]]; then
1145
- break
1146
- fi
1147
- printf '%s\n' "$issue_id"
1148
- done
1149
- }
1150
-
1151
- completed_workers() {
1152
- ensure_completed_workers_cache
1153
- printf '%s\n' "$completed_workers_cache"
1154
- }
1155
-
1156
- reconciled_marker_matches_run() {
1157
- local run_dir="${1:?run dir required}"
1158
- local marker_file="${run_dir}/reconciled.ok"
1159
- local run_env="${run_dir}/run.env"
1160
- local marker_started_at=""
1161
- local run_started_at=""
1162
-
1163
- [[ -f "${marker_file}" && -f "${run_env}" ]] || return 1
1164
-
1165
- marker_started_at="$(awk -F= '/^STARTED_AT=/{print $2}' "${marker_file}" 2>/dev/null | tr -d '"' | tail -n 1 || true)"
1166
- run_started_at="$(awk -F= '/^STARTED_AT=/{print $2}' "${run_env}" 2>/dev/null | tr -d '"' | tail -n 1 || true)"
1167
-
1168
- [[ -n "${marker_started_at}" && -n "${run_started_at}" && "${marker_started_at}" == "${run_started_at}" ]]
1169
- }
1170
-
1171
- ensure_completed_workers_cache() {
1172
- local dir session issue_id status_line status
1173
- if [[ "$completed_workers_cache_loaded" == "yes" ]]; then
1174
- return 0
1175
- fi
1176
- completed_workers_cache=""
1177
- for dir in "$runs_root"/*; do
1178
- [[ -d "$dir" ]] || continue
1179
- session="${dir##*/}"
1180
- session_matches_prefix "$session" || continue
1181
- if reconciled_marker_matches_run "$dir"; then
1182
- continue
1183
- fi
1184
- if [[ "$session" == "${issue_prefix}"* ]]; then
1185
- issue_id="$(issue_id_from_session "$session" || true)"
1186
- if [[ -n "${issue_id}" ]] && pending_issue_launch_active "${issue_id}"; then
1187
- continue
1188
- fi
1189
- fi
1190
- status_line="$(
1191
- "${shared_agent_home}/tools/bin/agent-project-worker-status" \
1192
- --runs-root "$runs_root" \
1193
- --session "$session" \
1194
- | awk -F= '/^STATUS=/{print $2}' || true
1195
- )"
1196
- status="${status_line:-UNKNOWN}"
1197
- if [[ "$status" == "SUCCEEDED" || "$status" == "FAILED" ]]; then
1198
- completed_workers_cache+="${session}"$'\n'
1199
- fi
1200
- done
1201
- completed_workers_cache="${completed_workers_cache%$'\n'}"
1202
- completed_workers_cache_loaded="yes"
1203
- }
1204
-
1205
- ready_issue_ids() {
1206
- ensure_ready_issue_ids_cache
1207
- printf '%s\n' "$ready_issue_ids_cache"
1208
- }
1209
-
1210
- ensure_ready_issue_ids_cache() {
1211
- if [[ "$ready_issue_ids_cache_loaded" != "yes" ]]; then
1212
- ready_issue_ids_cache="$(heartbeat_list_ready_issue_ids)"
1213
- ready_issue_ids_cache_loaded="yes"
1214
- fi
1215
- }
1216
-
1217
- last_launched_recurring_issue_id() {
1218
- if [[ -f "$recurring_rotation_file" ]]; then
1219
- tr -d '[:space:]' <"$recurring_rotation_file"
1220
- fi
1221
- }
1222
-
1223
- record_recurring_issue_launch() {
1224
- local issue_id="${1:?issue id required}"
1225
- printf '%s\n' "$issue_id" >"$recurring_rotation_file"
1226
- }
1227
-
1228
- scheduled_state_file() {
1229
- local issue_id="${1:?issue id required}"
1230
- printf '%s\n' "${scheduled_state_dir}/${issue_id}.env"
1231
- }
1232
-
1233
- scheduled_issue_due_epoch() {
1234
- local issue_id="${1:?issue id required}"
1235
- local state_file next_due_epoch
1236
- state_file="$(scheduled_state_file "$issue_id")"
1237
- if [[ ! -f "$state_file" ]]; then
1238
- printf '0\n'
1239
- return 0
1240
- fi
1241
-
1242
- next_due_epoch="$(awk -F= '/^NEXT_DUE_EPOCH=/{print $2}' "$state_file" 2>/dev/null | tr -d '[:space:]' || true)"
1243
- if ! [[ "${next_due_epoch:-}" =~ ^[0-9]+$ ]]; then
1244
- printf '0\n'
1245
- return 0
1246
- fi
1247
-
1248
- printf '%s\n' "$next_due_epoch"
1249
- }
1250
-
1251
- scheduled_issue_is_due() {
1252
- local issue_id="${1:?issue id required}"
1253
- local interval_seconds due_epoch now_epoch
1254
- interval_seconds="$(cached_issue_attr schedule_interval_seconds "$issue_id")"
1255
- if ! [[ "${interval_seconds:-}" =~ ^[1-9][0-9]*$ ]]; then
1256
- return 1
1257
- fi
1258
-
1259
- due_epoch="$(scheduled_issue_due_epoch "$issue_id")"
1260
- now_epoch="$(date +%s)"
1261
- if ! [[ "${due_epoch:-}" =~ ^[0-9]+$ ]] || (( due_epoch == 0 || due_epoch <= now_epoch )); then
1262
- return 0
1263
- fi
1264
- return 1
1265
- }
1266
-
1267
- record_scheduled_issue_launch() {
1268
- local issue_id="${1:?issue id required}"
1269
- local interval_seconds state_file now_epoch due_epoch next_due_epoch
1270
-
1271
- interval_seconds="$(cached_issue_attr schedule_interval_seconds "$issue_id")"
1272
- if ! [[ "${interval_seconds:-}" =~ ^[1-9][0-9]*$ ]]; then
1273
- return 0
1274
- fi
1275
-
1276
- now_epoch="$(date +%s)"
1277
- due_epoch="$(scheduled_issue_due_epoch "$issue_id")"
1278
- if ! [[ "${due_epoch:-}" =~ ^[0-9]+$ ]] || (( due_epoch <= 0 )); then
1279
- next_due_epoch=$((now_epoch + interval_seconds))
1280
- else
1281
- next_due_epoch="$due_epoch"
1282
- while (( next_due_epoch <= now_epoch )); do
1283
- next_due_epoch=$((next_due_epoch + interval_seconds))
1284
- done
1285
- fi
1286
-
1287
- state_file="$(scheduled_state_file "$issue_id")"
1288
- cat >"$state_file" <<EOF
1289
- INTERVAL_SECONDS=${interval_seconds}
1290
- LAST_STARTED_EPOCH=${now_epoch}
1291
- LAST_STARTED_AT=$(date -u -r "$now_epoch" +"%Y-%m-%dT%H:%M:%SZ")
1292
- NEXT_DUE_EPOCH=${next_due_epoch}
1293
- NEXT_DUE_AT=$(date -u -r "$next_due_epoch" +"%Y-%m-%dT%H:%M:%SZ")
1294
- UPDATED_AT=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
1295
- EOF
1296
- }
1297
-
1298
- record_scheduled_issue_result() {
1299
- local issue_id="${1:?issue id required}"
1300
- local result_status="${2:-unknown}"
1301
- local state_file interval_seconds last_started_epoch next_due_epoch now_epoch
1302
-
1303
- state_file="$(scheduled_state_file "$issue_id")"
1304
- interval_seconds="$(cached_issue_attr schedule_interval_seconds "$issue_id")"
1305
- last_started_epoch="$(awk -F= '/^LAST_STARTED_EPOCH=/{print $2}' "$state_file" 2>/dev/null | tr -d '[:space:]' || true)"
1306
- next_due_epoch="$(awk -F= '/^NEXT_DUE_EPOCH=/{print $2}' "$state_file" 2>/dev/null | tr -d '[:space:]' || true)"
1307
- now_epoch="$(date +%s)"
1308
-
1309
- if ! [[ "${interval_seconds:-}" =~ ^[1-9][0-9]*$ ]]; then
1310
- interval_seconds=0
1311
- fi
1312
- if ! [[ "${last_started_epoch:-}" =~ ^[0-9]+$ ]]; then
1313
- last_started_epoch=0
1314
- fi
1315
- if ! [[ "${next_due_epoch:-}" =~ ^[0-9]+$ ]]; then
1316
- next_due_epoch=0
1317
- fi
1318
-
1319
- cat >"$state_file" <<EOF
1320
- INTERVAL_SECONDS=${interval_seconds}
1321
- LAST_STARTED_EPOCH=${last_started_epoch}
1322
- LAST_STARTED_AT=$(if [[ "$last_started_epoch" =~ ^[0-9]+$ ]] && (( last_started_epoch > 0 )); then date -u -r "$last_started_epoch" +"%Y-%m-%dT%H:%M:%SZ"; fi)
1323
- LAST_RESULT_STATUS=${result_status}
1324
- LAST_RESULT_EPOCH=${now_epoch}
1325
- LAST_RESULT_AT=$(date -u -r "$now_epoch" +"%Y-%m-%dT%H:%M:%SZ")
1326
- NEXT_DUE_EPOCH=${next_due_epoch}
1327
- NEXT_DUE_AT=$(if [[ "$next_due_epoch" =~ ^[0-9]+$ ]] && (( next_due_epoch > 0 )); then date -u -r "$next_due_epoch" +"%Y-%m-%dT%H:%M:%SZ"; fi)
1328
- UPDATED_AT=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
1329
- EOF
1330
- }
1331
-
1332
- blocked_recovery_state_file() {
1333
- local issue_id="${1:?issue id required}"
1334
- printf '%s\n' "${blocked_recovery_state_dir}/${issue_id}.env"
1335
- }
1336
-
1337
- blocked_recovery_issue_has_state() {
1338
- local issue_id="${1:?issue id required}"
1339
- [[ -f "$(blocked_recovery_state_file "$issue_id")" ]]
1340
- }
1341
-
1342
- blocked_recovery_issue_due_epoch() {
1343
- local issue_id="${1:?issue id required}"
1344
- local state_file next_due_epoch
1345
- state_file="$(blocked_recovery_state_file "$issue_id")"
1346
- if [[ ! -f "$state_file" ]]; then
1347
- printf '0\n'
1348
- return 0
1349
- fi
1350
-
1351
- next_due_epoch="$(awk -F= '/^NEXT_DUE_EPOCH=/{print $2}' "$state_file" 2>/dev/null | tr -d '[:space:]' || true)"
1352
- if ! [[ "${next_due_epoch:-}" =~ ^[0-9]+$ ]]; then
1353
- printf '0\n'
1354
- return 0
1355
- fi
1356
-
1357
- printf '%s\n' "$next_due_epoch"
1358
- }
1359
-
1360
- blocked_recovery_issue_is_due() {
1361
- local issue_id="${1:?issue id required}"
1362
- local due_epoch now_epoch
1363
- if ! [[ "${blocked_recovery_cooldown_seconds:-}" =~ ^[1-9][0-9]*$ ]]; then
1364
- return 0
1365
- fi
1366
-
1367
- due_epoch="$(blocked_recovery_issue_due_epoch "$issue_id")"
1368
- now_epoch="$(date +%s)"
1369
- if ! [[ "${due_epoch:-}" =~ ^[0-9]+$ ]] || (( due_epoch == 0 || due_epoch <= now_epoch )); then
1370
- return 0
1371
- fi
1372
- return 1
1373
- }
1374
-
1375
- record_blocked_recovery_issue_launch() {
1376
- local issue_id="${1:?issue id required}"
1377
- local state_file now_epoch next_due_epoch next_due_at
1378
-
1379
- now_epoch="$(date +%s)"
1380
- next_due_epoch=0
1381
- next_due_at=""
1382
- if [[ "${blocked_recovery_cooldown_seconds:-}" =~ ^[1-9][0-9]*$ ]]; then
1383
- next_due_epoch=$((now_epoch + blocked_recovery_cooldown_seconds))
1384
- next_due_at="$(date -u -r "$next_due_epoch" +"%Y-%m-%dT%H:%M:%SZ")"
1385
- fi
1386
-
1387
- state_file="$(blocked_recovery_state_file "$issue_id")"
1388
- cat >"$state_file" <<EOF
1389
- LANE=blocked-recovery
1390
- LAST_STARTED_EPOCH=${now_epoch}
1391
- LAST_STARTED_AT=$(date -u -r "$now_epoch" +"%Y-%m-%dT%H:%M:%SZ")
1392
- NEXT_DUE_EPOCH=${next_due_epoch}
1393
- NEXT_DUE_AT=${next_due_at}
1394
- UPDATED_AT=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
1395
- EOF
1396
- }
1397
-
1398
- clear_blocked_recovery_issue_state() {
1399
- local issue_id="${1:?issue id required}"
1400
- rm -f "$(blocked_recovery_state_file "$issue_id")"
1401
- }
1402
-
1403
- open_agent_pr_ids() {
1404
- ensure_open_agent_pr_ids_cache
1405
- printf '%s\n' "$open_agent_pr_ids_cache"
1406
- }
1407
-
1408
- ensure_open_agent_pr_ids_cache() {
1409
- if [[ "$open_agent_pr_ids_cache_loaded" != "yes" ]]; then
1410
- open_agent_pr_ids_cache="$(heartbeat_list_open_agent_pr_ids)"
1411
- open_agent_pr_ids_cache_loaded="yes"
1412
- fi
1413
- }
1414
-
1415
- running_issue_ids() {
1416
- ensure_running_issue_ids_cache
1417
- printf '%s\n' "$running_issue_ids_cache"
1418
- }
1419
-
1420
- exclusive_issue_ids() {
1421
- ensure_exclusive_issue_ids_cache
1422
- printf '%s\n' "$exclusive_issue_ids_cache"
1423
- }
1424
-
1425
- exclusive_pr_ids() {
1426
- ensure_exclusive_pr_ids_cache
1427
- printf '%s\n' "$exclusive_pr_ids_cache"
1428
- }
1429
-
1430
- ensure_running_issue_ids_cache() {
1431
- if [[ "$running_issue_ids_cache_loaded" != "yes" ]]; then
1432
- running_issue_ids_cache="$(heartbeat_list_running_issue_ids)"
1433
- running_issue_ids_cache_loaded="yes"
1434
- fi
1435
- }
1436
-
1437
- ensure_exclusive_issue_ids_cache() {
1438
- if [[ "$exclusive_issue_ids_cache_loaded" != "yes" ]]; then
1439
- exclusive_issue_ids_cache="$(heartbeat_list_exclusive_issue_ids)"
1440
- exclusive_issue_ids_cache_loaded="yes"
1441
- fi
1442
- }
1443
-
1444
- ensure_exclusive_pr_ids_cache() {
1445
- if [[ "$exclusive_pr_ids_cache_loaded" != "yes" ]]; then
1446
- exclusive_pr_ids_cache="$(heartbeat_list_exclusive_pr_ids)"
1447
- exclusive_pr_ids_cache_loaded="yes"
1448
- fi
1449
- }
1450
-
1451
- ensure_ordered_ready_issue_ids_cache() {
1452
- if [[ "$ordered_ready_issue_ids_cache_loaded" != "yes" ]]; then
1453
- ordered_ready_issue_ids_cache="$(build_ordered_ready_issue_ids_cache)"
1454
- ordered_ready_issue_ids_cache_loaded="yes"
1455
- fi
1456
- }
1457
-
1458
- ensure_blocked_recovery_issue_ids_cache() {
1459
- if [[ "$blocked_recovery_issue_ids_cache_loaded" != "yes" ]]; then
1460
- blocked_recovery_issue_ids_cache="$(heartbeat_list_blocked_recovery_issue_ids)"
1461
- blocked_recovery_issue_ids_cache_loaded="yes"
1462
- fi
1463
- }
1464
-
1465
- sync_open_agent_issues() {
1466
- local issue_id status_out status
1467
- ensure_running_issue_ids_cache
1468
- while IFS= read -r issue_id; do
1469
- [[ -n "$issue_id" ]] || continue
1470
- if tmux has-session -t "${issue_prefix}${issue_id}" 2>/dev/null; then
1471
- continue
1472
- fi
1473
- if pending_issue_launch_active "$issue_id"; then
1474
- if pending_issue_launch_counts_toward_capacity "$issue_id"; then
1475
- heartbeat_mark_issue_running "$issue_id" "$(cached_issue_attr heavy "$issue_id")" >/dev/null || true
1476
- fi
1477
- continue
1478
- fi
1479
- status_out="$(
1480
- "${shared_agent_home}/tools/bin/agent-project-worker-status" \
1481
- --runs-root "$runs_root" \
1482
- --session "${issue_prefix}${issue_id}"
1483
- )"
1484
- status="$(awk -F= '/^STATUS=/{print $2}' <<<"$status_out")"
1485
- if [[ "$status" == "UNKNOWN" ]]; then
1486
- heartbeat_sync_issue_labels "$issue_id" >/dev/null || true
1487
- fi
1488
- done <<<"$running_issue_ids_cache"
1489
- }
1490
-
1491
- sync_open_agent_prs() {
1492
- local pr_number status_out status
1493
- ensure_open_agent_pr_ids_cache
1494
- while IFS= read -r pr_number; do
1495
- [[ -n "$pr_number" ]] || continue
1496
- if tmux has-session -t "${pr_prefix}${pr_number}" 2>/dev/null; then
1497
- continue
1498
- fi
1499
- if pending_pr_launch_active "$pr_number"; then
1500
- heartbeat_mark_pr_running "$pr_number" >/dev/null || true
1501
- continue
1502
- fi
1503
- status_out="$(
1504
- "${shared_agent_home}/tools/bin/agent-project-worker-status" \
1505
- --runs-root "$runs_root" \
1506
- --session "${pr_prefix}${pr_number}"
1507
- )"
1508
- status="$(awk -F= '/^STATUS=/{print $2}' <<<"$status_out")"
1509
- case "$status" in
1510
- UNKNOWN)
1511
- heartbeat_clear_pr_running "$pr_number" >/dev/null || true
1512
- heartbeat_sync_pr_labels "$pr_number" >/dev/null || true
1513
- ;;
1514
- RUNNING)
1515
- ;;
1516
- *)
1517
- heartbeat_clear_pr_running "$pr_number" >/dev/null || true
1518
- heartbeat_sync_pr_labels "$pr_number" >/dev/null || true
1519
- ;;
1520
- esac
1521
- done <<<"$open_agent_pr_ids_cache"
1522
- }
1523
-
1524
- next_pr_candidate_json() {
1525
- local target_lane pr_number risk_json lane
1526
- ensure_open_agent_pr_ids_cache
1527
- for target_lane in double-check-2 double-check-1 automerge merge-repair fix ci-refresh; do
1528
- while IFS= read -r pr_number; do
1529
- [[ -n "$pr_number" ]] || continue
1530
- if tmux has-session -t "${pr_prefix}${pr_number}" 2>/dev/null; then
1531
- continue
1532
- fi
1533
- if pr_launch_reserved "$pr_number"; then
1534
- continue
1535
- fi
1536
- if pending_pr_launch_active "$pr_number"; then
1537
- continue
1538
- fi
1539
- if ! retry_ready pr "$pr_number"; then
1540
- continue
1541
- fi
1542
- risk_json="$(cached_pr_risk_json "$pr_number")"
1543
- lane="$(jq -r '.agentLane' <<<"$risk_json")"
1544
- if [[ "$lane" == "$target_lane" ]]; then
1545
- printf '%s\n' "$risk_json"
1546
- return 0
1547
- fi
1548
- done <<<"$open_agent_pr_ids_cache"
1549
- done
1550
- }
1551
-
1552
- next_priority_review_pr_candidate_json() {
1553
- local target_lane pr_number risk_json lane
1554
- ensure_open_agent_pr_ids_cache
1555
- for target_lane in double-check-2 double-check-1; do
1556
- while IFS= read -r pr_number; do
1557
- [[ -n "$pr_number" ]] || continue
1558
- if tmux has-session -t "${pr_prefix}${pr_number}" 2>/dev/null; then
1559
- continue
1560
- fi
1561
- if pr_launch_reserved "$pr_number"; then
1562
- continue
1563
- fi
1564
- if pending_pr_launch_active "$pr_number"; then
1565
- continue
1566
- fi
1567
- if ! retry_ready pr "$pr_number"; then
1568
- continue
1569
- fi
1570
- risk_json="$(cached_pr_risk_json "$pr_number")"
1571
- lane="$(jq -r '.agentLane' <<<"$risk_json")"
1572
- if [[ "$lane" == "$target_lane" ]]; then
1573
- printf '%s\n' "$risk_json"
1574
- return 0
1575
- fi
1576
- done <<<"$open_agent_pr_ids_cache"
1577
- done
1578
- }
1579
-
1580
- eligible_pr_backlog_count() {
1581
- local pr_number risk_json lane count=0
1582
- ensure_open_agent_pr_ids_cache
1583
- while IFS= read -r pr_number; do
1584
- [[ -n "$pr_number" ]] || continue
1585
- if tmux has-session -t "${pr_prefix}${pr_number}" 2>/dev/null; then
1586
- continue
1587
- fi
1588
- if pr_launch_reserved "$pr_number"; then
1589
- continue
1590
- fi
1591
- if pending_pr_launch_active "$pr_number"; then
1592
- continue
1593
- fi
1594
- if ! retry_ready pr "$pr_number"; then
1595
- continue
1596
- fi
1597
- risk_json="$(cached_pr_risk_json "$pr_number")"
1598
- lane="$(jq -r '.agentLane' <<<"$risk_json")"
1599
- case "$lane" in
1600
- double-check-1|double-check-2|automerge|merge-repair|fix)
1601
- count=$((count + 1))
1602
- ;;
1603
- esac
1604
- done <<<"$open_agent_pr_ids_cache"
1605
- printf '%s\n' "$count"
1606
- }
1607
-
1608
- priority_review_backlog_count() {
1609
- local pr_number risk_json lane count=0
1610
- ensure_open_agent_pr_ids_cache
1611
- while IFS= read -r pr_number; do
1612
- [[ -n "$pr_number" ]] || continue
1613
- if tmux has-session -t "${pr_prefix}${pr_number}" 2>/dev/null; then
1614
- continue
1615
- fi
1616
- if pr_launch_reserved "$pr_number"; then
1617
- continue
1618
- fi
1619
- if pending_pr_launch_active "$pr_number"; then
1620
- continue
1621
- fi
1622
- if ! retry_ready pr "$pr_number"; then
1623
- continue
1624
- fi
1625
- risk_json="$(cached_pr_risk_json "$pr_number")"
1626
- lane="$(jq -r '.agentLane' <<<"$risk_json")"
1627
- case "$lane" in
1628
- double-check-1|double-check-2)
1629
- count=$((count + 1))
1630
- ;;
1631
- esac
1632
- done <<<"$open_agent_pr_ids_cache"
1633
- printf '%s\n' "$count"
1634
- }
1635
-
1636
- next_exclusive_pr_candidate_json() {
1637
- local target_lane pr_number risk_json lane
1638
- ensure_exclusive_pr_ids_cache
1639
- for target_lane in double-check-2 double-check-1 automerge merge-repair fix ci-refresh; do
1640
- while IFS= read -r pr_number; do
1641
- [[ -n "$pr_number" ]] || continue
1642
- if tmux has-session -t "${pr_prefix}${pr_number}" 2>/dev/null; then
1643
- continue
1644
- fi
1645
- if pr_launch_reserved "$pr_number"; then
1646
- continue
1647
- fi
1648
- if pending_pr_launch_active "$pr_number"; then
1649
- continue
1650
- fi
1651
- if ! retry_ready pr "$pr_number"; then
1652
- continue
1653
- fi
1654
- risk_json="$(cached_pr_risk_json "$pr_number")"
1655
- lane="$(jq -r '.agentLane' <<<"$risk_json")"
1656
- # Skip PRs requiring human review; they should not hold exclusive lock
1657
- if [[ "$lane" == "human-review" ]]; then
1658
- continue
1659
- fi
1660
- if [[ "$lane" == "$target_lane" ]]; then
1661
- printf '%s\n' "$risk_json"
1662
- return 0
1663
- fi
1664
- done <<<"$exclusive_pr_ids_cache"
1665
- done
1666
- }
1667
-
1668
- next_exclusive_issue_id() {
1669
- local issue_id
1670
- ensure_exclusive_issue_ids_cache
1671
- while IFS= read -r issue_id; do
1672
- [[ -n "$issue_id" ]] || continue
1673
- if tmux has-session -t "${issue_prefix}${issue_id}" 2>/dev/null; then
1674
- continue
1675
- fi
1676
- if pending_issue_launch_active "$issue_id"; then
1677
- continue
1678
- fi
1679
- if ! retry_ready issue "$issue_id"; then
1680
- continue
1681
- fi
1682
- printf '%s\n' "$issue_id"
1683
- return 0
1684
- done <<<"$exclusive_issue_ids_cache"
1685
- }
1686
-
1687
- count_pr_lane() {
1688
- local target_lane="${1:?target lane required}"
1689
- local pr_number risk_json lane count=0
1690
- ensure_open_agent_pr_ids_cache
1691
- while IFS= read -r pr_number; do
1692
- [[ -n "$pr_number" ]] || continue
1693
- risk_json="$(cached_pr_risk_json "$pr_number")"
1694
- lane="$(jq -r '.agentLane' <<<"$risk_json")"
1695
- if [[ "$lane" == "$target_lane" ]]; then
1696
- count=$((count + 1))
1697
- fi
1698
- done <<<"$open_agent_pr_ids_cache"
1699
- printf '%s\n' "$count"
1700
- }
1701
-
1702
- human_review_pr_ids() {
1703
- local pr_number risk_json lane
1704
- ensure_open_agent_pr_ids_cache
1705
- while IFS= read -r pr_number; do
1706
- [[ -n "$pr_number" ]] || continue
1707
- risk_json="$(cached_pr_risk_json "$pr_number")"
1708
- lane="$(jq -r '.agentLane' <<<"$risk_json")"
1709
- if [[ "$lane" == "human-review" ]]; then
1710
- printf '%s\n' "$pr_number"
1711
- fi
1712
- done <<<"$open_agent_pr_ids_cache"
1713
- }
1714
-
1715
- log_phase "reconcile-completed-workers:start"
1716
- ensure_completed_workers_cache
1717
- while IFS= read -r completed_session; do
1718
- [[ -n "$completed_session" ]] || continue
1719
- case "$completed_session" in
1720
- "${issue_prefix}"*)
1721
- if reconcile_out="$(heartbeat_reconcile_issue "$completed_session" 2>&1)"; then
1722
- record_memory "reconciled issue worker ${completed_session}"
1723
- print_block "RECONCILED_SESSION=${completed_session}" "$reconcile_out"
1724
- else
1725
- record_memory "failed to reconcile issue worker ${completed_session}"
1726
- print_block "RECONCILE_FAILED_SESSION=${completed_session}" "$reconcile_out"
1727
- fi
1728
- ;;
1729
- "${pr_prefix}"*)
1730
- if reconcile_out="$(heartbeat_reconcile_pr "$completed_session" 2>&1)"; then
1731
- record_memory "reconciled PR worker ${completed_session}"
1732
- print_block "RECONCILED_SESSION=${completed_session}" "$reconcile_out"
1733
- else
1734
- completed_pr_number="${completed_session#${pr_prefix}}"
1735
- if [[ -n "$completed_pr_number" ]]; then
1736
- heartbeat_clear_pr_running "$completed_pr_number" >/dev/null || true
1737
- heartbeat_sync_pr_labels "$completed_pr_number" >/dev/null || true
1738
- fi
1739
- record_memory "failed to reconcile PR worker ${completed_session}"
1740
- print_block "RECONCILE_FAILED_SESSION=${completed_session}" "$reconcile_out"
1741
- fi
1742
- ;;
1743
- *)
1744
- echo "unknown completed worker session: ${completed_session}" >&2
1745
- exit 1
1746
- ;;
1747
- esac
1748
- done <<<"$completed_workers_cache"
1749
- log_phase "reconcile-completed-workers:end"
440
+ log_phase "reconcile-completed-workers:start"
441
+ ensure_completed_workers_cache
442
+ while IFS= read -r completed_session; do
443
+ [[ -n "$completed_session" ]] || continue
444
+ case "$completed_session" in
445
+ "${issue_prefix}"*)
446
+ if reconcile_out="$(heartbeat_reconcile_issue "$completed_session" 2>&1)"; then
447
+ record_memory "reconciled issue worker ${completed_session}"
448
+ print_block "RECONCILED_SESSION=${completed_session}" "$reconcile_out"
449
+ else
450
+ record_memory "failed to reconcile issue worker ${completed_session}"
451
+ print_block "RECONCILE_FAILED_SESSION=${completed_session}" "$reconcile_out"
452
+ fi
453
+ ;;
454
+ "${pr_prefix}"*)
455
+ if reconcile_out="$(heartbeat_reconcile_pr "$completed_session" 2>&1)"; then
456
+ record_memory "reconciled PR worker ${completed_session}"
457
+ print_block "RECONCILED_SESSION=${completed_session}" "$reconcile_out"
458
+ else
459
+ completed_pr_number="${completed_session#${pr_prefix}}"
460
+ if [[ -n "$completed_pr_number" ]]; then
461
+ heartbeat_clear_pr_running "$completed_pr_number" >/dev/null || true
462
+ heartbeat_sync_pr_labels "$completed_pr_number" >/dev/null || true
463
+ fi
464
+ record_memory "failed to reconcile PR worker ${completed_session}"
465
+ print_block "RECONCILE_FAILED_SESSION=${completed_session}" "$reconcile_out"
466
+ fi
467
+ ;;
468
+ *)
469
+ echo "unknown completed worker session: ${completed_session}" >&2
470
+ exit 1
471
+ ;;
472
+ esac
473
+ done <<<"$completed_workers_cache"
474
+ log_phase "reconcile-completed-workers:end"
1750
475
 
1751
476
  log_phase "sync-open-agent-issues:start"
1752
477
  sync_open_agent_issues
@@ -1850,76 +575,6 @@ if provider_cooldown_out="$(provider_cooldown_state 2>/dev/null || true)"; then
1850
575
  fi
1851
576
  fi
1852
577
 
1853
- launch_pr_candidate_json() {
1854
- local pr_candidate_json="${1:?pr candidate json required}"
1855
- local pr_number pr_lane launch_out
1856
-
1857
- pr_number="$(jq -r '.number' <<<"$pr_candidate_json")"
1858
- pr_lane="$(jq -r '.agentLane' <<<"$pr_candidate_json")"
1859
- stage_pr_launch "$pr_number"
1860
-
1861
- case "$pr_lane" in
1862
- double-check-1|double-check-2|automerge)
1863
- if ! launch_out="$(heartbeat_start_pr_review_worker "$pr_number" 2>&1)"; then
1864
- heartbeat_clear_pr_running "$pr_number" || true
1865
- clear_launch_in_progress
1866
- record_memory "failed to launch PR review worker for #${pr_number}"
1867
- print_block "LAUNCH_FAILED_PR=${pr_number}" "$launch_out"
1868
- exit 1
1869
- fi
1870
- record_memory "launched PR review worker for #${pr_number}"
1871
- ;;
1872
- merge-repair)
1873
- if ! launch_out="$(heartbeat_start_pr_merge_repair_worker "$pr_number" 2>&1)"; then
1874
- heartbeat_clear_pr_running "$pr_number" || true
1875
- clear_launch_in_progress
1876
- record_memory "failed to launch PR merge-repair worker for #${pr_number}"
1877
- print_block "LAUNCH_FAILED_PR=${pr_number}" "$launch_out"
1878
- exit 1
1879
- fi
1880
- record_memory "launched PR merge-repair worker for #${pr_number}"
1881
- ;;
1882
- ci-refresh)
1883
- if ! launch_out="$(heartbeat_start_pr_ci_refresh "$pr_number" 2>&1)"; then
1884
- heartbeat_clear_pr_running "$pr_number" || true
1885
- clear_launch_in_progress
1886
- record_memory "failed to trigger PR ci-refresh for #${pr_number}"
1887
- print_block "LAUNCH_FAILED_PR=${pr_number}" "$launch_out"
1888
- exit 1
1889
- fi
1890
- heartbeat_clear_pr_running "$pr_number" || true
1891
- record_memory "triggered PR ci-refresh for #${pr_number}"
1892
- ;;
1893
- fix)
1894
- if ! launch_out="$(heartbeat_start_pr_fix_worker "$pr_number" 2>&1)"; then
1895
- heartbeat_clear_pr_running "$pr_number" || true
1896
- clear_launch_in_progress
1897
- record_memory "failed to launch PR fix worker for #${pr_number}"
1898
- print_block "LAUNCH_FAILED_PR=${pr_number}" "$launch_out"
1899
- exit 1
1900
- fi
1901
- record_memory "launched PR fix worker for #${pr_number}"
1902
- ;;
1903
- *)
1904
- launch_out="Unsupported PR lane: ${pr_lane}"
1905
- heartbeat_clear_pr_running "$pr_number" || true
1906
- clear_launch_in_progress
1907
- print_block "LAUNCH_FAILED_PR=${pr_number}" "$launch_out"
1908
- exit 1
1909
- ;;
1910
- esac
1911
-
1912
- clear_launch_in_progress
1913
- print_block "LAUNCHED_PR=${pr_number}" "$(printf 'LANE=%s\n%s' "$pr_lane" "$launch_out")"
1914
- if [[ "$pr_lane" != "ci-refresh" ]]; then
1915
- running_total_count=$((running_total_count + 1))
1916
- running_pr_count=$((running_pr_count + 1))
1917
- fi
1918
- launched_pr_count=$((launched_pr_count + 1))
1919
- if (( launch_budget_remaining > 0 )); then
1920
- launch_budget_remaining=$((launch_budget_remaining - 1))
1921
- fi
1922
- }
1923
578
 
1924
579
  if [[ "$exclusive_lock_mode" == "pending" && "$exclusive_lock_kind" == "pr" ]]; then
1925
580
  pr_number="$exclusive_lock_item"