agent-control-plane 0.2.0 → 0.4.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +69 -19
- package/assets/workflow-catalog.json +1 -1
- package/bin/pr-risk.sh +22 -7
- package/bin/sync-pr-labels.sh +1 -1
- package/hooks/heartbeat-hooks.sh +125 -12
- package/hooks/issue-reconcile-hooks.sh +1 -1
- package/hooks/pr-reconcile-hooks.sh +1 -1
- package/npm/bin/agent-control-plane.js +296 -61
- package/package.json +11 -7
- package/tools/bin/agent-github-update-labels +36 -2
- package/tools/bin/agent-project-catch-up-merged-prs +4 -2
- package/tools/bin/agent-project-cleanup-session +49 -5
- package/tools/bin/agent-project-heartbeat-loop +119 -1471
- package/tools/bin/agent-project-publish-issue-pr +6 -3
- package/tools/bin/agent-project-reconcile-issue-session +78 -106
- package/tools/bin/agent-project-reconcile-pr-session +166 -143
- package/tools/bin/agent-project-retry-state +18 -7
- package/tools/bin/agent-project-run-claude-session +10 -0
- package/tools/bin/agent-project-run-codex-resilient +99 -14
- package/tools/bin/agent-project-run-codex-session +16 -5
- package/tools/bin/agent-project-run-kilo-session +10 -0
- package/tools/bin/agent-project-run-openclaw-session +10 -0
- package/tools/bin/agent-project-run-opencode-session +10 -0
- package/tools/bin/agent-project-sync-source-repo-main +163 -0
- package/tools/bin/agent-project-worker-status +10 -7
- package/tools/bin/cleanup-worktree.sh +6 -1
- package/tools/bin/flow-config-lib.sh +1257 -34
- package/tools/bin/flow-resident-worker-lib.sh +119 -1
- package/tools/bin/flow-shell-lib.sh +56 -0
- package/tools/bin/github-core-rate-limit-state.sh +77 -0
- package/tools/bin/github-write-outbox.sh +470 -0
- package/tools/bin/heartbeat-loop-cache-lib.sh +164 -0
- package/tools/bin/heartbeat-loop-counting-lib.sh +306 -0
- package/tools/bin/heartbeat-loop-pr-strategy-lib.sh +199 -0
- package/tools/bin/heartbeat-loop-scheduling-lib.sh +506 -0
- package/tools/bin/heartbeat-loop-worker-lib.sh +319 -0
- package/tools/bin/heartbeat-recovery-preflight.sh +12 -1
- package/tools/bin/heartbeat-safe-auto.sh +56 -3
- package/tools/bin/install-project-launchd.sh +17 -2
- package/tools/bin/project-init.sh +21 -1
- package/tools/bin/project-launchd-bootstrap.sh +16 -9
- package/tools/bin/project-runtimectl.sh +46 -2
- package/tools/bin/reconcile-bootstrap-lib.sh +113 -0
- package/tools/bin/resident-issue-controller-lib.sh +448 -0
- package/tools/bin/scaffold-profile.sh +61 -3
- package/tools/bin/start-pr-fix-worker.sh +47 -10
- package/tools/bin/start-resident-issue-loop.sh +28 -439
- package/tools/dashboard/app.js +37 -1
- package/tools/dashboard/dashboard_snapshot.py +65 -26
- package/tools/templates/pr-fix-template.md +3 -1
- package/tools/templates/pr-merge-repair-template.md +2 -1
- package/SKILL.md +0 -149
- package/references/architecture.md +0 -217
- package/references/commands.md +0 -128
- package/references/control-plane-map.md +0 -124
- package/references/docs-map.md +0 -73
- package/references/release-checklist.md +0 -65
- package/references/repo-map.md +0 -36
- package/tools/bin/split-retained-slice.sh +0 -124
|
@@ -158,269 +158,6 @@ log_phase() {
|
|
|
158
158
|
printf 'HEARTBEAT_LOOP_PHASE=%s\n' "${1:?phase required}"
|
|
159
159
|
}
|
|
160
160
|
|
|
161
|
-
all_tmux_sessions() {
|
|
162
|
-
ensure_tmux_sessions_cache
|
|
163
|
-
printf '%s\n' "$tmux_sessions_cache"
|
|
164
|
-
}
|
|
165
|
-
|
|
166
|
-
session_matches_prefix() {
|
|
167
|
-
local session="${1:?session required}"
|
|
168
|
-
[[ "$session" == "${issue_prefix}"* || "$session" == "${pr_prefix}"* ]]
|
|
169
|
-
}
|
|
170
|
-
|
|
171
|
-
session_runner_state() {
|
|
172
|
-
local session="${1:?session required}"
|
|
173
|
-
local runner_state_file="${runs_root}/${session}/runner.env"
|
|
174
|
-
if [[ ! -f "$runner_state_file" ]]; then
|
|
175
|
-
return 1
|
|
176
|
-
fi
|
|
177
|
-
awk -F= '/^RUNNER_STATE=/{print $2; exit}' "$runner_state_file"
|
|
178
|
-
}
|
|
179
|
-
|
|
180
|
-
session_is_auth_waiting() {
|
|
181
|
-
local session="${1:?session required}"
|
|
182
|
-
local runner_state=""
|
|
183
|
-
runner_state="$(session_runner_state "$session" || true)"
|
|
184
|
-
[[ "$runner_state" == "waiting-auth-refresh" || "$runner_state" == "switching-account" ]]
|
|
185
|
-
}
|
|
186
|
-
|
|
187
|
-
all_running_workers() {
|
|
188
|
-
ensure_all_running_workers_cache
|
|
189
|
-
printf '%s\n' "$all_running_workers_cache"
|
|
190
|
-
}
|
|
191
|
-
|
|
192
|
-
running_issue_workers() {
|
|
193
|
-
ensure_running_issue_workers_cache
|
|
194
|
-
printf '%s\n' "$running_issue_workers_cache"
|
|
195
|
-
}
|
|
196
|
-
|
|
197
|
-
pending_launch_pid() {
|
|
198
|
-
local kind="${1:?kind required}"
|
|
199
|
-
local item_id="${2:?item id required}"
|
|
200
|
-
local pending_file pid
|
|
201
|
-
|
|
202
|
-
pending_file="${pending_launch_dir}/${kind}-${item_id}.pid"
|
|
203
|
-
if [[ ! -f "$pending_file" ]]; then
|
|
204
|
-
return 1
|
|
205
|
-
fi
|
|
206
|
-
|
|
207
|
-
pid="$(tr -d '[:space:]' <"$pending_file" 2>/dev/null || true)"
|
|
208
|
-
if [[ -z "$pid" ]]; then
|
|
209
|
-
rm -f "$pending_file"
|
|
210
|
-
return 1
|
|
211
|
-
fi
|
|
212
|
-
|
|
213
|
-
if kill -0 "$pid" 2>/dev/null; then
|
|
214
|
-
printf '%s\n' "$pid"
|
|
215
|
-
return 0
|
|
216
|
-
fi
|
|
217
|
-
|
|
218
|
-
rm -f "$pending_file"
|
|
219
|
-
return 1
|
|
220
|
-
}
|
|
221
|
-
|
|
222
|
-
pending_issue_launch_active() {
|
|
223
|
-
local issue_id="${1:?issue id required}"
|
|
224
|
-
if tmux has-session -t "${issue_prefix}${issue_id}" 2>/dev/null; then
|
|
225
|
-
rm -f "${pending_launch_dir}/issue-${issue_id}.pid" 2>/dev/null || true
|
|
226
|
-
return 1
|
|
227
|
-
fi
|
|
228
|
-
pending_launch_pid issue "$issue_id" >/dev/null
|
|
229
|
-
}
|
|
230
|
-
|
|
231
|
-
resident_issue_controller_file() {
|
|
232
|
-
local issue_id="${1:?issue id required}"
|
|
233
|
-
printf '%s/resident-workers/issues/%s/controller.env\n' "${state_root}" "${issue_id}"
|
|
234
|
-
}
|
|
235
|
-
|
|
236
|
-
resident_issue_controller_state() {
|
|
237
|
-
local issue_id="${1:?issue id required}"
|
|
238
|
-
local controller_file state=""
|
|
239
|
-
|
|
240
|
-
controller_file="$(resident_issue_controller_file "$issue_id")"
|
|
241
|
-
[[ -f "${controller_file}" ]] || return 1
|
|
242
|
-
|
|
243
|
-
state="$(awk -F= '/^CONTROLLER_STATE=/{print $2; exit}' "${controller_file}" 2>/dev/null | tr -d '"' || true)"
|
|
244
|
-
[[ -n "${state}" ]] || return 1
|
|
245
|
-
printf '%s\n' "${state}"
|
|
246
|
-
}
|
|
247
|
-
|
|
248
|
-
pending_issue_launch_counts_toward_capacity() {
|
|
249
|
-
local issue_id="${1:?issue id required}"
|
|
250
|
-
local controller_state=""
|
|
251
|
-
|
|
252
|
-
if ! pending_issue_launch_active "${issue_id}"; then
|
|
253
|
-
return 1
|
|
254
|
-
fi
|
|
255
|
-
|
|
256
|
-
controller_state="$(resident_issue_controller_state "${issue_id}" || true)"
|
|
257
|
-
if [[ -n "${controller_state}" ]]; then
|
|
258
|
-
case "${controller_state}" in
|
|
259
|
-
idle|sleeping|waiting-due|waiting-open-pr|waiting-provider)
|
|
260
|
-
return 1
|
|
261
|
-
;;
|
|
262
|
-
esac
|
|
263
|
-
fi
|
|
264
|
-
|
|
265
|
-
return 0
|
|
266
|
-
}
|
|
267
|
-
|
|
268
|
-
pending_pr_launch_active() {
|
|
269
|
-
local pr_id="${1:?pr id required}"
|
|
270
|
-
if tmux has-session -t "${pr_prefix}${pr_id}" 2>/dev/null; then
|
|
271
|
-
rm -f "${pending_launch_dir}/pr-${pr_id}.pid" 2>/dev/null || true
|
|
272
|
-
return 1
|
|
273
|
-
fi
|
|
274
|
-
pending_launch_pid pr "$pr_id" >/dev/null
|
|
275
|
-
}
|
|
276
|
-
|
|
277
|
-
running_pr_workers() {
|
|
278
|
-
ensure_running_pr_workers_cache
|
|
279
|
-
printf '%s\n' "$running_pr_workers_cache"
|
|
280
|
-
}
|
|
281
|
-
|
|
282
|
-
ensure_tmux_sessions_cache() {
|
|
283
|
-
if [[ "$tmux_sessions_cache_loaded" != "yes" ]]; then
|
|
284
|
-
tmux_sessions_cache="$(tmux list-sessions -F '#S' 2>/dev/null || true)"
|
|
285
|
-
tmux_sessions_cache_loaded="yes"
|
|
286
|
-
fi
|
|
287
|
-
}
|
|
288
|
-
|
|
289
|
-
ensure_all_running_workers_cache() {
|
|
290
|
-
local session
|
|
291
|
-
if [[ "$all_running_workers_cache_loaded" == "yes" ]]; then
|
|
292
|
-
return 0
|
|
293
|
-
fi
|
|
294
|
-
ensure_tmux_sessions_cache
|
|
295
|
-
all_running_workers_cache=""
|
|
296
|
-
while IFS= read -r session; do
|
|
297
|
-
[[ -n "$session" ]] || continue
|
|
298
|
-
if session_matches_prefix "$session"; then
|
|
299
|
-
all_running_workers_cache+="${session}"$'\n'
|
|
300
|
-
fi
|
|
301
|
-
done <<<"$tmux_sessions_cache"
|
|
302
|
-
all_running_workers_cache="${all_running_workers_cache%$'\n'}"
|
|
303
|
-
all_running_workers_cache_loaded="yes"
|
|
304
|
-
}
|
|
305
|
-
|
|
306
|
-
auth_wait_workers() {
|
|
307
|
-
ensure_auth_wait_workers_cache
|
|
308
|
-
printf '%s\n' "$auth_wait_workers_cache"
|
|
309
|
-
}
|
|
310
|
-
|
|
311
|
-
ensure_auth_wait_workers_cache() {
|
|
312
|
-
local session
|
|
313
|
-
if [[ "$auth_wait_workers_cache_loaded" == "yes" ]]; then
|
|
314
|
-
return 0
|
|
315
|
-
fi
|
|
316
|
-
ensure_tmux_sessions_cache
|
|
317
|
-
auth_wait_workers_cache=""
|
|
318
|
-
while IFS= read -r session; do
|
|
319
|
-
[[ -n "$session" ]] || continue
|
|
320
|
-
session_matches_prefix "$session" || continue
|
|
321
|
-
if session_is_auth_waiting "$session"; then
|
|
322
|
-
auth_wait_workers_cache+="${session}"$'\n'
|
|
323
|
-
fi
|
|
324
|
-
done <<<"$tmux_sessions_cache"
|
|
325
|
-
auth_wait_workers_cache="${auth_wait_workers_cache%$'\n'}"
|
|
326
|
-
auth_wait_workers_cache_loaded="yes"
|
|
327
|
-
}
|
|
328
|
-
|
|
329
|
-
ensure_running_issue_workers_cache() {
|
|
330
|
-
local session
|
|
331
|
-
if [[ "$running_issue_workers_cache_loaded" == "yes" ]]; then
|
|
332
|
-
return 0
|
|
333
|
-
fi
|
|
334
|
-
ensure_tmux_sessions_cache
|
|
335
|
-
running_issue_workers_cache=""
|
|
336
|
-
while IFS= read -r session; do
|
|
337
|
-
[[ -n "$session" ]] || continue
|
|
338
|
-
if [[ "$session" == "${issue_prefix}"* ]]; then
|
|
339
|
-
if session_is_auth_waiting "$session"; then
|
|
340
|
-
continue
|
|
341
|
-
fi
|
|
342
|
-
running_issue_workers_cache+="${session}"$'\n'
|
|
343
|
-
fi
|
|
344
|
-
done <<<"$tmux_sessions_cache"
|
|
345
|
-
running_issue_workers_cache="${running_issue_workers_cache%$'\n'}"
|
|
346
|
-
running_issue_workers_cache_loaded="yes"
|
|
347
|
-
}
|
|
348
|
-
|
|
349
|
-
ensure_running_pr_workers_cache() {
|
|
350
|
-
local session
|
|
351
|
-
if [[ "$running_pr_workers_cache_loaded" == "yes" ]]; then
|
|
352
|
-
return 0
|
|
353
|
-
fi
|
|
354
|
-
ensure_tmux_sessions_cache
|
|
355
|
-
running_pr_workers_cache=""
|
|
356
|
-
while IFS= read -r session; do
|
|
357
|
-
[[ -n "$session" ]] || continue
|
|
358
|
-
if [[ "$session" == "${pr_prefix}"* ]]; then
|
|
359
|
-
if session_is_auth_waiting "$session"; then
|
|
360
|
-
continue
|
|
361
|
-
fi
|
|
362
|
-
running_pr_workers_cache+="${session}"$'\n'
|
|
363
|
-
fi
|
|
364
|
-
done <<<"$tmux_sessions_cache"
|
|
365
|
-
running_pr_workers_cache="${running_pr_workers_cache%$'\n'}"
|
|
366
|
-
running_pr_workers_cache_loaded="yes"
|
|
367
|
-
}
|
|
368
|
-
|
|
369
|
-
worker_count() {
|
|
370
|
-
local workers="${1:-}"
|
|
371
|
-
if [[ -z "$workers" ]]; then
|
|
372
|
-
printf '0\n'
|
|
373
|
-
return
|
|
374
|
-
fi
|
|
375
|
-
printf '%s\n' "$workers" | sed '/^$/d' | wc -l | tr -d ' '
|
|
376
|
-
}
|
|
377
|
-
|
|
378
|
-
retry_ready() {
|
|
379
|
-
local kind="${1:?kind required}"
|
|
380
|
-
local item_id="${2:?item id required}"
|
|
381
|
-
local retry_out ready
|
|
382
|
-
|
|
383
|
-
retry_out="$(
|
|
384
|
-
"${shared_agent_home}/tools/bin/agent-project-retry-state" \
|
|
385
|
-
--state-root "$state_root" \
|
|
386
|
-
--kind "$kind" \
|
|
387
|
-
--item-id "$item_id" \
|
|
388
|
-
--action get
|
|
389
|
-
)"
|
|
390
|
-
ready="$(awk -F= '/^READY=/{print $2}' <<<"$retry_out")"
|
|
391
|
-
[[ "$ready" == "yes" ]]
|
|
392
|
-
}
|
|
393
|
-
|
|
394
|
-
provider_cooldown_state() {
|
|
395
|
-
"${shared_agent_home}/tools/bin/provider-cooldown-state.sh" get
|
|
396
|
-
}
|
|
397
|
-
|
|
398
|
-
issue_id_from_session() {
|
|
399
|
-
local session="${1:?session required}"
|
|
400
|
-
local issue_id=""
|
|
401
|
-
if [[ "$session" == "${issue_prefix}"* ]]; then
|
|
402
|
-
issue_id="${session#${issue_prefix}}"
|
|
403
|
-
fi
|
|
404
|
-
if [[ "$issue_id" =~ ^[0-9]+$ ]]; then
|
|
405
|
-
printf '%s\n' "$issue_id"
|
|
406
|
-
return 0
|
|
407
|
-
fi
|
|
408
|
-
return 1
|
|
409
|
-
}
|
|
410
|
-
|
|
411
|
-
pr_id_from_session() {
|
|
412
|
-
local session="${1:?session required}"
|
|
413
|
-
local pr_id=""
|
|
414
|
-
if [[ "$session" == "${pr_prefix}"* ]]; then
|
|
415
|
-
pr_id="${session#${pr_prefix}}"
|
|
416
|
-
fi
|
|
417
|
-
if [[ "$pr_id" =~ ^[0-9]+$ ]]; then
|
|
418
|
-
printf '%s\n' "$pr_id"
|
|
419
|
-
return 0
|
|
420
|
-
fi
|
|
421
|
-
return 1
|
|
422
|
-
}
|
|
423
|
-
|
|
424
161
|
if [[ ! -f "$hook_file" ]]; then
|
|
425
162
|
echo "missing hook file: $hook_file" >&2
|
|
426
163
|
exit 1
|
|
@@ -493,6 +230,23 @@ if ! declare -F heartbeat_sync_issue_labels >/dev/null 2>&1; then
|
|
|
493
230
|
heartbeat_sync_issue_labels() { :; }
|
|
494
231
|
fi
|
|
495
232
|
|
|
233
|
+
|
|
234
|
+
# --- Source modular libraries ---
|
|
235
|
+
|
|
236
|
+
heartbeat_lib_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
237
|
+
# shellcheck source=/dev/null
|
|
238
|
+
source "${heartbeat_lib_dir}/heartbeat-loop-worker-lib.sh"
|
|
239
|
+
# shellcheck source=/dev/null
|
|
240
|
+
source "${heartbeat_lib_dir}/heartbeat-loop-cache-lib.sh"
|
|
241
|
+
# shellcheck source=/dev/null
|
|
242
|
+
source "${heartbeat_lib_dir}/heartbeat-loop-counting-lib.sh"
|
|
243
|
+
# shellcheck source=/dev/null
|
|
244
|
+
source "${heartbeat_lib_dir}/heartbeat-loop-scheduling-lib.sh"
|
|
245
|
+
# shellcheck source=/dev/null
|
|
246
|
+
source "${heartbeat_lib_dir}/heartbeat-loop-pr-strategy-lib.sh"
|
|
247
|
+
|
|
248
|
+
# --- Launch staging & rollback ---
|
|
249
|
+
|
|
496
250
|
launch_in_progress_kind=""
|
|
497
251
|
launch_in_progress_id=""
|
|
498
252
|
launch_in_progress_issue_is_heavy="no"
|
|
@@ -606,1161 +360,125 @@ rollback_launch_in_progress() {
|
|
|
606
360
|
|
|
607
361
|
trap rollback_launch_in_progress EXIT INT TERM
|
|
608
362
|
|
|
609
|
-
|
|
610
|
-
local raw_prefix="${issue_prefix:-${pr_prefix:-agent-control-plane}}"
|
|
611
|
-
local sanitized=""
|
|
612
|
-
|
|
613
|
-
sanitized="$(printf '%s' "${raw_prefix}" | tr '/[:space:]' '-' | tr -cd '[:alnum:]_.-')"
|
|
614
|
-
if [[ -z "${sanitized}" ]]; then
|
|
615
|
-
sanitized="agent-control-plane"
|
|
616
|
-
fi
|
|
617
|
-
|
|
618
|
-
printf '%s\n' "${sanitized}"
|
|
619
|
-
}
|
|
620
|
-
|
|
621
|
-
ensure_issue_attr_cache_dir() {
|
|
622
|
-
if [[ -z "${issue_attr_cache_dir:-}" || ! -d "${issue_attr_cache_dir:-}" ]]; then
|
|
623
|
-
issue_attr_cache_dir="$(mktemp -d "${TMPDIR:-/tmp}/$(cache_prefix)-issue-attrs.XXXXXX")"
|
|
624
|
-
fi
|
|
625
|
-
}
|
|
626
|
-
|
|
627
|
-
ensure_pr_attr_cache_dir() {
|
|
628
|
-
if [[ -z "${pr_attr_cache_dir:-}" || ! -d "${pr_attr_cache_dir:-}" ]]; then
|
|
629
|
-
pr_attr_cache_dir="$(mktemp -d "${TMPDIR:-/tmp}/$(cache_prefix)-pr-attrs.XXXXXX")"
|
|
630
|
-
fi
|
|
631
|
-
}
|
|
632
|
-
|
|
633
|
-
ensure_pr_risk_cache_dir() {
|
|
634
|
-
if [[ -z "${pr_risk_cache_dir:-}" || ! -d "${pr_risk_cache_dir:-}" ]]; then
|
|
635
|
-
pr_risk_cache_dir="$(mktemp -d "${TMPDIR:-/tmp}/$(cache_prefix)-pr-risk.XXXXXX")"
|
|
636
|
-
fi
|
|
637
|
-
}
|
|
638
|
-
|
|
639
|
-
pr_risk_runtime_cache_fresh() {
|
|
640
|
-
local cache_file="${1:?cache file required}"
|
|
641
|
-
local modified_at now age
|
|
642
|
-
[[ -f "$cache_file" ]] || return 1
|
|
643
|
-
modified_at="$(stat -f '%m' "$cache_file" 2>/dev/null || true)"
|
|
644
|
-
[[ "$modified_at" =~ ^[0-9]+$ ]] || return 1
|
|
645
|
-
now="$(date +%s)"
|
|
646
|
-
age=$((now - modified_at))
|
|
647
|
-
(( age >= 0 && age <= pr_risk_runtime_cache_ttl_seconds ))
|
|
648
|
-
}
|
|
363
|
+
# --- PR launch dispatcher ---
|
|
649
364
|
|
|
650
|
-
|
|
651
|
-
local
|
|
652
|
-
local
|
|
653
|
-
local cache_file attr_value
|
|
365
|
+
launch_pr_candidate_json() {
|
|
366
|
+
local pr_candidate_json="${1:?pr candidate json required}"
|
|
367
|
+
local pr_number pr_lane launch_out
|
|
654
368
|
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
cat "${cache_file}"
|
|
659
|
-
return 0
|
|
660
|
-
fi
|
|
369
|
+
pr_number="$(jq -r '.number' <<<"$pr_candidate_json")"
|
|
370
|
+
pr_lane="$(jq -r '.agentLane' <<<"$pr_candidate_json")"
|
|
371
|
+
stage_pr_launch "$pr_number"
|
|
661
372
|
|
|
662
|
-
case "$
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
373
|
+
case "$pr_lane" in
|
|
374
|
+
double-check-1|double-check-2|automerge)
|
|
375
|
+
if ! launch_out="$(heartbeat_start_pr_review_worker "$pr_number" 2>&1)"; then
|
|
376
|
+
heartbeat_clear_pr_running "$pr_number" || true
|
|
377
|
+
clear_launch_in_progress
|
|
378
|
+
record_memory "failed to launch PR review worker for #${pr_number}"
|
|
379
|
+
print_block "LAUNCH_FAILED_PR=${pr_number}" "$launch_out"
|
|
380
|
+
exit 1
|
|
381
|
+
fi
|
|
382
|
+
record_memory "launched PR review worker for #${pr_number}"
|
|
668
383
|
;;
|
|
669
|
-
|
|
670
|
-
|
|
384
|
+
merge-repair)
|
|
385
|
+
if ! launch_out="$(heartbeat_start_pr_merge_repair_worker "$pr_number" 2>&1)"; then
|
|
386
|
+
heartbeat_clear_pr_running "$pr_number" || true
|
|
387
|
+
clear_launch_in_progress
|
|
388
|
+
record_memory "failed to launch PR merge-repair worker for #${pr_number}"
|
|
389
|
+
print_block "LAUNCH_FAILED_PR=${pr_number}" "$launch_out"
|
|
390
|
+
exit 1
|
|
391
|
+
fi
|
|
392
|
+
record_memory "launched PR merge-repair worker for #${pr_number}"
|
|
671
393
|
;;
|
|
672
|
-
|
|
673
|
-
|
|
394
|
+
ci-refresh)
|
|
395
|
+
if ! launch_out="$(heartbeat_start_pr_ci_refresh "$pr_number" 2>&1)"; then
|
|
396
|
+
heartbeat_clear_pr_running "$pr_number" || true
|
|
397
|
+
clear_launch_in_progress
|
|
398
|
+
record_memory "failed to trigger PR ci-refresh for #${pr_number}"
|
|
399
|
+
print_block "LAUNCH_FAILED_PR=${pr_number}" "$launch_out"
|
|
400
|
+
exit 1
|
|
401
|
+
fi
|
|
402
|
+
heartbeat_clear_pr_running "$pr_number" || true
|
|
403
|
+
record_memory "triggered PR ci-refresh for #${pr_number}"
|
|
674
404
|
;;
|
|
675
|
-
|
|
676
|
-
|
|
405
|
+
fix)
|
|
406
|
+
if ! launch_out="$(heartbeat_start_pr_fix_worker "$pr_number" 2>&1)"; then
|
|
407
|
+
heartbeat_clear_pr_running "$pr_number" || true
|
|
408
|
+
clear_launch_in_progress
|
|
409
|
+
record_memory "failed to launch PR fix worker for #${pr_number}"
|
|
410
|
+
print_block "LAUNCH_FAILED_PR=${pr_number}" "$launch_out"
|
|
411
|
+
exit 1
|
|
412
|
+
fi
|
|
413
|
+
record_memory "launched PR fix worker for #${pr_number}"
|
|
677
414
|
;;
|
|
678
415
|
*)
|
|
679
|
-
|
|
680
|
-
|
|
416
|
+
launch_out="Unsupported PR lane: ${pr_lane}"
|
|
417
|
+
heartbeat_clear_pr_running "$pr_number" || true
|
|
418
|
+
clear_launch_in_progress
|
|
419
|
+
print_block "LAUNCH_FAILED_PR=${pr_number}" "$launch_out"
|
|
420
|
+
exit 1
|
|
681
421
|
;;
|
|
682
422
|
esac
|
|
683
423
|
|
|
684
|
-
|
|
685
|
-
printf '
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
local pr_number="${1:?pr number required}"
|
|
690
|
-
local cache_file attr_value
|
|
691
|
-
|
|
692
|
-
ensure_pr_attr_cache_dir
|
|
693
|
-
cache_file="${pr_attr_cache_dir}/${pr_number}.exclusive"
|
|
694
|
-
if [[ -f "${cache_file}" ]]; then
|
|
695
|
-
cat "${cache_file}"
|
|
696
|
-
return 0
|
|
697
|
-
fi
|
|
698
|
-
|
|
699
|
-
attr_value="$(heartbeat_pr_is_exclusive "${pr_number}")"
|
|
700
|
-
printf '%s\n' "${attr_value}" >"${cache_file}"
|
|
701
|
-
printf '%s\n' "${attr_value}"
|
|
702
|
-
}
|
|
703
|
-
|
|
704
|
-
cached_pr_risk_json() {
|
|
705
|
-
local pr_number="${1:?pr number required}"
|
|
706
|
-
local cache_file runtime_cache_file risk_json
|
|
707
|
-
|
|
708
|
-
ensure_pr_risk_cache_dir
|
|
709
|
-
cache_file="${pr_risk_cache_dir}/${pr_number}.json"
|
|
710
|
-
runtime_cache_file="${pr_risk_runtime_cache_dir}/${pr_number}.json"
|
|
711
|
-
if [[ -f "${cache_file}" ]]; then
|
|
712
|
-
cat "${cache_file}"
|
|
713
|
-
return 0
|
|
424
|
+
clear_launch_in_progress
|
|
425
|
+
print_block "LAUNCHED_PR=${pr_number}" "$(printf 'LANE=%s\n%s' "$pr_lane" "$launch_out")"
|
|
426
|
+
if [[ "$pr_lane" != "ci-refresh" ]]; then
|
|
427
|
+
running_total_count=$((running_total_count + 1))
|
|
428
|
+
running_pr_count=$((running_pr_count + 1))
|
|
714
429
|
fi
|
|
715
|
-
|
|
716
|
-
if
|
|
717
|
-
|
|
718
|
-
cat "${cache_file}"
|
|
719
|
-
return 0
|
|
430
|
+
launched_pr_count=$((launched_pr_count + 1))
|
|
431
|
+
if (( launch_budget_remaining > 0 )); then
|
|
432
|
+
launch_budget_remaining=$((launch_budget_remaining - 1))
|
|
720
433
|
fi
|
|
721
|
-
|
|
722
|
-
risk_json="$(heartbeat_pr_risk_json "${pr_number}")"
|
|
723
|
-
printf '%s\n' "${risk_json}" >"${cache_file}"
|
|
724
|
-
printf '%s\n' "${risk_json}" >"${runtime_cache_file}"
|
|
725
|
-
printf '%s\n' "${risk_json}"
|
|
726
434
|
}
|
|
727
435
|
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
while IFS= read -r session; do
|
|
732
|
-
[[ -n "$session" ]] || continue
|
|
733
|
-
issue_id="$(issue_id_from_session "$session" || true)"
|
|
734
|
-
[[ -n "$issue_id" ]] || continue
|
|
735
|
-
is_heavy="$(cached_issue_attr heavy "$issue_id")"
|
|
736
|
-
if [[ "$is_heavy" == "yes" ]]; then
|
|
737
|
-
count=$((count + 1))
|
|
738
|
-
fi
|
|
739
|
-
done <<<"$running_issue_workers_cache"
|
|
740
|
-
printf '%s\n' "$count"
|
|
741
|
-
}
|
|
436
|
+
# ============================================================================
|
|
437
|
+
# MAIN EXECUTION
|
|
438
|
+
# ============================================================================
|
|
742
439
|
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
440
|
+
log_phase "reconcile-completed-workers:start"
|
|
441
|
+
ensure_completed_workers_cache
|
|
442
|
+
while IFS= read -r completed_session; do
|
|
443
|
+
[[ -n "$completed_session" ]] || continue
|
|
444
|
+
case "$completed_session" in
|
|
445
|
+
"${issue_prefix}"*)
|
|
446
|
+
if reconcile_out="$(heartbeat_reconcile_issue "$completed_session" 2>&1)"; then
|
|
447
|
+
record_memory "reconciled issue worker ${completed_session}"
|
|
448
|
+
print_block "RECONCILED_SESSION=${completed_session}" "$reconcile_out"
|
|
449
|
+
else
|
|
450
|
+
record_memory "failed to reconcile issue worker ${completed_session}"
|
|
451
|
+
print_block "RECONCILE_FAILED_SESSION=${completed_session}" "$reconcile_out"
|
|
452
|
+
fi
|
|
453
|
+
;;
|
|
454
|
+
"${pr_prefix}"*)
|
|
455
|
+
if reconcile_out="$(heartbeat_reconcile_pr "$completed_session" 2>&1)"; then
|
|
456
|
+
record_memory "reconciled PR worker ${completed_session}"
|
|
457
|
+
print_block "RECONCILED_SESSION=${completed_session}" "$reconcile_out"
|
|
458
|
+
else
|
|
459
|
+
completed_pr_number="${completed_session#${pr_prefix}}"
|
|
460
|
+
if [[ -n "$completed_pr_number" ]]; then
|
|
461
|
+
heartbeat_clear_pr_running "$completed_pr_number" >/dev/null || true
|
|
462
|
+
heartbeat_sync_pr_labels "$completed_pr_number" >/dev/null || true
|
|
463
|
+
fi
|
|
464
|
+
record_memory "failed to reconcile PR worker ${completed_session}"
|
|
465
|
+
print_block "RECONCILE_FAILED_SESSION=${completed_session}" "$reconcile_out"
|
|
466
|
+
fi
|
|
467
|
+
;;
|
|
468
|
+
*)
|
|
469
|
+
echo "unknown completed worker session: ${completed_session}" >&2
|
|
470
|
+
exit 1
|
|
471
|
+
;;
|
|
472
|
+
esac
|
|
473
|
+
done <<<"$completed_workers_cache"
|
|
474
|
+
log_phase "reconcile-completed-workers:end"
|
|
759
475
|
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
[[ -n "$pr_id" ]] || continue
|
|
767
|
-
if tmux has-session -t "${pr_prefix}${pr_id}" 2>/dev/null; then
|
|
768
|
-
continue
|
|
769
|
-
fi
|
|
770
|
-
if pending_pr_launch_active "$pr_id"; then
|
|
771
|
-
count=$((count + 1))
|
|
772
|
-
fi
|
|
773
|
-
done
|
|
774
|
-
printf '%s\n' "$count"
|
|
775
|
-
}
|
|
776
|
-
|
|
777
|
-
pending_heavy_issue_launch_count() {
|
|
778
|
-
local pending_file issue_id count=0
|
|
779
|
-
for pending_file in "${pending_launch_dir}"/issue-*.pid; do
|
|
780
|
-
[[ -f "$pending_file" ]] || continue
|
|
781
|
-
issue_id="${pending_file##*/issue-}"
|
|
782
|
-
issue_id="${issue_id%.pid}"
|
|
783
|
-
[[ -n "$issue_id" ]] || continue
|
|
784
|
-
if tmux has-session -t "${issue_prefix}${issue_id}" 2>/dev/null; then
|
|
785
|
-
continue
|
|
786
|
-
fi
|
|
787
|
-
if pending_issue_launch_counts_toward_capacity "$issue_id" && [[ "$(cached_issue_attr heavy "$issue_id")" == "yes" ]]; then
|
|
788
|
-
count=$((count + 1))
|
|
789
|
-
fi
|
|
790
|
-
done
|
|
791
|
-
printf '%s\n' "$count"
|
|
792
|
-
}
|
|
793
|
-
|
|
794
|
-
pending_scheduled_issue_launch_count() {
|
|
795
|
-
local pending_file issue_id count=0
|
|
796
|
-
for pending_file in "${pending_launch_dir}"/issue-*.pid; do
|
|
797
|
-
[[ -f "$pending_file" ]] || continue
|
|
798
|
-
issue_id="${pending_file##*/issue-}"
|
|
799
|
-
issue_id="${issue_id%.pid}"
|
|
800
|
-
[[ -n "$issue_id" ]] || continue
|
|
801
|
-
if tmux has-session -t "${issue_prefix}${issue_id}" 2>/dev/null; then
|
|
802
|
-
continue
|
|
803
|
-
fi
|
|
804
|
-
if pending_issue_launch_counts_toward_capacity "$issue_id" && [[ "$(cached_issue_attr scheduled "$issue_id")" == "yes" ]]; then
|
|
805
|
-
count=$((count + 1))
|
|
806
|
-
fi
|
|
807
|
-
done
|
|
808
|
-
printf '%s\n' "$count"
|
|
809
|
-
}
|
|
810
|
-
|
|
811
|
-
pending_scheduled_heavy_issue_launch_count() {
|
|
812
|
-
local pending_file issue_id count=0
|
|
813
|
-
for pending_file in "${pending_launch_dir}"/issue-*.pid; do
|
|
814
|
-
[[ -f "$pending_file" ]] || continue
|
|
815
|
-
issue_id="${pending_file##*/issue-}"
|
|
816
|
-
issue_id="${issue_id%.pid}"
|
|
817
|
-
[[ -n "$issue_id" ]] || continue
|
|
818
|
-
if tmux has-session -t "${issue_prefix}${issue_id}" 2>/dev/null; then
|
|
819
|
-
continue
|
|
820
|
-
fi
|
|
821
|
-
if pending_issue_launch_counts_toward_capacity "$issue_id" \
|
|
822
|
-
&& [[ "$(cached_issue_attr scheduled "$issue_id")" == "yes" ]] \
|
|
823
|
-
&& [[ "$(cached_issue_attr heavy "$issue_id")" == "yes" ]]; then
|
|
824
|
-
count=$((count + 1))
|
|
825
|
-
fi
|
|
826
|
-
done
|
|
827
|
-
printf '%s\n' "$count"
|
|
828
|
-
}
|
|
829
|
-
|
|
830
|
-
pending_recurring_issue_launch_count() {
|
|
831
|
-
local pending_file issue_id count=0
|
|
832
|
-
for pending_file in "${pending_launch_dir}"/issue-*.pid; do
|
|
833
|
-
[[ -f "$pending_file" ]] || continue
|
|
834
|
-
issue_id="${pending_file##*/issue-}"
|
|
835
|
-
issue_id="${issue_id%.pid}"
|
|
836
|
-
[[ -n "$issue_id" ]] || continue
|
|
837
|
-
if tmux has-session -t "${issue_prefix}${issue_id}" 2>/dev/null; then
|
|
838
|
-
continue
|
|
839
|
-
fi
|
|
840
|
-
if pending_issue_launch_counts_toward_capacity "$issue_id" \
|
|
841
|
-
&& [[ "$(cached_issue_attr scheduled "$issue_id")" != "yes" ]] \
|
|
842
|
-
&& [[ "$(cached_issue_attr recurring "$issue_id")" == "yes" ]]; then
|
|
843
|
-
count=$((count + 1))
|
|
844
|
-
fi
|
|
845
|
-
done
|
|
846
|
-
printf '%s\n' "$count"
|
|
847
|
-
}
|
|
848
|
-
|
|
849
|
-
pending_blocked_recovery_issue_launch_count() {
|
|
850
|
-
local pending_file issue_id count=0
|
|
851
|
-
for pending_file in "${pending_launch_dir}"/issue-*.pid; do
|
|
852
|
-
[[ -f "$pending_file" ]] || continue
|
|
853
|
-
issue_id="${pending_file##*/issue-}"
|
|
854
|
-
issue_id="${issue_id%.pid}"
|
|
855
|
-
[[ -n "$issue_id" ]] || continue
|
|
856
|
-
if tmux has-session -t "${issue_prefix}${issue_id}" 2>/dev/null; then
|
|
857
|
-
continue
|
|
858
|
-
fi
|
|
859
|
-
if pending_issue_launch_counts_toward_capacity "$issue_id" && blocked_recovery_issue_has_state "$issue_id"; then
|
|
860
|
-
count=$((count + 1))
|
|
861
|
-
fi
|
|
862
|
-
done
|
|
863
|
-
printf '%s\n' "$count"
|
|
864
|
-
}
|
|
865
|
-
|
|
866
|
-
pending_exclusive_issue_launch_count() {
|
|
867
|
-
local pending_file issue_id count=0
|
|
868
|
-
for pending_file in "${pending_launch_dir}"/issue-*.pid; do
|
|
869
|
-
[[ -f "$pending_file" ]] || continue
|
|
870
|
-
issue_id="${pending_file##*/issue-}"
|
|
871
|
-
issue_id="${issue_id%.pid}"
|
|
872
|
-
[[ -n "$issue_id" ]] || continue
|
|
873
|
-
if tmux has-session -t "${issue_prefix}${issue_id}" 2>/dev/null; then
|
|
874
|
-
continue
|
|
875
|
-
fi
|
|
876
|
-
if pending_issue_launch_counts_toward_capacity "$issue_id" && [[ "$(cached_issue_attr exclusive "$issue_id")" == "yes" ]]; then
|
|
877
|
-
count=$((count + 1))
|
|
878
|
-
fi
|
|
879
|
-
done
|
|
880
|
-
printf '%s\n' "$count"
|
|
881
|
-
}
|
|
882
|
-
|
|
883
|
-
pending_exclusive_pr_launch_count() {
|
|
884
|
-
local pending_file pr_id count=0
|
|
885
|
-
for pending_file in "${pending_launch_dir}"/pr-*.pid; do
|
|
886
|
-
[[ -f "$pending_file" ]] || continue
|
|
887
|
-
pr_id="${pending_file##*/pr-}"
|
|
888
|
-
pr_id="${pr_id%.pid}"
|
|
889
|
-
[[ -n "$pr_id" ]] || continue
|
|
890
|
-
if tmux has-session -t "${pr_prefix}${pr_id}" 2>/dev/null; then
|
|
891
|
-
continue
|
|
892
|
-
fi
|
|
893
|
-
if pending_pr_launch_active "$pr_id" && [[ "$(cached_pr_is_exclusive "$pr_id")" == "yes" ]]; then
|
|
894
|
-
count=$((count + 1))
|
|
895
|
-
fi
|
|
896
|
-
done
|
|
897
|
-
printf '%s\n' "$count"
|
|
898
|
-
}
|
|
899
|
-
|
|
900
|
-
running_non_recurring_issue_workers() {
|
|
901
|
-
local session issue_id is_recurring is_scheduled count=0
|
|
902
|
-
ensure_running_issue_workers_cache
|
|
903
|
-
while IFS= read -r session; do
|
|
904
|
-
[[ -n "$session" ]] || continue
|
|
905
|
-
issue_id="$(issue_id_from_session "$session" || true)"
|
|
906
|
-
[[ -n "$issue_id" ]] || continue
|
|
907
|
-
is_scheduled="$(cached_issue_attr scheduled "$issue_id")"
|
|
908
|
-
if [[ "$is_scheduled" == "yes" ]]; then
|
|
909
|
-
continue
|
|
910
|
-
fi
|
|
911
|
-
is_recurring="$(cached_issue_attr recurring "$issue_id")"
|
|
912
|
-
if [[ "$is_recurring" != "yes" ]]; then
|
|
913
|
-
count=$((count + 1))
|
|
914
|
-
fi
|
|
915
|
-
done <<<"$running_issue_workers_cache"
|
|
916
|
-
printf '%s\n' "$count"
|
|
917
|
-
}
|
|
918
|
-
|
|
919
|
-
running_recurring_issue_workers() {
|
|
920
|
-
local session issue_id is_recurring is_scheduled count=0
|
|
921
|
-
ensure_running_issue_workers_cache
|
|
922
|
-
while IFS= read -r session; do
|
|
923
|
-
[[ -n "$session" ]] || continue
|
|
924
|
-
issue_id="$(issue_id_from_session "$session" || true)"
|
|
925
|
-
[[ -n "$issue_id" ]] || continue
|
|
926
|
-
is_scheduled="$(cached_issue_attr scheduled "$issue_id")"
|
|
927
|
-
if [[ "$is_scheduled" == "yes" ]]; then
|
|
928
|
-
continue
|
|
929
|
-
fi
|
|
930
|
-
is_recurring="$(cached_issue_attr recurring "$issue_id")"
|
|
931
|
-
if [[ "$is_recurring" == "yes" ]]; then
|
|
932
|
-
count=$((count + 1))
|
|
933
|
-
fi
|
|
934
|
-
done <<<"$running_issue_workers_cache"
|
|
935
|
-
# Also count pending recurring launches that are still in progress
|
|
936
|
-
# (prevents infinite respawning when workers die before creating tmux sessions)
|
|
937
|
-
count=$((count + $(pending_recurring_issue_launch_count)))
|
|
938
|
-
printf '%s\n' "$count"
|
|
939
|
-
}
|
|
940
|
-
|
|
941
|
-
running_blocked_recovery_issue_workers() {
|
|
942
|
-
local session issue_id count=0
|
|
943
|
-
ensure_running_issue_workers_cache
|
|
944
|
-
while IFS= read -r session; do
|
|
945
|
-
[[ -n "$session" ]] || continue
|
|
946
|
-
issue_id="$(issue_id_from_session "$session" || true)"
|
|
947
|
-
[[ -n "$issue_id" ]] || continue
|
|
948
|
-
if blocked_recovery_issue_has_state "$issue_id"; then
|
|
949
|
-
count=$((count + 1))
|
|
950
|
-
fi
|
|
951
|
-
done <<<"$running_issue_workers_cache"
|
|
952
|
-
printf '%s\n' "$count"
|
|
953
|
-
}
|
|
954
|
-
|
|
955
|
-
running_exclusive_issue_workers() {
|
|
956
|
-
local session issue_id is_exclusive count=0
|
|
957
|
-
ensure_running_issue_workers_cache
|
|
958
|
-
while IFS= read -r session; do
|
|
959
|
-
[[ -n "$session" ]] || continue
|
|
960
|
-
issue_id="$(issue_id_from_session "$session" || true)"
|
|
961
|
-
[[ -n "$issue_id" ]] || continue
|
|
962
|
-
is_exclusive="$(cached_issue_attr exclusive "$issue_id")"
|
|
963
|
-
if [[ "$is_exclusive" == "yes" ]]; then
|
|
964
|
-
count=$((count + 1))
|
|
965
|
-
fi
|
|
966
|
-
done <<<"$running_issue_workers_cache"
|
|
967
|
-
printf '%s\n' "$count"
|
|
968
|
-
}
|
|
969
|
-
|
|
970
|
-
running_exclusive_pr_workers() {
|
|
971
|
-
local session pr_id is_exclusive count=0
|
|
972
|
-
ensure_running_pr_workers_cache
|
|
973
|
-
while IFS= read -r session; do
|
|
974
|
-
[[ -n "$session" ]] || continue
|
|
975
|
-
pr_id="$(pr_id_from_session "$session" || true)"
|
|
976
|
-
[[ -n "$pr_id" ]] || continue
|
|
977
|
-
is_exclusive="$(cached_pr_is_exclusive "$pr_id")"
|
|
978
|
-
if [[ "$is_exclusive" == "yes" ]]; then
|
|
979
|
-
count=$((count + 1))
|
|
980
|
-
fi
|
|
981
|
-
done <<<"$running_pr_workers_cache"
|
|
982
|
-
printf '%s\n' "$count"
|
|
983
|
-
}
|
|
984
|
-
|
|
985
|
-
running_scheduled_issue_workers() {
|
|
986
|
-
local session issue_id is_scheduled count=0
|
|
987
|
-
ensure_running_issue_workers_cache
|
|
988
|
-
while IFS= read -r session; do
|
|
989
|
-
[[ -n "$session" ]] || continue
|
|
990
|
-
issue_id="$(issue_id_from_session "$session" || true)"
|
|
991
|
-
[[ -n "$issue_id" ]] || continue
|
|
992
|
-
is_scheduled="$(cached_issue_attr scheduled "$issue_id")"
|
|
993
|
-
if [[ "$is_scheduled" == "yes" ]]; then
|
|
994
|
-
count=$((count + 1))
|
|
995
|
-
fi
|
|
996
|
-
done <<<"$running_issue_workers_cache"
|
|
997
|
-
printf '%s\n' "$count"
|
|
998
|
-
}
|
|
999
|
-
|
|
1000
|
-
running_scheduled_heavy_issue_workers() {
|
|
1001
|
-
local session issue_id is_scheduled is_heavy count=0
|
|
1002
|
-
ensure_running_issue_workers_cache
|
|
1003
|
-
while IFS= read -r session; do
|
|
1004
|
-
[[ -n "$session" ]] || continue
|
|
1005
|
-
issue_id="$(issue_id_from_session "$session" || true)"
|
|
1006
|
-
[[ -n "$issue_id" ]] || continue
|
|
1007
|
-
is_scheduled="$(cached_issue_attr scheduled "$issue_id")"
|
|
1008
|
-
is_heavy="$(cached_issue_attr heavy "$issue_id")"
|
|
1009
|
-
if [[ "$is_scheduled" == "yes" && "$is_heavy" == "yes" ]]; then
|
|
1010
|
-
count=$((count + 1))
|
|
1011
|
-
fi
|
|
1012
|
-
done <<<"$running_issue_workers_cache"
|
|
1013
|
-
printf '%s\n' "$count"
|
|
1014
|
-
}
|
|
1015
|
-
|
|
1016
|
-
ready_non_recurring_issue_count() {
|
|
1017
|
-
local issue_id is_recurring count=0
|
|
1018
|
-
ensure_ready_issue_ids_cache
|
|
1019
|
-
while IFS= read -r issue_id; do
|
|
1020
|
-
[[ -n "$issue_id" ]] || continue
|
|
1021
|
-
if [[ "$(cached_issue_attr scheduled "$issue_id")" == "yes" ]]; then
|
|
1022
|
-
continue
|
|
1023
|
-
fi
|
|
1024
|
-
is_recurring="$(cached_issue_attr recurring "$issue_id")"
|
|
1025
|
-
if [[ "$is_recurring" != "yes" ]]; then
|
|
1026
|
-
count=$((count + 1))
|
|
1027
|
-
fi
|
|
1028
|
-
done <<<"$ready_issue_ids_cache"
|
|
1029
|
-
printf '%s\n' "$count"
|
|
1030
|
-
}
|
|
1031
|
-
|
|
1032
|
-
blocked_recovery_issue_ids() {
|
|
1033
|
-
ensure_blocked_recovery_issue_ids_cache
|
|
1034
|
-
printf '%s\n' "$blocked_recovery_issue_ids_cache"
|
|
1035
|
-
}
|
|
1036
|
-
|
|
1037
|
-
ordered_ready_issue_ids() {
|
|
1038
|
-
ensure_ordered_ready_issue_ids_cache
|
|
1039
|
-
printf '%s\n' "$ordered_ready_issue_ids_cache"
|
|
1040
|
-
}
|
|
1041
|
-
|
|
1042
|
-
due_scheduled_issue_ids() {
|
|
1043
|
-
ensure_due_scheduled_issue_ids_cache
|
|
1044
|
-
printf '%s\n' "$due_scheduled_issue_ids_cache"
|
|
1045
|
-
}
|
|
1046
|
-
|
|
1047
|
-
due_blocked_recovery_issue_ids() {
|
|
1048
|
-
ensure_due_blocked_recovery_issue_ids_cache
|
|
1049
|
-
printf '%s\n' "$due_blocked_recovery_issue_ids_cache"
|
|
1050
|
-
}
|
|
1051
|
-
|
|
1052
|
-
ensure_due_scheduled_issue_ids_cache() {
|
|
1053
|
-
if [[ "$due_scheduled_issue_ids_cache_loaded" != "yes" ]]; then
|
|
1054
|
-
due_scheduled_issue_ids_cache="$(build_due_scheduled_issue_ids_cache)"
|
|
1055
|
-
due_scheduled_issue_ids_cache_loaded="yes"
|
|
1056
|
-
fi
|
|
1057
|
-
}
|
|
1058
|
-
|
|
1059
|
-
ensure_due_blocked_recovery_issue_ids_cache() {
|
|
1060
|
-
if [[ "$due_blocked_recovery_issue_ids_cache_loaded" != "yes" ]]; then
|
|
1061
|
-
due_blocked_recovery_issue_ids_cache="$(build_due_blocked_recovery_issue_ids_cache)"
|
|
1062
|
-
due_blocked_recovery_issue_ids_cache_loaded="yes"
|
|
1063
|
-
fi
|
|
1064
|
-
}
|
|
1065
|
-
|
|
1066
|
-
build_due_scheduled_issue_ids_cache() {
|
|
1067
|
-
local issue_id now_epoch due_epoch
|
|
1068
|
-
now_epoch="$(date +%s)"
|
|
1069
|
-
ensure_ready_issue_ids_cache
|
|
1070
|
-
while IFS= read -r issue_id; do
|
|
1071
|
-
[[ -n "$issue_id" ]] || continue
|
|
1072
|
-
if [[ "$(cached_issue_attr scheduled "$issue_id")" != "yes" ]]; then
|
|
1073
|
-
continue
|
|
1074
|
-
fi
|
|
1075
|
-
if ! scheduled_issue_is_due "$issue_id"; then
|
|
1076
|
-
continue
|
|
1077
|
-
fi
|
|
1078
|
-
due_epoch="$(scheduled_issue_due_epoch "$issue_id")"
|
|
1079
|
-
if ! [[ "${due_epoch:-}" =~ ^[0-9]+$ ]]; then
|
|
1080
|
-
due_epoch=0
|
|
1081
|
-
fi
|
|
1082
|
-
printf '%s\t%s\n' "$due_epoch" "$issue_id"
|
|
1083
|
-
done <<<"$ready_issue_ids_cache" | sort -n -k1,1 -k2,2n | cut -f2
|
|
1084
|
-
}
|
|
1085
|
-
|
|
1086
|
-
build_due_blocked_recovery_issue_ids_cache() {
|
|
1087
|
-
local issue_id due_epoch
|
|
1088
|
-
if (( max_concurrent_blocked_recovery_issue_workers <= 0 )); then
|
|
1089
|
-
return 0
|
|
1090
|
-
fi
|
|
1091
|
-
|
|
1092
|
-
ensure_blocked_recovery_issue_ids_cache
|
|
1093
|
-
while IFS= read -r issue_id; do
|
|
1094
|
-
[[ -n "$issue_id" ]] || continue
|
|
1095
|
-
if ! blocked_recovery_issue_is_due "$issue_id"; then
|
|
1096
|
-
continue
|
|
1097
|
-
fi
|
|
1098
|
-
due_epoch="$(blocked_recovery_issue_due_epoch "$issue_id")"
|
|
1099
|
-
if ! [[ "${due_epoch:-}" =~ ^[0-9]+$ ]]; then
|
|
1100
|
-
due_epoch=0
|
|
1101
|
-
fi
|
|
1102
|
-
printf '%s\t%s\n' "$due_epoch" "$issue_id"
|
|
1103
|
-
done <<<"$blocked_recovery_issue_ids_cache" | sort -n -k1,1 -k2,2n | cut -f2
|
|
1104
|
-
}
|
|
1105
|
-
|
|
1106
|
-
build_ordered_ready_issue_ids_cache() {
|
|
1107
|
-
local issue_id is_recurring last_recurring_issue seen_last="no"
|
|
1108
|
-
local -a recurring_ids=()
|
|
1109
|
-
ensure_ready_issue_ids_cache
|
|
1110
|
-
while IFS= read -r issue_id; do
|
|
1111
|
-
[[ -n "$issue_id" ]] || continue
|
|
1112
|
-
if [[ "$(cached_issue_attr scheduled "$issue_id")" == "yes" ]]; then
|
|
1113
|
-
continue
|
|
1114
|
-
fi
|
|
1115
|
-
is_recurring="$(cached_issue_attr recurring "$issue_id")"
|
|
1116
|
-
if [[ "$is_recurring" != "yes" ]]; then
|
|
1117
|
-
printf '%s\n' "$issue_id"
|
|
1118
|
-
else
|
|
1119
|
-
recurring_ids+=("$issue_id")
|
|
1120
|
-
fi
|
|
1121
|
-
done <<<"$ready_issue_ids_cache"
|
|
1122
|
-
|
|
1123
|
-
if (( ${#recurring_ids[@]} == 0 )); then
|
|
1124
|
-
return 0
|
|
1125
|
-
fi
|
|
1126
|
-
|
|
1127
|
-
last_recurring_issue="$(last_launched_recurring_issue_id || true)"
|
|
1128
|
-
if [[ -n "$last_recurring_issue" ]]; then
|
|
1129
|
-
local emitted_after_last=0
|
|
1130
|
-
for issue_id in "${recurring_ids[@]}"; do
|
|
1131
|
-
if [[ "$seen_last" == "yes" ]]; then
|
|
1132
|
-
printf '%s\n' "$issue_id"
|
|
1133
|
-
emitted_after_last=$((emitted_after_last + 1))
|
|
1134
|
-
fi
|
|
1135
|
-
if [[ "$issue_id" == "$last_recurring_issue" ]]; then
|
|
1136
|
-
seen_last="yes"
|
|
1137
|
-
fi
|
|
1138
|
-
done
|
|
1139
|
-
fi
|
|
1140
|
-
|
|
1141
|
-
for issue_id in "${recurring_ids[@]}"; do
|
|
1142
|
-
# Stop the wrap-around once we reach the last-launched issue, but only
|
|
1143
|
-
# when the first loop already emitted at least one issue after it.
|
|
1144
|
-
# When there is exactly one recurring issue (or the last-launched issue
|
|
1145
|
-
# is the final element), emitted_after_last is 0, so we must still
|
|
1146
|
-
# include it here to avoid producing an empty list.
|
|
1147
|
-
if [[ -n "$last_recurring_issue" && "$seen_last" == "yes" && "$issue_id" == "$last_recurring_issue" && "$emitted_after_last" -gt 0 ]]; then
|
|
1148
|
-
break
|
|
1149
|
-
fi
|
|
1150
|
-
printf '%s\n' "$issue_id"
|
|
1151
|
-
done
|
|
1152
|
-
}
|
|
1153
|
-
|
|
1154
|
-
completed_workers() {
|
|
1155
|
-
ensure_completed_workers_cache
|
|
1156
|
-
printf '%s\n' "$completed_workers_cache"
|
|
1157
|
-
}
|
|
1158
|
-
|
|
1159
|
-
reconciled_marker_matches_run() {
|
|
1160
|
-
local run_dir="${1:?run dir required}"
|
|
1161
|
-
local marker_file="${run_dir}/reconciled.ok"
|
|
1162
|
-
local run_env="${run_dir}/run.env"
|
|
1163
|
-
local marker_started_at=""
|
|
1164
|
-
local run_started_at=""
|
|
1165
|
-
|
|
1166
|
-
[[ -f "${marker_file}" && -f "${run_env}" ]] || return 1
|
|
1167
|
-
|
|
1168
|
-
marker_started_at="$(awk -F= '/^STARTED_AT=/{print $2}' "${marker_file}" 2>/dev/null | tr -d '"' | tail -n 1 || true)"
|
|
1169
|
-
run_started_at="$(awk -F= '/^STARTED_AT=/{print $2}' "${run_env}" 2>/dev/null | tr -d '"' | tail -n 1 || true)"
|
|
1170
|
-
|
|
1171
|
-
[[ -n "${marker_started_at}" && -n "${run_started_at}" && "${marker_started_at}" == "${run_started_at}" ]]
|
|
1172
|
-
}
|
|
1173
|
-
|
|
1174
|
-
ensure_completed_workers_cache() {
|
|
1175
|
-
local dir session issue_id status_line status
|
|
1176
|
-
if [[ "$completed_workers_cache_loaded" == "yes" ]]; then
|
|
1177
|
-
return 0
|
|
1178
|
-
fi
|
|
1179
|
-
completed_workers_cache=""
|
|
1180
|
-
for dir in "$runs_root"/*; do
|
|
1181
|
-
[[ -d "$dir" ]] || continue
|
|
1182
|
-
session="${dir##*/}"
|
|
1183
|
-
session_matches_prefix "$session" || continue
|
|
1184
|
-
if reconciled_marker_matches_run "$dir"; then
|
|
1185
|
-
continue
|
|
1186
|
-
fi
|
|
1187
|
-
if [[ "$session" == "${issue_prefix}"* ]]; then
|
|
1188
|
-
issue_id="$(issue_id_from_session "$session" || true)"
|
|
1189
|
-
if [[ -n "${issue_id}" ]] && pending_issue_launch_active "${issue_id}"; then
|
|
1190
|
-
continue
|
|
1191
|
-
fi
|
|
1192
|
-
fi
|
|
1193
|
-
status_line="$(
|
|
1194
|
-
"${shared_agent_home}/tools/bin/agent-project-worker-status" \
|
|
1195
|
-
--runs-root "$runs_root" \
|
|
1196
|
-
--session "$session" \
|
|
1197
|
-
| awk -F= '/^STATUS=/{print $2}' || true
|
|
1198
|
-
)"
|
|
1199
|
-
status="${status_line:-UNKNOWN}"
|
|
1200
|
-
if [[ "$status" == "SUCCEEDED" || "$status" == "FAILED" ]]; then
|
|
1201
|
-
completed_workers_cache+="${session}"$'\n'
|
|
1202
|
-
fi
|
|
1203
|
-
done
|
|
1204
|
-
completed_workers_cache="${completed_workers_cache%$'\n'}"
|
|
1205
|
-
completed_workers_cache_loaded="yes"
|
|
1206
|
-
}
|
|
1207
|
-
|
|
1208
|
-
ready_issue_ids() {
|
|
1209
|
-
ensure_ready_issue_ids_cache
|
|
1210
|
-
printf '%s\n' "$ready_issue_ids_cache"
|
|
1211
|
-
}
|
|
1212
|
-
|
|
1213
|
-
ensure_ready_issue_ids_cache() {
|
|
1214
|
-
if [[ "$ready_issue_ids_cache_loaded" != "yes" ]]; then
|
|
1215
|
-
ready_issue_ids_cache="$(heartbeat_list_ready_issue_ids)"
|
|
1216
|
-
ready_issue_ids_cache_loaded="yes"
|
|
1217
|
-
fi
|
|
1218
|
-
}
|
|
1219
|
-
|
|
1220
|
-
last_launched_recurring_issue_id() {
|
|
1221
|
-
if [[ -f "$recurring_rotation_file" ]]; then
|
|
1222
|
-
tr -d '[:space:]' <"$recurring_rotation_file"
|
|
1223
|
-
fi
|
|
1224
|
-
}
|
|
1225
|
-
|
|
1226
|
-
record_recurring_issue_launch() {
|
|
1227
|
-
local issue_id="${1:?issue id required}"
|
|
1228
|
-
printf '%s\n' "$issue_id" >"$recurring_rotation_file"
|
|
1229
|
-
}
|
|
1230
|
-
|
|
1231
|
-
scheduled_state_file() {
|
|
1232
|
-
local issue_id="${1:?issue id required}"
|
|
1233
|
-
printf '%s\n' "${scheduled_state_dir}/${issue_id}.env"
|
|
1234
|
-
}
|
|
1235
|
-
|
|
1236
|
-
scheduled_issue_due_epoch() {
|
|
1237
|
-
local issue_id="${1:?issue id required}"
|
|
1238
|
-
local state_file next_due_epoch
|
|
1239
|
-
state_file="$(scheduled_state_file "$issue_id")"
|
|
1240
|
-
if [[ ! -f "$state_file" ]]; then
|
|
1241
|
-
printf '0\n'
|
|
1242
|
-
return 0
|
|
1243
|
-
fi
|
|
1244
|
-
|
|
1245
|
-
next_due_epoch="$(awk -F= '/^NEXT_DUE_EPOCH=/{print $2}' "$state_file" 2>/dev/null | tr -d '[:space:]' || true)"
|
|
1246
|
-
if ! [[ "${next_due_epoch:-}" =~ ^[0-9]+$ ]]; then
|
|
1247
|
-
printf '0\n'
|
|
1248
|
-
return 0
|
|
1249
|
-
fi
|
|
1250
|
-
|
|
1251
|
-
printf '%s\n' "$next_due_epoch"
|
|
1252
|
-
}
|
|
1253
|
-
|
|
1254
|
-
scheduled_issue_is_due() {
|
|
1255
|
-
local issue_id="${1:?issue id required}"
|
|
1256
|
-
local interval_seconds due_epoch now_epoch
|
|
1257
|
-
interval_seconds="$(cached_issue_attr schedule_interval_seconds "$issue_id")"
|
|
1258
|
-
if ! [[ "${interval_seconds:-}" =~ ^[1-9][0-9]*$ ]]; then
|
|
1259
|
-
return 1
|
|
1260
|
-
fi
|
|
1261
|
-
|
|
1262
|
-
due_epoch="$(scheduled_issue_due_epoch "$issue_id")"
|
|
1263
|
-
now_epoch="$(date +%s)"
|
|
1264
|
-
if ! [[ "${due_epoch:-}" =~ ^[0-9]+$ ]] || (( due_epoch == 0 || due_epoch <= now_epoch )); then
|
|
1265
|
-
return 0
|
|
1266
|
-
fi
|
|
1267
|
-
return 1
|
|
1268
|
-
}
|
|
1269
|
-
|
|
1270
|
-
record_scheduled_issue_launch() {
|
|
1271
|
-
local issue_id="${1:?issue id required}"
|
|
1272
|
-
local interval_seconds state_file now_epoch due_epoch next_due_epoch
|
|
1273
|
-
|
|
1274
|
-
interval_seconds="$(cached_issue_attr schedule_interval_seconds "$issue_id")"
|
|
1275
|
-
if ! [[ "${interval_seconds:-}" =~ ^[1-9][0-9]*$ ]]; then
|
|
1276
|
-
return 0
|
|
1277
|
-
fi
|
|
1278
|
-
|
|
1279
|
-
now_epoch="$(date +%s)"
|
|
1280
|
-
due_epoch="$(scheduled_issue_due_epoch "$issue_id")"
|
|
1281
|
-
if ! [[ "${due_epoch:-}" =~ ^[0-9]+$ ]] || (( due_epoch <= 0 )); then
|
|
1282
|
-
next_due_epoch=$((now_epoch + interval_seconds))
|
|
1283
|
-
else
|
|
1284
|
-
next_due_epoch="$due_epoch"
|
|
1285
|
-
while (( next_due_epoch <= now_epoch )); do
|
|
1286
|
-
next_due_epoch=$((next_due_epoch + interval_seconds))
|
|
1287
|
-
done
|
|
1288
|
-
fi
|
|
1289
|
-
|
|
1290
|
-
state_file="$(scheduled_state_file "$issue_id")"
|
|
1291
|
-
cat >"$state_file" <<EOF
|
|
1292
|
-
INTERVAL_SECONDS=${interval_seconds}
|
|
1293
|
-
LAST_STARTED_EPOCH=${now_epoch}
|
|
1294
|
-
LAST_STARTED_AT=$(date -u -r "$now_epoch" +"%Y-%m-%dT%H:%M:%SZ")
|
|
1295
|
-
NEXT_DUE_EPOCH=${next_due_epoch}
|
|
1296
|
-
NEXT_DUE_AT=$(date -u -r "$next_due_epoch" +"%Y-%m-%dT%H:%M:%SZ")
|
|
1297
|
-
UPDATED_AT=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
|
1298
|
-
EOF
|
|
1299
|
-
}
|
|
1300
|
-
|
|
1301
|
-
record_scheduled_issue_result() {
|
|
1302
|
-
local issue_id="${1:?issue id required}"
|
|
1303
|
-
local result_status="${2:-unknown}"
|
|
1304
|
-
local state_file interval_seconds last_started_epoch next_due_epoch now_epoch
|
|
1305
|
-
|
|
1306
|
-
state_file="$(scheduled_state_file "$issue_id")"
|
|
1307
|
-
interval_seconds="$(cached_issue_attr schedule_interval_seconds "$issue_id")"
|
|
1308
|
-
last_started_epoch="$(awk -F= '/^LAST_STARTED_EPOCH=/{print $2}' "$state_file" 2>/dev/null | tr -d '[:space:]' || true)"
|
|
1309
|
-
next_due_epoch="$(awk -F= '/^NEXT_DUE_EPOCH=/{print $2}' "$state_file" 2>/dev/null | tr -d '[:space:]' || true)"
|
|
1310
|
-
now_epoch="$(date +%s)"
|
|
1311
|
-
|
|
1312
|
-
if ! [[ "${interval_seconds:-}" =~ ^[1-9][0-9]*$ ]]; then
|
|
1313
|
-
interval_seconds=0
|
|
1314
|
-
fi
|
|
1315
|
-
if ! [[ "${last_started_epoch:-}" =~ ^[0-9]+$ ]]; then
|
|
1316
|
-
last_started_epoch=0
|
|
1317
|
-
fi
|
|
1318
|
-
if ! [[ "${next_due_epoch:-}" =~ ^[0-9]+$ ]]; then
|
|
1319
|
-
next_due_epoch=0
|
|
1320
|
-
fi
|
|
1321
|
-
|
|
1322
|
-
cat >"$state_file" <<EOF
|
|
1323
|
-
INTERVAL_SECONDS=${interval_seconds}
|
|
1324
|
-
LAST_STARTED_EPOCH=${last_started_epoch}
|
|
1325
|
-
LAST_STARTED_AT=$(if [[ "$last_started_epoch" =~ ^[0-9]+$ ]] && (( last_started_epoch > 0 )); then date -u -r "$last_started_epoch" +"%Y-%m-%dT%H:%M:%SZ"; fi)
|
|
1326
|
-
LAST_RESULT_STATUS=${result_status}
|
|
1327
|
-
LAST_RESULT_EPOCH=${now_epoch}
|
|
1328
|
-
LAST_RESULT_AT=$(date -u -r "$now_epoch" +"%Y-%m-%dT%H:%M:%SZ")
|
|
1329
|
-
NEXT_DUE_EPOCH=${next_due_epoch}
|
|
1330
|
-
NEXT_DUE_AT=$(if [[ "$next_due_epoch" =~ ^[0-9]+$ ]] && (( next_due_epoch > 0 )); then date -u -r "$next_due_epoch" +"%Y-%m-%dT%H:%M:%SZ"; fi)
|
|
1331
|
-
UPDATED_AT=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
|
1332
|
-
EOF
|
|
1333
|
-
}
|
|
1334
|
-
|
|
1335
|
-
blocked_recovery_state_file() {
|
|
1336
|
-
local issue_id="${1:?issue id required}"
|
|
1337
|
-
printf '%s\n' "${blocked_recovery_state_dir}/${issue_id}.env"
|
|
1338
|
-
}
|
|
1339
|
-
|
|
1340
|
-
blocked_recovery_issue_has_state() {
|
|
1341
|
-
local issue_id="${1:?issue id required}"
|
|
1342
|
-
[[ -f "$(blocked_recovery_state_file "$issue_id")" ]]
|
|
1343
|
-
}
|
|
1344
|
-
|
|
1345
|
-
blocked_recovery_issue_due_epoch() {
|
|
1346
|
-
local issue_id="${1:?issue id required}"
|
|
1347
|
-
local state_file next_due_epoch
|
|
1348
|
-
state_file="$(blocked_recovery_state_file "$issue_id")"
|
|
1349
|
-
if [[ ! -f "$state_file" ]]; then
|
|
1350
|
-
printf '0\n'
|
|
1351
|
-
return 0
|
|
1352
|
-
fi
|
|
1353
|
-
|
|
1354
|
-
next_due_epoch="$(awk -F= '/^NEXT_DUE_EPOCH=/{print $2}' "$state_file" 2>/dev/null | tr -d '[:space:]' || true)"
|
|
1355
|
-
if ! [[ "${next_due_epoch:-}" =~ ^[0-9]+$ ]]; then
|
|
1356
|
-
printf '0\n'
|
|
1357
|
-
return 0
|
|
1358
|
-
fi
|
|
1359
|
-
|
|
1360
|
-
printf '%s\n' "$next_due_epoch"
|
|
1361
|
-
}
|
|
1362
|
-
|
|
1363
|
-
blocked_recovery_issue_is_due() {
|
|
1364
|
-
local issue_id="${1:?issue id required}"
|
|
1365
|
-
local due_epoch now_epoch
|
|
1366
|
-
if ! [[ "${blocked_recovery_cooldown_seconds:-}" =~ ^[1-9][0-9]*$ ]]; then
|
|
1367
|
-
return 0
|
|
1368
|
-
fi
|
|
1369
|
-
|
|
1370
|
-
due_epoch="$(blocked_recovery_issue_due_epoch "$issue_id")"
|
|
1371
|
-
now_epoch="$(date +%s)"
|
|
1372
|
-
if ! [[ "${due_epoch:-}" =~ ^[0-9]+$ ]] || (( due_epoch == 0 || due_epoch <= now_epoch )); then
|
|
1373
|
-
return 0
|
|
1374
|
-
fi
|
|
1375
|
-
return 1
|
|
1376
|
-
}
|
|
1377
|
-
|
|
1378
|
-
record_blocked_recovery_issue_launch() {
|
|
1379
|
-
local issue_id="${1:?issue id required}"
|
|
1380
|
-
local state_file now_epoch next_due_epoch next_due_at
|
|
1381
|
-
|
|
1382
|
-
now_epoch="$(date +%s)"
|
|
1383
|
-
next_due_epoch=0
|
|
1384
|
-
next_due_at=""
|
|
1385
|
-
if [[ "${blocked_recovery_cooldown_seconds:-}" =~ ^[1-9][0-9]*$ ]]; then
|
|
1386
|
-
next_due_epoch=$((now_epoch + blocked_recovery_cooldown_seconds))
|
|
1387
|
-
next_due_at="$(date -u -r "$next_due_epoch" +"%Y-%m-%dT%H:%M:%SZ")"
|
|
1388
|
-
fi
|
|
1389
|
-
|
|
1390
|
-
state_file="$(blocked_recovery_state_file "$issue_id")"
|
|
1391
|
-
cat >"$state_file" <<EOF
|
|
1392
|
-
LANE=blocked-recovery
|
|
1393
|
-
LAST_STARTED_EPOCH=${now_epoch}
|
|
1394
|
-
LAST_STARTED_AT=$(date -u -r "$now_epoch" +"%Y-%m-%dT%H:%M:%SZ")
|
|
1395
|
-
NEXT_DUE_EPOCH=${next_due_epoch}
|
|
1396
|
-
NEXT_DUE_AT=${next_due_at}
|
|
1397
|
-
UPDATED_AT=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
|
1398
|
-
EOF
|
|
1399
|
-
}
|
|
1400
|
-
|
|
1401
|
-
clear_blocked_recovery_issue_state() {
|
|
1402
|
-
local issue_id="${1:?issue id required}"
|
|
1403
|
-
rm -f "$(blocked_recovery_state_file "$issue_id")"
|
|
1404
|
-
}
|
|
1405
|
-
|
|
1406
|
-
open_agent_pr_ids() {
|
|
1407
|
-
ensure_open_agent_pr_ids_cache
|
|
1408
|
-
printf '%s\n' "$open_agent_pr_ids_cache"
|
|
1409
|
-
}
|
|
1410
|
-
|
|
1411
|
-
ensure_open_agent_pr_ids_cache() {
|
|
1412
|
-
if [[ "$open_agent_pr_ids_cache_loaded" != "yes" ]]; then
|
|
1413
|
-
open_agent_pr_ids_cache="$(heartbeat_list_open_agent_pr_ids)"
|
|
1414
|
-
open_agent_pr_ids_cache_loaded="yes"
|
|
1415
|
-
fi
|
|
1416
|
-
}
|
|
1417
|
-
|
|
1418
|
-
running_issue_ids() {
|
|
1419
|
-
ensure_running_issue_ids_cache
|
|
1420
|
-
printf '%s\n' "$running_issue_ids_cache"
|
|
1421
|
-
}
|
|
1422
|
-
|
|
1423
|
-
exclusive_issue_ids() {
|
|
1424
|
-
ensure_exclusive_issue_ids_cache
|
|
1425
|
-
printf '%s\n' "$exclusive_issue_ids_cache"
|
|
1426
|
-
}
|
|
1427
|
-
|
|
1428
|
-
exclusive_pr_ids() {
|
|
1429
|
-
ensure_exclusive_pr_ids_cache
|
|
1430
|
-
printf '%s\n' "$exclusive_pr_ids_cache"
|
|
1431
|
-
}
|
|
1432
|
-
|
|
1433
|
-
ensure_running_issue_ids_cache() {
|
|
1434
|
-
if [[ "$running_issue_ids_cache_loaded" != "yes" ]]; then
|
|
1435
|
-
running_issue_ids_cache="$(heartbeat_list_running_issue_ids)"
|
|
1436
|
-
running_issue_ids_cache_loaded="yes"
|
|
1437
|
-
fi
|
|
1438
|
-
}
|
|
1439
|
-
|
|
1440
|
-
ensure_exclusive_issue_ids_cache() {
|
|
1441
|
-
if [[ "$exclusive_issue_ids_cache_loaded" != "yes" ]]; then
|
|
1442
|
-
exclusive_issue_ids_cache="$(heartbeat_list_exclusive_issue_ids)"
|
|
1443
|
-
exclusive_issue_ids_cache_loaded="yes"
|
|
1444
|
-
fi
|
|
1445
|
-
}
|
|
1446
|
-
|
|
1447
|
-
ensure_exclusive_pr_ids_cache() {
|
|
1448
|
-
if [[ "$exclusive_pr_ids_cache_loaded" != "yes" ]]; then
|
|
1449
|
-
exclusive_pr_ids_cache="$(heartbeat_list_exclusive_pr_ids)"
|
|
1450
|
-
exclusive_pr_ids_cache_loaded="yes"
|
|
1451
|
-
fi
|
|
1452
|
-
}
|
|
1453
|
-
|
|
1454
|
-
ensure_ordered_ready_issue_ids_cache() {
|
|
1455
|
-
if [[ "$ordered_ready_issue_ids_cache_loaded" != "yes" ]]; then
|
|
1456
|
-
ordered_ready_issue_ids_cache="$(build_ordered_ready_issue_ids_cache)"
|
|
1457
|
-
ordered_ready_issue_ids_cache_loaded="yes"
|
|
1458
|
-
fi
|
|
1459
|
-
}
|
|
1460
|
-
|
|
1461
|
-
ensure_blocked_recovery_issue_ids_cache() {
|
|
1462
|
-
if [[ "$blocked_recovery_issue_ids_cache_loaded" != "yes" ]]; then
|
|
1463
|
-
blocked_recovery_issue_ids_cache="$(heartbeat_list_blocked_recovery_issue_ids)"
|
|
1464
|
-
blocked_recovery_issue_ids_cache_loaded="yes"
|
|
1465
|
-
fi
|
|
1466
|
-
}
|
|
1467
|
-
|
|
1468
|
-
sync_open_agent_issues() {
|
|
1469
|
-
local issue_id status_out status
|
|
1470
|
-
ensure_running_issue_ids_cache
|
|
1471
|
-
while IFS= read -r issue_id; do
|
|
1472
|
-
[[ -n "$issue_id" ]] || continue
|
|
1473
|
-
if tmux has-session -t "${issue_prefix}${issue_id}" 2>/dev/null; then
|
|
1474
|
-
continue
|
|
1475
|
-
fi
|
|
1476
|
-
if pending_issue_launch_active "$issue_id"; then
|
|
1477
|
-
if pending_issue_launch_counts_toward_capacity "$issue_id"; then
|
|
1478
|
-
heartbeat_mark_issue_running "$issue_id" "$(cached_issue_attr heavy "$issue_id")" >/dev/null || true
|
|
1479
|
-
fi
|
|
1480
|
-
continue
|
|
1481
|
-
fi
|
|
1482
|
-
status_out="$(
|
|
1483
|
-
"${shared_agent_home}/tools/bin/agent-project-worker-status" \
|
|
1484
|
-
--runs-root "$runs_root" \
|
|
1485
|
-
--session "${issue_prefix}${issue_id}"
|
|
1486
|
-
)"
|
|
1487
|
-
status="$(awk -F= '/^STATUS=/{print $2}' <<<"$status_out")"
|
|
1488
|
-
case "$status" in
|
|
1489
|
-
RUNNING)
|
|
1490
|
-
;;
|
|
1491
|
-
*)
|
|
1492
|
-
heartbeat_sync_issue_labels "$issue_id" >/dev/null || true
|
|
1493
|
-
;;
|
|
1494
|
-
esac
|
|
1495
|
-
done <<<"$running_issue_ids_cache"
|
|
1496
|
-
}
|
|
1497
|
-
|
|
1498
|
-
sync_open_agent_prs() {
|
|
1499
|
-
local pr_number status_out status
|
|
1500
|
-
ensure_open_agent_pr_ids_cache
|
|
1501
|
-
while IFS= read -r pr_number; do
|
|
1502
|
-
[[ -n "$pr_number" ]] || continue
|
|
1503
|
-
if tmux has-session -t "${pr_prefix}${pr_number}" 2>/dev/null; then
|
|
1504
|
-
continue
|
|
1505
|
-
fi
|
|
1506
|
-
if pending_pr_launch_active "$pr_number"; then
|
|
1507
|
-
heartbeat_mark_pr_running "$pr_number" >/dev/null || true
|
|
1508
|
-
continue
|
|
1509
|
-
fi
|
|
1510
|
-
status_out="$(
|
|
1511
|
-
"${shared_agent_home}/tools/bin/agent-project-worker-status" \
|
|
1512
|
-
--runs-root "$runs_root" \
|
|
1513
|
-
--session "${pr_prefix}${pr_number}"
|
|
1514
|
-
)"
|
|
1515
|
-
status="$(awk -F= '/^STATUS=/{print $2}' <<<"$status_out")"
|
|
1516
|
-
case "$status" in
|
|
1517
|
-
UNKNOWN)
|
|
1518
|
-
heartbeat_clear_pr_running "$pr_number" >/dev/null || true
|
|
1519
|
-
heartbeat_sync_pr_labels "$pr_number" >/dev/null || true
|
|
1520
|
-
;;
|
|
1521
|
-
RUNNING)
|
|
1522
|
-
;;
|
|
1523
|
-
*)
|
|
1524
|
-
heartbeat_clear_pr_running "$pr_number" >/dev/null || true
|
|
1525
|
-
heartbeat_sync_pr_labels "$pr_number" >/dev/null || true
|
|
1526
|
-
;;
|
|
1527
|
-
esac
|
|
1528
|
-
done <<<"$open_agent_pr_ids_cache"
|
|
1529
|
-
}
|
|
1530
|
-
|
|
1531
|
-
next_pr_candidate_json() {
|
|
1532
|
-
local target_lane pr_number risk_json lane
|
|
1533
|
-
ensure_open_agent_pr_ids_cache
|
|
1534
|
-
for target_lane in double-check-2 double-check-1 automerge merge-repair fix ci-refresh; do
|
|
1535
|
-
while IFS= read -r pr_number; do
|
|
1536
|
-
[[ -n "$pr_number" ]] || continue
|
|
1537
|
-
if tmux has-session -t "${pr_prefix}${pr_number}" 2>/dev/null; then
|
|
1538
|
-
continue
|
|
1539
|
-
fi
|
|
1540
|
-
if pr_launch_reserved "$pr_number"; then
|
|
1541
|
-
continue
|
|
1542
|
-
fi
|
|
1543
|
-
if pending_pr_launch_active "$pr_number"; then
|
|
1544
|
-
continue
|
|
1545
|
-
fi
|
|
1546
|
-
if ! retry_ready pr "$pr_number"; then
|
|
1547
|
-
continue
|
|
1548
|
-
fi
|
|
1549
|
-
risk_json="$(cached_pr_risk_json "$pr_number")"
|
|
1550
|
-
lane="$(jq -r '.agentLane' <<<"$risk_json")"
|
|
1551
|
-
if [[ "$lane" == "$target_lane" ]]; then
|
|
1552
|
-
printf '%s\n' "$risk_json"
|
|
1553
|
-
return 0
|
|
1554
|
-
fi
|
|
1555
|
-
done <<<"$open_agent_pr_ids_cache"
|
|
1556
|
-
done
|
|
1557
|
-
}
|
|
1558
|
-
|
|
1559
|
-
next_priority_review_pr_candidate_json() {
|
|
1560
|
-
local target_lane pr_number risk_json lane
|
|
1561
|
-
ensure_open_agent_pr_ids_cache
|
|
1562
|
-
for target_lane in double-check-2 double-check-1; do
|
|
1563
|
-
while IFS= read -r pr_number; do
|
|
1564
|
-
[[ -n "$pr_number" ]] || continue
|
|
1565
|
-
if tmux has-session -t "${pr_prefix}${pr_number}" 2>/dev/null; then
|
|
1566
|
-
continue
|
|
1567
|
-
fi
|
|
1568
|
-
if pr_launch_reserved "$pr_number"; then
|
|
1569
|
-
continue
|
|
1570
|
-
fi
|
|
1571
|
-
if pending_pr_launch_active "$pr_number"; then
|
|
1572
|
-
continue
|
|
1573
|
-
fi
|
|
1574
|
-
if ! retry_ready pr "$pr_number"; then
|
|
1575
|
-
continue
|
|
1576
|
-
fi
|
|
1577
|
-
risk_json="$(cached_pr_risk_json "$pr_number")"
|
|
1578
|
-
lane="$(jq -r '.agentLane' <<<"$risk_json")"
|
|
1579
|
-
if [[ "$lane" == "$target_lane" ]]; then
|
|
1580
|
-
printf '%s\n' "$risk_json"
|
|
1581
|
-
return 0
|
|
1582
|
-
fi
|
|
1583
|
-
done <<<"$open_agent_pr_ids_cache"
|
|
1584
|
-
done
|
|
1585
|
-
}
|
|
1586
|
-
|
|
1587
|
-
eligible_pr_backlog_count() {
|
|
1588
|
-
local pr_number risk_json lane count=0
|
|
1589
|
-
ensure_open_agent_pr_ids_cache
|
|
1590
|
-
while IFS= read -r pr_number; do
|
|
1591
|
-
[[ -n "$pr_number" ]] || continue
|
|
1592
|
-
if tmux has-session -t "${pr_prefix}${pr_number}" 2>/dev/null; then
|
|
1593
|
-
continue
|
|
1594
|
-
fi
|
|
1595
|
-
if pr_launch_reserved "$pr_number"; then
|
|
1596
|
-
continue
|
|
1597
|
-
fi
|
|
1598
|
-
if pending_pr_launch_active "$pr_number"; then
|
|
1599
|
-
continue
|
|
1600
|
-
fi
|
|
1601
|
-
if ! retry_ready pr "$pr_number"; then
|
|
1602
|
-
continue
|
|
1603
|
-
fi
|
|
1604
|
-
risk_json="$(cached_pr_risk_json "$pr_number")"
|
|
1605
|
-
lane="$(jq -r '.agentLane' <<<"$risk_json")"
|
|
1606
|
-
case "$lane" in
|
|
1607
|
-
double-check-1|double-check-2|automerge|merge-repair|fix)
|
|
1608
|
-
count=$((count + 1))
|
|
1609
|
-
;;
|
|
1610
|
-
esac
|
|
1611
|
-
done <<<"$open_agent_pr_ids_cache"
|
|
1612
|
-
printf '%s\n' "$count"
|
|
1613
|
-
}
|
|
1614
|
-
|
|
1615
|
-
priority_review_backlog_count() {
|
|
1616
|
-
local pr_number risk_json lane count=0
|
|
1617
|
-
ensure_open_agent_pr_ids_cache
|
|
1618
|
-
while IFS= read -r pr_number; do
|
|
1619
|
-
[[ -n "$pr_number" ]] || continue
|
|
1620
|
-
if tmux has-session -t "${pr_prefix}${pr_number}" 2>/dev/null; then
|
|
1621
|
-
continue
|
|
1622
|
-
fi
|
|
1623
|
-
if pr_launch_reserved "$pr_number"; then
|
|
1624
|
-
continue
|
|
1625
|
-
fi
|
|
1626
|
-
if pending_pr_launch_active "$pr_number"; then
|
|
1627
|
-
continue
|
|
1628
|
-
fi
|
|
1629
|
-
if ! retry_ready pr "$pr_number"; then
|
|
1630
|
-
continue
|
|
1631
|
-
fi
|
|
1632
|
-
risk_json="$(cached_pr_risk_json "$pr_number")"
|
|
1633
|
-
lane="$(jq -r '.agentLane' <<<"$risk_json")"
|
|
1634
|
-
case "$lane" in
|
|
1635
|
-
double-check-1|double-check-2)
|
|
1636
|
-
count=$((count + 1))
|
|
1637
|
-
;;
|
|
1638
|
-
esac
|
|
1639
|
-
done <<<"$open_agent_pr_ids_cache"
|
|
1640
|
-
printf '%s\n' "$count"
|
|
1641
|
-
}
|
|
1642
|
-
|
|
1643
|
-
next_exclusive_pr_candidate_json() {
|
|
1644
|
-
local target_lane pr_number risk_json lane
|
|
1645
|
-
ensure_exclusive_pr_ids_cache
|
|
1646
|
-
for target_lane in double-check-2 double-check-1 automerge merge-repair fix ci-refresh; do
|
|
1647
|
-
while IFS= read -r pr_number; do
|
|
1648
|
-
[[ -n "$pr_number" ]] || continue
|
|
1649
|
-
if tmux has-session -t "${pr_prefix}${pr_number}" 2>/dev/null; then
|
|
1650
|
-
continue
|
|
1651
|
-
fi
|
|
1652
|
-
if pr_launch_reserved "$pr_number"; then
|
|
1653
|
-
continue
|
|
1654
|
-
fi
|
|
1655
|
-
if pending_pr_launch_active "$pr_number"; then
|
|
1656
|
-
continue
|
|
1657
|
-
fi
|
|
1658
|
-
if ! retry_ready pr "$pr_number"; then
|
|
1659
|
-
continue
|
|
1660
|
-
fi
|
|
1661
|
-
risk_json="$(cached_pr_risk_json "$pr_number")"
|
|
1662
|
-
lane="$(jq -r '.agentLane' <<<"$risk_json")"
|
|
1663
|
-
# Skip PRs requiring human review; they should not hold exclusive lock
|
|
1664
|
-
if [[ "$lane" == "human-review" ]]; then
|
|
1665
|
-
continue
|
|
1666
|
-
fi
|
|
1667
|
-
if [[ "$lane" == "$target_lane" ]]; then
|
|
1668
|
-
printf '%s\n' "$risk_json"
|
|
1669
|
-
return 0
|
|
1670
|
-
fi
|
|
1671
|
-
done <<<"$exclusive_pr_ids_cache"
|
|
1672
|
-
done
|
|
1673
|
-
}
|
|
1674
|
-
|
|
1675
|
-
next_exclusive_issue_id() {
|
|
1676
|
-
local issue_id
|
|
1677
|
-
ensure_exclusive_issue_ids_cache
|
|
1678
|
-
while IFS= read -r issue_id; do
|
|
1679
|
-
[[ -n "$issue_id" ]] || continue
|
|
1680
|
-
if tmux has-session -t "${issue_prefix}${issue_id}" 2>/dev/null; then
|
|
1681
|
-
continue
|
|
1682
|
-
fi
|
|
1683
|
-
if pending_issue_launch_active "$issue_id"; then
|
|
1684
|
-
continue
|
|
1685
|
-
fi
|
|
1686
|
-
if ! retry_ready issue "$issue_id"; then
|
|
1687
|
-
continue
|
|
1688
|
-
fi
|
|
1689
|
-
printf '%s\n' "$issue_id"
|
|
1690
|
-
return 0
|
|
1691
|
-
done <<<"$exclusive_issue_ids_cache"
|
|
1692
|
-
}
|
|
1693
|
-
|
|
1694
|
-
count_pr_lane() {
|
|
1695
|
-
local target_lane="${1:?target lane required}"
|
|
1696
|
-
local pr_number risk_json lane count=0
|
|
1697
|
-
ensure_open_agent_pr_ids_cache
|
|
1698
|
-
while IFS= read -r pr_number; do
|
|
1699
|
-
[[ -n "$pr_number" ]] || continue
|
|
1700
|
-
risk_json="$(cached_pr_risk_json "$pr_number")"
|
|
1701
|
-
lane="$(jq -r '.agentLane' <<<"$risk_json")"
|
|
1702
|
-
if [[ "$lane" == "$target_lane" ]]; then
|
|
1703
|
-
count=$((count + 1))
|
|
1704
|
-
fi
|
|
1705
|
-
done <<<"$open_agent_pr_ids_cache"
|
|
1706
|
-
printf '%s\n' "$count"
|
|
1707
|
-
}
|
|
1708
|
-
|
|
1709
|
-
human_review_pr_ids() {
|
|
1710
|
-
local pr_number risk_json lane
|
|
1711
|
-
ensure_open_agent_pr_ids_cache
|
|
1712
|
-
while IFS= read -r pr_number; do
|
|
1713
|
-
[[ -n "$pr_number" ]] || continue
|
|
1714
|
-
risk_json="$(cached_pr_risk_json "$pr_number")"
|
|
1715
|
-
lane="$(jq -r '.agentLane' <<<"$risk_json")"
|
|
1716
|
-
if [[ "$lane" == "human-review" ]]; then
|
|
1717
|
-
printf '%s\n' "$pr_number"
|
|
1718
|
-
fi
|
|
1719
|
-
done <<<"$open_agent_pr_ids_cache"
|
|
1720
|
-
}
|
|
1721
|
-
|
|
1722
|
-
log_phase "reconcile-completed-workers:start"
|
|
1723
|
-
ensure_completed_workers_cache
|
|
1724
|
-
while IFS= read -r completed_session; do
|
|
1725
|
-
[[ -n "$completed_session" ]] || continue
|
|
1726
|
-
case "$completed_session" in
|
|
1727
|
-
"${issue_prefix}"*)
|
|
1728
|
-
if reconcile_out="$(heartbeat_reconcile_issue "$completed_session" 2>&1)"; then
|
|
1729
|
-
record_memory "reconciled issue worker ${completed_session}"
|
|
1730
|
-
print_block "RECONCILED_SESSION=${completed_session}" "$reconcile_out"
|
|
1731
|
-
else
|
|
1732
|
-
record_memory "failed to reconcile issue worker ${completed_session}"
|
|
1733
|
-
print_block "RECONCILE_FAILED_SESSION=${completed_session}" "$reconcile_out"
|
|
1734
|
-
fi
|
|
1735
|
-
;;
|
|
1736
|
-
"${pr_prefix}"*)
|
|
1737
|
-
if reconcile_out="$(heartbeat_reconcile_pr "$completed_session" 2>&1)"; then
|
|
1738
|
-
record_memory "reconciled PR worker ${completed_session}"
|
|
1739
|
-
print_block "RECONCILED_SESSION=${completed_session}" "$reconcile_out"
|
|
1740
|
-
else
|
|
1741
|
-
completed_pr_number="${completed_session#${pr_prefix}}"
|
|
1742
|
-
if [[ -n "$completed_pr_number" ]]; then
|
|
1743
|
-
heartbeat_clear_pr_running "$completed_pr_number" >/dev/null || true
|
|
1744
|
-
heartbeat_sync_pr_labels "$completed_pr_number" >/dev/null || true
|
|
1745
|
-
fi
|
|
1746
|
-
record_memory "failed to reconcile PR worker ${completed_session}"
|
|
1747
|
-
print_block "RECONCILE_FAILED_SESSION=${completed_session}" "$reconcile_out"
|
|
1748
|
-
fi
|
|
1749
|
-
;;
|
|
1750
|
-
*)
|
|
1751
|
-
echo "unknown completed worker session: ${completed_session}" >&2
|
|
1752
|
-
exit 1
|
|
1753
|
-
;;
|
|
1754
|
-
esac
|
|
1755
|
-
done <<<"$completed_workers_cache"
|
|
1756
|
-
log_phase "reconcile-completed-workers:end"
|
|
1757
|
-
|
|
1758
|
-
log_phase "sync-open-agent-issues:start"
|
|
1759
|
-
sync_open_agent_issues
|
|
1760
|
-
log_phase "sync-open-agent-issues:end"
|
|
1761
|
-
log_phase "sync-open-agent-prs:start"
|
|
1762
|
-
sync_open_agent_prs
|
|
1763
|
-
log_phase "sync-open-agent-prs:end"
|
|
476
|
+
log_phase "sync-open-agent-issues:start"
|
|
477
|
+
sync_open_agent_issues
|
|
478
|
+
log_phase "sync-open-agent-issues:end"
|
|
479
|
+
log_phase "sync-open-agent-prs:start"
|
|
480
|
+
sync_open_agent_prs
|
|
481
|
+
log_phase "sync-open-agent-prs:end"
|
|
1764
482
|
|
|
1765
483
|
log_phase "snapshot-running-counts:start"
|
|
1766
484
|
running_workers_now="$(all_running_workers)"
|
|
@@ -1857,76 +575,6 @@ if provider_cooldown_out="$(provider_cooldown_state 2>/dev/null || true)"; then
|
|
|
1857
575
|
fi
|
|
1858
576
|
fi
|
|
1859
577
|
|
|
1860
|
-
launch_pr_candidate_json() {
|
|
1861
|
-
local pr_candidate_json="${1:?pr candidate json required}"
|
|
1862
|
-
local pr_number pr_lane launch_out
|
|
1863
|
-
|
|
1864
|
-
pr_number="$(jq -r '.number' <<<"$pr_candidate_json")"
|
|
1865
|
-
pr_lane="$(jq -r '.agentLane' <<<"$pr_candidate_json")"
|
|
1866
|
-
stage_pr_launch "$pr_number"
|
|
1867
|
-
|
|
1868
|
-
case "$pr_lane" in
|
|
1869
|
-
double-check-1|double-check-2|automerge)
|
|
1870
|
-
if ! launch_out="$(heartbeat_start_pr_review_worker "$pr_number" 2>&1)"; then
|
|
1871
|
-
heartbeat_clear_pr_running "$pr_number" || true
|
|
1872
|
-
clear_launch_in_progress
|
|
1873
|
-
record_memory "failed to launch PR review worker for #${pr_number}"
|
|
1874
|
-
print_block "LAUNCH_FAILED_PR=${pr_number}" "$launch_out"
|
|
1875
|
-
exit 1
|
|
1876
|
-
fi
|
|
1877
|
-
record_memory "launched PR review worker for #${pr_number}"
|
|
1878
|
-
;;
|
|
1879
|
-
merge-repair)
|
|
1880
|
-
if ! launch_out="$(heartbeat_start_pr_merge_repair_worker "$pr_number" 2>&1)"; then
|
|
1881
|
-
heartbeat_clear_pr_running "$pr_number" || true
|
|
1882
|
-
clear_launch_in_progress
|
|
1883
|
-
record_memory "failed to launch PR merge-repair worker for #${pr_number}"
|
|
1884
|
-
print_block "LAUNCH_FAILED_PR=${pr_number}" "$launch_out"
|
|
1885
|
-
exit 1
|
|
1886
|
-
fi
|
|
1887
|
-
record_memory "launched PR merge-repair worker for #${pr_number}"
|
|
1888
|
-
;;
|
|
1889
|
-
ci-refresh)
|
|
1890
|
-
if ! launch_out="$(heartbeat_start_pr_ci_refresh "$pr_number" 2>&1)"; then
|
|
1891
|
-
heartbeat_clear_pr_running "$pr_number" || true
|
|
1892
|
-
clear_launch_in_progress
|
|
1893
|
-
record_memory "failed to trigger PR ci-refresh for #${pr_number}"
|
|
1894
|
-
print_block "LAUNCH_FAILED_PR=${pr_number}" "$launch_out"
|
|
1895
|
-
exit 1
|
|
1896
|
-
fi
|
|
1897
|
-
heartbeat_clear_pr_running "$pr_number" || true
|
|
1898
|
-
record_memory "triggered PR ci-refresh for #${pr_number}"
|
|
1899
|
-
;;
|
|
1900
|
-
fix)
|
|
1901
|
-
if ! launch_out="$(heartbeat_start_pr_fix_worker "$pr_number" 2>&1)"; then
|
|
1902
|
-
heartbeat_clear_pr_running "$pr_number" || true
|
|
1903
|
-
clear_launch_in_progress
|
|
1904
|
-
record_memory "failed to launch PR fix worker for #${pr_number}"
|
|
1905
|
-
print_block "LAUNCH_FAILED_PR=${pr_number}" "$launch_out"
|
|
1906
|
-
exit 1
|
|
1907
|
-
fi
|
|
1908
|
-
record_memory "launched PR fix worker for #${pr_number}"
|
|
1909
|
-
;;
|
|
1910
|
-
*)
|
|
1911
|
-
launch_out="Unsupported PR lane: ${pr_lane}"
|
|
1912
|
-
heartbeat_clear_pr_running "$pr_number" || true
|
|
1913
|
-
clear_launch_in_progress
|
|
1914
|
-
print_block "LAUNCH_FAILED_PR=${pr_number}" "$launch_out"
|
|
1915
|
-
exit 1
|
|
1916
|
-
;;
|
|
1917
|
-
esac
|
|
1918
|
-
|
|
1919
|
-
clear_launch_in_progress
|
|
1920
|
-
print_block "LAUNCHED_PR=${pr_number}" "$(printf 'LANE=%s\n%s' "$pr_lane" "$launch_out")"
|
|
1921
|
-
if [[ "$pr_lane" != "ci-refresh" ]]; then
|
|
1922
|
-
running_total_count=$((running_total_count + 1))
|
|
1923
|
-
running_pr_count=$((running_pr_count + 1))
|
|
1924
|
-
fi
|
|
1925
|
-
launched_pr_count=$((launched_pr_count + 1))
|
|
1926
|
-
if (( launch_budget_remaining > 0 )); then
|
|
1927
|
-
launch_budget_remaining=$((launch_budget_remaining - 1))
|
|
1928
|
-
fi
|
|
1929
|
-
}
|
|
1930
578
|
|
|
1931
579
|
if [[ "$exclusive_lock_mode" == "pending" && "$exclusive_lock_kind" == "pr" ]]; then
|
|
1932
580
|
pr_number="$exclusive_lock_item"
|