agent-control-plane 0.1.13 → 0.1.16

package/tools/bin/agent-project-reconcile-pr-session

@@ -227,23 +227,30 @@ schedule_provider_quota_cooldown() {
   local reason="${1:-provider-quota-limit}"
   [[ "${failure_reason:-}" == "provider-quota-limit" ]] || return 0
   [[ -x "${provider_cooldown_script}" ]] || return 0
+  [[ "${CODING_WORKER:-}" == "codex" ]] && return 0
 
   "${provider_cooldown_script}" schedule "${reason}" >/dev/null || true
 }
 
 clear_provider_quota_cooldown() {
   [[ -x "${provider_cooldown_script}" ]] || return 0
+  [[ "${CODING_WORKER:-}" == "codex" ]] && return 0
 
   "${provider_cooldown_script}" clear >/dev/null || true
 }
 
 blocked_runtime_reason=""
+host_github_rate_limited="no"
+host_github_rate_limit_detail=""
 
 owner="${repo_slug%%/*}"
 repo="${repo_slug#*/}"
 pr_view_json="$(flow_github_pr_view_json "$repo_slug" "$pr_number")"
 pr_state="$(jq -r '.state' <<<"$pr_view_json")"
-pr_base_ref="$(jq -r '.baseRefName // "main"' <<<"$pr_view_json")"
+pr_base_ref="$(jq -r '.baseRefName // empty' <<<"$pr_view_json")"
+if [[ -z "${pr_base_ref}" ]]; then
+  pr_base_ref="main"
+fi
 
 if [[ "$status" == "RUNNING" && "$pr_state" != "MERGED" && "$pr_state" != "CLOSED" ]]; then
   printf 'STATUS=%s\n' "$status"
@@ -395,7 +402,9 @@ post_pr_comment_if_present() {
   if pr_comment_already_posted; then
     return 0
   fi
-  flow_github_api_repo "${repo_slug}" "issues/${pr_number}/comments" --method POST -f body="$(cat "$comment_file")" >/dev/null
+  if ! host_github_post_issue_comment "${pr_number}" "$(cat "$comment_file")"; then
+    return 1
+  fi
 }
 
 pr_comment_already_posted() {
@@ -408,6 +417,120 @@ pr_comment_already_posted() {
   jq -e --arg body "$comment_body" 'any(.comments[]?; .body == $body)' >/dev/null <<<"$comments_json"
 }
 
+host_github_output_indicates_rate_limit() {
+  grep -Eiq 'API rate limit exceeded|secondary rate limit|rate limit exceeded|HTTP 403' <<<"${1:-}"
+}
+
+record_host_github_rate_limit() {
+  local output="${1:-}"
+  local detail_file="${run_dir}/host-github-rate-limit.log"
+  host_github_rate_limited="yes"
+  host_github_rate_limit_detail="${output}"
+  printf '%s\n' "${output}" >"${detail_file}"
+}
+
+host_github_post_issue_comment() {
+  local issue_number="${1:?issue number required}"
+  local body="${2:-}"
+  local output=""
+
+  flow_export_github_cli_auth_env "${repo_slug}"
+  if output="$(
+    gh api "repos/${repo_slug}/issues/${issue_number}/comments" \
+      --method POST \
+      -f body="${body}" 2>&1
+  )"; then
+    return 0
+  fi
+
+  if host_github_output_indicates_rate_limit "${output}"; then
+    record_host_github_rate_limit "${output}"
+    return 1
+  fi
+
+  printf '%s\n' "${output}" >&2
+  return 1
+}
+
+host_github_submit_pr_approval() {
+  local output=""
+
+  flow_export_github_cli_auth_env "${repo_slug}"
+  if output="$(
+    gh api "repos/${repo_slug}/pulls/${pr_number}/reviews" \
+      --method POST \
+      -f event=APPROVE \
+      -f body="Automated final review passed. Safe low-risk scope, green checks, and host-side merge approved." \
+      2>&1
+  )"; then
+    return 0
+  fi
+
+  if grep -q "Can not approve your own pull request" <<<"${output}"; then
+    return 0
+  fi
+
+  if host_github_output_indicates_rate_limit "${output}"; then
+    record_host_github_rate_limit "${output}"
+    return 1
+  fi
+
+  printf '%s\n' "${output}" >&2
+  return 1
+}
+
+append_host_rate_limit_comment() {
+  local detail="${1:-GitHub API rate limit blocked host actions.}"
+  local reset_line=""
+
+  if grep -Eiq 'resets at ' <<<"${detail}"; then
+    reset_line="$(grep -Eio 'resets at [^.]+' <<<"${detail}" | head -n 1 || true)"
+  fi
+
+  {
+    if [[ -s "${pr_comment_file}" ]]; then
+      printf '\n\n'
+    fi
+    printf '## Host action blocked\n\n'
+    printf 'GitHub API rate limit blocked ACP from posting the PR review outcome or merge action.\n'
+    if [[ -n "${reset_line}" ]]; then
+      printf '\n- %s\n' "${reset_line}"
+    fi
+    printf -- '- ACP kept the local review artifacts and scheduled an automatic retry for the host action.\n'
+  } >>"${pr_comment_file}"
+}
+
+handle_host_github_rate_limit_retry() {
+  local reason="${1:-github-api-rate-limit}"
+  local result_action_override="${2:-host-rate-limit-retry}"
+
+  append_host_rate_limit_comment "${host_github_rate_limit_detail:-}"
+  require_transition "pr_schedule_retry" pr_schedule_retry "${reason}"
+  require_transition "pr_after_blocked" pr_after_blocked "${pr_number}"
+  cleanup_pr_session
+  result_outcome="blocked"
+  result_action="${result_action_override}"
+  failure_reason="${reason}"
+  notify_pr_reconciled
+  mark_reconciled
+  printf 'STATUS=FAILED\n'
+  printf 'PR_NUMBER=%s\n' "${pr_number}"
+  printf 'PR_STATE=%s\n' "${pr_state}"
+  printf 'OUTCOME=%s\n' "${result_outcome}"
+  printf 'ACTION=%s\n' "${result_action}"
+  printf 'FAILURE_REASON=%s\n' "${failure_reason}"
+  exit 0
+}
+
+maybe_handle_host_github_rate_limit() {
+  local reason="${1:-github-api-rate-limit}"
+  local result_action_override="${2:-host-rate-limit-retry}"
+  if [[ "${host_github_rate_limited}" == "yes" ]]; then
+    handle_host_github_rate_limit_retry "${reason}" "${result_action_override}"
+  fi
+  return 1
+}
+
 blocked_result_indicates_local_bind_failure() {
   local candidate_file
   for candidate_file in "$pr_comment_file" "$session_log_file"; do
@@ -430,6 +553,31 @@ classify_pr_blocked_runtime_reason() {
     return 0
   fi
 
+  if [[ -f "$session_log_file" ]] && grep -Eiq 'no-codex-output-before-stall-threshold|no-codex-progress-before-stall-threshold' "$session_log_file" 2>/dev/null; then
+    printf 'codex-stalled\n'
+    return 0
+  fi
+
+  if [[ -f "$session_log_file" ]] && grep -Eiq 'no-agent-output-before-stall-threshold|no-agent-progress-before-stall-threshold' "$session_log_file" 2>/dev/null; then
+    printf 'agent-stalled\n'
+    return 0
+  fi
+
+  if [[ -f "$session_log_file" ]] && grep -Eiq 'provider-quota-limit|quota.*exhausted|rate.limit.*exceeded' "$session_log_file" 2>/dev/null; then
+    printf 'provider-quota-limit\n'
+    return 0
+  fi
+
+  if [[ -f "$pr_comment_file" ]] && grep -Eiq 'no-codex-output-before-stall-threshold|no-codex-progress-before-stall-threshold' "$pr_comment_file" 2>/dev/null; then
+    printf 'codex-stalled\n'
+    return 0
+  fi
+
+  if [[ -f "$pr_comment_file" ]] && grep -Eiq 'no-agent-output-before-stall-threshold|no-agent-progress-before-stall-threshold' "$pr_comment_file" 2>/dev/null; then
+    printf 'agent-stalled\n'
+    return 0
+  fi
+
   return 1
 }
 
@@ -774,23 +922,38 @@ pr_remote_already_has_final_head() {
 }
 
 approve_and_merge() {
-  local approve_output
   if ! pr_is_self_authored_for_current_actor; then
-    if ! approve_output="$(
-      flow_github_api_repo "${repo_slug}" "pulls/${pr_number}/reviews" \
-        --method POST \
-        -f event=APPROVE \
-        -f body="Automated final review passed. Safe low-risk scope, green checks, and host-side merge approved." \
-        2>&1
-    )"; then
-      if ! grep -q "Can not approve your own pull request" <<<"$approve_output"; then
-        printf '%s\n' "$approve_output" >&2
-        return 1
+    if ! host_github_submit_pr_approval; then
+      if [[ "${host_github_rate_limited}" == "yes" ]]; then
+        return 2
       fi
+      return 1
     fi
   fi
 
-  flow_github_pr_merge "$repo_slug" "$pr_number" "squash" "yes"
+  flow_export_github_cli_auth_env "${repo_slug}"
+  if ! gh pr merge "${pr_number}" -R "${repo_slug}" --squash --delete-branch --admin >"${run_dir}/host-github-merge.out" 2>"${run_dir}/host-github-merge.err"; then
+    local merge_output=""
+    merge_output="$(cat "${run_dir}/host-github-merge.err" 2>/dev/null || true)"
+    if host_github_output_indicates_rate_limit "${merge_output}"; then
+      record_host_github_rate_limit "${merge_output}"
+      return 2
+    fi
+    if flow_github_pr_merge "$repo_slug" "$pr_number" "squash" "yes" 2>"${run_dir}/host-github-merge.err"; then
+      return 0
+    fi
+    merge_output="$(cat "${run_dir}/host-github-merge.err" 2>/dev/null || true)"
+    if host_github_output_indicates_rate_limit "${merge_output}"; then
+      record_host_github_rate_limit "${merge_output}"
+      return 2
+    fi
+    if [[ -n "${merge_output}" ]]; then
+      printf '%s\n' "${merge_output}" >&2
+    fi
+    return 1
+  fi
+
+  return 0
 }
 
 cleanup_pr_session() {
@@ -829,7 +992,7 @@ handle_linked_issue_merge_cleanup() {
 handle_updated_branch_result() {
   if [[ -z "$pr_worktree" || ! -d "$pr_worktree" ]]; then
     if pr_remote_already_has_final_head; then
-      post_pr_comment_if_present
+      post_pr_comment_if_present || maybe_handle_host_github_rate_limit "github-api-rate-limit" "host-comment-rate-limit-retry"
       require_transition "pr_clear_retry" pr_clear_retry
       require_transition "pr_after_updated_branch" pr_after_updated_branch "$pr_number"
       cleanup_pr_session
@@ -895,7 +1058,7 @@ handle_updated_branch_result() {
   fi
 
   push_pr_branch
-  post_pr_comment_if_present
+  post_pr_comment_if_present || maybe_handle_host_github_rate_limit "github-api-rate-limit" "host-comment-rate-limit-retry"
   require_transition "pr_clear_retry" pr_clear_retry
   require_transition "pr_after_updated_branch" pr_after_updated_branch "$pr_number"
   cleanup_pr_session
@@ -936,7 +1099,7 @@ elif [[ "$pr_state" == "CLOSED" ]]; then
   result_action="${result_action:-cleaned-closed-pr}"
   notify_pr_reconciled
 elif [[ "$status" == "SUCCEEDED" && "$result_outcome" == "approved-local-review-passed" ]]; then
-  if ! review_pass_action="$(review_pass_action_from_result_action "${result_action:-}" 2>/dev/null)"; then
+      if ! review_pass_action="$(review_pass_action_from_result_action "${result_action:-}" 2>/dev/null)"; then
     review_pass_action="$(pr_review_pass_action "$pr_number")"
   fi
   case "$review_pass_action" in
@@ -963,8 +1126,13 @@ elif [[ "$status" == "SUCCEEDED" && "$result_outcome" == "approved-local-review-
       fi
 
       require_transition "pr_clear_retry" pr_clear_retry
-      approve_and_merge
-      pr_state="$(flow_github_pr_view_json "$repo_slug" "$pr_number" | jq -r '.state')"
+      if ! approve_and_merge; then
+        if [[ "${host_github_rate_limited}" == "yes" ]]; then
+          handle_host_github_rate_limit_retry "github-api-rate-limit" "host-merge-rate-limit-retry"
+        fi
+        exit 1
+      fi
+      pr_state="MERGED"
       if [[ "$pr_state" != "MERGED" ]]; then
         echo "PR ${pr_number} did not merge successfully" >&2
         exit 1
@@ -1018,7 +1186,7 @@ elif [[ "$status" == "SUCCEEDED" && "$result_outcome" == "no-change-needed" ]];
           result_action="host-rejected-noop-promotion"
         else
           push_pr_branch
-          post_pr_comment_if_present
+          post_pr_comment_if_present || maybe_handle_host_github_rate_limit "github-api-rate-limit" "host-comment-rate-limit-retry"
           require_transition "pr_clear_retry" pr_clear_retry
           require_transition "pr_after_updated_branch" pr_after_updated_branch "$pr_number"
           cleanup_pr_session
@@ -1041,7 +1209,7 @@ elif [[ "$status" == "SUCCEEDED" && "$result_outcome" == "no-change-needed" ]];
       result_action="host-rejected-no-change-needed"
       notify_pr_reconciled
     else
-      post_pr_comment_if_present
+      post_pr_comment_if_present || maybe_handle_host_github_rate_limit "github-api-rate-limit" "host-comment-rate-limit-retry"
       require_transition "pr_clear_retry" pr_clear_retry
       require_transition "pr_after_succeeded" pr_after_succeeded "$pr_number"
       cleanup_pr_session
@@ -1062,7 +1230,7 @@ elif [[ "$status" == "SUCCEEDED" && "$result_outcome" == "blocked" ]]; then
   elif attempt_blocked_pr_host_verification_recovery; then
     handle_updated_branch_result
   else
-    post_pr_comment_if_present
+    post_pr_comment_if_present || maybe_handle_host_github_rate_limit "github-api-rate-limit" "host-comment-rate-limit-retry"
     require_transition "pr_clear_retry" pr_clear_retry
     require_transition "pr_after_blocked" pr_after_blocked "$pr_number"
     cleanup_pr_session
@@ -1079,6 +1247,7 @@ elif [[ "$status" == "FAILED" ]]; then
   schedule_provider_quota_cooldown "${failure_reason:-worker-exit-failed}"
   require_transition "pr_schedule_retry" pr_schedule_retry "${failure_reason:-worker-exit-failed}"
   require_transition "pr_after_failed" pr_after_failed "$pr_number"
+  cleanup_pr_session
   notify_pr_reconciled
 fi
 

package/tools/bin/agent-project-run-codex-resilient

@@ -618,6 +618,22 @@ classify_failure_reason() {
   fi
 }
 
+failure_chunk_indicates_startup_stall() {
+  local chunk="${1:-}"
+  local recent_chunk
+
+  recent_chunk="$(tail -n 120 <<<"$chunk")"
+  grep -q '"type":"thread.started"' <<<"$recent_chunk" || return 1
+  grep -q '"type":"turn.started"' <<<"$recent_chunk" || return 1
+  if grep -Eq '"type":"item\.(started|completed)"' <<<"$recent_chunk"; then
+    return 1
+  fi
+  if grep -q '"type":"turn.completed"' <<<"$recent_chunk"; then
+    return 1
+  fi
+  return 0
+}
+
 resume_prompt() {
   cat <<EOF
 The previous Codex exec turn in this same thread was interrupted because the host refreshed Codex authentication after a quota or auth failure.
@@ -729,7 +745,7 @@ run_resume_exec() {
 }
 
 attempt_run() {
-  local reason auth_before_switch quota_label_before_switch quota_switch_signature_before_switch quota_switch_result shell_flags_before_quota_switch
+  local reason auth_before_switch quota_label_before_switch quota_switch_signature_before_switch quota_switch_result shell_flags_before_quota_switch failure_chunk startup_stall
 
   attempt=$((attempt + 1))
   last_quota_switch_status=""
@@ -750,8 +766,15 @@ attempt_run() {
     return 0
   fi
 
-  reason="$(classify_failure_reason "$(new_output_since "$last_attempt_start_size")")"
+  failure_chunk="$(new_output_since "$last_attempt_start_size")"
+  reason="$(classify_failure_reason "$failure_chunk")"
   last_failure_reason="${reason:-worker-exit-failed}"
+  startup_stall="no"
+  if [[ "$last_failure_reason" == "no-codex-output-before-stall-threshold" || "$last_failure_reason" == "no-codex-progress-before-stall-threshold" ]]; then
+    if failure_chunk_indicates_startup_stall "$failure_chunk"; then
+      startup_stall="yes"
+    fi
+  fi
 
   case "$last_failure_reason" in
     usage-limit|auth-failure|auth-401|account-banned)
@@ -796,6 +819,38 @@ attempt_run() {
       resume_count=$((resume_count + 1))
       return 2
       ;;
+    no-codex-output-before-stall-threshold|no-codex-progress-before-stall-threshold)
+      if [[ "$startup_stall" == "yes" && $quota_autoswitch_attempt_count -lt $max_quota_autoswitch_attempts ]]; then
+        auth_before_switch="$(auth_fingerprint)"
+        quota_label_before_switch="$last_attempt_start_quota_label"
+        quota_switch_signature_before_switch="$(quota_switch_signature)"
+        last_auth_fingerprint="$auth_before_switch"
+        write_state "switching-account" "$last_failure_reason"
+        log_runner "startup-stall detected before first Codex tool activity; attempting Codex account rotation"
+        shell_flags_before_quota_switch="$-"
+        set +e
+        run_quota_autoswitch
+        quota_switch_result=$?
+        case "$shell_flags_before_quota_switch" in
+          *e*) set -e ;;
+          *) set +e ;;
+        esac
+        if [[ "$quota_switch_result" == "0" ]]; then
+          thread_id=""
+          auth_wait_started_at=""
+          write_state "running" ""
+          return 2
+        fi
+        if [[ "$quota_switch_result" == "10" ]]; then
+          log_runner "startup-stall rotation deferred until ${last_quota_next_retry_at:-unknown}"
+          last_failure_reason="quota-switch-deferred"
+          write_state "failed" "$last_failure_reason"
+          return 1
+        fi
+      fi
+      write_state "failed" "$last_failure_reason"
+      return 1
+      ;;
     *)
       write_state "failed" "$last_failure_reason"
       return 1

package/tools/bin/agent-project-run-openclaw-session

@@ -115,6 +115,7 @@ result_file="${artifact_dir}/result.env"
 runner_state_file="${artifact_dir}/runner.env"
 sandbox_artifact_dir="${worktree%/}/${sandbox_subdir}"
 sandbox_run_dir="${worktree%/}/${sandbox_subdir}/${session}"
+retained_repo_root="${ACP_RETAINED_REPO_ROOT:-${F_LOSNING_RETAINED_REPO_ROOT:-}}"
 started_at="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
 openclaw_bin="$(command -v openclaw)"
 default_openclaw_agent_id="$(
@@ -154,6 +155,7 @@ printf -v runner_state_q '%q' "$runner_state_file"
 printf -v branch_q '%q' "$branch_name"
 printf -v sandbox_artifact_dir_q '%q' "$sandbox_artifact_dir"
 printf -v sandbox_run_dir_q '%q' "$sandbox_run_dir"
+printf -v retained_repo_root_q '%q' "$retained_repo_root"
 printf -v adapter_id_q '%q' "$adapter_id"
 printf -v started_at_q '%q' "$started_at"
 printf -v openclaw_bin_q '%q' "$openclaw_bin"
@@ -230,18 +232,21 @@ export AGENT_PROJECT_RUN_DIR=${sandbox_run_dir_q}
 export AGENT_PROJECT_HOST_RUN_DIR=${artifact_dir_q}
 export AGENT_PROJECT_RESULT_FILE=${sandbox_run_dir_q}/result.env
 export AGENT_PROJECT_OPENCLAW_BIN=${openclaw_bin_q}
+export AGENT_PROJECT_RETAINED_REPO_ROOT=${retained_repo_root_q}
 export ACP_SESSION=${session_q}
 export ACP_RUN_DIR=${sandbox_run_dir_q}
 export ACP_HOST_RUN_DIR=${artifact_dir_q}
 export ACP_RESULT_FILE=${sandbox_run_dir_q}/result.env
 export ACP_OPENCLAW_BIN=${openclaw_bin_q}
 export ACP_OPENCLAW_SESSION_ID=${openclaw_session_id_q}
+export ACP_RETAINED_REPO_ROOT=${retained_repo_root_q}
 export F_LOSNING_SESSION=${session_q}
 export F_LOSNING_RUN_DIR=${sandbox_run_dir_q}
 export F_LOSNING_HOST_RUN_DIR=${artifact_dir_q}
 export F_LOSNING_RESULT_FILE=${sandbox_run_dir_q}/result.env
 export F_LOSNING_OPENCLAW_BIN=${openclaw_bin_q}
 export F_LOSNING_OPENCLAW_SESSION_ID=${openclaw_session_id_q}
+export F_LOSNING_RETAINED_REPO_ROOT=${retained_repo_root_q}
 export OPENCLAW_STATE_DIR=${openclaw_state_dir_q}
 export OPENCLAW_CONFIG_PATH=${openclaw_config_path_q}
 EOF
@@ -294,6 +299,7 @@ runner_state_file=${runner_state_q}
 output_file=${output_q}
 sandbox_artifact_dir=${sandbox_artifact_dir_q}
 sandbox_run_dir=${sandbox_run_dir_q}
+retained_repo_root=${retained_repo_root_q}
 artifact_dir=${artifact_dir_q}
 run_dir=${artifact_dir_q}
 task_kind=${task_kind_q}
@@ -554,6 +560,45 @@ recover_literal_runtime_artifacts() {
   return 0
 }
 
+recover_retained_repo_artifact_leaks() {
+  local retained_worktree_root=""
+  local leaked_run_dir=""
+  local worktree_name=""
+  local session_name=""
+  local artifact_name=""
+  local recovered="no"
+
+  [[ -n "\${retained_repo_root}" ]] || return 0
+  worktree_name="\$(basename "\${worktree}")"
+  session_name="\${AGENT_PROJECT_SESSION:-}"
+  [[ -n "\${session_name}" ]] || return 0
+  retained_worktree_root="\${retained_repo_root%/}/worktrees"
+  leaked_run_dir="\${retained_worktree_root}/\${worktree_name}/.openclaw-artifacts/\${session_name}"
+
+  if [[ ! -d "\${leaked_run_dir}" || "\${leaked_run_dir}" == "\${sandbox_run_dir}" ]]; then
+    return 0
+  fi
+
+  for artifact_name in result.env verification.jsonl issue-comment.md pr-comment.md; do
+    if [[ -f "\${leaked_run_dir}/\${artifact_name}" ]]; then
+      cp "\${leaked_run_dir}/\${artifact_name}" "\${sandbox_run_dir}/\${artifact_name}" 2>/dev/null || true
+      cp "\${leaked_run_dir}/\${artifact_name}" "\${artifact_dir}/\${artifact_name}" 2>/dev/null || true
+      recovered="yes"
+    fi
+  done
+
+  rm -rf "\${leaked_run_dir}" 2>/dev/null || true
+  rmdir "\${retained_worktree_root}/\${worktree_name}/.openclaw-artifacts" 2>/dev/null || true
+  rmdir "\${retained_worktree_root}/\${worktree_name}" 2>/dev/null || true
+  rmdir "\${retained_worktree_root}" 2>/dev/null || true
+
+  if [[ "\${recovered}" == "yes" ]]; then
+    printf '[openclaw] recovered retained-repo artifact leak: %s\n' "\${leaked_run_dir}" >>"\${output_file}" 2>/dev/null || true
+  fi
+
+  return 0
+}
+
 reset_sandbox_run_dir() {
   mkdir -p "\${sandbox_run_dir}"
   find "\${sandbox_run_dir}" -mindepth 1 -maxdepth 1 -exec rm -rf {} + 2>/dev/null || true
@@ -1217,6 +1262,7 @@ while true; do
   break
 done
 recover_literal_runtime_artifacts
+recover_retained_repo_artifact_leaks
 infer_result_from_output
 synthesize_comment_artifact_from_output
 if [[ "\${status}" -eq 0 ]]; then

package/tools/bin/agent-project-worker-status

@@ -44,6 +44,36 @@ runner_state=""
 thread_id=""
 last_exit_code=""
 
+failure_reason_from_output() {
+  [[ -f "$output_file" ]] || return 1
+
+  if rg -qi "You've hit your usage limit|You have reached your Codex usage limits|visit https://chatgpt.com/codex/settings/usage|Upgrade to Pro|rate limit exceeded|quota exceeded|usage cap (reached|exceeded)|usage quota (reached|exceeded)" "$output_file"; then
+    printf 'usage-limit\n'
+    return 0
+  fi
+
+  if rg -qi 'stale-run no-codex-output-before-stall-threshold|no-codex-output-before-stall-threshold' "$output_file"; then
+    printf 'no-codex-output-before-stall-threshold\n'
+    return 0
+  fi
+
+  if rg -qi 'stale-run no-codex-progress-before-stall-threshold|no-codex-progress-before-stall-threshold' "$output_file"; then
+    printf 'no-codex-progress-before-stall-threshold\n'
+    return 0
+  fi
+
+  # Recover Codex startup stalls when the wrapper was archived before it could
+  # flush a terminal runner.env state. This is intentionally narrow: the log
+  # must show a turn started, but no tool activity or turn completion.
+  if rg -q '"type":"turn.started"' "$output_file" \
+    && ! rg -q '"type":"item.started"|"type":"item.completed"|"type":"turn.completed"' "$output_file"; then
+    printf 'no-codex-progress-before-stall-threshold\n'
+    return 0
+  fi
+
+  return 1
+}
+
 if tmux has-session -t "$session" 2>/dev/null; then
   status="RUNNING"
 fi
@@ -87,6 +117,13 @@ if [[ "$status" == "UNKNOWN" && -f "$output_file" ]]; then
   fi
 fi
 
+if [[ "$status" == "UNKNOWN" && -z "$failure_reason" ]]; then
+  failure_reason="$(failure_reason_from_output || true)"
+  if [[ -n "$failure_reason" ]]; then
+    status="FAILED"
+  fi
+fi
+
 if [[ "$status" == "UNKNOWN" && -n "$runner_state" ]]; then
   case "$runner_state" in
     running|waiting-auth-refresh|switching-account)

package/tools/bin/flow-config-lib.sh

@@ -2039,6 +2039,7 @@ flow_export_execution_env() {
   local openclaw_model=""
   local openclaw_thinking=""
   local openclaw_timeout=""
+  local openclaw_stall=""
 
   repo_id="$(flow_resolve_repo_id "${config_file}")"
   provider_quota_cooldowns="$(flow_resolve_provider_quota_cooldowns "${config_file}")"
@@ -2071,6 +2072,7 @@ flow_export_execution_env() {
     openclaw_model="$(flow_kv_get "${provider_pool_selection}" "OPENCLAW_MODEL")"
     openclaw_thinking="$(flow_kv_get "${provider_pool_selection}" "OPENCLAW_THINKING")"
     openclaw_timeout="$(flow_kv_get "${provider_pool_selection}" "OPENCLAW_TIMEOUT_SECONDS")"
+    openclaw_stall="$(flow_kv_get "${provider_pool_selection}" "OPENCLAW_STALL_SECONDS")"
   else
     if [[ -n "${explicit_coding_worker}" ]]; then
       active_provider_selection_reason="env-override"
@@ -2087,6 +2089,7 @@ flow_export_execution_env() {
     openclaw_model="$(flow_env_or_config "${config_file}" "ACP_OPENCLAW_MODEL F_LOSNING_OPENCLAW_MODEL" "execution.openclaw.model" "")"
     openclaw_thinking="$(flow_env_or_config "${config_file}" "ACP_OPENCLAW_THINKING F_LOSNING_OPENCLAW_THINKING" "execution.openclaw.thinking" "")"
     openclaw_timeout="$(flow_env_or_config "${config_file}" "ACP_OPENCLAW_TIMEOUT_SECONDS F_LOSNING_OPENCLAW_TIMEOUT_SECONDS" "execution.openclaw.timeout_seconds" "")"
+    openclaw_stall="$(flow_env_or_config "${config_file}" "ACP_OPENCLAW_STALL_SECONDS F_LOSNING_OPENCLAW_STALL_SECONDS" "execution.openclaw.stall_seconds" "")"
   fi
 
   if [[ -n "${coding_worker}" ]]; then
@@ -2167,6 +2170,10 @@ flow_export_execution_env() {
     export F_LOSNING_OPENCLAW_TIMEOUT_SECONDS="${openclaw_timeout}"
     export ACP_OPENCLAW_TIMEOUT_SECONDS="${openclaw_timeout}"
   fi
+  if [[ -n "${openclaw_stall}" ]]; then
+    export F_LOSNING_OPENCLAW_STALL_SECONDS="${openclaw_stall}"
+    export ACP_OPENCLAW_STALL_SECONDS="${openclaw_stall}"
+  fi
 
   flow_export_github_cli_auth_env "$(flow_resolve_repo_slug "${config_file}")"
   flow_export_project_env_aliases

package/tools/bin/flow-shell-lib.sh

@@ -52,6 +52,7 @@ flow_export_compat_env_aliases() {
   flow_export_env_alias_if_unset F_LOSNING_OPENCLAW_MODEL ACP_OPENCLAW_MODEL
   flow_export_env_alias_if_unset F_LOSNING_OPENCLAW_THINKING ACP_OPENCLAW_THINKING
   flow_export_env_alias_if_unset F_LOSNING_OPENCLAW_TIMEOUT_SECONDS ACP_OPENCLAW_TIMEOUT_SECONDS
+  flow_export_env_alias_if_unset F_LOSNING_OPENCLAW_STALL_SECONDS ACP_OPENCLAW_STALL_SECONDS
   flow_export_env_alias_if_unset F_LOSNING_ALLOW_INFRA_CI_BYPASS ACP_ALLOW_INFRA_CI_BYPASS
   flow_export_env_alias_if_unset F_LOSNING_LOCAL_FIRST_PR_POLICY ACP_LOCAL_FIRST_PR_POLICY
   flow_export_env_alias_if_unset F_LOSNING_PR_RISK_CACHE_TTL_SECONDS ACP_PR_RISK_CACHE_TTL_SECONDS
@@ -100,6 +101,7 @@ flow_export_canonical_env_aliases() {
   flow_export_env_alias_if_unset ACP_OPENCLAW_MODEL F_LOSNING_OPENCLAW_MODEL
   flow_export_env_alias_if_unset ACP_OPENCLAW_THINKING F_LOSNING_OPENCLAW_THINKING
   flow_export_env_alias_if_unset ACP_OPENCLAW_TIMEOUT_SECONDS F_LOSNING_OPENCLAW_TIMEOUT_SECONDS
+  flow_export_env_alias_if_unset ACP_OPENCLAW_STALL_SECONDS F_LOSNING_OPENCLAW_STALL_SECONDS
   flow_export_env_alias_if_unset ACP_ALLOW_INFRA_CI_BYPASS F_LOSNING_ALLOW_INFRA_CI_BYPASS
   flow_export_env_alias_if_unset ACP_LOCAL_FIRST_PR_POLICY F_LOSNING_LOCAL_FIRST_PR_POLICY
   flow_export_env_alias_if_unset ACP_PR_RISK_CACHE_TTL_SECONDS F_LOSNING_PR_RISK_CACHE_TTL_SECONDS

package/tools/bin/heartbeat-safe-auto.sh

@@ -345,24 +345,28 @@ PY
   local effective_pools=""
   healthy_pools="$(
     jq -r --argjson primaryThresh "${CODEX_QUOTA_THRESHOLD}" --argjson weeklyThresh "${CODEX_QUOTA_WEEKLY_THRESHOLD}" '
-      group_by(.accountId)
+      map(. + {poolKey: (.label // .trackedLabel // .email // .accountId // "")})
       | map(select(
-          ((.[0].usage.rate_limit.limit_reached // false) | not)
-          and ((.[0].usage.rate_limit.primary_window.used_percent // 100) < $primaryThresh)
-          and ((.[0].usage.rate_limit.secondary_window.used_percent // 100) < $weeklyThresh)
-        ))
+          (.poolKey != "")
+          and ((.usage.rate_limit.limit_reached // false) | not)
+          and ((.usage.rate_limit.primary_window.used_percent // 100) < $primaryThresh)
+          and ((.usage.rate_limit.secondary_window.used_percent // 100) < $weeklyThresh)
+        ) | .poolKey)
+      | unique
       | length
     ' "${CODEX_QUOTA_FULL_CACHE_FILE}" 2>/dev/null || true
   )"
 
   rotation_pools="$(
     jq -r --argjson weeklyThresh "${CODEX_QUOTA_WEEKLY_THRESHOLD}" '
-      group_by(.accountId)
+      map(. + {poolKey: (.label // .trackedLabel // .email // .accountId // "")})
       | map(select(
-          ((.[0].usage.rate_limit.limit_reached // false) | not)
-          and ((.[0].usage.rate_limit.secondary_window.used_percent // 100) < $weeklyThresh)
-          and ((.[0].planType // "") != "free")
-        ))
+          (.poolKey != "")
+          and ((.usage.rate_limit.limit_reached // false) | not)
+          and ((.usage.rate_limit.secondary_window.used_percent // 100) < $weeklyThresh)
+          and ((.planType // "") != "free")
+        ) | .poolKey)
+      | unique
       | length
     ' "${CODEX_QUOTA_FULL_CACHE_FILE}" 2>/dev/null || true
   )"
@@ -471,6 +475,12 @@ fi
 
 run_codex_quota_preflight
 
+# Sync skill files to runtime-home if source has changed since last sync.
+# This ensures start-issue-worker.sh and other scripts are always up to date.
+if [[ -x "${FLOW_TOOLS_DIR}/ensure-runtime-sync.sh" ]]; then
+  "${FLOW_TOOLS_DIR}/ensure-runtime-sync.sh" --quiet 2>/dev/null || true
+fi
+
 acquire_lock
 
 reap_orphan_shared_loop_groups