agent-control-plane 0.3.0 → 0.6.0

package/tools/bin/agent-project-run-codex-resilient

@@ -1,955 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-# shellcheck source=/dev/null
-source "${SCRIPT_DIR}/flow-config-lib.sh"
-
-usage() {
-  cat <<'EOF'
-Usage:
-  agent-project-run-codex-resilient --mode safe|bypass --worktree <path> --prompt-file <path> --output-file <path> --host-run-dir <path> --sandbox-run-dir <path> --codex-bin <path> [options]
-
-Run Codex with persisted thread recovery for quota/auth interruptions.
-
-Options:
-  --safe-profile <name>                   Codex profile for safe mode
-  --bypass-profile <name>                 Codex profile for bypass mode
-  --max-resume-attempts <count>           Maximum resume attempts after interruption
-  --auth-refresh-timeout-seconds <secs>   How long to wait for refreshed auth before failing
-  --auth-refresh-poll-seconds <secs>      Poll interval while waiting for refreshed auth
-  --stall-seconds <secs>                  Fail if Codex stops producing output for too long
-  --help                                  Show this help
-EOF
-}
-
-mode=""
-worktree=""
-prompt_file=""
-output_file=""
-host_run_dir=""
-sandbox_run_dir=""
-safe_profile="default"
-bypass_profile="default"
-codex_bin=""
-max_resume_attempts="${ACP_CODEX_MAX_RESUME_ATTEMPTS:-${F_LOSNING_CODEX_MAX_RESUME_ATTEMPTS:-6}}"
-auth_refresh_timeout_seconds="${ACP_CODEX_AUTH_REFRESH_TIMEOUT_SECONDS:-${F_LOSNING_CODEX_AUTH_REFRESH_TIMEOUT_SECONDS:-900}}"
-auth_refresh_poll_seconds="${ACP_CODEX_AUTH_REFRESH_POLL_SECONDS:-${F_LOSNING_CODEX_AUTH_REFRESH_POLL_SECONDS:-10}}"
-max_quota_autoswitch_attempts="${ACP_CODEX_MAX_AUTOSWITCH_ATTEMPTS:-${F_LOSNING_CODEX_MAX_AUTOSWITCH_ATTEMPTS:-1}}"
-codex_progress_heartbeat_seconds="${ACP_CODEX_PROGRESS_HEARTBEAT_SECONDS:-${F_LOSNING_CODEX_PROGRESS_HEARTBEAT_SECONDS:-30}}"
-codex_stall_seconds="${ACP_CODEX_STALL_SECONDS:-${F_LOSNING_CODEX_STALL_SECONDS:-300}}"
-python_bin=""
-
-resolve_python_bin() {
-  if command -v python3 >/dev/null 2>&1; then
-    command -v python3
-    return 0
-  fi
-  if [[ -x /opt/homebrew/bin/python3 ]]; then
-    printf '%s\n' "/opt/homebrew/bin/python3"
-    return 0
-  fi
-  if command -v python >/dev/null 2>&1; then
-    command -v python
-    return 0
-  fi
-  return 1
-}
-
-while [[ $# -gt 0 ]]; do
-  case "$1" in
-    --mode) mode="${2:-}"; shift 2 ;;
-    --worktree) worktree="${2:-}"; shift 2 ;;
-    --prompt-file) prompt_file="${2:-}"; shift 2 ;;
-    --output-file) output_file="${2:-}"; shift 2 ;;
-    --host-run-dir) host_run_dir="${2:-}"; shift 2 ;;
-    --sandbox-run-dir) sandbox_run_dir="${2:-}"; shift 2 ;;
-    --safe-profile) safe_profile="${2:-}"; shift 2 ;;
-    --bypass-profile) bypass_profile="${2:-}"; shift 2 ;;
-    --codex-bin) codex_bin="${2:-}"; shift 2 ;;
-    --max-resume-attempts) max_resume_attempts="${2:-}"; shift 2 ;;
-    --auth-refresh-timeout-seconds) auth_refresh_timeout_seconds="${2:-}"; shift 2 ;;
-    --auth-refresh-poll-seconds) auth_refresh_poll_seconds="${2:-}"; shift 2 ;;
-    --stall-seconds) codex_stall_seconds="${2:-}"; shift 2 ;;
-    --help|-h) usage; exit 0 ;;
-    *) echo "Unknown argument: $1" >&2; usage >&2; exit 1 ;;
-  esac
-done
-
-if [[ -z "$mode" || -z "$worktree" || -z "$prompt_file" || -z "$output_file" || -z "$host_run_dir" || -z "$sandbox_run_dir" || -z "$codex_bin" ]]; then
-  usage >&2
-  exit 1
-fi
-
-case "$mode" in
-  safe|bypass) ;;
-  *)
-    echo "--mode must be safe or bypass" >&2
-    exit 1
-    ;;
-esac
-
-case "$max_resume_attempts" in
-  ''|*[!0-9]*) echo "--max-resume-attempts must be numeric" >&2; exit 1 ;;
-esac
-case "$auth_refresh_timeout_seconds" in
-  ''|*[!0-9]*) echo "--auth-refresh-timeout-seconds must be numeric" >&2; exit 1 ;;
-esac
-case "$auth_refresh_poll_seconds" in
-  ''|*[!0-9]*) echo "--auth-refresh-poll-seconds must be numeric" >&2; exit 1 ;;
-esac
-case "$max_quota_autoswitch_attempts" in
-  ''|*[!0-9]*) echo "ACP_CODEX_MAX_AUTOSWITCH_ATTEMPTS must be numeric" >&2; exit 1 ;;
-esac
-case "$codex_progress_heartbeat_seconds" in
-  ''|*[!0-9]*) echo "ACP_CODEX_PROGRESS_HEARTBEAT_SECONDS must be numeric" >&2; exit 1 ;;
-  0) echo "ACP_CODEX_PROGRESS_HEARTBEAT_SECONDS must be greater than zero" >&2; exit 1 ;;
-esac
-case "$codex_stall_seconds" in
-  ''|*[!0-9]*) echo "ACP_CODEX_STALL_SECONDS must be numeric" >&2; exit 1 ;;
-esac
-
-python_bin="$(resolve_python_bin || true)"
-if [[ -z "$python_bin" || ! -x "$python_bin" ]]; then
-  echo "unable to resolve a runnable python interpreter for codex supervision" >&2
-  exit 1
-fi
-
-FLOW_SKILL_DIR="$(resolve_flow_skill_dir "${BASH_SOURCE[0]}")"
-state_file="${host_run_dir}/runner.env"
-auth_file="${HOME}/.codex/auth.json"
-shared_agent_home="${SHARED_AGENT_HOME:-$(resolve_shared_agent_home "${FLOW_SKILL_DIR}")}"
-quota_tool_bin="$(flow_resolve_codex_quota_bin "${FLOW_SKILL_DIR}")"
-quota_manager_script="$(flow_resolve_codex_quota_manager_script "${FLOW_SKILL_DIR}")"
-quota_autoswitch_enabled="${ACP_CODEX_QUOTA_AUTOSWITCH_ENABLED:-${F_LOSNING_CODEX_QUOTA_AUTOSWITCH_ENABLED:-1}}"
-quota_threshold="${ACP_CODEX_QUOTA_THRESHOLD:-${F_LOSNING_CODEX_QUOTA_THRESHOLD:-70}}"
-quota_weekly_threshold="${ACP_CODEX_QUOTA_WEEKLY_THRESHOLD:-${F_LOSNING_CODEX_QUOTA_WEEKLY_THRESHOLD:-90}}"
-quota_soft_threshold="${ACP_CODEX_QUOTA_SOFT_THRESHOLD:-${F_LOSNING_CODEX_QUOTA_SOFT_THRESHOLD:-55}}"
-quota_soft_worker_threshold="${ACP_CODEX_QUOTA_SOFT_WORKER_THRESHOLD:-${F_LOSNING_CODEX_QUOTA_SOFT_WORKER_THRESHOLD:-8}}"
-quota_emergency_threshold="${ACP_CODEX_QUOTA_EMERGENCY_THRESHOLD:-${F_LOSNING_CODEX_QUOTA_EMERGENCY_THRESHOLD:-65}}"
-quota_emergency_worker_threshold="${ACP_CODEX_QUOTA_EMERGENCY_WORKER_THRESHOLD:-${F_LOSNING_CODEX_QUOTA_EMERGENCY_WORKER_THRESHOLD:-12}}"
-quota_switch_cooldown_seconds="${ACP_CODEX_QUOTA_SWITCH_COOLDOWN_SECONDS:-${F_LOSNING_CODEX_QUOTA_SWITCH_COOLDOWN_SECONDS:-600}}"
-quota_timeout_seconds="${ACP_CODEX_QUOTA_TIMEOUT_SECONDS:-${F_LOSNING_CODEX_QUOTA_TIMEOUT_SECONDS:-45}}"
-quota_prefer_label="${ACP_CODEX_QUOTA_PREFER_LABEL:-${F_LOSNING_CODEX_QUOTA_PREFER_LABEL:-}}"
-quota_switch_state_file="${CODEX_QUOTA_MANAGER_SWITCH_STATE_FILE:-${XDG_CACHE_HOME:-$HOME/.cache}/codex-quota-manager/last-switch.env}"
-config_yaml="$(resolve_flow_config_yaml "${BASH_SOURCE[0]}")"
-issue_session_prefix="$(flow_resolve_issue_session_prefix "${config_yaml}")"
-pr_session_prefix="$(flow_resolve_pr_session_prefix "${config_yaml}")"
-
-thread_id=""
-attempt=0
-resume_count=0
-last_exit_code=""
-last_failure_reason=""
-last_trigger_reason=""
-auth_wait_started_at=""
-last_auth_fingerprint=""
-last_attempt_start_size=0
-last_attempt_start_quota_label=""
-last_quota_switch_status=""
-last_quota_next_retry_at=""
-last_quota_selected_label=""
-quota_autoswitch_attempt_count=0
-last_attempt_started_epoch=0
-
-mkdir -p "$host_run_dir"
-touch "$output_file"
-
-log_runner() {
-  local message="${1:-}"
-  [[ -n "$message" ]] || return 0
-  printf '[%s] %s\n' "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" "$message" | tee -a "$output_file"
-}
-
-write_state() {
-  local runner_state="${1:?runner state required}"
-  local failure_reason="${2:-$last_failure_reason}"
-  local updated_at
-  local tmp_file="${state_file}.tmp.$$"
-
-  updated_at="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
-  {
-    printf 'RUNNER_STATE=%q\n' "$runner_state"
-    printf 'THREAD_ID=%q\n' "$thread_id"
-    printf 'ATTEMPT=%s\n' "$attempt"
-    printf 'RESUME_COUNT=%s\n' "$resume_count"
-    printf 'LAST_EXIT_CODE=%q\n' "$last_exit_code"
-    printf 'LAST_FAILURE_REASON=%q\n' "$failure_reason"
-    printf 'LAST_TRIGGER_REASON=%q\n' "$last_trigger_reason"
-    printf 'AUTH_WAIT_STARTED_AT=%q\n' "$auth_wait_started_at"
-    printf 'LAST_AUTH_FINGERPRINT=%q\n' "$last_auth_fingerprint"
-    printf 'UPDATED_AT=%q\n' "$updated_at"
-  } >"$tmp_file"
-  mv "$tmp_file" "$state_file"
-}
-
-run_codex_command() {
-  # Nested workers must not inherit a parent thread id; the wrapper persists the child thread explicitly.
-  env -u CODEX_THREAD_ID "$codex_bin" "$@"
-}
-
-codex_recovery_target() {
-  if [[ -n "$thread_id" ]]; then
-    printf 'thread %s' "$thread_id"
-    return 0
-  fi
-  printf 'initial Codex exec'
-}
-
-run_with_timeout() {
-  local timeout_seconds="${1:?timeout seconds required}"
-  shift
-
-  "$python_bin" - "$timeout_seconds" "$@" <<'PY'
-import os
-import signal
-import subprocess
-import sys
-
-timeout_seconds = float(sys.argv[1])
-argv = sys.argv[2:]
-
-if not argv:
-    sys.exit(64)
-
-proc = subprocess.Popen(argv, start_new_session=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-
-try:
-    stdout, stderr = proc.communicate(timeout=timeout_seconds)
-except subprocess.TimeoutExpired:
-    try:
-        os.killpg(proc.pid, signal.SIGTERM)
-    except ProcessLookupError:
-        pass
-    try:
-        stdout, stderr = proc.communicate(timeout=2)
-    except subprocess.TimeoutExpired:
-        try:
-            os.killpg(proc.pid, signal.SIGKILL)
-        except ProcessLookupError:
-            pass
-        stdout, stderr = proc.communicate()
-    if stdout:
-        sys.stdout.buffer.write(stdout)
-    if stderr:
-        sys.stderr.buffer.write(stderr)
-    sys.exit(124)
-
-if stdout:
-    sys.stdout.buffer.write(stdout)
-if stderr:
-    sys.stderr.buffer.write(stderr)
-sys.exit(proc.returncode)
-PY
-}
-
-stat_file_size() {
-  local path="${1:?path required}"
-  local value=""
-
-  value="$(stat -f %z "$path" 2>/dev/null || true)"
-  if [[ "$value" =~ ^[0-9]+$ ]]; then
-    printf '%s\n' "$value"
-    return 0
-  fi
-
-  value="$(stat -c %s "$path" 2>/dev/null || true)"
-  if [[ "$value" =~ ^[0-9]+$ ]]; then
-    printf '%s\n' "$value"
-    return 0
-  fi
-
-  "$python_bin" - "$path" <<'PY'
-import os
-import sys
-
-try:
-    print(os.path.getsize(sys.argv[1]))
-except OSError:
-    print("0")
-PY
-}
-
-stat_file_mtime() {
-  local path="${1:?path required}"
-  local value=""
-
-  value="$(stat -f %m "$path" 2>/dev/null || true)"
-  if [[ "$value" =~ ^[0-9]+$ ]]; then
-    printf '%s\n' "$value"
-    return 0
-  fi
-
-  value="$(stat -c %Y "$path" 2>/dev/null || true)"
-  if [[ "$value" =~ ^[0-9]+$ ]]; then
-    printf '%s\n' "$value"
-    return 0
-  fi
-
-  "$python_bin" - "$path" <<'PY'
-import os
-import sys
-
-try:
-    print(int(os.path.getmtime(sys.argv[1])))
-except OSError:
-    print("0")
-PY
-}
-
-auth_fingerprint() {
-  if [[ ! -f "$auth_file" ]]; then
-    printf 'missing\n'
-    return 0
-  fi
-
-  local mtime size sha
-  mtime="$(stat_file_mtime "$auth_file" 2>/dev/null || printf '0')"
-  size="$(stat_file_size "$auth_file" 2>/dev/null || printf '0')"
-  sha="$(shasum -a 256 "$auth_file" | awk '{print $1}')"
-  printf '%s:%s:%s\n' "$mtime" "$size" "$sha"
-}
-
-quota_active_label() {
-  if [[ ! -x "$quota_tool_bin" ]] || ! command -v jq >/dev/null 2>&1; then
-    printf '\n'
-    return 0
-  fi
-
-  "$quota_tool_bin" codex list --json 2>/dev/null \
-    | jq -r '
-        .activeInfo.trackedLabel
-        // .activeInfo.activeLabel
-        // ([.accounts[]? | select(.isActive == true or .isNativeActive == true)][0].label)
-        // empty
-      ' 2>/dev/null \
-    || printf '\n'
-}
-
-quota_switch_signature() {
-  if [[ ! -f "$quota_switch_state_file" ]]; then
-    printf 'missing\n'
-    return 0
-  fi
-
-  local mtime size sha
-  mtime="$(stat_file_mtime "$quota_switch_state_file" 2>/dev/null || printf '0')"
-  size="$(stat_file_size "$quota_switch_state_file" 2>/dev/null || printf '0')"
-  sha="$(shasum -a 256 "$quota_switch_state_file" | awk '{print $1}')"
-  printf '%s:%s:%s\n' "$mtime" "$size" "$sha"
-}
-
-running_workers() {
-  if ! command -v tmux >/dev/null 2>&1; then
-    printf '0\n'
-    return 0
-  fi
-
-  { tmux list-sessions -F '#S' 2>/dev/null || true; } \
-    | awk -v issue_prefix="$issue_session_prefix" -v pr_prefix="$pr_session_prefix" '
-        index($0, issue_prefix) == 1 || index($0, pr_prefix) == 1 { count++ }
-        END { print count + 0 }
-      '
-}
-
-extract_kv_value() {
-  local key="${1:?key required}"
-  local payload="${2:-}"
-  sed -nE "s/^${key}=(.*)$/\\1/p" <<<"$payload" | tail -n 1
-}
-
-run_quota_autoswitch() {
-  local quota_output quota_status shell_flags_before_quota_exec
-  local -a quota_cmd
-  local worker_count
-
-  quota_autoswitch_unavailable_reason() {
-    if [[ "$quota_autoswitch_enabled" == "0" ]]; then
-      printf 'disabled\n'
-      return 0
-    fi
-    if [[ ! -x "$quota_manager_script" ]]; then
-      printf 'missing-script\n'
-      return 0
-    fi
-    if [[ ! -x "$quota_tool_bin" ]]; then
-      printf 'missing-codex-quota\n'
-      return 0
-    fi
-    if ! command -v jq >/dev/null 2>&1; then
-      printf 'missing-jq\n'
-      return 0
-    fi
-    printf 'ok\n'
-  }
-
-  local unavailable_reason=""
-  last_quota_next_retry_at=""
-  last_quota_selected_label=""
-  if [[ "$quota_autoswitch_enabled" == "0" ]]; then
-    log_runner "quota auto-switch disabled; waiting for external Codex auth refresh"
-    printf 'CODEX_QUOTA_AUTOSWITCH_ENABLED=0\n' | tee -a "$output_file"
-    last_quota_switch_status="disabled"
-    return 1
-  fi
-
-  unavailable_reason="$(quota_autoswitch_unavailable_reason)"
-  if [[ "$unavailable_reason" != "ok" ]]; then
-    log_runner "quota auto-switch unavailable (${unavailable_reason}); waiting for external Codex auth refresh"
-    printf 'CODEX_QUOTA_MANAGER_UNAVAILABLE=yes\n' | tee -a "$output_file"
-    printf 'CODEX_QUOTA_MANAGER_REASON=%s\n' "$unavailable_reason" | tee -a "$output_file"
-    last_quota_switch_status="unavailable"
-    return 1
-  fi
-
-  quota_autoswitch_attempt_count=$((quota_autoswitch_attempt_count + 1))
-  worker_count="$(running_workers)"
-  quota_cmd=(
-    env
-    "CODEX_QUOTA_BIN=${quota_tool_bin}"
-    bash "$quota_manager_script"
-    --trigger-reason "$last_failure_reason"
-    --current-label "${last_attempt_start_quota_label}"
-    --five-hour-threshold "$quota_threshold"
-    --weekly-threshold "$quota_weekly_threshold"
-    --running-workers "$worker_count"
-  )
-  if [[ -n "$quota_prefer_label" ]]; then
-    quota_cmd+=(--prefer-label "$quota_prefer_label")
-  fi
-
-  log_runner "${last_failure_reason} detected; attempting failure-driven Codex account switch"
-  shell_flags_before_quota_exec="$-"
-  set +e
-  quota_output="$(run_with_timeout "$quota_timeout_seconds" "${quota_cmd[@]}" 2>&1)"
-  quota_status=$?
-  case "$shell_flags_before_quota_exec" in
-    *e*) set -e ;;
-    *) set +e ;;
-  esac
-
-  if [[ "$quota_status" == "0" ]]; then
-    last_quota_switch_status="$(extract_kv_value "SWITCH_DECISION" "$quota_output")"
-    last_quota_next_retry_at="$(extract_kv_value "NEXT_RETRY_AT" "$quota_output")"
-    last_quota_selected_label="$(extract_kv_value "SELECTED_LABEL" "$quota_output")"
-    [[ -n "$quota_output" ]] && printf '%s\n' "$quota_output" | tee -a "$output_file"
-    case "$last_quota_switch_status" in
-      switched|current-ok)
-        return 0
-        ;;
-      deferred)
-        return 10
-        ;;
-      *)
-        last_quota_switch_status="failed"
-        return 1
-        ;;
-    esac
-  fi
-
-  last_quota_next_retry_at="$(extract_kv_value "NEXT_RETRY_AT" "${quota_output:-}")"
-  last_quota_selected_label="$(extract_kv_value "SELECTED_LABEL" "${quota_output:-}")"
-  [[ -n "${quota_output:-}" ]] && printf '%s\n' "$quota_output" | tee -a "$output_file"
-  if [[ "$quota_status" == "10" ]]; then
-    last_quota_switch_status="deferred"
-    log_runner "no eligible Codex account is ready yet; waiting for the next reset window"
-    return 10
-  fi
-  last_quota_switch_status="failed"
-  if [[ "$quota_status" == "124" ]]; then
-    log_runner "quota auto-switch timed out after ${quota_timeout_seconds}s"
-  else
-    log_runner "quota auto-switch exited with status ${quota_status}"
-  fi
-  return "$quota_status"
-}
-
-new_output_since() {
-  local start_size="${1:?start size required}"
-  local file_size
-  file_size="$(stat_file_size "$output_file" 2>/dev/null || printf '0')"
-  if (( file_size <= start_size )); then
-    return 0
-  fi
-  tail -c "+$((start_size + 1))" "$output_file"
-}
-
-update_thread_id_from_output() {
-  local start_size="${1:?start size required}"
-  local new_thread_id
-
-  new_thread_id="$(new_output_since "$start_size" | extract_thread_id || true)"
-  if [[ -n "$new_thread_id" ]]; then
-    thread_id="$new_thread_id"
-  fi
-}
-
-extract_thread_id_from_line() {
-  local line="${1:-}"
-  sed -nE 's/.*"type"[[:space:]]*:[[:space:]]*"thread\.started".*"thread_id"[[:space:]]*:[[:space:]]*"([^"]+)".*/\1/p' <<<"$line"
-}
-
-persist_thread_id_from_line() {
-  local line="${1:-}"
-  local new_thread_id=""
-
-  new_thread_id="$(extract_thread_id_from_line "$line")"
-  if [[ -n "$new_thread_id" && "$new_thread_id" != "$thread_id" ]]; then
-    thread_id="$new_thread_id"
-    write_state "running" ""
-  fi
-}
-
-terminate_codex_producer_tree() {
-  local pid="${1:?pid required}"
-  local deadline=""
-
-  if ! kill -0 "$pid" 2>/dev/null; then
-    return 0
-  fi
-
-  pkill -TERM -P "$pid" 2>/dev/null || true
-  kill "$pid" 2>/dev/null || true
-
-  deadline=$(( $(date +%s) + 2 ))
-  while kill -0 "$pid" 2>/dev/null; do
-    if (( $(date +%s) >= deadline )); then
-      break
-    fi
-    sleep 0.1
-  done
-
-  if kill -0 "$pid" 2>/dev/null; then
-    pkill -KILL -P "$pid" 2>/dev/null || true
-    kill -9 "$pid" 2>/dev/null || true
-  fi
-}
-
-stream_codex_exec() {
-  local phase="${1:?phase required}"
-  local stream_fifo=""
-  local producer_pid=""
-  local heartbeat_pid=""
-  local progress_file=""
-  local line=""
-
-  last_attempt_start_size="$(stat_file_size "$output_file" 2>/dev/null || printf '0')"
-  last_attempt_started_epoch="$(date +%s)"
-  progress_file="${host_run_dir}/.codex-progress.$$"
-  rm -f "$progress_file"
-  stream_fifo="$(mktemp -u "${TMPDIR:-/tmp}/codex-stream.XXXXXX")"
-  mkfifo "$stream_fifo"
-
-  case "$phase" in
-    initial)
-      (
-        case "$mode" in
-          safe)
-            run_codex_command exec --json --profile "$safe_profile" --full-auto <"$prompt_file"
-            ;;
-          bypass)
-            run_codex_command exec --json --profile "$bypass_profile" --dangerously-bypass-approvals-and-sandbox <"$prompt_file"
-            ;;
-        esac
-      ) >"$stream_fifo" 2>&1 &
-      ;;
-    resume)
-      (
-        case "$mode" in
-          safe)
-            resume_prompt | run_codex_command exec resume --json --full-auto "$thread_id" -
-            ;;
-          bypass)
-            resume_prompt | run_codex_command exec resume --json --dangerously-bypass-approvals-and-sandbox "$thread_id" -
-            ;;
-        esac
-      ) >"$stream_fifo" 2>&1 &
-      ;;
-    *)
-      rm -f "$stream_fifo"
-      echo "unknown codex exec phase: $phase" >&2
-      exit 1
-      ;;
-  esac
-
-  producer_pid="$!"
-  (
-    local now elapsed last_progress_epoch idle_for
-    while kill -0 "$producer_pid" 2>/dev/null; do
-      sleep "$codex_progress_heartbeat_seconds"
-      if ! kill -0 "$producer_pid" 2>/dev/null; then
-        break
-      fi
-      now="$(date +%s)"
-      elapsed=$((now - last_attempt_started_epoch))
-      if (( codex_stall_seconds > 0 )); then
-        if [[ ! -f "$progress_file" ]]; then
-          if (( elapsed >= codex_stall_seconds )); then
-            write_state "running" ""
-            log_runner "stale-run no-codex-output-before-stall-threshold elapsed=${elapsed}s"
-            terminate_codex_producer_tree "$producer_pid"
-            break
-          fi
-        else
-          last_progress_epoch="$(stat_file_mtime "$progress_file" 2>/dev/null || printf '0')"
-          if [[ -n "$last_progress_epoch" && "$last_progress_epoch" != "0" ]]; then
-            idle_for=$((now - last_progress_epoch))
-            if (( idle_for >= codex_stall_seconds )); then
-              write_state "running" ""
-              log_runner "stale-run no-codex-progress-before-stall-threshold elapsed=${elapsed}s idle=${idle_for}s"
-              terminate_codex_producer_tree "$producer_pid"
-              break
-            fi
-          fi
-        fi
-      fi
-      write_state "running" ""
-      log_runner "heartbeat waiting-for-codex-output elapsed=${elapsed}s"
-    done
-  ) &
-  heartbeat_pid="$!"
-
-  while IFS= read -r line || [[ -n "$line" ]]; do
-    printf '%s\n' "$line" | tee -a "$output_file"
-    touch "$progress_file" 2>/dev/null || true
-    persist_thread_id_from_line "$line"
-  done <"$stream_fifo"
-
-  if [[ -n "$heartbeat_pid" ]] && kill -0 "$heartbeat_pid" 2>/dev/null; then
-    kill "$heartbeat_pid" 2>/dev/null || true
-    wait "$heartbeat_pid" 2>/dev/null || true
-  fi
-
-  rm -f "$stream_fifo"
-  rm -f "$progress_file"
-
-  if wait "$producer_pid"; then
-    last_exit_code="0"
-  else
-    last_exit_code="$?"
-  fi
-
-  update_thread_id_from_output "$last_attempt_start_size"
-}
-
-extract_thread_id() {
-  "$python_bin" -c '
-import json
-import sys
-
-thread_id = ""
-for raw in sys.stdin:
-    line = raw.strip()
-    if not line.startswith("{"):
-        continue
-    try:
-        payload = json.loads(line)
-    except Exception:
-        continue
-    if payload.get("type") == "thread.started" and payload.get("thread_id"):
-        thread_id = str(payload["thread_id"])
-
-if thread_id:
-    sys.stdout.write(thread_id)
-'
-}
-
-classify_failure_reason() {
-  local chunk="${1:-}"
-  local recent_chunk
-
-  recent_chunk="$(tail -n 120 <<<"$chunk")"
-
-  if grep -Eiq 'stale-run no-codex-output-before-stall-threshold|no-codex-output-before-stall-threshold' <<<"$recent_chunk"; then
-    printf 'no-codex-output-before-stall-threshold\n'
-    return 0
-  fi
-
-  if grep -Eiq 'stale-run no-codex-progress-before-stall-threshold|no-codex-progress-before-stall-threshold' <<<"$recent_chunk"; then
-    printf 'no-codex-progress-before-stall-threshold\n'
-    return 0
-  fi
-
-  if grep -Eiq "You've hit your usage limit|You have reached your Codex usage limits|visit https://chatgpt.com/codex/settings/usage|Upgrade to Pro|rate limit exceeded|quota exceeded|usage cap (reached|exceeded)|usage quota (reached|exceeded)" <<<"$recent_chunk"; then
-    printf 'usage-limit\n'
-    return 0
-  fi
-
-  if grep -Eiq "(HTTP[^0-9]*)?401([^0-9]|$)|unauthorized|invalid credentials|invalid api key|authentication failed with status 401|received 401" <<<"$recent_chunk"; then
-    printf 'auth-401\n'
-    return 0
-  fi
-
-  if grep -Eiq "account (is )?(banned|suspended|disabled)|access revoked|account revoked|forbidden due to policy|account blocked|policy violation" <<<"$recent_chunk"; then
-    printf 'account-banned\n'
-    return 0
-  fi
-
-  if grep -Eiq "Authentication required|Please log in|Please login|Please authenticate|login required|run codex login|codex login required|logged out|not logged in|expired session|session expired|token expired|reauthenticate|unauthenticated|auth(entication)? failed|credentials expired" <<<"$recent_chunk"; then
-    printf 'auth-failure\n'
-    return 0
-  fi
-
-  if [[ -n "$last_exit_code" && "$last_exit_code" != "0" ]]; then
-    printf 'worker-exit-failed\n'
-  fi
-}
-
-failure_chunk_indicates_startup_stall() {
-  local chunk="${1:-}"
-  local recent_chunk
-
-  recent_chunk="$(tail -n 120 <<<"$chunk")"
-  grep -q '"type":"thread.started"' <<<"$recent_chunk" || return 1
-  grep -q '"type":"turn.started"' <<<"$recent_chunk" || return 1
-  if grep -Eq '"type":"item\.(started|completed)"' <<<"$recent_chunk"; then
-    return 1
-  fi
-  if grep -q '"type":"turn.completed"' <<<"$recent_chunk"; then
-    return 1
-  fi
-  return 0
-}
-
-resume_prompt() {
-  cat <<EOF
-The previous Codex exec turn in this same thread was interrupted because the host refreshed Codex authentication after a quota or auth failure.
-
-Continue the same task from the next unfinished step only. Do not restart completed work unless you need to verify or repair it.
-
-If you need to reorient, inspect the current git status plus the existing run artifacts in:
-- Host run dir: ${host_run_dir}
-- Sandbox run dir: ${sandbox_run_dir}
-EOF
-}
-
-codex_login_healthy() {
-  run_codex_command login status >/dev/null 2>&1
-}
-
-wait_for_auth_refresh() {
-  local baseline_fingerprint="${1:?baseline fingerprint required}"
-  local trigger_reason="${2:?trigger reason required}"
-  local baseline_quota_label="${3:-}"
-  local baseline_switch_signature="${4:-}"
-  local deadline now current_fingerprint current_quota_label current_switch_signature
-  local sleep_seconds
-  local recovery_target
-
-  recovery_target="$(codex_recovery_target)"
-  auth_wait_started_at="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
-  last_trigger_reason="$trigger_reason"
-  write_state "waiting-auth-refresh" "$trigger_reason"
-
-  deadline=$(( $(date +%s) + auth_refresh_timeout_seconds ))
-  while :; do
-    current_fingerprint="$(auth_fingerprint)"
-    last_auth_fingerprint="$current_fingerprint"
-    case "$trigger_reason" in
-      usage-limit|auth-401|account-banned)
-        current_quota_label="$(quota_active_label)"
-        current_switch_signature="$(quota_switch_signature)"
-        if codex_login_healthy; then
-          if [[ "$current_fingerprint" != "$baseline_fingerprint" ]]; then
-            log_runner "detected refreshed Codex auth after quota interruption; resuming ${recovery_target}"
-            auth_wait_started_at=""
-            write_state "running" ""
-            return 0
-          fi
-
-          if [[ -n "$baseline_quota_label" && -n "$current_quota_label" && "$current_quota_label" != "$baseline_quota_label" ]]; then
-            log_runner "detected rotated Codex quota account (${baseline_quota_label} -> ${current_quota_label}); resuming ${recovery_target}"
-            auth_wait_started_at=""
-            write_state "running" ""
-            return 0
-          fi
-
-          if [[ -n "$baseline_switch_signature" && -n "$current_switch_signature" && "$current_switch_signature" != "$baseline_switch_signature" ]]; then
-            log_runner "detected quota switch state refresh; resuming ${recovery_target}"
-            auth_wait_started_at=""
-            write_state "running" ""
-            return 0
-          fi
-
-          if [[ "$last_quota_switch_status" == "switched" && -n "$current_quota_label" ]]; then
-            log_runner "quota manager reports healthy Codex account ${current_quota_label}; resuming ${recovery_target}"
-            auth_wait_started_at=""
-            write_state "running" ""
-            return 0
-          fi
-        fi
-
-        ;;
-      *)
-        if codex_login_healthy; then
-          if [[ "$current_fingerprint" != "$baseline_fingerprint" ]]; then
-            log_runner "detected refreshed Codex auth; resuming ${recovery_target}"
-          else
-            log_runner "Codex auth is healthy again; resuming ${recovery_target}"
-          fi
-          auth_wait_started_at=""
-          write_state "running" ""
-          return 0
-        fi
-        ;;
-    esac
-
-    now="$(date +%s)"
-    if (( now >= deadline )); then
-      last_failure_reason="auth-refresh-timeout"
-      write_state "failed" "$last_failure_reason"
-      return 1
-    fi
-
-    sleep_seconds="$auth_refresh_poll_seconds"
-    if (( sleep_seconds > deadline - now )); then
-      sleep_seconds=$(( deadline - now ))
-    fi
-
-    if (( sleep_seconds < 1 )); then
-      sleep_seconds=1
-    fi
-    sleep "$sleep_seconds"
-  done
-}
-
-run_initial_exec() {
-  stream_codex_exec initial
-}
-
-run_resume_exec() {
-  stream_codex_exec resume
-}
-
-attempt_run() {
-  local reason auth_before_switch quota_label_before_switch quota_switch_signature_before_switch quota_switch_result shell_flags_before_quota_switch failure_chunk startup_stall
-
-  attempt=$((attempt + 1))
-  last_quota_switch_status=""
-  last_attempt_start_quota_label="$(quota_active_label)"
-  write_state "running" ""
-
-  if [[ -z "$thread_id" ]]; then
-    log_runner "starting Codex exec attempt ${attempt}"
-    run_initial_exec
-  else
-    log_runner "resuming Codex thread ${thread_id} (resume ${resume_count}/${max_resume_attempts})"
-    run_resume_exec
-  fi
-
-  if [[ "${last_exit_code}" == "0" ]]; then
-    last_failure_reason=""
-    write_state "succeeded" ""
-    return 0
-  fi
-
-  failure_chunk="$(new_output_since "$last_attempt_start_size")"
-  reason="$(classify_failure_reason "$failure_chunk")"
-  last_failure_reason="${reason:-worker-exit-failed}"
-  startup_stall="no"
-  if [[ "$last_failure_reason" == "no-codex-output-before-stall-threshold" || "$last_failure_reason" == "no-codex-progress-before-stall-threshold" ]]; then
-    if failure_chunk_indicates_startup_stall "$failure_chunk"; then
-      startup_stall="yes"
-    fi
-  fi
-
-  case "$last_failure_reason" in
-    usage-limit|auth-failure|auth-401|account-banned)
-      if (( resume_count >= max_resume_attempts )); then
-        last_failure_reason="resume-attempts-exhausted"
-        write_state "failed" "$last_failure_reason"
-        return 1
-      fi
-
-      auth_before_switch="$(auth_fingerprint)"
-      quota_label_before_switch="$last_attempt_start_quota_label"
-      quota_switch_signature_before_switch="$(quota_switch_signature)"
-      last_auth_fingerprint="$auth_before_switch"
-      if [[ "$last_failure_reason" == "usage-limit" || "$last_failure_reason" == "auth-401" || "$last_failure_reason" == "account-banned" ]]; then
-        if (( quota_autoswitch_attempt_count >= max_quota_autoswitch_attempts )); then
-          log_runner "automatic Codex quota switching already ran ${quota_autoswitch_attempt_count} time(s) in this worker; refusing another rotation"
-          last_failure_reason="quota-switch-attempt-limit"
-          write_state "failed" "$last_failure_reason"
-          return 1
-        fi
-        write_state "switching-account" "$last_failure_reason"
-        shell_flags_before_quota_switch="$-"
-        set +e
-        run_quota_autoswitch
-        quota_switch_result=$?
-        case "$shell_flags_before_quota_switch" in
-          *e*) set -e ;;
-          *) set +e ;;
-        esac
-        if [[ "$quota_switch_result" == "10" ]]; then
-          log_runner "quota manager deferred rotation until ${last_quota_next_retry_at:-unknown}; automatic timed re-tries are disabled for safety"
-          last_failure_reason="quota-switch-deferred"
-          write_state "failed" "$last_failure_reason"
-          return 1
-        fi
-      fi
-
-      if ! wait_for_auth_refresh "$auth_before_switch" "$last_failure_reason" "$quota_label_before_switch" "$quota_switch_signature_before_switch"; then
-        return 1
-      fi
-
-      resume_count=$((resume_count + 1))
-      return 2
-      ;;
-    no-codex-output-before-stall-threshold|no-codex-progress-before-stall-threshold)
-      if [[ "$startup_stall" == "yes" && $quota_autoswitch_attempt_count -lt $max_quota_autoswitch_attempts ]]; then
-        auth_before_switch="$(auth_fingerprint)"
-        quota_label_before_switch="$last_attempt_start_quota_label"
-        quota_switch_signature_before_switch="$(quota_switch_signature)"
-        last_auth_fingerprint="$auth_before_switch"
-        write_state "switching-account" "$last_failure_reason"
-        log_runner "startup-stall detected before first Codex tool activity; attempting Codex account rotation"
-        shell_flags_before_quota_switch="$-"
-        set +e
-        run_quota_autoswitch
-        quota_switch_result=$?
-        case "$shell_flags_before_quota_switch" in
-          *e*) set -e ;;
-          *) set +e ;;
-        esac
-        if [[ "$quota_switch_result" == "0" ]]; then
-          thread_id=""
-          auth_wait_started_at=""
-          write_state "running" ""
-          return 2
-        fi
-        if [[ "$quota_switch_result" == "10" ]]; then
-          log_runner "startup-stall rotation deferred until ${last_quota_next_retry_at:-unknown}"
-          last_failure_reason="quota-switch-deferred"
-          write_state "failed" "$last_failure_reason"
-          return 1
-        fi
-      fi
-      write_state "failed" "$last_failure_reason"
-      return 1
-      ;;
-    *)
-      write_state "failed" "$last_failure_reason"
-      return 1
-      ;;
-  esac
-}
-
-write_state "running" ""
-
-while :; do
-  set +e
-  attempt_run
-  attempt_status=$?
-  set -e
-
-  if [[ "$attempt_status" == "0" ]]; then
-    exit 0
-  fi
-
-  if [[ "$attempt_status" == "2" ]]; then
-    continue
-  fi
-
-  exit "${last_exit_code:-1}"
-done