agent-control-plane 0.1.16 → 0.3.0

package/tools/bin/resident-issue-controller-lib.sh

@@ -0,0 +1,448 @@
+#!/usr/bin/env bash
+# resident-issue-controller-lib.sh — controller_* functions for the resident
+# issue loop.  Sourced by start-resident-issue-loop.sh to keep the main script
+# focused on the top-level loop logic.
+
+controller_unregister_pending_issue() {
+  local issue_id="${1:-${ISSUE_ID:-}}"
+  [[ -n "${issue_id}" ]] || return 0
+  rm -f "$(issue_pending_file "${issue_id}")"
+}
+
+controller_register_pending_issue() {
+  [[ -n "${ISSUE_ID:-}" ]] || return 0
+  printf '%s\n' "$$" >"$(issue_pending_file "${ISSUE_ID}")"
+}
+
+controller_refresh_execution_context() {
+  unset \
+    ACP_CODING_WORKER \
+    ACP_CODEX_PROFILE_SAFE F_LOSNING_CODEX_PROFILE_SAFE \
+    ACP_CODEX_PROFILE_BYPASS F_LOSNING_CODEX_PROFILE_BYPASS \
+    ACP_CLAUDE_MODEL F_LOSNING_CLAUDE_MODEL \
+    ACP_CLAUDE_PERMISSION_MODE F_LOSNING_CLAUDE_PERMISSION_MODE \
+    ACP_CLAUDE_EFFORT F_LOSNING_CLAUDE_EFFORT \
+    ACP_CLAUDE_TIMEOUT_SECONDS F_LOSNING_CLAUDE_TIMEOUT_SECONDS \
+    ACP_CLAUDE_MAX_ATTEMPTS F_LOSNING_CLAUDE_MAX_ATTEMPTS \
+    ACP_CLAUDE_RETRY_BACKOFF_SECONDS F_LOSNING_CLAUDE_RETRY_BACKOFF_SECONDS \
+    ACP_OPENCLAW_MODEL F_LOSNING_OPENCLAW_MODEL \
+    ACP_OPENCLAW_THINKING F_LOSNING_OPENCLAW_THINKING \
+    ACP_OPENCLAW_TIMEOUT_SECONDS F_LOSNING_OPENCLAW_TIMEOUT_SECONDS \
+    ACP_OPENCLAW_STALL_SECONDS F_LOSNING_OPENCLAW_STALL_SECONDS \
+    ACP_ACTIVE_PROVIDER_POOL_NAME F_LOSNING_ACTIVE_PROVIDER_POOL_NAME \
+    ACP_ACTIVE_PROVIDER_BACKEND F_LOSNING_ACTIVE_PROVIDER_BACKEND \
+    ACP_ACTIVE_PROVIDER_MODEL F_LOSNING_ACTIVE_PROVIDER_MODEL \
+    ACP_ACTIVE_PROVIDER_KEY F_LOSNING_ACTIVE_PROVIDER_KEY \
+    ACP_PROVIDER_POOLS_EXHAUSTED F_LOSNING_PROVIDER_POOLS_EXHAUSTED \
+    ACP_PROVIDER_POOL_SELECTION_REASON F_LOSNING_PROVIDER_POOL_SELECTION_REASON \
+    ACP_PROVIDER_POOL_NEXT_ATTEMPT_EPOCH F_LOSNING_PROVIDER_POOL_NEXT_ATTEMPT_EPOCH \
+    ACP_PROVIDER_POOL_NEXT_ATTEMPT_AT F_LOSNING_PROVIDER_POOL_NEXT_ATTEMPT_AT \
+    ACP_PROVIDER_POOL_LAST_REASON F_LOSNING_PROVIDER_POOL_LAST_REASON
+  flow_export_execution_env "${CONFIG_YAML}"
+  flow_export_project_env_aliases
+  CODING_WORKER="${ACP_CODING_WORKER:-codex}"
+  controller_capture_active_provider_context
+}
+
+controller_refresh_issue_lane_context() {
+  local is_scheduled="${1:-no}"
+  local schedule_interval_seconds="${2:-0}"
+
+  if [[ "${is_scheduled}" == "yes" ]]; then
+    ACTIVE_RESIDENT_LANE_KIND="scheduled"
+    ACTIVE_RESIDENT_LANE_VALUE="${schedule_interval_seconds}"
+  else
+    ACTIVE_RESIDENT_LANE_KIND="recurring"
+    ACTIVE_RESIDENT_LANE_VALUE="general"
+  fi
+
+  ACTIVE_RESIDENT_WORKER_KEY="$(flow_resident_issue_lane_key "${CODING_WORKER}" "${MODE}" "${ACTIVE_RESIDENT_LANE_KIND}" "${ACTIVE_RESIDENT_LANE_VALUE}")"
+  ACTIVE_RESIDENT_META_FILE="$(flow_resident_issue_lane_meta_file "${CONFIG_YAML}" "${ACTIVE_RESIDENT_WORKER_KEY}")"
+}
+
+controller_live_lane_peer() {
+  [[ -n "${ACTIVE_RESIDENT_WORKER_KEY}" ]] || return 1
+  flow_resident_live_issue_controller_for_key "${CONFIG_YAML}" "${ACTIVE_RESIDENT_WORKER_KEY}" "$$" || return 1
+}
+
+controller_yield_to_live_lane_peer() {
+  local live_controller=""
+  local controller_issue_id=""
+  local controller_state=""
+
+  live_controller="$(controller_live_lane_peer || true)"
+  [[ -n "${live_controller}" ]] || return 1
+
+  controller_issue_id="$(awk -F= '/^ISSUE_ID=/{print $2; exit}' <<<"${live_controller}")"
+  controller_state="$(awk -F= '/^CONTROLLER_STATE=/{print $2; exit}' <<<"${live_controller}")"
+
+  if [[ -n "${controller_issue_id}" && "${controller_issue_id}" != "${ISSUE_ID}" ]]; then
+    flow_resident_issue_enqueue "${CONFIG_YAML}" "${ISSUE_ID}" "resident-live-lane" >/dev/null || true
+    CONTROLLER_REASON="live-lane-controller-${controller_issue_id}-${controller_state:-running}"
+  else
+    CONTROLLER_REASON="duplicate-live-lane-controller"
+  fi
+
+  return 0
+}
+
+controller_capture_active_provider_context() {
+  ACTIVE_PROVIDER_POOL_NAME="${ACP_ACTIVE_PROVIDER_POOL_NAME:-${F_LOSNING_ACTIVE_PROVIDER_POOL_NAME:-}}"
+  ACTIVE_PROVIDER_BACKEND="${ACP_ACTIVE_PROVIDER_BACKEND:-${F_LOSNING_ACTIVE_PROVIDER_BACKEND:-${CODING_WORKER:-}}}"
+  ACTIVE_PROVIDER_MODEL="${ACP_ACTIVE_PROVIDER_MODEL:-${F_LOSNING_ACTIVE_PROVIDER_MODEL:-}}"
+  ACTIVE_PROVIDER_KEY="${ACP_ACTIVE_PROVIDER_KEY:-${F_LOSNING_ACTIVE_PROVIDER_KEY:-}}"
+  ACTIVE_PROVIDER_SELECTION_REASON="${ACP_PROVIDER_POOL_SELECTION_REASON:-${F_LOSNING_PROVIDER_POOL_SELECTION_REASON:-}}"
+  ACTIVE_PROVIDER_NEXT_ATTEMPT_EPOCH="${ACP_PROVIDER_POOL_NEXT_ATTEMPT_EPOCH:-${F_LOSNING_PROVIDER_POOL_NEXT_ATTEMPT_EPOCH:-}}"
+  ACTIVE_PROVIDER_NEXT_ATTEMPT_AT="${ACP_PROVIDER_POOL_NEXT_ATTEMPT_AT:-${F_LOSNING_PROVIDER_POOL_NEXT_ATTEMPT_AT:-}}"
+  ACTIVE_PROVIDER_LAST_REASON="${ACP_PROVIDER_POOL_LAST_REASON:-${F_LOSNING_PROVIDER_POOL_LAST_REASON:-}}"
+
+  if [[ -z "${ACTIVE_PROVIDER_MODEL}" ]]; then
+    case "${ACTIVE_PROVIDER_BACKEND}" in
+      openclaw)
+        ACTIVE_PROVIDER_MODEL="${ACP_OPENCLAW_MODEL:-${F_LOSNING_OPENCLAW_MODEL:-}}"
+        ;;
+      claude)
+        ACTIVE_PROVIDER_MODEL="${ACP_CLAUDE_MODEL:-${F_LOSNING_CLAUDE_MODEL:-}}"
+        ;;
+      codex)
+        ACTIVE_PROVIDER_MODEL="${ACP_CODEX_PROFILE_SAFE:-${F_LOSNING_CODEX_PROFILE_SAFE:-}}"
+        ;;
+      opencode)
+        ACTIVE_PROVIDER_MODEL="${ACP_OPENCODE_MODEL:-${F_LOSNING_OPENCODE_MODEL:-}}"
+        ;;
+      kilo)
+        ACTIVE_PROVIDER_MODEL="${ACP_KILO_MODEL:-${F_LOSNING_KILO_MODEL:-}}"
+        ;;
+    esac
+  fi
+
+  if [[ -z "${ACTIVE_PROVIDER_KEY}" && -n "${ACTIVE_PROVIDER_BACKEND}" && -n "${ACTIVE_PROVIDER_MODEL}" ]]; then
+    ACTIVE_PROVIDER_KEY="$(flow_sanitize_provider_key "${ACTIVE_PROVIDER_BACKEND}-${ACTIVE_PROVIDER_MODEL}")"
+  fi
+}
+
+controller_set_recorded_provider_from_active() {
+  LAST_RECORDED_PROVIDER_POOL_NAME="${ACTIVE_PROVIDER_POOL_NAME}"
+  LAST_RECORDED_PROVIDER_BACKEND="${ACTIVE_PROVIDER_BACKEND}"
+  LAST_RECORDED_PROVIDER_MODEL="${ACTIVE_PROVIDER_MODEL}"
+  LAST_RECORDED_PROVIDER_KEY="${ACTIVE_PROVIDER_KEY}"
+}
+
+controller_mark_provider_launched() {
+  LAST_LAUNCHED_PROVIDER_POOL_NAME="${ACTIVE_PROVIDER_POOL_NAME}"
+  LAST_LAUNCHED_PROVIDER_BACKEND="${ACTIVE_PROVIDER_BACKEND}"
+  LAST_LAUNCHED_PROVIDER_MODEL="${ACTIVE_PROVIDER_MODEL}"
+  LAST_LAUNCHED_PROVIDER_KEY="${ACTIVE_PROVIDER_KEY}"
+
+  if [[ -z "${LAST_RECORDED_PROVIDER_KEY}" ]]; then
+    controller_set_recorded_provider_from_active
+  fi
+}
+
+controller_track_provider_selection() {
+  local reason="${1:-provider-selection}"
+  local now_at=""
+
+  [[ -n "${ACTIVE_PROVIDER_KEY}" ]] || return 0
+
+  if [[ -z "${LAST_RECORDED_PROVIDER_KEY}" ]]; then
+    controller_set_recorded_provider_from_active
+    return 0
+  fi
+
+  if [[ "${ACTIVE_PROVIDER_KEY}" == "${LAST_RECORDED_PROVIDER_KEY}" ]]; then
+    return 0
+  fi
+
+  now_at="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
+  PROVIDER_SWITCH_COUNT=$((PROVIDER_SWITCH_COUNT + 1))
+  LAST_PROVIDER_SWITCH_AT="${now_at}"
+  LAST_PROVIDER_SWITCH_REASON="${reason}"
+  LAST_PROVIDER_FROM_POOL_NAME="${LAST_RECORDED_PROVIDER_POOL_NAME}"
+  LAST_PROVIDER_FROM_BACKEND="${LAST_RECORDED_PROVIDER_BACKEND}"
+  LAST_PROVIDER_FROM_MODEL="${LAST_RECORDED_PROVIDER_MODEL}"
+  LAST_PROVIDER_FROM_KEY="${LAST_RECORDED_PROVIDER_KEY}"
+  LAST_PROVIDER_TO_POOL_NAME="${ACTIVE_PROVIDER_POOL_NAME}"
+  LAST_PROVIDER_TO_BACKEND="${ACTIVE_PROVIDER_BACKEND}"
+  LAST_PROVIDER_TO_MODEL="${ACTIVE_PROVIDER_MODEL}"
+  LAST_PROVIDER_TO_KEY="${ACTIVE_PROVIDER_KEY}"
+
+  if [[ "${reason}" == "provider-failover" ]]; then
+    PROVIDER_FAILOVER_COUNT=$((PROVIDER_FAILOVER_COUNT + 1))
+    LAST_PROVIDER_FAILOVER_AT="${now_at}"
+  fi
+
+  controller_set_recorded_provider_from_active
+}
+
+controller_adopt_issue() {
+  local next_issue_id="${1:?issue id required}"
+  local previous_issue_id="${ISSUE_ID:-}"
+  local previous_controller_file="${CONTROLLER_FILE:-}"
+
+  if [[ -n "${previous_issue_id}" && "${previous_issue_id}" != "${next_issue_id}" ]]; then
+    controller_unregister_pending_issue "${previous_issue_id}"
+    if [[ -n "${previous_controller_file}" && -f "${previous_controller_file}" ]]; then
+      rm -f "${previous_controller_file}"
+    fi
+  fi
+
+  ISSUE_ID="${next_issue_id}"
+  SESSION="${ISSUE_SESSION_PREFIX}${ISSUE_ID}"
+  CONTROLLER_FILE="$(flow_resident_issue_controller_file "${CONFIG_YAML}" "${ISSUE_ID}")"
+  RESIDENT_META_FILE="$(flow_resident_issue_meta_file "${CONFIG_YAML}" "${ISSUE_ID}")"
+  CONTROLLER_LOOP_COUNT="0"
+  NEXT_WAKE_EPOCH=""
+  NEXT_WAKE_AT=""
+  IDLE_WAIT_STARTED_EPOCH=""
+  ACTIVE_RESIDENT_WORKER_KEY=""
+  ACTIVE_RESIDENT_META_FILE=""
+  ACTIVE_RESIDENT_LANE_KIND=""
+  ACTIVE_RESIDENT_LANE_VALUE=""
+}
+
+controller_mark_issue_running() {
+  local is_heavy="no"
+
+  if declare -F heartbeat_issue_is_heavy >/dev/null 2>&1; then
+    is_heavy="$(heartbeat_issue_is_heavy "${ISSUE_ID}" 2>/dev/null || printf 'no\n')"
+  fi
+
+  if declare -F heartbeat_mark_issue_running >/dev/null 2>&1; then
+    heartbeat_mark_issue_running "${ISSUE_ID}" "${is_heavy}" >/dev/null 2>&1 || true
+  fi
+}
+
+controller_rollback_issue_launch() {
+  if declare -F heartbeat_issue_launch_failed >/dev/null 2>&1; then
+    heartbeat_issue_launch_failed "${ISSUE_ID}" >/dev/null 2>&1 || true
+  fi
+}
+
+controller_adopt_next_recurring_issue() {
+  local next_issue_id=""
+  local claim_out=""
+  local claim_file=""
+
+  claim_out="$(flow_resident_issue_claim_next "${CONFIG_YAML}" "${SESSION}" "${ISSUE_ID}" || true)"
+  next_issue_id="$(awk -F= '/^ISSUE_ID=/{print $2}' <<<"${claim_out}")"
+  claim_file="$(awk -F= '/^CLAIM_FILE=/{print $2}' <<<"${claim_out}")"
+  if [[ -z "${next_issue_id}" ]]; then
+    next_issue_id="$(select_next_recurring_issue_id || true)"
+  fi
+  [[ -n "${next_issue_id}" ]] || return 1
+
+  controller_adopt_issue "${next_issue_id}"
+  flow_resident_issue_release_claim "${claim_file}"
+  CONTROLLER_REASON="adopted-next-recurring-issue"
+  controller_write_state "adopting-issue" ""
+  return 0
+}
+
+controller_wait_for_leased_issue() {
+  local idle_timeout="${IDLE_TIMEOUT_SECONDS:-0}"
+  local now_epoch=""
+
+  case "${idle_timeout}" in
+    ''|*[!0-9]*) idle_timeout="0" ;;
+  esac
+
+  if [[ "${idle_timeout}" -le 0 ]]; then
+    return 1
+  fi
+
+  if [[ -z "${IDLE_WAIT_STARTED_EPOCH}" ]]; then
+    IDLE_WAIT_STARTED_EPOCH="$(date +%s)"
+  fi
+
+  while true; do
+    if controller_adopt_next_recurring_issue; then
+      return 0
+    fi
+
+    now_epoch="$(date +%s)"
+    if (( now_epoch - IDLE_WAIT_STARTED_EPOCH >= idle_timeout )); then
+      CONTROLLER_REASON="idle-timeout"
+      return 1
+    fi
+
+    controller_write_state "idle" ""
+    sleep "${POLL_SECONDS}"
+  done
+}
+
+controller_write_state() {
+  local state="${1:?state required}"
+  local reason="${2:-${CONTROLLER_REASON}}"
+
+  CONTROLLER_STATE="${state}"
+  CONTROLLER_REASON="${reason}"
+  flow_resident_write_metadata "${CONTROLLER_FILE}" \
+    "ISSUE_ID=${ISSUE_ID}" \
+    "SESSION=${SESSION}" \
+    "CONTROLLER_PID=$$" \
+    "CONTROLLER_MODE=${MODE}" \
+    "CONTROLLER_LOOP_COUNT=${CONTROLLER_LOOP_COUNT}" \
+    "CONTROLLER_STATE=${CONTROLLER_STATE}" \
+    "CONTROLLER_REASON=${CONTROLLER_REASON}" \
+    "ACTIVE_RESIDENT_WORKER_KEY=${ACTIVE_RESIDENT_WORKER_KEY}" \
+    "ACTIVE_RESIDENT_LANE_KIND=${ACTIVE_RESIDENT_LANE_KIND}" \
+    "ACTIVE_RESIDENT_LANE_VALUE=${ACTIVE_RESIDENT_LANE_VALUE}" \
+    "ACTIVE_PROVIDER_POOL_NAME=${ACTIVE_PROVIDER_POOL_NAME}" \
+    "ACTIVE_PROVIDER_BACKEND=${ACTIVE_PROVIDER_BACKEND}" \
+    "ACTIVE_PROVIDER_MODEL=${ACTIVE_PROVIDER_MODEL}" \
+    "ACTIVE_PROVIDER_KEY=${ACTIVE_PROVIDER_KEY}" \
+    "ACTIVE_PROVIDER_SELECTION_REASON=${ACTIVE_PROVIDER_SELECTION_REASON}" \
+    "ACTIVE_PROVIDER_NEXT_ATTEMPT_EPOCH=${ACTIVE_PROVIDER_NEXT_ATTEMPT_EPOCH}" \
+    "ACTIVE_PROVIDER_NEXT_ATTEMPT_AT=${ACTIVE_PROVIDER_NEXT_ATTEMPT_AT}" \
+    "ACTIVE_PROVIDER_LAST_REASON=${ACTIVE_PROVIDER_LAST_REASON}" \
+    "LAST_LAUNCHED_PROVIDER_POOL_NAME=${LAST_LAUNCHED_PROVIDER_POOL_NAME}" \
+    "LAST_LAUNCHED_PROVIDER_BACKEND=${LAST_LAUNCHED_PROVIDER_BACKEND}" \
+    "LAST_LAUNCHED_PROVIDER_MODEL=${LAST_LAUNCHED_PROVIDER_MODEL}" \
+    "LAST_LAUNCHED_PROVIDER_KEY=${LAST_LAUNCHED_PROVIDER_KEY}" \
+    "PROVIDER_SWITCH_COUNT=${PROVIDER_SWITCH_COUNT}" \
+    "PROVIDER_FAILOVER_COUNT=${PROVIDER_FAILOVER_COUNT}" \
+    "LAST_PROVIDER_SWITCH_AT=${LAST_PROVIDER_SWITCH_AT}" \
+    "LAST_PROVIDER_SWITCH_REASON=${LAST_PROVIDER_SWITCH_REASON}" \
+    "LAST_PROVIDER_FROM_POOL_NAME=${LAST_PROVIDER_FROM_POOL_NAME}" \
+    "LAST_PROVIDER_FROM_BACKEND=${LAST_PROVIDER_FROM_BACKEND}" \
+    "LAST_PROVIDER_FROM_MODEL=${LAST_PROVIDER_FROM_MODEL}" \
+    "LAST_PROVIDER_FROM_KEY=${LAST_PROVIDER_FROM_KEY}" \
+    "LAST_PROVIDER_TO_POOL_NAME=${LAST_PROVIDER_TO_POOL_NAME}" \
+    "LAST_PROVIDER_TO_BACKEND=${LAST_PROVIDER_TO_BACKEND}" \
+    "LAST_PROVIDER_TO_MODEL=${LAST_PROVIDER_TO_MODEL}" \
+    "LAST_PROVIDER_TO_KEY=${LAST_PROVIDER_TO_KEY}" \
+    "LAST_PROVIDER_FAILOVER_AT=${LAST_PROVIDER_FAILOVER_AT}" \
+    "PROVIDER_WAIT_COUNT=${PROVIDER_WAIT_COUNT}" \
+    "PROVIDER_WAIT_TOTAL_SECONDS=${PROVIDER_WAIT_TOTAL_SECONDS}" \
+    "PROVIDER_LAST_WAIT_SECONDS=${PROVIDER_LAST_WAIT_SECONDS}" \
+    "PROVIDER_LAST_WAIT_STARTED_AT=${PROVIDER_LAST_WAIT_STARTED_AT}" \
+    "PROVIDER_LAST_WAIT_COMPLETED_AT=${PROVIDER_LAST_WAIT_COMPLETED_AT}" \
+    "NEXT_WAKE_EPOCH=${NEXT_WAKE_EPOCH}" \
+    "NEXT_WAKE_AT=${NEXT_WAKE_AT}" \
+    "UPDATED_AT=$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
+
+  if [[ "${CONTROLLER_STATE}" == "stopped" ]]; then
+    controller_unregister_pending_issue "${ISSUE_ID}"
+  elif flow_resident_issue_controller_counts_as_pending "${CONTROLLER_STATE}"; then
+    controller_register_pending_issue
+  else
+    controller_unregister_pending_issue "${ISSUE_ID}"
+  fi
+}
+
+controller_last_failure_reason() {
+  local metadata_file="${ACTIVE_RESIDENT_META_FILE:-${RESIDENT_META_FILE:-}}"
+  [[ -n "${metadata_file}" && -f "${metadata_file}" ]] || return 1
+  awk -F= '/^LAST_FAILURE_REASON=/{print $2; exit}' "${metadata_file}" 2>/dev/null | tr -d '"' || true
+}
+
+controller_provider_state() {
+  local provider_state_script="${FLOW_TOOLS_DIR}/provider-cooldown-state.sh"
+  local provider_state=""
+
+  if [[ ! -x "${provider_state_script}" ]]; then
+    printf 'READY=yes\n'
+    return 0
+  fi
+
+  provider_state="$(
+    env \
+      -u ACP_CODING_WORKER \
+      -u ACP_CODEX_PROFILE_SAFE -u F_LOSNING_CODEX_PROFILE_SAFE \
+      -u ACP_CODEX_PROFILE_BYPASS -u F_LOSNING_CODEX_PROFILE_BYPASS \
+      -u ACP_CLAUDE_MODEL -u F_LOSNING_CLAUDE_MODEL \
+      -u ACP_CLAUDE_PERMISSION_MODE -u F_LOSNING_CLAUDE_PERMISSION_MODE \
+      -u ACP_CLAUDE_EFFORT -u F_LOSNING_CLAUDE_EFFORT \
+      -u ACP_CLAUDE_TIMEOUT_SECONDS -u F_LOSNING_CLAUDE_TIMEOUT_SECONDS \
+      -u ACP_CLAUDE_MAX_ATTEMPTS -u F_LOSNING_CLAUDE_MAX_ATTEMPTS \
+      -u ACP_CLAUDE_RETRY_BACKOFF_SECONDS -u F_LOSNING_CLAUDE_RETRY_BACKOFF_SECONDS \
+      -u ACP_OPENCLAW_MODEL -u F_LOSNING_OPENCLAW_MODEL \
+      -u ACP_OPENCLAW_THINKING -u F_LOSNING_OPENCLAW_THINKING \
+      -u ACP_OPENCLAW_TIMEOUT_SECONDS -u F_LOSNING_OPENCLAW_TIMEOUT_SECONDS \
+      -u ACP_ACTIVE_PROVIDER_POOL_NAME -u F_LOSNING_ACTIVE_PROVIDER_POOL_NAME \
+      -u ACP_ACTIVE_PROVIDER_BACKEND -u F_LOSNING_ACTIVE_PROVIDER_BACKEND \
+      -u ACP_ACTIVE_PROVIDER_MODEL -u F_LOSNING_ACTIVE_PROVIDER_MODEL \
+      -u ACP_ACTIVE_PROVIDER_KEY -u F_LOSNING_ACTIVE_PROVIDER_KEY \
+      -u ACP_PROVIDER_POOLS_EXHAUSTED -u F_LOSNING_PROVIDER_POOLS_EXHAUSTED \
+      -u ACP_PROVIDER_POOL_SELECTION_REASON -u F_LOSNING_PROVIDER_POOL_SELECTION_REASON \
+      -u ACP_PROVIDER_POOL_NEXT_ATTEMPT_EPOCH -u F_LOSNING_PROVIDER_POOL_NEXT_ATTEMPT_EPOCH \
+      -u ACP_PROVIDER_POOL_NEXT_ATTEMPT_AT -u F_LOSNING_PROVIDER_POOL_NEXT_ATTEMPT_AT \
+      -u ACP_PROVIDER_POOL_LAST_REASON -u F_LOSNING_PROVIDER_POOL_LAST_REASON \
+      "${provider_state_script}" get 2>/dev/null || true
+  )"
+  if [[ -z "${provider_state}" ]]; then
+    printf 'READY=yes\n'
+    return 0
+  fi
+
+  printf '%s\n' "${provider_state}"
+}
+
+controller_wait_for_provider_capacity() {
+  local provider_state=""
+  local provider_ready=""
+  local provider_next_epoch=""
+  local provider_next_at=""
+  local now_epoch=""
+  local remaining=""
+  local sleep_seconds=""
+  local wait_started_epoch=""
+  local wait_completed_epoch=""
+
+  PROVIDER_WAITED="no"
+
+  while true; do
+    provider_state="$(controller_provider_state)"
+    provider_ready="$(flow_kv_get "${provider_state}" "READY")"
+    if [[ "${provider_ready}" == "yes" ]]; then
+      if [[ -n "${wait_started_epoch}" ]]; then
+        wait_completed_epoch="$(date +%s)"
+        if (( wait_completed_epoch >= wait_started_epoch )); then
+          PROVIDER_LAST_WAIT_SECONDS=$((wait_completed_epoch - wait_started_epoch))
+          PROVIDER_WAIT_TOTAL_SECONDS=$((PROVIDER_WAIT_TOTAL_SECONDS + PROVIDER_LAST_WAIT_SECONDS))
+          PROVIDER_LAST_WAIT_COMPLETED_AT="$(date -u -r "${wait_completed_epoch}" +"%Y-%m-%dT%H:%M:%SZ")"
+        fi
+      fi
+      NEXT_WAKE_EPOCH=""
+      NEXT_WAKE_AT=""
+      return 0
+    fi
+
+    provider_next_epoch="$(flow_kv_get "${provider_state}" "NEXT_ATTEMPT_EPOCH")"
+    provider_next_at="$(flow_kv_get "${provider_state}" "NEXT_ATTEMPT_AT")"
+    if ! [[ "${provider_next_epoch}" =~ ^[0-9]+$ ]] || [[ "${provider_next_epoch}" == "0" ]]; then
+      return 1
+    fi
+
+    if [[ -z "${wait_started_epoch}" ]]; then
+      wait_started_epoch="$(date +%s)"
+      PROVIDER_WAIT_COUNT=$((PROVIDER_WAIT_COUNT + 1))
+      PROVIDER_LAST_WAIT_STARTED_AT="$(date -u -r "${wait_started_epoch}" +"%Y-%m-%dT%H:%M:%SZ")"
+    fi
+
+    PROVIDER_WAITED="yes"
+    NEXT_WAKE_EPOCH="${provider_next_epoch}"
+    NEXT_WAKE_AT="${provider_next_at}"
+    CONTROLLER_REASON="provider-cooldown"
+    controller_write_state "waiting-provider" ""
+
+    now_epoch="$(date +%s)"
+    remaining=$((provider_next_epoch - now_epoch))
+    sleep_seconds="${POLL_SECONDS}"
+    if ! [[ "${sleep_seconds}" =~ ^[1-9][0-9]*$ ]]; then
+      sleep_seconds="60"
+    fi
+    if (( remaining > 0 && remaining < sleep_seconds )); then
+      sleep_seconds="${remaining}"
+    fi
+    if (( sleep_seconds <= 0 )); then
+      sleep_seconds="1"
+    fi
+    sleep "${sleep_seconds}"
+  done
+}
+
+controller_cleanup() {
+  controller_write_state "stopped" "${CONTROLLER_REASON:-stopped}"
+}

package/tools/bin/resident-issue-queue-status.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+
+
+ROOT_DIR = Path(__file__).resolve().parents[1]
+DASHBOARD_DIR = ROOT_DIR / "dashboard"
+if str(DASHBOARD_DIR) not in sys.path:
+    sys.path.insert(0, str(DASHBOARD_DIR))
+
+from issue_queue_state import collect_issue_queue
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Render resident issue queue state as JSON.")
+    parser.add_argument("--state-root", default=os.environ.get("ACP_STATE_ROOT", "").strip(), help="ACP runtime state root")
+    parser.add_argument("--pretty", action="store_true", help="Pretty-print JSON output")
+    args = parser.parse_args()
+
+    if not args.state_root:
+        parser.error("--state-root is required")
+
+    payload = collect_issue_queue(Path(args.state_root).expanduser())
+    json.dump(payload, sys.stdout, indent=2 if args.pretty else None, sort_keys=True)
+    sys.stdout.write("\n")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/tools/bin/run-codex-task.sh

@@ -35,7 +35,7 @@ RECONCILE_COMMAND=""
 ADAPTER_ID="$(flow_resolve_adapter_id "${CONFIG_YAML}")"
 ISSUE_SESSION_PREFIX="$(flow_resolve_issue_session_prefix "${CONFIG_YAML}")"
 PR_SESSION_PREFIX="$(flow_resolve_pr_session_prefix "${CONFIG_YAML}")"
-CODING_WORKER="${ACP_CODING_WORKER:-${F_LOSNING_CODING_WORKER:-codex}}"
+CODING_WORKER="${ACP_CODING_WORKER:-codex}"
 CODEX_PROFILE_SAFE="${ACP_CODEX_PROFILE_SAFE:-${F_LOSNING_CODEX_PROFILE_SAFE:-${CODEX_PROFILE_SAFE:-f_losning_safe_auto}}}"
 CODEX_PROFILE_BYPASS="${ACP_CODEX_PROFILE_BYPASS:-${F_LOSNING_CODEX_PROFILE_BYPASS:-${CODEX_PROFILE_BYPASS:-f_losning_yolo}}}"
 if [[ "$MODE" == "bypass" ]]; then
@@ -64,23 +64,47 @@ RESIDENT_OPENCLAW_CONFIG_PATH="${ACP_RESIDENT_OPENCLAW_CONFIG_PATH:-${F_LOSNING_
 OPENCLAW_MODEL="${OPENCLAW_MODEL:-${ACP_OPENCLAW_MODEL:-${F_LOSNING_OPENCLAW_MODEL:-openrouter/qwen/qwen3.6-plus-preview:free}}}"
 OPENCLAW_THINKING="${OPENCLAW_THINKING:-${ACP_OPENCLAW_THINKING:-${F_LOSNING_OPENCLAW_THINKING:-low}}}"
 OPENCLAW_TIMEOUT_SECONDS="${OPENCLAW_TIMEOUT_SECONDS:-${ACP_OPENCLAW_TIMEOUT_SECONDS:-${F_LOSNING_OPENCLAW_TIMEOUT_SECONDS:-900}}}"
+OLLAMA_MODEL="${ACP_OLLAMA_MODEL:-${F_LOSNING_OLLAMA_MODEL:-qwen2.5-coder:7b}}"
+OLLAMA_BASE_URL="${ACP_OLLAMA_BASE_URL:-${F_LOSNING_OLLAMA_BASE_URL:-http://localhost:11434}}"
+OLLAMA_TIMEOUT_SECONDS="${ACP_OLLAMA_TIMEOUT_SECONDS:-${F_LOSNING_OLLAMA_TIMEOUT_SECONDS:-900}}"
+PI_MODEL="${ACP_PI_MODEL:-${F_LOSNING_PI_MODEL:-openrouter/qwen/qwen3.6-plus:free}}"
+PI_THINKING="${ACP_PI_THINKING:-${F_LOSNING_PI_THINKING:-low}}"
+PI_TIMEOUT_SECONDS="${ACP_PI_TIMEOUT_SECONDS:-${F_LOSNING_PI_TIMEOUT_SECONDS:-900}}"
+PI_STALL_SECONDS="${ACP_PI_STALL_SECONDS:-${F_LOSNING_PI_STALL_SECONDS:-300}}"
+OPENCODE_MODEL="${ACP_OPENCODE_MODEL:-${F_LOSNING_OPENCODE_MODEL:-anthropic/claude-sonnet-4-20250514}}"
+OPENCODE_TIMEOUT_SECONDS="${ACP_OPENCODE_TIMEOUT_SECONDS:-${F_LOSNING_OPENCODE_TIMEOUT_SECONDS:-900}}"
+KILO_MODEL="${ACP_KILO_MODEL:-${F_LOSNING_KILO_MODEL:-anthropic/claude-sonnet-4-20250514}}"
+KILO_TIMEOUT_SECONDS="${ACP_KILO_TIMEOUT_SECONDS:-${F_LOSNING_KILO_TIMEOUT_SECONDS:-900}}"
 printf -v SESSION_Q '%q' "$SESSION"
 printf -v CONFIG_YAML_Q '%q' "$CONFIG_YAML"
 printf -v ADAPTER_ID_Q '%q' "$ADAPTER_ID"
-RECONCILE_ENV_PREFIX="ACP_PROJECT_ID=${ADAPTER_ID_Q} AGENT_PROJECT_ID=${ADAPTER_ID_Q} AGENT_CONTROL_PLANE_CONFIG=${CONFIG_YAML_Q} ACP_CONFIG=${CONFIG_YAML_Q}"
+printf -v CANONICAL_REPO_ROOT_Q '%q' "$CANONICAL_REPO_ROOT"
+RECONCILE_ENV_PREFIX="ACP_PROJECT_ID=${ADAPTER_ID_Q} AGENT_PROJECT_ID=${ADAPTER_ID_Q} AGENT_CONTROL_PLANE_CONFIG=${CONFIG_YAML_Q} ACP_CONFIG=${CONFIG_YAML_Q} ACP_ROOT=${CANONICAL_REPO_ROOT_Q} AGENT_CONTROL_PLANE_ROOT=${CANONICAL_REPO_ROOT_Q}"
+
+# The materialized skills surface is cleaned up after each heartbeat cycle.
+# Reconcile runs post-tmux via nohup, after the skills dir is gone.
+# Use the permanent runtime-home/tools/bin path derived from AGENT_PLATFORM_HOME.
+_runtime_tools_bin="${AGENT_PLATFORM_HOME:-${HOME}/.agent-runtime}/runtime-home/tools/bin"
+if [[ -f "${_runtime_tools_bin}/reconcile-pr-worker.sh" ]]; then
+  RECONCILE_ISSUE_BIN="${_runtime_tools_bin}/reconcile-issue-worker.sh"
+  RECONCILE_PR_BIN="${_runtime_tools_bin}/reconcile-pr-worker.sh"
+else
+  RECONCILE_ISSUE_BIN="${WORKSPACE_DIR}/bin/reconcile-issue-worker.sh"
+  RECONCILE_PR_BIN="${WORKSPACE_DIR}/bin/reconcile-pr-worker.sh"
+fi
 
 case "$SESSION" in
   "${ISSUE_SESSION_PREFIX}"*)
     TASK_KIND="issue"
     TASK_ID="${ISSUE_ID:-${SESSION#${ISSUE_SESSION_PREFIX}}}"
     if [[ "${RESIDENT_WORKER_ENABLED}" != "yes" ]]; then
-      RECONCILE_COMMAND="${RECONCILE_ENV_PREFIX} ${WORKSPACE_DIR}/bin/reconcile-issue-worker.sh ${SESSION_Q}"
+      RECONCILE_COMMAND="${RECONCILE_ENV_PREFIX} ${RECONCILE_ISSUE_BIN} ${SESSION_Q}"
     fi
     ;;
   "${PR_SESSION_PREFIX}"*)
     TASK_KIND="pr"
     TASK_ID="${PR_NUMBER:-${SESSION#${PR_SESSION_PREFIX}}}"
-    RECONCILE_COMMAND="${RECONCILE_ENV_PREFIX} ${WORKSPACE_DIR}/bin/reconcile-pr-worker.sh ${SESSION_Q}"
+    RECONCILE_COMMAND="${RECONCILE_ENV_PREFIX} ${RECONCILE_PR_BIN} ${SESSION_Q}"
     ;;
 esac
 
@@ -219,11 +243,36 @@ case "$CODING_WORKER" in
     bash "${FLOW_TOOLS_DIR}/agent-project-run-openclaw-session" "${ARGS[@]}"
     ;;
   opencode)
+    ARGS+=(
+      --opencode-model "${OPENCODE_MODEL}"
+      --opencode-timeout-seconds "${OPENCODE_TIMEOUT_SECONDS}"
+    )
     bash "${FLOW_TOOLS_DIR}/agent-project-run-opencode-session" "${ARGS[@]}"
     ;;
   kilo)
+    ARGS+=(
+      --kilo-model "${KILO_MODEL}"
+      --kilo-timeout-seconds "${KILO_TIMEOUT_SECONDS}"
+    )
     bash "${FLOW_TOOLS_DIR}/agent-project-run-kilo-session" "${ARGS[@]}"
     ;;
+  ollama)
+    ARGS+=(
+      --ollama-model "${OLLAMA_MODEL}"
+      --ollama-base-url "${OLLAMA_BASE_URL}"
+      --ollama-timeout-seconds "${OLLAMA_TIMEOUT_SECONDS}"
+    )
+    bash "${FLOW_TOOLS_DIR}/agent-project-run-ollama-session" "${ARGS[@]}"
+    ;;
+  pi)
+    ARGS+=(
+      --pi-model "${PI_MODEL}"
+      --pi-thinking "${PI_THINKING}"
+      --pi-timeout-seconds "${PI_TIMEOUT_SECONDS}"
+      --pi-stall-seconds "${PI_STALL_SECONDS}"
+    )
+    bash "${FLOW_TOOLS_DIR}/agent-project-run-pi-session" "${ARGS[@]}"
+    ;;
   *)
     echo "unsupported coding worker: ${CODING_WORKER}" >&2
     exit 1

package/tools/bin/scaffold-profile.sh

@@ -22,7 +22,7 @@ Options:
   --worktree-root <path>             Worktree parent root
   --retained-repo-root <path>        Optional retained/manual checkout root
   --vscode-workspace-file <path>     Optional VS Code workspace file
-  --coding-worker <codex|openclaw|claude>
+  --coding-worker <codex|openclaw|claude|ollama|pi|opencode|kilo>
                                      Default coding backend (default: openclaw)
   --claude-model <model>             Claude model alias or full name
   --claude-permission-mode <mode>    Claude permission mode (default: acceptEdits)
@@ -97,9 +97,9 @@ if [[ ! "$profile_id" =~ ^[a-z0-9][a-z0-9-]*$ ]]; then
 fi
 
 case "$coding_worker" in
-  codex|openclaw|claude) ;;
+  codex|openclaw|claude|ollama|pi|opencode|kilo) ;;
   *)
-    echo "--coding-worker must be codex, openclaw, or claude" >&2
+    echo "--coding-worker must be codex, openclaw, claude, ollama, pi, opencode, or kilo" >&2
     exit 1
     ;;
 esac
@@ -271,6 +271,21 @@ execution:
     model: "${openclaw_model}"
     thinking: "${openclaw_thinking}"
     timeout_seconds: ${openclaw_timeout_seconds}
+  ollama:
+    model: "qwen2.5-coder:7b"
+    base_url: "http://localhost:11434"
+    timeout_seconds: 900
+  pi:
+    model: "openrouter/qwen/qwen3.6-plus:free"
+    thinking: "low"
+    timeout_seconds: 900
+    stall_seconds: 300
+  opencode:
+    model: "anthropic/claude-sonnet-4-20250514"
+    timeout_seconds: 900
+  kilo:
+    model: "anthropic/claude-sonnet-4-20250514"
+    timeout_seconds: 900
   review_requires_independent_final_review: true
   verification:
     web_playwright_command: "pnpm exec playwright test"

package/tools/bin/start-issue-worker.sh

@@ -30,7 +30,7 @@ LOCAL_INSTALL_POLICY_BIN="${WORKSPACE_DIR}/bin/issue-requires-local-workspace-in
 SESSION="${ISSUE_SESSION_PREFIX}${ISSUE_ID}"
 RUN_DIR="${RUNS_ROOT}/${SESSION}"
 UPDATE_LABELS_BIN="${WORKSPACE_DIR}/bin/agent-github-update-labels"
-CODING_WORKER="${ACP_CODING_WORKER:-${F_LOSNING_CODING_WORKER:-codex}}"
+CODING_WORKER="${ACP_CODING_WORKER:-codex}"
 launch_success="no"
 label_rollback_armed="no"
 RECURRING_CHECKLIST_SYNC_BIN="${FLOW_TOOLS_DIR}/sync-recurring-issue-checklist.sh"