agent-control-plane 0.2.0 → 0.3.0

package/tools/bin/heartbeat-loop-worker-lib.sh

@@ -0,0 +1,319 @@
+#!/usr/bin/env bash
+# heartbeat-loop-worker-lib.sh — tmux session queries, worker enumeration, and cache helpers
+
+all_tmux_sessions() {
+  ensure_tmux_sessions_cache
+  printf '%s\n' "$tmux_sessions_cache"
+}
+
+session_matches_prefix() {
+  local session="${1:?session required}"
+  [[ "$session" == "${issue_prefix}"* || "$session" == "${pr_prefix}"* ]]
+}
+
+session_runner_state() {
+  local session="${1:?session required}"
+  local runner_state_file="${runs_root}/${session}/runner.env"
+  if [[ ! -f "$runner_state_file" ]]; then
+    return 1
+  fi
+  awk -F= '/^RUNNER_STATE=/{print $2; exit}' "$runner_state_file"
+}
+
+session_is_auth_waiting() {
+  local session="${1:?session required}"
+  local runner_state=""
+  runner_state="$(session_runner_state "$session" || true)"
+  [[ "$runner_state" == "waiting-auth-refresh" || "$runner_state" == "switching-account" ]]
+}
+
+all_running_workers() {
+  ensure_all_running_workers_cache
+  printf '%s\n' "$all_running_workers_cache"
+}
+
+running_issue_workers() {
+  ensure_running_issue_workers_cache
+  printf '%s\n' "$running_issue_workers_cache"
+}
+
+running_pr_workers() {
+  ensure_running_pr_workers_cache
+  printf '%s\n' "$running_pr_workers_cache"
+}
+
+auth_wait_workers() {
+  ensure_auth_wait_workers_cache
+  printf '%s\n' "$auth_wait_workers_cache"
+}
+
+pending_launch_pid() {
+  local kind="${1:?kind required}"
+  local item_id="${2:?item id required}"
+  local pending_file pid
+
+  pending_file="${pending_launch_dir}/${kind}-${item_id}.pid"
+  if [[ ! -f "$pending_file" ]]; then
+    return 1
+  fi
+
+  pid="$(tr -d '[:space:]' <"$pending_file" 2>/dev/null || true)"
+  if [[ -z "$pid" ]]; then
+    rm -f "$pending_file"
+    return 1
+  fi
+
+  if kill -0 "$pid" 2>/dev/null; then
+    printf '%s\n' "$pid"
+    return 0
+  fi
+
+  rm -f "$pending_file"
+  return 1
+}
+
+pending_issue_launch_active() {
+  local issue_id="${1:?issue id required}"
+  if tmux has-session -t "${issue_prefix}${issue_id}" 2>/dev/null; then
+    rm -f "${pending_launch_dir}/issue-${issue_id}.pid" 2>/dev/null || true
+    return 1
+  fi
+  pending_launch_pid issue "$issue_id" >/dev/null
+}
+
+pending_pr_launch_active() {
+  local pr_id="${1:?pr id required}"
+  if tmux has-session -t "${pr_prefix}${pr_id}" 2>/dev/null; then
+    rm -f "${pending_launch_dir}/pr-${pr_id}.pid" 2>/dev/null || true
+    return 1
+  fi
+  pending_launch_pid pr "$pr_id" >/dev/null
+}
+
+pending_issue_launch_counts_toward_capacity() {
+  local issue_id="${1:?issue id required}"
+  local controller_state=""
+
+  if ! pending_issue_launch_active "${issue_id}"; then
+    return 1
+  fi
+
+  controller_state="$(resident_issue_controller_state "${issue_id}" || true)"
+  if [[ -n "${controller_state}" ]]; then
+    case "${controller_state}" in
+      idle|sleeping|waiting-due|waiting-open-pr|waiting-provider)
+        return 1
+        ;;
+    esac
+  fi
+
+  return 0
+}
+
+resident_issue_controller_file() {
+  local issue_id="${1:?issue id required}"
+  printf '%s/resident-workers/issues/%s/controller.env\n' "${state_root}" "${issue_id}"
+}
+
+resident_issue_controller_state() {
+  local issue_id="${1:?issue id required}"
+  local controller_file state=""
+
+  controller_file="$(resident_issue_controller_file "$issue_id")"
+  [[ -f "${controller_file}" ]] || return 1
+
+  state="$(awk -F= '/^CONTROLLER_STATE=/{print $2; exit}' "${controller_file}" 2>/dev/null | tr -d '"' || true)"
+  [[ -n "${state}" ]] || return 1
+  printf '%s\n' "${state}"
+}
+
+issue_id_from_session() {
+  local session="${1:?session required}"
+  local issue_id=""
+  if [[ "$session" == "${issue_prefix}"* ]]; then
+    issue_id="${session#${issue_prefix}}"
+  fi
+  if [[ "$issue_id" =~ ^[0-9]+$ ]]; then
+    printf '%s\n' "$issue_id"
+    return 0
+  fi
+  return 1
+}
+
+pr_id_from_session() {
+  local session="${1:?session required}"
+  local pr_id=""
+  if [[ "$session" == "${pr_prefix}"* ]]; then
+    pr_id="${session#${pr_prefix}}"
+  fi
+  if [[ "$pr_id" =~ ^[0-9]+$ ]]; then
+    printf '%s\n' "$pr_id"
+    return 0
+  fi
+  return 1
+}
+
+worker_count() {
+  local workers="${1:-}"
+  if [[ -z "$workers" ]]; then
+    printf '0\n'
+    return
+  fi
+  printf '%s\n' "$workers" | sed '/^$/d' | wc -l | tr -d ' '
+}
+
+retry_ready() {
+  local kind="${1:?kind required}"
+  local item_id="${2:?item id required}"
+  local retry_out ready
+
+  retry_out="$(
+    "${shared_agent_home}/tools/bin/agent-project-retry-state" \
+      --state-root "$state_root" \
+      --kind "$kind" \
+      --item-id "$item_id" \
+      --action get
+  )"
+  ready="$(awk -F= '/^READY=/{print $2}' <<<"$retry_out")"
+  [[ "$ready" == "yes" ]]
+}
+
+provider_cooldown_state() {
+  "${shared_agent_home}/tools/bin/provider-cooldown-state.sh" get
+}
+
+completed_workers() {
+  ensure_completed_workers_cache
+  printf '%s\n' "$completed_workers_cache"
+}
+
+reconciled_marker_matches_run() {
+  local run_dir="${1:?run dir required}"
+  local marker_file="${run_dir}/reconciled.ok"
+  local run_env="${run_dir}/run.env"
+  local marker_started_at=""
+  local run_started_at=""
+
+  [[ -f "${marker_file}" && -f "${run_env}" ]] || return 1
+
+  marker_started_at="$(awk -F= '/^STARTED_AT=/{print $2}' "${marker_file}" 2>/dev/null | tr -d '"' | tail -n 1 || true)"
+  run_started_at="$(awk -F= '/^STARTED_AT=/{print $2}' "${run_env}" 2>/dev/null | tr -d '"' | tail -n 1 || true)"
+
+  [[ -n "${marker_started_at}" && -n "${run_started_at}" && "${marker_started_at}" == "${run_started_at}" ]]
+}
+
+ensure_tmux_sessions_cache() {
+  if [[ "$tmux_sessions_cache_loaded" != "yes" ]]; then
+    tmux_sessions_cache="$(tmux list-sessions -F '#S' 2>/dev/null || true)"
+    tmux_sessions_cache_loaded="yes"
+  fi
+}
+
+ensure_all_running_workers_cache() {
+  local session
+  if [[ "$all_running_workers_cache_loaded" == "yes" ]]; then
+    return 0
+  fi
+  ensure_tmux_sessions_cache
+  all_running_workers_cache=""
+  while IFS= read -r session; do
+    [[ -n "$session" ]] || continue
+    if session_matches_prefix "$session"; then
+      all_running_workers_cache+="${session}"$'\n'
+    fi
+  done <<<"$tmux_sessions_cache"
+  all_running_workers_cache="${all_running_workers_cache%$'\n'}"
+  all_running_workers_cache_loaded="yes"
+}
+
+ensure_auth_wait_workers_cache() {
+  local session
+  if [[ "$auth_wait_workers_cache_loaded" == "yes" ]]; then
+    return 0
+  fi
+  ensure_tmux_sessions_cache
+  auth_wait_workers_cache=""
+  while IFS= read -r session; do
+    [[ -n "$session" ]] || continue
+    session_matches_prefix "$session" || continue
+    if session_is_auth_waiting "$session"; then
+      auth_wait_workers_cache+="${session}"$'\n'
+    fi
+  done <<<"$tmux_sessions_cache"
+  auth_wait_workers_cache="${auth_wait_workers_cache%$'\n'}"
+  auth_wait_workers_cache_loaded="yes"
+}
+
+ensure_running_issue_workers_cache() {
+  local session
+  if [[ "$running_issue_workers_cache_loaded" == "yes" ]]; then
+    return 0
+  fi
+  ensure_tmux_sessions_cache
+  running_issue_workers_cache=""
+  while IFS= read -r session; do
+    [[ -n "$session" ]] || continue
+    if [[ "$session" == "${issue_prefix}"* ]]; then
+      if session_is_auth_waiting "$session"; then
+        continue
+      fi
+      running_issue_workers_cache+="${session}"$'\n'
+    fi
+  done <<<"$tmux_sessions_cache"
+  running_issue_workers_cache="${running_issue_workers_cache%$'\n'}"
+  running_issue_workers_cache_loaded="yes"
+}
+
+ensure_running_pr_workers_cache() {
+  local session
+  if [[ "$running_pr_workers_cache_loaded" == "yes" ]]; then
+    return 0
+  fi
+  ensure_tmux_sessions_cache
+  running_pr_workers_cache=""
+  while IFS= read -r session; do
+    [[ -n "$session" ]] || continue
+    if [[ "$session" == "${pr_prefix}"* ]]; then
+      if session_is_auth_waiting "$session"; then
+        continue
+      fi
+      running_pr_workers_cache+="${session}"$'\n'
+    fi
+  done <<<"$tmux_sessions_cache"
+  running_pr_workers_cache="${running_pr_workers_cache%$'\n'}"
+  running_pr_workers_cache_loaded="yes"
+}
+
+ensure_completed_workers_cache() {
+  local dir session issue_id status_line status
+  if [[ "$completed_workers_cache_loaded" == "yes" ]]; then
+    return 0
+  fi
+  completed_workers_cache=""
+  for dir in "$runs_root"/*; do
+    [[ -d "$dir" ]] || continue
+    session="${dir##*/}"
+    session_matches_prefix "$session" || continue
+    if reconciled_marker_matches_run "$dir"; then
+      continue
+    fi
+    if [[ "$session" == "${issue_prefix}"* ]]; then
+      issue_id="$(issue_id_from_session "$session" || true)"
+      if [[ -n "${issue_id}" ]] && pending_issue_launch_active "${issue_id}"; then
+        continue
+      fi
+    fi
+    status_line="$(
+      "${shared_agent_home}/tools/bin/agent-project-worker-status" \
+        --runs-root "$runs_root" \
+        --session "$session" \
+        | awk -F= '/^STATUS=/{print $2}' || true
+    )"
+    status="${status_line:-UNKNOWN}"
+    if [[ "$status" == "SUCCEEDED" || "$status" == "FAILED" ]]; then
+      completed_workers_cache+="${session}"$'\n'
+    fi
+  done
+  completed_workers_cache="${completed_workers_cache%$'\n'}"
+  completed_workers_cache_loaded="yes"
+}

package/tools/bin/heartbeat-recovery-preflight.sh

@@ -2,14 +2,25 @@
 set -euo pipefail
 
 FLOW_TOOLS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+FLOW_SHELL_LIB="${FLOW_TOOLS_DIR}/flow-shell-lib.sh"
 TEST_DIR="${FLOW_TOOLS_DIR%/bin}/tests"
 TEST_TIMEOUT_SECONDS="${F_LOSNING_HEARTBEAT_PREFLIGHT_TEST_TIMEOUT_SECONDS:-120}"
+python_bin=""
+
+# shellcheck source=/dev/null
+source "${FLOW_SHELL_LIB}"
+
+python_bin="$(flow_resolve_python_bin || true)"
+if [[ -z "${python_bin}" || ! -x "${python_bin}" ]]; then
+  echo "unable to resolve a runnable python interpreter for heartbeat-recovery-preflight.sh" >&2
+  exit 1
+fi
 
 run_with_timeout() {
   local timeout_seconds="${1:?timeout seconds required}"
   shift
 
-  /opt/homebrew/bin/python3 - "$timeout_seconds" "$@" <<'PY'
+  "${python_bin}" - "$timeout_seconds" "$@" <<'PY'
 import os
 import signal
 import subprocess

package/tools/bin/heartbeat-safe-auto.sh

@@ -38,6 +38,7 @@ CATCHUP_TIMEOUT_SECONDS="${ACP_CATCHUP_TIMEOUT_SECONDS:-${F_LOSNING_CATCHUP_TIME
 HEARTBEAT_LOOP_TIMEOUT_SECONDS="${ACP_HEARTBEAT_LOOP_TIMEOUT_SECONDS:-${F_LOSNING_HEARTBEAT_LOOP_TIMEOUT_SECONDS:-720}}"
 CONFIG_YAML="$(resolve_flow_config_yaml "${BASH_SOURCE[0]}")"
 SHARED_AGENT_HOME="$(resolve_shared_agent_home "${FLOW_SKILL_DIR}")"
+RUNTIME_HOME_DIR="$(resolve_runtime_home)"
 FLOW_TOOLS_DIR="${FLOW_SKILL_DIR}/tools/bin"
 ISSUE_SESSION_PREFIX="$(flow_resolve_issue_session_prefix "${CONFIG_YAML}")"
 PR_SESSION_PREFIX="$(flow_resolve_pr_session_prefix "${CONFIG_YAML}")"
@@ -72,9 +73,15 @@ SHARED_LOOP_PID_FILE="${STATE_ROOT}/shared-heartbeat-loop.pid"
 SHARED_LOOP_STATUS_FILE="${STATE_ROOT}/shared-heartbeat-loop.env"
 QUOTA_LOCK_DIR="${STATE_ROOT}/quota-preflight.lock"
 QUOTA_PID_FILE="${QUOTA_LOCK_DIR}/pid"
+python_bin="$(flow_resolve_python_bin || true)"
 
 mkdir -p "${AGENT_ROOT}" "${RUNS_ROOT}" "${STATE_ROOT}" "${HISTORY_ROOT}" "${WORKTREE_ROOT}" "${MEMORY_DIR}"
 
+if [[ -z "${python_bin}" || ! -x "${python_bin}" ]]; then
+  echo "unable to resolve a runnable python interpreter for heartbeat-safe-auto.sh" >&2
+  exit 1
+fi
+
 acquire_lock() {
   mkdir -p "${STATE_ROOT}"
 
@@ -156,7 +163,7 @@ run_with_timeout() {
   local timeout_seconds="${1:?timeout seconds required}"
   shift
 
-  /opt/homebrew/bin/python3 - "${timeout_seconds}" "$@" <<'PY'
+  "${python_bin}" - "${timeout_seconds}" "$@" <<'PY'
 import os
 from pathlib import Path
 import signal
@@ -330,7 +337,7 @@ EFFECTIVE_QUOTA_POOLS=""
 
   local quota_cache_age_seconds=""
   quota_cache_age_seconds="$(
-    /opt/homebrew/bin/python3 - "${CODEX_QUOTA_FULL_CACHE_FILE}" <<'PY' 2>/dev/null || true
+    "${python_bin}" - "${CODEX_QUOTA_FULL_CACHE_FILE}" <<'PY' 2>/dev/null || true
 import os
 import sys
 import time
@@ -506,7 +513,11 @@ run_codex_quota_preflight
 # Sync skill files to runtime-home if source has changed since last sync.
 # This ensures start-issue-worker.sh and other scripts are always up to date.
 if [[ -x "${FLOW_TOOLS_DIR}/ensure-runtime-sync.sh" ]]; then
-  "${FLOW_TOOLS_DIR}/ensure-runtime-sync.sh" --quiet 2>/dev/null || true
+  if [[ "${FLOW_SKILL_DIR}" == "${RUNTIME_HOME_DIR}"/* ]]; then
+    printf 'RUNTIME_SYNC_SKIPPED=active-runtime-home\n'
+  else
+    "${FLOW_TOOLS_DIR}/ensure-runtime-sync.sh" --quiet 2>/dev/null || true
+  fi
 fi
 
 acquire_lock

package/tools/bin/project-launchd-bootstrap.sh

@@ -4,16 +4,9 @@ set -euo pipefail
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 FLOW_SKILL_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)"
 HOME_DIR="${ACP_PROJECT_RUNTIME_HOME_DIR:-${HOME:-}}"
-SOURCE_HOME="${ACP_PROJECT_RUNTIME_SOURCE_HOME:-}"
-RUNTIME_HOME="${ACP_PROJECT_RUNTIME_RUNTIME_HOME:-${HOME_DIR}/.agent-runtime/runtime-home}"
 PROFILE_REGISTRY_ROOT="${ACP_PROJECT_RUNTIME_PROFILE_REGISTRY_ROOT:-${ACP_PROFILE_REGISTRY_ROOT:-${HOME_DIR}/.agent-runtime/control-plane/profiles}}"
 PROFILE_ID="${ACP_PROJECT_RUNTIME_PROFILE_ID:-${ACP_PROJECT_ID:-${AGENT_PROJECT_ID:-}}}"
 ENV_FILE="${ACP_PROJECT_RUNTIME_ENV_FILE:-${PROFILE_REGISTRY_ROOT}/${PROFILE_ID}/runtime.env}"
-BASE_PATH="${ACP_PROJECT_RUNTIME_PATH:-/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin}"
-SYNC_SCRIPT="${ACP_PROJECT_RUNTIME_SYNC_SCRIPT:-${FLOW_SKILL_DIR}/tools/bin/sync-shared-agent-home.sh}"
-ENSURE_SYNC_SCRIPT="${ACP_PROJECT_RUNTIME_ENSURE_SYNC_SCRIPT:-${FLOW_SKILL_DIR}/tools/bin/ensure-runtime-sync.sh}"
-RUNTIME_HEARTBEAT_SCRIPT="${ACP_PROJECT_RUNTIME_HEARTBEAT_SCRIPT:-${RUNTIME_HOME}/skills/openclaw/agent-control-plane/tools/bin/heartbeat-safe-auto.sh}"
-ALWAYS_SYNC="${ACP_PROJECT_RUNTIME_ALWAYS_SYNC:-0}"
 
 if [[ -z "${HOME_DIR}" ]]; then
   echo "project launchd bootstrap requires HOME or ACP_PROJECT_RUNTIME_HOME_DIR" >&2
@@ -26,7 +19,6 @@ if [[ -z "${PROFILE_ID}" ]]; then
 fi
 
 export HOME="${HOME_DIR}"
-export PATH="${BASE_PATH}"
 export ACP_PROFILE_REGISTRY_ROOT="${PROFILE_REGISTRY_ROOT}"
 export ACP_PROJECT_ID="${PROFILE_ID}"
 export AGENT_PROJECT_ID="${PROFILE_ID}"
@@ -38,6 +30,17 @@ if [[ -f "${ENV_FILE}" ]]; then
   set +a
 fi
 
+# Resolve launch paths after runtime.env overrides are loaded so launchd can
+# pin the project runtime to a source checkout or alternate runtime home.
+SOURCE_HOME="${ACP_PROJECT_RUNTIME_SOURCE_HOME:-}"
+RUNTIME_HOME="${ACP_PROJECT_RUNTIME_RUNTIME_HOME:-${HOME_DIR}/.agent-runtime/runtime-home}"
+BASE_PATH="${ACP_PROJECT_RUNTIME_PATH:-/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin}"
+SYNC_SCRIPT="${ACP_PROJECT_RUNTIME_SYNC_SCRIPT:-${FLOW_SKILL_DIR}/tools/bin/sync-shared-agent-home.sh}"
+ENSURE_SYNC_SCRIPT="${ACP_PROJECT_RUNTIME_ENSURE_SYNC_SCRIPT:-${FLOW_SKILL_DIR}/tools/bin/ensure-runtime-sync.sh}"
+RUNTIME_HEARTBEAT_SCRIPT="${ACP_PROJECT_RUNTIME_HEARTBEAT_SCRIPT:-${RUNTIME_HOME}/skills/openclaw/agent-control-plane/tools/bin/heartbeat-safe-auto.sh}"
+ALWAYS_SYNC="${ACP_PROJECT_RUNTIME_ALWAYS_SYNC:-0}"
+export PATH="${BASE_PATH}"
+
 if [[ ! -x "${ENSURE_SYNC_SCRIPT}" && ! -x "${SYNC_SCRIPT}" ]]; then
   echo "project launchd bootstrap missing sync helper: ${ENSURE_SYNC_SCRIPT}" >&2
   exit 65

package/tools/bin/reconcile-bootstrap-lib.sh

@@ -0,0 +1,113 @@
+#!/usr/bin/env bash
+# reconcile-bootstrap-lib.sh — shared bootstrap helpers for reconcile scripts.
+# Sourced by both agent-project-reconcile-pr-session and
+# agent-project-reconcile-issue-session to avoid duplicating the bootstrap
+# preamble.
+
+bootstrap_flow_shell_lib() {
+  local candidate=""
+  local skill_name=""
+
+  for candidate in \
+    "${SCRIPT_DIR}/flow-shell-lib.sh" \
+    "${AGENT_CONTROL_PLANE_ROOT:-}/tools/bin/flow-shell-lib.sh" \
+    "${ACP_ROOT:-}/tools/bin/flow-shell-lib.sh" \
+    "${F_LOSNING_FLOW_ROOT:-}/tools/bin/flow-shell-lib.sh" \
+    "${AGENT_FLOW_SKILL_ROOT:-}/tools/bin/flow-shell-lib.sh" \
+    "${SHARED_AGENT_HOME:-}/tools/bin/flow-shell-lib.sh" \
+    "$(pwd)/tools/bin/flow-shell-lib.sh"; do
+    if [[ -n "${candidate}" && -f "${candidate}" ]]; then
+      printf '%s\n' "${candidate}"
+      return 0
+    fi
+  done
+
+  if [[ -n "${SHARED_AGENT_HOME:-}" ]]; then
+    for skill_name in "${AGENT_CONTROL_PLANE_SKILL_NAME:-agent-control-plane}" "${AGENT_CONTROL_PLANE_COMPAT_ALIAS:-}"; do
+      [[ -n "${skill_name}" ]] || continue
+      candidate="${SHARED_AGENT_HOME}/skills/openclaw/${skill_name}/tools/bin/flow-shell-lib.sh"
+      if [[ -f "${candidate}" ]]; then
+        printf '%s\n' "${candidate}"
+        return 0
+      fi
+    done
+  fi
+
+  echo "unable to locate flow-shell-lib.sh for reconcile bootstrap" >&2
+  return 1
+}
+
+FLOW_SHELL_LIB_PATH="$(bootstrap_flow_shell_lib)"
+BOOTSTRAP_TOOLS_DIR="$(cd "$(dirname "${FLOW_SHELL_LIB_PATH}")" && pwd)"
+# shellcheck source=/dev/null
+source "${FLOW_SHELL_LIB_PATH}"
+
+resolve_reconcile_tools_dir() {
+  local candidate_root=""
+  local skill_name=""
+
+  for candidate_root in \
+    "${AGENT_CONTROL_PLANE_ROOT:-}" \
+    "${ACP_ROOT:-}" \
+    "${F_LOSNING_FLOW_ROOT:-}" \
+    "${AGENT_FLOW_SKILL_ROOT:-}"; do
+    if [[ -n "${candidate_root}" && -d "${candidate_root}/tools/bin" ]]; then
+      printf '%s/tools/bin\n' "${candidate_root}"
+      return 0
+    fi
+  done
+
+  if [[ -n "${SHARED_AGENT_HOME:-}" ]]; then
+    if [[ -d "${SHARED_AGENT_HOME}/tools/bin" ]]; then
+      printf '%s/tools/bin\n' "${SHARED_AGENT_HOME}"
+      return 0
+    fi
+    for skill_name in "${AGENT_CONTROL_PLANE_SKILL_NAME:-agent-control-plane}" "${AGENT_CONTROL_PLANE_COMPAT_ALIAS:-}"; do
+      [[ -n "${skill_name}" ]] || continue
+      candidate_root="${SHARED_AGENT_HOME}/skills/openclaw/${skill_name}"
+      if [[ -d "${candidate_root}/tools/bin" ]]; then
+        printf '%s/tools/bin\n' "${candidate_root}"
+        return 0
+      fi
+    done
+  fi
+
+  if [[ -d "${SCRIPT_DIR}" ]]; then
+    printf '%s\n' "${SCRIPT_DIR}"
+    return 0
+  fi
+
+  printf '%s\n' "${BOOTSTRAP_TOOLS_DIR}"
+}
+
+shared_tools_dir="$(resolve_reconcile_tools_dir)"
+resolve_reconcile_helper_path() {
+  local helper_name="${1:?helper name required}"
+  local candidate=""
+
+  for candidate in \
+    "${SCRIPT_DIR}/${helper_name}" \
+    "${BOOTSTRAP_TOOLS_DIR}/${helper_name}" \
+    "${shared_tools_dir}/${helper_name}"; do
+    if [[ -n "${candidate}" && -f "${candidate}" ]]; then
+      printf '%s\n' "${candidate}"
+      return 0
+    fi
+  done
+
+  echo "unable to locate ${helper_name} for reconcile bootstrap" >&2
+  return 1
+}
+
+FLOW_CONFIG_LIB_PATH="$(resolve_reconcile_helper_path "flow-config-lib.sh")"
+# shellcheck source=/dev/null
+source "${FLOW_CONFIG_LIB_PATH}"
+
+require_transition() {
+  local step="${1:?step required}"
+  shift
+  if ! "$@"; then
+    echo "reconcile transition failed: ${step}" >&2
+    exit 1
+  fi
+}