agent-control-plane 0.1.0

package/tools/tests/test-project-runtimectl-status-supervisor-running.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+FLOW_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+RUNTIMECTL_BIN="${FLOW_ROOT}/tools/bin/project-runtimectl.sh"
+
+tmpdir="$(mktemp -d)"
+cleanup() {
+  rm -rf "${tmpdir}"
+}
+trap cleanup EXIT
+
+profile_registry_root="${tmpdir}/profiles"
+profile_dir="${profile_registry_root}/demo"
+runs_root="${tmpdir}/runtime/demo/runs"
+state_root="${tmpdir}/runtime/demo/state"
+mkdir -p "${profile_dir}" "${runs_root}" "${state_root}"
+
+cat >"${profile_dir}/control-plane.yaml" <<EOF
+schema_version: "1"
+id: "demo"
+repo:
+  slug: "example/demo"
+  root: "${tmpdir}/repo"
+  default_branch: "main"
+runtime:
+  orchestrator_agent_root: "${tmpdir}/runtime/demo"
+  worktree_root: "${tmpdir}/worktrees"
+  agent_repo_root: "${tmpdir}/repo"
+  runs_root: "${runs_root}"
+  state_root: "${state_root}"
+  history_root: "${tmpdir}/runtime/demo/history"
+  retained_repo_root: "${tmpdir}/repo"
+  vscode_workspace_file: "${tmpdir}/demo.code-workspace"
+execution:
+  coding_worker: "openclaw"
+  openclaw:
+    model: "primary/model"
+    thinking: "adaptive"
+    timeout_seconds: 900
+EOF
+
+sleep 60 >/dev/null 2>&1 &
+supervisor_pid="$!"
+printf '%s\n' "${supervisor_pid}" >"${state_root}/runtime-supervisor.pid"
+
+status_output="$(
+  ACP_PROFILE_REGISTRY_ROOT="${profile_registry_root}" \
+  ACP_PROJECT_RUNTIME_LAUNCHCTL_BIN="/nonexistent" \
+    bash "${RUNTIMECTL_BIN}" status --profile-id demo
+)"
+
+grep -q 'RUNTIME_STATUS=running' <<<"${status_output}"
+grep -q "SUPERVISOR_PID=${supervisor_pid}" <<<"${status_output}"
+
+kill "${supervisor_pid}" >/dev/null 2>&1 || true
+wait "${supervisor_pid}" >/dev/null 2>&1 || true
+
+echo "project runtimectl supervisor status test passed"

package/tools/tests/test-project-runtimectl-stop-cancels-pending-kick.sh

@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+FLOW_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+RUNTIMECTL_BIN="${FLOW_ROOT}/tools/bin/project-runtimectl.sh"
+KICK_SCRIPT="${FLOW_ROOT}/tools/bin/kick-scheduler.sh"
+
+tmpdir="$(mktemp -d)"
+cleanup() {
+  rm -rf "${tmpdir}"
+}
+trap cleanup EXIT
+
+profile_registry_root="${tmpdir}/profiles"
+profile_dir="${profile_registry_root}/demo"
+runs_root="${tmpdir}/runtime/demo/runs"
+state_root="${tmpdir}/runtime/demo/state"
+bootstrap_log="${tmpdir}/bootstrap.log"
+bootstrap_script="${tmpdir}/bootstrap.sh"
+mkdir -p "${profile_dir}" "${runs_root}" "${state_root}" "${tmpdir}/repo" "${tmpdir}/worktrees"
+
+cat >"${profile_dir}/control-plane.yaml" <<EOF
+schema_version: "1"
+id: "demo"
+repo:
+  slug: "example/demo"
+  root: "${tmpdir}/repo"
+  default_branch: "main"
+runtime:
+  orchestrator_agent_root: "${tmpdir}/runtime/demo"
+  worktree_root: "${tmpdir}/worktrees"
+  agent_repo_root: "${tmpdir}/repo"
+  runs_root: "${runs_root}"
+  state_root: "${state_root}"
+  history_root: "${tmpdir}/runtime/demo/history"
+  retained_repo_root: "${tmpdir}/repo"
+  vscode_workspace_file: "${tmpdir}/demo.code-workspace"
+execution:
+  coding_worker: "openclaw"
+  openclaw:
+    model: "primary/model"
+    thinking: "adaptive"
+    timeout_seconds: 900
+EOF
+
+cat >"${bootstrap_script}" <<EOF
+#!/usr/bin/env bash
+set -euo pipefail
+printf 'bootstrap profile=%s\n' "\${ACP_PROJECT_ID:-}" >>"${bootstrap_log}"
+mkdir -p "${state_root}/heartbeat-loop.lock"
+sleep 60 >/dev/null 2>&1 &
+printf '%s\n' "\$!" >"${state_root}/heartbeat-loop.lock/pid"
+EOF
+chmod +x "${bootstrap_script}"
+
+kick_output="$(
+  ACP_PROFILE_REGISTRY_ROOT="${profile_registry_root}" \
+  ACP_PROJECT_ID=demo \
+  ACP_BOOTSTRAP_SCRIPT="${bootstrap_script}" \
+    bash "${KICK_SCRIPT}" 2
+)"
+
+grep -q 'KICK_STATUS=scheduled' <<<"${kick_output}"
+kick_pid_file="${state_root}/kick-scheduler/pid"
+[[ -f "${kick_pid_file}" ]]
+
+stop_output="$(
+  ACP_PROFILE_REGISTRY_ROOT="${profile_registry_root}" \
+  ACP_PROJECT_RUNTIME_LAUNCHCTL_BIN="/nonexistent" \
+    bash "${RUNTIMECTL_BIN}" stop --profile-id demo --wait-seconds 1
+)"
+
+grep -q 'ACTION=stop' <<<"${stop_output}"
+grep -q 'RUNTIME_STATUS=stopped' <<<"${stop_output}"
+[[ ! -f "${kick_pid_file}" ]]
+
+sleep 3
+
+if [[ -f "${bootstrap_log}" ]]; then
+  echo "expected pending kick to be cancelled before bootstrap ran" >&2
+  cat "${bootstrap_log}" >&2
+  exit 1
+fi
+
+echo "project runtimectl cancels pending kick test passed"

package/tools/tests/test-project-runtimectl-stop-clears-running-labels.sh

@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+FLOW_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+RUNTIMECTL_BIN="${FLOW_ROOT}/tools/bin/project-runtimectl.sh"
+
+tmpdir="$(mktemp -d)"
+trap 'rm -rf "${tmpdir}"' EXIT
+
+profile_registry_root="${tmpdir}/profiles"
+profile_dir="${profile_registry_root}/demo"
+fake_bin="${tmpdir}/bin"
+update_log="${tmpdir}/update.log"
+
+mkdir -p "${profile_dir}" "${fake_bin}" "${tmpdir}/repo" "${tmpdir}/runtime/demo"
+
+cat >"${profile_dir}/control-plane.yaml" <<EOF
+schema_version: "1"
+id: "demo"
+repo:
+  slug: "example/demo"
+  root: "${tmpdir}/repo"
+  default_branch: "main"
+runtime:
+  orchestrator_agent_root: "${tmpdir}/runtime/demo"
+  worktree_root: "${tmpdir}/worktrees"
+  agent_repo_root: "${tmpdir}/repo"
+  runs_root: "${tmpdir}/runtime/demo/runs"
+  state_root: "${tmpdir}/runtime/demo/state"
+  history_root: "${tmpdir}/runtime/demo/history"
+  retained_repo_root: "${tmpdir}/repo"
+  vscode_workspace_file: "${tmpdir}/demo.code-workspace"
+execution:
+  coding_worker: "openclaw"
+EOF
+
+cat >"${fake_bin}/gh" <<'EOF'
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ "${1:-}" == "issue" && "${2:-}" == "list" ]]; then
+  printf '[{"number":1,"labels":[{"name":"agent-running"}]},{"number":2,"labels":[{"name":"agent-keep-open"}]}]\n'
+  exit 0
+fi
+
+if [[ "${1:-}" == "pr" && "${2:-}" == "list" ]]; then
+  printf '[{"number":7,"labels":[{"name":"agent-running"}]}]\n'
+  exit 0
+fi
+
+exit 64
+EOF
+chmod +x "${fake_bin}/gh"
+
+cat >"${fake_bin}/update-labels.sh" <<EOF
+#!/usr/bin/env bash
+set -euo pipefail
+printf '%s\n' "\$*" >>"${update_log}"
+EOF
+chmod +x "${fake_bin}/update-labels.sh"
+
+output="$(
+  PATH="${fake_bin}:$PATH" \
+  ACP_PROFILE_REGISTRY_ROOT="${profile_registry_root}" \
+  ACP_PROJECT_RUNTIME_LAUNCHCTL_BIN="/nonexistent" \
+  ACP_PROJECT_RUNTIME_UPDATE_LABELS_SCRIPT="${fake_bin}/update-labels.sh" \
+    bash "${RUNTIMECTL_BIN}" stop --profile-id demo
+)"
+
+grep -q 'ACTION=stop' <<<"${output}"
+grep -q '^--repo-slug example/demo --number 1 --remove agent-running$' "${update_log}"
+grep -q '^--repo-slug example/demo --number 7 --remove agent-running$' "${update_log}"
+if grep -q '^--repo-slug example/demo --number 2 --remove agent-running$' "${update_log}"; then
+  echo "non-running issue label update should not be attempted" >&2
+  exit 1
+fi
+
+echo "project runtimectl stop clears running labels test passed"

package/tools/tests/test-project-runtimectl.sh

@@ -0,0 +1,212 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+FLOW_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+RUNTIMECTL_BIN="${FLOW_ROOT}/tools/bin/project-runtimectl.sh"
+
+tmpdir="$(mktemp -d)"
+sleep_pids=()
+cleanup() {
+  local pid=""
+  for pid in "${sleep_pids[@]:-}"; do
+    kill "${pid}" >/dev/null 2>&1 || true
+    wait "${pid}" >/dev/null 2>&1 || true
+  done
+  rm -rf "${tmpdir}"
+}
+trap cleanup EXIT
+
+profile_registry_root="${tmpdir}/profiles"
+profile_dir="${profile_registry_root}/demo"
+runs_root="${tmpdir}/runtime/demo/runs"
+state_root="${tmpdir}/runtime/demo/state"
+tmux_sessions_file="${tmpdir}/tmux-sessions.txt"
+tmux_kill_log="${tmpdir}/tmux-kill.log"
+kick_log="${tmpdir}/kick.log"
+fake_bin="${tmpdir}/bin"
+mkdir -p "${profile_dir}" "${runs_root}/demo-issue-1" "${runs_root}/demo-pr-2" "${state_root}/heartbeat-loop.lock" "${state_root}/pending-launches" "${state_root}/resident-workers/issues/1" "${fake_bin}"
+
+cat >"${profile_dir}/control-plane.yaml" <<EOF
+schema_version: "1"
+id: "demo"
+repo:
+  slug: "example/demo"
+  root: "${tmpdir}/repo"
+  default_branch: "main"
+runtime:
+  orchestrator_agent_root: "${tmpdir}/runtime/demo"
+  worktree_root: "${tmpdir}/worktrees"
+  agent_repo_root: "${tmpdir}/repo"
+  runs_root: "${runs_root}"
+  state_root: "${state_root}"
+  history_root: "${tmpdir}/runtime/demo/history"
+  retained_repo_root: "${tmpdir}/repo"
+  vscode_workspace_file: "${tmpdir}/demo.code-workspace"
+execution:
+  coding_worker: "openclaw"
+  openclaw:
+    model: "primary/model"
+    thinking: "adaptive"
+    timeout_seconds: 900
+EOF
+
+cat >"${runs_root}/demo-issue-1/run.env" <<'EOF'
+SESSION=demo-issue-1
+TASK_KIND=issue
+TASK_ID=1
+EOF
+
+cat >"${runs_root}/demo-pr-2/run.env" <<'EOF'
+SESSION=demo-pr-2
+TASK_KIND=pr
+TASK_ID=2
+EOF
+
+printf 'demo-issue-1\ndemo-pr-2\n' >"${tmux_sessions_file}"
+
+cat >"${fake_bin}/tmux" <<EOF
+#!/usr/bin/env bash
+set -euo pipefail
+sessions_file="${tmux_sessions_file}"
+kill_log="${tmux_kill_log}"
+cmd="\${1:-}"
+case "\${cmd}" in
+  has-session)
+    shift
+    [[ "\${1:-}" == "-t" ]] || exit 1
+    session="\${2:-}"
+    grep -Fxq "\${session}" "\${sessions_file}"
+    ;;
+  kill-session)
+    shift
+    [[ "\${1:-}" == "-t" ]] || exit 1
+    session="\${2:-}"
+    printf '%s\n' "\${session}" >>"\${kill_log}"
+    grep -Fxv "\${session}" "\${sessions_file}" >"\${sessions_file}.next" || true
+    mv "\${sessions_file}.next" "\${sessions_file}"
+    ;;
+  *)
+    exit 1
+    ;;
+esac
+EOF
+chmod +x "${fake_bin}/tmux"
+
+cat >"${fake_bin}/kick-scheduler.sh" <<EOF
+#!/usr/bin/env bash
+set -euo pipefail
+printf 'kick profile=%s delay=%s\n' "\${ACP_PROJECT_ID:-}" "\${1:-}" >>"${kick_log}"
+mkdir -p "${state_root}/heartbeat-loop.lock"
+sleep 60 >/dev/null 2>&1 &
+echo \$! > "${state_root}/heartbeat-loop.lock/pid"
+printf 'KICK_STATUS=scheduled\nPID=%s\n' "\$!"
+EOF
+chmod +x "${fake_bin}/kick-scheduler.sh"
+
+sleep 60 &
+heartbeat_pid="$!"
+sleep_pids+=("${heartbeat_pid}")
+printf '%s\n' "${heartbeat_pid}" >"${state_root}/heartbeat-loop.lock/pid"
+
+sleep 60 &
+shared_loop_pid="$!"
+sleep_pids+=("${shared_loop_pid}")
+printf '%s\n' "${shared_loop_pid}" >"${state_root}/shared-heartbeat-loop.pid"
+
+sleep 60 &
+controller_pid="$!"
+sleep_pids+=("${controller_pid}")
+cat >"${state_root}/resident-workers/issues/1/controller.env" <<EOF
+ISSUE_ID=1
+CONTROLLER_PID=${controller_pid}
+EOF
+
+sleep 60 &
+pending_pid="$!"
+sleep_pids+=("${pending_pid}")
+printf '%s\n' "${pending_pid}" >"${state_root}/pending-launches/issue-1.pid"
+
+status_output="$(
+  ACP_PROFILE_REGISTRY_ROOT="${profile_registry_root}" \
+  ACP_PROJECT_RUNTIME_TMUX_BIN="${fake_bin}/tmux" \
+  ACP_PROJECT_RUNTIME_KICK_SCRIPT="${fake_bin}/kick-scheduler.sh" \
+  ACP_PROJECT_RUNTIME_LAUNCHCTL_BIN="/nonexistent" \
+    bash "${RUNTIMECTL_BIN}" status --profile-id demo
+)"
+
+grep -q 'RUNTIME_STATUS=running' <<<"${status_output}"
+grep -q 'CONTROLLER_COUNT=1' <<<"${status_output}"
+grep -q 'ACTIVE_TMUX_SESSION_COUNT=2' <<<"${status_output}"
+
+stop_output="$(
+  ACP_PROFILE_REGISTRY_ROOT="${profile_registry_root}" \
+  ACP_PROJECT_RUNTIME_TMUX_BIN="${fake_bin}/tmux" \
+  ACP_PROJECT_RUNTIME_KICK_SCRIPT="${fake_bin}/kick-scheduler.sh" \
+  ACP_PROJECT_RUNTIME_LAUNCHCTL_BIN="/nonexistent" \
+    bash "${RUNTIMECTL_BIN}" stop --profile-id demo --wait-seconds 1
+)"
+
+grep -q 'ACTION=stop' <<<"${stop_output}"
+grep -q 'RUNTIME_STATUS=stopped' <<<"${stop_output}"
+grep -q 'STOPPED_TMUX_SESSION_COUNT=2' <<<"${stop_output}"
+grep -q 'STOPPED_PID_COUNT=3' <<<"${stop_output}"
+
+! kill -0 "${heartbeat_pid}" 2>/dev/null
+! kill -0 "${shared_loop_pid}" 2>/dev/null
+! kill -0 "${controller_pid}" 2>/dev/null
+! kill -0 "${pending_pid}" 2>/dev/null
+[[ ! -s "${tmux_sessions_file}" ]]
+
+start_output="$(
+  ACP_PROFILE_REGISTRY_ROOT="${profile_registry_root}" \
+  ACP_PROJECT_RUNTIME_TMUX_BIN="${fake_bin}/tmux" \
+  ACP_PROJECT_RUNTIME_KICK_SCRIPT="${fake_bin}/kick-scheduler.sh" \
+  ACP_PROJECT_RUNTIME_LAUNCHCTL_BIN="/nonexistent" \
+    bash "${RUNTIMECTL_BIN}" start --profile-id demo --delay-seconds 0
+)"
+
+grep -q 'ACTION=start' <<<"${start_output}"
+grep -q 'START_MODE=kick-scheduler' <<<"${start_output}"
+grep -q 'RUNTIME_STATUS=running' <<<"${start_output}"
+grep -q 'kick profile=demo delay=0' "${kick_log}"
+if grep -q 'FALLBACK_SUPERVISOR_PID=' <<<"${start_output}"; then
+  echo "expected runtime started by kick-scheduler to avoid fallback supervisor" >&2
+  exit 1
+fi
+
+new_heartbeat_pid="$(tr -d '[:space:]' <"${state_root}/heartbeat-loop.lock/pid")"
+if [[ -n "${new_heartbeat_pid}" ]]; then
+  sleep_pids+=("${new_heartbeat_pid}")
+fi
+
+start_again_output="$(
+  ACP_PROFILE_REGISTRY_ROOT="${profile_registry_root}" \
+  ACP_PROJECT_RUNTIME_TMUX_BIN="${fake_bin}/tmux" \
+  ACP_PROJECT_RUNTIME_KICK_SCRIPT="${fake_bin}/kick-scheduler.sh" \
+  ACP_PROJECT_RUNTIME_LAUNCHCTL_BIN="/nonexistent" \
+    bash "${RUNTIMECTL_BIN}" start --profile-id demo --delay-seconds 0
+)"
+
+grep -q 'ACTION=start' <<<"${start_again_output}"
+grep -q 'START_MODE=already-running' <<<"${start_again_output}"
+grep -q 'NOOP=yes' <<<"${start_again_output}"
+[[ "$(wc -l <"${kick_log}")" -eq 1 ]]
+
+restart_output="$(
+  ACP_PROFILE_REGISTRY_ROOT="${profile_registry_root}" \
+  ACP_PROJECT_RUNTIME_TMUX_BIN="${fake_bin}/tmux" \
+  ACP_PROJECT_RUNTIME_KICK_SCRIPT="${fake_bin}/kick-scheduler.sh" \
+  ACP_PROJECT_RUNTIME_LAUNCHCTL_BIN="/nonexistent" \
+    bash "${RUNTIMECTL_BIN}" restart --profile-id demo --delay-seconds 0 --wait-seconds 1
+)"
+
+grep -q 'ACTION=start' <<<"${restart_output}"
+grep -q 'START_MODE=kick-scheduler' <<<"${restart_output}"
+grep -q 'RUNTIME_STATUS=running' <<<"${restart_output}"
+[[ "$(wc -l <"${kick_log}")" -eq 2 ]]
+if grep -q 'FALLBACK_SUPERVISOR_PID=' <<<"${restart_output}"; then
+  echo "expected restart to avoid fallback supervisor when kick-scheduler already restored runtime" >&2
+  exit 1
+fi
+
+echo "project runtimectl test passed"

package/tools/tests/test-provider-cooldown-state-prefers-runtime-worker-context.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+FLOW_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+SCRIPT="${FLOW_ROOT}/tools/bin/provider-cooldown-state.sh"
+
+tmpdir="$(mktemp -d)"
+trap 'rm -rf "$tmpdir"' EXIT
+
+state_root="$tmpdir/state"
+config_file="$tmpdir/control-plane.yaml"
+
+cat >"$config_file" <<EOF
+id: "quota-demo"
+runtime:
+  orchestrator_agent_root: "${tmpdir}/agent"
+  state_root: "${state_root}"
+execution:
+  coding_worker: "openclaw"
+  provider_quota:
+    cooldowns: "5,10"
+  openclaw:
+    model: "primary/model"
+    thinking: "adaptive"
+    timeout_seconds: 900
+EOF
+
+get_output="$(
+  AGENT_CONTROL_PLANE_CONFIG="$config_file" \
+  CODING_WORKER="claude" \
+  CLAUDE_MODEL="fallback-sonnet" \
+  bash "$SCRIPT" get
+)"
+
+grep -q '^BACKEND=claude$' <<<"$get_output"
+grep -q '^MODEL=fallback-sonnet$' <<<"$get_output"
+grep -q '^PROVIDER_KEY=claude-fallback-sonnet$' <<<"$get_output"
+
+echo "provider cooldown state runtime worker context test passed"

package/tools/tests/test-provider-cooldown-state.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+FLOW_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+SCRIPT="${FLOW_ROOT}/tools/bin/provider-cooldown-state.sh"
+
+tmpdir="$(mktemp -d)"
+trap 'rm -rf "$tmpdir"' EXIT
+
+state_root="$tmpdir/state"
+config_file="$tmpdir/control-plane.yaml"
+model="openrouter/stepfun/step-3.5-flash:free"
+
+cat >"$config_file" <<EOF
+id: "quota-demo"
+runtime:
+  orchestrator_agent_root: "${tmpdir}/agent"
+  state_root: "${state_root}"
+execution:
+  coding_worker: "openclaw"
+  provider_quota:
+    cooldowns: "5,10"
+  openclaw:
+    model: "${model}"
+    thinking: "adaptive"
+    timeout_seconds: 900
+EOF
+
+get_output="$(
+  AGENT_CONTROL_PLANE_CONFIG="$config_file" \
+  bash "$SCRIPT" get
+)"
+
+grep -q '^BACKEND=openclaw$' <<<"$get_output"
+grep -q "^MODEL=${model}$" <<<"$get_output"
+grep -q '^ATTEMPTS=0$' <<<"$get_output"
+grep -q '^READY=yes$' <<<"$get_output"
+
+schedule_output="$(
+  AGENT_CONTROL_PLANE_CONFIG="$config_file" \
+  bash "$SCRIPT" schedule provider-quota-limit
+)"
+
+grep -q '^PROVIDER_KEY=openclaw-openrouter-stepfun-step-3.5-flash-free$' <<<"$schedule_output"
+grep -q '^ATTEMPTS=1$' <<<"$schedule_output"
+grep -q '^READY=no$' <<<"$schedule_output"
+grep -q '^LAST_REASON=provider-quota-limit$' <<<"$schedule_output"
+test -f "$state_root/retries/providers/openclaw-openrouter-stepfun-step-3.5-flash-free.env"
+
+clear_output="$(
+  AGENT_CONTROL_PLANE_CONFIG="$config_file" \
+  bash "$SCRIPT" clear
+)"
+
+grep -q '^ATTEMPTS=0$' <<<"$clear_output"
+grep -q '^READY=yes$' <<<"$clear_output"
+test ! -f "$state_root/retries/providers/openclaw-openrouter-stepfun-step-3.5-flash-free.env"
+
+echo "provider cooldown state test passed"