ovtools 0.0.0-stage → 1.0.1

package/assets/deploy/10-prepare-host.sh

@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+# ===================================================================
+# 10-prepare-host.sh - turn a fresh VPS into a deploy target
+#
+#   * swap + node data store (delegates to the shared /root/pre-run.sh, which
+#     detects the volume, formats it when it holds no Octra data set, mounts it
+#     and records its decision in prerun.env)
+#   * the small tooling every later step needs (git/curl/jq/rsync/psmisc)
+#   * the directory layout, and /root/deploy/host.env with the facts the
+#     orchestrator and the GUI display
+#
+# Usage:  sudo bash 10-prepare-host.sh
+# Env:    WIPE_VOLUME=1   reformat the data volume even if it holds a data set
+# Idempotent.
+# ===================================================================
+
+set -euo pipefail
+source "$(dirname "$0")/lib.sh"
+
+require_root
+set_status RUNNING "prepare-host"
+
+log "=== preparing $(hostname) ==="
+
+# --- 1. swap + data store -------------------------------------------
+if [[ -f /root/pre-run.sh ]]; then
+  log "running /root/pre-run.sh (swap + data store: auto detect/format/mount)"
+  # keep this program's state separate: the decision file goes to /root/deploy
+  # Project settings decide the swap size and which block device is the data store
+  PRERUN_STATE="${DEPLOY_DIR}/prerun.env" \
+    WIPE_VOLUME="${WIPE_VOLUME:-0}" \
+    SWAP_GB="${SWAP_GB:-8}" \
+    BLOCK_DEVICE="${BLOCK_DEVICE:-/dev/sdb}" \
+    DATA_MIN_GB="${DATA_MIN_GB:-1000}" \
+    bash /root/pre-run.sh
+else
+  warn "/root/pre-run.sh is missing - run the deploy step again after the script push"
+fi
+
+# --- 2. tooling: check -> install -> continue ------------------------
+# A re-installed VPS is a bare image, so nothing here is assumed: every binary
+# this and the later steps call is checked first and only installed when it is
+# really missing (apt is skipped completely when everything is present, which
+# makes a retry after a dropped SSH cheap).
+export DEBIAN_FRONTEND=noninteractive
+export TERM="${TERM:-dumb}"
+log "checking the base tooling (git, curl, wget, rsync, jq, psmisc, procps)"
+ensure_package ca-certificates || die "cannot install ca-certificates"
+ensure_tool git   git    || die "git is missing and could not be installed"
+ensure_tool curl  curl   || die "curl is missing and could not be installed"
+ensure_tool wget  wget   || die "wget is missing and could not be installed"
+ensure_tool rsync rsync  || die "rsync is missing and could not be installed"
+ensure_tool jq    jq     || die "jq is missing and could not be installed"
+ensure_tool pgrep psmisc || die "pgrep (psmisc) is missing and could not be installed"
+ensure_tool pkill psmisc || die "pkill (psmisc) is missing and could not be installed"
+ensure_package procps    || die "cannot install procps"
+ensure_package gnupg     || die "cannot install gnupg"
+ensure_package lsb-release || die "cannot install lsb-release"
+
+# --- 3. layout + facts ----------------------------------------------
+if ! id "${NODE_USER}" >/dev/null 2>&1; then
+  log "creating system user '${NODE_USER}'"
+  useradd -r -m -d "/home/${NODE_USER}" -s /bin/bash "${NODE_USER}"
+else
+  log "user '${NODE_USER}' already exists"
+fi
+ensure_dirs
+mkdir -p /root/deploy
+
+PUBLIC_HOST=$(curl -s --max-time 5 https://api.ipify.org 2>/dev/null || true)
+if [[ -z "${PUBLIC_HOST}" ]]; then
+  PUBLIC_HOST=$(hostname -I | awk '{print $1}')
+fi
+
+STORE_DEV=$(findmnt -n -o SOURCE "${NODE_DATA_ROOT}" 2>/dev/null || echo none)
+STORE_FREE_GB=$(df -BG --output=avail "${NODE_DATA_ROOT}" 2>/dev/null | tail -1 | tr -dc '0-9')
+STORE_TOTAL_GB=$(df -BG --output=size "${NODE_DATA_ROOT}" 2>/dev/null | tail -1 | tr -dc '0-9')
+{
+  echo "PUBLIC_HOST=${PUBLIC_HOST}"
+  echo "HOSTNAME=$(hostname)"
+  echo "CORES=$(nproc)"
+  echo "MEM_MB=$(free -m | awk '/^Mem:/{print $2}')"
+  echo "SWAP=$(free -h | awk '/^Swap:/{print $2}')"
+  echo "STORE_DEV=${STORE_DEV}"
+  echo "STORE_TOTAL_GB=${STORE_TOTAL_GB:-0}"
+  echo "STORE_FREE_GB=${STORE_FREE_GB:-0}"
+  echo "PREPARED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+} > "${DEPLOY_HOST_ENV}"
+log "host facts:"
+sed 's/^/    /' "${DEPLOY_HOST_ENV}"
+
+if [[ -f "${DEPLOY_DIR}/prerun.env" ]]; then
+  log "data store decision:"
+  sed 's/^/    /' "${DEPLOY_DIR}/prerun.env"
+fi
+
+log "=== host ready: ${PUBLIC_HOST} ($(nproc) cores, ${STORE_FREE_GB:-?} GB free on ${STORE_DEV}) ==="
+set_status DONE "prepare-host" "${PUBLIC_HOST}"

package/assets/deploy/20-install-toolchain.sh

@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+# ===================================================================
+# 20-install-toolchain.sh - build the Octra runtime (apt deps + OCaml + dune)
+#
+# Runs the project's own /root/install-deps.sh DETACHED as a systemd unit:
+# it compiles for 15-30 minutes, and a dropped SSH must not kill it. The exit
+# code lands in /root/deploy/octra-deploy-toolchain.rc, so re-running this step
+# after a reconnect does not repeat a finished build.
+#
+# Usage:  sudo bash 20-install-toolchain.sh
+# Env:    TOOLCHAIN_TIMEOUT (default 5400s)
+# ===================================================================
+
+set -euo pipefail
+source "$(dirname "$0")/lib.sh"
+
+require_root
+set_status RUNNING "install-toolchain"
+
+# tools this step calls itself (install-deps.sh installs its own compiler deps)
+ensure_tool git git || die "git is missing and could not be installed"
+ensure_tool curl curl || die "curl is missing and could not be installed"
+ensure_tool tar tar || die "tar is missing and could not be installed"
+
+UNIT="octra-deploy-toolchain"
+RC_FILE="${DEPLOY_DIR}/${UNIT}.rc"
+
+# A cached "exit 0" is NOT proof the toolchain is usable: a failed
+# `opam switch link` empties the switch prefix (opam 2.3, see config.sh) and the
+# unit still reports success - 2026-11-02 the deploy then reported "toolchain
+# ready" and the next step found no switch at all.
+if unit_done "${UNIT}" && [[ "$(unit_rc ${UNIT})" == "0" ]]; then
+  if ocaml_switch_built; then
+    log "toolchain already installed (unit exit 0, OCaml switch present) - nothing to do"
+    set_status DONE "install-toolchain" "cached"
+    exit 0
+  fi
+  warn "the toolchain unit exited 0 but the OCaml switch is missing/incomplete"
+  warn "  (opam 2.3 empties the switch when 'opam switch link' is called on an existing _opam)"
+  drop_broken_ocaml_switch "${RC_FILE}"
+  log "the stale result was dropped - rebuilding the runtime"
+fi
+
+if ! [[ -f /root/install-deps.sh ]]; then
+  die "install-deps.sh is missing at /root/install-deps.sh - push the scripts first"
+fi
+
+if [[ "$(unit_active ${UNIT})" == "yes" ]]; then
+  log "toolchain build already running (unit ${UNIT}) - re-attaching"
+else
+  log "=== building the runtime (install-deps.sh, 15-30 min on 4 cores) ==="
+  # install.sh itself calls `opam switch link`: opam 2.3 dies on an existing
+  # <repo>/_opam and empties the switch prefix, so clear both leftovers first
+  # (drop_broken_ocaml_switch keeps a real switch untouched).
+  drop_broken_ocaml_switch "${RC_FILE}"
+  unlink_ocaml_switch_link "install.sh runs opam switch link next"
+  unit_start "${UNIT}" "/root" \
+    "DEBIAN_FRONTEND=noninteractive" "TERM=dumb" \
+    -- "bash /root/install-deps.sh"
+fi
+
+unit_wait "${UNIT}" "${TOOLCHAIN_TIMEOUT:-5400}" "install-deps.sh (toolchain)" \
+  "Cloning|Checking out|Building|Compiling|success|error" \
+  || die "the toolchain build failed - see ${DEPLOY_DIR}/${UNIT}.log"
+
+# --- proof the build is usable ---------------------------------------
+if [[ ! -d "${NODE_SRC_DIR}/.git" ]]; then
+  die "the node repository is missing at ${NODE_SRC_DIR}"
+fi
+if [[ ! -f "${NODE_SRC_DIR}/controls/check.sh" ]]; then
+  die "controls/check.sh is missing - the repository looks incomplete"
+fi
+COMMIT=$(node_repo_commit)
+BRANCH=$(node_repo_branch)
+REPO_URL=$(node_repo_url)
+if [[ -n "${COMMIT}" ]]; then
+  log "repository: ${COMMIT}${BRANCH:+ on ${BRANCH}} -> ${NODE_SRC_DIR}${REPO_URL:+ (${REPO_URL})}"
+else
+  # never print a bare "unknown": name what is wrong and what git said instead
+  REPO_ERR=$(node_repo_error)
+  warn "repository: no readable commit at ${NODE_SRC_DIR}${REPO_ERR:+ - git said: ${REPO_ERR}}"
+fi
+# install-deps.sh can exit 0 while leaving no usable OCaml switch behind (the
+# opam 2.3 `switch link` wipe). Drop the empty prefix and the cached result, so
+# the NEXT run really rebuilds instead of reporting "already installed".
+if ! assert_opam_switch; then
+  warn "the build did not leave a usable OCaml switch - dropping the leftovers so the next run rebuilds"
+  drop_broken_ocaml_switch "${RC_FILE}"
+  set_status WAITING-BUILD "no usable OCaml switch - Start deploy again rebuilds the runtime"
+  exit 2
+fi
+log "toolchain ready"
+set_status DONE "install-toolchain"

package/assets/deploy/30-setup-node.sh

@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+# ===================================================================
+# 30-setup-node.sh - configure the node: identity, binary, chain state
+#
+# Runs the project's own /root/node-setup.sh DETACHED (config_val.sh --build
+# compiles the node, --sync downloads ~38 GB). Both take a long time, so the
+# unit + rc-file pattern is used here too.
+#
+# Result: /opt/octra/libv_litecore/.keys/validator/{wallet.json,node.env}
+#
+# Usage:  sudo bash 30-setup-node.sh
+# Env:    NODE_NAME (default octra-node-<hostname>)
+#         PUBLIC_HOST (default: from host.env, else auto-detected)
+#         SETUP_TIMEOUT (default 5400s)
+# ===================================================================
+
+set -euo pipefail
+source "$(dirname "$0")/lib.sh"
+
+require_root
+set_status RUNNING "setup-node"
+
+# tools used below and by node-setup.sh: check first, install when missing
+ensure_tool curl  curl   || die "curl is missing and could not be installed"
+ensure_tool jq    jq     || die "jq is missing and could not be installed"
+ensure_tool git   git    || die "git is missing and could not be installed"
+ensure_tool rsync rsync  || die "rsync is missing and could not be installed"
+
+# host.env from the prep step carries the public address the node advertises
+if [[ -f "${DEPLOY_HOST_ENV}" ]]; then
+  # shellcheck disable=SC1090
+  source "${DEPLOY_HOST_ENV}"
+fi
+PUBLIC_HOST="${PUBLIC_HOST:-$(curl -s --max-time 5 https://api.ipify.org 2>/dev/null || hostname -I | awk '{print $1}')}"
+NODE_NAME="${NODE_NAME:-octra-node-$(hostname)}"
+UNIT="octra-deploy-setup"
+
+# --- already configured? ---------------------------------------------
+if [[ -f "${NODE_ENV}" && -f "${NODE_WALLET}" ]]; then
+  log "node is already configured (node.env + wallet.json present)"
+  log "  wallet: $(wallet_address || echo unknown)"
+  log "  role:   $(grep -m1 '^OCTRA_OPERATOR_ROLE=' "${NODE_ENV}" | cut -d= -f2)"
+  set_status DONE "setup-node" "cached"
+  exit 0
+fi
+
+if [[ ! -f /root/node-setup.sh ]]; then
+  die "node-setup.sh is missing at /root/node-setup.sh - push the scripts first"
+fi
+if [[ ! -d "${NODE_SRC_DIR}/_build" && ! -d "${NODE_SRC_DIR}/runtime_data" ]]; then
+  warn "the toolchain does not look installed yet - run the toolchain step first"
+fi
+
+if [[ "$(unit_active ${UNIT})" == "yes" ]]; then
+  log "node setup already running (unit ${UNIT}) - re-attaching"
+else
+  log "=== configuring ${NODE_NAME} @ ${PUBLIC_HOST} (build + 38 GB sync, 10-30 min) ==="
+  # a missing/broken OCaml switch would fail deep inside config_val.sh: say so
+  # here instead (soft stop - the operator can re-run the toolchain step)
+  if ! assert_opam_switch; then
+    set_status WAITING-BUILD "OCaml switch missing - re-run the toolchain step (see the log)"
+    exit 2
+  fi
+  unit_start "${UNIT}" "/root" \
+    "PUBLIC_HOST=${PUBLIC_HOST}" "NODE_NAME=${NODE_NAME}" "ROLE=observer" \
+    "DEBIAN_FRONTEND=noninteractive" "TERM=dumb" \
+    -- "bash /root/node-setup.sh"
+fi
+
+# the sync prints progress percentages; show the last one on the heartbeat
+unit_wait "${UNIT}" "${SETUP_TIMEOUT:-5400}" "node-setup.sh (build + sync)" \
+  "[0-9]+%|Downloaded|sync|Building|Compiling" \
+  || die "node setup failed - see ${DEPLOY_DIR}/${UNIT}.log"
+
+# --- proof the identity exists ---------------------------------------
+[[ -f "${NODE_WALLET}" ]] || die "wallet.json was not created (${NODE_WALLET})"
+[[ -f "${NODE_ENV}" ]] || die "node.env was not created (${NODE_ENV})"
+log "wallet address : $(wallet_address || echo unknown)"
+log "role           : $(grep -m1 '^OCTRA_OPERATOR_ROLE=' "${NODE_ENV}" | cut -d= -f2)"
+log "data dir       : $(du -sh "${NODE_DATA_DIR}" 2>/dev/null | awk '{print $1}') in ${NODE_DATA_DIR}"
+log "advertise      : $(grep -m1 '^OCTRA_ADVERTISE_ENDPOINT=' "${NODE_ENV}" | cut -d= -f2)"
+set_status DONE "setup-node"

package/assets/deploy/40-start-node.sh

@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+# ===================================================================
+# 40-start-node.sh - start the node and wait until its RPC answers
+#
+# The first start has to open the whole chain store and replay the epoch log, so
+# "started" is not enough: this step waits for `rpc = ready` (bounded) and prints
+# one status line a minute while it waits.
+#
+# Usage:  sudo bash 40-start-node.sh
+# Env:    WAIT_SEC (default 1800)
+# ===================================================================
+
+set -euo pipefail
+source "$(dirname "$0")/lib.sh"
+
+require_root
+set_status RUNNING "start-node"
+
+# tools this step calls (node_stat -> controls/stat.sh, pm2, curl, jq)
+ensure_tool curl  curl   || die "curl is missing and could not be installed"
+ensure_tool jq    jq     || die "jq is missing and could not be installed"
+ensure_tool pgrep psmisc || die "pgrep (psmisc) is missing and could not be installed"
+
+[[ -f "${NODE_ENV}" ]] || die "node.env is missing - run the setup step first"
+[[ -f /root/node-start.sh ]] || die "node-start.sh is missing at /root - push the scripts first"
+
+if [[ -n "$(node_processes)" ]]; then
+  log "node already running - not restarting"
+else
+  log "=== starting the node ($(wallet_address || echo 'no wallet yet')) ==="
+  # node-start.sh verifies pm2 itself and fails fast with the node's own log when
+  # the process does not come up (waiting 30 min for rpc would tell nobody why)
+  bash /root/node-start.sh ||
+    die "the node did not start - see the node log above (pm2 list, ${NODE_SRC_DIR}/data/node.log)"
+fi
+
+WAIT_SEC="${WAIT_SEC:-3600}"         # 2-core hosts need 30-40 min (4-core ~7-9 min)
+DEADLINE=$(( $(date +%s) + WAIT_SEC ))
+START=$(date +%s)
+LAST=$START
+RPC=""
+# read the real pm2 process name the node registered itself under (OCTRA_OPERATOR_PM2_NAME
+# in node.env); the lib.sh default (octra-node-1) is a fallback and often differs
+REAL_PM2=$(grep -m1 '^OCTRA_OPERATOR_PM2_NAME=' "${NODE_ENV:-${NODE_SRC_DIR}/.keys/validator/node.env}" 2>/dev/null | cut -d= -f2- || true)
+REAL_PM2="${REAL_PM2:-${NODE_PM2_NAME}}"
+log "waiting for rpc = ready (up to $(elapsed_minutes "${WAIT_SEC}")) - the first start replays the epoch log..."
+while (( $(date +%s) < DEADLINE )); do
+  SNAP=$(node_stat)
+  RPC=$(printf '%s\n' "${SNAP}" | stat_field rpc)
+  PROC=$(printf '%s\n' "${SNAP}" | stat_field process)
+  HEAD=$(printf '%s\n' "${SNAP}" | stat_field head_epoch)
+  if [[ "${RPC}" == "ready" ]]; then
+    log "rpc is ready after $(( $(date +%s) - START ))s (process=${PROC} head_epoch=${HEAD})"
+    break
+  fi
+  if (( $(date +%s) - LAST >= 60 )); then
+    log "  waiting for rpc: $(elapsed_minutes $(( $(date +%s) - START ))) of $(elapsed_minutes "${WAIT_SEC}") (rpc=${RPC:-?} process=${PROC:-?} head=${HEAD:-?})"
+    LAST=$(date +%s)
+  fi
+  sleep 5
+done
+
+if [[ "${RPC}" != "ready" ]]; then
+  warn "rpc did not become ready in $(elapsed_minutes "${WAIT_SEC}") - pm2 log (process: ${REAL_PM2}):"
+  as_octra "pm2 logs ${REAL_PM2} --lines 15 --nostream" 2>/dev/null | sed 's/^/    /' || true
+  die "node rpc is not ready after ${WAIT_SEC}s (set WAIT_SEC=7200 in the env to double the limit)"
+fi
+
+log "--- pm2 ---"
+as_octra "pm2 list" 2>/dev/null | sed 's/^/    /' || warn "pm2 list failed"
+log "--- status ---"
+node_stat | tee "${DEPLOY_DIR}/start-stat.txt" | \
+  grep -E '^(node|role|address|process|pid|rpc|epoch|head_epoch|voting|p2p_connected|validator_active|validator_enrollment) =' || true
+
+set_status DONE "start-node"

package/assets/deploy/50-verify.sh

@@ -0,0 +1,259 @@
+#!/usr/bin/env bash
+# ===================================================================
+# 50-verify.sh - prove the freshly deployed node works
+#
+# For ROLE=observer (a fresh deploy): process online, rpc ready, role correct,
+# p2p peers > 0, and the epoch advancing between samples. A brand-new node needs
+# a moment to find peers, so it WAITS (PEER_WAIT) instead of failing on the first
+# sample - that wait is what lets the enrollment popup appear on a healthy node.
+# For ROLE=validator (already enrolled): also voting enabled, validator_active
+# true and validator_enrollment ready - the same bar the migration uses. The
+# validator only starts voting after the set fold, so this waits (VOTE_WAIT)
+# instead of failing: the step ends as PASS exactly when voting = enabled.
+#
+# Usage:  sudo SAMPLES=3 INTERVAL=20 bash 50-verify.sh
+# Env:    SAMPLES (default 3), INTERVAL (default 20), ROLE (default observer),
+#         PEER_WAIT (default 300 s), VOTE_WAIT (default 900 s),
+#         VERIFY_RETRY (default 2), VERIFY_RETRY_WAIT (default 8 s)
+#
+# A sample that fails the bar is re-read (VERIFY_RETRY times, VERIFY_RETRY_WAIT
+# apart) before it counts: the node's own state flaps for one window right after
+# a set fold (`voting=disabled`) or while it replays an epoch (`rpc=unavailable`),
+# and one flapping sample must not turn a healthy validator into a FAILED deploy
+# (2026-11-04). A transient blip is counted, logged and named in the result; more
+# than half of the samples blipping is still a failure.
+#
+# Running out of the wait budget is NOT a broken deploy either: a node that is
+# still replaying (`rpc != ready`) or a validator the network has not admitted /
+# activated yet is a matter of minutes, not of a wrong command. Those two end as
+# soft stops (`WAITING-RPC` / `WAITING-FOLD`, exit 2) so the console asks for
+# another run instead of showing a red failure. A dead process, a wrong role or
+# an observer that never finds a peer is still a real failure.
+# ===================================================================
+
+set -euo pipefail
+source "$(dirname "$0")/lib.sh"
+
+require_root
+# the verify bar reads the node through curl/jq and watches the process with pgrep
+ensure_tool curl  curl   || die "curl is missing and could not be installed"
+ensure_tool jq    jq     || die "jq is missing and could not be installed"
+ensure_tool pgrep psmisc || die "pgrep (psmisc) is missing and could not be installed"
+SAMPLES="${SAMPLES:-3}"
+INTERVAL="${INTERVAL:-20}"
+ROLE="${ROLE:-observer}"
+# How long to keep waiting for the state a healthy node reaches on its own:
+# peers for an observer, the set fold + voting for a validator.
+PEER_WAIT="${PEER_WAIT:-300}"
+VOTE_WAIT="${VOTE_WAIT:-900}"
+EXPECTED_ADDRESS="${EXPECTED_ADDRESS:-$(wallet_address || true)}"
+
+set_status RUNNING "verify"
+# per-sample dumps of earlier attempts (a test host carried eleven
+# verify-sample-*.txt from older runs): this run writes its own, so drop the old
+rm -f "${DEPLOY_DIR}"/verify-sample-*.txt 2>/dev/null || true
+log "=== verifying the node on $(hostname) - ${SAMPLES} samples every ${INTERVAL}s (role=${ROLE}) ==="
+[[ "${ROLE}" == "validator" ]] && log "=== the step only passes once voting = enabled (waiting up to ${VOTE_WAIT}s for the fold) ==="
+
+FAIL=0
+PREV_EPOCH=""
+ADDR=""
+P2P=""
+VOTING=""
+ACTIVE=""
+ENROLL=""
+
+epoch_num() { # epoch_num <value> -> a number, 0 when empty/not numeric
+  case "${1:-}" in
+    ''|*[!0-9]*) printf '0' ;;
+    *) printf '%s' "$1" ;;
+  esac
+}
+
+# --- a flapping sample is not a broken node ---------------------------
+# The node's own state flaps by design: `voting` can read `disabled` for one
+# window right after a set fold, and the RPC answers `unavailable` while it
+# replays an epoch log (observed on both). 2026-11-04: two deploys of a perfectly
+# healthy ACTIVE validator were reported FAILED because exactly ONE of five
+# samples flapped (sample 3 `voting=disabled`, resp. sample 2 `rpc=unavailable`),
+# so the console showed a red verify although every other sample proved voting,
+# active=true, enrollment=ready and an advancing epoch. A sample that fails the
+# bar is therefore RE-READ up to VERIFY_RETRY times (VERIFY_RETRY_WAIT apart):
+# any healthy re-read makes it a TRANSIENT BLIP (counted, logged, named in the
+# result). Only a sample that stays bad - or a run where more than half of the
+# samples blipped - fails.
+BLIPS=0
+VERIFY_RETRY="${VERIFY_RETRY:-2}"
+VERIFY_RETRY_WAIT="${VERIFY_RETRY_WAIT:-8}"
+
+sample_reason() {   # why the current field values fail the bar ("" = healthy)
+  if [[ "${PROC}" != "online" ]]; then printf 'process is %s (not online)' "${PROC:-?}"; return 0; fi
+  if [[ "${RPC}" != "ready" ]]; then printf 'rpc is %s' "${RPC:-?}"; return 0; fi
+  if [[ "${DEV_ROLE}" != "${ROLE}" ]]; then
+    printf 'role is %s, expected %s' "${DEV_ROLE:-?}" "${ROLE}"; return 0
+  fi
+  if [[ "${P2P:-0}" -le 0 ]]; then printf 'no p2p peers connected'; return 0; fi
+  if [[ -n "${EXPECTED_ADDRESS}" && "${ADDR}" != "${EXPECTED_ADDRESS}" ]]; then
+    printf 'address is %s, expected %s' "${ADDR:-?}" "${EXPECTED_ADDRESS}"; return 0
+  fi
+  if [[ "${ROLE}" == "validator" ]]; then
+    if [[ "${VOTING}" != "enabled" ]]; then printf "voting is %s (must be 'enabled')" "${VOTING:-?}"; return 0; fi
+    if [[ "${ACTIVE}" != "true" ]]; then printf 'validator_active is %s' "${ACTIVE:-?}"; return 0; fi
+    if [[ "${ACTIVE}" != "true" && "${ENROLL}" != "ready" ]]; then printf 'validator_enrollment is %s' "${ENROLL:-?}"; return 0; fi
+  fi
+  printf ''
+  return 0
+}
+
+sample_state() {   # read the node once into the field variables
+  STAT=$(node_stat)
+  printf '%s\n' "${STAT}" > "${DEPLOY_DIR}/verify-sample-${SAMPLE_N}.txt"
+  FIELD() { printf '%s\n' "${STAT}" | stat_field "$1"; }
+  DEV_ROLE=$(FIELD role); ADDR=$(FIELD address); PROC=$(FIELD process)
+  RPC=$(FIELD rpc); EPOCH=$(FIELD epoch); HEAD=$(FIELD head_epoch)
+  VOTING=$(FIELD voting); ACTIVE=$(FIELD validator_active)
+  ENROLL=$(FIELD validator_enrollment); P2P=$(FIELD p2p_connected)
+  NEXT_SET=$(FIELD validator_next_set_epoch)
+  UPTIME=$(FIELD uptime); RESTARTS=$(FIELD restarts)
+}
+
+# A validator only votes after the set fold; a fresh observer needs a moment for
+# its first peers. Waiting here (instead of failing on the first sample) is what
+# makes the run end exactly when the node is healthy - and therefore what lets the
+# enrollment popup follow a *verified* node.
+WAIT_BUDGET="${PEER_WAIT}"
+[[ "${ROLE}" == "validator" ]] && WAIT_BUDGET="${VOTE_WAIT}"
+WAITED=0
+SAMPLE_N=0
+READY=0
+while :; do
+  SAMPLE_N=$(( SAMPLE_N + 1 ))
+  sample_state
+  log "waiting ${SAMPLE_N}: process=${PROC} rpc=${RPC} role=${DEV_ROLE} p2p=${P2P} voting=${VOTING} active=${ACTIVE} enrollment=${ENROLL}${NEXT_SET:+ - set fold at epoch ${NEXT_SET}}"
+  if [[ "${PROC}" == "online" && "${RPC}" == "ready" && "${P2P:-0}" -gt 0 ]]; then
+    if [[ "${ROLE}" != "validator" ]]; then READY=1; fi
+    if [[ "${ROLE}" == "validator" \
+          && "${VOTING}" == "enabled" && "${ACTIVE}" == "true" ]]; then
+      # active + voting is authoritative; the enrollment projection can still say
+      # 'unavailable' while the RPC warms up (observed on live migrations)
+      if [[ "${ENROLL}" != "ready" ]]; then
+        warn "enrollment projection reads '${ENROLL:-unavailable}' while the validator is active and voting - accepting active membership"
+      fi
+      READY=1
+    fi
+  fi
+  (( READY == 1 )) && break
+  if (( WAITED >= WAIT_BUDGET )); then
+    # The budget is out. Only a node that is really broken fails here: the two
+    # states a healthy node has not reached yet are timing (see the header).
+    if [[ "${PROC}" != "online" ]]; then
+      err "the node process is ${PROC:-?} after ${WAIT_BUDGET}s - it is not running"
+      err "  check on the host: pm2 status ; tail -n 50 ${NODE_SRC_DIR}/data/operator_logs/node.log"
+      set_status FAILED "verify-not-ready" "process=${PROC:-?} after ${WAIT_BUDGET}s"
+      exit 1
+    fi
+    if [[ "${RPC}" != "ready" ]]; then
+      warn "the node is still starting or catching up after ${WAIT_BUDGET}s (rpc=${RPC:-?})"
+      warn "  nothing is broken: the rpc answers 'ready' once the node replayed its state"
+      set_status WAITING-RPC "rpc=${RPC:-?} after ${WAIT_BUDGET}s - the node is still catching up, re-run verify"
+      exit 2
+    fi
+    if [[ "${ROLE}" == "validator" && "${P2P:-0}" -gt 0 && "${VOTING}" != "enabled" ]]; then
+      warn "the validator is not voting after ${WAIT_BUDGET}s (voting=${VOTING:-?} active=${ACTIVE:-?} enrollment=${ENROLL:-?})"
+      warn "  the network admits a new validator at the set fold and starts its duty epoch after that;"
+      warn "  the node keeps running and votes on its own - re-run verify in a few minutes"
+      set_status WAITING-FOLD "voting=${VOTING:-?} active=${ACTIVE:-?} enrollment=${ENROLL:-?} after ${WAIT_BUDGET}s${NEXT_SET:+ - set fold at epoch ${NEXT_SET}} - re-run verify later"
+      exit 2
+    fi
+    err "the node did not reach the ${ROLE} state within ${WAIT_BUDGET}s"
+    err "process=${PROC} rpc=${RPC} p2p=${P2P:-0} voting=${VOTING:-?} active=${ACTIVE:-?} enrollment=${ENROLL:-?}"
+    [[ "${ROLE}" == "validator" ]] && err "voting must be 'enabled' - check: sh controls/stat.sh on the host"
+    set_status FAILED "verify-not-ready"
+    exit 1
+  fi
+  log "  not there yet - waiting ${INTERVAL}s (${WAITED}s of ${WAIT_BUDGET}s)"
+  WAITED=$(( WAITED + INTERVAL ))
+  sleep "${INTERVAL}"
+done
+if [[ "${ROLE}" == "validator" ]]; then
+  log "=== the node is an ACTIVE validator and voting (voting=${VOTING} active=${ACTIVE} enrollment=${ENROLL}) ==="
+else
+  log "=== the node is online with ${P2P} peers ==="
+fi
+
+for ((i = 1; i <= SAMPLES; i++)); do
+  SAMPLE_N="${i}"
+  sample_state
+
+  log "sample ${i}/${SAMPLES}: epoch=${EPOCH} head=${HEAD} role=${DEV_ROLE} process=${PROC} rpc=${RPC} p2p=${P2P} voting=${VOTING} active=${ACTIVE} enrollment=${ENROLL} uptime=${UPTIME} restarts=${RESTARTS}"
+
+  WHY=$(sample_reason)
+  if [[ -n "${WHY}" ]]; then
+    RECOVERED=""
+    for ((r = 1; r <= VERIFY_RETRY; r++)); do
+      log "  sample ${i}/${SAMPLES} read '${WHY}' - re-reading in ${VERIFY_RETRY_WAIT}s (${r}/${VERIFY_RETRY})"
+      sleep "${VERIFY_RETRY_WAIT}"
+      sample_state
+      WHY=$(sample_reason)
+      if [[ -z "${WHY}" ]]; then RECOVERED="yes"; break; fi
+    done
+    if [[ -n "${RECOVERED}" ]]; then
+      BLIPS=$(( BLIPS + 1 ))
+      log "    transient blip ${BLIPS}: recovered - process=${PROC} rpc=${RPC} role=${DEV_ROLE} p2p=${P2P} voting=${VOTING} active=${ACTIVE} enrollment=${ENROLL}"
+    else
+      err "sample ${i}/${SAMPLES}: ${WHY}"
+      FAIL=1
+    fi
+  fi
+
+  if [[ -n "${PREV_EPOCH}" ]]; then
+    PREV_NUM=$(epoch_num "${PREV_EPOCH}")
+    NOW_NUM=$(epoch_num "${EPOCH}")
+    if (( NOW_NUM > PREV_NUM )); then
+      log "  epoch advanced ${PREV_EPOCH} -> ${EPOCH} (+$((NOW_NUM - PREV_NUM)))"
+    elif (( NOW_NUM == 0 )); then
+      warn "  the node does not report an epoch yet - it is probably still catching up"
+    else
+      warn "  epoch did not advance (${PREV_EPOCH} -> ${EPOCH}) - the node may still be catching up"
+    fi
+  fi
+  PREV_EPOCH="${EPOCH}"
+
+  if (( i < SAMPLES )); then
+    log "  sleeping ${INTERVAL}s before the next sample"
+    sleep "${INTERVAL}"
+  fi
+done
+
+# A node that is mostly unhealthy must not pass: one or two blips are noise, a
+# majority of bad samples is not (a node that flaps most of the time is broken).
+if (( BLIPS * 2 > SAMPLES )); then
+  err "most samples (${BLIPS}/${SAMPLES}) had to be re-read before they looked healthy - that is not a stable node"
+  FAIL=1
+fi
+
+echo
+if (( FAIL == 0 )); then
+  if (( BLIPS > 0 )); then
+    log "=== RESULT: PASS - the node is online and following the chain (${BLIPS}/${SAMPLES} transient blip(s) recovered) ==="
+  else
+    log "=== RESULT: PASS - the node is online and following the chain ==="
+  fi
+  log "wallet address: ${ADDR}"
+  if [[ "${ROLE}" == "validator" ]]; then
+    log "voting = enabled - this node is an ACTIVE validator"
+  else
+    log "this node is an observer. Next: fund ${ADDR} and run the enroll step,"
+    log "then the node can be promoted to a validator (ROLE=validator)."
+  fi
+  if (( BLIPS > 0 )); then
+    set_status DONE "verify-pass" "address=${ADDR}" "transient blips=${BLIPS}/${SAMPLES} (recovered)"
+  else
+    set_status DONE "verify-pass" "address=${ADDR}"
+  fi
+else
+  err "=== RESULT: FAIL - see the errors above ==="
+  set_status FAILED "verify-failed"
+  exit 1
+fi
+