ovtools 0.0.0-stage → 1.0.1

package/assets/migrate/07-verify.sh

@@ -0,0 +1,237 @@
+#!/usr/bin/env bash
+# ===================================================================
+# 07-verify.sh - prove the migrated node is a live voting validator
+#
+# Checks (all must pass):
+#   process = online, rpc = ready
+#   role = validator, voting = enabled
+#   validator_active = true, validator_enrollment = ready
+#   epoch advances between samples (node follows the network head)
+#   p2p_connected > 0
+#
+# Usage (on the NEW vps):
+#   EXPECTED_ADDRESS=oct... bash 07-verify.sh          # 3 samples / 30s
+#   SAMPLES=10 INTERVAL=30 bash 07-verify.sh           # longer watch
+# ===================================================================
+
+set -euo pipefail
+source "$(dirname "$0")/lib.sh"
+
+require_root
+SAMPLES="${SAMPLES:-3}"
+INTERVAL="${INTERVAL:-30}"
+EXPECTED_ADDRESS="${EXPECTED_ADDRESS:-}"
+if [[ -z "${EXPECTED_ADDRESS}" && -s "${MIG_DIR}/source-address.txt" ]]; then
+  EXPECTED_ADDRESS=$(cat "${MIG_DIR}/source-address.txt")
+fi
+if [[ -z "${EXPECTED_ADDRESS}" && -f "${MIG_DIR}/source-final-stat.txt" ]]; then
+  EXPECTED_ADDRESS=$(grep -m1 '^address = ' "${MIG_DIR}/source-final-stat.txt" | awk -F' = ' '{print $2}' || true)
+fi
+
+set_status RUNNING "verify"
+log "=== verifying validator on $(hostname) - ${SAMPLES} samples every ${INTERVAL}s ==="
+
+# --- wait for the node to finish opening its store -------------------
+# A freshly started node must open a ~38 GB irmin-pack store, replay the
+# epochlog/rule graph and rejoin consensus before its RPC answers. On 2
+# vCPU that takes minutes, so poll instead of failing straight away.
+WAIT_SEC="${WAIT_SEC:-900}"
+DEADLINE=$(( $(date +%s) + WAIT_SEC ))
+WAIT_START=$(date +%s)
+LAST_LOG=$WAIT_START
+log "waiting for rpc = ready (up to ${WAIT_SEC}s)..."
+while (( $(date +%s) < DEADLINE )); do
+  SNAP=$(node_stat)
+  RPC_NOW=$(printf '%s\n' "${SNAP}" | stat_field rpc)
+  PROC_NOW=$(printf '%s\n' "${SNAP}" | stat_field process)
+  HEAD_NOW=$(printf '%s\n' "${SNAP}" | stat_field head_epoch)
+  if [[ "${RPC_NOW}" == "ready" ]]; then
+    log "rpc ready after $(( $(date +%s) - WAIT_START ))s (head_epoch=${HEAD_NOW})"
+    break
+  fi
+  # One line a minute instead of a carriage-return animation: every \r update
+  # arrives as its own log row in the console, which buried the real steps.
+  if (( $(date +%s) - LAST_LOG >= 60 )); then
+    log "  waiting for rpc: $(( $(date +%s) - WAIT_START ))s of ${WAIT_SEC}s (rpc=${RPC_NOW:-?} process=${PROC_NOW:-?} head=${HEAD_NOW:-?})"
+    LAST_LOG=$(date +%s)
+  fi
+  sleep 10
+done
+RPC_NOW=$(node_stat | stat_field rpc)
+if [[ "${RPC_NOW}" != "ready" ]]; then
+  warn "rpc still unavailable after ${WAIT_SEC}s - running the checks anyway"
+fi
+echo
+
+# --- wait for the set fold to re-admit the validator -----------------
+# After any downtime the network drops a validator from the active set
+# (`set_fold_refused ... reason = marks_short`). It is re-admitted by the
+# fold once its duty pulses span `rejoin_span = 64` epochs - measured at
+# ~27 minutes on this devnet. Nothing has to be re-enrolled: the bond and
+# the committed `ready` state stay valid, the node pulses automatically.
+MEMBER_WAIT_SEC="${MEMBER_WAIT_SEC:-2700}"
+ACTIVE_NOW=$(node_stat | stat_field validator_active)
+if [[ "${ACTIVE_NOW}" != "true" ]]; then
+  log "validator is online but not a member yet - waiting up to ${MEMBER_WAIT_SEC}s for the set fold"
+  MEMBER_DEADLINE=$(( $(date +%s) + MEMBER_WAIT_SEC ))
+  MEMBER_START=$(date +%s)
+  LAST_LOG=$MEMBER_START
+  LAST_SCHED=""
+  FIRST_EPOCH=""
+  while (( $(date +%s) < MEMBER_DEADLINE )); do
+    SNAP=$(node_stat)
+    ACTIVE_NOW=$(printf '%s\n' "${SNAP}" | stat_field validator_active)
+    SCHED_NOW=$(printf '%s\n' "${SNAP}"  | stat_field validator_scheduled)
+    EPOCH_NOW=$(printf '%s\n' "${SNAP}"  | stat_field epoch)
+    if [[ "${ACTIVE_NOW}" == "true" ]]; then
+      log "validator re-admitted to the active set at epoch ${EPOCH_NOW} (after $(( $(date +%s) - MEMBER_START ))s)"
+      break
+    fi
+    if [[ -z "${FIRST_EPOCH}" && -n "${EPOCH_NOW}" ]]; then
+      FIRST_EPOCH="${EPOCH_NOW}"
+    fi
+    NOW=$(date +%s)
+    # A state change is news, so it is logged immediately; otherwise one status
+    # line every two minutes. (A \r progress line becomes its own log row in the
+    # console, which produced ~50 identical "epoch=... active=false" rows.)
+    if [[ -n "${SCHED_NOW}" && "${SCHED_NOW}" != "${LAST_SCHED}" ]]; then
+      log "  validator_scheduled=${SCHED_NOW} at epoch ${EPOCH_NOW} - the fold re-admits after ~64 duty epochs"
+      LAST_SCHED="${SCHED_NOW}"
+      LAST_LOG=$NOW
+    elif (( NOW - LAST_LOG >= 120 )); then
+      log "  still waiting: $(( (NOW - MEMBER_START) / 60 ))m of $(( MEMBER_WAIT_SEC / 60 ))m, epoch +$(( EPOCH_NOW - FIRST_EPOCH )) since the wait began (active=${ACTIVE_NOW:-?} scheduled=${SCHED_NOW:-false})"
+      LAST_LOG=$NOW
+    fi
+    sleep 20
+  done
+  [[ "${ACTIVE_NOW}" == "true" ]] || \
+    warn "still not active after ${MEMBER_WAIT_SEC}s - the fold re-admits on its own, keep monitoring"
+fi
+
+# The enrollment projection can lag behind the membership view right after the
+# RPC opens (observed on a live migration: active=true and voting=enabled for
+# three 30 s samples while validator_enrollment still read 'unavailable', then
+# 'ready'). Wait for the projection before the strict samples instead of turning
+# that race into a false "migration failed".
+ENROLL_WAIT_SEC="${ENROLL_WAIT_SEC:-900}"
+ACTIVE_NOW=$(node_stat | stat_field validator_active)
+ENROLL_NOW=$(node_stat | stat_field validator_enrollment)
+if [[ "${ACTIVE_NOW}" == "true" && "${ENROLL_NOW}" != "ready" && "${ENROLL_NOW}" != "bonded" ]]; then
+  log "validator is active but enrollment reads '${ENROLL_NOW:-unavailable}' - waiting up to ${ENROLL_WAIT_SEC}s for the projection"
+  ENROLL_DEADLINE=$(( $(date +%s) + ENROLL_WAIT_SEC ))
+  ENROLL_START=$(date +%s)
+  LAST_LOG=$ENROLL_START
+  while (( $(date +%s) < ENROLL_DEADLINE )); do
+    SNAP=$(node_stat)
+    ACTIVE_NOW=$(printf '%s\n' "${SNAP}" | stat_field validator_active)
+    ENROLL_NOW=$(printf '%s\n' "${SNAP}" | stat_field validator_enrollment)
+    if [[ "${ENROLL_NOW}" == "ready" || "${ENROLL_NOW}" == "bonded" ]]; then
+      log "enrollment projection ready after $(( $(date +%s) - ENROLL_START ))s (${ENROLL_NOW})"
+      break
+    fi
+    if (( $(date +%s) - LAST_LOG >= 60 )); then
+      log "  waiting for enrollment: $(( $(date +%s) - ENROLL_START ))s of ${ENROLL_WAIT_SEC}s (active=${ACTIVE_NOW:-?} enrollment=${ENROLL_NOW:-?})"
+      LAST_LOG=$(date +%s)
+    fi
+    sleep 10
+  done
+  if [[ "${ENROLL_NOW}" != "ready" && "${ENROLL_NOW}" != "bonded" ]]; then
+    warn "enrollment still reads '${ENROLL_NOW:-unavailable}' after ${ENROLL_WAIT_SEC}s - active membership is authoritative, the samples below decide"
+  fi
+fi
+
+# A migrated STUCK validator is in sync and healthy but the chain has not admitted
+# it yet: that is a schedule, not a failure (2026-11-06, .99 -> .231). Ending the
+# phase with FAILED there made the operator think the migration broke, while the
+# bond and the readiness were fine and only the fold was missing.
+if [[ "${ACTIVE_NOW}" != "true" ]]; then
+  MEM_STAT=$(node_stat)
+  MEM_ENROLL=$(printf "%s\n" "${MEM_STAT}" | stat_field validator_enrollment)
+  MEM_PROC=$(printf "%s\n" "${MEM_STAT}"   | stat_field process)
+  MEM_RPC=$(printf "%s\n" "${MEM_STAT}"    | stat_field rpc)
+  MEM_EPOCH=$(printf "%s\n" "${MEM_STAT}"  | stat_field epoch)
+  if [[ "${MEM_ENROLL}" =~ ^(ready|bonded)$ && "${MEM_PROC}" == "online" && "${MEM_RPC}" == "ready" ]]; then
+    log "=== RESULT: WAITING-FOLD - the node runs, is in sync and holds the bond (enrollment=${MEM_ENROLL}) ==="
+    log "the set has not admitted it yet: the fold does that on its own (~64 duty epochs), nothing was re-enrolled"
+    log "nothing to fix: the console keeps watching and opens the activation popup when it happens"
+    set_status WAITING-FOLD "migrated and in sync; waiting for the set fold (enrollment=${MEM_ENROLL} epoch=${MEM_EPOCH})"
+    echo "MIGRATE_VERIFY state=waiting_fold enrollment=${MEM_ENROLL} epoch=${MEM_EPOCH}"
+    exit 2
+  fi
+  warn "the node is not active AND not ready (process=${MEM_PROC:-?} rpc=${MEM_RPC:-?} enrollment=${MEM_ENROLL:-?}) - the samples below show why"
+fi
+
+FAIL=0
+PREV_EPOCH=""
+
+for ((i = 1; i <= SAMPLES; i++)); do
+  STAT=$(node_stat)
+  printf '%s\n' "${STAT}" > "${MIG_DIR}/verify-sample-${i}.txt"
+
+  ROLE=$(printf '%s\n' "${STAT}"  | stat_field role)
+  ADDR=$(printf '%s\n' "${STAT}"  | stat_field address)
+  PROC=$(printf '%s\n' "${STAT}"  | stat_field process)
+  RPC=$(printf '%s\n' "${STAT}"   | stat_field rpc)
+  EPOCH=$(printf '%s\n' "${STAT}" | stat_field epoch)
+  HEAD=$(printf '%s\n' "${STAT}"  | stat_field head_epoch)
+  VOTING=$(printf '%s\n' "${STAT}" | stat_field voting)
+  ACTIVE=$(printf '%s\n' "${STAT}" | stat_field validator_active)
+  ENROLL=$(printf '%s\n' "${STAT}" | stat_field validator_enrollment)
+  P2P=$(printf '%s\n' "${STAT}"   | stat_field p2p_connected)
+  AGREED=$(printf '%s\n' "${STAT}"| stat_field round_agreed)
+  STEP=$(printf '%s\n' "${STAT}"  | stat_field round_step)
+  PEERS=$(printf '%s\n' "${STAT}" | stat_field round_peers)
+  UPTIME=$(printf '%s\n' "${STAT}"| stat_field uptime)
+  RESTARTS=$(printf '%s\n' "${STAT}" | stat_field restarts)
+
+  log "sample ${i}/${SAMPLES}: epoch=${EPOCH} head=${HEAD} role=${ROLE} voting=${VOTING} active=${ACTIVE} p2p=${P2P} round_peers=${PEERS} agreed=${AGREED} step=${STEP} uptime=${UPTIME} restarts=${RESTARTS}"
+
+  [[ "${PROC}" == "online" ]]        || { err "process is not online: ${PROC}"; FAIL=1; }
+  [[ "${RPC}" == "ready" ]]          || { err "rpc is not ready: ${RPC}"; FAIL=1; }
+  [[ "${ROLE}" == "validator" ]]     || { err "role is not validator: ${ROLE}"; FAIL=1; }
+  [[ "${VOTING}" == "enabled" ]]     || { err "voting is not enabled: ${VOTING}"; FAIL=1; }
+  [[ "${ACTIVE}" == "true" ]]        || { err "validator_active is not true: ${ACTIVE}"; FAIL=1; }
+  if [[ "${ENROLL}" != "ready" ]]; then
+    if [[ "${ACTIVE}" == "true" && "${VOTING}" == "enabled" ]]; then
+      # active + voting is stronger evidence than the enrollment projection; the
+      # projection can answer 'unavailable' while the RPC warms up.
+      warn "enrollment projection reads '${ENROLL:-unavailable}' while the validator is active and voting - accepting active membership"
+    else
+      err "validator_enrollment is not ready: ${ENROLL}"
+      FAIL=1
+    fi
+  fi
+  [[ -z "${EXPECTED_ADDRESS}" || "${ADDR}" == "${EXPECTED_ADDRESS}" ]] || { err "address mismatch: ${ADDR} != ${EXPECTED_ADDRESS}"; FAIL=1; }
+  [[ "${P2P:-0}" -gt 0 ]]            || { err "no p2p peers connected"; FAIL=1; }
+
+  if [[ -n "${PREV_EPOCH}" ]]; then
+    if (( EPOCH > PREV_EPOCH )); then
+      log "  epoch advanced ${PREV_EPOCH} -> ${EPOCH} (+$((EPOCH - PREV_EPOCH)))"
+    else
+      warn "  epoch did not advance (${PREV_EPOCH} -> ${EPOCH}) - node may still be catching up"
+    fi
+  fi
+  PREV_EPOCH="${EPOCH}"
+
+  if (( i < SAMPLES )); then
+    log "  sleeping ${INTERVAL}s before the next sample"
+    sleep "${INTERVAL}"
+  fi
+done
+
+echo
+if (( FAIL == 0 )); then
+  log "=== RESULT: PASS - validator is online, voting and following the chain ==="
+  if [[ -s "${MIG_DIR}/downtime-start.txt" ]]; then
+    START_TS=$(date -u -d "$(cat "${MIG_DIR}/downtime-start.txt")" +%s 2>/dev/null || true)
+    if [[ -n "${START_TS}" ]]; then
+      DOWN=$(( $(date +%s) - START_TS ))
+      log "total time from source stop to verified voting: $((DOWN / 60))m$((DOWN % 60))s"
+    fi
+  fi
+  set_status DONE "verify-pass" "epoch=${PREV_EPOCH} address=${ADDR}"
+else
+  err "=== RESULT: FAIL - see the errors above ==="
+  set_status FAILED "verify-failed"
+  exit 1
+fi

package/assets/migrate/08-rollback.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+# ===================================================================
+# 08-rollback.sh — abort the migration and hand the validator back
+#
+# Only valid while the NEW node is fenced: the target node is stopped
+# first, and only then is the source node started again. Both nodes
+# must never run at the same time (same wallet = equivocation).
+#
+# Usage (on the NEW vps):
+#   OLD_HOST=203.0.113.27 bash 08-rollback.sh
+# ===================================================================
+
+set -euo pipefail
+source "$(dirname "$0")/lib.sh"
+
+require_root
+require_old_host
+set_status RUNNING "rollback"
+
+log "=== ROLLBACK — stopping the new node first ==="
+if [[ -f "${NODE_ENV}" ]] && [[ -n "$(node_processes)" ]]; then
+  as_octra "cd ${NODE_SRC_DIR} && sh controls/stop.sh" 2>&1 | tee -a "${MIG_LOG}" || true
+fi
+
+LEFT=$(node_processes)
+for i in {1..20}; do
+  [[ -z "${LEFT}" ]] && break
+  sleep 1
+  LEFT=$(node_processes)
+done
+if [[ -n "${LEFT}" ]]; then
+  warn "target processes still alive — SIGTERM"
+  pkill -TERM -f "${NODE_PROC_RE}" || true
+  sleep 5
+  LEFT=$(node_processes)
+fi
+[[ -z "${LEFT}" ]] || die "cannot stop the target node — aborting rollback (source stays down)"
+log "Target node stopped."
+
+log "=== restarting the node on the old host ${OLD_HOST} ==="
+if on_old "systemctl is-enabled --quiet pm2-${NODE_USER} 2>/dev/null"; then
+  log "pm2 boot unit already enabled on the source"
+else
+  on_old "systemctl enable pm2-${NODE_USER} >/dev/null 2>&1 || true"
+  log "pm2 boot unit re-enabled on the source"
+fi
+on_old "su - ${NODE_USER} -c 'cd ${NODE_SRC_DIR} && sh controls/run.sh'" 2>&1 | tee -a "${MIG_LOG}"
+
+sleep 10
+log "=== source status after rollback ==="
+on_old "su - ${NODE_USER} -c 'cd ${NODE_SRC_DIR} && sh controls/stat.sh'" 2>&1 | \
+  grep -E '^(role|process|epoch|voting|validator_active) =' || true
+
+set_status DONE "rollback"

package/assets/migrate/lib.sh

@@ -0,0 +1,149 @@
+#!/usr/bin/env bash
+# ===================================================================
+# lib.sh — shared helpers for the Octra node migration toolkit
+#
+# Sourced by every script in scripts/migrate/.
+# ===================================================================
+
+set -euo pipefail
+
+# --- Paths ---------------------------------------------------------
+MIG_DIR="${MIG_DIR:-/root/migration}"
+MIG_LOG="${MIG_LOG:-${MIG_DIR}/migrate.log}"
+MIG_STATUS="${MIG_STATUS:-${MIG_DIR}/status}"
+
+# --- Node paths (must match scripts/config.sh) ----------------------
+NODE_SRC_DIR="${NODE_SRC_DIR:-/opt/octra/libv_litecore}"
+NODE_DATA_ROOT="${NODE_DATA_ROOT:-/var/lib/octra}"
+NODE_DATA_DIR="${NODE_DATA_DIR:-${NODE_DATA_ROOT}/devnet}"
+NODE_SYNC_STAGE="${NODE_SYNC_STAGE:-${NODE_DATA_ROOT}/devnet.state_sync}"
+NODE_ENV="${NODE_ENV:-${NODE_SRC_DIR}/.keys/validator/node.env}"
+NODE_USER="${NODE_USER:-octra}"
+CONSENSUS_PORT="${CONSENSUS_PORT:-19000}"
+
+mkdir -p "${MIG_DIR}"
+
+# --- Tool policy: check -> install -> continue ----------------------
+# The shared implementation lives in scripts/config.sh, which the push puts at
+# /root/config.sh on BOTH hosts. A host that still carries an old copy gets a
+# loud warning instead of a bare "command not found" half way through a pull.
+# Sourced BEFORE the logging helpers below, so migrate/log/err keep tee-ing into
+# /root/migration/migrate.log.
+if [[ -f /root/config.sh ]]; then
+  # shellcheck disable=SC1091
+  source /root/config.sh
+fi
+
+# --- Logging -------------------------------------------------------
+log()  { echo -e "\033[1;32m[migrate]\033[0m $*" | tee -a "${MIG_LOG}"; }
+warn() { echo -e "\033[1;33m[warn]\033[0m    $*" | tee -a "${MIG_LOG}" >&2; }
+err()  { echo -e "\033[1;31m[err]\033[0m     $*" | tee -a "${MIG_LOG}" >&2; }
+
+if ! declare -F ensure_tool >/dev/null 2>&1; then
+  warn "/root/config.sh has no ensure_tool (old copy on the host) - pushing the scripts again fixes it"
+  ensure_tool() {
+    command -v "$1" >/dev/null 2>&1 || die "tool $1 is missing and /root/config.sh cannot install it (push the scripts again)"
+  }
+fi
+
+die() { err "$*"; set_status FAILED "$*"; exit 1; }
+
+# --- Status markers (read by the Windows orchestrator) -------------
+set_status() {  # set_status <STATE> [detail...]
+  # all arguments after the state form the detail (a 3rd argument used to be
+  # dropped, which is why the console showed the step label instead of the result)
+  local state="$1" detail
+  shift || true
+  detail="$*"
+  printf '%s %s %s\n' "${state}" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${detail}" > "${MIG_STATUS}"
+  log "status = ${state} ${detail}"
+}
+
+require_root() {
+  [[ $EUID -eq 0 ]] || die "run as root"
+}
+
+# --- a step must never die leaving status = RUNNING -------------------
+# The migration orchestrator follows /root/migration/status (Wait-RemoteState).
+# A script that died from an unexpected error - a helper that no longer exists, an
+# unset variable with `set -u`, a full disk - used to leave that file at RUNNING,
+# so the step looked hung until its timeout. This trap writes FAILED immediately.
+_migrate_on_err() {
+  local rc="$1" line="$2" cmd="$3" state
+  state=$(awk 'NR==1 {print $1}' "${MIG_STATUS}" 2>/dev/null || true)
+  case "${state}" in
+    DONE|FAILED|WAITING-*) return 0 ;;        # the script already decided
+  esac
+  err "unexpected failure (exit ${rc}) at line ${line}: ${cmd}"
+  set_status FAILED "line ${line}: ${cmd} (exit ${rc})"
+}
+trap '_migrate_on_err "$?" "$LINENO" "$BASH_COMMAND"' ERR
+
+# Create every directory the node and the migration write into. A fresh host
+# has none of them, and "no such file or directory" half-way through a pull is
+# exactly the kind of failure this toolkit must not have.
+ensure_dirs() {
+  local dir
+  for dir in "${MIG_DIR}" "${MIG_DIR}/backups" "${NODE_SRC_DIR}" \
+             "${NODE_DATA_ROOT}" "${NODE_DATA_DIR}" "${NODE_SYNC_STAGE}" \
+             "${NODE_DATA_ROOT}/validator-control"; do
+    mkdir -p "${dir}"
+  done
+  if id "${NODE_USER}" >/dev/null 2>&1; then
+    chown "${NODE_USER}:${NODE_USER}" "${NODE_DATA_DIR}" "${NODE_SYNC_STAGE}" 2>/dev/null || true
+  fi
+}
+
+require_cmd() {
+  command -v "$1" >/dev/null 2>&1 || die "missing command: $1"
+}
+
+# --- Node helpers --------------------------------------------------
+as_octra() { su - "${NODE_USER}" -c "$*"; }
+
+node_stat() {
+  as_octra "cd ${NODE_SRC_DIR} && sh controls/stat.sh 2>/dev/null" || true
+}
+
+stat_field() {  # stat_field <name>  (reads node_stat from stdin when piped)
+  # the node prints Python's `None` for a field it cannot read: that is an EMPTY
+  # value, not a value (see scripts/deploy/lib.sh for the incident)
+  awk -F' = ' -v k="$1" '$1==k { v=$2; if (v=="None" || v=="null") v=""; print v; exit }'
+}
+
+countdown() {  # countdown <seconds> <message>
+  local n="$1" msg="${2:-waiting}"
+  while (( n > 0 )); do
+    printf '\r%s %ds   ' "${msg}" "${n}"
+    sleep 1
+    n=$((n - 1))
+  done
+  printf '\r%s done\n' "${msg}"
+}
+
+# --- Remote (source VPS) helpers ----------------------------------
+OLD_HOST="${OLD_HOST:-}"
+OLD_USER="${OLD_USER:-root}"
+SSH_OPTS=(-o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=10)
+
+require_old_host() {
+  [[ -n "${OLD_HOST}" ]] || die "OLD_HOST is not set (export OLD_HOST=<old-vps-ip>)"
+  ssh "${SSH_OPTS[@]}" "${OLD_USER}@${OLD_HOST}" "echo ok" >/dev/null 2>&1 \
+    || die "cannot ssh ${OLD_USER}@${OLD_HOST} as root (key not installed?)"
+}
+
+on_old() { ssh "${SSH_OPTS[@]}" "${OLD_USER}@${OLD_HOST}" "$*"; }
+
+# --- Node process detection ----------------------------------------
+# IMPORTANT: pgrep/pkill -f patterns would also match the shell that runs
+# the check itself (its command line contains the pattern), which makes a
+# dead node look alive. The bracket trick ([o]ctra_node) avoids that.
+NODE_PROC_RE='[o]ctra_node\.exe|[o]ctra_pvac_worker\.exe'
+
+node_processes() {  # processes of the node on THIS host
+  ps -eo pid,args --no-headers 2>/dev/null | grep -E "${NODE_PROC_RE}" || true
+}
+
+on_old_node_processes() {  # processes of the node on the SOURCE host
+  on_old "ps -eo pid,args --no-headers 2>/dev/null | grep -E '${NODE_PROC_RE}' || true"
+}

package/assets/upgrade/10-preflight.sh

@@ -0,0 +1,120 @@
+#!/usr/bin/env bash
+# ===================================================================
+# 10-preflight.sh - is this host a node we may update, and what is on it?
+#
+# Read-only. Nothing is pulled, built or restarted here: the step reports the
+# checkout (commit/branch/dirty), the running binary, the chain role and the disk
+# head-room, and refuses clearly when this is not an Octra node at all.
+#
+# Upstream rules this step reminds the operator of (README, "How to update"):
+# never run `--sync` again, never change config/network.env, never re-enroll.
+#
+# Usage:  sudo bash 10-preflight.sh
+# Env:    MIN_FREE_GB (20) - warn below that much free space on the data volume
+#         STATE_TIMEOUT (60)
+# ===================================================================
+
+set -euo pipefail
+source "$(dirname "$0")/lib.sh"
+
+require_root
+set_status RUNNING "preflight"
+
+MIN_FREE_GB="${MIN_FREE_GB:-20}"
+WARNINGS=0
+
+[[ -d "${NODE_SRC_DIR}" ]] || die "no node at ${NODE_SRC_DIR} - this host does not run an Octra node"
+[[ -d "${NODE_SRC_DIR}/.git" ]] || die "${NODE_SRC_DIR} is not a git checkout - the node cannot be updated from the source tree"
+[[ -f "${NODE_WALLET}" ]] || die "wallet.json is missing (${NODE_WALLET}) - this is not the validator host"
+[[ -f "${NODE_ENV}" ]] || die "node.env is missing (${NODE_ENV}) - the node has no configuration to update"
+[[ -d "${NODE_DATA_DIR}" ]] || die "the data directory is missing (${NODE_DATA_DIR}) - nothing to keep in sync"
+
+for tool in git make opam; do
+  if command -v "${tool}" >/dev/null 2>&1; then
+    log "tool ${tool}: present ($(command -v "${tool}"))"
+  else
+    warn "tool ${tool}: MISSING - the update needs it (the Apply step installs it via apt)"
+    WARNINGS=$(( WARNINGS + 1 ))
+  fi
+done
+
+# --- what the checkout says -------------------------------------------
+COMMIT=$(source_commit)
+HEAD=$(git_head)
+BRANCH=$(git_branch)
+DIRTY=$(git_dirty)
+REMOTE=$(git_remote)
+log "repository: ${REMOTE}"
+log "checkout  : SOURCE_COMMIT=${COMMIT:-unknown} git HEAD=${HEAD:-unknown}${BRANCH:+ on ${BRANCH}}"
+if [[ -n "${DIRTY}" ]]; then
+  warn "the working tree has local changes - the official tool refuses to update a dirty checkout"
+  WARNINGS=$(( WARNINGS + 1 ))
+fi
+
+# --- the control scripts the update needs -----------------------------
+[[ -f "${CHECK_SH}" ]] || warn "controls/check.sh is missing (the package cannot be verified)"
+if [[ -f "${UPGRADE_SH}" ]]; then
+  log "official tool: controls/upgrade.sh present"
+else
+  warn "controls/upgrade.sh is missing - use the manual update (fallback) on this node version"
+  WARNINGS=$(( WARNINGS + 1 ))
+fi
+if [[ -f "${STORAGE_SH}" ]]; then
+  log "storage tool : controls/storage.sh present"
+else
+  warn "controls/storage.sh is missing - 'Remove preserved states' is not available on this node"
+  WARNINGS=$(( WARNINGS + 1 ))
+fi
+[[ -f "${STOP_SH}" && -f "${RUN_SH}" ]] || warn "controls/stop.sh or controls/run.sh is missing - the restart may fail"
+
+# --- the running node --------------------------------------------------
+read_state
+log "node      : role=${ROLE:-?} process=${PROCESS:-?} rpc=${RPC:-?} voting=${VOTING:-?} epoch=${EPOCH:-?} uptime=${PROCESS_UP:-?}"
+log "membership: active=${ACTIVE:-?} scheduled=${SCHED:-?} enrollment=${ENROLL:-?}"
+log "binary    : ${BINARY:-unknown}${PVAC:+ pvac ${PVAC}}"
+if [[ "${PROCESS}" != "online" ]]; then
+  warn "the node process is ${PROCESS:-unknown} - the update restarts it (controls/run.sh)"
+  WARNINGS=$(( WARNINGS + 1 ))
+fi
+if [[ "${RPC}" != "ready" ]]; then
+  warn "the node rpc is ${RPC:-unknown} - the upgrade tool waits for it, so the update may take longer"
+  WARNINGS=$(( WARNINGS + 1 ))
+fi
+
+# a join is running: the upgrade restarts the node and could interrupt it
+if systemctl is-active --quiet octra-enroll-join 2>/dev/null; then
+  warn "an enrollment join is running (unit octra-enroll-join) - finish the enrollment before updating"
+  WARNINGS=$(( WARNINGS + 1 ))
+fi
+if [[ -f "${NODE_DATA_DIR}/validator-control/bond-increase.json" ]]; then
+  warn "a signed bond increase is stored on this host - make sure it is confirmed before restarting the node"
+fi
+
+# --- disk head-room (a build + a state sync need room) -----------------
+FREE_GB=""
+if command -v df >/dev/null 2>&1; then
+  FREE_GB=$(df -B1 --output=avail "${NODE_DATA_ROOT}" 2>/dev/null | tail -1 | tr -d ' ' || true)
+  if [[ "${FREE_GB}" =~ ^[0-9]+$ ]]; then
+    FREE_H=$(awk -v b="${FREE_GB}" 'BEGIN { printf "%.1f GiB", b / 1073741824 }')
+    log "data volume: ${FREE_H} free on ${NODE_DATA_ROOT}"
+    if (( FREE_GB < MIN_FREE_GB * 1073741824 )); then
+      warn "less than ${MIN_FREE_GB} GiB free - free space before updating (controls/storage.sh --prune-prior --yes)"
+      WARNINGS=$(( WARNINGS + 1 ))
+    fi
+  fi
+fi
+if command -v du >/dev/null 2>&1; then
+  USED=$(du -sh "${NODE_DATA_DIR}" 2>/dev/null | cut -f1 || true)
+  [[ -n "${USED}" ]] && log "data store : ${USED} in ${NODE_DATA_DIR}"
+fi
+
+if [[ -f "${STORAGE_SH}" ]]; then
+  SUMMARY=$(storage_summary)
+  [[ -n "${SUMMARY}" ]] && log "storage    : ${SUMMARY}"
+fi
+
+log "upstream rules: do NOT run --sync again, do NOT change config/network.env, do NOT re-enroll"
+if (( WARNINGS > 0 )); then
+  log "${WARNINGS} warning(s) above - the update can still run"
+fi
+set_status DONE "commit=${HEAD:-unknown}${BRANCH:+ on ${BRANCH}} running=${ROLE:-?} active=${ACTIVE:-?} warnings=${WARNINGS}"