ovtools 0.0.0-stage → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +121 -2
- package/assets/common/bond-inspect.sh +368 -0
- package/assets/common/config.sh +263 -0
- package/assets/common/extract-data.sh +197 -0
- package/assets/common/harden-host.sh +37 -0
- package/assets/common/install-deps.sh +93 -0
- package/assets/common/node-enroll.sh +65 -0
- package/assets/common/node-setup.sh +150 -0
- package/assets/common/node-start.sh +90 -0
- package/assets/common/node-stop.sh +18 -0
- package/assets/common/node-upgrade.sh +45 -0
- package/assets/common/pre-run.sh +497 -0
- package/assets/common/promote-validator.sh +104 -0
- package/assets/deploy/10-prepare-host.sh +98 -0
- package/assets/deploy/20-install-toolchain.sh +93 -0
- package/assets/deploy/30-setup-node.sh +82 -0
- package/assets/deploy/40-start-node.sh +75 -0
- package/assets/deploy/50-verify.sh +259 -0
- package/assets/deploy/60-enroll.sh +304 -0
- package/assets/deploy/70-promote.sh +152 -0
- package/assets/deploy/lib.sh +398 -0
- package/assets/manage/10-bond-increase.sh +149 -0
- package/assets/manage/10-diagnose-node.sh +169 -0
- package/assets/manage/20-exit-validator.sh +259 -0
- package/assets/manage/25-recover-node.sh +291 -0
- package/assets/manage/30-status.sh +64 -0
- package/assets/manage/40-exit-withdraw.sh +133 -0
- package/assets/manage/bond_inc.sh +318 -0
- package/assets/manage/dashboard_monitor/app.js +1544 -0
- package/assets/manage/dashboard_monitor/assets/v2.2/css/base.css +113 -0
- package/assets/manage/dashboard_monitor/assets/v2.2/css/components/content.css +133 -0
- package/assets/manage/dashboard_monitor/assets/v2.2/css/components/controls.css +229 -0
- package/assets/manage/dashboard_monitor/assets/v2.2/css/components/data.css +374 -0
- package/assets/manage/dashboard_monitor/assets/v2.2/css/components/icons.css +35 -0
- package/assets/manage/dashboard_monitor/assets/v2.2/css/components/motion.css +57 -0
- package/assets/manage/dashboard_monitor/assets/v2.2/css/components/shell.css +51 -0
- package/assets/manage/dashboard_monitor/assets/v2.2/css/components/sidebar.css +242 -0
- package/assets/manage/dashboard_monitor/assets/v2.2/css/components/topbar.css +138 -0
- package/assets/manage/dashboard_monitor/assets/v2.2/css/fonts.css +30 -0
- package/assets/manage/dashboard_monitor/assets/v2.2/css/synteq-ui-v2.css +25 -0
- package/assets/manage/dashboard_monitor/assets/v2.2/css/theme-dark.css +38 -0
- package/assets/manage/dashboard_monitor/assets/v2.2/css/theme-light.css +36 -0
- package/assets/manage/dashboard_monitor/assets/v2.2/css/tokens.css +61 -0
- package/assets/manage/dashboard_monitor/assets/v2.2/fonts/GoogleSansCode-Medium.ttf +0 -0
- package/assets/manage/dashboard_monitor/assets/v2.2/fonts/GoogleSansCode-Regular.ttf +0 -0
- package/assets/manage/dashboard_monitor/assets/v2.2/fonts/GoogleSansCode-SemiBold.ttf +0 -0
- package/assets/manage/dashboard_monitor/assets/v2.2/js/synteq-ui-v2.js +869 -0
- package/assets/manage/dashboard_monitor/deploy-dashboard.sh +230 -0
- package/assets/manage/dashboard_monitor/favicon.svg +13 -0
- package/assets/manage/dashboard_monitor/index.html +1197 -0
- package/assets/manage/dashboard_monitor/nginx-octra.conf +76 -0
- package/assets/manage/dashboard_monitor/octra-dashboard.service +29 -0
- package/assets/manage/dashboard_monitor/octra-node-prune.cron +12 -0
- package/assets/manage/dashboard_monitor/octra-node-prune.service +16 -0
- package/assets/manage/dashboard_monitor/octra-node-prune.timer +12 -0
- package/assets/manage/dashboard_monitor/prune-node-junk.sh +329 -0
- package/assets/manage/dashboard_monitor/server.py +1887 -0
- package/assets/manage/dashboard_monitor/ssl-certbot.sh +90 -0
- package/assets/manage/dashboard_monitor/styles.css +327 -0
- package/assets/manage/lib.sh +347 -0
- package/assets/manifest.json +143 -0
- package/assets/migrate/00-backup-identity.sh +67 -0
- package/assets/migrate/01-prepare-target.sh +145 -0
- package/assets/migrate/02-authorize-pull.sh +43 -0
- package/assets/migrate/03-pull-state.sh +102 -0
- package/assets/migrate/04-stop-source.sh +85 -0
- package/assets/migrate/05-finalize-target.sh +140 -0
- package/assets/migrate/06-start-target.sh +57 -0
- package/assets/migrate/07-verify.sh +237 -0
- package/assets/migrate/08-rollback.sh +54 -0
- package/assets/migrate/lib.sh +149 -0
- package/assets/upgrade/10-preflight.sh +120 -0
- package/assets/upgrade/20-upgrade.sh +142 -0
- package/assets/upgrade/30-manual-upgrade.sh +129 -0
- package/assets/upgrade/40-storage-prune.sh +65 -0
- package/assets/upgrade/50-status.sh +58 -0
- package/assets/upgrade/lib.sh +232 -0
- package/bin/ovtools.js +32 -0
- package/dist/api/checks.d.ts +6 -0
- package/dist/api/checks.d.ts.map +1 -0
- package/dist/api/checks.js +173 -0
- package/dist/api/checks.js.map +1 -0
- package/dist/api/hostkeys.d.ts +10 -0
- package/dist/api/hostkeys.d.ts.map +1 -0
- package/dist/api/hostkeys.js +58 -0
- package/dist/api/hostkeys.js.map +1 -0
- package/dist/api/jobs.d.ts +33 -0
- package/dist/api/jobs.d.ts.map +1 -0
- package/dist/api/jobs.js +486 -0
- package/dist/api/jobs.js.map +1 -0
- package/dist/api/nodes.d.ts +16 -0
- package/dist/api/nodes.d.ts.map +1 -0
- package/dist/api/nodes.js +126 -0
- package/dist/api/nodes.js.map +1 -0
- package/dist/api/server.d.ts +3 -0
- package/dist/api/server.d.ts.map +1 -0
- package/dist/api/server.js +443 -0
- package/dist/api/server.js.map +1 -0
- package/dist/cli.d.ts +3 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +301 -0
- package/dist/cli.js.map +1 -0
- package/dist/constants.d.ts +76 -0
- package/dist/constants.d.ts.map +1 -0
- package/dist/constants.js +150 -0
- package/dist/constants.js.map +1 -0
- package/dist/index.d.ts +18 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +60 -0
- package/dist/index.js.map +1 -0
- package/dist/orchestrator/base.d.ts +52 -0
- package/dist/orchestrator/base.d.ts.map +1 -0
- package/dist/orchestrator/base.js +219 -0
- package/dist/orchestrator/base.js.map +1 -0
- package/dist/orchestrator/deploy.d.ts +33 -0
- package/dist/orchestrator/deploy.d.ts.map +1 -0
- package/dist/orchestrator/deploy.js +313 -0
- package/dist/orchestrator/deploy.js.map +1 -0
- package/dist/orchestrator/manage.d.ts +28 -0
- package/dist/orchestrator/manage.d.ts.map +1 -0
- package/dist/orchestrator/manage.js +267 -0
- package/dist/orchestrator/manage.js.map +1 -0
- package/dist/orchestrator/migrate.d.ts +31 -0
- package/dist/orchestrator/migrate.d.ts.map +1 -0
- package/dist/orchestrator/migrate.js +442 -0
- package/dist/orchestrator/migrate.js.map +1 -0
- package/dist/ssh/client.d.ts +39 -0
- package/dist/ssh/client.d.ts.map +1 -0
- package/dist/ssh/client.js +418 -0
- package/dist/ssh/client.js.map +1 -0
- package/dist/test/core.test.d.ts +2 -0
- package/dist/test/core.test.d.ts.map +1 -0
- package/dist/test/core.test.js +188 -0
- package/dist/test/core.test.js.map +1 -0
- package/dist/types.d.ts +267 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +3 -0
- package/dist/types.js.map +1 -0
- package/dist/utils/assets.d.ts +28 -0
- package/dist/utils/assets.d.ts.map +1 -0
- package/dist/utils/assets.js +583 -0
- package/dist/utils/assets.js.map +1 -0
- package/dist/utils/browser.d.ts +2 -0
- package/dist/utils/browser.d.ts.map +1 -0
- package/dist/utils/browser.js +23 -0
- package/dist/utils/browser.js.map +1 -0
- package/dist/utils/dashboard.d.ts +16 -0
- package/dist/utils/dashboard.d.ts.map +1 -0
- package/dist/utils/dashboard.js +177 -0
- package/dist/utils/dashboard.js.map +1 -0
- package/dist/utils/helpers.d.ts +29 -0
- package/dist/utils/helpers.d.ts.map +1 -0
- package/dist/utils/helpers.js +204 -0
- package/dist/utils/helpers.js.map +1 -0
- package/dist/utils/scripts.d.ts +60 -0
- package/dist/utils/scripts.d.ts.map +1 -0
- package/dist/utils/scripts.js +174 -0
- package/dist/utils/scripts.js.map +1 -0
- package/dist/utils/state.d.ts +12 -0
- package/dist/utils/state.d.ts.map +1 -0
- package/dist/utils/state.js +44 -0
- package/dist/utils/state.js.map +1 -0
- package/dist/utils/tar.d.ts +11 -0
- package/dist/utils/tar.d.ts.map +1 -0
- package/dist/utils/tar.js +93 -0
- package/dist/utils/tar.js.map +1 -0
- package/dist/utils/update.d.ts +9 -0
- package/dist/utils/update.d.ts.map +1 -0
- package/dist/utils/update.js +134 -0
- package/dist/utils/update.js.map +1 -0
- package/dist/utils/websocket.d.ts +10 -0
- package/dist/utils/websocket.d.ts.map +1 -0
- package/dist/utils/websocket.js +203 -0
- package/dist/utils/websocket.js.map +1 -0
- package/package.json +55 -4
- package/public/app.js +2736 -0
- package/public/assets/v2.2/css/base.css +113 -0
- package/public/assets/v2.2/css/components/content.css +133 -0
- package/public/assets/v2.2/css/components/controls.css +229 -0
- package/public/assets/v2.2/css/components/data.css +373 -0
- package/public/assets/v2.2/css/components/icons.css +35 -0
- package/public/assets/v2.2/css/components/motion.css +57 -0
- package/public/assets/v2.2/css/components/shell.css +51 -0
- package/public/assets/v2.2/css/components/sidebar.css +242 -0
- package/public/assets/v2.2/css/components/topbar.css +138 -0
- package/public/assets/v2.2/css/fonts.css +30 -0
- package/public/assets/v2.2/css/synteq-ui-v2.css +25 -0
- package/public/assets/v2.2/css/theme-dark.css +38 -0
- package/public/assets/v2.2/css/theme-light.css +36 -0
- package/public/assets/v2.2/css/tokens.css +61 -0
- package/public/assets/v2.2/fonts/GoogleSansCode-Medium.ttf +0 -0
- package/public/assets/v2.2/fonts/GoogleSansCode-Regular.ttf +0 -0
- package/public/assets/v2.2/fonts/GoogleSansCode-SemiBold.ttf +0 -0
- package/public/assets/v2.2/js/synteq-ui-v2.js +869 -0
- package/public/favicon.svg +13 -0
- package/public/index.html +1083 -0
- package/public/styles.css +580 -0
|
@@ -0,0 +1,304 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# ===================================================================
|
|
3
|
+
# 60-enroll.sh - join the validator set (needs a funded wallet)
|
|
4
|
+
#
|
|
5
|
+
# * refuses to enroll unless the node RPC is ready (a node that is still
|
|
6
|
+
# syncing cannot commit the join)
|
|
7
|
+
# * calls the node's own controls/enroll.sh join --amount <raw>
|
|
8
|
+
# * when the wallet is not funded yet it exits 2 with state WAITING-FUNDING and
|
|
9
|
+
# prints the address to fund - that is not a failure, it is the next step
|
|
10
|
+
#
|
|
11
|
+
# Usage: sudo bash 60-enroll.sh [amount_raw]
|
|
12
|
+
# sudo bash 60-enroll.sh status
|
|
13
|
+
# Env: BOND_AMOUNT (default 1000000 raw = the protocol minimum)
|
|
14
|
+
# ENROLL_WAIT (600 s) - how long to follow the join before the soft stop
|
|
15
|
+
# ENROLL_PROGRESS_SEC (60) / STATUS_POLL (10) - progress line + poll interval
|
|
16
|
+
# JOIN_ATTEMPTS (3) / JOIN_RETRY_WAIT (10 s) - the node can answer a join
|
|
17
|
+
# with "bond submission unresolved; signed transaction retained; retry
|
|
18
|
+
# same command", so the same command is retried before anything is
|
|
19
|
+
# reported (2026-11-05)
|
|
20
|
+
# ===================================================================
|
|
21
|
+
|
|
22
|
+
set -euo pipefail
|
|
23
|
+
source "$(dirname "$0")/lib.sh"
|
|
24
|
+
|
|
25
|
+
require_root
|
|
26
|
+
ensure_tool curl curl || die "curl is missing and could not be installed"
|
|
27
|
+
ensure_tool jq jq || die "jq is missing and could not be installed"
|
|
28
|
+
ACTION="${1:-join}"
|
|
29
|
+
BOND_AMOUNT="${2:-${BOND_AMOUNT:-1000000}}"
|
|
30
|
+
|
|
31
|
+
if [[ "${ACTION}" == "status" ]]; then
|
|
32
|
+
set_status RUNNING "enroll-status"
|
|
33
|
+
as_octra "cd ${NODE_SRC_DIR} && sh controls/enroll.sh status" 2>&1 | sed 's/^/ /' || true
|
|
34
|
+
set_status DONE "enroll-status"
|
|
35
|
+
exit 0
|
|
36
|
+
fi
|
|
37
|
+
|
|
38
|
+
if [[ "${ACTION}" != "join" ]]; then
|
|
39
|
+
die "unknown action '${ACTION}' (join|status)"
|
|
40
|
+
fi
|
|
41
|
+
|
|
42
|
+
set_status RUNNING "enroll"
|
|
43
|
+
|
|
44
|
+
ENROLL_HELPER="${ENROLL_HELPER:-/root/node-enroll.sh}"
|
|
45
|
+
[[ -f "${NODE_WALLET}" ]] || die "wallet.json is missing - run the setup step first"
|
|
46
|
+
[[ -f "${ENROLL_HELPER}" ]] || die "node-enroll.sh is missing at ${ENROLL_HELPER} - push the scripts first"
|
|
47
|
+
|
|
48
|
+
ADDR=$(wallet_address || echo unknown)
|
|
49
|
+
log "=== enroll: ${ADDR} with amount ${BOND_AMOUNT} (raw) ==="
|
|
50
|
+
|
|
51
|
+
# 2026-11-03 (real incident): the bond transaction was confirmed in the explorer
|
|
52
|
+
# while this step sat on `controls/enroll.sh join` for 33 minutes with NO output -
|
|
53
|
+
# the old code captured the join in $(...) and had no timeout. The join is the
|
|
54
|
+
# node's whole enrollment (bond -> readiness via its duty loop -> wait for the
|
|
55
|
+
# set fold -> activation commit -> ROLE=validator + restart), and the public RPC
|
|
56
|
+
# kept refusing the readiness post. So:
|
|
57
|
+
# * the join runs in its OWN unit and is streamed, so it neither looks stuck
|
|
58
|
+
# nor dies when this step stops,
|
|
59
|
+
# * the state on the host/chain decides the verdict, not the exit code,
|
|
60
|
+
# * a membership that is already on chain never gets a second join,
|
|
61
|
+
# * the reason the network gives ("validator ready head has expired", "invalid
|
|
62
|
+
# nonce", HTTP 429) is quoted from the node log.
|
|
63
|
+
STATE_TIMEOUT="${STATE_TIMEOUT:-120}" # stat.sh is read-only and must not hang
|
|
64
|
+
|
|
65
|
+
read_state() { # fills SNAP/RPC/ACTIVE/SCHED/ENROLL/BOND/EPOCH/NEXT_SET/ACT_EPOCH
|
|
66
|
+
SNAP=$(timeout "${STATE_TIMEOUT}" su - "${NODE_USER}" -c \
|
|
67
|
+
"cd ${NODE_SRC_DIR} && sh controls/stat.sh" 2>/dev/null || true)
|
|
68
|
+
RPC=$(printf '%s\n' "${SNAP}" | stat_field rpc)
|
|
69
|
+
ACTIVE=$(printf '%s\n' "${SNAP}" | stat_field validator_active)
|
|
70
|
+
SCHED=$(printf '%s\n' "${SNAP}" | stat_field validator_scheduled)
|
|
71
|
+
ENROLL=$(printf '%s\n' "${SNAP}" | stat_field validator_enrollment)
|
|
72
|
+
BOND=$(printf '%s\n' "${SNAP}" | stat_field validator_bond)
|
|
73
|
+
EPOCH=$(printf '%s\n' "${SNAP}" | stat_field epoch)
|
|
74
|
+
# the chain tells us WHEN the validator is admitted (the set fold) and when it
|
|
75
|
+
# was activated - these two epochs are what the operator needs to read instead
|
|
76
|
+
# of a red error (2026-11-05)
|
|
77
|
+
NEXT_SET=$(printf '%s\n' "${SNAP}" | stat_field validator_next_set_epoch)
|
|
78
|
+
ACT_EPOCH=$(printf '%s\n' "${SNAP}" | stat_field validator_activation_epoch)
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
# What the chain says will happen next, in one line the console can read back:
|
|
82
|
+
# SCHEDULE next_set_epoch=1798156 activation_epoch=1798156 bond=24000000
|
|
83
|
+
schedule_line() {
|
|
84
|
+
printf 'SCHEDULE next_set_epoch=%s activation_epoch=%s bond=%s epoch=%s' \
|
|
85
|
+
"${NEXT_SET:--}" "${ACT_EPOCH:--}" "${BOND:--}" "${EPOCH:--}"
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
# The enrollment state is the authority: `bonded`/`ready` means the wallet is in
|
|
89
|
+
# the protocol's enrollment table, and `exiting` means it is still in it on its way
|
|
90
|
+
# out - signing another join in any of those is a SECOND bond. `validator_bond` is
|
|
91
|
+
# a number when the node can read it and EMPTY when it cannot (`stat_field` maps
|
|
92
|
+
# the node's Python `None` to an empty value): the old test compared the literal
|
|
93
|
+
# string "None" and counted it as a bond, so a missing bond looked like a bond.
|
|
94
|
+
enrolled_on_chain() {
|
|
95
|
+
case "${ENROLL:-}" in
|
|
96
|
+
bonded|ready|exiting) return 0 ;;
|
|
97
|
+
esac
|
|
98
|
+
[[ "${BOND:-0}" =~ ^[0-9]+$ && "${BOND:-0}" != "0" ]]
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
read_state
|
|
102
|
+
log "node: rpc=${RPC:-?} epoch=${EPOCH:-?}"
|
|
103
|
+
log "membership: active=${ACTIVE:-?} scheduled=${SCHED:-?} enrollment=${ENROLL:-?} bond=${BOND:--}"
|
|
104
|
+
|
|
105
|
+
if [[ "${ACTIVE}" == "true" ]]; then
|
|
106
|
+
log "this wallet is already an active validator - nothing to enroll"
|
|
107
|
+
set_status DONE "already active (bond ${BOND:-?})"
|
|
108
|
+
exit 0
|
|
109
|
+
fi
|
|
110
|
+
if [[ "${SCHED}" == "true" ]]; then
|
|
111
|
+
log "this wallet is already scheduled for the next set - nothing to enroll"
|
|
112
|
+
set_status DONE "already scheduled (bond ${BOND:-?})"
|
|
113
|
+
exit 0
|
|
114
|
+
fi
|
|
115
|
+
if enrolled_on_chain; then
|
|
116
|
+
log "this wallet is already enrolled on chain (bond ${BOND:--}, enrollment ${ENROLL})"
|
|
117
|
+
if [[ "${ENROLL}" == "ready" ]]; then
|
|
118
|
+
log "readiness is committed - the set fold admits it on its own (the promote step waits for that)"
|
|
119
|
+
else
|
|
120
|
+
log "the bond is committed; the node publishes readiness from its duty loop"
|
|
121
|
+
log "re-run -Phase enroll to follow it, or -Phase promote once the node reports scheduled"
|
|
122
|
+
fi
|
|
123
|
+
set_status DONE "already enrolled (bond ${BOND:--}, enrollment ${ENROLL})"
|
|
124
|
+
exit 0
|
|
125
|
+
fi
|
|
126
|
+
|
|
127
|
+
if [[ "${RPC}" != "ready" ]]; then
|
|
128
|
+
die "the node rpc is not ready (${RPC:-unknown}) - start the node and wait for sync first"
|
|
129
|
+
fi
|
|
130
|
+
|
|
131
|
+
# --- submit the join in its OWN transient unit -------------------------
|
|
132
|
+
# `controls/enroll.sh join` is the whole enrollment: bond -> (the node's duty
|
|
133
|
+
# loop publishes readiness) -> wait for the set fold -> wait for the activation
|
|
134
|
+
# commit -> write ROLE=validator + CONSENSUS_MODE=bft and restart the node.
|
|
135
|
+
# 2026-11-03 that took ~33 minutes because the public RPC kept refusing the
|
|
136
|
+
# readiness post ("validator ready head has expired", "invalid nonce", HTTP 429).
|
|
137
|
+
# So the join runs as its own unit: this step can report progress and stop
|
|
138
|
+
# WITHOUT killing the work the node is doing. post_hint() (lib.sh) quotes the
|
|
139
|
+
# last refusal.
|
|
140
|
+
JOIN_UNIT="${JOIN_UNIT:-octra-enroll-join}"
|
|
141
|
+
JOIN_OUT="${DEPLOY_DIR}/enroll-join.out"
|
|
142
|
+
ENROLL_WAIT="${ENROLL_WAIT:-900}"
|
|
143
|
+
|
|
144
|
+
join_active() { systemctl is-active --quiet "${JOIN_UNIT}" 2>/dev/null; }
|
|
145
|
+
|
|
146
|
+
start_join() {
|
|
147
|
+
: > "${JOIN_OUT}"
|
|
148
|
+
systemctl reset-failed "${JOIN_UNIT}" 2>/dev/null || true
|
|
149
|
+
systemd-run --unit="${JOIN_UNIT}" --collect --no-block --working-directory=/root \
|
|
150
|
+
--setenv=TERM=dumb --setenv=DEBIAN_FRONTEND=noninteractive \
|
|
151
|
+
bash -c "su - ${NODE_USER} -c 'cd ${NODE_SRC_DIR} && sh controls/enroll.sh join --amount ${BOND_AMOUNT}' >> '${JOIN_OUT}' 2>&1" \
|
|
152
|
+
|| die "could not start the enroll join unit ${JOIN_UNIT}"
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
log "the node publishes readiness itself, then waits for the set fold - this can take 10-40 min"
|
|
156
|
+
JOIN_ATTEMPTS="${JOIN_ATTEMPTS:-3}" # "retry same command" is the node's own advice
|
|
157
|
+
JOIN_RETRY_WAIT="${JOIN_RETRY_WAIT:-10}"
|
|
158
|
+
|
|
159
|
+
# The node can answer a join with
|
|
160
|
+
# status = refused reason = bond submission unresolved; signed transaction
|
|
161
|
+
# retained; retry same command; tx = d8cad812...
|
|
162
|
+
# The bond WAS signed and kept - only the submission did not go through in that
|
|
163
|
+
# window. 2026-11-05 (.231) this was reported FAILED while a plain retry landed the
|
|
164
|
+
# bond seconds later, so the statement "retry same command" is followed here.
|
|
165
|
+
join_retry_reason() {
|
|
166
|
+
{ grep -aoE 'bond submission unresolved[^"]{0,80}' "${JOIN_OUT}" 2>/dev/null | tail -1 || true; }
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
join_tx() {
|
|
170
|
+
{ grep -aoE 'tx = [0-9a-fA-F]{8,}' "${JOIN_OUT}" 2>/dev/null | tail -1 | awk '{print $3}' || true; }
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
# run the join once and follow it -> VERDICT (membership|ready|join-gone|wait)
|
|
174
|
+
run_join_once() {
|
|
175
|
+
log "submitting the join in unit ${JOIN_UNIT} (controls/enroll.sh join --amount ${BOND_AMOUNT})"
|
|
176
|
+
start_join
|
|
177
|
+
sleep 3
|
|
178
|
+
tail -n +1 -F "${JOIN_OUT}" 2>/dev/null &
|
|
179
|
+
TAIL_PID=$!
|
|
180
|
+
|
|
181
|
+
START=$(date +%s)
|
|
182
|
+
LAST=${START}
|
|
183
|
+
VERDICT=""
|
|
184
|
+
POLL="${STATUS_POLL:-10}" # how often the state is re-read (tests shrink it)
|
|
185
|
+
while :; do
|
|
186
|
+
read_state
|
|
187
|
+
if [[ "${ACTIVE}" == "true" || "${SCHED}" == "true" ]]; then VERDICT="membership"; break; fi
|
|
188
|
+
if [[ "${ENROLL}" == "ready" ]]; then VERDICT="ready"; break; fi
|
|
189
|
+
if ! join_active; then VERDICT="join-gone"; break; fi
|
|
190
|
+
NOW=$(date +%s)
|
|
191
|
+
if (( NOW - START >= ENROLL_WAIT )); then VERDICT="wait"; break; fi
|
|
192
|
+
if (( NOW - LAST >= ENROLL_PROGRESS_SEC )); then
|
|
193
|
+
HINT=$(post_hint)
|
|
194
|
+
log " enrolling: $(( NOW - START ))s of ${ENROLL_WAIT}s - enrollment=${ENROLL:-?} scheduled=${SCHED:-?}${NEXT_SET:+ set fold at epoch ${NEXT_SET}}${HINT:+ (network said: ${HINT})}"
|
|
195
|
+
LAST=${NOW}
|
|
196
|
+
fi
|
|
197
|
+
sleep "${POLL}"
|
|
198
|
+
done
|
|
199
|
+
kill "${TAIL_PID}" 2>/dev/null || true
|
|
200
|
+
wait "${TAIL_PID}" 2>/dev/null || true
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
ATTEMPT=0
|
|
204
|
+
while :; do
|
|
205
|
+
ATTEMPT=$(( ATTEMPT + 1 ))
|
|
206
|
+
run_join_once
|
|
207
|
+
if [[ "${VERDICT}" == "join-gone" && -n "$(join_retry_reason)" ]]; then
|
|
208
|
+
read_state
|
|
209
|
+
if ! enrolled_on_chain && (( ATTEMPT < JOIN_ATTEMPTS )); then
|
|
210
|
+
warn "the node retained the signed bond and asks for a retry (attempt ${ATTEMPT}/${JOIN_ATTEMPTS}, tx $(join_tx))"
|
|
211
|
+
sleep "${JOIN_RETRY_WAIT}"
|
|
212
|
+
continue
|
|
213
|
+
fi
|
|
214
|
+
fi
|
|
215
|
+
break
|
|
216
|
+
done
|
|
217
|
+
|
|
218
|
+
# the join restarts the node as validator when it is done: give it a moment
|
|
219
|
+
# before the state is read for the verdict
|
|
220
|
+
SETTLE_SEC="${SETTLE_SEC:-120}"
|
|
221
|
+
if [[ "${VERDICT}" == "join-gone" ]]; then
|
|
222
|
+
SETTLE_START=$(date +%s)
|
|
223
|
+
while (( $(date +%s) - SETTLE_START < SETTLE_SEC )); do
|
|
224
|
+
read_state
|
|
225
|
+
[[ "${ACTIVE}" == "true" || "${SCHED}" == "true" || "${ENROLL}" == "ready" ]] && break
|
|
226
|
+
sleep 5
|
|
227
|
+
done
|
|
228
|
+
fi
|
|
229
|
+
|
|
230
|
+
read_state
|
|
231
|
+
ROLE_NOW=$(printf '%s\n' "${SNAP}" | stat_field role)
|
|
232
|
+
log "after the join: role=${ROLE_NOW:-?} active=${ACTIVE:-?} scheduled=${SCHED:-?} enrollment=${ENROLL:-?} bond=${BOND:--}"
|
|
233
|
+
# The chain says WHEN this validator is admitted (the set fold) and when it was
|
|
234
|
+
# activated: that sentence is what the operator needs instead of a red error.
|
|
235
|
+
if [[ -n "${NEXT_SET}" ]]; then
|
|
236
|
+
log "$(schedule_line)"
|
|
237
|
+
log "the set fold admits this validator at epoch ${NEXT_SET} (now ${EPOCH:-?}) - it activates itself and starts voting"
|
|
238
|
+
fi
|
|
239
|
+
|
|
240
|
+
if [[ "${ACTIVE}" == "true" || "${SCHED}" == "true" ]]; then
|
|
241
|
+
log "the node is in the set (active=${ACTIVE} scheduled=${SCHED} role=${ROLE_NOW:-?}${ACT_EPOCH:+, activation epoch ${ACT_EPOCH}})"
|
|
242
|
+
set_status DONE "membership active=${ACTIVE} scheduled=${SCHED} role=${ROLE_NOW:-?}"
|
|
243
|
+
exit 0
|
|
244
|
+
fi
|
|
245
|
+
if [[ "${ENROLL}" == "ready" ]]; then
|
|
246
|
+
log "readiness is committed on chain (validator_enrollment = ready)"
|
|
247
|
+
log "the set fold schedules it next and the node switches its own role - the promote step waits for that"
|
|
248
|
+
set_status DONE "enrollment ready (bond ${BOND}) - set fold at epoch ${NEXT_SET:-unknown}"
|
|
249
|
+
exit 0
|
|
250
|
+
fi
|
|
251
|
+
|
|
252
|
+
HINT=$(post_hint)
|
|
253
|
+
# A bond that is already on chain is NOT a failure: the chain admits the validator
|
|
254
|
+
# at the set fold and the node promotes itself. Report the epoch instead.
|
|
255
|
+
if enrolled_on_chain; then
|
|
256
|
+
if [[ -n "${NEXT_SET}" ]]; then
|
|
257
|
+
log "the bond is on chain (bond ${BOND:--}, enrollment ${ENROLL}) - set fold at epoch ${NEXT_SET}"
|
|
258
|
+
else
|
|
259
|
+
log "the bond is on chain (bond ${BOND:--}, enrollment ${ENROLL}) - the node publishes readiness from its duty loop"
|
|
260
|
+
fi
|
|
261
|
+
if [[ -n "$(join_retry_reason)" ]]; then
|
|
262
|
+
# the tx is signed and kept, only the submission did not go through
|
|
263
|
+
set_status WAITING-CONFIRM "bond ${BOND:--} on chain, signed tx $(join_tx) still unresolved - re-run enroll to submit it again"
|
|
264
|
+
exit 2
|
|
265
|
+
fi
|
|
266
|
+
set_status WAITING-FOLD "bond ${BOND:--} on chain - scheduled for set epoch ${NEXT_SET:-unknown}; re-run enroll or promote later"
|
|
267
|
+
exit 2
|
|
268
|
+
fi
|
|
269
|
+
|
|
270
|
+
if [[ "${VERDICT}" == "wait" ]]; then
|
|
271
|
+
warn "enrollment is still pending after ${ENROLL_WAIT}s${HINT:+ - the network refused the readiness post: ${HINT}}"
|
|
272
|
+
warn "the node's own join keeps running in unit ${JOIN_UNIT} (follow: tail -f ${JOIN_OUT})"
|
|
273
|
+
warn "this is not a failure: the node waits for the set fold${NEXT_SET:+ at epoch ${NEXT_SET}}"
|
|
274
|
+
warn "re-run this step (Start deploy again / -Phase enroll), or -Phase promote once it reports ready"
|
|
275
|
+
set_status WAITING-FOLD "enrollment=${ENROLL:-unknown} pending after ${ENROLL_WAIT}s${NEXT_SET:+ - set fold at epoch ${NEXT_SET}}${HINT:+ - ${HINT}} - the join keeps running"
|
|
276
|
+
exit 2
|
|
277
|
+
fi
|
|
278
|
+
|
|
279
|
+
# The node kept the signed transaction: the same command submits it again. Never
|
|
280
|
+
# report this as a failure (2026-11-05: the bond landed on the retry).
|
|
281
|
+
if [[ -n "$(join_retry_reason)" ]]; then
|
|
282
|
+
warn "the node retained a signed bond transaction but the submission stayed unresolved"
|
|
283
|
+
warn "re-run this step (Start deploy again / -Phase enroll): the same command submits the retained tx"
|
|
284
|
+
set_status WAITING-CONFIRM "signed bond tx $(join_tx) retained after ${JOIN_ATTEMPTS} attempt(s) - re-run enroll to submit it"
|
|
285
|
+
exit 2
|
|
286
|
+
fi
|
|
287
|
+
|
|
288
|
+
if grep -qiE 'insufficient|not enough|balance|fund|stake|amount' "${JOIN_OUT}" 2>/dev/null; then
|
|
289
|
+
log "--- the wallet is not funded yet ---"
|
|
290
|
+
log "send OCTRA to: ${ADDR}"
|
|
291
|
+
log "then re-run this step (the console: Start deploy again / -Phase enroll)"
|
|
292
|
+
set_status WAITING-FUNDING "${ADDR} needs ${BOND_AMOUNT} raw - fund it and re-run enroll"
|
|
293
|
+
exit 2
|
|
294
|
+
fi
|
|
295
|
+
if grep -qi 'selection status = pending' "${JOIN_OUT}" 2>/dev/null; then
|
|
296
|
+
warn "the network has not selected this validator yet${HINT:+ (last refusal: ${HINT})}"
|
|
297
|
+
warn "re-run -Phase enroll later, or -Phase promote once the node reports scheduled"
|
|
298
|
+
set_status WAITING-FOLD "not selected yet${NEXT_SET:+ - set fold at epoch ${NEXT_SET}}${HINT:+ - ${HINT}}"
|
|
299
|
+
exit 2
|
|
300
|
+
fi
|
|
301
|
+
|
|
302
|
+
warn "--- last join output ---"
|
|
303
|
+
tail -n 20 "${JOIN_OUT}" 2>/dev/null | sed 's/^/ /' || true
|
|
304
|
+
die "controls/enroll.sh join ended without a membership${HINT:+ (last network refusal: ${HINT})}"
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# ===================================================================
|
|
3
|
+
# 70-promote.sh - switch an enrolled node from observer to validator
|
|
4
|
+
#
|
|
5
|
+
# The network admits a new validator only after the set fold (observed: ~64
|
|
6
|
+
# duty epochs). This step waits (bounded) for validator_active/validator_scheduled
|
|
7
|
+
# and then for the first duty epoch that actually activates the validator, and it
|
|
8
|
+
# applies the validator config and restarts the node when the node's own join has
|
|
9
|
+
# not done that already:
|
|
10
|
+
#
|
|
11
|
+
# ROLE=validator node-setup.sh (patches OCTRA_OPERATOR_ROLE / CONSENSUS_MODE)
|
|
12
|
+
# node-start.sh (pm2 start)
|
|
13
|
+
#
|
|
14
|
+
# Usage: sudo bash 70-promote.sh
|
|
15
|
+
# Env: PROMOTE_WAIT_SEC (default 3600) - wait for the set fold
|
|
16
|
+
# PROMOTE_ACTIVE_WAIT_SEC (default 900) - wait for validator_active=true
|
|
17
|
+
# PROMOTE_PROGRESS_SEC (120) / STATUS_POLL (20) - progress line + poll interval
|
|
18
|
+
# ===================================================================
|
|
19
|
+
|
|
20
|
+
set -euo pipefail
|
|
21
|
+
source "$(dirname "$0")/lib.sh"
|
|
22
|
+
|
|
23
|
+
require_root
|
|
24
|
+
set_status RUNNING "promote"
|
|
25
|
+
|
|
26
|
+
ensure_tool curl curl || die "curl is missing and could not be installed"
|
|
27
|
+
ensure_tool jq jq || die "jq is missing and could not be installed"
|
|
28
|
+
ensure_tool pgrep psmisc || die "pgrep (psmisc) is missing and could not be installed"
|
|
29
|
+
|
|
30
|
+
[[ -f "${NODE_ENV}" ]] || die "node.env is missing - run the setup step first"
|
|
31
|
+
# both helpers live at /root when the toolkit pushed them; overridable so the
|
|
32
|
+
# step is testable without writing to /root
|
|
33
|
+
NODE_SETUP_SH="${NODE_SETUP_SH:-/root/node-setup.sh}"
|
|
34
|
+
NODE_START_SH="${NODE_START_SH:-/root/node-start.sh}"
|
|
35
|
+
[[ -f "${NODE_SETUP_SH}" && -f "${NODE_START_SH}" ]] || die "node-setup.sh/node-start.sh missing at /root"
|
|
36
|
+
|
|
37
|
+
JOIN_UNIT="${JOIN_UNIT:-octra-enroll-join}"
|
|
38
|
+
# how often the wait re-prints its progress line (and the network hint)
|
|
39
|
+
PROMOTE_PROGRESS_SEC="${PROMOTE_PROGRESS_SEC:-120}"
|
|
40
|
+
|
|
41
|
+
SNAP=$(node_stat)
|
|
42
|
+
ACTIVE=$(printf '%s\n' "${SNAP}" | stat_field validator_active)
|
|
43
|
+
SCHED=$(printf '%s\n' "${SNAP}" | stat_field validator_scheduled)
|
|
44
|
+
ENROLL=$(printf '%s\n' "${SNAP}" | stat_field validator_enrollment)
|
|
45
|
+
# the chain announces WHEN the validator is admitted and when it activated: the
|
|
46
|
+
# operator gets those epochs instead of a red error (2026-11-05)
|
|
47
|
+
NEXT_SET=$(printf '%s\n' "${SNAP}" | stat_field validator_next_set_epoch)
|
|
48
|
+
ACT_EPOCH=$(printf '%s\n' "${SNAP}" | stat_field validator_activation_epoch)
|
|
49
|
+
|
|
50
|
+
if [[ "${ACTIVE}" == "true" ]]; then
|
|
51
|
+
log "this node is already an active validator (enrollment=${ENROLL}${ACT_EPOCH:+, activation epoch ${ACT_EPOCH}})"
|
|
52
|
+
else
|
|
53
|
+
WAIT="${PROMOTE_WAIT_SEC:-3600}"
|
|
54
|
+
DEADLINE=$(( $(date +%s) + WAIT ))
|
|
55
|
+
START=$(date +%s)
|
|
56
|
+
LAST=$START
|
|
57
|
+
FIRST_EPOCH=""
|
|
58
|
+
POLL="${STATUS_POLL:-20}" # how often the state is re-read (tests shrink it)
|
|
59
|
+
log "waiting for the set fold (active/scheduled) up to ${WAIT}s - enrollment=${ENROLL:-unknown}${NEXT_SET:+ - the chain schedules it for epoch ${NEXT_SET}}"
|
|
60
|
+
if systemctl is-active --quiet "${JOIN_UNIT}" 2>/dev/null; then
|
|
61
|
+
log "the node's own enrollment join is still running (unit ${JOIN_UNIT}) - it applies ROLE=validator itself"
|
|
62
|
+
log "follow it with: journalctl -u ${JOIN_UNIT} -f"
|
|
63
|
+
fi
|
|
64
|
+
while (( $(date +%s) < DEADLINE )); do
|
|
65
|
+
SNAP=$(node_stat)
|
|
66
|
+
ACTIVE=$(printf '%s\n' "${SNAP}" | stat_field validator_active)
|
|
67
|
+
SCHED=$(printf '%s\n' "${SNAP}" | stat_field validator_scheduled)
|
|
68
|
+
EPOCH=$(printf '%s\n' "${SNAP}" | stat_field epoch)
|
|
69
|
+
NEXT_SET=$(printf '%s\n' "${SNAP}" | stat_field validator_next_set_epoch)
|
|
70
|
+
[[ -z "${FIRST_EPOCH}" && -n "${EPOCH}" ]] && FIRST_EPOCH="${EPOCH}"
|
|
71
|
+
if [[ "${ACTIVE}" == "true" || "${SCHED}" == "true" ]]; then
|
|
72
|
+
log "membership ready: active=${ACTIVE} scheduled=${SCHED} at epoch ${EPOCH}${NEXT_SET:+ (set fold epoch ${NEXT_SET})}"
|
|
73
|
+
break
|
|
74
|
+
fi
|
|
75
|
+
if (( $(date +%s) - LAST >= PROMOTE_PROGRESS_SEC )); then
|
|
76
|
+
# the fold is gated by the node's readiness post: say why when the
|
|
77
|
+
# network refused it (that is what made a 2026-11-03 wait look endless)
|
|
78
|
+
HINT=$(post_hint)
|
|
79
|
+
log " still waiting: $(( ($(date +%s) - START) / 60 ))m of $(( WAIT / 60 ))m, epoch +$(( ${EPOCH:-0} - ${FIRST_EPOCH:-0} )) since the wait began (enrollment=${ENROLL:-unknown}${NEXT_SET:+ - set fold at epoch ${NEXT_SET}})${HINT:+ - network said: ${HINT}}"
|
|
80
|
+
LAST=$(date +%s)
|
|
81
|
+
fi
|
|
82
|
+
sleep "${POLL}"
|
|
83
|
+
done
|
|
84
|
+
if [[ "${ACTIVE}" != "true" && "${SCHED}" != "true" ]]; then
|
|
85
|
+
HINT=$(post_hint)
|
|
86
|
+
log "the fold has not admitted this validator yet${NEXT_SET:+ (the chain schedules it for epoch ${NEXT_SET})}${HINT:+ (the network refused the readiness post: ${HINT})} - it will on its own;"
|
|
87
|
+
log "re-run this step (or leave it to the next console run) when the node is scheduled"
|
|
88
|
+
set_status WAITING-FOLD "enrollment=${ENROLL:-unknown}${NEXT_SET:+ - set fold at epoch ${NEXT_SET}}${HINT:+ - ${HINT}} - re-run promote later"
|
|
89
|
+
exit 2
|
|
90
|
+
fi
|
|
91
|
+
|
|
92
|
+
# The fold only SCHEDULES the validator: it starts voting in its first duty
|
|
93
|
+
# epoch after that (2026-11-05: node .27 read scheduled=true at 00:12 and
|
|
94
|
+
# validator_active=true at 00:17). Waiting for the activation here means the
|
|
95
|
+
# verify that follows PASSes on its first sample instead of waiting for the
|
|
96
|
+
# epoch itself - bounded, and a timeout is still a soft "run me again".
|
|
97
|
+
if [[ "${ACTIVE}" != "true" ]]; then
|
|
98
|
+
AW="${PROMOTE_ACTIVE_WAIT_SEC:-900}"
|
|
99
|
+
if (( AW > 0 )); then
|
|
100
|
+
log "waiting up to ${AW}s for the first duty epoch to activate the validator (validator_active=true)"
|
|
101
|
+
ASTART=$(date +%s)
|
|
102
|
+
ALAST=$ASTART
|
|
103
|
+
while (( $(date +%s) - ASTART < AW )); do
|
|
104
|
+
SNAP=$(node_stat)
|
|
105
|
+
ACTIVE=$(printf '%s\n' "${SNAP}" | stat_field validator_active)
|
|
106
|
+
SCHED=$(printf '%s\n' "${SNAP}" | stat_field validator_scheduled)
|
|
107
|
+
VOTE_NOW=$(printf '%s\n' "${SNAP}" | stat_field voting)
|
|
108
|
+
if [[ "${ACTIVE}" == "true" ]]; then
|
|
109
|
+
log "the validator is ACTIVE (validator_active=true, voting=${VOTE_NOW:-?})"
|
|
110
|
+
break
|
|
111
|
+
fi
|
|
112
|
+
if (( $(date +%s) - ALAST >= PROMOTE_PROGRESS_SEC )); then
|
|
113
|
+
log " still waiting for activation: $(( ($(date +%s) - ASTART) / 60 ))m of $(( AW / 60 ))m (active=${ACTIVE:-?} voting=${VOTE_NOW:-?})"
|
|
114
|
+
ALAST=$(date +%s)
|
|
115
|
+
fi
|
|
116
|
+
sleep "${POLL}"
|
|
117
|
+
done
|
|
118
|
+
fi
|
|
119
|
+
fi
|
|
120
|
+
fi
|
|
121
|
+
|
|
122
|
+
# The node's own `controls/enroll.sh join` switches the role as its last step
|
|
123
|
+
# (ROLE=validator, CONSENSUS_MODE=bft, controls/run.sh). Re-configuring here
|
|
124
|
+
# would restart a node that is already correct, so check before touching it.
|
|
125
|
+
ROLE_NOW=$(printf '%s\n' "$(node_stat)" | stat_field role)
|
|
126
|
+
MODE_NOW=$(grep -m1 '^OCTRA_CONSENSUS_MODE=' "${NODE_ENV}" 2>/dev/null | cut -d= -f2)
|
|
127
|
+
if [[ "${ROLE_NOW}" == "validator" && "${MODE_NOW}" == "bft" ]]; then
|
|
128
|
+
log "role=${ROLE_NOW} and consensus mode=${MODE_NOW} are already set - nothing to reconfigure"
|
|
129
|
+
# Say which of the three real states this is - "already validator" while
|
|
130
|
+
# active=false was a false claim the console repeated (2026-11-05).
|
|
131
|
+
if [[ "${ACTIVE}" == "true" ]]; then
|
|
132
|
+
set_status DONE "active validator (validator_active=true scheduled=${SCHED:-?})"
|
|
133
|
+
elif [[ "${SCHED}" == "true" ]]; then
|
|
134
|
+
set_status DONE "scheduled for the set fold (validator_active=false) - verify waits for the duty epoch"
|
|
135
|
+
else
|
|
136
|
+
set_status DONE "role=validator configured (membership not active yet)"
|
|
137
|
+
fi
|
|
138
|
+
exit 0
|
|
139
|
+
fi
|
|
140
|
+
|
|
141
|
+
log "=== applying the validator configuration (role=${ROLE_NOW:-?} mode=${MODE_NOW:-?}) ==="
|
|
142
|
+
log "stopping the node (the role change must not race the running process)"
|
|
143
|
+
as_octra "cd ${NODE_SRC_DIR} && sh controls/stop.sh" 2>&1 | sed 's/^/ /' || warn "stop.sh reported an error"
|
|
144
|
+
sleep 3
|
|
145
|
+
|
|
146
|
+
ROLE=validator bash "${NODE_SETUP_SH}" 2>&1 | sed 's/^/ /' || die "node-setup.sh ROLE=validator failed"
|
|
147
|
+
|
|
148
|
+
log "starting the node as validator"
|
|
149
|
+
bash "${NODE_START_SH}" 2>&1 | sed 's/^/ /' || die "node-start.sh failed after the role change"
|
|
150
|
+
|
|
151
|
+
log "role now: $(grep -m1 '^OCTRA_OPERATOR_ROLE=' "${NODE_ENV}" | cut -d= -f2) / $(grep -m1 '^OCTRA_CONSENSUS_MODE=' "${NODE_ENV}" | cut -d= -f2)"
|
|
152
|
+
set_status DONE "promote"
|