ovtools 0.0.0-stage → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (198) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +121 -2
  3. package/assets/common/bond-inspect.sh +368 -0
  4. package/assets/common/config.sh +263 -0
  5. package/assets/common/extract-data.sh +197 -0
  6. package/assets/common/harden-host.sh +37 -0
  7. package/assets/common/install-deps.sh +93 -0
  8. package/assets/common/node-enroll.sh +65 -0
  9. package/assets/common/node-setup.sh +150 -0
  10. package/assets/common/node-start.sh +90 -0
  11. package/assets/common/node-stop.sh +18 -0
  12. package/assets/common/node-upgrade.sh +45 -0
  13. package/assets/common/pre-run.sh +497 -0
  14. package/assets/common/promote-validator.sh +104 -0
  15. package/assets/deploy/10-prepare-host.sh +98 -0
  16. package/assets/deploy/20-install-toolchain.sh +93 -0
  17. package/assets/deploy/30-setup-node.sh +82 -0
  18. package/assets/deploy/40-start-node.sh +75 -0
  19. package/assets/deploy/50-verify.sh +259 -0
  20. package/assets/deploy/60-enroll.sh +304 -0
  21. package/assets/deploy/70-promote.sh +152 -0
  22. package/assets/deploy/lib.sh +398 -0
  23. package/assets/manage/10-bond-increase.sh +149 -0
  24. package/assets/manage/10-diagnose-node.sh +169 -0
  25. package/assets/manage/20-exit-validator.sh +259 -0
  26. package/assets/manage/25-recover-node.sh +291 -0
  27. package/assets/manage/30-status.sh +64 -0
  28. package/assets/manage/40-exit-withdraw.sh +133 -0
  29. package/assets/manage/bond_inc.sh +318 -0
  30. package/assets/manage/dashboard_monitor/app.js +1544 -0
  31. package/assets/manage/dashboard_monitor/assets/v2.2/css/base.css +113 -0
  32. package/assets/manage/dashboard_monitor/assets/v2.2/css/components/content.css +133 -0
  33. package/assets/manage/dashboard_monitor/assets/v2.2/css/components/controls.css +229 -0
  34. package/assets/manage/dashboard_monitor/assets/v2.2/css/components/data.css +374 -0
  35. package/assets/manage/dashboard_monitor/assets/v2.2/css/components/icons.css +35 -0
  36. package/assets/manage/dashboard_monitor/assets/v2.2/css/components/motion.css +57 -0
  37. package/assets/manage/dashboard_monitor/assets/v2.2/css/components/shell.css +51 -0
  38. package/assets/manage/dashboard_monitor/assets/v2.2/css/components/sidebar.css +242 -0
  39. package/assets/manage/dashboard_monitor/assets/v2.2/css/components/topbar.css +138 -0
  40. package/assets/manage/dashboard_monitor/assets/v2.2/css/fonts.css +30 -0
  41. package/assets/manage/dashboard_monitor/assets/v2.2/css/synteq-ui-v2.css +25 -0
  42. package/assets/manage/dashboard_monitor/assets/v2.2/css/theme-dark.css +38 -0
  43. package/assets/manage/dashboard_monitor/assets/v2.2/css/theme-light.css +36 -0
  44. package/assets/manage/dashboard_monitor/assets/v2.2/css/tokens.css +61 -0
  45. package/assets/manage/dashboard_monitor/assets/v2.2/fonts/GoogleSansCode-Medium.ttf +0 -0
  46. package/assets/manage/dashboard_monitor/assets/v2.2/fonts/GoogleSansCode-Regular.ttf +0 -0
  47. package/assets/manage/dashboard_monitor/assets/v2.2/fonts/GoogleSansCode-SemiBold.ttf +0 -0
  48. package/assets/manage/dashboard_monitor/assets/v2.2/js/synteq-ui-v2.js +869 -0
  49. package/assets/manage/dashboard_monitor/deploy-dashboard.sh +230 -0
  50. package/assets/manage/dashboard_monitor/favicon.svg +13 -0
  51. package/assets/manage/dashboard_monitor/index.html +1197 -0
  52. package/assets/manage/dashboard_monitor/nginx-octra.conf +76 -0
  53. package/assets/manage/dashboard_monitor/octra-dashboard.service +29 -0
  54. package/assets/manage/dashboard_monitor/octra-node-prune.cron +12 -0
  55. package/assets/manage/dashboard_monitor/octra-node-prune.service +16 -0
  56. package/assets/manage/dashboard_monitor/octra-node-prune.timer +12 -0
  57. package/assets/manage/dashboard_monitor/prune-node-junk.sh +329 -0
  58. package/assets/manage/dashboard_monitor/server.py +1887 -0
  59. package/assets/manage/dashboard_monitor/ssl-certbot.sh +90 -0
  60. package/assets/manage/dashboard_monitor/styles.css +327 -0
  61. package/assets/manage/lib.sh +347 -0
  62. package/assets/manifest.json +143 -0
  63. package/assets/migrate/00-backup-identity.sh +67 -0
  64. package/assets/migrate/01-prepare-target.sh +145 -0
  65. package/assets/migrate/02-authorize-pull.sh +43 -0
  66. package/assets/migrate/03-pull-state.sh +102 -0
  67. package/assets/migrate/04-stop-source.sh +85 -0
  68. package/assets/migrate/05-finalize-target.sh +140 -0
  69. package/assets/migrate/06-start-target.sh +57 -0
  70. package/assets/migrate/07-verify.sh +237 -0
  71. package/assets/migrate/08-rollback.sh +54 -0
  72. package/assets/migrate/lib.sh +149 -0
  73. package/assets/upgrade/10-preflight.sh +120 -0
  74. package/assets/upgrade/20-upgrade.sh +142 -0
  75. package/assets/upgrade/30-manual-upgrade.sh +129 -0
  76. package/assets/upgrade/40-storage-prune.sh +65 -0
  77. package/assets/upgrade/50-status.sh +58 -0
  78. package/assets/upgrade/lib.sh +232 -0
  79. package/bin/ovtools.js +32 -0
  80. package/dist/api/checks.d.ts +6 -0
  81. package/dist/api/checks.d.ts.map +1 -0
  82. package/dist/api/checks.js +173 -0
  83. package/dist/api/checks.js.map +1 -0
  84. package/dist/api/hostkeys.d.ts +10 -0
  85. package/dist/api/hostkeys.d.ts.map +1 -0
  86. package/dist/api/hostkeys.js +58 -0
  87. package/dist/api/hostkeys.js.map +1 -0
  88. package/dist/api/jobs.d.ts +33 -0
  89. package/dist/api/jobs.d.ts.map +1 -0
  90. package/dist/api/jobs.js +486 -0
  91. package/dist/api/jobs.js.map +1 -0
  92. package/dist/api/nodes.d.ts +16 -0
  93. package/dist/api/nodes.d.ts.map +1 -0
  94. package/dist/api/nodes.js +126 -0
  95. package/dist/api/nodes.js.map +1 -0
  96. package/dist/api/server.d.ts +3 -0
  97. package/dist/api/server.d.ts.map +1 -0
  98. package/dist/api/server.js +443 -0
  99. package/dist/api/server.js.map +1 -0
  100. package/dist/cli.d.ts +3 -0
  101. package/dist/cli.d.ts.map +1 -0
  102. package/dist/cli.js +301 -0
  103. package/dist/cli.js.map +1 -0
  104. package/dist/constants.d.ts +76 -0
  105. package/dist/constants.d.ts.map +1 -0
  106. package/dist/constants.js +150 -0
  107. package/dist/constants.js.map +1 -0
  108. package/dist/index.d.ts +18 -0
  109. package/dist/index.d.ts.map +1 -0
  110. package/dist/index.js +60 -0
  111. package/dist/index.js.map +1 -0
  112. package/dist/orchestrator/base.d.ts +52 -0
  113. package/dist/orchestrator/base.d.ts.map +1 -0
  114. package/dist/orchestrator/base.js +219 -0
  115. package/dist/orchestrator/base.js.map +1 -0
  116. package/dist/orchestrator/deploy.d.ts +33 -0
  117. package/dist/orchestrator/deploy.d.ts.map +1 -0
  118. package/dist/orchestrator/deploy.js +313 -0
  119. package/dist/orchestrator/deploy.js.map +1 -0
  120. package/dist/orchestrator/manage.d.ts +28 -0
  121. package/dist/orchestrator/manage.d.ts.map +1 -0
  122. package/dist/orchestrator/manage.js +267 -0
  123. package/dist/orchestrator/manage.js.map +1 -0
  124. package/dist/orchestrator/migrate.d.ts +31 -0
  125. package/dist/orchestrator/migrate.d.ts.map +1 -0
  126. package/dist/orchestrator/migrate.js +442 -0
  127. package/dist/orchestrator/migrate.js.map +1 -0
  128. package/dist/ssh/client.d.ts +39 -0
  129. package/dist/ssh/client.d.ts.map +1 -0
  130. package/dist/ssh/client.js +418 -0
  131. package/dist/ssh/client.js.map +1 -0
  132. package/dist/test/core.test.d.ts +2 -0
  133. package/dist/test/core.test.d.ts.map +1 -0
  134. package/dist/test/core.test.js +188 -0
  135. package/dist/test/core.test.js.map +1 -0
  136. package/dist/types.d.ts +267 -0
  137. package/dist/types.d.ts.map +1 -0
  138. package/dist/types.js +3 -0
  139. package/dist/types.js.map +1 -0
  140. package/dist/utils/assets.d.ts +28 -0
  141. package/dist/utils/assets.d.ts.map +1 -0
  142. package/dist/utils/assets.js +583 -0
  143. package/dist/utils/assets.js.map +1 -0
  144. package/dist/utils/browser.d.ts +2 -0
  145. package/dist/utils/browser.d.ts.map +1 -0
  146. package/dist/utils/browser.js +23 -0
  147. package/dist/utils/browser.js.map +1 -0
  148. package/dist/utils/dashboard.d.ts +16 -0
  149. package/dist/utils/dashboard.d.ts.map +1 -0
  150. package/dist/utils/dashboard.js +177 -0
  151. package/dist/utils/dashboard.js.map +1 -0
  152. package/dist/utils/helpers.d.ts +29 -0
  153. package/dist/utils/helpers.d.ts.map +1 -0
  154. package/dist/utils/helpers.js +204 -0
  155. package/dist/utils/helpers.js.map +1 -0
  156. package/dist/utils/scripts.d.ts +60 -0
  157. package/dist/utils/scripts.d.ts.map +1 -0
  158. package/dist/utils/scripts.js +174 -0
  159. package/dist/utils/scripts.js.map +1 -0
  160. package/dist/utils/state.d.ts +12 -0
  161. package/dist/utils/state.d.ts.map +1 -0
  162. package/dist/utils/state.js +44 -0
  163. package/dist/utils/state.js.map +1 -0
  164. package/dist/utils/tar.d.ts +11 -0
  165. package/dist/utils/tar.d.ts.map +1 -0
  166. package/dist/utils/tar.js +93 -0
  167. package/dist/utils/tar.js.map +1 -0
  168. package/dist/utils/update.d.ts +9 -0
  169. package/dist/utils/update.d.ts.map +1 -0
  170. package/dist/utils/update.js +134 -0
  171. package/dist/utils/update.js.map +1 -0
  172. package/dist/utils/websocket.d.ts +10 -0
  173. package/dist/utils/websocket.d.ts.map +1 -0
  174. package/dist/utils/websocket.js +203 -0
  175. package/dist/utils/websocket.js.map +1 -0
  176. package/package.json +55 -4
  177. package/public/app.js +2736 -0
  178. package/public/assets/v2.2/css/base.css +113 -0
  179. package/public/assets/v2.2/css/components/content.css +133 -0
  180. package/public/assets/v2.2/css/components/controls.css +229 -0
  181. package/public/assets/v2.2/css/components/data.css +373 -0
  182. package/public/assets/v2.2/css/components/icons.css +35 -0
  183. package/public/assets/v2.2/css/components/motion.css +57 -0
  184. package/public/assets/v2.2/css/components/shell.css +51 -0
  185. package/public/assets/v2.2/css/components/sidebar.css +242 -0
  186. package/public/assets/v2.2/css/components/topbar.css +138 -0
  187. package/public/assets/v2.2/css/fonts.css +30 -0
  188. package/public/assets/v2.2/css/synteq-ui-v2.css +25 -0
  189. package/public/assets/v2.2/css/theme-dark.css +38 -0
  190. package/public/assets/v2.2/css/theme-light.css +36 -0
  191. package/public/assets/v2.2/css/tokens.css +61 -0
  192. package/public/assets/v2.2/fonts/GoogleSansCode-Medium.ttf +0 -0
  193. package/public/assets/v2.2/fonts/GoogleSansCode-Regular.ttf +0 -0
  194. package/public/assets/v2.2/fonts/GoogleSansCode-SemiBold.ttf +0 -0
  195. package/public/assets/v2.2/js/synteq-ui-v2.js +869 -0
  196. package/public/favicon.svg +13 -0
  197. package/public/index.html +1083 -0
  198. package/public/styles.css +580 -0
@@ -0,0 +1,304 @@
1
+ #!/usr/bin/env bash
2
+ # ===================================================================
3
+ # 60-enroll.sh - join the validator set (needs a funded wallet)
4
+ #
5
+ # * refuses to enroll unless the node RPC is ready (a node that is still
6
+ # syncing cannot commit the join)
7
+ # * calls the node's own controls/enroll.sh join --amount <raw>
8
+ # * when the wallet is not funded yet it exits 2 with state WAITING-FUNDING and
9
+ # prints the address to fund - that is not a failure, it is the next step
10
+ #
11
+ # Usage: sudo bash 60-enroll.sh [amount_raw]
12
+ # sudo bash 60-enroll.sh status
13
+ # Env: BOND_AMOUNT (default 1000000 raw = the protocol minimum)
14
+ # ENROLL_WAIT (600 s) - how long to follow the join before the soft stop
15
+ # ENROLL_PROGRESS_SEC (60) / STATUS_POLL (10) - progress line + poll interval
16
+ # JOIN_ATTEMPTS (3) / JOIN_RETRY_WAIT (10 s) - the node can answer a join
17
+ # with "bond submission unresolved; signed transaction retained; retry
18
+ # same command", so the same command is retried before anything is
19
+ # reported (2026-11-05)
20
+ # ===================================================================
21
+
22
+ set -euo pipefail
23
+ source "$(dirname "$0")/lib.sh"
24
+
25
+ require_root
26
+ ensure_tool curl curl || die "curl is missing and could not be installed"
27
+ ensure_tool jq jq || die "jq is missing and could not be installed"
28
+ ACTION="${1:-join}"
29
+ BOND_AMOUNT="${2:-${BOND_AMOUNT:-1000000}}"
30
+
31
+ if [[ "${ACTION}" == "status" ]]; then
32
+ set_status RUNNING "enroll-status"
33
+ as_octra "cd ${NODE_SRC_DIR} && sh controls/enroll.sh status" 2>&1 | sed 's/^/ /' || true
34
+ set_status DONE "enroll-status"
35
+ exit 0
36
+ fi
37
+
38
+ if [[ "${ACTION}" != "join" ]]; then
39
+ die "unknown action '${ACTION}' (join|status)"
40
+ fi
41
+
42
+ set_status RUNNING "enroll"
43
+
44
+ ENROLL_HELPER="${ENROLL_HELPER:-/root/node-enroll.sh}"
45
+ [[ -f "${NODE_WALLET}" ]] || die "wallet.json is missing - run the setup step first"
46
+ [[ -f "${ENROLL_HELPER}" ]] || die "node-enroll.sh is missing at ${ENROLL_HELPER} - push the scripts first"
47
+
48
+ ADDR=$(wallet_address || echo unknown)
49
+ log "=== enroll: ${ADDR} with amount ${BOND_AMOUNT} (raw) ==="
50
+
51
+ # 2026-11-03 (real incident): the bond transaction was confirmed in the explorer
52
+ # while this step sat on `controls/enroll.sh join` for 33 minutes with NO output -
53
+ # the old code captured the join in $(...) and had no timeout. The join is the
54
+ # node's whole enrollment (bond -> readiness via its duty loop -> wait for the
55
+ # set fold -> activation commit -> ROLE=validator + restart), and the public RPC
56
+ # kept refusing the readiness post. So:
57
+ # * the join runs in its OWN unit and is streamed, so it neither looks stuck
58
+ # nor dies when this step stops,
59
+ # * the state on the host/chain decides the verdict, not the exit code,
60
+ # * a membership that is already on chain never gets a second join,
61
+ # * the reason the network gives ("validator ready head has expired", "invalid
62
+ # nonce", HTTP 429) is quoted from the node log.
63
+ STATE_TIMEOUT="${STATE_TIMEOUT:-120}" # stat.sh is read-only and must not hang
64
+
65
+ read_state() { # fills SNAP/RPC/ACTIVE/SCHED/ENROLL/BOND/EPOCH/NEXT_SET/ACT_EPOCH
66
+ SNAP=$(timeout "${STATE_TIMEOUT}" su - "${NODE_USER}" -c \
67
+ "cd ${NODE_SRC_DIR} && sh controls/stat.sh" 2>/dev/null || true)
68
+ RPC=$(printf '%s\n' "${SNAP}" | stat_field rpc)
69
+ ACTIVE=$(printf '%s\n' "${SNAP}" | stat_field validator_active)
70
+ SCHED=$(printf '%s\n' "${SNAP}" | stat_field validator_scheduled)
71
+ ENROLL=$(printf '%s\n' "${SNAP}" | stat_field validator_enrollment)
72
+ BOND=$(printf '%s\n' "${SNAP}" | stat_field validator_bond)
73
+ EPOCH=$(printf '%s\n' "${SNAP}" | stat_field epoch)
74
+ # the chain tells us WHEN the validator is admitted (the set fold) and when it
75
+ # was activated - these two epochs are what the operator needs to read instead
76
+ # of a red error (2026-11-05)
77
+ NEXT_SET=$(printf '%s\n' "${SNAP}" | stat_field validator_next_set_epoch)
78
+ ACT_EPOCH=$(printf '%s\n' "${SNAP}" | stat_field validator_activation_epoch)
79
+ }
80
+
81
+ # What the chain says will happen next, in one line the console can read back:
82
+ # SCHEDULE next_set_epoch=1798156 activation_epoch=1798156 bond=24000000
83
+ schedule_line() {
84
+ printf 'SCHEDULE next_set_epoch=%s activation_epoch=%s bond=%s epoch=%s' \
85
+ "${NEXT_SET:--}" "${ACT_EPOCH:--}" "${BOND:--}" "${EPOCH:--}"
86
+ }
87
+
88
+ # The enrollment state is the authority: `bonded`/`ready` means the wallet is in
89
+ # the protocol's enrollment table, and `exiting` means it is still in it on its way
90
+ # out - signing another join in any of those is a SECOND bond. `validator_bond` is
91
+ # a number when the node can read it and EMPTY when it cannot (`stat_field` maps
92
+ # the node's Python `None` to an empty value): the old test compared the literal
93
+ # string "None" and counted it as a bond, so a missing bond looked like a bond.
94
+ enrolled_on_chain() {
95
+ case "${ENROLL:-}" in
96
+ bonded|ready|exiting) return 0 ;;
97
+ esac
98
+ [[ "${BOND:-0}" =~ ^[0-9]+$ && "${BOND:-0}" != "0" ]]
99
+ }
100
+
101
+ read_state
102
+ log "node: rpc=${RPC:-?} epoch=${EPOCH:-?}"
103
+ log "membership: active=${ACTIVE:-?} scheduled=${SCHED:-?} enrollment=${ENROLL:-?} bond=${BOND:--}"
104
+
105
+ if [[ "${ACTIVE}" == "true" ]]; then
106
+ log "this wallet is already an active validator - nothing to enroll"
107
+ set_status DONE "already active (bond ${BOND:-?})"
108
+ exit 0
109
+ fi
110
+ if [[ "${SCHED}" == "true" ]]; then
111
+ log "this wallet is already scheduled for the next set - nothing to enroll"
112
+ set_status DONE "already scheduled (bond ${BOND:-?})"
113
+ exit 0
114
+ fi
115
+ if enrolled_on_chain; then
116
+ log "this wallet is already enrolled on chain (bond ${BOND:--}, enrollment ${ENROLL})"
117
+ if [[ "${ENROLL}" == "ready" ]]; then
118
+ log "readiness is committed - the set fold admits it on its own (the promote step waits for that)"
119
+ else
120
+ log "the bond is committed; the node publishes readiness from its duty loop"
121
+ log "re-run -Phase enroll to follow it, or -Phase promote once the node reports scheduled"
122
+ fi
123
+ set_status DONE "already enrolled (bond ${BOND:--}, enrollment ${ENROLL})"
124
+ exit 0
125
+ fi
126
+
127
+ if [[ "${RPC}" != "ready" ]]; then
128
+ die "the node rpc is not ready (${RPC:-unknown}) - start the node and wait for sync first"
129
+ fi
130
+
131
+ # --- submit the join in its OWN transient unit -------------------------
132
+ # `controls/enroll.sh join` is the whole enrollment: bond -> (the node's duty
133
+ # loop publishes readiness) -> wait for the set fold -> wait for the activation
134
+ # commit -> write ROLE=validator + CONSENSUS_MODE=bft and restart the node.
135
+ # 2026-11-03 that took ~33 minutes because the public RPC kept refusing the
136
+ # readiness post ("validator ready head has expired", "invalid nonce", HTTP 429).
137
+ # So the join runs as its own unit: this step can report progress and stop
138
+ # WITHOUT killing the work the node is doing. post_hint() (lib.sh) quotes the
139
+ # last refusal.
140
+ JOIN_UNIT="${JOIN_UNIT:-octra-enroll-join}"
141
+ JOIN_OUT="${DEPLOY_DIR}/enroll-join.out"
142
+ ENROLL_WAIT="${ENROLL_WAIT:-900}"
143
+
144
+ join_active() { systemctl is-active --quiet "${JOIN_UNIT}" 2>/dev/null; }
145
+
146
+ start_join() {
147
+ : > "${JOIN_OUT}"
148
+ systemctl reset-failed "${JOIN_UNIT}" 2>/dev/null || true
149
+ systemd-run --unit="${JOIN_UNIT}" --collect --no-block --working-directory=/root \
150
+ --setenv=TERM=dumb --setenv=DEBIAN_FRONTEND=noninteractive \
151
+ bash -c "su - ${NODE_USER} -c 'cd ${NODE_SRC_DIR} && sh controls/enroll.sh join --amount ${BOND_AMOUNT}' >> '${JOIN_OUT}' 2>&1" \
152
+ || die "could not start the enroll join unit ${JOIN_UNIT}"
153
+ }
154
+
155
+ log "the node publishes readiness itself, then waits for the set fold - this can take 10-40 min"
156
+ JOIN_ATTEMPTS="${JOIN_ATTEMPTS:-3}" # "retry same command" is the node's own advice
157
+ JOIN_RETRY_WAIT="${JOIN_RETRY_WAIT:-10}"
158
+
159
+ # The node can answer a join with
160
+ # status = refused reason = bond submission unresolved; signed transaction
161
+ # retained; retry same command; tx = d8cad812...
162
+ # The bond WAS signed and kept - only the submission did not go through in that
163
+ # window. 2026-11-05 (.231) this was reported FAILED while a plain retry landed the
164
+ # bond seconds later, so the statement "retry same command" is followed here.
165
+ join_retry_reason() {
166
+ { grep -aoE 'bond submission unresolved[^"]{0,80}' "${JOIN_OUT}" 2>/dev/null | tail -1 || true; }
167
+ }
168
+
169
+ join_tx() {
170
+ { grep -aoE 'tx = [0-9a-fA-F]{8,}' "${JOIN_OUT}" 2>/dev/null | tail -1 | awk '{print $3}' || true; }
171
+ }
172
+
173
+ # run the join once and follow it -> VERDICT (membership|ready|join-gone|wait)
174
+ run_join_once() {
175
+ log "submitting the join in unit ${JOIN_UNIT} (controls/enroll.sh join --amount ${BOND_AMOUNT})"
176
+ start_join
177
+ sleep 3
178
+ tail -n +1 -F "${JOIN_OUT}" 2>/dev/null &
179
+ TAIL_PID=$!
180
+
181
+ START=$(date +%s)
182
+ LAST=${START}
183
+ VERDICT=""
184
+ POLL="${STATUS_POLL:-10}" # how often the state is re-read (tests shrink it)
185
+ while :; do
186
+ read_state
187
+ if [[ "${ACTIVE}" == "true" || "${SCHED}" == "true" ]]; then VERDICT="membership"; break; fi
188
+ if [[ "${ENROLL}" == "ready" ]]; then VERDICT="ready"; break; fi
189
+ if ! join_active; then VERDICT="join-gone"; break; fi
190
+ NOW=$(date +%s)
191
+ if (( NOW - START >= ENROLL_WAIT )); then VERDICT="wait"; break; fi
192
+ if (( NOW - LAST >= ENROLL_PROGRESS_SEC )); then
193
+ HINT=$(post_hint)
194
+ log " enrolling: $(( NOW - START ))s of ${ENROLL_WAIT}s - enrollment=${ENROLL:-?} scheduled=${SCHED:-?}${NEXT_SET:+ set fold at epoch ${NEXT_SET}}${HINT:+ (network said: ${HINT})}"
195
+ LAST=${NOW}
196
+ fi
197
+ sleep "${POLL}"
198
+ done
199
+ kill "${TAIL_PID}" 2>/dev/null || true
200
+ wait "${TAIL_PID}" 2>/dev/null || true
201
+ }
202
+
203
+ ATTEMPT=0
204
+ while :; do
205
+ ATTEMPT=$(( ATTEMPT + 1 ))
206
+ run_join_once
207
+ if [[ "${VERDICT}" == "join-gone" && -n "$(join_retry_reason)" ]]; then
208
+ read_state
209
+ if ! enrolled_on_chain && (( ATTEMPT < JOIN_ATTEMPTS )); then
210
+ warn "the node retained the signed bond and asks for a retry (attempt ${ATTEMPT}/${JOIN_ATTEMPTS}, tx $(join_tx))"
211
+ sleep "${JOIN_RETRY_WAIT}"
212
+ continue
213
+ fi
214
+ fi
215
+ break
216
+ done
217
+
218
+ # the join restarts the node as validator when it is done: give it a moment
219
+ # before the state is read for the verdict
220
+ SETTLE_SEC="${SETTLE_SEC:-120}"
221
+ if [[ "${VERDICT}" == "join-gone" ]]; then
222
+ SETTLE_START=$(date +%s)
223
+ while (( $(date +%s) - SETTLE_START < SETTLE_SEC )); do
224
+ read_state
225
+ [[ "${ACTIVE}" == "true" || "${SCHED}" == "true" || "${ENROLL}" == "ready" ]] && break
226
+ sleep 5
227
+ done
228
+ fi
229
+
230
+ read_state
231
+ ROLE_NOW=$(printf '%s\n' "${SNAP}" | stat_field role)
232
+ log "after the join: role=${ROLE_NOW:-?} active=${ACTIVE:-?} scheduled=${SCHED:-?} enrollment=${ENROLL:-?} bond=${BOND:--}"
233
+ # The chain says WHEN this validator is admitted (the set fold) and when it was
234
+ # activated: that sentence is what the operator needs instead of a red error.
235
+ if [[ -n "${NEXT_SET}" ]]; then
236
+ log "$(schedule_line)"
237
+ log "the set fold admits this validator at epoch ${NEXT_SET} (now ${EPOCH:-?}) - it activates itself and starts voting"
238
+ fi
239
+
240
+ if [[ "${ACTIVE}" == "true" || "${SCHED}" == "true" ]]; then
241
+ log "the node is in the set (active=${ACTIVE} scheduled=${SCHED} role=${ROLE_NOW:-?}${ACT_EPOCH:+, activation epoch ${ACT_EPOCH}})"
242
+ set_status DONE "membership active=${ACTIVE} scheduled=${SCHED} role=${ROLE_NOW:-?}"
243
+ exit 0
244
+ fi
245
+ if [[ "${ENROLL}" == "ready" ]]; then
246
+ log "readiness is committed on chain (validator_enrollment = ready)"
247
+ log "the set fold schedules it next and the node switches its own role - the promote step waits for that"
248
+ set_status DONE "enrollment ready (bond ${BOND}) - set fold at epoch ${NEXT_SET:-unknown}"
249
+ exit 0
250
+ fi
251
+
252
+ HINT=$(post_hint)
253
+ # A bond that is already on chain is NOT a failure: the chain admits the validator
254
+ # at the set fold and the node promotes itself. Report the epoch instead.
255
+ if enrolled_on_chain; then
256
+ if [[ -n "${NEXT_SET}" ]]; then
257
+ log "the bond is on chain (bond ${BOND:--}, enrollment ${ENROLL}) - set fold at epoch ${NEXT_SET}"
258
+ else
259
+ log "the bond is on chain (bond ${BOND:--}, enrollment ${ENROLL}) - the node publishes readiness from its duty loop"
260
+ fi
261
+ if [[ -n "$(join_retry_reason)" ]]; then
262
+ # the tx is signed and kept, only the submission did not go through
263
+ set_status WAITING-CONFIRM "bond ${BOND:--} on chain, signed tx $(join_tx) still unresolved - re-run enroll to submit it again"
264
+ exit 2
265
+ fi
266
+ set_status WAITING-FOLD "bond ${BOND:--} on chain - scheduled for set epoch ${NEXT_SET:-unknown}; re-run enroll or promote later"
267
+ exit 2
268
+ fi
269
+
270
+ if [[ "${VERDICT}" == "wait" ]]; then
271
+ warn "enrollment is still pending after ${ENROLL_WAIT}s${HINT:+ - the network refused the readiness post: ${HINT}}"
272
+ warn "the node's own join keeps running in unit ${JOIN_UNIT} (follow: tail -f ${JOIN_OUT})"
273
+ warn "this is not a failure: the node waits for the set fold${NEXT_SET:+ at epoch ${NEXT_SET}}"
274
+ warn "re-run this step (Start deploy again / -Phase enroll), or -Phase promote once it reports ready"
275
+ set_status WAITING-FOLD "enrollment=${ENROLL:-unknown} pending after ${ENROLL_WAIT}s${NEXT_SET:+ - set fold at epoch ${NEXT_SET}}${HINT:+ - ${HINT}} - the join keeps running"
276
+ exit 2
277
+ fi
278
+
279
+ # The node kept the signed transaction: the same command submits it again. Never
280
+ # report this as a failure (2026-11-05: the bond landed on the retry).
281
+ if [[ -n "$(join_retry_reason)" ]]; then
282
+ warn "the node retained a signed bond transaction but the submission stayed unresolved"
283
+ warn "re-run this step (Start deploy again / -Phase enroll): the same command submits the retained tx"
284
+ set_status WAITING-CONFIRM "signed bond tx $(join_tx) retained after ${JOIN_ATTEMPTS} attempt(s) - re-run enroll to submit it"
285
+ exit 2
286
+ fi
287
+
288
+ if grep -qiE 'insufficient|not enough|balance|fund|stake|amount' "${JOIN_OUT}" 2>/dev/null; then
289
+ log "--- the wallet is not funded yet ---"
290
+ log "send OCTRA to: ${ADDR}"
291
+ log "then re-run this step (the console: Start deploy again / -Phase enroll)"
292
+ set_status WAITING-FUNDING "${ADDR} needs ${BOND_AMOUNT} raw - fund it and re-run enroll"
293
+ exit 2
294
+ fi
295
+ if grep -qi 'selection status = pending' "${JOIN_OUT}" 2>/dev/null; then
296
+ warn "the network has not selected this validator yet${HINT:+ (last refusal: ${HINT})}"
297
+ warn "re-run -Phase enroll later, or -Phase promote once the node reports scheduled"
298
+ set_status WAITING-FOLD "not selected yet${NEXT_SET:+ - set fold at epoch ${NEXT_SET}}${HINT:+ - ${HINT}}"
299
+ exit 2
300
+ fi
301
+
302
+ warn "--- last join output ---"
303
+ tail -n 20 "${JOIN_OUT}" 2>/dev/null | sed 's/^/ /' || true
304
+ die "controls/enroll.sh join ended without a membership${HINT:+ (last network refusal: ${HINT})}"
@@ -0,0 +1,152 @@
1
+ #!/usr/bin/env bash
2
+ # ===================================================================
3
+ # 70-promote.sh - switch an enrolled node from observer to validator
4
+ #
5
+ # The network admits a new validator only after the set fold (observed: ~64
6
+ # duty epochs). This step waits (bounded) for validator_active/validator_scheduled
7
+ # and then for the first duty epoch that actually activates the validator, and it
8
+ # applies the validator config and restarts the node when the node's own join has
9
+ # not done that already:
10
+ #
11
+ # ROLE=validator node-setup.sh (patches OCTRA_OPERATOR_ROLE / CONSENSUS_MODE)
12
+ # node-start.sh (pm2 start)
13
+ #
14
+ # Usage: sudo bash 70-promote.sh
15
+ # Env: PROMOTE_WAIT_SEC (default 3600) - wait for the set fold
16
+ # PROMOTE_ACTIVE_WAIT_SEC (default 900) - wait for validator_active=true
17
+ # PROMOTE_PROGRESS_SEC (120) / STATUS_POLL (20) - progress line + poll interval
18
+ # ===================================================================
19
+
20
+ set -euo pipefail
21
+ source "$(dirname "$0")/lib.sh"
22
+
23
+ require_root
24
+ set_status RUNNING "promote"
25
+
26
+ ensure_tool curl curl || die "curl is missing and could not be installed"
27
+ ensure_tool jq jq || die "jq is missing and could not be installed"
28
+ ensure_tool pgrep psmisc || die "pgrep (psmisc) is missing and could not be installed"
29
+
30
+ [[ -f "${NODE_ENV}" ]] || die "node.env is missing - run the setup step first"
31
+ # both helpers live at /root when the toolkit pushed them; overridable so the
32
+ # step is testable without writing to /root
33
+ NODE_SETUP_SH="${NODE_SETUP_SH:-/root/node-setup.sh}"
34
+ NODE_START_SH="${NODE_START_SH:-/root/node-start.sh}"
35
+ [[ -f "${NODE_SETUP_SH}" && -f "${NODE_START_SH}" ]] || die "node-setup.sh/node-start.sh missing at /root"
36
+
37
+ JOIN_UNIT="${JOIN_UNIT:-octra-enroll-join}"
38
+ # how often the wait re-prints its progress line (and the network hint)
39
+ PROMOTE_PROGRESS_SEC="${PROMOTE_PROGRESS_SEC:-120}"
40
+
41
+ SNAP=$(node_stat)
42
+ ACTIVE=$(printf '%s\n' "${SNAP}" | stat_field validator_active)
43
+ SCHED=$(printf '%s\n' "${SNAP}" | stat_field validator_scheduled)
44
+ ENROLL=$(printf '%s\n' "${SNAP}" | stat_field validator_enrollment)
45
+ # the chain announces WHEN the validator is admitted and when it activated: the
46
+ # operator gets those epochs instead of a red error (2026-11-05)
47
+ NEXT_SET=$(printf '%s\n' "${SNAP}" | stat_field validator_next_set_epoch)
48
+ ACT_EPOCH=$(printf '%s\n' "${SNAP}" | stat_field validator_activation_epoch)
49
+
50
+ if [[ "${ACTIVE}" == "true" ]]; then
51
+ log "this node is already an active validator (enrollment=${ENROLL}${ACT_EPOCH:+, activation epoch ${ACT_EPOCH}})"
52
+ else
53
+ WAIT="${PROMOTE_WAIT_SEC:-3600}"
54
+ DEADLINE=$(( $(date +%s) + WAIT ))
55
+ START=$(date +%s)
56
+ LAST=$START
57
+ FIRST_EPOCH=""
58
+ POLL="${STATUS_POLL:-20}" # how often the state is re-read (tests shrink it)
59
+ log "waiting for the set fold (active/scheduled) up to ${WAIT}s - enrollment=${ENROLL:-unknown}${NEXT_SET:+ - the chain schedules it for epoch ${NEXT_SET}}"
60
+ if systemctl is-active --quiet "${JOIN_UNIT}" 2>/dev/null; then
61
+ log "the node's own enrollment join is still running (unit ${JOIN_UNIT}) - it applies ROLE=validator itself"
62
+ log "follow it with: journalctl -u ${JOIN_UNIT} -f"
63
+ fi
64
+ while (( $(date +%s) < DEADLINE )); do
65
+ SNAP=$(node_stat)
66
+ ACTIVE=$(printf '%s\n' "${SNAP}" | stat_field validator_active)
67
+ SCHED=$(printf '%s\n' "${SNAP}" | stat_field validator_scheduled)
68
+ EPOCH=$(printf '%s\n' "${SNAP}" | stat_field epoch)
69
+ NEXT_SET=$(printf '%s\n' "${SNAP}" | stat_field validator_next_set_epoch)
70
+ [[ -z "${FIRST_EPOCH}" && -n "${EPOCH}" ]] && FIRST_EPOCH="${EPOCH}"
71
+ if [[ "${ACTIVE}" == "true" || "${SCHED}" == "true" ]]; then
72
+ log "membership ready: active=${ACTIVE} scheduled=${SCHED} at epoch ${EPOCH}${NEXT_SET:+ (set fold epoch ${NEXT_SET})}"
73
+ break
74
+ fi
75
+ if (( $(date +%s) - LAST >= PROMOTE_PROGRESS_SEC )); then
76
+ # the fold is gated by the node's readiness post: say why when the
77
+ # network refused it (that is what made a 2026-11-03 wait look endless)
78
+ HINT=$(post_hint)
79
+ log " still waiting: $(( ($(date +%s) - START) / 60 ))m of $(( WAIT / 60 ))m, epoch +$(( ${EPOCH:-0} - ${FIRST_EPOCH:-0} )) since the wait began (enrollment=${ENROLL:-unknown}${NEXT_SET:+ - set fold at epoch ${NEXT_SET}})${HINT:+ - network said: ${HINT}}"
80
+ LAST=$(date +%s)
81
+ fi
82
+ sleep "${POLL}"
83
+ done
84
+ if [[ "${ACTIVE}" != "true" && "${SCHED}" != "true" ]]; then
85
+ HINT=$(post_hint)
86
+ log "the fold has not admitted this validator yet${NEXT_SET:+ (the chain schedules it for epoch ${NEXT_SET})}${HINT:+ (the network refused the readiness post: ${HINT})} - it will on its own;"
87
+ log "re-run this step (or leave it to the next console run) when the node is scheduled"
88
+ set_status WAITING-FOLD "enrollment=${ENROLL:-unknown}${NEXT_SET:+ - set fold at epoch ${NEXT_SET}}${HINT:+ - ${HINT}} - re-run promote later"
89
+ exit 2
90
+ fi
91
+
92
+ # The fold only SCHEDULES the validator: it starts voting in its first duty
93
+ # epoch after that (2026-11-05: node .27 read scheduled=true at 00:12 and
94
+ # validator_active=true at 00:17). Waiting for the activation here means the
95
+ # verify that follows PASSes on its first sample instead of waiting for the
96
+ # epoch itself - bounded, and a timeout is still a soft "run me again".
97
+ if [[ "${ACTIVE}" != "true" ]]; then
98
+ AW="${PROMOTE_ACTIVE_WAIT_SEC:-900}"
99
+ if (( AW > 0 )); then
100
+ log "waiting up to ${AW}s for the first duty epoch to activate the validator (validator_active=true)"
101
+ ASTART=$(date +%s)
102
+ ALAST=$ASTART
103
+ while (( $(date +%s) - ASTART < AW )); do
104
+ SNAP=$(node_stat)
105
+ ACTIVE=$(printf '%s\n' "${SNAP}" | stat_field validator_active)
106
+ SCHED=$(printf '%s\n' "${SNAP}" | stat_field validator_scheduled)
107
+ VOTE_NOW=$(printf '%s\n' "${SNAP}" | stat_field voting)
108
+ if [[ "${ACTIVE}" == "true" ]]; then
109
+ log "the validator is ACTIVE (validator_active=true, voting=${VOTE_NOW:-?})"
110
+ break
111
+ fi
112
+ if (( $(date +%s) - ALAST >= PROMOTE_PROGRESS_SEC )); then
113
+ log " still waiting for activation: $(( ($(date +%s) - ASTART) / 60 ))m of $(( AW / 60 ))m (active=${ACTIVE:-?} voting=${VOTE_NOW:-?})"
114
+ ALAST=$(date +%s)
115
+ fi
116
+ sleep "${POLL}"
117
+ done
118
+ fi
119
+ fi
120
+ fi
121
+
122
+ # The node's own `controls/enroll.sh join` switches the role as its last step
123
+ # (ROLE=validator, CONSENSUS_MODE=bft, controls/run.sh). Re-configuring here
124
+ # would restart a node that is already correct, so check before touching it.
125
+ ROLE_NOW=$(printf '%s\n' "$(node_stat)" | stat_field role)
126
+ MODE_NOW=$(grep -m1 '^OCTRA_CONSENSUS_MODE=' "${NODE_ENV}" 2>/dev/null | cut -d= -f2)
127
+ if [[ "${ROLE_NOW}" == "validator" && "${MODE_NOW}" == "bft" ]]; then
128
+ log "role=${ROLE_NOW} and consensus mode=${MODE_NOW} are already set - nothing to reconfigure"
129
+ # Say which of the three real states this is - "already validator" while
130
+ # active=false was a false claim the console repeated (2026-11-05).
131
+ if [[ "${ACTIVE}" == "true" ]]; then
132
+ set_status DONE "active validator (validator_active=true scheduled=${SCHED:-?})"
133
+ elif [[ "${SCHED}" == "true" ]]; then
134
+ set_status DONE "scheduled for the set fold (validator_active=false) - verify waits for the duty epoch"
135
+ else
136
+ set_status DONE "role=validator configured (membership not active yet)"
137
+ fi
138
+ exit 0
139
+ fi
140
+
141
+ log "=== applying the validator configuration (role=${ROLE_NOW:-?} mode=${MODE_NOW:-?}) ==="
142
+ log "stopping the node (the role change must not race the running process)"
143
+ as_octra "cd ${NODE_SRC_DIR} && sh controls/stop.sh" 2>&1 | sed 's/^/ /' || warn "stop.sh reported an error"
144
+ sleep 3
145
+
146
+ ROLE=validator bash "${NODE_SETUP_SH}" 2>&1 | sed 's/^/ /' || die "node-setup.sh ROLE=validator failed"
147
+
148
+ log "starting the node as validator"
149
+ bash "${NODE_START_SH}" 2>&1 | sed 's/^/ /' || die "node-start.sh failed after the role change"
150
+
151
+ log "role now: $(grep -m1 '^OCTRA_OPERATOR_ROLE=' "${NODE_ENV}" | cut -d= -f2) / $(grep -m1 '^OCTRA_CONSENSUS_MODE=' "${NODE_ENV}" | cut -d= -f2)"
152
+ set_status DONE "promote"