ovtools 0.0.0-stage → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (198) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +121 -2
  3. package/assets/common/bond-inspect.sh +368 -0
  4. package/assets/common/config.sh +263 -0
  5. package/assets/common/extract-data.sh +197 -0
  6. package/assets/common/harden-host.sh +37 -0
  7. package/assets/common/install-deps.sh +93 -0
  8. package/assets/common/node-enroll.sh +65 -0
  9. package/assets/common/node-setup.sh +150 -0
  10. package/assets/common/node-start.sh +90 -0
  11. package/assets/common/node-stop.sh +18 -0
  12. package/assets/common/node-upgrade.sh +45 -0
  13. package/assets/common/pre-run.sh +497 -0
  14. package/assets/common/promote-validator.sh +104 -0
  15. package/assets/deploy/10-prepare-host.sh +98 -0
  16. package/assets/deploy/20-install-toolchain.sh +93 -0
  17. package/assets/deploy/30-setup-node.sh +82 -0
  18. package/assets/deploy/40-start-node.sh +75 -0
  19. package/assets/deploy/50-verify.sh +259 -0
  20. package/assets/deploy/60-enroll.sh +304 -0
  21. package/assets/deploy/70-promote.sh +152 -0
  22. package/assets/deploy/lib.sh +398 -0
  23. package/assets/manage/10-bond-increase.sh +149 -0
  24. package/assets/manage/10-diagnose-node.sh +169 -0
  25. package/assets/manage/20-exit-validator.sh +259 -0
  26. package/assets/manage/25-recover-node.sh +291 -0
  27. package/assets/manage/30-status.sh +64 -0
  28. package/assets/manage/40-exit-withdraw.sh +133 -0
  29. package/assets/manage/bond_inc.sh +318 -0
  30. package/assets/manage/dashboard_monitor/app.js +1544 -0
  31. package/assets/manage/dashboard_monitor/assets/v2.2/css/base.css +113 -0
  32. package/assets/manage/dashboard_monitor/assets/v2.2/css/components/content.css +133 -0
  33. package/assets/manage/dashboard_monitor/assets/v2.2/css/components/controls.css +229 -0
  34. package/assets/manage/dashboard_monitor/assets/v2.2/css/components/data.css +374 -0
  35. package/assets/manage/dashboard_monitor/assets/v2.2/css/components/icons.css +35 -0
  36. package/assets/manage/dashboard_monitor/assets/v2.2/css/components/motion.css +57 -0
  37. package/assets/manage/dashboard_monitor/assets/v2.2/css/components/shell.css +51 -0
  38. package/assets/manage/dashboard_monitor/assets/v2.2/css/components/sidebar.css +242 -0
  39. package/assets/manage/dashboard_monitor/assets/v2.2/css/components/topbar.css +138 -0
  40. package/assets/manage/dashboard_monitor/assets/v2.2/css/fonts.css +30 -0
  41. package/assets/manage/dashboard_monitor/assets/v2.2/css/synteq-ui-v2.css +25 -0
  42. package/assets/manage/dashboard_monitor/assets/v2.2/css/theme-dark.css +38 -0
  43. package/assets/manage/dashboard_monitor/assets/v2.2/css/theme-light.css +36 -0
  44. package/assets/manage/dashboard_monitor/assets/v2.2/css/tokens.css +61 -0
  45. package/assets/manage/dashboard_monitor/assets/v2.2/fonts/GoogleSansCode-Medium.ttf +0 -0
  46. package/assets/manage/dashboard_monitor/assets/v2.2/fonts/GoogleSansCode-Regular.ttf +0 -0
  47. package/assets/manage/dashboard_monitor/assets/v2.2/fonts/GoogleSansCode-SemiBold.ttf +0 -0
  48. package/assets/manage/dashboard_monitor/assets/v2.2/js/synteq-ui-v2.js +869 -0
  49. package/assets/manage/dashboard_monitor/deploy-dashboard.sh +230 -0
  50. package/assets/manage/dashboard_monitor/favicon.svg +13 -0
  51. package/assets/manage/dashboard_monitor/index.html +1197 -0
  52. package/assets/manage/dashboard_monitor/nginx-octra.conf +76 -0
  53. package/assets/manage/dashboard_monitor/octra-dashboard.service +29 -0
  54. package/assets/manage/dashboard_monitor/octra-node-prune.cron +12 -0
  55. package/assets/manage/dashboard_monitor/octra-node-prune.service +16 -0
  56. package/assets/manage/dashboard_monitor/octra-node-prune.timer +12 -0
  57. package/assets/manage/dashboard_monitor/prune-node-junk.sh +329 -0
  58. package/assets/manage/dashboard_monitor/server.py +1887 -0
  59. package/assets/manage/dashboard_monitor/ssl-certbot.sh +90 -0
  60. package/assets/manage/dashboard_monitor/styles.css +327 -0
  61. package/assets/manage/lib.sh +347 -0
  62. package/assets/manifest.json +143 -0
  63. package/assets/migrate/00-backup-identity.sh +67 -0
  64. package/assets/migrate/01-prepare-target.sh +145 -0
  65. package/assets/migrate/02-authorize-pull.sh +43 -0
  66. package/assets/migrate/03-pull-state.sh +102 -0
  67. package/assets/migrate/04-stop-source.sh +85 -0
  68. package/assets/migrate/05-finalize-target.sh +140 -0
  69. package/assets/migrate/06-start-target.sh +57 -0
  70. package/assets/migrate/07-verify.sh +237 -0
  71. package/assets/migrate/08-rollback.sh +54 -0
  72. package/assets/migrate/lib.sh +149 -0
  73. package/assets/upgrade/10-preflight.sh +120 -0
  74. package/assets/upgrade/20-upgrade.sh +142 -0
  75. package/assets/upgrade/30-manual-upgrade.sh +129 -0
  76. package/assets/upgrade/40-storage-prune.sh +65 -0
  77. package/assets/upgrade/50-status.sh +58 -0
  78. package/assets/upgrade/lib.sh +232 -0
  79. package/bin/ovtools.js +32 -0
  80. package/dist/api/checks.d.ts +6 -0
  81. package/dist/api/checks.d.ts.map +1 -0
  82. package/dist/api/checks.js +173 -0
  83. package/dist/api/checks.js.map +1 -0
  84. package/dist/api/hostkeys.d.ts +10 -0
  85. package/dist/api/hostkeys.d.ts.map +1 -0
  86. package/dist/api/hostkeys.js +58 -0
  87. package/dist/api/hostkeys.js.map +1 -0
  88. package/dist/api/jobs.d.ts +33 -0
  89. package/dist/api/jobs.d.ts.map +1 -0
  90. package/dist/api/jobs.js +483 -0
  91. package/dist/api/jobs.js.map +1 -0
  92. package/dist/api/nodes.d.ts +16 -0
  93. package/dist/api/nodes.d.ts.map +1 -0
  94. package/dist/api/nodes.js +126 -0
  95. package/dist/api/nodes.js.map +1 -0
  96. package/dist/api/server.d.ts +3 -0
  97. package/dist/api/server.d.ts.map +1 -0
  98. package/dist/api/server.js +443 -0
  99. package/dist/api/server.js.map +1 -0
  100. package/dist/cli.d.ts +3 -0
  101. package/dist/cli.d.ts.map +1 -0
  102. package/dist/cli.js +301 -0
  103. package/dist/cli.js.map +1 -0
  104. package/dist/constants.d.ts +76 -0
  105. package/dist/constants.d.ts.map +1 -0
  106. package/dist/constants.js +150 -0
  107. package/dist/constants.js.map +1 -0
  108. package/dist/index.d.ts +18 -0
  109. package/dist/index.d.ts.map +1 -0
  110. package/dist/index.js +60 -0
  111. package/dist/index.js.map +1 -0
  112. package/dist/orchestrator/base.d.ts +52 -0
  113. package/dist/orchestrator/base.d.ts.map +1 -0
  114. package/dist/orchestrator/base.js +219 -0
  115. package/dist/orchestrator/base.js.map +1 -0
  116. package/dist/orchestrator/deploy.d.ts +33 -0
  117. package/dist/orchestrator/deploy.d.ts.map +1 -0
  118. package/dist/orchestrator/deploy.js +313 -0
  119. package/dist/orchestrator/deploy.js.map +1 -0
  120. package/dist/orchestrator/manage.d.ts +28 -0
  121. package/dist/orchestrator/manage.d.ts.map +1 -0
  122. package/dist/orchestrator/manage.js +267 -0
  123. package/dist/orchestrator/manage.js.map +1 -0
  124. package/dist/orchestrator/migrate.d.ts +31 -0
  125. package/dist/orchestrator/migrate.d.ts.map +1 -0
  126. package/dist/orchestrator/migrate.js +442 -0
  127. package/dist/orchestrator/migrate.js.map +1 -0
  128. package/dist/ssh/client.d.ts +39 -0
  129. package/dist/ssh/client.d.ts.map +1 -0
  130. package/dist/ssh/client.js +418 -0
  131. package/dist/ssh/client.js.map +1 -0
  132. package/dist/test/core.test.d.ts +2 -0
  133. package/dist/test/core.test.d.ts.map +1 -0
  134. package/dist/test/core.test.js +188 -0
  135. package/dist/test/core.test.js.map +1 -0
  136. package/dist/types.d.ts +267 -0
  137. package/dist/types.d.ts.map +1 -0
  138. package/dist/types.js +3 -0
  139. package/dist/types.js.map +1 -0
  140. package/dist/utils/assets.d.ts +28 -0
  141. package/dist/utils/assets.d.ts.map +1 -0
  142. package/dist/utils/assets.js +583 -0
  143. package/dist/utils/assets.js.map +1 -0
  144. package/dist/utils/browser.d.ts +2 -0
  145. package/dist/utils/browser.d.ts.map +1 -0
  146. package/dist/utils/browser.js +23 -0
  147. package/dist/utils/browser.js.map +1 -0
  148. package/dist/utils/dashboard.d.ts +16 -0
  149. package/dist/utils/dashboard.d.ts.map +1 -0
  150. package/dist/utils/dashboard.js +177 -0
  151. package/dist/utils/dashboard.js.map +1 -0
  152. package/dist/utils/helpers.d.ts +29 -0
  153. package/dist/utils/helpers.d.ts.map +1 -0
  154. package/dist/utils/helpers.js +204 -0
  155. package/dist/utils/helpers.js.map +1 -0
  156. package/dist/utils/scripts.d.ts +60 -0
  157. package/dist/utils/scripts.d.ts.map +1 -0
  158. package/dist/utils/scripts.js +174 -0
  159. package/dist/utils/scripts.js.map +1 -0
  160. package/dist/utils/state.d.ts +12 -0
  161. package/dist/utils/state.d.ts.map +1 -0
  162. package/dist/utils/state.js +44 -0
  163. package/dist/utils/state.js.map +1 -0
  164. package/dist/utils/tar.d.ts +11 -0
  165. package/dist/utils/tar.d.ts.map +1 -0
  166. package/dist/utils/tar.js +93 -0
  167. package/dist/utils/tar.js.map +1 -0
  168. package/dist/utils/update.d.ts +9 -0
  169. package/dist/utils/update.d.ts.map +1 -0
  170. package/dist/utils/update.js +134 -0
  171. package/dist/utils/update.js.map +1 -0
  172. package/dist/utils/websocket.d.ts +10 -0
  173. package/dist/utils/websocket.d.ts.map +1 -0
  174. package/dist/utils/websocket.js +203 -0
  175. package/dist/utils/websocket.js.map +1 -0
  176. package/package.json +55 -4
  177. package/public/app.js +2712 -0
  178. package/public/assets/v2.2/css/base.css +113 -0
  179. package/public/assets/v2.2/css/components/content.css +133 -0
  180. package/public/assets/v2.2/css/components/controls.css +229 -0
  181. package/public/assets/v2.2/css/components/data.css +373 -0
  182. package/public/assets/v2.2/css/components/icons.css +35 -0
  183. package/public/assets/v2.2/css/components/motion.css +57 -0
  184. package/public/assets/v2.2/css/components/shell.css +51 -0
  185. package/public/assets/v2.2/css/components/sidebar.css +242 -0
  186. package/public/assets/v2.2/css/components/topbar.css +138 -0
  187. package/public/assets/v2.2/css/fonts.css +30 -0
  188. package/public/assets/v2.2/css/synteq-ui-v2.css +25 -0
  189. package/public/assets/v2.2/css/theme-dark.css +38 -0
  190. package/public/assets/v2.2/css/theme-light.css +36 -0
  191. package/public/assets/v2.2/css/tokens.css +61 -0
  192. package/public/assets/v2.2/fonts/GoogleSansCode-Medium.ttf +0 -0
  193. package/public/assets/v2.2/fonts/GoogleSansCode-Regular.ttf +0 -0
  194. package/public/assets/v2.2/fonts/GoogleSansCode-SemiBold.ttf +0 -0
  195. package/public/assets/v2.2/js/synteq-ui-v2.js +869 -0
  196. package/public/favicon.svg +7 -0
  197. package/public/index.html +1080 -0
  198. package/public/styles.css +552 -0
@@ -0,0 +1,398 @@
1
+ #!/usr/bin/env bash
2
+ # ===================================================================
3
+ # scripts/deploy/lib.sh - shared helpers for the DEPLOY toolkit
4
+ #
5
+ # Sourced by every script in scripts/deploy/. It mirrors scripts/migrate/lib.sh
6
+ # (paths, logging, status markers) but keeps its own state directory, so the two
7
+ # programs never read the files of the other on the host:
8
+ #
9
+ # /root/deploy/ this program (/root/migration/ = migration)
10
+ #
11
+ # The long children (toolchain build, node build + 38 GB sync) run as systemd
12
+ # units and write an exit code to a file, so a dropped SSH never kills them and a
13
+ # re-run picks the finished result up.
14
+ # ===================================================================
15
+ set -euo pipefail
16
+
17
+ # Node paths: reuse the pushed config.sh (/root/config.sh) when present so both
18
+ # programs agree, and fall back to the same absolute defaults otherwise.
19
+ CONFIG_FILE="${CONFIG_FILE:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/../config.sh}"
20
+ if [[ -f "${CONFIG_FILE}" ]]; then
21
+ # shellcheck disable=SC1090
22
+ source "${CONFIG_FILE}"
23
+ fi
24
+
25
+ # Tool policy (check -> install -> continue) lives in config.sh, which is pushed
26
+ # to /root/config.sh next to these scripts. A host that still has an OLD config.sh
27
+ # gets a loud warning instead of a "command not found" ten minutes later.
28
+ if ! declare -F ensure_tool >/dev/null 2>&1; then
29
+ echo "[warn] ${CONFIG_FILE} has no ensure_tool (old copy on the host) - pushing the scripts again fixes it" >&2
30
+ ensure_tool() {
31
+ command -v "$1" >/dev/null 2>&1 || die "tool $1 is missing and ${CONFIG_FILE} cannot install it (push the scripts again)"
32
+ }
33
+ fi
34
+
35
+ DEPLOY_DIR="${DEPLOY_DIR:-/root/deploy}"
36
+ DEPLOY_LOG="${DEPLOY_LOG:-${DEPLOY_DIR}/deploy.log}"
37
+ DEPLOY_STATUS="${DEPLOY_STATUS:-${DEPLOY_DIR}/status}"
38
+ DEPLOY_HOST_ENV="${DEPLOY_HOST_ENV:-${DEPLOY_DIR}/host.env}"
39
+
40
+ NODE_SRC_DIR="${NODE_SRC_DIR:-/opt/octra/libv_litecore}"
41
+ NODE_DATA_ROOT="${NODE_DATA_ROOT:-/var/lib/octra}"
42
+ NODE_DATA_DIR="${NODE_DATA_DIR:-${NODE_DATA_ROOT}/devnet}"
43
+ NODE_SYNC_STAGE="${NODE_SYNC_STAGE:-${NODE_DATA_ROOT}/devnet.state_sync}"
44
+ NODE_ENV="${NODE_ENV:-${NODE_SRC_DIR}/.keys/validator/node.env}"
45
+ NODE_WALLET="${NODE_WALLET:-${NODE_SRC_DIR}/.keys/validator/wallet.json}"
46
+ NODE_USER="${NODE_USER:-octra}"
47
+ API_PORT="${API_PORT:-8080}"
48
+ CONSENSUS_PORT="${CONSENSUS_PORT:-19000}"
49
+ P2P_PORT="${P2P_PORT:-9000}"
50
+ NODE_PM2_NAME="${NODE_PM2_NAME:-octra-node-1}"
51
+ # the node's own operator log: the enroll/promote steps quote it when the
52
+ # network refuses a readiness post (the real reason a fold wait is slow).
53
+ # Defined AFTER NODE_SRC_DIR: with `set -u` the other order dies on a host whose
54
+ # /root/config.sh is missing (2026-11-04).
55
+ NODE_LOG="${NODE_LOG:-${NODE_SRC_DIR}/data/operator_logs/node.log}"
56
+ # how often a waiting step re-prints its progress line (and the network hint).
57
+ # Overridable so the wait path is testable without a 2-minute test.
58
+ ENROLL_PROGRESS_SEC="${ENROLL_PROGRESS_SEC:-60}"
59
+
60
+ mkdir -p "${DEPLOY_DIR}"
61
+
62
+ # The host keeps this log across runs (a redeploy appends to it): rotate it
63
+ # instead of letting it grow forever - a full deploy writes well under 1 MB.
64
+ log_bytes=$(stat -c%s "${DEPLOY_LOG}" 2>/dev/null || echo 0)
65
+ if [[ "${log_bytes}" =~ ^[0-9]+$ ]] && (( log_bytes > 5242880 )); then
66
+ mv -f "${DEPLOY_LOG}" "${DEPLOY_LOG}.1" 2>/dev/null || true
67
+ fi
68
+
69
+ log() { echo -e "[deploy] $*" | tee -a "${DEPLOY_LOG}"; }
70
+ warn() { echo -e "[warn] $*" | tee -a "${DEPLOY_LOG}" >&2; }
71
+ err() { echo -e "[err] $*" | tee -a "${DEPLOY_LOG}" >&2; }
72
+
73
+ set_status() { # set_status <STATE> [detail...] - read by the Windows orchestrator
74
+ # every argument after the state is the detail: several steps used to pass the
75
+ # label and the interesting part as two arguments (set_status DONE "enroll"
76
+ # "${ADDR}") and the third one was silently dropped, so the console showed
77
+ # "enroll" instead of the result.
78
+ local state="$1" detail
79
+ shift || true
80
+ detail="$*"
81
+ printf '%s %s %s\n' "${state}" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${detail}" > "${DEPLOY_STATUS}"
82
+ log "status = ${state} ${detail}"
83
+ }
84
+
85
+ die() { err "$*"; set_status FAILED "$*"; exit 1; }
86
+
87
+ require_root() { [[ $EUID -eq 0 ]] || die "run as root"; }
88
+
89
+ # --- a step must never die leaving status = RUNNING -------------------
90
+ # These scripts run detached (systemd unit) and the Windows orchestrator follows
91
+ # /root/deploy/status. A script that died from an unexpected error - a helper that
92
+ # no longer exists (2026-11-02: heal_opam_link), an unset variable with `set -u`, a
93
+ # full disk - wrote nothing, so the status stayed RUNNING and the console followed
94
+ # it until the 2-hour timeout instead of showing the error. This trap turns that
95
+ # class of failure into an immediate, readable FAILED with the failing line.
96
+ _deploy_on_err() {
97
+ local rc="$1" line="$2" cmd="$3" state
98
+ state=$(awk 'NR==1 {print $1}' "${DEPLOY_STATUS}" 2>/dev/null || true)
99
+ case "${state}" in
100
+ DONE|FAILED|WAITING-*) return 0 ;; # the script already decided
101
+ esac
102
+ err "unexpected failure (exit ${rc}) at line ${line}: ${cmd}"
103
+ set_status FAILED "line ${line}: ${cmd} (exit ${rc})"
104
+ }
105
+ trap '_deploy_on_err "$?" "$LINENO" "$BASH_COMMAND"' ERR
106
+
107
+ # --- node process/status helpers (same conventions as the migration lib) ---
108
+ # pgrep/pkill -f would match the shell running the check itself; [o]ctra avoids it
109
+ NODE_PROC_RE='[o]ctra_node\.exe|[o]ctra_pvac_worker\.exe'
110
+ node_processes() { ps -eo pid,args --no-headers 2>/dev/null | grep -E "${NODE_PROC_RE}" || true; }
111
+ as_octra() { su - "${NODE_USER}" -c "$*"; }
112
+ node_stat() { as_octra "cd ${NODE_SRC_DIR} && sh controls/stat.sh 2>/dev/null" || true; }
113
+ # ends with `sed`: the node prints Python's `None` for a field it cannot read
114
+ # (e.g. `validator_bond = None`), which used to be logged verbatim (`bond=None`)
115
+ # and compared as if it were a value. An unset field is an EMPTY value, so every
116
+ # `${BOND:-0}` default works as intended.
117
+ stat_field() {
118
+ awk -F' = ' -v k="$1" '$1==k { v=$2; if (v=="None" || v=="null") v=""; print v; exit }'
119
+ }
120
+
121
+ # The node publishes its readiness through its own duty loop; when the public
122
+ # RPC refuses that post the set fold never schedules us and the enroll/promote
123
+ # waits look stalled. This quotes the last refusal so the operator sees the
124
+ # real reason ("validator ready head has expired", "invalid nonce", HTTP 429).
125
+ #
126
+ # 2026-11-04 (real incident): the helper was called as `HINT=$(post_hint)` and
127
+ # returned the status of its own pipeline. With `set -o pipefail`, `grep` finds
128
+ # nothing on a HEALTHY node (no `set_post_failed` line yet) and exits 1 - so the
129
+ # assignment failed, `set -e` fired and the step died as
130
+ # `unexpected failure (exit 1) at line 62: HINT=$(post_hint)`
131
+ # before the fold wait could ever finish: the promote step kept dying at its
132
+ # 2-minute progress line and the node stayed an observer. A helper that only
133
+ # produces OPTIONAL text must never return non-zero - the callers rely on it.
134
+ post_hint() {
135
+ local line=""
136
+ [[ -f "${NODE_LOG}" ]] || return 0
137
+ line=$(tail -n 400 "${NODE_LOG}" 2>/dev/null | grep -a "set_post_failed" | tail -1 || true)
138
+ if [[ -n "${line}" ]]; then
139
+ printf '%s' "${line##*reason = }"
140
+ fi
141
+ return 0
142
+ }
143
+
144
+ api_get() { # api_get <path> - node REST API on localhost
145
+ curl -fsS --max-time 8 "http://127.0.0.1:${API_PORT}$1" 2>/dev/null || true
146
+ }
147
+
148
+ wallet_address() {
149
+ if [[ -f "${NODE_WALLET}" ]] && command -v jq >/dev/null 2>&1; then
150
+ jq -r '.address // empty' "${NODE_WALLET}" 2>/dev/null || true
151
+ fi
152
+ }
153
+
154
+ # --- the node repository: what is really checked out? ------------------
155
+ # The clone belongs to the node user, so `git` run as root refuses with
156
+ # "fatal: detected dubious ownership in repository" and every probe came back
157
+ # empty - the toolchain step then logged `repository: unknown` although the
158
+ # checkout was right there (2026-11-04). Ask as the node user, then retry as root
159
+ # with an explicit safe.directory, then read the ref out of .git directly, and
160
+ # report what git said when everything fails: the log line is never a bare
161
+ # "unknown" (a name, a date and a path are what an operator can act on).
162
+ repo_commit_from_dotgit() { # the ref the checkout points at, read by hand
163
+ local head ref sha=""
164
+ head="${NODE_SRC_DIR}/.git/HEAD"
165
+ [[ -f "${head}" ]] || return 0
166
+ read -r ref < "${head}" 2>/dev/null || ref=""
167
+ case "${ref}" in
168
+ "ref: "*)
169
+ ref="${ref#ref: }"
170
+ sha=$(head -c 64 "${NODE_SRC_DIR}/.git/${ref}" 2>/dev/null | tr -d '[:space:]' || true)
171
+ if [[ ! "${sha}" =~ ^[0-9a-f]{40}$ ]]; then
172
+ sha=$(awk -v r="${ref}" '$2 == r { print $1; exit }' \
173
+ "${NODE_SRC_DIR}/.git/packed-refs" 2>/dev/null || true)
174
+ fi
175
+ ;;
176
+ *) sha="${ref}" ;;
177
+ esac
178
+ [[ "${sha}" =~ ^[0-9a-f]{7,40}$ ]] || return 0
179
+ printf '%s' "${sha:0:8}"
180
+ }
181
+
182
+ node_repo_commit() { # short commit of the node repository ("" when unreadable)
183
+ local out=""
184
+ if id "${NODE_USER}" >/dev/null 2>&1; then
185
+ out=$(as_octra "cd ${NODE_SRC_DIR} && git rev-parse --short HEAD 2>/dev/null" 2>/dev/null || true)
186
+ fi
187
+ if [[ -z "${out}" ]]; then
188
+ out=$(git -c safe.directory="${NODE_SRC_DIR}" -C "${NODE_SRC_DIR}" \
189
+ rev-parse --short HEAD 2>/dev/null || true)
190
+ fi
191
+ if [[ -z "${out}" ]]; then
192
+ # git unusable (no binary / ownership rules / no safe.directory on git 2.35):
193
+ # .git/HEAD still names the commit
194
+ out=$(repo_commit_from_dotgit)
195
+ fi
196
+ printf '%s' "${out}"
197
+ }
198
+
199
+ node_repo_branch() { # branch of the checkout ("" when detached / unknown)
200
+ local out=""
201
+ out=$(git -c safe.directory="${NODE_SRC_DIR}" -C "${NODE_SRC_DIR}" \
202
+ rev-parse --abbrev-ref HEAD 2>/dev/null || true)
203
+ [[ "${out}" == "HEAD" ]] && out=""
204
+ if [[ -z "${out}" && -f "${NODE_SRC_DIR}/.git/HEAD" ]]; then
205
+ read -r out < "${NODE_SRC_DIR}/.git/HEAD" 2>/dev/null || out=""
206
+ case "${out}" in
207
+ "ref: refs/heads/"*) out="${out#ref: refs/heads/}" ;;
208
+ *) out="" ;;
209
+ esac
210
+ fi
211
+ printf '%s' "${out}"
212
+ }
213
+
214
+ node_repo_url() { # the clone URL this toolkit installs (config.sh)
215
+ printf '%s' "${NODE_REPO:-}"
216
+ }
217
+
218
+ node_repo_error() { # why the probes failed (diagnostics only, may be empty)
219
+ local as_user="" as_root=""
220
+ if id "${NODE_USER}" >/dev/null 2>&1; then
221
+ as_user=$(as_octra "cd ${NODE_SRC_DIR} && git rev-parse --short HEAD 2>&1 >/dev/null" 2>/dev/null \
222
+ | head -1 || true)
223
+ fi
224
+ as_root=$(git -c safe.directory="${NODE_SRC_DIR}" -C "${NODE_SRC_DIR}" \
225
+ rev-parse --short HEAD 2>&1 >/dev/null | head -1 || true)
226
+ if [[ -n "${as_user}" && "${as_user}" != "${as_root}" ]]; then
227
+ printf '%s; %s' "${as_user}" "${as_root}"
228
+ else
229
+ printf '%s' "${as_root:-${as_user}}"
230
+ fi
231
+ }
232
+
233
+ ensure_dirs() {
234
+ local dir
235
+ for dir in "${DEPLOY_DIR}" "${NODE_SRC_DIR}" "$(dirname "${NODE_SRC_DIR}")" \
236
+ "${NODE_DATA_ROOT}" "${NODE_DATA_DIR}" "${NODE_SYNC_STAGE}"; do
237
+ mkdir -p "${dir}"
238
+ done
239
+ if id "${NODE_USER}" >/dev/null 2>&1; then
240
+ chown "${NODE_USER}:${NODE_USER}" "${NODE_DATA_DIR}" "${NODE_SYNC_STAGE}" 2>/dev/null || true
241
+ fi
242
+ }
243
+
244
+ # --- is the OCaml switch usable? -------------------------------------
245
+ # `controls/install.sh --source-build` builds the switch inside
246
+ # runtime_data/toolchains/ocaml (a real switch root holds `.opam-switch`) and links
247
+ # it as <repo>/_opam. The danger is the follow-up `opam switch link` call: on
248
+ # opam 2.3 (Debian 13) a second call on the existing link dies with
249
+ # «"rmdir" failed on <repo>/_opam: Not a directory» AND empties the switch prefix,
250
+ # i.e. it silently throws the build away (proven 2026-11-02). config.sh therefore
251
+ # unlinks <repo>/_opam before every `switch link` call, and this check decides
252
+ # whether a build is really usable.
253
+ assert_opam_switch() {
254
+ local root link
255
+ root=$(ocaml_switch_root)
256
+ link="${NODE_SRC_DIR}/_opam"
257
+ if ocaml_switch_built; then
258
+ return 0
259
+ fi
260
+ warn "the OCaml switch does not look built: ${root}/_opam/.opam-switch is missing"
261
+ if [[ -L "${link}" ]]; then
262
+ warn " ${link} -> $(readlink "${link}") (a link, not a switch)"
263
+ else
264
+ warn " ${link} does not exist"
265
+ fi
266
+ warn " the toolchain step reported success but left no switch - the leftovers are dropped"
267
+ warn " automatically, so pressing Start deploy again rebuilds the runtime:"
268
+ warn " rm -f ${DEPLOY_DIR}/octra-deploy-toolchain.rc && bash ${DEPLOY_DIR}/20-install-toolchain.sh"
269
+ warn " for a closer look at the node's own installer:"
270
+ warn " su - ${NODE_USER} -c 'opam switch list' ; opam --version ; ls -la ${root}/_opam"
271
+ warn " (opam 2.3 'switch link' empties the switch when <repo>/_opam already exists -"
272
+ warn " config.sh unlinks that link before every switch link call since 2026-11-02)"
273
+ return 1
274
+ }
275
+
276
+ # --- long children as systemd units ---------------------------------
277
+ # install-deps.sh (toolchain, 15-30 min) and node-setup.sh (build + 38 GB sync)
278
+ # must survive a dropped SSH and be resumable: they run detached, write their
279
+ # log, and leave the exit code in a <unit>.rc file that the next run reads.
280
+ unit_start() { # unit_start <unit> <workdir> [K=V ...] -- <command>
281
+ local unit="$1" workdir="$2"
282
+ shift 2
283
+ local envs=()
284
+ while [[ $# -gt 0 && "$1" != "--" ]]; do envs+=("--setenv=$1"); shift; done
285
+ [[ "${1:-}" == "--" ]] && shift
286
+ local logfile="${DEPLOY_DIR}/${unit}.log" rcfile="${DEPLOY_DIR}/${unit}.rc"
287
+ rm -f "${logfile}" "${rcfile}"
288
+ systemctl reset-failed "${unit}" 2>/dev/null || true
289
+ systemd-run --unit="${unit}" --collect --no-block --working-directory="${workdir}" \
290
+ "${envs[@]}" bash -c "$* > '${logfile}' 2>&1; echo \$? > '${rcfile}'"
291
+ log "started unit ${unit} - log ${logfile}"
292
+ }
293
+
294
+ unit_active() { systemctl is-active --quiet "$1" && echo yes || echo no; }
295
+ unit_done() { [[ -s "${DEPLOY_DIR}/$1.rc" ]]; }
296
+ unit_rc() { tr -dc '0-9' < "${DEPLOY_DIR}/$1.rc" 2>/dev/null || echo 1; }
297
+ unit_tail() { tail -n "${2:-8}" "${DEPLOY_DIR}/$1.log" 2>/dev/null | sed 's/^/ /' || true; }
298
+
299
+ # --- human progress ---------------------------------------------------
300
+ # The long children print machine events (`event = sync_progress done = ...`)
301
+ # and apt bookkeeping. The operator needs one readable line per minute, so:
302
+ # progress_summary: sync event -> "sync 42.1% - 16.0 of 38.0 GiB"
303
+ # elapsed_minutes: 1500s -> "25 min"
304
+ human_gib() {
305
+ awk -v b="${1:-0}" 'BEGIN { if (b <= 0) { printf "?"; exit 0 } printf "%.1f GiB", b / 1073741824 }'
306
+ }
307
+
308
+ progress_summary() { # progress_summary "<last matching log line>"
309
+ local line="${1:-}"
310
+ if [[ "${line}" =~ pct\ =\ ([0-9.]+) ]]; then
311
+ local pct="${BASH_REMATCH[1]}" done_b="" total_b=""
312
+ [[ "${line}" =~ done\ =\ ([0-9]+) ]] && done_b="${BASH_REMATCH[1]}"
313
+ [[ "${line}" =~ total\ =\ ([0-9]+) ]] && total_b="${BASH_REMATCH[1]}"
314
+ printf 'sync %.1f%% - %s of %s' "${pct}" "$(human_gib "${done_b}")" "$(human_gib "${total_b}")"
315
+ return 0
316
+ fi
317
+ # anything else: keep it short enough for one row
318
+ printf '%s' "${line:0:150}"
319
+ }
320
+
321
+ elapsed_minutes() { # elapsed_minutes <seconds>
322
+ local sec="${1:-0}"
323
+ if (( sec < 90 )); then
324
+ printf '%ds' "${sec}"
325
+ else
326
+ printf '%d min' "$(( sec / 60 ))"
327
+ fi
328
+ }
329
+
330
+ unit_wait() { # unit_wait <unit> <timeout_sec> <label> [progress_grep]
331
+ local unit="$1" timeout="$2" label="${3:-$1}" progress_re="${4:-}"
332
+ local start now last rc extra tailpid="" logfile="${DEPLOY_DIR}/${unit}.log"
333
+ local last_note="" last_note_ts=0
334
+ start=$(date +%s)
335
+ last=$start
336
+ # The long child writes its own log. Follow it into THIS script's stdout, so
337
+ # the orchestrator (and the console) sees the real apt/compile/sync output
338
+ # live instead of only a status marker.
339
+ if [[ -f "${logfile}" ]]; then
340
+ tail -n +1 -F "${logfile}" 2>/dev/null &
341
+ tailpid=$!
342
+ fi
343
+ rc=0
344
+ while ! unit_done "${unit}"; do
345
+ now=$(date +%s)
346
+ if (( now - start > timeout )); then
347
+ warn "${label} did not finish in ${timeout}s - the unit may still be running (systemctl status ${unit})"
348
+ rc=1
349
+ break
350
+ fi
351
+ if [[ "$(unit_active "${unit}")" != "yes" ]]; then
352
+ # the unit vanished without an rc file: give systemd a moment, then fail
353
+ sleep 3
354
+ if ! unit_done "${unit}"; then
355
+ warn "${label}: unit ${unit} is gone without a status file - last log lines:"
356
+ unit_tail "${unit}" 12
357
+ rc=1
358
+ break
359
+ fi
360
+ fi
361
+ # One readable line a minute on top of the streamed output - and ONLY when the
362
+ # progress actually changed. Repeating "still running... 240s | Setting up
363
+ # node-es6-error" every minute is what made the log unreadable; when nothing
364
+ # new is in the child log, a quiet reminder appears every 5 minutes instead.
365
+ if (( now - last >= 60 )); then
366
+ extra=""
367
+ if [[ -n "${progress_re}" ]]; then
368
+ extra=$(progress_summary "$(grep -aE "${progress_re}" "${logfile}" 2>/dev/null | tail -1 || true)")
369
+ fi
370
+ if [[ -n "${extra}" && "${extra}" != "${last_note}" ]]; then
371
+ log " ${label}: ${extra} ($(elapsed_minutes $(( now - start ))) elapsed)"
372
+ last_note="${extra}"
373
+ last_note_ts=${now}
374
+ elif (( now - last_note_ts >= 300 )); then
375
+ log " ${label} still running - $(elapsed_minutes $(( now - start ))) elapsed, no new output"
376
+ last_note_ts=${now}
377
+ fi
378
+ last=$now
379
+ fi
380
+ sleep 15
381
+ done
382
+ # stop following before the verdict is printed, so the log stays ordered
383
+ if [[ -n "${tailpid}" ]]; then
384
+ kill "${tailpid}" 2>/dev/null || true
385
+ wait "${tailpid}" 2>/dev/null || true
386
+ fi
387
+ if (( rc != 0 )); then
388
+ return 1
389
+ fi
390
+ rc=$(unit_rc "${unit}")
391
+ if [[ "${rc}" != "0" ]]; then
392
+ warn "${label} failed (exit ${rc}) - last log lines:"
393
+ unit_tail "${unit}" 15
394
+ return 1
395
+ fi
396
+ log "${label} finished in $(( $(date +%s) - start ))s"
397
+ return 0
398
+ }
@@ -0,0 +1,149 @@
1
+ #!/usr/bin/env bash
2
+ # ===================================================================
3
+ # 10-bond-increase.sh - add OCT to the bond of the ACTIVE validator
4
+ #
5
+ # Runs the node's own `bond_inc.sh` as the `octra` user, from the node
6
+ # directory, and verifies the registry afterwards:
7
+ #
8
+ # bash bond_inc.sh -a <OCT> -w <wallet.json> (first increase)
9
+ # bash bond_inc.sh -a <OCT> -w <wallet.json> --new (every later one)
10
+ #
11
+ # `bond_inc.sh` keeps ONE signed increase in
12
+ # $OCTRA_DATA_DIR/validator-control/bond-increase.json:
13
+ # * no record -> plain run (the first increase)
14
+ # * record, tx confirmed -> `--new` is required, otherwise the call is a
15
+ # silent no-op (it just reports "confirmed")
16
+ # * record, tx pending/staging -> nothing new may be signed, wait instead
17
+ # This step therefore decides `--new` from the record itself; the operator only
18
+ # types the amount.
19
+ #
20
+ # Usage: BOND_OCT=100 sudo bash 10-bond-increase.sh
21
+ # Env: BOND_OCT=<OCT> additional amount, e.g. 100 or 12.5 (min 1)
22
+ # BOND_NEW=1 force --new (the console sets it when it knows better)
23
+ # BOND_WAIT=<sec> how long to wait for the registry (default 300)
24
+ # ===================================================================
25
+
26
+ set -euo pipefail
27
+ source "$(dirname "$0")/lib.sh"
28
+
29
+ require_root
30
+ set_status RUNNING "bond-increase"
31
+
32
+ BOND_OCT="${BOND_OCT:-}"
33
+ [[ -n "${BOND_OCT}" ]] || die "BOND_OCT is empty - type the amount to add (e.g. 100 for +100 OCT)"
34
+ AMOUNT_RAW=$(oct_to_raw "${BOND_OCT}")
35
+ (( AMOUNT_RAW >= TOKEN_UNIT )) || die "the protocol minimum is 1 OCT (got ${BOND_OCT})"
36
+ FORCE_NEW="${BOND_NEW:-0}"
37
+ WAIT="${BOND_WAIT:-300}"
38
+
39
+ ensure_tool jq jq || die "jq is missing and could not be installed"
40
+ ensure_tool python3 python3 || die "python3 is missing and could not be installed"
41
+
42
+ [[ -f "${NODE_WALLET}" ]] || die "wallet.json is missing (${NODE_WALLET}) - is this the validator host?"
43
+ [[ -f "${NODE_ENV}" ]] || die "node.env is missing (${NODE_ENV})"
44
+
45
+ read_state
46
+ ADDR=$(wallet_address || echo unknown)
47
+ log "=== increase bond: +${BOND_OCT} OCT (${AMOUNT_RAW} raw) on ${ADDR} ==="
48
+ log "node: role=${ROLE:-?} process=${PROCESS:-?} rpc=${RPC:-?} epoch=${EPOCH:-?}"
49
+ log "membership: active=${ACTIVE:-?} scheduled=${SCHED:-?} enrollment=${ENROLL:-?} bond=$(raw_to_oct "${BOND:-0}") OCT"
50
+
51
+ # The bond of a validator can only be raised on the host that runs it, and only
52
+ # while it is in the active set - `bond_inc.sh` refuses otherwise with
53
+ # "active ready validator required".
54
+ [[ "${ACTIVE}" == "true" ]] || die "this host is not an active validator (active=${ACTIVE:-?}, scheduled=${SCHED:-?}) - increase the bond on the host that votes"
55
+ [[ "${RPC}" == "ready" ]] || die "the node rpc is not ready (${RPC:-unknown}) - wait for the sync, then try again"
56
+
57
+ BEFORE="${BOND:-0}"
58
+ [[ "${BEFORE}" =~ ^[0-9]+$ ]] || BEFORE=0
59
+
60
+ install_bond_inc
61
+ show_saved_increase
62
+ SAVED=$(saved_increase_path)
63
+
64
+ NEW_FLAG=""
65
+ if [[ -n "${SAVED}" ]]; then
66
+ # a signed increase is stored: only `--new` may sign another one, and
67
+ # bond_inc.sh checks that the previous one is confirmed before it does
68
+ NEW_FLAG="--new"
69
+ log "a saved increase exists - running with --new (the previous one must be confirmed)"
70
+ elif [[ "${FORCE_NEW}" == "1" ]]; then
71
+ NEW_FLAG="--new"
72
+ log "the console asked for --new"
73
+ fi
74
+
75
+ run_bond_inc() { # run_bond_inc [--new] -> prints the tool output, returns its code
76
+ local extra="${1:-}" rc
77
+ set +e
78
+ # shellcheck disable=SC2086
79
+ ( cd "${NODE_SRC_DIR}" && timeout 300 su - "${NODE_USER}" -c \
80
+ "cd ${NODE_SRC_DIR} && sh ./bond_inc.sh -a ${BOND_OCT} -w ${NODE_WALLET} ${extra}" ) 2>&1
81
+ rc=$?
82
+ set -e
83
+ return "${rc}"
84
+ }
85
+
86
+ log "running bond_inc.sh -a ${BOND_OCT} -w ${NODE_WALLET} ${NEW_FLAG}"
87
+
88
+ # Poll the registry until the bond really moved. The tool prints "accepted" as
89
+ # soon as the RPC took the transaction; the bond is only final after the epoch
90
+ # that includes it, so the step (not the tool) decides when the increase landed.
91
+ wait_for_registry() { # wait_for_registry <before_raw> <label>
92
+ local before="$1" label="${2:-bond increase submitted}"
93
+ local target=$(( before + AMOUNT_RAW )) current now start last
94
+ start=$(date +%s)
95
+ last=${start}
96
+ while :; do
97
+ read_state
98
+ current="${BOND:-0}"
99
+ [[ "${current}" =~ ^[0-9]+$ ]] || current=0
100
+ if (( current >= target )); then
101
+ log "bond is now $(raw_to_oct "${current}") OCT (+$(raw_to_oct "$(( current - before ))") OCT) - tx ${TX:-?}"
102
+ set_status DONE "bond $(raw_to_oct "${current}") OCT (added $(raw_to_oct "$(( current - before ))") OCT, tx ${TX:-unknown})"
103
+ exit 0
104
+ fi
105
+ now=$(date +%s)
106
+ if (( now - start >= WAIT )); then
107
+ warn "${label}: the registry still shows $(raw_to_oct "${current}") OCT after ${WAIT}s (expected $(raw_to_oct "${target}"))"
108
+ warn "check the transaction with: su - ${NODE_USER} -c 'cd ${NODE_SRC_DIR} && sh controls/enroll.sh status'"
109
+ set_status WAITING-CONFIRM "tx ${TX:-unknown} pending - bond $(raw_to_oct "${current}") of $(raw_to_oct "${target}") OCT; re-run to re-check"
110
+ exit 2
111
+ fi
112
+ if (( now - last >= 30 )); then
113
+ log " waiting for the registry: bond $(raw_to_oct "${current}") of $(raw_to_oct "${target}") OCT ($(( now - start ))s of ${WAIT}s)"
114
+ last=${now}
115
+ fi
116
+ sleep 10
117
+ done
118
+ }
119
+
120
+ set +e
121
+ OUT=$(run_bond_inc "${NEW_FLAG}")
122
+ BOND_RC=$?
123
+ set -e
124
+ printf '%s\n' "${OUT}" | sed 's/^/ /'
125
+ TX=$(printf '%s\n' "${OUT}" | awk '/^tx = /{print $3; exit}')
126
+ STATUS_LINE=$(printf '%s\n' "${OUT}" | awk '/event = bond_inc /{print; exit}')
127
+
128
+ # `saved increase differs` means the stored record no longer matches this wallet
129
+ # or the registry: the documented remedy is bond-inspect.sh + --new once the
130
+ # previous transaction is confirmed. Never sign blindly a second time.
131
+ if printf '%s' "${OUT}" | grep -q "saved increase differs"; then
132
+ log "the stored increase no longer matches the registry - the tool refuses to sign another one"
133
+ log "inspect it with: sudo bash ${MANAGE_DIR}/bond-inspect.sh -a ${BOND_OCT}"
134
+ die "saved increase differs; inspect the stored record before requesting another increase"
135
+ fi
136
+ if printf '%s' "${OUT}" | grep -q "previous increase is not confirmed"; then
137
+ wait_for_registry "${BEFORE}" "the previous increase is not confirmed yet"
138
+ fi
139
+ if printf '%s' "${OUT}" | grep -qE "account has pending transactions|submission unresolved|submission not acknowledged"; then
140
+ warn "the submission is unresolved (tx ${TX:-unknown}) - the payment is stored on the host"
141
+ warn "re-run the same amount to resubmit it (bond_inc.sh reuses the saved payment)"
142
+ set_status WAITING-CONFIRM "tx ${TX:-unknown} submitted but not acknowledged - re-run increase ${BOND_OCT} OCT"
143
+ exit 2
144
+ fi
145
+ if (( BOND_RC != 0 )); then
146
+ die "bond_inc.sh refused (exit ${BOND_RC}): ${STATUS_LINE:-see the output above}"
147
+ fi
148
+
149
+ wait_for_registry "${BEFORE}" "bond increase submitted"