@ouro.bot/cli 0.1.0-alpha.834 → 0.1.0-alpha.835
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.json +8 -0
- package/deploy/unraid/sanctuary-butler-upgrade.mjs +151 -33
- package/deploy/unraid/sanctuary.ouro/bundle-meta.json +1 -1
- package/deploy/unraid/sanctuary.xml +1 -1
- package/dist/heart/daemon/sanctuary-authority-epoch.js +25 -0
- package/dist/heart/daemon/sanctuary-authority-root-lifecycle.js +282 -22
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
package/changelog.json
CHANGED
|
@@ -1,6 +1,14 @@
|
|
|
1
1
|
{
|
|
2
2
|
"_note": "This changelog is maintained as part of the PR/version-bump workflow. Agent-curated, not auto-generated. Agents read this file directly via read_file to understand what changed between versions.",
|
|
3
3
|
"versions": [
|
|
4
|
+
{
|
|
5
|
+
"version": "0.1.0-alpha.835",
|
|
6
|
+
"changes": [
|
|
7
|
+
"Sanctuary: in-place upgrade of an installed root authority (D-043). The authority keeps its epoch, token, issuer and Telegram cursor; only the reviewed package, its pins and the resident image change. The new package's lifecycle runs stop, switch, resident, migrate and start from a journal, backs up every root record it rewrites, and rolls back exactly on any failure. An interrupted upgrade rolls back on the next authority boot. No token rotation.",
|
|
8
|
+
"Upgrade orchestrator: new upgrade phase stages the package beside the live one, pauses host supervision behind a self-expiring maintenance flag, runs the lifecycle upgrade with automatic rollback, pins the DockerMan template, and resumes supervision. --rehearse <step> stops after a step and rolls back, proving the rollback on the real host. The host keeper now also restarts a resident left stopped.",
|
|
9
|
+
"Sanctuary root lifecycle failures are recorded with their real cause in a root-only log (the authority directory), and stderr names that file (D-034). A stale copy of the live token left in incoming-token is removed after an upgrade (D-044)."
|
|
10
|
+
]
|
|
11
|
+
},
|
|
4
12
|
{
|
|
5
13
|
"version": "0.1.0-alpha.834",
|
|
6
14
|
"changes": [
|
|
@@ -19,11 +19,17 @@
|
|
|
19
19
|
// automatic rollback on ANY failure so the Butler is never left
|
|
20
20
|
// down. Requires a freshly rotated token at incoming-token.
|
|
21
21
|
// verify health + preservation (Jellyfin, steward policy) + readback.
|
|
22
|
+
// upgrade in-place upgrade of an INSTALLED authority (D-043): same epoch,
|
|
23
|
+
// token and cursor; new package, pins and resident image. The new
|
|
24
|
+
// package's own lifecycle does the work and rolls back exactly on
|
|
25
|
+
// any failure. `--rehearse <step>` stops after that step and rolls
|
|
26
|
+
// back, proving the rollback on real hardware. No token rotation.
|
|
22
27
|
//
|
|
23
|
-
// Run `preflight` and `prepare` freely. `install` is the
|
|
24
|
-
//
|
|
28
|
+
// Run `preflight` and `prepare` freely. `install` is the first-install phase and
|
|
29
|
+
// refuses without a rotated incoming-token. Run `upgrade` detached so an SSH drop
|
|
30
|
+
// cannot interrupt it: setsid nohup node <this> upgrade <version> > /var/log/ouro-upgrade.log 2>&1 &
|
|
25
31
|
//
|
|
26
|
-
// Usage: sanctuary-butler-upgrade.mjs <preflight|prepare|install|verify> <version>
|
|
32
|
+
// Usage: sanctuary-butler-upgrade.mjs <preflight|prepare|install|verify|upgrade> <version> [--rehearse <step>]
|
|
27
33
|
|
|
28
34
|
import { execFileSync } from "node:child_process"
|
|
29
35
|
import { createHash } from "node:crypto"
|
|
@@ -47,6 +53,13 @@ const REQUIRED_PROGRAMS = [
|
|
|
47
53
|
"deploy/unraid/sanctuary-authority-service.sh",
|
|
48
54
|
]
|
|
49
55
|
const MIN_FREE_GB = 2
|
|
56
|
+
const INCOMING_MANIFEST = `${ROOT}/incoming-package-manifest.json`
|
|
57
|
+
const INCOMING_REQUEST = `${ROOT}/incoming-request.json`
|
|
58
|
+
const UPGRADE_JOURNAL = `${ROOT}/upgrade.json`
|
|
59
|
+
// Host supervision skips its work while this flag is fresh (< 30 min), so a crashed
|
|
60
|
+
// upgrade can never leave self-heal off for longer than that.
|
|
61
|
+
const MAINTENANCE = "/run/ouro-authority-maintenance"
|
|
62
|
+
const UPGRADE_STEPS = ["stop", "switch", "resident", "migrate", "start"]
|
|
50
63
|
|
|
51
64
|
const sh = (file, args, opts = {}) => execFileSync(file, args, { encoding: "utf8", maxBuffer: 64 << 20, ...opts })
|
|
52
65
|
const docker = (args, opts = {}) => sh("/usr/bin/docker", args, opts)
|
|
@@ -109,8 +122,11 @@ function preflight(version) {
|
|
|
109
122
|
} catch { bad(`the ${CONTAINER} container is absent`) }
|
|
110
123
|
if (id && runningImage && id === runningImage) bad("target and running image are identical — nothing to upgrade")
|
|
111
124
|
|
|
125
|
+
const installed = existsSync(`${ROOT}/active.json`)
|
|
126
|
+
if (installed) console.log(" note: an authority is installed, so this is an in-place upgrade (run `upgrade`, no token rotation)")
|
|
127
|
+
|
|
112
128
|
say("no stale authority runtime state")
|
|
113
|
-
{
|
|
129
|
+
if (!installed) {
|
|
114
130
|
const stale = ["/sys/fs/cgroup/ouro-authority", "/var/lib/ouro-authority", "/run/ouro-authority", `${ROOT}/epochs`].filter((d) => existsSync(d))
|
|
115
131
|
if (stale.length && !existsSync(`${ROOT}/active.json`)) console.log(` note: stale dirs present, install will clear them: ${stale.join(", ")}`)
|
|
116
132
|
else if (!stale.length) ok("no stale authority runtime dirs")
|
|
@@ -119,7 +135,8 @@ function preflight(version) {
|
|
|
119
135
|
|
|
120
136
|
say("no upgrade already in flight")
|
|
121
137
|
existsSync(JOURNAL) ? bad(`a template transaction journal is pending at ${JOURNAL}; resolve it first`) : ok("no pending template transaction")
|
|
122
|
-
existsSync(
|
|
138
|
+
if (installed) existsSync(UPGRADE_JOURNAL) ? console.log(" note: an in-place upgrade is pending; `upgrade` resumes it") : ok("no pending in-place upgrade")
|
|
139
|
+
if (installed && existsSync(`${ROOT}/incoming-token`)) console.log(" note: a stale incoming-token copy is present; the upgrade removes it (D-044)")
|
|
123
140
|
|
|
124
141
|
say("host primitives (root-owned, not group/world-writable)")
|
|
125
142
|
for (const f of PRIMITIVES) {
|
|
@@ -140,7 +157,8 @@ function preflight(version) {
|
|
|
140
157
|
} catch { bad("could not inspect the image contents") }
|
|
141
158
|
|
|
142
159
|
say("D-018: the fenced vault read the install depends on")
|
|
143
|
-
if (
|
|
160
|
+
if (installed) ok("not needed: the installed gateway already holds the token in root custody")
|
|
161
|
+
else if (id) {
|
|
144
162
|
// Run the read fenced exactly as THIS version's install will fence it, so
|
|
145
163
|
// the rehearsal is faithful: an unfixed image is tested without the fix and
|
|
146
164
|
// correctly fails here rather than during a real install.
|
|
@@ -169,7 +187,7 @@ function preflight(version) {
|
|
|
169
187
|
catch { bad("jellyfin container not inspectable") }
|
|
170
188
|
|
|
171
189
|
console.log("")
|
|
172
|
-
if (RED === 0) console.log("PREFLIGHT GREEN — the upgrade can proceed. Next: prepare, rotate the token, install.")
|
|
190
|
+
if (RED === 0) console.log(installed ? "PREFLIGHT GREEN — run `upgrade` (detached)." : "PREFLIGHT GREEN — the upgrade can proceed. Next: prepare, rotate the token, install.")
|
|
173
191
|
else console.log(`PREFLIGHT RED — ${RED} blocker(s) above. Nothing was changed; fix these before rotating the token.`)
|
|
174
192
|
process.exit(RED === 0 ? 0 : 1)
|
|
175
193
|
}
|
|
@@ -215,7 +233,41 @@ function prepare(version) {
|
|
|
215
233
|
say(`prepare inputs for ${version}`)
|
|
216
234
|
const id = imageId(version)
|
|
217
235
|
if (!id) fail(`image ${image(version)} is not present; pull it first`)
|
|
236
|
+
if (existsSync(`${ROOT}/active.json`)) fail("an authority is installed; use `upgrade` (in place, no token rotation) instead of prepare/install")
|
|
218
237
|
const epochId = `${version.replace(/[^A-Za-z0-9_-]/g, "-")}-${new Date().toISOString().slice(0, 10).replace(/-/g, "")}`
|
|
238
|
+
const manifestBytes = stageIncomingPackage(version)
|
|
239
|
+
writePrivate(`${ROOT}/package-manifest.json`, manifestBytes)
|
|
240
|
+
const creds = JSON.parse(readFileSync(`/mnt/user/appdata/ouro-butler/runtime/container-credentials.json`, "utf8")).credentials[0].runtimeConfig
|
|
241
|
+
const request = {
|
|
242
|
+
schemaVersion: 1, epochId,
|
|
243
|
+
botId: String(creds.telegramBotToken).split(":")[0],
|
|
244
|
+
ownerUserId: String(creds.telegramAuthorizedUserId), ownerChatId: String(creds.telegramAuthorizedChatId),
|
|
245
|
+
...packagePins(manifestBytes),
|
|
246
|
+
}
|
|
247
|
+
if (request.ownerUserId !== request.ownerChatId) fail("owner user and chat must match")
|
|
248
|
+
writePrivate(`${ROOT}/request.json`, Buffer.from(JSON.stringify(request)))
|
|
249
|
+
ok(`epoch ${epochId}, package ${request.packageDigest.slice(7, 19)}…`)
|
|
250
|
+
|
|
251
|
+
say("verify the prepared package against the installer's own rules")
|
|
252
|
+
verifyPackage(`${ROOT}/incoming-package`, `${ROOT}/package-manifest.json`, `${ROOT}/request.json`)
|
|
253
|
+
ok("package verifies")
|
|
254
|
+
|
|
255
|
+
console.log("\nPREPARE done. Next: rotate the bot token into ${ROOT}/incoming-token, then install.".replace("${ROOT}", ROOT))
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
// The package digest plus the host primitive pins every request carries.
|
|
259
|
+
function packagePins(manifestBytes) {
|
|
260
|
+
const d = (f) => digest(readFileSync(f))
|
|
261
|
+
return {
|
|
262
|
+
packageDigest: digest(manifestBytes),
|
|
263
|
+
nodeDigest: d("/usr/local/bin/node"), prlimitDigest: d("/usr/bin/prlimit"),
|
|
264
|
+
setsidDigest: d("/usr/bin/setsid"), shellDigest: d(sh("/bin/readlink", ["-f", "/bin/sh"]).trim()),
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
// Extract the exact package from the image into incoming-package, normalised to the
|
|
269
|
+
// installer's rules, and return its manifest bytes. The running Butler is untouched.
|
|
270
|
+
function stageIncomingPackage(version) {
|
|
219
271
|
const incoming = `${ROOT}/incoming-package`
|
|
220
272
|
|
|
221
273
|
say("extract the exact package from the image")
|
|
@@ -252,27 +304,8 @@ function prepare(version) {
|
|
|
252
304
|
}
|
|
253
305
|
}
|
|
254
306
|
walk("")
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
const creds = JSON.parse(readFileSync(`/mnt/user/appdata/ouro-butler/runtime/container-credentials.json`, "utf8")).credentials[0].runtimeConfig
|
|
258
|
-
const d = (f) => digest(readFileSync(f))
|
|
259
|
-
const request = {
|
|
260
|
-
schemaVersion: 1, epochId,
|
|
261
|
-
botId: String(creds.telegramBotToken).split(":")[0],
|
|
262
|
-
ownerUserId: String(creds.telegramAuthorizedUserId), ownerChatId: String(creds.telegramAuthorizedChatId),
|
|
263
|
-
packageDigest: digest(manifestBytes),
|
|
264
|
-
nodeDigest: d("/usr/local/bin/node"), prlimitDigest: d("/usr/bin/prlimit"),
|
|
265
|
-
setsidDigest: d("/usr/bin/setsid"), shellDigest: d(sh("/bin/readlink", ["-f", "/bin/sh"]).trim()),
|
|
266
|
-
}
|
|
267
|
-
if (request.ownerUserId !== request.ownerChatId) fail("owner user and chat must match")
|
|
268
|
-
writePrivate(`${ROOT}/request.json`, Buffer.from(JSON.stringify(request)))
|
|
269
|
-
ok(`epoch ${epochId}, ${Object.keys(files).length} files pinned, package ${request.packageDigest.slice(7, 19)}…`)
|
|
270
|
-
|
|
271
|
-
say("verify the prepared package against the installer's own rules")
|
|
272
|
-
verifyPackage(incoming, `${ROOT}/package-manifest.json`, `${ROOT}/request.json`)
|
|
273
|
-
ok("package verifies")
|
|
274
|
-
|
|
275
|
-
console.log("\nPREPARE done. Next: rotate the bot token into ${ROOT}/incoming-token, then install.".replace("${ROOT}", ROOT))
|
|
307
|
+
ok(`${Object.keys(files).length} files pinned`)
|
|
308
|
+
return Buffer.from(JSON.stringify({ schemaVersion: 1, files }))
|
|
276
309
|
}
|
|
277
310
|
|
|
278
311
|
function writePrivate(path, bytes) { writeFileSync(path, bytes, { mode: 0o600 }); sh("/bin/chown", ["0:0", path]); chmodSync(path, 0o600) }
|
|
@@ -440,18 +473,23 @@ start_gw() {
|
|
|
440
473
|
/bin/sh /boot/config/custom/ouro-authority/start.sh --boot >> "$LOG" 2>&1 \
|
|
441
474
|
|| echo "$(date) keeper: authority boot failed; will retry" >> "$LOG"
|
|
442
475
|
}
|
|
476
|
+
# An in-place upgrade pauses us with a flag; a stale flag (> 30 min) is ignored.
|
|
477
|
+
maintenance() { [ -n "$(find /run/ouro-authority-maintenance -mmin -30 2>/dev/null)" ]; }
|
|
443
478
|
pause_reaper; ensure_cg; ensure_sock
|
|
444
479
|
last_res=0
|
|
445
480
|
while true; do
|
|
481
|
+
if maintenance; then sleep 15; continue; fi
|
|
446
482
|
pause_reaper
|
|
447
483
|
ensure_cg
|
|
448
484
|
ensure_sock
|
|
449
485
|
[ -n "$(gw_pid)" ] || { start_gw; sleep 8; }
|
|
450
486
|
rd=$(jq -rc .status "$R"/epochs/*/readiness.json 2>/dev/null)
|
|
451
487
|
st=$(docker inspect ouro-butler --format '{{.State.Health.Status}}' 2>/dev/null)
|
|
488
|
+
ss=$(docker inspect ouro-butler --format '{{.State.Status}}' 2>/dev/null)
|
|
452
489
|
now=$(date +%s)
|
|
453
|
-
|
|
454
|
-
|
|
490
|
+
# Unhealthy, or left stopped (an interrupted upgrade or restart): the gateway is ready, so bring it back.
|
|
491
|
+
if [ "$rd" = ready ] && { [ "$st" = unhealthy ] || [ "$ss" = exited ] || [ "$ss" = created ]; } && [ $((now - last_res)) -gt 90 ]; then
|
|
492
|
+
echo "$(date) keeper: reconnecting resident ($ss/$st)" >> "$LOG"
|
|
455
493
|
docker restart ouro-butler >/dev/null 2>&1
|
|
456
494
|
last_res=$now
|
|
457
495
|
fi
|
|
@@ -468,6 +506,8 @@ const GATEWAY_WATCHDOG_SH = String.raw`#!/bin/sh
|
|
|
468
506
|
set -u
|
|
469
507
|
# The stop hook sets this during shutdown; never resurrect the supervisor then.
|
|
470
508
|
[ -e /run/ouro-authority-shutdown ] && exit 0
|
|
509
|
+
# An in-place upgrade pauses supervision; a stale flag (> 30 min) is ignored.
|
|
510
|
+
[ -n "$(find /run/ouro-authority-maintenance -mmin -30 2>/dev/null)" ] && exit 0
|
|
471
511
|
SUP=/boot/config/custom/ouro-authority/gateway-supervisor.sh
|
|
472
512
|
LOG=/var/log/ouro-gateway.log
|
|
473
513
|
if [ ! -f "$SUP" ]; then
|
|
@@ -737,6 +777,83 @@ function buildFinalProof(version, out) {
|
|
|
737
777
|
writeFileSync(out, JSON.stringify(proof), { mode: 0o600 })
|
|
738
778
|
}
|
|
739
779
|
|
|
780
|
+
// ---- upgrade (in place, installed authority) --------------------------------
|
|
781
|
+
|
|
782
|
+
function pauseSupervision() {
|
|
783
|
+
writeFileSync(MAINTENANCE, `${new Date().toISOString()} ${process.pid}\n`)
|
|
784
|
+
sh("/bin/sh", ["-c", "for p in $(pgrep -f '^/bin/sh /boot/config/custom/ouro-authority/[g]ateway-supervisor' || true); do kill $p; done"])
|
|
785
|
+
ok("host supervision paused (its flag expires on its own after 30 min)")
|
|
786
|
+
}
|
|
787
|
+
function resumeSupervision() {
|
|
788
|
+
rmSync(MAINTENANCE, { force: true })
|
|
789
|
+
installGatewaySupervisor()
|
|
790
|
+
}
|
|
791
|
+
|
|
792
|
+
// Keep Unraid's DockerMan template (what the UI's Apply/Update uses) on the live image.
|
|
793
|
+
function pinTemplate(version) {
|
|
794
|
+
if (!existsSync(TEMPLATE)) { console.log(" note: no DockerMan template to update"); return }
|
|
795
|
+
const text = readFileSync(TEMPLATE, "utf8")
|
|
796
|
+
const next = text.replace(/<Repository>ghcr\.io\/ourostack\/ouroboros-butler:[^<]+<\/Repository>/u, `<Repository>${image(version)}</Repository>`)
|
|
797
|
+
if (!next.includes(`<Repository>${image(version)}</Repository>`)) { console.log(" WARN: DockerMan template Repository not recognised; left unchanged"); return }
|
|
798
|
+
if (next !== text) { writeFileSync(`${TEMPLATE}.prev`, text); writeFileSync(TEMPLATE, next) }
|
|
799
|
+
ok(`DockerMan template pins ${image(version)}`)
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
function lifecycleFailure(e) {
|
|
803
|
+
const detail = [e.stderr, e.stdout].map((x) => (x ? String(x).trim() : "")).filter(Boolean).join(" | ")
|
|
804
|
+
let last = ""
|
|
805
|
+
try { last = readFileSync(`${ROOT}/lifecycle-failure.log`, "utf8").trim().split("\n").filter((l) => /^\d{4}-/.test(l)).at(-1) ?? "" } catch { /* none */ }
|
|
806
|
+
return `${detail || e.message}${last ? `\n last root failure: ${last}` : ""}`
|
|
807
|
+
}
|
|
808
|
+
|
|
809
|
+
function upgrade(version, rehearse) {
|
|
810
|
+
say(`in-place upgrade to ${version}${rehearse ? ` — REHEARSAL: stop after "${rehearse}", then roll back` : ""}`)
|
|
811
|
+
if (rehearse && !UPGRADE_STEPS.includes(rehearse)) fail(`--rehearse takes one of: ${UPGRADE_STEPS.join(", ")}`)
|
|
812
|
+
if (!existsSync(`${ROOT}/active.json`) || !existsSync(`${ROOT}/activation.json`)) fail("no installed authority; use prepare + install")
|
|
813
|
+
if (existsSync(JOURNAL)) fail("a template transaction is pending; resolve it first")
|
|
814
|
+
let id = imageId(version)
|
|
815
|
+
if (!id) { docker(["pull", image(version)]); id = imageId(version) }
|
|
816
|
+
if (!id) fail(`image ${image(version)} could not be pulled`)
|
|
817
|
+
ok(`image ${id.slice(0, 19)}…`)
|
|
818
|
+
if (existsSync(UPGRADE_JOURNAL)) console.log(" note: resuming the pending in-place upgrade with its staged package")
|
|
819
|
+
else {
|
|
820
|
+
const current = JSON.parse(readFileSync(`${ROOT}/request.json`, "utf8"))
|
|
821
|
+
const manifestBytes = stageIncomingPackage(version)
|
|
822
|
+
const pkgVersion = JSON.parse(readFileSync(`${ROOT}/incoming-package/deploy/unraid/sanctuary.ouro/bundle-meta.json`, "utf8")).runtimeVersion
|
|
823
|
+
if (pkgVersion !== version) fail(`staged package is ${pkgVersion}, not ${version}`)
|
|
824
|
+
writePrivate(INCOMING_MANIFEST, manifestBytes)
|
|
825
|
+
const request = { schemaVersion: 1, epochId: current.epochId, botId: current.botId, ownerUserId: current.ownerUserId, ownerChatId: current.ownerChatId, ...packagePins(manifestBytes) }
|
|
826
|
+
writePrivate(INCOMING_REQUEST, Buffer.from(JSON.stringify(request)))
|
|
827
|
+
verifyPackage(`${ROOT}/incoming-package`, INCOMING_MANIFEST, INCOMING_REQUEST)
|
|
828
|
+
ok("new package staged beside the live one and verified")
|
|
829
|
+
}
|
|
830
|
+
const lifecycle = `${ROOT}/incoming-package/dist/heart/daemon/sanctuary-authority-root-lifecycle.js`
|
|
831
|
+
const jellyfinBefore = docker(["inspect", "jellyfin", "--format", "{{.Id}}|{{.Image}}|{{.RestartCount}}|{{.State.StartedAt}}"]).trim()
|
|
832
|
+
const policyBefore = existsSync(POLICY) ? sha12(POLICY) : fail("steward policy missing")
|
|
833
|
+
pauseSupervision()
|
|
834
|
+
let failure = null
|
|
835
|
+
try {
|
|
836
|
+
say("lifecycle upgrade (stop → switch → resident → migrate → start)")
|
|
837
|
+
ok(sh("/usr/local/bin/node", [lifecycle, "upgrade", id, image(version), ...(rehearse ? ["--fail-after", rehearse] : [])], { stdio: ["ignore", "pipe", "pipe"] }).trim())
|
|
838
|
+
} catch (e) { failure = e }
|
|
839
|
+
if (failure) {
|
|
840
|
+
console.log(`\n!! ${rehearse ? "rehearsal stopped as planned" : "upgrade failed"}: ${lifecycleFailure(failure)}\n!! rolling back to the predecessor`)
|
|
841
|
+
try { ok(`rollback: ${sh("/usr/local/bin/node", [lifecycle, "upgrade-rollback"], { stdio: ["ignore", "pipe", "pipe"] }).trim()}`) }
|
|
842
|
+
catch (e) {
|
|
843
|
+
resumeSupervision()
|
|
844
|
+
fail(`ROLLBACK FAILED: ${lifecycleFailure(e)}\nThe journal is kept; the next authority boot (or \`upgrade-rollback\`) retries it.`)
|
|
845
|
+
}
|
|
846
|
+
} else pinTemplate(version)
|
|
847
|
+
resumeSupervision()
|
|
848
|
+
say("preservation")
|
|
849
|
+
docker(["inspect", "jellyfin", "--format", "{{.Id}}|{{.Image}}|{{.RestartCount}}|{{.State.StartedAt}}"]).trim() === jellyfinBefore ? ok("jellyfin unchanged") : fail("JELLYFIN CHANGED")
|
|
850
|
+
sha12(POLICY) === policyBefore ? ok("steward policy unchanged") : fail("STEWARD POLICY CHANGED")
|
|
851
|
+
const live = docker(["inspect", CONTAINER, "--format", "{{.Config.Image}} {{.State.Status}}/{{.State.Health.Status}}"]).trim()
|
|
852
|
+
ok(`butler: ${live}`)
|
|
853
|
+
if (failure && !rehearse) fail("upgrade rolled back; the Butler is on its prior version (see above)")
|
|
854
|
+
console.log(rehearse ? "\nREHEARSAL done — the rollback restored the predecessor." : `\nUPGRADE done — ${version} live. Run verify.`)
|
|
855
|
+
}
|
|
856
|
+
|
|
740
857
|
// ---- verify ---------------------------------------------------------------
|
|
741
858
|
|
|
742
859
|
function verify() {
|
|
@@ -755,9 +872,9 @@ function fail(m) { console.error(`\nREFUSING: ${m}`); process.exit(1) }
|
|
|
755
872
|
|
|
756
873
|
// ---- entry ----------------------------------------------------------------
|
|
757
874
|
|
|
758
|
-
const [phase, version] = process.argv.slice(2)
|
|
759
|
-
if (!phase || !["preflight", "prepare", "install", "verify"].includes(phase)) {
|
|
760
|
-
console.error("Usage: sanctuary-butler-upgrade.mjs <preflight|prepare|install|verify> <version>")
|
|
875
|
+
const [phase, version, flag, rehearse] = process.argv.slice(2)
|
|
876
|
+
if (!phase || !["preflight", "prepare", "install", "verify", "upgrade"].includes(phase) || (flag !== undefined && !(phase === "upgrade" && flag === "--rehearse" && rehearse))) {
|
|
877
|
+
console.error("Usage: sanctuary-butler-upgrade.mjs <preflight|prepare|install|verify|upgrade> <version> [--rehearse <step>]")
|
|
761
878
|
process.exit(2)
|
|
762
879
|
}
|
|
763
880
|
if (phase !== "verify" && !/^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$/.test(version || "")) {
|
|
@@ -770,3 +887,4 @@ if (phase === "preflight") preflight(version)
|
|
|
770
887
|
else if (phase === "prepare") prepare(version)
|
|
771
888
|
else if (phase === "install") install(version)
|
|
772
889
|
else if (phase === "verify") verify()
|
|
890
|
+
else if (phase === "upgrade") upgrade(version, rehearse)
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
<?xml version="1.0"?>
|
|
2
2
|
<Container version="2">
|
|
3
3
|
<Name>ouro-butler</Name>
|
|
4
|
-
<Repository>ghcr.io/ourostack/ouroboros-butler:0.1.0-alpha.
|
|
4
|
+
<Repository>ghcr.io/ourostack/ouroboros-butler:0.1.0-alpha.835</Repository>
|
|
5
5
|
<Registry>https://github.com/ourostack/ouroboros/pkgs/container/ouroboros-butler</Registry>
|
|
6
6
|
<Network>host</Network>
|
|
7
7
|
<Shell>sh</Shell>
|
|
@@ -36,6 +36,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
36
36
|
exports.readSanctuaryAuthorityEpoch = readSanctuaryAuthorityEpoch;
|
|
37
37
|
exports.prepareSanctuaryAuthorityEpoch = prepareSanctuaryAuthorityEpoch;
|
|
38
38
|
exports.releaseSanctuaryAuthorityToken = releaseSanctuaryAuthorityToken;
|
|
39
|
+
exports.rebindSanctuaryAuthorityEpochPackage = rebindSanctuaryAuthorityEpochPackage;
|
|
39
40
|
exports.retireSanctuaryAuthorityEpoch = retireSanctuaryAuthorityEpoch;
|
|
40
41
|
const node_crypto_1 = require("node:crypto");
|
|
41
42
|
const fs = __importStar(require("node:fs"));
|
|
@@ -238,6 +239,30 @@ function releaseSanctuaryAuthorityToken(root, options) {
|
|
|
238
239
|
fs.unlinkSync(epoch.tokenPath);
|
|
239
240
|
syncRoot(root);
|
|
240
241
|
}
|
|
242
|
+
/**
|
|
243
|
+
* Move a live epoch to a new reviewed package without replacing its token, issuer
|
|
244
|
+
* or cursor. An in-place upgrade (and its rollback) rebinds the package the epoch
|
|
245
|
+
* trusts; the gateway still refuses any config whose package differs from the epoch's.
|
|
246
|
+
*/
|
|
247
|
+
function rebindSanctuaryAuthorityEpochPackage(root, options) {
|
|
248
|
+
if (!DIGEST.test(options.from) || !DIGEST.test(options.to))
|
|
249
|
+
throw new Error("Sanctuary authority package rebind is invalid");
|
|
250
|
+
const epochPath = path.join(root, "epoch.json");
|
|
251
|
+
return (0, session_transaction_1.withImmediateSessionTurnLease)(epochPath, (lease) => {
|
|
252
|
+
const transaction = (0, session_transaction_1.readSessionTransaction)(epochPath, lease);
|
|
253
|
+
const epoch = readSanctuaryAuthorityEpoch(root, options);
|
|
254
|
+
if (epoch.state !== "prepared")
|
|
255
|
+
throw new Error("Sanctuary authority epoch is retired");
|
|
256
|
+
if (epoch.packageDigest === options.to)
|
|
257
|
+
return epoch;
|
|
258
|
+
if (epoch.packageDigest !== options.from)
|
|
259
|
+
throw new Error("Sanctuary authority epoch package changed");
|
|
260
|
+
const rebound = { ...epoch, packageDigest: options.to };
|
|
261
|
+
(0, session_transaction_1.writeSessionTransaction)(epochPath, rebound, { lease, expectedRevision: transaction.revision });
|
|
262
|
+
(0, runtime_1.emitNervesEvent)({ component: "daemon", event: "daemon.sanctuary_authority_epoch_package_rebound", message: "Sanctuary authority epoch rebound to a reviewed package", meta: { epochId: epoch.epochId, from: options.from, to: options.to } });
|
|
263
|
+
return rebound;
|
|
264
|
+
});
|
|
265
|
+
}
|
|
241
266
|
function retireSanctuaryAuthorityEpoch(root, options) {
|
|
242
267
|
if (options.quiescent !== true)
|
|
243
268
|
throw new Error("Sanctuary authority execution state is not quiescent");
|
|
@@ -32,8 +32,10 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
32
32
|
return result;
|
|
33
33
|
};
|
|
34
34
|
})();
|
|
35
|
+
var _a;
|
|
35
36
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
-
exports.SanctuaryAuthorityRootLifecycle = void 0;
|
|
37
|
+
exports.SanctuaryAuthorityRootLifecycle = exports.SANCTUARY_UPGRADE_STEPS = void 0;
|
|
38
|
+
exports.recordSanctuaryRootLifecycleFailure = recordSanctuaryRootLifecycleFailure;
|
|
37
39
|
exports.runSanctuaryAuthorityRootCli = runSanctuaryAuthorityRootCli;
|
|
38
40
|
const node_child_process_1 = require("node:child_process");
|
|
39
41
|
const node_crypto_1 = require("node:crypto");
|
|
@@ -57,7 +59,32 @@ const CGROUP = "/sys/fs/cgroup/ouro-authority";
|
|
|
57
59
|
const BOOT = "/boot/config/custom/ouro-authority/start.sh";
|
|
58
60
|
const BOOT_LINE = `/bin/sh ${BOOT} --boot & # ouro-authority`;
|
|
59
61
|
const DIGEST = /^sha256:[a-f0-9]{64}$/u;
|
|
62
|
+
const TEMPLATE_JOURNAL = "/boot/config/custom/ouro-butler/docker-man-template-transaction.json";
|
|
63
|
+
// In-place upgrade of an installed authority: same epoch (token, issuer, cursor,
|
|
64
|
+
// gateway state), new reviewed package and resident image. Every root record it
|
|
65
|
+
// rewrites is backed up first so an interrupted or failed upgrade rolls back exactly.
|
|
66
|
+
const UPGRADE = `${ROOT}/upgrade.json`;
|
|
67
|
+
const PREVIOUS = `${ROOT}/upgrade-previous`;
|
|
68
|
+
const NEXT_PACKAGE = `${ROOT}/package-next`;
|
|
69
|
+
const INCOMING_REQUEST = `${ROOT}/incoming-request.json`;
|
|
70
|
+
const INCOMING_MANIFEST = `${ROOT}/incoming-package-manifest.json`;
|
|
71
|
+
const EVENTS = "/boot/config/custom/ouro-events/spool";
|
|
72
|
+
const ICON = "https://raw.githubusercontent.com/ourostack/ouroboros/main/assets/ouroboros.png";
|
|
73
|
+
const IMAGE_REFERENCE = /^ghcr\.io\/ourostack\/ouroboros-butler:[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?$/u;
|
|
74
|
+
exports.SANCTUARY_UPGRADE_STEPS = ["stop", "switch", "resident", "migrate", "start"];
|
|
75
|
+
const ROOT_RECORDS = [["request.json", 1024 * 1024], ["package-manifest.json", 8 * 1024 * 1024], ["active.json", 1024 * 1024], ["activation.json", 1024 * 1024]];
|
|
76
|
+
const EPOCH_RECORDS = ["migration.json", "stage.json", "configured.json"];
|
|
60
77
|
const digest = (value) => `sha256:${(0, node_crypto_1.createHash)("sha256").update(value).digest("hex")}`;
|
|
78
|
+
function validateRequest(value) {
|
|
79
|
+
const request = value;
|
|
80
|
+
if (!request || typeof request !== "object" || Object.keys(request).sort().join(",") !== "botId,epochId,nodeDigest,ownerChatId,ownerUserId,packageDigest,prlimitDigest,schemaVersion,setsidDigest,shellDigest"
|
|
81
|
+
|| request.schemaVersion !== 1 || !/^[A-Za-z0-9_-]{1,128}$/u.test(request.epochId)
|
|
82
|
+
|| ![request.botId, request.ownerUserId, request.ownerChatId].every((id) => typeof id === "string" && /^[1-9][0-9]*$/u.test(id))
|
|
83
|
+
|| request.ownerUserId !== request.ownerChatId
|
|
84
|
+
|| ![request.packageDigest, request.nodeDigest, request.prlimitDigest, request.setsidDigest, request.shellDigest].every((pin) => typeof pin === "string" && DIGEST.test(pin)))
|
|
85
|
+
throw new Error("Sanctuary root lifecycle request is invalid");
|
|
86
|
+
return request;
|
|
87
|
+
}
|
|
61
88
|
// The prefix is a filesystem-fixture seam; the production CLI never accepts it.
|
|
62
89
|
class SanctuaryAuthorityRootLifecycle {
|
|
63
90
|
#request;
|
|
@@ -66,7 +93,9 @@ class SanctuaryAuthorityRootLifecycle {
|
|
|
66
93
|
#uid;
|
|
67
94
|
#gid;
|
|
68
95
|
#socketGid;
|
|
96
|
+
#options;
|
|
69
97
|
constructor(transaction, options = {}) {
|
|
98
|
+
this.#options = options;
|
|
70
99
|
this.#prefix = options.prefix ?? "";
|
|
71
100
|
this.#uid = options.expectedUid ?? 0;
|
|
72
101
|
this.#gid = options.expectedGid ?? 0;
|
|
@@ -77,15 +106,8 @@ class SanctuaryAuthorityRootLifecycle {
|
|
|
77
106
|
throw new Error("Sanctuary root lifecycle image identity is invalid");
|
|
78
107
|
this.#transaction = { targetImageId: transaction.targetImageId, rollbackImageId: transaction.rollbackImageId };
|
|
79
108
|
this.#directory(ROOT);
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|| request.schemaVersion !== 1 || !/^[A-Za-z0-9_-]{1,128}$/u.test(request.epochId)
|
|
83
|
-
|| ![request.botId, request.ownerUserId, request.ownerChatId].every((id) => typeof id === "string" && /^[1-9][0-9]*$/u.test(id))
|
|
84
|
-
|| request.ownerUserId !== request.ownerChatId
|
|
85
|
-
|| ![request.packageDigest, request.nodeDigest, request.prlimitDigest, request.setsidDigest, request.shellDigest].every((pin) => typeof pin === "string" && DIGEST.test(pin)))
|
|
86
|
-
throw new Error("Sanctuary root lifecycle request is invalid");
|
|
87
|
-
this.#request = request;
|
|
88
|
-
(0, runtime_1.emitNervesEvent)({ component: "daemon", event: "daemon.sanctuary_root_lifecycle_loaded", message: "Sanctuary root lifecycle identity loaded", meta: { epochId: request.epochId, targetImageId: transaction.targetImageId } });
|
|
109
|
+
this.#request = validateRequest(JSON.parse(this.#private(`${ROOT}/request.json`)));
|
|
110
|
+
(0, runtime_1.emitNervesEvent)({ component: "daemon", event: "daemon.sanctuary_root_lifecycle_loaded", message: "Sanctuary root lifecycle identity loaded", meta: { epochId: this.#request.epochId, targetImageId: transaction.targetImageId } });
|
|
89
111
|
}
|
|
90
112
|
plan() {
|
|
91
113
|
const { epochId, botId, ownerUserId, ownerChatId, packageDigest } = this.#request;
|
|
@@ -94,6 +116,10 @@ class SanctuaryAuthorityRootLifecycle {
|
|
|
94
116
|
async boot() {
|
|
95
117
|
if (!fs.existsSync(this.#p(`${ROOT}/activation.json`)))
|
|
96
118
|
return false;
|
|
119
|
+
// An upgrade that never finished (process killed, host rebooted) is rolled back
|
|
120
|
+
// here, so the Butler comes back on its known-good version without a human.
|
|
121
|
+
if (fs.existsSync(this.#p(UPGRADE)))
|
|
122
|
+
return this.rollbackUpgrade();
|
|
97
123
|
this.#record(`${ROOT}/activation.json`, { ...this.#transaction, state: "active", epochId: this.#request.epochId });
|
|
98
124
|
if (fs.existsSync(this.#p("/boot/config/custom/ouro-butler/docker-man-template-transaction.json")))
|
|
99
125
|
throw new Error("Sanctuary pending installation requires recovery, not boot");
|
|
@@ -234,20 +260,20 @@ class SanctuaryAuthorityRootLifecycle {
|
|
|
234
260
|
if (!stat.isDirectory() || stat.uid !== this.#uid || stat.gid !== gid || (stat.mode & 0o7777) !== mode || fs.realpathSync(directory) !== directory)
|
|
235
261
|
throw new Error("Sanctuary root lifecycle directory is unsafe");
|
|
236
262
|
}
|
|
237
|
-
#verifyPackage(packageRoot = `${ROOT}/package`, hostExecution = true) {
|
|
263
|
+
#verifyPackage(packageRoot = `${ROOT}/package`, hostExecution = true, manifestPath = `${ROOT}/package-manifest.json`, request = this.#request) {
|
|
238
264
|
(0, sanctuary_authority_installation_1.verifySanctuaryAuthorityInstallation)({
|
|
239
|
-
packageRoot: this.#p(packageRoot), manifestPath: this.#p(
|
|
265
|
+
packageRoot: this.#p(packageRoot), manifestPath: this.#p(manifestPath), manifestDigest: request.packageDigest,
|
|
240
266
|
stateRoot: this.#p(this.#epochRoot()), stagingRoot: this.#p(STAGING), socketRoot: this.#p(SOCKET), cgroupRoot: this.#p(CGROUP),
|
|
241
267
|
expectedUid: this.#uid, expectedGid: this.#gid, socketGroupId: this.#socketGid, mountInfo: fs.readFileSync(this.#p("/proc/self/mountinfo"), "utf8"),
|
|
242
268
|
});
|
|
243
|
-
const manifest = JSON.parse(this.#private(
|
|
269
|
+
const manifest = JSON.parse(this.#private(manifestPath, 0o600, 8 * 1024 * 1024));
|
|
244
270
|
for (const program of ["dist/heart/daemon/sanctuary-telegram-authority-entry.js", "dist/heart/daemon/sanctuary-authority-root-lifecycle.js", "dist/heart/daemon/sanctuary-host-supervisor-entry.js", "deploy/unraid/sanctuary-host-launcher.sh", "deploy/unraid/sanctuary-authority-service.sh"]) {
|
|
245
271
|
if (!Object.hasOwn(manifest.files, program))
|
|
246
272
|
throw new Error("Sanctuary authority package program is absent");
|
|
247
273
|
}
|
|
248
274
|
for (const [file, expected] of [
|
|
249
|
-
["/usr/local/bin/node",
|
|
250
|
-
...(hostExecution ? [["/usr/bin/prlimit",
|
|
275
|
+
["/usr/local/bin/node", request.nodeDigest], ["/bin/sh", request.shellDigest],
|
|
276
|
+
...(hostExecution ? [["/usr/bin/prlimit", request.prlimitDigest], ["/usr/bin/setsid", request.setsidDigest]] : []),
|
|
251
277
|
]) {
|
|
252
278
|
this.#verifyPrimitive(file, expected);
|
|
253
279
|
}
|
|
@@ -626,6 +652,215 @@ class SanctuaryAuthorityRootLifecycle {
|
|
|
626
652
|
throw new Error("Sanctuary rollback readback failed");
|
|
627
653
|
this.#write(`${this.#epochRoot()}/rollback.json`, JSON.stringify({ epochId: this.#request.epochId, state: "retired" }));
|
|
628
654
|
}
|
|
655
|
+
/**
|
|
656
|
+
* Upgrade the installed authority in place to a reviewed package and resident image.
|
|
657
|
+
* The epoch (token, issuer, cursor, gateway state) is kept; only the package, its
|
|
658
|
+
* pins and the resident image change. Resumable from its journal; any failure is
|
|
659
|
+
* left for `rollbackUpgrade` (or the next boot) to undo exactly.
|
|
660
|
+
*/
|
|
661
|
+
async upgrade(input) {
|
|
662
|
+
if (!DIGEST.test(input.targetImageId) || !IMAGE_REFERENCE.test(input.imageReference)
|
|
663
|
+
|| (input.failAfter !== undefined && !exports.SANCTUARY_UPGRADE_STEPS.includes(input.failAfter)))
|
|
664
|
+
throw new Error("Sanctuary upgrade request is invalid");
|
|
665
|
+
if (fs.existsSync(this.#p(TEMPLATE_JOURNAL)))
|
|
666
|
+
throw new Error("Finish the pending installation before upgrading");
|
|
667
|
+
let journal = fs.existsSync(this.#p(UPGRADE)) ? this.#upgradeJournal() : this.#beginUpgrade(input);
|
|
668
|
+
if (journal.to.imageId !== input.targetImageId || journal.to.imageReference !== input.imageReference)
|
|
669
|
+
throw new Error("A different Sanctuary upgrade is pending; roll it back first");
|
|
670
|
+
const target = () => new _a({ targetImageId: journal.to.imageId, rollbackImageId: journal.from.imageId }, this.#options);
|
|
671
|
+
const steps = {
|
|
672
|
+
stop: () => this.#halt(),
|
|
673
|
+
switch: async () => { await this.#halt(); this.#switchPackage(journal); },
|
|
674
|
+
resident: () => this.#recreateResident(journal.to.imageReference, journal.to.imageId),
|
|
675
|
+
migrate: () => target().#bundle(["--operation", "migrate", "--rollback-image-id", journal.from.imageId, "--target-image-id", journal.to.imageId]),
|
|
676
|
+
start: () => target().#startUpgraded(),
|
|
677
|
+
};
|
|
678
|
+
for (const step of exports.SANCTUARY_UPGRADE_STEPS) {
|
|
679
|
+
if (!journal.completed.includes(step)) {
|
|
680
|
+
await steps[step]();
|
|
681
|
+
journal = { ...journal, completed: [...journal.completed, step] };
|
|
682
|
+
this.#write(UPGRADE, JSON.stringify(journal));
|
|
683
|
+
(0, runtime_1.emitNervesEvent)({ component: "daemon", event: "daemon.sanctuary_upgrade_step", message: "Sanctuary upgrade step completed", meta: { step, to: journal.to.imageReference } });
|
|
684
|
+
}
|
|
685
|
+
if (input.failAfter === step)
|
|
686
|
+
throw new Error(`Sanctuary upgrade rehearsal stopped after ${step}`);
|
|
687
|
+
}
|
|
688
|
+
target().#bundle(["--operation", "commit"]);
|
|
689
|
+
// A copy of the live token must not linger as if it were a fresh rotation (D-044).
|
|
690
|
+
this.#remove(`${ROOT}/incoming-token`);
|
|
691
|
+
this.#remove(UPGRADE);
|
|
692
|
+
this.#removeTree(PREVIOUS);
|
|
693
|
+
(0, runtime_1.emitNervesEvent)({ component: "daemon", event: "daemon.sanctuary_upgraded", message: "Sanctuary authority upgraded in place", meta: { from: journal.from.imageReference, to: journal.to.imageReference } });
|
|
694
|
+
}
|
|
695
|
+
/** Undo a pending in-place upgrade exactly, back to the recorded predecessor. */
|
|
696
|
+
async rollbackUpgrade() {
|
|
697
|
+
if (!fs.existsSync(this.#p(UPGRADE)))
|
|
698
|
+
return false;
|
|
699
|
+
let journal = this.#upgradeJournal();
|
|
700
|
+
await this.#halt();
|
|
701
|
+
if (journal.completed.includes("switch")) {
|
|
702
|
+
// A migration interrupted before the journal recorded it may or may not be
|
|
703
|
+
// pending; only a recorded migration must roll back. Once the bundle is back,
|
|
704
|
+
// record that so a retried rollback does not demand it again.
|
|
705
|
+
try {
|
|
706
|
+
this.#bundle(["--operation", "finalize-rollback"]);
|
|
707
|
+
}
|
|
708
|
+
catch (error) {
|
|
709
|
+
if (journal.completed.includes("migrate"))
|
|
710
|
+
throw error;
|
|
711
|
+
}
|
|
712
|
+
journal = { ...journal, completed: journal.completed.filter((step) => step !== "migrate") };
|
|
713
|
+
this.#write(UPGRADE, JSON.stringify(journal));
|
|
714
|
+
}
|
|
715
|
+
if (fs.existsSync(this.#p(`${PREVIOUS}/package`))) {
|
|
716
|
+
this.#removeTree(`${ROOT}/package`);
|
|
717
|
+
fs.renameSync(this.#p(`${PREVIOUS}/package`), this.#p(`${ROOT}/package`));
|
|
718
|
+
}
|
|
719
|
+
this.#removeTree(NEXT_PACKAGE);
|
|
720
|
+
for (const [name, max] of ROOT_RECORDS)
|
|
721
|
+
this.#write(`${ROOT}/${name}`, this.#private(`${PREVIOUS}/${name}`, 0o600, max));
|
|
722
|
+
(0, sanctuary_authority_epoch_1.rebindSanctuaryAuthorityEpochPackage)(this.#p(this.#epochRoot()), { expectedUid: this.#uid, expectedGid: this.#gid, from: journal.to.packageDigest, to: journal.from.packageDigest });
|
|
723
|
+
for (const name of EPOCH_RECORDS)
|
|
724
|
+
this.#write(`${this.#epochRoot()}/${name}`, this.#private(`${PREVIOUS}/epoch/${name}`));
|
|
725
|
+
const predecessor = new _a({ targetImageId: journal.from.imageId, rollbackImageId: journal.from.rollbackImageId }, this.#options);
|
|
726
|
+
predecessor.#publishBoot();
|
|
727
|
+
if (this.#containers().find((container) => container.Name === "/ouro-butler")?.Image !== journal.from.imageId) {
|
|
728
|
+
predecessor.#recreateResident(journal.from.imageReference, journal.from.imageId);
|
|
729
|
+
}
|
|
730
|
+
await predecessor.#startUpgraded();
|
|
731
|
+
this.#remove(UPGRADE);
|
|
732
|
+
this.#removeTree(PREVIOUS);
|
|
733
|
+
(0, runtime_1.emitNervesEvent)({ level: "warn", component: "daemon", event: "daemon.sanctuary_upgrade_rolled_back", message: "Sanctuary upgrade rolled back to its predecessor", meta: { from: journal.to.imageReference, to: journal.from.imageReference } });
|
|
734
|
+
return true;
|
|
735
|
+
}
|
|
736
|
+
#upgradeJournal() {
|
|
737
|
+
const journal = JSON.parse(this.#private(UPGRADE));
|
|
738
|
+
const sides = [journal?.from, journal?.to];
|
|
739
|
+
if (!journal || journal.schemaVersion !== 1 || journal.epochId !== this.#request.epochId || !Array.isArray(journal.completed)
|
|
740
|
+
|| !journal.completed.every((step) => exports.SANCTUARY_UPGRADE_STEPS.includes(step))
|
|
741
|
+
|| !sides.every((side) => side && DIGEST.test(side.imageId) && DIGEST.test(side.packageDigest) && IMAGE_REFERENCE.test(side.imageReference))
|
|
742
|
+
|| !DIGEST.test(journal.from.rollbackImageId))
|
|
743
|
+
throw new Error("Sanctuary upgrade journal is invalid");
|
|
744
|
+
return journal;
|
|
745
|
+
}
|
|
746
|
+
#beginUpgrade(input) {
|
|
747
|
+
this.#record(`${ROOT}/activation.json`, { ...this.#transaction, state: "active", epochId: this.#request.epochId });
|
|
748
|
+
const current = this.#target();
|
|
749
|
+
if (input.targetImageId === this.#transaction.targetImageId)
|
|
750
|
+
throw new Error("Sanctuary already runs the requested image");
|
|
751
|
+
if (this.#docker(["image", "inspect", "--format", "{{.Id}}", input.imageReference]).trim() !== input.targetImageId)
|
|
752
|
+
throw new Error("Sanctuary upgrade image identity does not match its reference");
|
|
753
|
+
const request = validateRequest(JSON.parse(this.#private(INCOMING_REQUEST)));
|
|
754
|
+
if (["epochId", "botId", "ownerUserId", "ownerChatId"].some((key) => request[key] !== this.#request[key]))
|
|
755
|
+
throw new Error("Sanctuary upgrade would change the authority identity");
|
|
756
|
+
if (request.packageDigest === this.#request.packageDigest)
|
|
757
|
+
throw new Error("Sanctuary upgrade package is already installed");
|
|
758
|
+
this.#verifyPackage(`${ROOT}/incoming-package`, true, INCOMING_MANIFEST, request);
|
|
759
|
+
this.#removeTree(PREVIOUS);
|
|
760
|
+
this.#directory(PREVIOUS);
|
|
761
|
+
this.#directory(`${PREVIOUS}/epoch`);
|
|
762
|
+
for (const [name, max] of ROOT_RECORDS)
|
|
763
|
+
this.#write(`${PREVIOUS}/${name}`, this.#private(`${ROOT}/${name}`, 0o600, max));
|
|
764
|
+
for (const name of EPOCH_RECORDS)
|
|
765
|
+
this.#write(`${PREVIOUS}/epoch/${name}`, this.#private(`${this.#epochRoot()}/${name}`));
|
|
766
|
+
const journal = {
|
|
767
|
+
schemaVersion: 1, epochId: this.#request.epochId, completed: [],
|
|
768
|
+
from: { imageId: this.#transaction.targetImageId, imageReference: current.Config.Image, packageDigest: this.#request.packageDigest, rollbackImageId: this.#transaction.rollbackImageId },
|
|
769
|
+
to: { imageId: input.targetImageId, imageReference: input.imageReference, packageDigest: request.packageDigest },
|
|
770
|
+
};
|
|
771
|
+
if (!IMAGE_REFERENCE.test(journal.from.imageReference))
|
|
772
|
+
throw new Error("Sanctuary current resident image reference is not a reviewed release");
|
|
773
|
+
this.#write(UPGRADE, JSON.stringify(journal));
|
|
774
|
+
return journal;
|
|
775
|
+
}
|
|
776
|
+
/** Stop the resident, then the gateway, without retiring anything. */
|
|
777
|
+
async #halt() {
|
|
778
|
+
this.#assertNoForeignPoller();
|
|
779
|
+
for (const container of this.#containers())
|
|
780
|
+
if (container.State.Running)
|
|
781
|
+
this.#docker(["stop", "--time", "30", container.Name.slice(1)]);
|
|
782
|
+
if (this.#containers().some((container) => container.State.Running))
|
|
783
|
+
throw new Error("Sanctuary resident did not stop");
|
|
784
|
+
const pid = this.#pid();
|
|
785
|
+
if (pid !== null) {
|
|
786
|
+
process.kill(pid, "SIGTERM");
|
|
787
|
+
await this.#wait(() => this.#pid() === null);
|
|
788
|
+
}
|
|
789
|
+
this.#remove(`${this.#epochRoot()}/readiness.json`);
|
|
790
|
+
}
|
|
791
|
+
#switchPackage(journal) {
|
|
792
|
+
const manifestText = this.#private(INCOMING_MANIFEST, 0o600, 8 * 1024 * 1024);
|
|
793
|
+
if (!fs.existsSync(this.#p(`${PREVIOUS}/package`))) {
|
|
794
|
+
this.#populate(NEXT_PACKAGE, `${ROOT}/incoming-package`, JSON.parse(manifestText));
|
|
795
|
+
fs.renameSync(this.#p(`${ROOT}/package`), this.#p(`${PREVIOUS}/package`));
|
|
796
|
+
}
|
|
797
|
+
if (!fs.existsSync(this.#p(`${ROOT}/package`)))
|
|
798
|
+
fs.renameSync(this.#p(NEXT_PACKAGE), this.#p(`${ROOT}/package`));
|
|
799
|
+
this.#write(`${ROOT}/package-manifest.json`, manifestText);
|
|
800
|
+
this.#write(`${ROOT}/request.json`, this.#private(INCOMING_REQUEST));
|
|
801
|
+
(0, sanctuary_authority_epoch_1.rebindSanctuaryAuthorityEpochPackage)(this.#p(this.#epochRoot()), { expectedUid: this.#uid, expectedGid: this.#gid, from: journal.from.packageDigest, to: journal.to.packageDigest });
|
|
802
|
+
const target = new _a({ targetImageId: journal.to.imageId, rollbackImageId: journal.from.imageId }, this.#options);
|
|
803
|
+
target.#rebindRecords();
|
|
804
|
+
}
|
|
805
|
+
/** Copy a reviewed package into a fresh private tree, pin by pin. */
|
|
806
|
+
#populate(destinationRoot, sourceRoot, manifest) {
|
|
807
|
+
this.#removeTree(destinationRoot);
|
|
808
|
+
this.#directory(destinationRoot);
|
|
809
|
+
for (const [relative, pin] of Object.entries(manifest.files)) {
|
|
810
|
+
const destination = `${destinationRoot}/${relative}`;
|
|
811
|
+
this.#directory(path.dirname(destination));
|
|
812
|
+
fs.copyFileSync(this.#p(`${sourceRoot}/${relative}`), this.#p(destination), fs.constants.COPYFILE_EXCL);
|
|
813
|
+
fs.chmodSync(this.#p(destination), pin.mode);
|
|
814
|
+
if (digest(fs.readFileSync(this.#p(destination))) !== pin.digest)
|
|
815
|
+
throw new Error("Sanctuary upgrade package conflicts with its pin");
|
|
816
|
+
}
|
|
817
|
+
}
|
|
818
|
+
/** Point every epoch record at this lifecycle's (new) package and image, keeping the cursor. */
|
|
819
|
+
#rebindRecords() {
|
|
820
|
+
const migration = JSON.parse(this.#private(`${this.#epochRoot()}/migration.json`));
|
|
821
|
+
if (!Number.isSafeInteger(migration.predecessorCursor) || migration.predecessorCursor < 0)
|
|
822
|
+
throw new Error("Sanctuary root migration cursor is invalid");
|
|
823
|
+
this.#write(`${this.#epochRoot()}/migration.json`, JSON.stringify({ ...this.plan(), ...this.#transaction, predecessorCursor: migration.predecessorCursor }));
|
|
824
|
+
this.#write(`${this.#epochRoot()}/stage.json`, JSON.stringify({ packageDigest: this.#request.packageDigest }));
|
|
825
|
+
this.#write(`${this.#epochRoot()}/configured.json`, JSON.stringify({ epochId: this.#request.epochId, targetImageId: this.#transaction.targetImageId }));
|
|
826
|
+
this.#verifyPackage();
|
|
827
|
+
this.#publishBoot();
|
|
828
|
+
this.#write(`${ROOT}/active.json`, JSON.stringify(this.#configuration()));
|
|
829
|
+
this.#write(`${ROOT}/activation.json`, JSON.stringify({ ...this.#transaction, state: "active", epochId: this.#request.epochId }));
|
|
830
|
+
}
|
|
831
|
+
/** Point the boot service script at the installed package. On Unraid the host keeper
|
|
832
|
+
* (not a direct go line) invokes it, so the go file is left as the operator has it. */
|
|
833
|
+
#publishBoot() {
|
|
834
|
+
this.#write(BOOT, fs.readFileSync(this.#p(`${ROOT}/package/deploy/unraid/sanctuary-authority-service.sh`), "utf8"));
|
|
835
|
+
}
|
|
836
|
+
#recreateResident(reference, imageId) {
|
|
837
|
+
if (this.#containers().some((container) => container.Name === "/ouro-butler"))
|
|
838
|
+
this.#docker(["rm", "-f", "ouro-butler"]);
|
|
839
|
+
this.#docker(["create", "--name", "ouro-butler", "--network", "host", "--restart", "unless-stopped", "--user", "10001:10001",
|
|
840
|
+
"-l", "net.unraid.docker.managed=dockerman", "-l", `net.unraid.docker.icon=${ICON}`, "-l", "org.opencontainers.image.source=https://github.com/ourostack/ouroboros",
|
|
841
|
+
"-v", `${RUNTIME}:/home/ouro/.ouro-cli:rw`, "-v", `${BUNDLE}:/home/ouro/AgentBundles/sanctuary.ouro:rw`,
|
|
842
|
+
"-v", `${EVENTS}:/run/ouro-events:ro`, "-v", `${SOCKET}:${SOCKET}:ro`, reference]);
|
|
843
|
+
const resident = this.#containers().find((container) => container.Name === "/ouro-butler");
|
|
844
|
+
if (resident?.Image !== imageId || resident.State.Running)
|
|
845
|
+
throw new Error("Sanctuary resident recreation readback failed");
|
|
846
|
+
}
|
|
847
|
+
/** Run the installed package's bundle migration CLI, then return the bundle to the resident. */
|
|
848
|
+
#bundle(operation) {
|
|
849
|
+
(0, node_child_process_1.execFileSync)(this.#p("/usr/local/bin/node"), [this.#p(`${ROOT}/package/deploy/unraid/migrate-sanctuary-bundle.mjs`),
|
|
850
|
+
"--package-root", this.#p(`${ROOT}/package/deploy/unraid/sanctuary.ouro`), "--agent-root", this.#p(BUNDLE), ...operation], {
|
|
851
|
+
encoding: "utf8", timeout: 300_000, stdio: ["ignore", "pipe", "pipe"], env: { PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" }, cwd: "/",
|
|
852
|
+
});
|
|
853
|
+
(0, node_child_process_1.execFileSync)("/bin/chown", ["-R", "10001:10001", this.#p(BUNDLE)], { stdio: "ignore" });
|
|
854
|
+
}
|
|
855
|
+
/** Start the gateway, then the resident, and prove both. */
|
|
856
|
+
async #startUpgraded() {
|
|
857
|
+
await this.#startGateway();
|
|
858
|
+
this.#docker(["start", "ouro-butler"]);
|
|
859
|
+
await this.#wait(async () => this.#target().State.Running && this.#target().State.Health?.Status === "healthy" && await this.#ready(), 300_000);
|
|
860
|
+
}
|
|
861
|
+
#removeTree(absolute) {
|
|
862
|
+
fs.rmSync(this.#p(absolute), { recursive: true, force: true });
|
|
863
|
+
}
|
|
629
864
|
#remove(absolute) {
|
|
630
865
|
if (!fs.existsSync(this.#p(absolute)))
|
|
631
866
|
return;
|
|
@@ -772,6 +1007,21 @@ class SanctuaryAuthorityRootLifecycle {
|
|
|
772
1007
|
}
|
|
773
1008
|
}
|
|
774
1009
|
exports.SanctuaryAuthorityRootLifecycle = SanctuaryAuthorityRootLifecycle;
|
|
1010
|
+
_a = SanctuaryAuthorityRootLifecycle;
|
|
1011
|
+
/**
|
|
1012
|
+
* Keep the real failure where only root can read it (D-034). Stderr stays generic
|
|
1013
|
+
* because a parse error can quote private bytes; the authority root is as private
|
|
1014
|
+
* as the token it already holds.
|
|
1015
|
+
*/
|
|
1016
|
+
function recordSanctuaryRootLifecycleFailure(error, file = `${ROOT}/lifecycle-failure.log`) {
|
|
1017
|
+
try {
|
|
1018
|
+
fs.appendFileSync(file, `${new Date().toISOString()} ${error instanceof Error ? error.stack : String(error)}\n`, { mode: 0o600 });
|
|
1019
|
+
return ` Details (root-only): ${file}`;
|
|
1020
|
+
}
|
|
1021
|
+
catch {
|
|
1022
|
+
return "";
|
|
1023
|
+
}
|
|
1024
|
+
}
|
|
775
1025
|
async function runSanctuaryAuthorityRootCli(argv, write = (text) => process.stdout.write(text)) {
|
|
776
1026
|
if (process.getuid() !== 0 || process.getgid() !== 0)
|
|
777
1027
|
throw new Error("Sanctuary root lifecycle requires root");
|
|
@@ -792,12 +1042,15 @@ async function runSanctuaryAuthorityRootCli(argv, write = (text) => process.stdo
|
|
|
792
1042
|
return;
|
|
793
1043
|
}
|
|
794
1044
|
const repin = argv.length === 3 && argv[0] === "repin-execution";
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
1045
|
+
const upgrade = argv[0] === "upgrade" && (argv.length === 3 || (argv.length === 5 && argv[3] === "--fail-after"));
|
|
1046
|
+
const rollback = argv.length === 1 && argv[0] === "upgrade-rollback";
|
|
1047
|
+
if (!repin && !upgrade && !rollback && (argv.length !== 1 || argv[0] !== "boot"))
|
|
1048
|
+
throw new Error("Usage: sanctuary-authority-root-lifecycle <boot|repin-execution <prlimit-sha256> <setsid-sha256>|upgrade <target-image-id> <image-reference> [--fail-after <step>]|upgrade-rollback|vault snapshot|vault presence|vault remove|vault restore>");
|
|
1049
|
+
// Boot never waits: the host keeper retries it. An operator upgrade waits out a boot in flight.
|
|
1050
|
+
await (0, session_transaction_1.withSessionTurnLease)(TEMPLATE_JOURNAL, async () => {
|
|
798
1051
|
const activationPath = `${ROOT}/activation.json`;
|
|
799
1052
|
if (!fs.existsSync(activationPath)) {
|
|
800
|
-
if (repin)
|
|
1053
|
+
if (repin || upgrade || rollback)
|
|
801
1054
|
throw new Error("Sanctuary authority is not active");
|
|
802
1055
|
return;
|
|
803
1056
|
}
|
|
@@ -807,13 +1060,20 @@ async function runSanctuaryAuthorityRootCli(argv, write = (text) => process.stdo
|
|
|
807
1060
|
lifecycle.repinExecution(argv[1], argv[2]);
|
|
808
1061
|
write('{"repinned":true,"gatewayRestartRequired":true}\n');
|
|
809
1062
|
}
|
|
1063
|
+
else if (upgrade) {
|
|
1064
|
+
await lifecycle.upgrade({ targetImageId: argv[1], imageReference: argv[2], ...(argv[4] ? { failAfter: argv[4] } : {}) });
|
|
1065
|
+
write(`${JSON.stringify({ upgraded: argv[2] })}\n`);
|
|
1066
|
+
}
|
|
1067
|
+
else if (rollback) {
|
|
1068
|
+
write(`${JSON.stringify({ rolledBack: await lifecycle.rollbackUpgrade() })}\n`);
|
|
1069
|
+
}
|
|
810
1070
|
else
|
|
811
1071
|
await lifecycle.boot();
|
|
812
|
-
}, { timeoutMs: 0, confinementRoot: "/boot/config/custom/ouro-butler" });
|
|
1072
|
+
}, { timeoutMs: upgrade || rollback ? 600_000 : 0, confinementRoot: "/boot/config/custom/ouro-butler" });
|
|
813
1073
|
}
|
|
814
1074
|
if (process.argv[1] && fs.existsSync(process.argv[1]) && fs.realpathSync(process.argv[1]) === __filename) {
|
|
815
|
-
void runSanctuaryAuthorityRootCli(process.argv.slice(2)).catch(() => {
|
|
816
|
-
process.stderr.write(
|
|
1075
|
+
void runSanctuaryAuthorityRootCli(process.argv.slice(2)).catch((error) => {
|
|
1076
|
+
process.stderr.write(`Sanctuary root lifecycle failed; inspect the root transaction and repair its failed boundary.${recordSanctuaryRootLifecycleFailure(error)}\n`);
|
|
817
1077
|
process.exitCode = 1;
|
|
818
1078
|
});
|
|
819
1079
|
}
|
package/npm-shrinkwrap.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ouro.bot/cli",
|
|
3
|
-
"version": "0.1.0-alpha.
|
|
3
|
+
"version": "0.1.0-alpha.835",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "@ouro.bot/cli",
|
|
9
|
-
"version": "0.1.0-alpha.
|
|
9
|
+
"version": "0.1.0-alpha.835",
|
|
10
10
|
"dependencies": {
|
|
11
11
|
"@anthropic-ai/sdk": "^0.78.0",
|
|
12
12
|
"@azure/identity": "^4.13.0",
|