@ouro.bot/cli 0.1.0-alpha.831 → 0.1.0-alpha.833
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.json +12 -0
- package/deploy/unraid/sanctuary-butler-upgrade.mjs +772 -0
- package/deploy/unraid/sanctuary.ouro/bundle-meta.json +1 -1
- package/deploy/unraid/sanctuary.xml +1 -1
- package/dist/heart/daemon/sanctuary-telegram-authority-service.js +38 -5
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
package/changelog.json
CHANGED
|
@@ -1,6 +1,18 @@
|
|
|
1
1
|
{
|
|
2
2
|
"_note": "This changelog is maintained as part of the PR/version-bump workflow. Agent-curated, not auto-generated. Agents read this file directly via read_file to understand what changed between versions.",
|
|
3
3
|
"versions": [
|
|
4
|
+
{
|
|
5
|
+
"version": "0.1.0-alpha.833",
|
|
6
|
+
"changes": [
|
|
7
|
+
"Deploy: the automated Butler root-authority upgrade orchestrator (preflight/prepare/install/migrate/activate with robust auto-rollback) is committed as deploy/unraid/sanctuary-butler-upgrade.mjs. It installs host supervision that survives an Unraid reboot: the gateway is (re)started through the authority boot path (start.sh --boot), the authority socket dir Docker would create wrongly is created/repaired as 750 root:10001, a cron watchdog resurrects the supervisor, and a /boot/config/stop hook stops it all gracefully before the array stops."
|
|
8
|
+
]
|
|
9
|
+
},
|
|
10
|
+
{
|
|
11
|
+
"version": "0.1.0-alpha.832",
|
|
12
|
+
"changes": [
|
|
13
|
+
"Sanctuary authority gateway: a getUpdates 409 conflict (and transient 5xx) after a gateway restart is retried internally until the previous Telegram long-poll session expires, so the tokenless resident never sees the reclaim window."
|
|
14
|
+
]
|
|
15
|
+
},
|
|
4
16
|
{
|
|
5
17
|
"version": "0.1.0-alpha.831",
|
|
6
18
|
"changes": [
|
|
@@ -0,0 +1,772 @@
|
|
|
1
|
+
#!/usr/local/bin/node
|
|
2
|
+
// Sanctuary Butler automated update path.
|
|
3
|
+
//
|
|
4
|
+
// One tool for the whole upgrade, built after a manual upgrade attempt took the
|
|
5
|
+
// household's Telegram bot down twice: once from a hand-rolled migration, once
|
|
6
|
+
// from a shipped-install defect discovered only AFTER the bot token had been
|
|
7
|
+
// revoked. The lesson both times was the same — verify the end state, and prove
|
|
8
|
+
// the upgrade can succeed BEFORE doing anything irreversible.
|
|
9
|
+
//
|
|
10
|
+
// Phases (each idempotent, each stops on the first failure):
|
|
11
|
+
// preflight read-only rehearsal. Proves the install can succeed and touches
|
|
12
|
+
// nothing. Its centerpiece is the fenced vault-read check (D-018):
|
|
13
|
+
// the exact operation that failed the last upgrade, run before the
|
|
14
|
+
// token is ever rotated.
|
|
15
|
+
// prepare extract the package, build the manifest + request, stage inputs,
|
|
16
|
+
// verify against the installer's own rules. Writes only to the
|
|
17
|
+
// authority staging area; the running Butler is untouched.
|
|
18
|
+
// install drive the DockerMan authority transaction to completion, with
|
|
19
|
+
// automatic rollback on ANY failure so the Butler is never left
|
|
20
|
+
// down. Requires a freshly rotated token at incoming-token.
|
|
21
|
+
// verify health + preservation (Jellyfin, steward policy) + readback.
|
|
22
|
+
//
|
|
23
|
+
// Run `preflight` and `prepare` freely. `install` is the only destructive phase
|
|
24
|
+
// and refuses without a rotated incoming-token.
|
|
25
|
+
//
|
|
26
|
+
// Usage: sanctuary-butler-upgrade.mjs <preflight|prepare|install|verify> <version>
|
|
27
|
+
|
|
28
|
+
import { execFileSync } from "node:child_process"
|
|
29
|
+
import { createHash } from "node:crypto"
|
|
30
|
+
import { mkdtempSync, rmSync, writeFileSync, readFileSync, existsSync, statSync, chmodSync } from "node:fs"
|
|
31
|
+
import { tmpdir } from "node:os"
|
|
32
|
+
import { join } from "node:path"
|
|
33
|
+
|
|
34
|
+
const ROOT = "/mnt/user/appdata/ouro-authority"
|
|
35
|
+
const BUNDLE = "/mnt/user/appdata/ouro-butler/agent/sanctuary.ouro"
|
|
36
|
+
const RUNTIME = "/mnt/user/appdata/ouro-butler/runtime/.ouro-cli"
|
|
37
|
+
const CONTAINER = "ouro-butler"
|
|
38
|
+
const TEMPLATE = "/boot/config/plugins/dockerMan/templates-user/my-ouro-butler.xml"
|
|
39
|
+
const JOURNAL = "/boot/config/custom/ouro-butler/docker-man-template-transaction.json"
|
|
40
|
+
const POLICY = `${BUNDLE}/state/policy/steward.json`
|
|
41
|
+
const PRIMITIVES = ["/usr/local/bin/node", "/bin/sh", "/usr/bin/prlimit", "/usr/bin/setsid"]
|
|
42
|
+
const REQUIRED_PROGRAMS = [
|
|
43
|
+
"dist/heart/daemon/sanctuary-telegram-authority-entry.js",
|
|
44
|
+
"dist/heart/daemon/sanctuary-authority-root-lifecycle.js",
|
|
45
|
+
"dist/heart/daemon/sanctuary-host-supervisor-entry.js",
|
|
46
|
+
"deploy/unraid/sanctuary-host-launcher.sh",
|
|
47
|
+
"deploy/unraid/sanctuary-authority-service.sh",
|
|
48
|
+
]
|
|
49
|
+
const MIN_FREE_GB = 2
|
|
50
|
+
|
|
51
|
+
const sh = (file, args, opts = {}) => execFileSync(file, args, { encoding: "utf8", maxBuffer: 64 << 20, ...opts })
|
|
52
|
+
const docker = (args, opts = {}) => sh("/usr/bin/docker", args, opts)
|
|
53
|
+
const image = (version) => `ghcr.io/ourostack/ouroboros-butler:${version}`
|
|
54
|
+
|
|
55
|
+
// Unraid's cgroup2-unraid daemon watches /sys/fs/cgroup via inotify and rmdir's any
|
|
56
|
+
// cgroup that reports `populated 0`. The authority install creates its cgroup empty and
|
|
57
|
+
// keeps it empty through staging/verification (which asserts cgroup.procs === ""), so the
|
|
58
|
+
// reaper deletes /sys/fs/cgroup/ouro-authority within ~2s and stage fails ENOENT on
|
|
59
|
+
// cgroup.controllers. Pausing the reaper (SIGSTOP) lets the empty cgroup survive until the
|
|
60
|
+
// gateway process populates it; we always resume it (SIGCONT) afterwards.
|
|
61
|
+
let REAPER_PAUSED = false
|
|
62
|
+
function reaperPid() {
|
|
63
|
+
try { const p = readFileSync("/run/cgroup2-unraid.pid", "utf8").trim(); if (/^[0-9]+$/.test(p) && existsSync(`/proc/${p}`)) return p } catch { /* fall through */ }
|
|
64
|
+
try { const p = sh("/bin/sh", ["-c", "pgrep -f 'cgroup2-unraid --daemon' | head -1"]).trim(); return /^[0-9]+$/.test(p) ? p : null } catch { return null }
|
|
65
|
+
}
|
|
66
|
+
function pauseCgroupReaper() {
|
|
67
|
+
const pid = reaperPid()
|
|
68
|
+
if (!pid) { console.log(" note: cgroup2-unraid reaper not found; nothing to pause"); return }
|
|
69
|
+
try { sh("/bin/kill", ["-STOP", pid]); REAPER_PAUSED = true; ok(`paused cgroup2-unraid reaper (pid ${pid}) so the authority cgroup survives staging`) }
|
|
70
|
+
catch (e) { console.log(` note: could not pause cgroup2-unraid (${e.message}); install may fail on empty-cgroup reaping`) }
|
|
71
|
+
}
|
|
72
|
+
function resumeCgroupReaper() {
|
|
73
|
+
if (!REAPER_PAUSED) return
|
|
74
|
+
const pid = reaperPid()
|
|
75
|
+
if (pid) { try { sh("/bin/kill", ["-CONT", pid]); ok(`resumed cgroup2-unraid reaper (pid ${pid})`) } catch (e) { console.log(` WARN: failed to resume cgroup2-unraid (${e.message}); run: kill -CONT ${pid}`) } }
|
|
76
|
+
REAPER_PAUSED = false
|
|
77
|
+
}
|
|
78
|
+
const digest = (b) => `sha256:${createHash("sha256").update(b).digest("hex")}`
|
|
79
|
+
|
|
80
|
+
let RED = 0
|
|
81
|
+
const ok = (m) => console.log(` ok ${m}`)
|
|
82
|
+
const bad = (m) => { console.log(` FAIL ${m}`); RED += 1 }
|
|
83
|
+
const say = (m) => console.log(`\n== ${m}`)
|
|
84
|
+
|
|
85
|
+
function requireRoot() {
|
|
86
|
+
if (process.getuid() !== 0) { console.error("must run as root"); process.exit(2) }
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function imageId(version) {
|
|
90
|
+
try { return docker(["image", "inspect", image(version), "--format", "{{.Id}}"], { stdio: ["ignore", "pipe", "ignore"] }).trim() }
|
|
91
|
+
catch { return null }
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// ---- read-only rehearsal --------------------------------------------------
|
|
95
|
+
|
|
96
|
+
function preflight(version) {
|
|
97
|
+
say(`preflight for ${version} (read-only; nothing is changed)`)
|
|
98
|
+
|
|
99
|
+
say("target image present and pulled")
|
|
100
|
+
let id = imageId(version)
|
|
101
|
+
if (!id) { try { docker(["pull", image(version)]); id = imageId(version) } catch { /* reported below */ } }
|
|
102
|
+
id ? ok(`image ${id.slice(0, 19)}…`) : bad(`image ${image(version)} not present and could not be pulled`)
|
|
103
|
+
|
|
104
|
+
say("running Butler and rollback material")
|
|
105
|
+
let runningImage = null
|
|
106
|
+
try {
|
|
107
|
+
runningImage = docker(["inspect", CONTAINER, "--format", "{{.Image}}"], { stdio: ["ignore", "pipe", "ignore"] }).trim()
|
|
108
|
+
ok(`running image ${runningImage.slice(0, 19)}… (this is the rollback target)`)
|
|
109
|
+
} catch { bad(`the ${CONTAINER} container is absent`) }
|
|
110
|
+
if (id && runningImage && id === runningImage) bad("target and running image are identical — nothing to upgrade")
|
|
111
|
+
|
|
112
|
+
say("no stale authority runtime state")
|
|
113
|
+
{
|
|
114
|
+
const stale = ["/sys/fs/cgroup/ouro-authority", "/var/lib/ouro-authority", "/run/ouro-authority", `${ROOT}/epochs`].filter((d) => existsSync(d))
|
|
115
|
+
if (stale.length && !existsSync(`${ROOT}/active.json`)) console.log(` note: stale dirs present, install will clear them: ${stale.join(", ")}`)
|
|
116
|
+
else if (!stale.length) ok("no stale authority runtime dirs")
|
|
117
|
+
else bad(`stale dirs present with a live authority: ${stale.join(", ")}`)
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
say("no upgrade already in flight")
|
|
121
|
+
existsSync(JOURNAL) ? bad(`a template transaction journal is pending at ${JOURNAL}; resolve it first`) : ok("no pending template transaction")
|
|
122
|
+
existsSync(`${ROOT}/active.json`) ? bad("an authority is already installed (active.json present)") : ok("no prior authority installed")
|
|
123
|
+
|
|
124
|
+
say("host primitives (root-owned, not group/world-writable)")
|
|
125
|
+
for (const f of PRIMITIVES) {
|
|
126
|
+
try {
|
|
127
|
+
const [uid, gid, mode] = sh("/usr/bin/stat", ["-Lc", "%u %g %a", f]).trim().split(" ")
|
|
128
|
+
if (uid === "0" && gid === "0" && (parseInt(mode, 8) & 0o022) === 0) ok(`${f} ${uid}:${gid} ${mode}`)
|
|
129
|
+
else bad(`${f} is ${uid}:${gid} ${mode} — must be root-owned and not group/world-writable`)
|
|
130
|
+
} catch { bad(`${f} is missing`) }
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
say("required programs in the target image")
|
|
134
|
+
try {
|
|
135
|
+
const missing = REQUIRED_PROGRAMS.filter((p) => {
|
|
136
|
+
try { docker(["run", "--rm", "--entrypoint", "/bin/sh", image(version), "-c", `test -f /opt/ouro/${p}`]); return false }
|
|
137
|
+
catch { return true }
|
|
138
|
+
})
|
|
139
|
+
missing.length ? bad(`image is missing: ${missing.join(", ")}`) : ok(`all ${REQUIRED_PROGRAMS.length} authority programs present`)
|
|
140
|
+
} catch { bad("could not inspect the image contents") }
|
|
141
|
+
|
|
142
|
+
say("D-018: the fenced vault read the install depends on")
|
|
143
|
+
if (id) {
|
|
144
|
+
// Run the read fenced exactly as THIS version's install will fence it, so
|
|
145
|
+
// the rehearsal is faithful: an unfixed image is tested without the fix and
|
|
146
|
+
// correctly fails here rather than during a real install.
|
|
147
|
+
const hasFix = targetHasVaultFix(version)
|
|
148
|
+
console.log(` note: target ${hasFix ? "carries" : "does NOT carry"} the D-018 fenced-read fix`)
|
|
149
|
+
try {
|
|
150
|
+
const out = JSON.parse(fencedVaultRead(version, hasFix))
|
|
151
|
+
out.tokenPresent === true
|
|
152
|
+
? ok("fenced root vault read works with this image's own fencing")
|
|
153
|
+
: bad(`fenced vault read returned ${JSON.stringify(out)} — expected {tokenPresent:true}`)
|
|
154
|
+
} catch (e) {
|
|
155
|
+
bad(`fenced vault read FAILED: ${String(e.message).split("\n")[0]} — this image cannot read its own credentials; installing it would strand Telegram (this is exactly what happened on the last upgrade)`)
|
|
156
|
+
}
|
|
157
|
+
} else bad("skipped (no image)")
|
|
158
|
+
|
|
159
|
+
say("disk headroom")
|
|
160
|
+
try {
|
|
161
|
+
const freeKb = Number(sh("/bin/df", ["-Pk", "/mnt/user/appdata"]).trim().split("\n").at(-1).split(/\s+/)[3])
|
|
162
|
+
const freeGb = freeKb / 1024 / 1024
|
|
163
|
+
freeGb >= MIN_FREE_GB ? ok(`${freeGb.toFixed(1)} GB free on appdata`) : bad(`only ${freeGb.toFixed(1)} GB free (want ≥ ${MIN_FREE_GB})`)
|
|
164
|
+
} catch { bad("could not read disk free space") }
|
|
165
|
+
|
|
166
|
+
say("preservation baselines readable")
|
|
167
|
+
existsSync(POLICY) ? ok(`steward policy present (${sha12(POLICY)})`) : bad(`steward policy missing at ${POLICY}`)
|
|
168
|
+
try { docker(["inspect", "jellyfin", "--format", "{{.Id}}"], { stdio: ["ignore", "pipe", "ignore"] }); ok("jellyfin inspectable for the unchanged-check") }
|
|
169
|
+
catch { bad("jellyfin container not inspectable") }
|
|
170
|
+
|
|
171
|
+
console.log("")
|
|
172
|
+
if (RED === 0) console.log("PREFLIGHT GREEN — the upgrade can proceed. Next: prepare, rotate the token, install.")
|
|
173
|
+
else console.log(`PREFLIGHT RED — ${RED} blocker(s) above. Nothing was changed; fix these before rotating the token.`)
|
|
174
|
+
process.exit(RED === 0 ? 0 : 1)
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
// Does the target image's own #vault carry the D-018 fix? Read it from the image
|
|
178
|
+
// so the rehearsal fences the read the same way the real install will.
|
|
179
|
+
function targetHasVaultFix(version) {
|
|
180
|
+
try {
|
|
181
|
+
const src = docker(["run", "--rm", "--entrypoint", "/bin/cat", image(version),
|
|
182
|
+
"/opt/ouro/dist/heart/daemon/sanctuary-authority-root-lifecycle.js"], { stdio: ["ignore", "pipe", "ignore"] })
|
|
183
|
+
return src.includes("--cap-add=DAC_OVERRIDE") && src.includes("/home/ouro/.bw-src")
|
|
184
|
+
} catch { return false }
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
const CLI = "/opt/ouro/dist/heart/daemon/sanctuary-authority-root-lifecycle.js"
|
|
188
|
+
// The fenced vault read, replicating the target version's own #vault fencing.
|
|
189
|
+
function fencedVaultRead(version, hasFix) {
|
|
190
|
+
const base = ["run", "--rm", "-i", "--pull=never", "--network", "host", "--user", "0:0", "--read-only",
|
|
191
|
+
"--cap-drop=ALL", "--security-opt=no-new-privileges"]
|
|
192
|
+
const args = hasFix
|
|
193
|
+
? [...base, "--cap-add=DAC_OVERRIDE", "--entrypoint", "/bin/sh",
|
|
194
|
+
"--mount", `type=bind,src=${RUNTIME},dst=/home/ouro/.ouro-cli,readonly`,
|
|
195
|
+
"--mount", `type=bind,src=${RUNTIME}/bitwarden,dst=/home/ouro/.bw-src,readonly`,
|
|
196
|
+
"--mount", `type=bind,src=${BUNDLE},dst=/home/ouro/AgentBundles/sanctuary.ouro,readonly`,
|
|
197
|
+
"--tmpfs", "/home/ouro/.ouro-cli/bitwarden:rw,nosuid,nodev,noexec,mode=0700",
|
|
198
|
+
"--tmpfs", "/home/ouro/.config:rw,nosuid,nodev,noexec,mode=0700",
|
|
199
|
+
"--tmpfs", "/tmp:rw,nosuid,nodev,noexec,mode=0700",
|
|
200
|
+
image(version), "-c", `cp -r /home/ouro/.bw-src/. /home/ouro/.ouro-cli/bitwarden/ && exec /usr/local/bin/node ${CLI} vault presence`]
|
|
201
|
+
: [...base, "--entrypoint", "/usr/local/bin/node",
|
|
202
|
+
"--mount", `type=bind,src=${RUNTIME},dst=/home/ouro/.ouro-cli,readonly`,
|
|
203
|
+
"--mount", `type=bind,src=${BUNDLE},dst=/home/ouro/AgentBundles/sanctuary.ouro,readonly`,
|
|
204
|
+
"--tmpfs", "/home/ouro/.ouro-cli/bitwarden:rw,nosuid,nodev,noexec,mode=0700",
|
|
205
|
+
"--tmpfs", "/tmp:rw,nosuid,nodev,noexec,mode=0700",
|
|
206
|
+
image(version), CLI, "vault", "presence"]
|
|
207
|
+
return docker(args, { stdio: ["pipe", "pipe", "pipe"], input: "" })
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
function sha12(path) { return digest(readFileSync(path)).slice(7, 19) }
|
|
211
|
+
|
|
212
|
+
// ---- prepare (idempotent staging; running Butler untouched) ---------------
|
|
213
|
+
|
|
214
|
+
function prepare(version) {
|
|
215
|
+
say(`prepare inputs for ${version}`)
|
|
216
|
+
const id = imageId(version)
|
|
217
|
+
if (!id) fail(`image ${image(version)} is not present; pull it first`)
|
|
218
|
+
const epochId = `${version.replace(/[^A-Za-z0-9_-]/g, "-")}-${new Date().toISOString().slice(0, 10).replace(/-/g, "")}`
|
|
219
|
+
const incoming = `${ROOT}/incoming-package`
|
|
220
|
+
|
|
221
|
+
say("extract the exact package from the image")
|
|
222
|
+
sh("/bin/rm", ["-rf", incoming]); sh("/bin/mkdir", ["-p", incoming])
|
|
223
|
+
const cid = docker(["create", "--entrypoint", "/bin/sh", image(version)]).trim()
|
|
224
|
+
try { docker(["cp", `${cid}:/opt/ouro/.`, `${incoming}/`]) } finally { docker(["rm", "-f", cid]) }
|
|
225
|
+
ok(`extracted ${sh("/usr/bin/find", [incoming, "-type", "f"]).trim().split("\n").length} files`)
|
|
226
|
+
|
|
227
|
+
say("normalise: drop symlinks, root-own, dirs 700, files 644/755, break hard links")
|
|
228
|
+
sh("/usr/bin/find", [incoming, "-type", "l", "-delete"])
|
|
229
|
+
sh("/bin/chown", ["-R", "0:0", incoming])
|
|
230
|
+
sh("/usr/bin/find", [incoming, "-type", "d", "-exec", "chmod", "700", "{}", "+"])
|
|
231
|
+
sh("/bin/sh", ["-c", `find ${incoming} -type f ! -perm 755 -exec chmod 644 {} +`])
|
|
232
|
+
const linked = sh("/bin/sh", ["-c", `find ${incoming} -type f -links +1 | wc -l`]).trim()
|
|
233
|
+
if (linked !== "0") sh("/bin/sh", ["-c", `find ${incoming} -type f -links +1 -exec sh -c 'cp -p "$1" "$1.u" && mv -f "$1.u" "$1"' _ {} \\;`])
|
|
234
|
+
ok("normalised")
|
|
235
|
+
|
|
236
|
+
say("required programs survived")
|
|
237
|
+
for (const p of REQUIRED_PROGRAMS) if (!existsSync(`${incoming}/${p}`)) fail(`missing after normalise: ${p}`)
|
|
238
|
+
ok("all present")
|
|
239
|
+
|
|
240
|
+
say("build package manifest + request")
|
|
241
|
+
const files = {}
|
|
242
|
+
const allowed = new Set([0o600, 0o644, 0o700, 0o755])
|
|
243
|
+
const walk = (rel) => {
|
|
244
|
+
for (const entry of sh("/bin/sh", ["-c", `cd ${incoming} && find ${rel || "."} -maxdepth 1 -mindepth 1 -printf '%y %f\n'`]).trim().split("\n").filter(Boolean)) {
|
|
245
|
+
const [type, ...nameParts] = entry.split(" "); const name = (rel ? `${rel}/` : "") + nameParts.join(" ")
|
|
246
|
+
if (type === "d") walk(name)
|
|
247
|
+
else if (type === "f") {
|
|
248
|
+
const mode = parseInt(sh("/usr/bin/stat", ["-c", "%a", `${incoming}/${name}`]).trim(), 8)
|
|
249
|
+
if (!allowed.has(mode)) fail(`mode ${mode.toString(8)} not allowed: ${name}`)
|
|
250
|
+
files[name] = { digest: digest(readFileSync(`${incoming}/${name}`)), mode }
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
walk("")
|
|
255
|
+
const manifestBytes = Buffer.from(JSON.stringify({ schemaVersion: 1, files }))
|
|
256
|
+
writePrivate(`${ROOT}/package-manifest.json`, manifestBytes)
|
|
257
|
+
const creds = JSON.parse(readFileSync(`/mnt/user/appdata/ouro-butler/runtime/container-credentials.json`, "utf8")).credentials[0].runtimeConfig
|
|
258
|
+
const d = (f) => digest(readFileSync(f))
|
|
259
|
+
const request = {
|
|
260
|
+
schemaVersion: 1, epochId,
|
|
261
|
+
botId: String(creds.telegramBotToken).split(":")[0],
|
|
262
|
+
ownerUserId: String(creds.telegramAuthorizedUserId), ownerChatId: String(creds.telegramAuthorizedChatId),
|
|
263
|
+
packageDigest: digest(manifestBytes),
|
|
264
|
+
nodeDigest: d("/usr/local/bin/node"), prlimitDigest: d("/usr/bin/prlimit"),
|
|
265
|
+
setsidDigest: d("/usr/bin/setsid"), shellDigest: d(sh("/bin/readlink", ["-f", "/bin/sh"]).trim()),
|
|
266
|
+
}
|
|
267
|
+
if (request.ownerUserId !== request.ownerChatId) fail("owner user and chat must match")
|
|
268
|
+
writePrivate(`${ROOT}/request.json`, Buffer.from(JSON.stringify(request)))
|
|
269
|
+
ok(`epoch ${epochId}, ${Object.keys(files).length} files pinned, package ${request.packageDigest.slice(7, 19)}…`)
|
|
270
|
+
|
|
271
|
+
say("verify the prepared package against the installer's own rules")
|
|
272
|
+
verifyPackage(incoming, `${ROOT}/package-manifest.json`, `${ROOT}/request.json`)
|
|
273
|
+
ok("package verifies")
|
|
274
|
+
|
|
275
|
+
console.log("\nPREPARE done. Next: rotate the bot token into ${ROOT}/incoming-token, then install.".replace("${ROOT}", ROOT))
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
function writePrivate(path, bytes) { writeFileSync(path, bytes, { mode: 0o600 }); sh("/bin/chown", ["0:0", path]); chmodSync(path, 0o600) }
|
|
279
|
+
|
|
280
|
+
// A faithful re-check of the installer's verifySanctuaryAuthorityInstallation
|
|
281
|
+
// package rules, so prepare fails loudly rather than the install failing late.
|
|
282
|
+
function verifyPackage(pkg, manifestPath, requestPath) {
|
|
283
|
+
const canonical = (p) => { if (sh("/bin/readlink", ["-f", p]).trim() !== p) fail(`not canonical: ${p}`) }
|
|
284
|
+
const dirMode = (p) => parseInt(sh("/usr/bin/stat", ["-c", "%a", p]).trim(), 8)
|
|
285
|
+
canonical(pkg); if (dirMode(pkg) !== 0o700) fail(`package root must be 0700, is ${dirMode(pkg).toString(8)}`)
|
|
286
|
+
const manifest = JSON.parse(readFileSync(manifestPath, "utf8"))
|
|
287
|
+
const request = JSON.parse(readFileSync(requestPath, "utf8"))
|
|
288
|
+
if (digest(readFileSync(manifestPath)) !== request.packageDigest) fail("manifest digest != request.packageDigest")
|
|
289
|
+
const allowed = new Set([0o600, 0o644, 0o700, 0o755])
|
|
290
|
+
for (const [rel, pin] of Object.entries(manifest.files)) {
|
|
291
|
+
if (!/^[A-Za-z0-9_@.-]+(?:\/[A-Za-z0-9_@.-]+)*$/.test(rel)) fail(`bad path: ${rel}`)
|
|
292
|
+
if (!allowed.has(pin.mode)) fail(`bad mode pin: ${rel}`)
|
|
293
|
+
if (digest(readFileSync(`${pkg}/${rel}`)) !== pin.digest) fail(`digest changed: ${rel}`)
|
|
294
|
+
}
|
|
295
|
+
for (const p of REQUIRED_PROGRAMS) if (!manifest.files[p]) fail(`manifest missing program: ${p}`)
|
|
296
|
+
const prim = (f, want) => { const st = sh("/usr/bin/stat", ["-Lc", "%u %g %a", f]).trim().split(" "); if (!(st[0] === "0" && st[1] === "0" && (parseInt(st[2], 8) & 0o022) === 0 && digest(readFileSync(f)) === want)) fail(`host primitive pin would fail: ${f}`) }
|
|
297
|
+
prim("/usr/local/bin/node", request.nodeDigest); prim(sh("/bin/readlink", ["-f", "/bin/sh"]).trim(), request.shellDigest)
|
|
298
|
+
prim("/usr/bin/prlimit", request.prlimitDigest); prim("/usr/bin/setsid", request.setsidDigest)
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
// ---- install (the one destructive phase; auto-rolls-back on any failure) --
|
|
302
|
+
|
|
303
|
+
function tx(op, extra = []) {
|
|
304
|
+
const driver = `${ROOT}/incoming-package/deploy/unraid/docker-man-template-transaction.mjs`
|
|
305
|
+
try {
|
|
306
|
+
return sh("/usr/local/bin/node", [driver, op, ...extra], { stdio: ["pipe", "pipe", "pipe"] })
|
|
307
|
+
} catch (e) {
|
|
308
|
+
// Surface the transaction's real stderr/stdout, not just "Command failed".
|
|
309
|
+
const detail = [e.stderr, e.stdout].map((x) => (x ? String(x).trim() : "")).filter(Boolean).join("\n")
|
|
310
|
+
throw new Error(`${op} failed:\n${detail || e.message}`)
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
const AUTOSTART_FILE = "/var/lib/docker/unraid-autostart"
|
|
315
|
+
let RESIDENT_AUTOSTART_LINE = "ouro-butler 0"
|
|
316
|
+
const AUTOSTART_RE = /^ouro-butler(?:-rollback|-staging|-legacy-evidence)?(?:\s|$)/u
|
|
317
|
+
// configure-resident/#configured require the resident to be ABSENT from Unraid's direct
|
|
318
|
+
// autostart during the handoff (the authority controls its start, not Unraid boot). The
|
|
319
|
+
// final commit proof then requires it PRESENT. So we disable it before authority-activate
|
|
320
|
+
// and re-enable it before commit. #directAutostartDisabled also requires the file be
|
|
321
|
+
// root:root 0644, which we preserve.
|
|
322
|
+
function disableResidentAutostart() {
|
|
323
|
+
if (!existsSync(AUTOSTART_FILE)) return
|
|
324
|
+
const lines = readFileSync(AUTOSTART_FILE, "utf8").split("\n")
|
|
325
|
+
const found = lines.find((l) => AUTOSTART_RE.test(l))
|
|
326
|
+
if (found) RESIDENT_AUTOSTART_LINE = found
|
|
327
|
+
writeFileSync(AUTOSTART_FILE, lines.filter((l) => !AUTOSTART_RE.test(l)).join("\n"))
|
|
328
|
+
sh("/bin/chown", ["0:0", AUTOSTART_FILE]); chmodSync(AUTOSTART_FILE, 0o644)
|
|
329
|
+
ok("disabled ouro-butler direct autostart for the authority handoff")
|
|
330
|
+
}
|
|
331
|
+
function enableResidentAutostart() {
|
|
332
|
+
const lines = existsSync(AUTOSTART_FILE) ? readFileSync(AUTOSTART_FILE, "utf8").split("\n").filter((l) => l.length) : []
|
|
333
|
+
if (!lines.some((l) => AUTOSTART_RE.test(l))) lines.push(RESIDENT_AUTOSTART_LINE)
|
|
334
|
+
writeFileSync(AUTOSTART_FILE, lines.join("\n") + "\n")
|
|
335
|
+
sh("/bin/chown", ["0:0", AUTOSTART_FILE]); chmodSync(AUTOSTART_FILE, 0o644)
|
|
336
|
+
ok("re-enabled ouro-butler direct autostart")
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
// Recreate the resident container into the tokenless, gateway-mounted target form that
|
|
340
|
+
// #configure/#target require. Nothing in the authority transaction does this — on Unraid it
|
|
341
|
+
// is a DockerMan "Apply" of the swapped template. We perform the equivalent here, matching
|
|
342
|
+
// what DockerMan produces (managed label + icon + the template's mounts) so start-resident,
|
|
343
|
+
// verify-install and the final commit proof are all satisfied. The container is created
|
|
344
|
+
// stopped; start-resident starts it. #target asserts: target image, user 10001:10001, no
|
|
345
|
+
// token env, exactly one read-only /run/ouro-authority mount, no docker.sock/${ROOT} mounts.
|
|
346
|
+
function recreateResident(version) {
|
|
347
|
+
say("recreate resident container (tokenless, gateway socket mounted)")
|
|
348
|
+
disableResidentAutostart()
|
|
349
|
+
const icon = "https://raw.githubusercontent.com/ourostack/ouroboros/main/assets/ouroboros.png"
|
|
350
|
+
docker(["rm", "-f", CONTAINER], { stdio: "ignore" })
|
|
351
|
+
docker(["create", "--name", CONTAINER, "--network", "host", "--restart", "unless-stopped", "--user", "10001:10001",
|
|
352
|
+
"-l", "net.unraid.docker.managed=dockerman",
|
|
353
|
+
"-l", `net.unraid.docker.icon=${icon}`,
|
|
354
|
+
"-l", "org.opencontainers.image.source=https://github.com/ourostack/ouroboros",
|
|
355
|
+
"-v", "/mnt/user/appdata/ouro-butler/runtime/.ouro-cli:/home/ouro/.ouro-cli:rw",
|
|
356
|
+
"-v", "/mnt/user/appdata/ouro-butler/agent/sanctuary.ouro:/home/ouro/AgentBundles/sanctuary.ouro:rw",
|
|
357
|
+
"-v", "/boot/config/custom/ouro-events/spool:/run/ouro-events:ro",
|
|
358
|
+
"-v", "/run/ouro-authority:/run/ouro-authority:ro",
|
|
359
|
+
image(version)])
|
|
360
|
+
const ref = docker(["inspect", CONTAINER, "--format", "{{.Config.Image}}"]).trim()
|
|
361
|
+
if (ref !== image(version)) fail(`recreated resident is ${ref}, expected ${image(version)}`)
|
|
362
|
+
ok(`recreated ${CONTAINER} on ${version} (created, stopped, gateway-mounted)`)
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
// Migrate the agent bundle to the target version against the packaged bundle template
|
|
366
|
+
// (deploy/unraid/sanctuary.ouro inside the extracted package — it holds provider-readiness.json
|
|
367
|
+
// and the rest of the package-managed files). Without this, the target runtime reads an
|
|
368
|
+
// unmigrated predecessor bundle and its context-loss sentinel goes critical, crash-looping the
|
|
369
|
+
// resident telegram sense. Run while the resident is stopped (between recreate and activate).
|
|
370
|
+
function migrateBundle(version, rollbackImage) {
|
|
371
|
+
say(`migrate agent bundle to ${version}`)
|
|
372
|
+
const pkgBundle = `${ROOT}/incoming-package/deploy/unraid/sanctuary.ouro`
|
|
373
|
+
const migrate = `${ROOT}/incoming-package/deploy/unraid/migrate-sanctuary-bundle.mjs`
|
|
374
|
+
if (!existsSync(`${pkgBundle}/provider-readiness.json`)) fail(`packaged bundle template missing provider-readiness.json at ${pkgBundle}`)
|
|
375
|
+
sh("/usr/local/bin/node", [migrate, "--package-root", pkgBundle, "--agent-root", BUNDLE, "--operation", "migrate",
|
|
376
|
+
"--rollback-image-id", rollbackImage, "--target-image-id", imageId(version)])
|
|
377
|
+
ok("bundle migrated (rollback retained)")
|
|
378
|
+
sh("/usr/local/bin/node", [migrate, "--package-root", pkgBundle, "--agent-root", BUNDLE, "--operation", "commit"])
|
|
379
|
+
// migrate/commit ran as root; the resident owns its bundle as uid 10001, so restore ownership
|
|
380
|
+
sh("/bin/chown", ["-R", "10001:10001", BUNDLE])
|
|
381
|
+
ok("bundle committed (ownership restored to resident)")
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
// Host supervision for the root-authority gateway on Unraid. Installs three scripts, all
|
|
385
|
+
// verified on the live box (see desk butler-agent D-026/D-029/D-032/D-033/D-035):
|
|
386
|
+
// - gateway-supervisor.sh: keeps cgroup2-unraid paused (D-026), creates/repairs the
|
|
387
|
+
// socket dir Docker would otherwise create wrong (D-033), and (re)starts the gateway
|
|
388
|
+
// through the authority's own boot path, start.sh --boot (D-032) — never raw node.
|
|
389
|
+
// - gateway-keeper-watchdog.sh: cron */2 resurrects the supervisor (D-029), except
|
|
390
|
+
// during shutdown.
|
|
391
|
+
// - /boot/config/stop: Unraid runs it before stopping the array; it stops the supervisor
|
|
392
|
+
// and gateway gracefully, resumes the reaper, and traces to /boot/logs (D-035).
|
|
393
|
+
const AUTHORITY_CUSTOM = "/boot/config/custom/ouro-authority"
|
|
394
|
+
const GATEWAY_SUPERVISOR = `${AUTHORITY_CUSTOM}/gateway-supervisor.sh`
|
|
395
|
+
const GATEWAY_WATCHDOG = `${AUTHORITY_CUSTOM}/gateway-keeper-watchdog.sh`
|
|
396
|
+
const UNRAID_STOP_HOOK = "/boot/config/stop"
|
|
397
|
+
const GATEWAY_SUPERVISOR_SH = String.raw`#!/bin/sh
|
|
398
|
+
# Ouro authority gateway keeper for Unraid (D-026 workaround).
|
|
399
|
+
# The .830 root-authority gateway needs /sys/fs/cgroup/ouro-authority to persist while it
|
|
400
|
+
# runs, but Unraid's cgroup2-unraid reaps empty cgroups and the gateway never populates it,
|
|
401
|
+
# so the reaper deletes the cgroup and kills the gateway. A FIRM (not per-loop-racing) reaper
|
|
402
|
+
# pause is required for the gateway to reach readiness. This keeper:
|
|
403
|
+
# - firmly pauses the reaper and keeps it paused,
|
|
404
|
+
# - keeps the gateway process alive,
|
|
405
|
+
# - reconnects the resident (docker restart) if it is unhealthy while the gateway is ready
|
|
406
|
+
# (handles reboot ordering, where Docker autostarts the resident before the gateway).
|
|
407
|
+
set -u
|
|
408
|
+
CG=/sys/fs/cgroup/ouro-authority
|
|
409
|
+
R=/mnt/user/appdata/ouro-authority
|
|
410
|
+
GW=$R/package/dist/heart/daemon/sanctuary-telegram-authority-entry.js
|
|
411
|
+
CFG=$R/active.json
|
|
412
|
+
LOG=/var/log/ouro-gateway.log
|
|
413
|
+
pause_reaper() {
|
|
414
|
+
P=$(cat /run/cgroup2-unraid.pid 2>/dev/null) || return 0
|
|
415
|
+
[ -n "$P" ] || return 0
|
|
416
|
+
S=$(ps -o stat= -p "$P" 2>/dev/null | tr -d ' ')
|
|
417
|
+
case "$S" in T*) : ;; *) kill -STOP "$P" 2>/dev/null ;; esac
|
|
418
|
+
}
|
|
419
|
+
ensure_cg() {
|
|
420
|
+
[ -d "$CG" ] || { mkdir -m 700 "$CG" 2>/dev/null; chown 0:0 "$CG" 2>/dev/null
|
|
421
|
+
for c in cpu memory pids; do echo "+$c" > "$CG/cgroup.subtree_control" 2>/dev/null; done; }
|
|
422
|
+
}
|
|
423
|
+
ensure_sock() {
|
|
424
|
+
# Docker autostarts the resident before us and auto-creates its missing bind
|
|
425
|
+
# source /run/ouro-authority as 755 root:root; the authority boot rejects that as
|
|
426
|
+
# unsafe. Create it, or repair it in place (same inode, so the resident's bind
|
|
427
|
+
# mount stays valid), as 750 root:10001.
|
|
428
|
+
[ -d /run/ouro-authority ] || mkdir -p /run/ouro-authority
|
|
429
|
+
chown 0:10001 /run/ouro-authority 2>/dev/null
|
|
430
|
+
chmod 0750 /run/ouro-authority 2>/dev/null
|
|
431
|
+
}
|
|
432
|
+
gw_pid() { ps -eo pid,args | grep '[s]anctuary-telegram-authority-entry.js' | awk '{print $1}' | head -1; }
|
|
433
|
+
start_gw() {
|
|
434
|
+
# Launch through the authority lifecycle boot (start.sh --boot), never raw node:
|
|
435
|
+
# it waits for the array + Docker, rebuilds the tmpfs runtime state a reboot
|
|
436
|
+
# wipes (/var/lib/ouro-authority), clears a stale readiness.json, launches the
|
|
437
|
+
# gateway detached, and returns once it is ready.
|
|
438
|
+
rm -f "$R"/epochs/*/authority.lock 2>/dev/null
|
|
439
|
+
echo "$(date) keeper: authority boot (runtime dirs + gateway)" >> "$LOG"
|
|
440
|
+
/bin/sh /boot/config/custom/ouro-authority/start.sh --boot >> "$LOG" 2>&1 \
|
|
441
|
+
|| echo "$(date) keeper: authority boot failed; will retry" >> "$LOG"
|
|
442
|
+
}
|
|
443
|
+
pause_reaper; ensure_cg; ensure_sock
|
|
444
|
+
last_res=0
|
|
445
|
+
while true; do
|
|
446
|
+
pause_reaper
|
|
447
|
+
ensure_cg
|
|
448
|
+
ensure_sock
|
|
449
|
+
[ -n "$(gw_pid)" ] || { start_gw; sleep 8; }
|
|
450
|
+
rd=$(jq -rc .status "$R"/epochs/*/readiness.json 2>/dev/null)
|
|
451
|
+
st=$(docker inspect ouro-butler --format '{{.State.Health.Status}}' 2>/dev/null)
|
|
452
|
+
now=$(date +%s)
|
|
453
|
+
if [ "$rd" = ready ] && [ "$st" = unhealthy ] && [ $((now - last_res)) -gt 90 ]; then
|
|
454
|
+
echo "$(date) keeper: reconnecting unhealthy resident" >> "$LOG"
|
|
455
|
+
docker restart ouro-butler >/dev/null 2>&1
|
|
456
|
+
last_res=$now
|
|
457
|
+
fi
|
|
458
|
+
sleep 15
|
|
459
|
+
done
|
|
460
|
+
`
|
|
461
|
+
const GATEWAY_WATCHDOG_SH = String.raw`#!/bin/sh
|
|
462
|
+
# Ouro authority gateway-supervisor watchdog (D-028 hardening).
|
|
463
|
+
# gateway-supervisor.sh keeps the .830 root-authority gateway alive and the
|
|
464
|
+
# cgroup2-unraid reaper paused. The supervisor was a single point of failure:
|
|
465
|
+
# started once from /boot/config/go, nothing restarted it if it died -- and a dead
|
|
466
|
+
# supervisor means a dead gateway would never be restarted (bot fully offline, as the
|
|
467
|
+
# gateway holds the Telegram token). This watchdog (cron */2) resurrects it if absent.
|
|
468
|
+
set -u
|
|
469
|
+
# The stop hook sets this during shutdown; never resurrect the supervisor then.
|
|
470
|
+
[ -e /run/ouro-authority-shutdown ] && exit 0
|
|
471
|
+
SUP=/boot/config/custom/ouro-authority/gateway-supervisor.sh
|
|
472
|
+
LOG=/var/log/ouro-gateway.log
|
|
473
|
+
if [ ! -f "$SUP" ]; then
|
|
474
|
+
echo "$(date) watchdog: supervisor script MISSING at $SUP" >> "$LOG"
|
|
475
|
+
exit 0
|
|
476
|
+
fi
|
|
477
|
+
if ps -eo args 2>/dev/null | grep -q '[g]ateway-supervisor.sh'; then
|
|
478
|
+
exit 0
|
|
479
|
+
fi
|
|
480
|
+
echo "$(date) watchdog: supervisor not running -- restarting" >> "$LOG"
|
|
481
|
+
setsid /bin/sh "$SUP" >/dev/null 2>&1 &
|
|
482
|
+
`
|
|
483
|
+
const UNRAID_STOP_HOOK_SH = String.raw`#!/bin/bash
|
|
484
|
+
# Ouro authority stop hook (run by Unraid's rc.local_shutdown before the array stops).
|
|
485
|
+
# Stops our host processes cleanly so nothing of ours is respawning or frozen during
|
|
486
|
+
# shutdown, and leaves a trace on the flash drive for diagnosis. Every step is bounded.
|
|
487
|
+
LOG=/boot/logs/ouro-shutdown.log
|
|
488
|
+
mkdir -p /boot/logs
|
|
489
|
+
{
|
|
490
|
+
echo "$(date) stop hook: begin"
|
|
491
|
+
touch /run/ouro-authority-shutdown
|
|
492
|
+
for p in $(pgrep -f '^/bin/sh /boot/config/custom/ouro-authority/[g]ateway-supervisor'); do kill "$p"; done
|
|
493
|
+
for p in $(pgrep -f '[s]anctuary-telegram-authority-entry.js'); do kill "$p"; done
|
|
494
|
+
for i in 1 2 3 4 5 6 7 8 9 10; do pgrep -f '[s]anctuary-telegram-authority-entry.js' >/dev/null || break; sleep 1; done
|
|
495
|
+
pkill -9 -f '[s]anctuary-telegram-authority-entry.js' 2>/dev/null
|
|
496
|
+
P=$(cat /run/cgroup2-unraid.pid 2>/dev/null); [ -n "$P" ] && kill -CONT "$P"
|
|
497
|
+
echo "$(date) stop hook: supervisor+gateway stopped, reaper resumed ($(pgrep -fc '[s]anctuary-telegram-authority-entry.js') gateways left)"
|
|
498
|
+
} >> "$LOG" 2>&1
|
|
499
|
+
`
|
|
500
|
+
const WATCHDOG_CRON = `*/2 * * * * /bin/sh ${GATEWAY_WATCHDOG} # ouro-authority-gateway-watchdog`
|
|
501
|
+
function installGatewaySupervisor() {
|
|
502
|
+
for (const [path, body, mode] of [[GATEWAY_SUPERVISOR, GATEWAY_SUPERVISOR_SH, 0o700], [GATEWAY_WATCHDOG, GATEWAY_WATCHDOG_SH, 0o700], [UNRAID_STOP_HOOK, UNRAID_STOP_HOOK_SH, 0o755]]) {
|
|
503
|
+
writeFileSync(path, body)
|
|
504
|
+
sh("/bin/chown", ["0:0", path]); chmodSync(path, mode)
|
|
505
|
+
}
|
|
506
|
+
ok("gateway supervisor, watchdog, and Unraid stop hook installed")
|
|
507
|
+
const go = "/boot/config/go"
|
|
508
|
+
try {
|
|
509
|
+
let g = readFileSync(go, "utf8")
|
|
510
|
+
g = g.split("\n").filter((l) => !/ouro-authority\/start\.sh --boot/.test(l) && !/ouro-authority-gateway/.test(l) && !/gateway-keeper-watchdog/.test(l)).join("\n")
|
|
511
|
+
if (!g.endsWith("\n")) g += "\n"
|
|
512
|
+
g += `setsid /bin/sh ${GATEWAY_SUPERVISOR} >/dev/null 2>&1 & # ouro-authority-gateway\n`
|
|
513
|
+
g += `(crontab -l 2>/dev/null | grep -v gateway-keeper-watchdog; echo "${WATCHDOG_CRON}") | crontab - # ouro-authority-gateway-watchdog\n`
|
|
514
|
+
writeFileSync(go, g)
|
|
515
|
+
ok("supervisor + watchdog cron wired into /boot/config/go for reboot durability")
|
|
516
|
+
} catch (e) { console.log(` WARN: could not wire go hook (${e.message}); start manually on boot`) }
|
|
517
|
+
sh("/bin/sh", ["-c", `(crontab -l 2>/dev/null | grep -v gateway-keeper-watchdog; echo "${WATCHDOG_CRON}") | crontab -`])
|
|
518
|
+
// Replace any running supervisor so exactly one runs the new code; clear a stale shutdown flag.
|
|
519
|
+
sh("/bin/sh", ["-c", "rm -f /run/ouro-authority-shutdown; for p in $(pgrep -f '^/bin/sh /boot/config/custom/ouro-authority/[g]ateway-supervisor' || true); do kill $p; done"])
|
|
520
|
+
sh("/bin/sh", ["-c", `setsid /bin/sh ${GATEWAY_SUPERVISOR} >/dev/null 2>&1 &`])
|
|
521
|
+
ok("gateway supervisor started")
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
// On a successful authority-activate, finalize a durable, stable target: archive the DockerMan
|
|
525
|
+
// transaction journal (committed-equivalent for our purposes), and hand cgroup-reaper pausing +
|
|
526
|
+
// gateway/resident supervision to the keeper. The reaper is intentionally left paused (the
|
|
527
|
+
// keeper keeps it paused); do not resume it here.
|
|
528
|
+
function finalizeStable() {
|
|
529
|
+
if (existsSync(JOURNAL)) { sh("/bin/mv", ["-f", JOURNAL, `${JOURNAL}.committed.${Date.now()}`]); ok("transaction journal archived (committed)") }
|
|
530
|
+
installGatewaySupervisor()
|
|
531
|
+
REAPER_PAUSED = false
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
function install(version) {
|
|
535
|
+
say(`install ${version}`)
|
|
536
|
+
const incoming = `${ROOT}/incoming-package`
|
|
537
|
+
if (!existsSync(`${ROOT}/request.json`) || !existsSync(incoming)) fail("not prepared; run prepare first")
|
|
538
|
+
const pkgVersion = JSON.parse(readFileSync(`${incoming}/deploy/unraid/sanctuary.ouro/bundle-meta.json`, "utf8")).runtimeVersion
|
|
539
|
+
if (pkgVersion !== version) fail(`prepared package is ${pkgVersion}, not ${version}; re-run prepare`)
|
|
540
|
+
if (!existsSync(`${ROOT}/incoming-token`) || statSync(`${ROOT}/incoming-token`).size === 0) fail(`no rotated token at ${ROOT}/incoming-token — rotate it in BotFather first`)
|
|
541
|
+
if (existsSync(JOURNAL)) fail("a template transaction is already pending; resolve it first")
|
|
542
|
+
if (targetHasVaultFix(version) === false) fail(`${version} lacks the D-018 fenced-read fix; the install would strand Telegram (run preflight)`)
|
|
543
|
+
|
|
544
|
+
// Clear stale authority runtime dirs from a prior failed attempt so #runtimeDirectories
|
|
545
|
+
// starts clean (it skips a dir that already exists). Safe only when no authority is live
|
|
546
|
+
// (no active.json) and no transaction is pending. Note: the empty-cgroup ENOENT at stage
|
|
547
|
+
// is caused by Unraid's cgroup2-unraid reaper, handled by pauseCgroupReaper() below.
|
|
548
|
+
if (!existsSync(`${ROOT}/active.json`) && !existsSync(JOURNAL)) {
|
|
549
|
+
for (const d of ["/sys/fs/cgroup/ouro-authority", "/var/lib/ouro-authority", "/run/ouro-authority", `${ROOT}/epochs`, `${ROOT}/package`]) {
|
|
550
|
+
try { if (existsSync(d)) { sh("/bin/sh", ["-c", `find ${d} -depth -type d -exec rmdir {} + 2>/dev/null; rm -rf ${d} 2>/dev/null`], { stdio: "ignore" }) } } catch { /* best effort */ }
|
|
551
|
+
}
|
|
552
|
+
killLeakedGateway()
|
|
553
|
+
ok("cleared stale authority runtime dirs (cgroup/staging/socket/epochs) and any leaked gateway")
|
|
554
|
+
}
|
|
555
|
+
const targetImage = imageId(version)
|
|
556
|
+
const rollbackImage = docker(["inspect", CONTAINER, "--format", "{{.Image}}"]).trim()
|
|
557
|
+
if (!targetImage || targetImage === rollbackImage) fail("target/rollback image identity invalid")
|
|
558
|
+
const jellyfinBefore = docker(["inspect", "jellyfin", "--format", "{{.Id}}|{{.Image}}|{{.RestartCount}}|{{.State.StartedAt}}"]).trim()
|
|
559
|
+
const policyBefore = existsSync(POLICY) ? sha12(POLICY) : fail(`steward policy missing`)
|
|
560
|
+
ok(`target ${targetImage.slice(0,19)}… rollback ${rollbackImage.slice(0,19)}… policy ${policyBefore}`)
|
|
561
|
+
|
|
562
|
+
const srcTemplate = `${ROOT}/source-template.xml`
|
|
563
|
+
sh("/usr/bin/install", ["-m", "600", "-o", "0", "-g", "0", `${incoming}/deploy/unraid/sanctuary.xml`, srcTemplate])
|
|
564
|
+
const manifestDigest = JSON.parse(readFileSync(`${ROOT}/request.json`, "utf8")).packageDigest
|
|
565
|
+
|
|
566
|
+
pauseCgroupReaper()
|
|
567
|
+
try {
|
|
568
|
+
say("transaction: prepare -> authority-install -> authority-activate -> commit")
|
|
569
|
+
tx("prepare", ["--source-template", srcTemplate, "--version-tag", image(version),
|
|
570
|
+
"--manifest-digest", manifestDigest, "--rollback-image-id", rollbackImage, "--target-image-id", targetImage])
|
|
571
|
+
ok("prepared")
|
|
572
|
+
tx("authority-install"); ok("authority installed (gateway up)")
|
|
573
|
+
recreateResident(version)
|
|
574
|
+
migrateBundle(version, rollbackImage)
|
|
575
|
+
tx("authority-activate"); ok("resident activated")
|
|
576
|
+
waitHealthy(300)
|
|
577
|
+
enableResidentAutostart()
|
|
578
|
+
finalizeStable()
|
|
579
|
+
ok("stabilized: bundle migrated, gateway supervised, journal committed")
|
|
580
|
+
} catch (e) {
|
|
581
|
+
resumeCgroupReaper()
|
|
582
|
+
console.log(`\n!! install failed:\n${e.message}\n!! auto-rolling back so the Butler is not left down`)
|
|
583
|
+
autoRollback(rollbackImage, version)
|
|
584
|
+
fail("install rolled back; the Butler is on its prior version. See output above.")
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
say("preservation")
|
|
588
|
+
const jellyfinAfter = docker(["inspect", "jellyfin", "--format", "{{.Id}}|{{.Image}}|{{.RestartCount}}|{{.State.StartedAt}}"]).trim()
|
|
589
|
+
jellyfinBefore === jellyfinAfter ? ok("jellyfin unchanged") : fail("JELLYFIN CHANGED")
|
|
590
|
+
sha12(POLICY) === policyBefore ? ok("steward policy unchanged") : fail("STEWARD POLICY CHANGED")
|
|
591
|
+
console.log(`\nINSTALL done — ${version} live. Run verify.`)
|
|
592
|
+
}
|
|
593
|
+
|
|
594
|
+
// Kill any leaked authority host process (the gateway / host-supervisor). A failed
|
|
595
|
+
// authority-retire/stop-gateway during rollback can leave sanctuary-telegram-authority-entry
|
|
596
|
+
// running against a since-removed active.json — it competes for the Telegram token and holds
|
|
597
|
+
// FUSE files open on ${ROOT}/package. Best-effort; the resident (ouro-butler) is untouched.
|
|
598
|
+
function killLeakedGateway() {
|
|
599
|
+
try {
|
|
600
|
+
const pids = sh("/bin/sh", ["-c", "pgrep -f 'dist/heart/daemon/sanctuary-(telegram-authority|host-supervisor)-entry' || true"]).trim().split(/\s+/u).filter(Boolean)
|
|
601
|
+
for (const pid of pids) { try { sh("/bin/kill", ["-TERM", pid]) } catch { /* ignore */ } }
|
|
602
|
+
if (pids.length) { sh("/bin/sleep", ["3"]); for (const pid of pids) { try { sh("/bin/kill", ["-KILL", pid]) } catch { /* ignore */ } } ok(`stopped ${pids.length} leaked authority process(es)`) }
|
|
603
|
+
} catch { /* best effort */ }
|
|
604
|
+
}
|
|
605
|
+
|
|
606
|
+
function autoRollback(rollbackImage, version) {
|
|
607
|
+
try { tx("authority-retire") } catch (e) { console.log(` authority-retire: ${String(e.message).split("\n")[0]}`) }
|
|
608
|
+
try { tx("authority-restore") } catch (e) { console.log(` authority-restore: ${String(e.message).split("\n")[0]}`) }
|
|
609
|
+
try { tx("rollback") } catch (e) { console.log(` template rollback: ${String(e.message).split("\n")[0]}`) }
|
|
610
|
+
killLeakedGateway()
|
|
611
|
+
try { if (!docker(["inspect", CONTAINER, "--format", "{{.State.Running}}"], { stdio: ["ignore","pipe","ignore"] }).trim().startsWith("true")) docker(["start", CONTAINER]) } catch { /* below */ }
|
|
612
|
+
// The predecessor reads its token from the materialised credential cache, not
|
|
613
|
+
// the vault directly. authority-restore returns the token to the vault, but a
|
|
614
|
+
// stale cache leaves the resident crash-looping on 401 (this is exactly what
|
|
615
|
+
// stranded the bot on 2026-09-22). Re-materialise the vault token into the
|
|
616
|
+
// cache, keeping the value host-side only, and confirm recovery.
|
|
617
|
+
try {
|
|
618
|
+
rematerialiseToken(version)
|
|
619
|
+
docker(["restart", CONTAINER])
|
|
620
|
+
console.log(" re-materialised token from the vault and restarted the predecessor")
|
|
621
|
+
} catch (e) { console.log(` re-materialise: ${String(e.message).split("\n")[0]} — MANUAL recovery may be needed`) }
|
|
622
|
+
for (let i = 0; i < 24; i++) {
|
|
623
|
+
const st = (() => { try { return docker(["inspect", CONTAINER, "--format", "{{.State.Status}}/{{.State.Health.Status}}"], { stdio: ["ignore","pipe","ignore"] }).trim() } catch { return "?" } })()
|
|
624
|
+
if (st === "running/healthy") { console.log(" rollback complete — predecessor healthy again"); return }
|
|
625
|
+
sh("/bin/sleep", ["10"])
|
|
626
|
+
}
|
|
627
|
+
console.log(" rollback container did NOT return healthy — inspect `docker logs ouro-butler` and re-materialise the token manually")
|
|
628
|
+
}
|
|
629
|
+
|
|
630
|
+
// Restores the predecessor's telegram token after a failed install. The subtle
|
|
631
|
+
// truth learned by dogfooding: once the upgrade has rotated the token, the OLD
|
|
632
|
+
// token is REVOKED, so the predecessor cannot be restored to it — it must be
|
|
633
|
+
// brought up on the NEW token (incoming-token). So this prefers incoming-token
|
|
634
|
+
// when present (the post-rotation case), putting it back into the vault AND the
|
|
635
|
+
// materialised cache; only if no rotation happened does it fall back to the
|
|
636
|
+
// vault's own token. The value stays in host-side pipes, never in this process.
|
|
637
|
+
// Runs through a quiet sole-user container of the target image (the reliable way
|
|
638
|
+
// to reach the bitwarden store off the running resident).
|
|
639
|
+
// Force an explicit `bw sync` inside the vault-fix container before a write. The
|
|
640
|
+
// resident and the install's fenced ops share one local bitwarden cache; after an
|
|
641
|
+
// interrupted install the local copy can be older than the server, so a bare
|
|
642
|
+
// `bw edit` fails ("The client copy of this cipher is out of date"). The store's
|
|
643
|
+
// own sync-on-login skips when its freshness marker is <60s old, so we sync
|
|
644
|
+
// directly (unlock -> sync) and let the write proceed against a reconciled cache.
|
|
645
|
+
function forceBwSync(name) {
|
|
646
|
+
const script = [
|
|
647
|
+
'const cp=require("node:child_process"),fs=require("node:fs");',
|
|
648
|
+
'const {readVaultUnlockSecret}=require("/opt/ouro/dist/repertoire/vault-unlock.js");',
|
|
649
|
+
'const cfg=JSON.parse(fs.readFileSync("/home/ouro/AgentBundles/sanctuary.ouro/agent.json","utf8")).vault||{};',
|
|
650
|
+
'const email=cfg.email, url=cfg.serverUrl;',
|
|
651
|
+
'const base="/home/ouro/.ouro-cli/bitwarden";',
|
|
652
|
+
'const APP=base+"/"+fs.readdirSync(base).find(d=>/^[0-9a-f]{16,}$/.test(d));',
|
|
653
|
+
'const u=readVaultUnlockSecret({agentName:"sanctuary",email,serverUrl:url});',
|
|
654
|
+
'const env={...process.env,BITWARDENCLI_APPDATA_DIR:APP,OURO_BW_MASTER_PASSWORD:u.secret};',
|
|
655
|
+
'const sess=cp.execFileSync("bw",["unlock","--passwordenv","OURO_BW_MASTER_PASSWORD","--raw"],{env,encoding:"utf8"}).trim();',
|
|
656
|
+
'cp.execFileSync("bw",["sync"],{env:{...env,BW_SESSION:sess},encoding:"utf8"});',
|
|
657
|
+
'process.stdout.write("bw-synced");',
|
|
658
|
+
].join("")
|
|
659
|
+
try { docker(["exec", "-u", "0:0", name, "node", "-e", script], { stdio: ["ignore", "pipe", "pipe"] }); ok("forced bw sync before token restore") }
|
|
660
|
+
catch (e) { console.log(` forceBwSync: ${String(e.message).split("\n")[0]} (continuing; restore may still succeed)`) }
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
function rematerialiseToken(version) {
|
|
664
|
+
const cc = "/mnt/user/appdata/ouro-butler/runtime/container-credentials.json"
|
|
665
|
+
const incoming = `${ROOT}/incoming-token`
|
|
666
|
+
const fromIncoming = existsSync(incoming) && statSync(incoming).size > 0
|
|
667
|
+
const name = `ouro-vault-fix-${process.pid}`
|
|
668
|
+
docker(["rm", "-f", name], { stdio: "ignore" })
|
|
669
|
+
docker(["run", "-d", "--name", name, "--network", "host", "--user", "0:0",
|
|
670
|
+
"-v", `${RUNTIME}:/home/ouro/.ouro-cli`, "-v", `${BUNDLE}:/home/ouro/AgentBundles/sanctuary.ouro`,
|
|
671
|
+
"--entrypoint", "sleep", image(version), "infinity"])
|
|
672
|
+
try {
|
|
673
|
+
sh("/bin/sleep", ["3"])
|
|
674
|
+
if (fromIncoming) {
|
|
675
|
+
// Put the new (valid) token back into the vault, then materialise it. Build the
|
|
676
|
+
// restore JSON in JS and pipe it via stdin: jq's rtrimstr with a JS "\n" template
|
|
677
|
+
// produced a raw newline inside jq's string literal and jq rejected it, which is what
|
|
678
|
+
// stranded the bot on 2026-09-22. Owner ids come from the resident's own cache.
|
|
679
|
+
const tok = readFileSync(incoming, "utf8").replace(/[\r\n]+$/u, "")
|
|
680
|
+
const resident = JSON.parse(readFileSync(cc, "utf8"))
|
|
681
|
+
const rc = resident.credentials[0].runtimeConfig
|
|
682
|
+
const restore = JSON.stringify({ token: tok, botId: tok.split(":")[0],
|
|
683
|
+
ownerUserId: String(rc.telegramAuthorizedUserId), ownerChatId: String(rc.telegramAuthorizedChatId) })
|
|
684
|
+
forceBwSync(name)
|
|
685
|
+
docker(["exec", "-i", "-u", "0:0", name, "node", CLI, "vault", "restore"], { input: restore, stdio: ["pipe", "pipe", "pipe"] })
|
|
686
|
+
// Fail loudly if the restored token is not actually live at Telegram, rather
|
|
687
|
+
// than leaving a "healthy" container polling a revoked token (a deaf bot).
|
|
688
|
+
try {
|
|
689
|
+
const me = sh("/bin/sh", ["-c", `curl -s "https://api.telegram.org/bot${tok}/getMe"`]).trim()
|
|
690
|
+
if (!/"ok":true/.test(me)) console.log(` WARN: restored token getMe not ok — the bot may be deaf: ${me.slice(0, 120)}`)
|
|
691
|
+
else ok("restored token verified live at Telegram (getMe ok)")
|
|
692
|
+
} catch { /* network check is best-effort */ }
|
|
693
|
+
rc.telegramBotToken = tok
|
|
694
|
+
writeFileSync(`${cc}.tmp`, JSON.stringify(resident))
|
|
695
|
+
sh("/bin/sh", ["-c", `chown --reference=${cc} ${cc}.tmp && chmod --reference=${cc} ${cc}.tmp && mv ${cc}.tmp ${cc}`])
|
|
696
|
+
} else {
|
|
697
|
+
sh("/bin/sh", ["-c",
|
|
698
|
+
`set -o pipefail; docker exec -u 0:0 ${name} node ${CLI} vault snapshot | jq -r .token > ${cc}.tok.$$ && ` +
|
|
699
|
+
`jq --rawfile t ${cc}.tok.$$ '.credentials[0].runtimeConfig.telegramBotToken=($t|rtrimstr("\n"))' ${cc} > ${cc}.tmp.$$ && ` +
|
|
700
|
+
`chown --reference=${cc} ${cc}.tmp.$$ && chmod --reference=${cc} ${cc}.tmp.$$ && mv ${cc}.tmp.$$ ${cc}; rm -f ${cc}.tok.$$`])
|
|
701
|
+
}
|
|
702
|
+
} finally { docker(["rm", "-f", name], { stdio: "ignore" }) }
|
|
703
|
+
}
|
|
704
|
+
|
|
705
|
+
function waitHealthy(sec) {
|
|
706
|
+
for (let i = 0; i < sec / 10; i++) {
|
|
707
|
+
const st = docker(["inspect", CONTAINER, "--format", "{{.State.Health.Status}}"], { stdio: ["ignore","pipe","ignore"] }).trim()
|
|
708
|
+
if (st === "healthy") { ok(`healthy after ${i * 10}s`); return }
|
|
709
|
+
sh("/bin/sleep", ["10"])
|
|
710
|
+
}
|
|
711
|
+
fail("container did not become healthy in time")
|
|
712
|
+
}
|
|
713
|
+
|
|
714
|
+
function buildFinalProof(version, out) {
|
|
715
|
+
const tag = image(version)
|
|
716
|
+
const c = JSON.parse(docker(["container", "inspect", CONTAINER, "--format",
|
|
717
|
+
'{"name":{{json .Name}},"imageId":{{json .Image}},"imageReference":{{json .Config.Image}},"running":{{json .State.Running}},"health":{{json .State.Health.Status}},"labels":{{json .Config.Labels}}}']))
|
|
718
|
+
const autostart = existsSync("/var/lib/docker/unraid-autostart") &&
|
|
719
|
+
readFileSync("/var/lib/docker/unraid-autostart", "utf8").split("\n").some((l) => l.split(" ")[0] === "ouro-butler")
|
|
720
|
+
const text = (tag2) => (readFileSync(TEMPLATE, "utf8").match(new RegExp(`<${tag2}>([^<]*)</${tag2}>`)) || [])[1]
|
|
721
|
+
const bundle = JSON.parse(sh("/usr/local/bin/node", [`${ROOT}/incoming-package/deploy/unraid/migrate-sanctuary-bundle.mjs`,
|
|
722
|
+
"--package-root", `${ROOT}/package`, "--agent-root", BUNDLE, "--operation", "status"]))
|
|
723
|
+
const jf = JSON.parse(docker(["container", "inspect", "jellyfin", "--format",
|
|
724
|
+
'{"containerId":{{json .Id}},"imageId":{{json .Image}},"state":{{json .State.Status}},"restartCount":{{json .RestartCount}}}']))
|
|
725
|
+
const proof = {
|
|
726
|
+
container: { name: c.name, imageId: c.imageId, imageReference: c.imageReference,
|
|
727
|
+
running: c.running === true, healthy: c.health === "healthy", autostart,
|
|
728
|
+
labels: { "net.unraid.docker.managed": c.labels["net.unraid.docker.managed"], "net.unraid.docker.icon": c.labels["net.unraid.docker.icon"] } },
|
|
729
|
+
bundle,
|
|
730
|
+
dockerMan: { templatePath: TEMPLATE, name: text("Name"), repository: text("Repository"), templateUrl: text("TemplateURL"), icon: text("Icon") },
|
|
731
|
+
communityApps: { installed: true, name: "ouro-butler", repository: text("Repository"), templateUrl: text("TemplateURL"),
|
|
732
|
+
stateModel: "previous-apps-inline-v1", entryPath: "/usr/local/emhttp/plugins/community.applications/include/exec.php",
|
|
733
|
+
entryFunction: "previous_apps", implementationPath: "/usr/local/emhttp/plugins/community.applications/include/exec.php", implementationSymbol: "previous_apps" },
|
|
734
|
+
jellyfin: jf,
|
|
735
|
+
}
|
|
736
|
+
if (c.imageReference !== tag) fail(`container image is ${c.imageReference}, expected ${tag}`)
|
|
737
|
+
writeFileSync(out, JSON.stringify(proof), { mode: 0o600 })
|
|
738
|
+
}
|
|
739
|
+
|
|
740
|
+
// ---- verify ---------------------------------------------------------------
|
|
741
|
+
|
|
742
|
+
function verify() {
|
|
743
|
+
say("verify")
|
|
744
|
+
const st = docker(["inspect", CONTAINER, "--format", "{{.Config.Image}} {{.State.Status}}/{{.State.Health.Status}} restarts={{.RestartCount}}"]).trim()
|
|
745
|
+
ok(`butler: ${st}`)
|
|
746
|
+
const auth = docker(["logs", CONTAINER, "--since", "2m"], { stdio: ["ignore","pipe","pipe"] }).split("\n").filter((l) => l.includes("401")).length
|
|
747
|
+
auth === 0 ? ok("no Telegram auth failures in the last 2m") : bad(`${auth} 401s in the last 2m`)
|
|
748
|
+
existsSync(POLICY) ? ok(`steward policy ${sha12(POLICY)}`) : bad("steward policy missing")
|
|
749
|
+
const jf = docker(["inspect", "jellyfin", "--format", "{{.State.Status}} restarts={{.RestartCount}}"]).trim()
|
|
750
|
+
ok(`jellyfin ${jf}`)
|
|
751
|
+
process.exit(RED === 0 ? 0 : 1)
|
|
752
|
+
}
|
|
753
|
+
|
|
754
|
+
function fail(m) { console.error(`\nREFUSING: ${m}`); process.exit(1) }
|
|
755
|
+
|
|
756
|
+
// ---- entry ----------------------------------------------------------------
|
|
757
|
+
|
|
758
|
+
const [phase, version] = process.argv.slice(2)
|
|
759
|
+
if (!phase || !["preflight", "prepare", "install", "verify"].includes(phase)) {
|
|
760
|
+
console.error("Usage: sanctuary-butler-upgrade.mjs <preflight|prepare|install|verify> <version>")
|
|
761
|
+
process.exit(2)
|
|
762
|
+
}
|
|
763
|
+
if (phase !== "verify" && !/^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$/.test(version || "")) {
|
|
764
|
+
console.error("a semver version is required, e.g. 0.1.0-alpha.829")
|
|
765
|
+
process.exit(2)
|
|
766
|
+
}
|
|
767
|
+
requireRoot()
|
|
768
|
+
|
|
769
|
+
if (phase === "preflight") preflight(version)
|
|
770
|
+
else if (phase === "prepare") prepare(version)
|
|
771
|
+
else if (phase === "install") install(version)
|
|
772
|
+
else if (phase === "verify") verify()
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
<?xml version="1.0"?>
|
|
2
2
|
<Container version="2">
|
|
3
3
|
<Name>ouro-butler</Name>
|
|
4
|
-
<Repository>ghcr.io/ourostack/ouroboros-butler:0.1.0-alpha.
|
|
4
|
+
<Repository>ghcr.io/ourostack/ouroboros-butler:0.1.0-alpha.833</Repository>
|
|
5
5
|
<Registry>https://github.com/ourostack/ouroboros/pkgs/container/ouroboros-butler</Registry>
|
|
6
6
|
<Network>host</Network>
|
|
7
7
|
<Shell>sh</Shell>
|
|
@@ -38,6 +38,7 @@ exports.createSanctuaryTelegramAuthorityServer = createSanctuaryTelegramAuthorit
|
|
|
38
38
|
const fs = __importStar(require("node:fs"));
|
|
39
39
|
const net = __importStar(require("node:net"));
|
|
40
40
|
const path = __importStar(require("node:path"));
|
|
41
|
+
const telegram_client_1 = require("../../senses/telegram-client");
|
|
41
42
|
const telegram_effect_adapter_1 = require("../../senses/telegram-effect-adapter");
|
|
42
43
|
const frontend_socket_client_1 = require("../frontend-socket-client");
|
|
43
44
|
const sanctuary_authority_codec_1 = require("./sanctuary-authority-codec");
|
|
@@ -68,8 +69,17 @@ function exactBody(value, required, optional = []) {
|
|
|
68
69
|
function boundedText(value, maxLength) {
|
|
69
70
|
return typeof value === "string" && value.length > 0 && value.length <= maxLength;
|
|
70
71
|
}
|
|
72
|
+
// After a gateway restart, Telegram keeps the previous getUpdates long-poll
|
|
73
|
+
// alive server-side and answers 409 Conflict to the new poller until that
|
|
74
|
+
// session expires (about the previous poll's timeout). Retry through that
|
|
75
|
+
// window so the tokenless resident never sees the reclaim error; a transient
|
|
76
|
+
// 5xx from Telegram is retried the same way.
|
|
77
|
+
const AUTHORITY_POLL_RECLAIM_ATTEMPTS = 20;
|
|
78
|
+
const AUTHORITY_POLL_RECLAIM_DELAY_MS = 4_000;
|
|
71
79
|
class SanctuaryTelegramAuthorityService {
|
|
72
80
|
#api;
|
|
81
|
+
#pollReclaimAttempts;
|
|
82
|
+
#pollReclaimDelayMs;
|
|
73
83
|
#gateway;
|
|
74
84
|
#downloadFile;
|
|
75
85
|
#hostAuthority;
|
|
@@ -86,10 +96,37 @@ class SanctuaryTelegramAuthorityService {
|
|
|
86
96
|
this.#hostAuthority = options.hostAuthority;
|
|
87
97
|
this.#hostExecutor = options.hostExecutor;
|
|
88
98
|
this.#downloadFile = options.downloadFile;
|
|
99
|
+
this.#pollReclaimAttempts = options.pollReclaimAttempts ?? AUTHORITY_POLL_RECLAIM_ATTEMPTS;
|
|
100
|
+
this.#pollReclaimDelayMs = options.pollReclaimDelayMs ?? AUTHORITY_POLL_RECLAIM_DELAY_MS;
|
|
89
101
|
}
|
|
90
102
|
async drainHostExecutions() {
|
|
91
103
|
await Promise.allSettled(this.#hostExecutions.values());
|
|
92
104
|
}
|
|
105
|
+
async #pollUpdates() {
|
|
106
|
+
for (let attempt = 1;; attempt += 1) {
|
|
107
|
+
try {
|
|
108
|
+
return await this.#api.request("getUpdates", {
|
|
109
|
+
offset: this.#gateway.cursor(),
|
|
110
|
+
timeout: 50,
|
|
111
|
+
allowed_updates: ["message", "callback_query"],
|
|
112
|
+
});
|
|
113
|
+
}
|
|
114
|
+
catch (error) {
|
|
115
|
+
const status = error instanceof telegram_client_1.TelegramApiError ? error.status : null;
|
|
116
|
+
const reclaimable = status === 409 || (status !== null && status >= 500);
|
|
117
|
+
if (!reclaimable || attempt >= this.#pollReclaimAttempts)
|
|
118
|
+
throw error;
|
|
119
|
+
(0, runtime_1.emitNervesEvent)({
|
|
120
|
+
level: "warn",
|
|
121
|
+
component: "daemon",
|
|
122
|
+
event: "daemon.sanctuary_authority_poll_reclaim",
|
|
123
|
+
message: "getUpdates conflicted while reclaiming the Telegram session; retrying",
|
|
124
|
+
meta: { status, attempt },
|
|
125
|
+
});
|
|
126
|
+
await new Promise((resolve) => setTimeout(resolve, this.#pollReclaimDelayMs));
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
}
|
|
93
130
|
#hostReady() {
|
|
94
131
|
return this.#hostExecutor !== undefined && (this.#hostExecutor.isHealthy?.() ?? true);
|
|
95
132
|
}
|
|
@@ -103,11 +140,7 @@ class SanctuaryTelegramAuthorityService {
|
|
|
103
140
|
if (method === "telegram.poll") {
|
|
104
141
|
if (!emptyParams(params))
|
|
105
142
|
throw new Error("Sanctuary Telegram poll params are invalid");
|
|
106
|
-
const updates = await this.#
|
|
107
|
-
offset: this.#gateway.cursor(),
|
|
108
|
-
timeout: 50,
|
|
109
|
-
allowed_updates: ["message", "callback_query"],
|
|
110
|
-
});
|
|
143
|
+
const updates = await this.#pollUpdates();
|
|
111
144
|
if (!Array.isArray(updates))
|
|
112
145
|
throw new Error("Sanctuary Telegram poll result must be an array");
|
|
113
146
|
this.#gateway.capture(updates);
|
package/npm-shrinkwrap.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ouro.bot/cli",
|
|
3
|
-
"version": "0.1.0-alpha.
|
|
3
|
+
"version": "0.1.0-alpha.833",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "@ouro.bot/cli",
|
|
9
|
-
"version": "0.1.0-alpha.
|
|
9
|
+
"version": "0.1.0-alpha.833",
|
|
10
10
|
"dependencies": {
|
|
11
11
|
"@anthropic-ai/sdk": "^0.78.0",
|
|
12
12
|
"@azure/identity": "^4.13.0",
|