@ouro.bot/cli 0.1.0-alpha.864 → 0.1.0-alpha.866

package/changelog.json

@@ -1,6 +1,23 @@
 {
   "_note": "This changelog is maintained as part of the PR/version-bump workflow. Agent-curated, not auto-generated. Agents read this file directly via read_file to understand what changed between versions.",
   "versions": [
+    {
+      "version": "0.1.0-alpha.866",
+      "changes": [
+        "The Sanctuary lifecycle now relaunches a gateway that exits before readiness (backoff of 2, 10 and 30 seconds) within one 15-minute readiness deadline and, if every attempt fails, throws promptly with the gateway.log tail (token-shaped text masked before the tail is cut) instead of waiting out a fresh budget.",
+        "The Sanctuary Telegram authority retries getMe with backoff on network-class errors (fetch failed, ECONNRESET, ETIMEDOUT, 5xx) and still fails at once on auth errors such as 401.",
+        "The Sanctuary authority socket server handles client connection errors by logging them and skips writes to destroyed connections, so a client that disconnects mid-response no longer crashes the gateway with write EPIPE.",
+        "The Butler upgrade script checks that api.telegram.org is reachable before pausing host supervision and aborts early with a clear message if it is not; it skips the check when an upgrade journal exists (resume or rollback)."
+      ]
+    },
+    {
+      "version": "0.1.0-alpha.865",
+      "changes": [
+        "Added an operator-set replay window: while the host's replay gate holds it open for a replay peer, that peer's delegated-command owner notices and A2A await deliveries go to a local sink instead of the owner's Telegram chat, with every authorization check unchanged, a failed sink write refusing the command, and a nerves event for each notice sunk. The window counts only when its directory is root-owned and not group- or other-writable and the window file, opened once with O_NOFOLLOW and checked by descriptor, is a root-owned regular file nobody else can write, and a sink record satisfies an await's noticed check only for the same friend inside such a window.",
+        "Added the host-side replay gate (sanctuary-replay-gate.mjs) that provisions two replay peers and replays seven real requests over A2A, reading each outcome back from tool traces, the steward policy hash, the books ledger, the Sonarr queue, await archives and Telegram effect records; unreadable evidence now fails a case instead of skipping or passing it.",
+        "Made Sanctuary upgrades hold the new version uncommitted until the replay gate passes: a pass commits, pins the template and prunes images, a failed gate rolls back to the predecessor and exits 1, and the commit is refused unless the held upgrade is still the live one. Added --no-gate, --plant and verify --gate, plus a --hold-commit option on the root lifecycle because a completed upgrade cannot be rolled back."
+      ]
+    },
     {
       "version": "0.1.0-alpha.864",
       "changes": [

package/deploy/unraid/sanctuary-butler-upgrade.mjs

@@ -29,11 +29,21 @@
 // refuses without a rotated incoming-token. Run `upgrade` detached so an SSH drop
 // cannot interrupt it:  setsid nohup node <this> upgrade <version> > /var/log/ouro-upgrade.log 2>&1 &
 //
-// Usage: sanctuary-butler-upgrade.mjs <preflight|prepare|install|verify|upgrade> <version> [--rehearse <step>]
+// An `upgrade` ends with the replay gate (sanctuary-replay-gate.mjs): the new Butler is held UNCOMMITTED (the lifecycle
+// keeps its journal and the predecessor's records, because a committed upgrade cannot be rolled back), the gate replays
+// real requests over A2A and reads the outcome back from machine state, and only a pass commits, pins the template and
+// prunes images. A failed gate runs `upgrade-rollback`, confirms the predecessor is healthy, and exits 1.
+//   --no-gate         skip the gate and commit straight away (emergencies; logged loudly)
+//   --plant <case>    make that gate case fail on purpose, proving the rollback end to end
+//   verify --gate     run the gate against the live version without upgrading
+//
+// Usage: sanctuary-butler-upgrade.mjs <preflight|prepare|install|verify|upgrade> <version> [--rehearse <step>] [--no-gate] [--plant <case>]
+//        sanctuary-butler-upgrade.mjs verify [--gate]
 
-import { execFileSync } from "node:child_process"
+import { execFileSync, spawnSync } from "node:child_process"
 import { createHash } from "node:crypto"
-import { mkdtempSync, rmSync, writeFileSync, readFileSync, existsSync, statSync, chmodSync } from "node:fs"
+import { mkdtempSync, rmSync, writeFileSync, readFileSync, existsSync, statSync, chmodSync, realpathSync } from "node:fs"
+import { fileURLToPath } from "node:url"
 import { tmpdir } from "node:os"
 import { join } from "node:path"
 
@@ -61,6 +71,10 @@ const UPGRADE_JOURNAL = `${ROOT}/upgrade.json`
 const MAINTENANCE = "/run/ouro-authority-maintenance"
 // The stop hook's flag: every keeper watchdog version stands down while it exists.
 const SHUTDOWN_FLAG = "/run/ouro-authority-shutdown"
+const GATE_SCRIPT = `${ROOT}/package/deploy/unraid/sanctuary-replay-gate.mjs`
+const GATE_PROVISION = `${BUNDLE}/state/replay-client/provision.json`
+// The whole gate (every case, including the six-minute await wait) must finish well inside this; a hung gate is a failed gate.
+export const GATE_TIMEOUT_MS = 45 * 60 * 1000
 const UPGRADE_STEPS = ["stop", "switch", "resident", "migrate", "start"]
 
 const sh = (file, args, opts = {}) => execFileSync(file, args, { encoding: "utf8", maxBuffer: 64 << 20, ...opts })
@@ -845,9 +859,86 @@ function lifecycleFailure(e) {
   return `${detail || e.message}${last ? `\n   last root failure: ${last}` : ""}`
 }
 
-function upgrade(version, rehearse) {
+/**
+ * The end of a gated upgrade, with every side effect injected so the decision is testable: a pass commits the held
+ * upgrade, pins the template and prunes; a fail rolls back to the predecessor and confirms it is healthy. Nothing is
+ * committed, pinned or pruned unless the gate passed.
+ */
+export function completeGatedUpgrade({ runGate, commit, rollback, confirmRollback, pin, prune, log = console.log }) {
+  const gate = runGate()
+  if (gate.ok) {
+    commit()
+    pin()
+    prune()
+    log("GATE PASS")
+    return { passed: true }
+  }
+  log(`gate failed: ${gate.detail ?? "see the case lines above"}`)
+  rollback()
+  const restored = confirmRollback()
+  log(`GATE FAIL — ROLLED BACK to ${restored}`)
+  return { passed: false, rolledBackTo: restored }
+}
+
+export function runReplayGate(plant, { spawn = spawnSync, exists = existsSync } = {}) {
+  if (!exists(GATE_SCRIPT)) return { ok: false, detail: `the replay gate is missing from the live package (${GATE_SCRIPT})` }
+  const result = spawn("/usr/local/bin/node", [GATE_SCRIPT, "run", ...(plant ? ["--plant", plant] : [])], { stdio: "inherit", timeout: GATE_TIMEOUT_MS, killSignal: "SIGKILL" })
+  return result.status === 0 ? { ok: true } : { ok: false, detail: `the replay gate exited ${result.status ?? result.signal}` }
+}
+
+/**
+ * The commit step runs the lifecycle's `upgrade` again, which would START A FRESH UNGATED UPGRADE if the keeper had
+ * rolled the held one back mid-gate. So commit only while the journal still exists and the new image is still live.
+ */
+export function heldUpgradeProblem({ journalExists, liveImage, expectedImage }) {
+  if (!journalExists) return "the held upgrade's journal is gone (the keeper or a boot rolled it back mid-gate)"
+  if (liveImage !== expectedImage) return `the live Butler is ${liveImage}, not the held ${expectedImage}`
+  return null
+}
+
+function butlerState() {
+  try { return docker(["inspect", CONTAINER, "--format", "{{.Config.Image}} {{.State.Status}}/{{.State.Health.Status}}"], { stdio: ["ignore", "pipe", "ignore"] }).trim() } catch { return "? ?/?" }
+}
+
+function waitForButler(imageRef, seconds) {
+  for (let waited = 0; waited <= seconds; waited += 10) {
+    const [liveImage, state] = butlerState().split(" ")
+    if (liveImage === imageRef && state === "running/healthy") return true
+    sh("/bin/sleep", ["10"])
+  }
+  return false
+}
+
+// The new gateway asks Telegram who it is (getMe) before it reports ready. When the host cannot reach
+// Telegram, that fails and the whole upgrade rolls back after host supervision was already paused
+// (the 863 -> 864 upgrade). Any HTTP answer from the API host proves the network path; no token is sent.
+export const TELEGRAM_API_URL = "https://api.telegram.org/"
+export async function telegramApiReachable({ fetchImpl = globalThis.fetch, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), tries = 3, timeoutMs = 8000, retryDelayMs = 3000 } = {}) {
+  let detail = "no attempt made"
+  for (let attempt = 1; attempt <= tries; attempt += 1) {
+    try {
+      const response = await fetchImpl(TELEGRAM_API_URL, { method: "GET", signal: AbortSignal.timeout(timeoutMs) })
+      return { ok: true, detail: `HTTP ${response.status}` }
+    } catch (error) {
+      detail = error instanceof Error ? (error.cause instanceof Error ? `${error.message}: ${error.cause.message}` : error.message) : String(error)
+      if (attempt < tries) await sleep(retryDelayMs)
+    }
+  }
+  return { ok: false, detail: `${detail} (${tries} attempts)` }
+}
+
+// A resume or rollback (an upgrade journal exists) must never be blocked by a network probe.
+export async function checkTelegramBeforePause({ journalExists, probe = telegramApiReachable }) {
+  if (journalExists) return { skipped: true, ok: true, detail: "upgrade journal present" }
+  return { skipped: false, ...(await probe()) }
+}
+
+async function upgrade(version, rehearse, { noGate = false, plant } = {}) {
   say(`in-place upgrade to ${version}${rehearse ? ` — REHEARSAL: stop after "${rehearse}", then roll back` : ""}`)
   if (rehearse && !UPGRADE_STEPS.includes(rehearse)) fail(`--rehearse takes one of: ${UPGRADE_STEPS.join(", ")}`)
+  const gated = !rehearse && !noGate
+  if (noGate && !rehearse) console.log("\n!! --no-gate: this upgrade will NOT be replay-gated or auto-rolled-back on a behavioural regression")
+  if (gated && !existsSync(GATE_PROVISION)) fail(`replay peers are not provisioned (${GATE_PROVISION}); run \`sanctuary-replay-gate.mjs provision\` first, or pass --no-gate`)
   if (!existsSync(`${ROOT}/active.json`) || !existsSync(`${ROOT}/activation.json`)) fail("no installed authority; use prepare + install")
   if (existsSync(JOURNAL)) fail("a template transaction is pending; resolve it first")
   let id = imageId(version)
@@ -867,13 +958,18 @@ function upgrade(version, rehearse) {
     ok("new package staged beside the live one and verified")
   }
   const lifecycle = `${ROOT}/incoming-package/dist/heart/daemon/sanctuary-authority-root-lifecycle.js`
+  const priorImage = butlerState().split(" ")[0]
   const jellyfinBefore = docker(["inspect", "jellyfin", "--format", "{{.Id}}|{{.Image}}|{{.RestartCount}}|{{.State.StartedAt}}"]).trim()
   const policyBefore = existsSync(POLICY) ? sha12(POLICY) : fail("steward policy missing")
+  const telegram = await checkTelegramBeforePause({ journalExists: existsSync(UPGRADE_JOURNAL) })
+  if (!telegram.ok) fail(`api.telegram.org is not reachable from this host (${telegram.detail}); the new gateway could not pass its Telegram identity check. Nothing was paused or changed; fix the network and re-run.`)
+  if (telegram.skipped) console.log("  note: resuming a pending upgrade; Telegram reachability not re-checked")
+  else ok(`api.telegram.org reachable (${telegram.detail})`)
   pauseSupervision()
   let failure = null
   try {
-    say("lifecycle upgrade (stop → switch → resident → migrate → start)")
-    ok(sh("/usr/local/bin/node", [lifecycle, "upgrade", id, image(version), ...(rehearse ? ["--fail-after", rehearse] : [])], { stdio: ["ignore", "pipe", "pipe"] }).trim())
+    say(`lifecycle upgrade (stop → switch → resident → migrate → start${gated ? ", held uncommitted for the gate" : ""})`)
+    ok(sh("/usr/local/bin/node", [lifecycle, "upgrade", id, image(version), ...(rehearse ? ["--fail-after", rehearse] : gated ? ["--hold-commit"] : [])], { stdio: ["ignore", "pipe", "pipe"] }).trim())
   } catch (e) { failure = e }
   // Only the lifecycle's own rehearsal stop is planned; anything else is a real failure,
   // even during a rehearsal (the first 830 -> 835 rehearsal was mislabelled "as planned").
@@ -886,20 +982,58 @@ function upgrade(version, rehearse) {
       resumeSupervision()
       fail(`ROLLBACK FAILED: ${lifecycleFailure(e)}\nThe journal is kept; the next authority boot (or \`upgrade-rollback\`) retries it.`)
     }
-  } else { pinTemplate(version); pruneButlerImages() }
+  } else if (!gated) { pinTemplate(version); pruneButlerImages() }
   resumeSupervision()
   say("preservation")
-  docker(["inspect", "jellyfin", "--format", "{{.Id}}|{{.Image}}|{{.RestartCount}}|{{.State.StartedAt}}"]).trim() === jellyfinBefore ? ok("jellyfin unchanged") : fail("JELLYFIN CHANGED")
-  sha12(POLICY) === policyBefore ? ok("steward policy unchanged") : fail("STEWARD POLICY CHANGED")
+  const preserved = []
+  const jellyfinNow = docker(["inspect", "jellyfin", "--format", "{{.Id}}|{{.Image}}|{{.RestartCount}}|{{.State.StartedAt}}"]).trim()
+  jellyfinNow === jellyfinBefore ? ok("jellyfin unchanged") : (gated && !failure ? (bad("JELLYFIN CHANGED"), preserved.push("jellyfin changed")) : fail("JELLYFIN CHANGED"))
+  sha12(POLICY) === policyBefore ? ok("steward policy unchanged") : (gated && !failure ? (bad("STEWARD POLICY CHANGED"), preserved.push("steward policy changed")) : fail("STEWARD POLICY CHANGED"))
   const live = docker(["inspect", CONTAINER, "--format", "{{.Config.Image}} {{.State.Status}}/{{.State.Health.Status}}"]).trim()
   ok(`butler: ${live}`)
   if (failure && !planned) fail("upgrade rolled back; the Butler is on its prior version (see above)")
-  console.log(rehearse ? "\nREHEARSAL done — the upgrade reached the planned step and the rollback restored the predecessor." : `\nUPGRADE done — ${version} live. Run verify.`)
+  if (gated && !failure) {
+    say(`replay gate${plant ? ` (planted failure: ${plant})` : ""}`)
+    const stillHeld = () => heldUpgradeProblem({ journalExists: existsSync(UPGRADE_JOURNAL), liveImage: butlerState().split(" ")[0], expectedImage: image(version) })
+    const outcome = completeGatedUpgrade({
+      runGate: () => {
+        if (preserved.length) return { ok: false, detail: preserved.join("; ") }
+        if (!waitForButler(image(version), 300)) return { ok: false, detail: `the new Butler did not become healthy on ${image(version)}` }
+        const result = runReplayGate(plant)
+        if (!result.ok) return result
+        const problem = stillHeld()
+        return problem ? { ok: false, detail: problem } : result
+      },
+      commit: () => {
+        const problem = stillHeld()
+        if (problem) throw new Error(`refusing to commit: ${problem}`)
+        pauseSupervision()
+        try { ok(`commit: ${sh("/usr/local/bin/node", [lifecycle, "upgrade", id, image(version)], { stdio: ["ignore", "pipe", "pipe"] }).trim()}`) }
+        finally { resumeSupervision() }
+      },
+      rollback: () => {
+        pauseSupervision()
+        try { ok(`rollback: ${sh("/usr/local/bin/node", [lifecycle, "upgrade-rollback"], { stdio: ["ignore", "pipe", "pipe"] }).trim()}`) }
+        catch (e) { fail(`ROLLBACK FAILED: ${lifecycleFailure(e)}\nThe journal is kept; the next authority boot (or \`upgrade-rollback\`) retries it.`) }
+        finally { resumeSupervision() }
+      },
+      confirmRollback: () => {
+        if (!waitForButler(priorImage, 300)) fail(`rolled back, but ${priorImage} is not running healthy; inspect \`docker logs ${CONTAINER}\``)
+        return priorImage
+      },
+      pin: () => pinTemplate(version),
+      prune: () => pruneButlerImages(),
+    })
+    if (!outcome.passed) process.exit(1)
+    console.log(`\nUPGRADE done — ${version} live and gate-verified. Run verify.`)
+    return
+  }
+  console.log(rehearse ? "\nREHEARSAL done — the upgrade reached the planned step and the rollback restored the predecessor." : `\nUPGRADE done — ${version} live (NOT gate-verified). Run verify.`)
 }
 
 // ---- verify ---------------------------------------------------------------
 
-function verify() {
+function verify(withGate = false) {
   say("verify")
   const st = docker(["inspect", CONTAINER, "--format", "{{.Config.Image}} {{.State.Status}}/{{.State.Health.Status}} restarts={{.RestartCount}}"]).trim()
   ok(`butler: ${st}`)
@@ -908,6 +1042,11 @@ function verify() {
   existsSync(POLICY) ? ok(`steward policy ${sha12(POLICY)}`) : bad("steward policy missing")
   const jf = docker(["inspect", "jellyfin", "--format", "{{.State.Status}} restarts={{.RestartCount}}"]).trim()
   ok(`jellyfin ${jf}`)
+  if (withGate) {
+    say("replay gate (live version, no upgrade)")
+    const gate = runReplayGate(undefined)
+    gate.ok ? ok("replay gate passed") : bad(`replay gate failed: ${gate.detail}`)
+  }
   process.exit(RED === 0 ? 0 : 1)
 }
 
@@ -915,19 +1054,35 @@ function fail(m) { console.error(`\nREFUSING: ${m}`); process.exit(1) }
 
 // ---- entry ----------------------------------------------------------------
 
-const [phase, version, flag, rehearse] = process.argv.slice(2)
-if (!phase || !["preflight", "prepare", "install", "verify", "upgrade"].includes(phase) || (flag !== undefined && !(phase === "upgrade" && flag === "--rehearse" && rehearse))) {
-  console.error("Usage: sanctuary-butler-upgrade.mjs <preflight|prepare|install|verify|upgrade> <version> [--rehearse <step>]")
-  process.exit(2)
+export function parseUpgradeArgs(argv) {
+  const usage = "Usage: sanctuary-butler-upgrade.mjs <preflight|prepare|install|upgrade> <version> [--rehearse <step>] [--no-gate] [--plant <case>] | verify [--gate]"
+  const [phase, ...rest] = argv
+  if (!phase || !["preflight", "prepare", "install", "verify", "upgrade"].includes(phase)) throw new Error(usage)
+  const parsed = { phase, version: undefined, rehearse: undefined, noGate: false, plant: undefined, gate: false }
+  for (let i = 0; i < rest.length; i += 1) {
+    const arg = rest[i]
+    if (arg === "--rehearse" && phase === "upgrade" && rest[i + 1]) parsed.rehearse = rest[++i]
+    else if (arg === "--no-gate" && phase === "upgrade") parsed.noGate = true
+    else if (arg === "--plant" && phase === "upgrade" && rest[i + 1]) parsed.plant = rest[++i]
+    else if (arg === "--gate" && phase === "verify") parsed.gate = true
+    else if (!arg.startsWith("--") && parsed.version === undefined && phase !== "verify") parsed.version = arg
+    else throw new Error(usage)
+  }
+  if (parsed.noGate && parsed.plant) throw new Error("--plant needs the gate; it cannot be combined with --no-gate")
+  if (parsed.rehearse && (parsed.noGate || parsed.plant)) throw new Error("--rehearse does not use the gate; do not combine it with --no-gate or --plant")
+  if (phase !== "verify" && !/^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$/.test(parsed.version || "")) throw new Error("a semver version is required, e.g. 0.1.0-alpha.829")
+  return parsed
 }
-if (phase !== "verify" && !/^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$/.test(version || "")) {
-  console.error("a semver version is required, e.g. 0.1.0-alpha.829")
-  process.exit(2)
+
+async function run(argv) {
+  let args
+  try { args = parseUpgradeArgs(argv) } catch (error) { console.error(error.message); process.exit(2) }
+  requireRoot()
+  if (args.phase === "preflight") preflight(args.version)
+  else if (args.phase === "prepare") prepare(args.version)
+  else if (args.phase === "install") install(args.version)
+  else if (args.phase === "verify") verify(args.gate)
+  else if (args.phase === "upgrade") await upgrade(args.version, args.rehearse, { noGate: args.noGate, plant: args.plant })
 }
-requireRoot()
 
-if (phase === "preflight") preflight(version)
-else if (phase === "prepare") prepare(version)
-else if (phase === "install") install(version)
-else if (phase === "verify") verify()
-else if (phase === "upgrade") upgrade(version, rehearse)
+if (process.argv[1] && fileURLToPath(import.meta.url) === realpathSync(process.argv[1])) run(process.argv.slice(2)).catch((error) => { console.error(error); process.exit(1) })