@ouro.bot/cli 0.1.0-alpha.771 → 0.1.0-alpha.772

package/changelog.json

@@ -1,6 +1,14 @@
 {
   "_note": "This changelog is maintained as part of the PR/version-bump workflow. Agent-curated, not auto-generated. Agents read this file directly via read_file to understand what changed between versions.",
   "versions": [
+    {
+      "version": "0.1.0-alpha.772",
+      "changes": [
+        "Make Sanctuary Unraid key rotation crash-resumable with exact transaction and rejection proofs.",
+        "Require durable zero-playback journal evidence before live callback injection.",
+        "Reject malformed Unraid rotation checkpoints before mutation with exact slot, attestation, revocation-order, and phase-state validation."
+      ]
+    },
     {
       "version": "0.1.0-alpha.771",
       "changes": [

package/deploy/unraid/README.txt

@@ -2440,6 +2440,9 @@ Packaged Unit 16 acceptance execution:
   failure. It never reads or changes Unraid autostart configuration. Every command
   also receives a freshly generated, redacted typed container-inspect snapshot;
   raw container environment or credential values are never captured.
+  Callback injection requires two stable zero observations from a durable callback playback journal under
+  `state/approvals`; the journal is keyed only by the full callback-coordinate
+  digest and stores no raw update, callback data, user, chat, message, or query ID.
   Evidence-snapshot keeps the bundle mount read-only and overlays only the exact
   canonical `state/acceptance` directory as writable. It first stops the exact
   healthy staging container under recovery responsibility, then opens the
@@ -2459,17 +2462,40 @@ Audit and safety verification:
   stop and restart mutations; only the separate write key may perform the one
   typed approved restart action.
   The packaged `unraid-key-rotate` command is the only canonical key-rotation
-  authority. It inventories the two occupied canonical names by exact immutable
-  ID, creates collision-safe `Butler RO Rotation <suffix>` and
+  authority. Its host launcher stops the exact production container and assumes
+  trap-backed recovery responsibility before key mutation. It inventories the
+  two occupied canonical names by exact immutable ID, creates collision-safe
+  `Butler RO Rotation <suffix>` and
   `Butler RW Rotation <suffix>` keys, stores and capability-probes both through
-  the Sanctuary machine vault, and requires an exact four-key inventory. It then
+  the Sanctuary machine vault, and on the uninterrupted path requires an exact
+  four-key inventory. Because
+  Unraid 7.2.3 cannot create a role-free key through its CLI, the root broker
+  creates each key as provisional `GUEST` with the intended scope plus only
+  `API_KEY:UPDATE_ANY`, then immediately uses that in-memory credential to
+  self-downgrade the same immutable ID to no roles and the exact intended scope.
+  Disk and live-auth proofs must pass before the credential may reach the vault.
+  The provisional credential never reaches argv, logs, evidence, the bundle, or
+  the model. It then
   revokes both old IDs and proves each old credential receives a 401 or 403.
   Only after an exact inventory contains the temporary pair alone does it create,
   store, and probe the canonical `Butler RO` and `Butler RW` pair. It requires an
-  exact four-key temporary-plus-canonical inventory, revokes and rejection-probes
+  exact four-key temporary-plus-canonical inventory on the uninterrupted path,
+  revokes and rejection-probes
   both temporary IDs, and finally requires exactly the canonical pair. Every
   unexpected key, duplicate, malformed permission set, role, name, inventory
-  delta, or probe response fails closed.
+  delta, or probe response fails closed. A resumed path may observe fewer keys
+  only when the transaction checkpoint plus a durable rejection receipt proves
+  that the missing immutable ID was already revoked; it never treats absence
+  alone as success.
+
+  A failed invocation resumes from the durable redacted transaction checkpoint
+  instead of reminting known keys. The checkpoint binds every created immutable
+  ID to its name, scope, vault field, temporary/canonical class, and attestation
+  state. Recovery reconciles exact live inventory and rejection proofs, skips
+  already-attested vault writes after canonical fields advance, and never
+  recreates a temporary key already known revoked. Root-owned recovery ownership
+  survives broker response loss and is cleared only after the harness has
+  checkpointed the immutable ID and the vault has read back the exact binding.
 
   Record only IDs, names, and permission sets in evidence; never raw key values.
   Raw credentials must not appear in argv, shell history, stdout, logs, or
@@ -2477,7 +2503,9 @@ Audit and safety verification:
   socket in process memory. The narrow revocation-retry recovery records under
   `/mnt/user/appdata/ouro-butler/acceptance/revoked-key-proof` are the sole file
   exception: they are root-owned mode 0600 beneath a root-owned mode 0700
-  directory, excluded from backup/evidence, streamed only on standard input to
-  the bounded rejection probe, retained across failed retries, and durably
-  deleted only after the exact authentication rejection is validated. Never
-  substitute names for IDs when revoking keys.
+  directory, excluded from backup/evidence, and readable only by the root broker.
+  Recovery state is retained across failed retries.
+  Creation intent is fsynced before the CLI side effect so an exact provisional
+  key can be adopted after process death. Raw recovery proof is durably deleted
+  only after the exact authentication rejection and its redacted replayable
+  receipt are fsynced. Never substitute names for IDs when revoking keys.

package/deploy/unraid/sanctuary-acceptance-contract.json

@@ -31,7 +31,7 @@
     "telegram-poller-quiescence": { "operation": "quiesce_telegram_poller", "authority": "fixed-managed-telegram-poller-stop-and-count", "modelReachable": false, "timeoutMs": 15000 },
     "telegram-vault-store": { "operation": "store_telegram_bootstrap", "authority": "fixed-runtime-vault-merge", "modelReachable": false, "timeoutMs": 15000 },
     "cursor-snapshot": { "operation": "snapshot", "authority": "fixed-telegram-state-read", "modelReachable": false, "timeoutMs": 15000 },
-    "callback-live": { "operation": "inject_callbacks_concurrently|inject_callback_replay", "authority": "fixed-live-telegram-callback-probe", "modelReachable": false, "timeoutMs": 15000 },
+    "callback-live": { "operation": "callback_playback_preflight|inject_callbacks_concurrently|inject_callback_replay", "authority": "fixed-live-telegram-callback-probe", "modelReachable": false, "timeoutMs": 15000 },
     "key-inventory": { "operation": "inventory_keys", "authority": "fixed-unraid-key-directory-read", "modelReachable": false, "timeoutMs": 15000 },
     "key-create": { "operation": "create_key", "authority": "fixed-unraid-cli-create-and-vault-store", "modelReachable": false, "timeoutMs": 15000 },
     "key-store": { "operation": "store_key", "authority": "fixed-machine-vault-readback", "modelReachable": false, "timeoutMs": 15000 },

package/deploy/unraid/sanctuary-unit16-host-broker.mjs

@@ -149,27 +149,113 @@ function runUnraid(args) {
   return object(JSON.parse(result.stdout), "Unraid API response")
 }
 
-function createUnraidKey(name, requested, dependencies = { runUnraid, inventoryRecords, persistRecovery }) {
+function permissionObjects(values) {
+  const grouped = new Map()
+  for (const value of values) {
+    const [resource, action] = value.split(":")
+    const actions = grouped.get(resource) ?? []
+    actions.push(action)
+    grouped.set(resource, actions)
+  }
+  return [...grouped.entries()].sort(([left], [right]) => left.localeCompare(right))
+    .map(([resource, actions]) => ({ resource, actions: actions.sort() }))
+}
+
+async function createUnraidKey(name, requested, dependencies = {
+  runUnraid, inventoryRecords, persistRecovery, readRecovery: optionalRecoveryRecord,
+  persistIntent: persistCreateIntent, readIntent: optionalCreateIntent, removeIntent: removeCreateIntent,
+  fetchImpl: fetch,
+}) {
   const before = dependencies.inventoryRecords()
   const beforeIds = new Set(before.map((record) => record.id))
+  const provisionalPermissions = [...requested, "API_KEY:UPDATE_ANY"].sort()
+  let recoveryId = null
+  let cliIdentity = null
   try {
-    const created = object(dependencies.runUnraid(["apikey", "--name", name, "--create", "--permissions", requested.join(","), "--json"]), "created Unraid key")
-    exactKeys(created, ["id", "key", "name"], "created Unraid key")
+    const occupied = before.filter((record) => record.name === name)
+    const existingIntent = dependencies.readIntent?.(name) ?? null
+    if (existingIntent === null) {
+      if (occupied.length !== 0) throw new Error("existing provisional key has no durable creation intent")
+      dependencies.persistIntent(name, provisionalPermissions)
+    } else if (existingIntent.name !== name || JSON.stringify(existingIntent.permissions) !== JSON.stringify(provisionalPermissions)) {
+      throw new Error("Unraid key creation intent is invalid")
+    }
+    let created
+    let provisional
+    let resumed = false
+    let resumedFinal = false
+    if (occupied.length === 1) {
+      provisional = occupied[0]
+      let recovery = dependencies.readRecovery?.(provisional.id) ?? null
+      const provisionalExact = JSON.stringify(flattened(provisional)) === JSON.stringify(provisionalPermissions)
+        && JSON.stringify(provisional.roles) === JSON.stringify(["GUEST"])
+      if (recovery === null && provisionalExact) {
+        dependencies.persistRecovery(provisional)
+        recovery = provisional
+      }
+      if (!recovery || recovery.id !== provisional.id || recovery.name !== provisional.name || recovery.key !== provisional.key
+        || JSON.stringify(flattened(recovery)) !== JSON.stringify(provisionalPermissions)
+        || JSON.stringify(recovery.roles) !== JSON.stringify(["GUEST"])
+        || !(provisionalExact
+          || (JSON.stringify(flattened(provisional)) === JSON.stringify(requested) && provisional.roles.length === 0))) {
+        throw new Error("existing provisional key is not exact owned recovery")
+      }
+      resumedFinal = provisional.roles.length === 0
+      created = { id: provisional.id, key: provisional.key, name: provisional.name }
+      resumed = true
+    } else if (occupied.length === 0) {
+      created = object(dependencies.runUnraid(["apikey", "--name", name, "--create", "--roles", "GUEST", "--permissions", provisionalPermissions.join(","), "--json"]), "created Unraid key")
+      exactKeys(created, ["id", "key", "name"], "created Unraid key")
+      cliIdentity = { id: text(created.id, "created key id", KEY_ID), key: text(created.key, "created key descriptor"), name: created.name }
+    } else {
+      throw new Error("created provisional key name is ambiguous")
+    }
     const id = text(created.id, "created key id", KEY_ID)
     const key = text(created.key, "created key descriptor")
-    if (created.name !== name || beforeIds.has(id)) throw new Error("created key identity is invalid")
-    const matches = dependencies.inventoryRecords().filter((record) => record.id === id)
+    if (created.name !== name || (!resumed && beforeIds.has(id))) throw new Error("created key identity is invalid")
+    const matches = resumed ? [provisional] : dependencies.inventoryRecords().filter((record) => record.id === id)
     if (matches.length !== 1 || matches[0].name !== name || matches[0].key !== key
-      || JSON.stringify(flattened(matches[0])) !== JSON.stringify(requested) || matches[0].roles.length !== 0) {
-      throw new Error("created key record does not match the exact CLI result")
+      || (!resumedFinal && (JSON.stringify(flattened(matches[0])) !== JSON.stringify(provisionalPermissions)
+        || JSON.stringify(matches[0].roles) !== JSON.stringify(["GUEST"])))) {
+      throw new Error("created provisional key record does not match the exact CLI result")
     }
-    return matches[0]
-  } catch (error) {
-    const recoveredIds = new Set()
-    for (const record of dependencies.inventoryRecords().filter((candidate) => !beforeIds.has(candidate.id))) {
-      if (!recoveredIds.has(record.id)) dependencies.persistRecovery(record)
-      recoveredIds.add(record.id)
+    if (!resumed) dependencies.persistRecovery(matches[0])
+    recoveryId = id
+    const headers = { "content-type": "application/json", "x-api-key": key }
+    if (!resumedFinal) try {
+      const updateResponse = await dependencies.fetchImpl(GRAPHQL_ENDPOINT, {
+        method: "POST", headers,
+        body: JSON.stringify({
+          query: "mutation AcceptanceApiKeyDowngrade($input: UpdateApiKeyInput!) { apiKey { update(input: $input) { id name roles permissions { resource actions } } } }",
+          variables: { input: { id, roles: [], permissions: permissionObjects(requested) } },
+        }),
+        signal: AbortSignal.timeout(10_000),
+      })
+      const updateEnvelope = object(await updateResponse.json(), "Unraid key downgrade response")
+      if (!updateResponse.ok || updateEnvelope.errors || !updateEnvelope.data) throw new Error("Unraid key downgrade was rejected")
+    } catch {
+      // A lost response is accepted only when disk and live-auth attestation below prove convergence.
     }
+    const finalized = dependencies.inventoryRecords().filter((record) => record.id === id)
+    if (finalized.length !== 1 || finalized[0].name !== name || finalized[0].key !== key
+      || JSON.stringify(flattened(finalized[0])) !== JSON.stringify(requested) || finalized[0].roles.length !== 0) {
+      throw new Error("created key did not converge to its exact final scope")
+    }
+    const probeResponse = await dependencies.fetchImpl(GRAPHQL_ENDPOINT, {
+      method: "POST", headers,
+      body: JSON.stringify({ query: "query AcceptanceAuthProbe { info { os { hostname } } }", variables: {} }),
+      signal: AbortSignal.timeout(10_000),
+    })
+    const probeEnvelope = object(await probeResponse.json(), "Unraid key readiness response")
+    if (!probeResponse.ok || probeEnvelope.errors || !probeEnvelope.data) throw new Error("created key readiness probe failed")
+    return finalized[0]
+  } catch (error) {
+    const candidates = dependencies.inventoryRecords().filter((record) => record.name === name
+      && !beforeIds.has(record.id)
+      && JSON.stringify(flattened(record)) === JSON.stringify(provisionalPermissions)
+      && JSON.stringify(record.roles) === JSON.stringify(["GUEST"])
+      && (cliIdentity === null || (record.id === cliIdentity.id && record.key === cliIdentity.key && record.name === cliIdentity.name)))
+    if (candidates.length === 1 && candidates[0].id !== recoveryId) dependencies.persistRecovery(candidates[0])
     throw error
   }
 }
@@ -570,27 +656,149 @@ function containerRestartSnapshot(expectedImage) {
 }
 
 function recoveryPath(id) { return `${RECOVERY_ROOT}/${id}.json` }
+function createIntentPath(name) { return `${RECOVERY_ROOT}/create-${createHash("sha256").update(name).digest("hex")}.json` }
+function rejectionReceiptPath(id) { return `${RECOVERY_ROOT}/${id}.rejected.json` }
 
-function persistRecovery(record) {
+function persistPrivateJson(target, value) {
   mkdirSync(RECOVERY_ROOT, { recursive: true, mode: 0o700 })
   chmodSync(RECOVERY_ROOT, 0o700)
   chownSync(RECOVERY_ROOT, 0, 0)
-  const target = recoveryPath(record.id)
   const temporary = `${target}.tmp-${process.pid}`
   const fd = openSync(temporary, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL, 0o600)
-  try { writeFileSync(fd, `${JSON.stringify(record)}\n`); fsyncSync(fd) } finally { closeSync(fd) }
-  chownSync(temporary, 0, 0)
+  try { chownSync(temporary, 0, 0); writeFileSync(fd, `${JSON.stringify(value)}\n`); fsyncSync(fd) } finally { closeSync(fd) }
   renameSync(temporary, target)
   const directory = openSync(RECOVERY_ROOT, constants.O_RDONLY)
   try { fsyncSync(directory) } finally { closeSync(directory) }
 }
 
+function optionalPrivateJson(target) {
+  try { return object(readPrivateJson(target), "private acceptance record") }
+  catch (error) { if (error?.code === "ENOENT") return null; throw error }
+}
+
+function persistCreateIntent(name, requested) {
+  mkdirSync(RECOVERY_ROOT, { recursive: true, mode: 0o700 })
+  chmodSync(RECOVERY_ROOT, 0o700)
+  chownSync(RECOVERY_ROOT, 0, 0)
+  const target = createIntentPath(name)
+  const fd = openSync(target, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL, 0o600)
+  try {
+    chownSync(target, 0, 0)
+    writeFileSync(fd, `${JSON.stringify({ schemaVersion: 1, name, permissions: requested })}\n`)
+    fsyncSync(fd)
+  } finally { closeSync(fd) }
+  const directory = openSync(RECOVERY_ROOT, constants.O_RDONLY)
+  try { fsyncSync(directory) } finally { closeSync(directory) }
+}
+
+function optionalCreateIntent(name) {
+  const value = optionalPrivateJson(createIntentPath(name))
+  if (value === null) return null
+  exactKeys(value, ["schemaVersion", "name", "permissions"], "Unraid key creation intent")
+  if (value.schemaVersion !== 1 || value.name !== name) throw new Error("Unraid key creation intent is invalid")
+  return { name, permissions: permissions(value.permissions) }
+}
+
+function removePrivateFile(target) {
+  try { unlinkSync(target) } catch (error) { if (error?.code !== "ENOENT") throw error }
+  const directory = openSync(RECOVERY_ROOT, constants.O_RDONLY)
+  try { fsyncSync(directory) } finally { closeSync(directory) }
+}
+
+function removeCreateIntent(name) { removePrivateFile(createIntentPath(name)) }
+
+function optionalRejectionReceipt(id) {
+  const value = optionalPrivateJson(rejectionReceiptPath(id))
+  if (value === null) return null
+  exactKeys(value, ["schemaVersion", "id", "status", "proofDigest"], "revoked key rejection receipt")
+  if (value.schemaVersion !== 1 || value.id !== id || (value.status !== 401 && value.status !== 403)
+    || value.proofDigest !== createHash("sha256").update(`revoked-key-rejected\0${id}\0${value.status}`).digest("hex")) {
+    throw new Error("revoked key rejection receipt is invalid")
+  }
+  return { valid: false, status: value.status, id }
+}
+
+function persistRejectionReceipt(id, status) {
+  persistPrivateJson(rejectionReceiptPath(id), {
+    schemaVersion: 1, id, status,
+    proofDigest: createHash("sha256").update(`revoked-key-rejected\0${id}\0${status}`).digest("hex"),
+  })
+}
+
+async function probeRevokedUnraidKey(id, dependencies = {
+  readReceipt: optionalRejectionReceipt,
+  readRecovery: optionalRecoveryRecord,
+  persistReceipt: persistRejectionReceipt,
+  removeRecovery,
+  fetchImpl: fetch,
+}) {
+  text(id, "key id", KEY_ID)
+  const receipt = dependencies.readReceipt(id)
+  if (receipt !== null) {
+    const staleRecovery = dependencies.readRecovery(id)
+    if (staleRecovery !== null) {
+      if (staleRecovery.id !== id) throw new Error("recovery key id is invalid")
+      dependencies.removeRecovery(id)
+    }
+    return receipt
+  }
+  const record = dependencies.readRecovery(id)
+  if (record === null || record.id !== id) throw new Error("recovery key id is invalid")
+  const response = await dependencies.fetchImpl(GRAPHQL_ENDPOINT, {
+    method: "POST", headers: { "content-type": "application/json", "x-api-key": record.key },
+    body: JSON.stringify({ query: "query AcceptanceAuthProbe { info { os { hostname } } }", variables: {} }),
+    signal: AbortSignal.timeout(10_000),
+  })
+  if (response.status !== 401 && response.status !== 403) throw new Error("revoked key still authenticates")
+  dependencies.persistReceipt(id, response.status)
+  dependencies.removeRecovery(id)
+  return { valid: false, status: response.status, id }
+}
+
+function optionalRecoveryRecord(id) {
+  try { return normalizeRecord(readPrivateJson(recoveryPath(id))) }
+  catch (error) { if (error?.code === "ENOENT") return null; throw error }
+}
+
+function persistRecovery(record) {
+  persistPrivateJson(recoveryPath(record.id), record)
+}
+
 function removeRecovery(id) {
   unlinkSync(recoveryPath(id))
   const directory = openSync(RECOVERY_ROOT, constants.O_RDONLY)
   try { fsyncSync(directory) } finally { closeSync(directory) }
 }
 
+function acknowledgeStoredUnraidKey(id, dependencies = {
+  readRecovery: optionalRecoveryRecord,
+  inventoryRecords,
+  removeRecovery,
+  removeIntent: removeCreateIntent,
+}) {
+  text(id, "key id", KEY_ID)
+  const matches = dependencies.inventoryRecords().filter((record) => record.id === id)
+  if (matches.length !== 1 || !KEY_NAME.test(matches[0].name)) throw new Error("Unraid key final storage acknowledgement is invalid")
+  const expected = matches[0].name.startsWith("Butler RO") ? RO_PERMISSIONS : [...RO_PERMISSIONS, "DOCKER:UPDATE_ANY"].sort()
+  if (matches[0].roles.length !== 0 || JSON.stringify(flattened(matches[0])) !== JSON.stringify(expected)) {
+    throw new Error("Unraid key final storage acknowledgement is invalid")
+  }
+  const recovery = dependencies.readRecovery(id)
+  if (recovery === null) {
+    dependencies.removeIntent?.(matches[0].name)
+    return { acknowledged: true, id }
+  }
+  const provisional = [...expected, "API_KEY:UPDATE_ANY"].sort()
+  if (recovery.id !== id || recovery.name !== matches[0].name || recovery.key !== matches[0].key
+    || JSON.stringify(recovery.roles) !== JSON.stringify(["GUEST"])
+    || JSON.stringify(flattened(recovery)) !== JSON.stringify(provisional)) {
+    throw new Error("Unraid key recovery ownership is invalid")
+  }
+  dependencies.removeRecovery(id)
+  dependencies.removeIntent?.(matches[0].name)
+  return { acknowledged: true, id }
+}
+
 function permissions(value) {
   if (!Array.isArray(value) || value.length === 0) throw new Error("permissions are invalid")
   const result = value.map((item) => text(item, "permission", PERMISSION)).sort()
@@ -1556,7 +1764,12 @@ async function dispatch(request, dependencies = {
     if (payload.targetServerId !== TARGET_SERVER) throw new Error("target server is invalid")
     const name = text(payload.name, "key name", KEY_NAME)
     const requested = permissions(payload.permissions)
-    return createUnraidKey(name, requested)
+    return await createUnraidKey(name, requested)
+  }
+  if (operation === "acknowledge_key_storage") {
+    exactKeys(payload, ["operation", "targetServerId", "keyId"], operation)
+    if (payload.targetServerId !== TARGET_SERVER) throw new Error("target server is invalid")
+    return acknowledgeStoredUnraidKey(text(payload.keyId, "key id", KEY_ID))
   }
   if (operation === "revoke_key") {
     exactKeys(payload, ["operation", "targetServerId", "keyId"], operation)
@@ -1574,17 +1787,7 @@ async function dispatch(request, dependencies = {
   if (operation === "probe_revoked_key") {
     exactKeys(payload, ["operation", "targetServerId", "keyId"], operation)
     if (payload.targetServerId !== TARGET_SERVER) throw new Error("target server is invalid")
-    const id = text(payload.keyId, "key id", KEY_ID)
-    const record = normalizeRecord(readPrivateJson(recoveryPath(id)))
-    if (record.id !== id) throw new Error("recovery key id is invalid")
-    const response = await fetch(GRAPHQL_ENDPOINT, {
-      method: "POST", headers: { "content-type": "application/json", "x-api-key": record.key },
-      body: JSON.stringify({ query: "query AcceptanceAuthProbe { info { os { hostname } } }", variables: {} }),
-      signal: AbortSignal.timeout(10_000),
-    })
-    if (response.status !== 401 && response.status !== 403) throw new Error("revoked key still authenticates")
-    removeRecovery(id)
-    return { valid: false, status: response.status, id }
+    return await probeRevokedUnraidKey(text(payload.keyId, "key id", KEY_ID))
   }
   if (operation === "request_reboot") {
     exactKeys(payload, ["operation", "targetId", "idempotencyKey", "preflightDigest", "processBindingDigest"], operation)
@@ -1721,7 +1924,11 @@ function writeClosedInventory(file) {
 
 function createBrokerServer(dispatchRequest = dispatch) {
   const dispatchDrain = createDispatchDrain()
+  const connections = new Set()
   const server = createServer({ allowHalfOpen: true }, (connection) => {
+    connections.add(connection)
+    connection.setTimeout(30_000, () => connection.destroy())
+    connection.once("close", () => connections.delete(connection))
     let input = ""
     connection.setEncoding("utf8")
     connection.on("data", (chunk) => {
@@ -1737,7 +1944,8 @@ function createBrokerServer(dispatchRequest = dispatch) {
       } catch { connection.end(`${JSON.stringify({ ok: false, error: "host operation failed" })}\n`) }
     })
   })
-  return { server, dispatchDrain }
+  const destroyConnections = () => { for (const connection of connections) connection.destroy() }
+  return { server, dispatchDrain, destroyConnections }
 }
 
 async function main() {
@@ -1754,7 +1962,7 @@ async function main() {
   const snapshotFd = openSync(initialSnapshot, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL, 0o600)
   try { writeFileSync(snapshotFd, `${JSON.stringify(snapshot)}\n`); fsyncSync(snapshotFd) } finally { closeSync(snapshotFd) }
   chownSync(initialSnapshot, 0, 0)
-  const { server, dispatchDrain } = createBrokerServer()
+  const { server, dispatchDrain, destroyConnections } = createBrokerServer()
   server.listen(socket, () => {
     chownSync(socket, 0, 10001)
     chmodSync(socket, 0o660)
@@ -1764,6 +1972,7 @@ async function main() {
     if (shuttingDown) return
     shuttingDown = true
     server.close()
+    destroyConnections()
     await dispatchDrain.stopAndDrain()
     await interactiveRestartDriver.stopAndDrain()
     let failed = false
@@ -1781,6 +1990,7 @@ if (process.argv[1]?.endsWith("sanctuary-unit16-host-broker.mjs")) {
 }
 
 export {
+  acknowledgeStoredUnraidKey,
   assertStableContainerProcess,
   attestHealthProbeProcessAbsent,
   armRestartAfterResponseClosed,
@@ -1805,6 +2015,7 @@ export {
   observeRebootPreflight,
   parseVaultStatus,
   optionalStoppedContainerExact,
+  probeRevokedUnraidKey,
   queryGraphqlAutostart,
   readBoundedProcStatus,
   driveDuplicateCallbacks,

package/deploy/unraid/sanctuary-unit16-run.sh

@@ -50,9 +50,22 @@ assert_health_probe_cleanup() {
   ' "$ACCEPTANCE_STATE_ROOT" "$BUNDLE_ROOT/state/health/sanctuary-health.json.lease"
 }
 
+terminate_broker() {
+  test -n "$BROKER_PID" || return 0
+  kill "$BROKER_PID" 2>/dev/null || true
+  BROKER_WAIT=0
+  while kill -0 "$BROKER_PID" 2>/dev/null && test "$BROKER_WAIT" -lt 10; do
+    sleep 1
+    BROKER_WAIT=$(( BROKER_WAIT + 1 ))
+  done
+  if kill -0 "$BROKER_PID" 2>/dev/null; then kill -KILL "$BROKER_PID" 2>/dev/null || true; fi
+  wait "$BROKER_PID" 2>/dev/null || true
+  BROKER_PID=
+}
+
 cleanup_unit16() {
   CLEANUP_STATUS=$?
-  if test -n "$BROKER_PID"; then kill "$BROKER_PID" 2>/dev/null || true; wait "$BROKER_PID" 2>/dev/null || true; fi
+  terminate_broker
   if test "$ACCEPTANCE_CANONICAL_PINNED" = yes && ! assert_health_probe_cleanup; then CLEANUP_STATUS=1; fi
   if test "$PRODUCTION_STOPPED" = yes && test "$HOST_REBOOT_COMMIT_STATE" = not_sent; then
     if ! restore_production_container; then CLEANUP_STATUS=1; fi
@@ -73,7 +86,10 @@ cleanup_unit16() {
   if test "$ACCEPTANCE_ALIAS_MOUNTED" = no && test -n "$PRIVATE_ROOT" && test -d "$PRIVATE_ROOT"; then rm -rf -- "$PRIVATE_ROOT"; fi
   return "$CLEANUP_STATUS"
 }
-trap cleanup_unit16 EXIT HUP INT TERM
+trap cleanup_unit16 EXIT
+trap 'exit 129' HUP
+trap 'exit 130' INT
+trap 'exit 143' TERM
 
 case "$MODE" in
   materialize)
@@ -347,6 +363,7 @@ cmp -s "$EXPECTED_CONFIG" "$CONFIG_PATH" || exit 1
 case "$COMMAND" in telegram-bootstrap|callback-inject) test -r /proc/self/fd/3 || exit 2 ;; esac
 prepare_live_facts
 if test "$COMMAND" = telegram-bootstrap; then quiesce_production_telegram_poller; fi
+if test "$COMMAND" = unraid-key-rotate; then stop_exact_production_container; fi
 
 case "$COMMAND" in
   telegram-bootstrap) TIME_LIMIT=900; NETWORK=host; INPUT=yes; BUNDLE_MODE=readonly; BROKER=no ;;

package/deploy/unraid/sanctuary.ouro/bundle-meta.json

@@ -1,5 +1,5 @@
 {
-  "runtimeVersion": "0.1.0-alpha.771",
+  "runtimeVersion": "0.1.0-alpha.772",
   "bundleSchemaVersion": 3,
   "lastUpdated": "2026-08-30T00:00:00.000Z"
 }

package/deploy/unraid/sanctuary.xml

@@ -1,7 +1,7 @@
 <?xml version="1.0"?>
 <Container version="2">
   <Name>Mendelow Cloud Butler</Name>
-  <Repository>ghcr.io/ourostack/ouroboros-butler:0.1.0-alpha.771</Repository>
+  <Repository>ghcr.io/ourostack/ouroboros-butler:0.1.0-alpha.772</Repository>
   <Registry>https://github.com/ourostack/ouroboros/pkgs/container/ouroboros-butler</Registry>
   <Network>host</Network>
   <Shell>sh</Shell>

package/dist/heart/daemon/sanctuary-acceptance-adapter.js

@@ -32,6 +32,9 @@ var __importStar = (this && this.__importStar) || (function () {
         return result;
     };
 })();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.proveAttemptedRecoveryWithoutRetry = void 0;
 exports.evaluateSanctuaryProviderReadinessContract = evaluateSanctuaryProviderReadinessContract;
@@ -45,6 +48,8 @@ exports.canonicalDockerIdFromUnraidPrefixedId = canonicalDockerIdFromUnraidPrefi
 exports.runSanctuaryProductionBoundaryProbe = runSanctuaryProductionBoundaryProbe;
 exports.createSanctuaryReadOnlyDenialScenarioDriver = createSanctuaryReadOnlyDenialScenarioDriver;
 exports.readDefaultSanctuaryScenarioFacts = readDefaultSanctuaryScenarioFacts;
+exports.callbackPlaybackSnapshot = callbackPlaybackSnapshot;
+exports.recordCallbackPlayback = recordCallbackPlayback;
 exports.executeSanctuaryAcceptanceAdapter = executeSanctuaryAcceptanceAdapter;
 exports.executeSanctuaryAcceptanceVaultProbe = executeSanctuaryAcceptanceVaultProbe;
 exports.executeSanctuaryAcceptanceRevokedProbe = executeSanctuaryAcceptanceRevokedProbe;
@@ -55,6 +60,7 @@ const node_crypto_1 = require("node:crypto");
 const node_fs_1 = require("node:fs");
 const node_net_1 = require("node:net");
 const path = __importStar(require("node:path"));
+const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
 const friends_1 = require("@ouro.bot/friends");
 const runtime_1 = require("../../nerves/runtime");
 const telegram_approval_runtime_1 = require("../../senses/telegram-approval-runtime");
@@ -99,6 +105,7 @@ const NETWORK_TIMEOUT_MS = 10_000;
 const KEY_DIRECTORY = "/boot/config/plugins/dynamix.my.servers/keys";
 const SELECTED_KEY_RECORD = "/run/ouro-acceptance/unraid-key.json";
 const TELEGRAM_OFFSET = "/home/ouro/AgentBundles/sanctuary.ouro/state/senses/telegram/offset.json";
+const CALLBACK_PLAYBACK_JOURNAL = "state/approvals/sanctuary-callback-playback.sqlite";
 const TELEGRAM_IDENTITY_KEY = "/home/ouro/AgentBundles/sanctuary.ouro/state/senses/telegram/identity.key";
 const TELEGRAM_AUDIT = `/home/ouro/AgentBundles/sanctuary.ouro/${telegram_audit_ledger_1.TELEGRAM_ACCEPTANCE_AUDIT_RELATIVE_PATH}`;
 const TELEGRAM_AUDIT_HEAD = `/home/ouro/AgentBundles/sanctuary.ouro/${telegram_audit_ledger_1.TELEGRAM_ACCEPTANCE_AUDIT_HEAD_RELATIVE_PATH}`;
@@ -244,6 +251,7 @@ function createSanctuaryAcceptanceAdapterDependencies(secretFd = 3, options = {}
         refreshMachine: runtime_credentials_1.refreshMachineRuntimeCredentialConfig,
         mergeMachine: runtime_credentials_1.mergeMachineRuntimeCredentialConfig,
         callbackProbe: executeSanctuaryAcceptanceCallbackProbe,
+        callbackPlaybackSnapshot: (coordinateDigest) => callbackPlaybackSnapshot((0, identity_1.getAgentRoot)(TARGET_ID), coordinateDigest),
         interactiveRuntime: executeSanctuaryInteractiveRuntimeOperation,
         hostRequest: options.hostRequest ?? ((payload) => defaultHostRequest(payload, hostBrokerSocket, adapterTimeoutMs)),
         telegramCredentials: () => (0, telegram_1.loadTelegramSenseCredentials)(TARGET_ID),
@@ -1912,6 +1920,94 @@ function callbackUpdate(value) {
     object(update.callback_query, "callback update callback_query");
     return update;
 }
+function withCallbackPlaybackJournal(agentRoot, operation) {
+    const databasePath = path.join(agentRoot, CALLBACK_PLAYBACK_JOURNAL);
+    (0, node_fs_1.mkdirSync)(path.dirname(databasePath), { recursive: true, mode: 0o700 });
+    const database = new better_sqlite3_1.default(databasePath);
+    try {
+        (0, node_fs_1.chmodSync)(databasePath, 0o600);
+        database.pragma("journal_mode = DELETE");
+        database.pragma("synchronous = FULL");
+        database.exec(`
+      CREATE TABLE IF NOT EXISTS callback_playback (
+        coordinate_digest TEXT PRIMARY KEY NOT NULL CHECK(length(coordinate_digest) = 64),
+        playback_count INTEGER NOT NULL CHECK(playback_count > 0)
+      )
+    `);
+        return operation(database);
+    }
+    finally {
+        database.close();
+    }
+}
+function callbackPlaybackSnapshot(agentRoot, coordinateDigest) {
+    if (!SHA256.test(coordinateDigest))
+        throw new Error("callback coordinate digest is invalid");
+    return withCallbackPlaybackJournal(agentRoot, (database) => {
+        const rows = database.prepare("SELECT coordinate_digest, playback_count FROM callback_playback ORDER BY coordinate_digest")
+            .all();
+        if (rows.some((row) => !SHA256.test(row.coordinate_digest) || !Number.isSafeInteger(row.playback_count) || row.playback_count <= 0)) {
+            throw new Error("callback playback journal is invalid");
+        }
+        const playbackCount = rows.find((row) => row.coordinate_digest === coordinateDigest)?.playback_count ?? 0;
+        return { playbackCount, coordinateDigest, journalDigest: sha256(JSON.stringify(rows)) };
+    });
+}
+function recordCallbackPlayback(agentRoot, coordinateDigest) {
+    if (!SHA256.test(coordinateDigest))
+        throw new Error("callback coordinate digest is invalid");
+    withCallbackPlaybackJournal(agentRoot, (database) => {
+        database.prepare(`
+      INSERT INTO callback_playback (coordinate_digest, playback_count) VALUES (?, 1)
+      ON CONFLICT(coordinate_digest) DO UPDATE SET playback_count = playback_count + 1
+    `).run(coordinateDigest);
+    });
+}
+function callbackCoordinate(update) {
+    const updateId = update.update_id;
+    if (!Number.isSafeInteger(updateId) || Number(updateId) < 0)
+        throw new Error("callback update_id is invalid");
+    const callback = object(update.callback_query, "callback update callback_query");
+    const from = object(callback.from, "callback update sender");
+    const message = object(callback.message, "callback update message");
+    const chat = object(message.chat, "callback update chat");
+    const callbackId = (value, label) => {
+        if (!Number.isSafeInteger(value) || Number(value) <= 0)
+            throw new Error(`${label} is invalid`);
+        return String(value);
+    };
+    const coordinate = {
+        updateId: Number(updateId),
+        queryId: text(callback.id, "callback query id"),
+        callbackData: text(callback.data, "callback data"),
+        userId: callbackId(from.id, "callback user id"),
+        chatId: callbackId(chat.id, "callback chat id"),
+        messageId: callbackId(message.message_id, "callback message id"),
+    };
+    return {
+        updateId: coordinate.updateId,
+        digest: sha256(`ouroboros.sanctuary.callback-coordinate.v1\0${JSON.stringify(coordinate)}`),
+    };
+}
+function callbackPlaybackPreflight(payload, deps) {
+    exactKeys(payload, ["operation", "update"], "callback playback preflight request");
+    const coordinate = callbackCoordinate(callbackUpdate(payload.update));
+    const offset = object(JSON.parse(fixedFile(deps, TELEGRAM_OFFSET)), "Telegram offset");
+    exactKeys(offset, ["nextUpdateId"], "Telegram offset");
+    if (!Number.isSafeInteger(offset.nextUpdateId) || Number(offset.nextUpdateId) < 0)
+        throw new Error("Telegram offset is invalid");
+    const snapshot = object(dependency(deps.callbackPlaybackSnapshot, "callback playback journal")(coordinate.digest), "callback playback journal snapshot");
+    exactKeys(snapshot, ["coordinateDigest", "journalDigest", "playbackCount"], "callback playback journal snapshot");
+    if (snapshot.coordinateDigest !== coordinate.digest || typeof snapshot.journalDigest !== "string" || !SHA256.test(snapshot.journalDigest)
+        || !Number.isSafeInteger(snapshot.playbackCount) || Number(snapshot.playbackCount) < 0) {
+        throw new Error("callback playback journal snapshot is invalid");
+    }
+    return {
+        playbackCount: Math.max(Number(snapshot.playbackCount), Number(offset.nextUpdateId) > coordinate.updateId ? 1 : 0),
+        coordinateDigest: coordinate.digest,
+        journalDigest: snapshot.journalDigest,
+    };
+}
 async function concurrentCallbackProbe(payload, deps) {
     const update = callbackUpdate(payload.update);
     if (!Number.isSafeInteger(payload.concurrency) || payload.concurrency < 2 || payload.concurrency > 16) {
@@ -2026,6 +2122,12 @@ async function storeKey(payload, deps) {
     if (text(handles[id], "vault-backed Unraid credential") !== text(stored[field], "vault-backed Unraid credential")) {
         throw new Error("Unraid key handle does not bind to the active vault field");
     }
+    const acknowledged = object(await dependency(deps.hostRequest, "Sanctuary host broker")({
+        operation: "acknowledge_key_storage", targetServerId: TARGET_SERVER_ID, keyId: id,
+    }), "Unraid key storage acknowledgement");
+    if (acknowledged.acknowledged !== true || acknowledged.id !== id) {
+        throw new Error("Unraid key storage acknowledgement failed");
+    }
     return { stored: true, keyId: id };
 }
 async function probeKey(payload, deps) {
@@ -2232,6 +2334,9 @@ async function executeSanctuaryAcceptanceAdapter(rawPayload, deps = createSanctu
             case "snapshot":
                 result = cursorSnapshot(payload, deps);
                 break;
+            case "callback_playback_preflight":
+                result = callbackPlaybackPreflight(payload, deps);
+                break;
             case "inject_callbacks_concurrently":
                 result = await concurrentCallbackProbe(payload, deps);
                 break;
@@ -2418,7 +2523,7 @@ async function executeSanctuaryInteractiveRuntimeOperation(rawPayload, supplied)
     };
     return (0, sanctuary_interactive_control_1.executeSanctuaryInteractiveEngine)(rawPayload, deps);
 }
-async function executeSanctuaryAcceptanceCallbackProbe(rawUpdate, _replay, deps = {
+async function executeSanctuaryAcceptanceCallbackProbe(rawUpdate, replay, deps = {
     refresh: runtime_credentials_1.refreshRuntimeCredentialConfig,
     credentials: telegram_1.loadTelegramSenseCredentials,
     identityKey: telegram_1.readOrCreateTelegramIdentityKey,
@@ -2426,6 +2531,7 @@ async function executeSanctuaryAcceptanceCallbackProbe(rawUpdate, _replay, deps
     createRuntime: telegram_approval_runtime_1.createTelegramApprovalRuntime,
     toolContext: sanctuary_runtime_1.createSanctuaryToolContext,
     effects: createAcceptanceProbeEffects,
+    recordCallbackPlayback: (coordinateDigest) => recordCallbackPlayback((0, identity_1.getAgentRoot)(TARGET_ID), coordinateDigest),
 }) {
     const update = callbackUpdate(rawUpdate);
     const refreshed = await deps.refresh(TARGET_ID);
@@ -2453,7 +2559,11 @@ async function executeSanctuaryAcceptanceCallbackProbe(rawUpdate, _replay, deps
         effects: effectBoundary?.port ?? unavailableEffects,
     });
     try {
+        deps.recordCallbackPlayback(callbackCoordinate(update).digest);
         const result = await runtime.transport.handleUpdate(update);
+        if (replay && (result.handled !== true || result.accepted !== false || result.reason !== "stale_callback")) {
+            throw new Error("Telegram callback replay did not settle as stale");
+        }
         return {
             settled: result.handled,
             claimed: result.reason === "accepted" || result.reason === "decision_refused",

package/dist/heart/daemon/sanctuary-acceptance-harness.js

@@ -1065,6 +1065,32 @@ async function callbackInject(config, deps) {
     const base = { schemaVersion: 1, operation: "callback-inject", phase: "preflight", updateDigest: digest(update), concurrency, replay: true };
     initializeCheckpoint(root, evidencePath, base);
     try {
+        const preflight = async () => {
+            const result = object(await deps.runAdapter(executable, { operation: "callback_playback_preflight", update }), "callback playback preflight");
+            if (JSON.stringify(Object.keys(result).sort()) !== JSON.stringify(["coordinateDigest", "journalDigest", "playbackCount"])) {
+                throw new Error("callback playback preflight shape is invalid");
+            }
+            if (!Number.isSafeInteger(result.playbackCount) || Number(result.playbackCount) < 0) {
+                throw new Error("callback playback preflight count is invalid");
+            }
+            if (typeof result.coordinateDigest !== "string" || !/^[0-9a-f]{64}$/u.test(result.coordinateDigest)) {
+                throw new Error("callback playback preflight coordinate digest is invalid");
+            }
+            if (typeof result.journalDigest !== "string" || !/^[0-9a-f]{64}$/u.test(result.journalDigest)) {
+                throw new Error("callback playback preflight journal digest is invalid");
+            }
+            return { playbackCount: Number(result.playbackCount), coordinateDigest: result.coordinateDigest, journalDigest: result.journalDigest };
+        };
+        const firstPreflight = await preflight();
+        if (firstPreflight.playbackCount !== 0)
+            throw new Error("callback injection requires zero prior playback");
+        const secondPreflight = await preflight();
+        if (secondPreflight.playbackCount !== 0)
+            throw new Error("callback injection requires zero prior playback");
+        if (secondPreflight.coordinateDigest !== firstPreflight.coordinateDigest)
+            throw new Error("callback playback coordinate changed between preflights");
+        if (secondPreflight.journalDigest !== firstPreflight.journalDigest)
+            throw new Error("callback playback journal changed between preflights");
         const batch = object(await deps.runAdapter(executable, { operation: "inject_callbacks_concurrently", update, concurrency }), "callback batch result");
         if (!Array.isArray(batch.results) || batch.results.length !== concurrency)
             throw new Error("callback batch result count mismatch");
@@ -1094,7 +1120,7 @@ async function callbackInject(config, deps) {
         }
         if (replayResult.claimed || replayResult.mutated)
             throw new Error("callback replay was claimed or mutated state");
-        replaceCheckpoint(root, evidencePath, { ...base, phase: "complete", claims, mutations, replayClaimed: false, replayMutated: false, completedAt: deps.now() });
+        replaceCheckpoint(root, evidencePath, { ...base, phase: "complete", coordinateDigest: firstPreflight.coordinateDigest, zeroPlayback: true, claims, mutations, replayClaimed: false, replayMutated: false, completedAt: deps.now() });
     }
     catch (error) {
         failedCheckpoint(root, evidencePath, base, error);
@@ -1135,99 +1161,267 @@ function sameInventory(left, right) {
 }
 async function rotateOccupiedCanonicalUnraidKeys(input) {
     const { root, evidencePath, targetServerId, adapters, desired, oldKeys, initial, deps } = input;
+    const prior = pathEntryExists(evidencePath) ? readCheckpoint(root, evidencePath) : null;
+    if (prior && (prior.schemaVersion !== 1 || prior.operation !== "unraid-key-rotate" || prior.targetServerId !== targetServerId)) {
+        throw new Error("Unraid rotation checkpoint identity mismatch");
+    }
+    const suffix = prior ? text(prior.transactionSuffix, "Unraid rotation transaction suffix") : deps.randomBytes(8).subarray(0, 8).toString("hex");
+    if (!/^[0-9a-f]{16}$/u.test(suffix))
+        throw new Error("Unraid rotation transaction suffix is invalid");
+    const createdKeyIds = prior ? stringArray(prior.createdKeyIds, "createdKeyIds") : [];
+    const revokedKeyIds = prior ? stringArray(prior.revokedKeyIds, "revokedKeyIds") : [];
+    const transactionKeys = prior ? (() => {
+        if (!Array.isArray(prior.transactionKeys))
+            throw new Error("Unraid rotation transaction keys are unavailable");
+        return prior.transactionKeys.map((raw) => {
+            const value = object(raw, "Unraid rotation transaction key");
+            return {
+                id: text(value.id, "transaction key id"), name: text(value.name, "transaction key name"),
+                vaultField: text(value.vaultField, "transaction key vault field"),
+                permissions: stringArray(value.permissions, "transaction key permissions").sort(),
+                temporary: boolean(value.temporary, "transaction key temporary"),
+                attested: boolean(value.attested, "transaction key attested"),
+            };
+        });
+    })() : [];
+    if (new Set(createdKeyIds).size !== createdKeyIds.length || new Set(revokedKeyIds).size !== revokedKeyIds.length) {
+        throw new Error("Unraid rotation checkpoint IDs are ambiguous");
+    }
+    if (new Set(transactionKeys.map((entry) => entry.id)).size !== transactionKeys.length
+        || new Set(transactionKeys.map((entry) => entry.name)).size !== transactionKeys.length
+        || !sameStrings(createdKeyIds, transactionKeys.map((entry) => entry.id))) {
+        throw new Error("Unraid rotation transaction key metadata is ambiguous");
+    }
+    const slots = desired.flatMap((key) => [
+        { ...key, name: `${key.name} Rotation ${suffix}`, temporary: true },
+        { ...key, temporary: false },
+    ]);
+    if (transactionKeys.some((entry) => {
+        const slot = slots.find((candidate) => candidate.name === entry.name);
+        return !slot || entry.temporary !== slot.temporary || entry.vaultField !== slot.vaultField
+            || !sameStrings(entry.permissions, slot.permissions);
+    }))
+        throw new Error("Unraid rotation checkpoint transaction slots are invalid");
+    const revocableKeyIds = new Set([
+        ...oldKeys.map((entry) => entry.id),
+        ...transactionKeys.filter((entry) => entry.temporary).map((entry) => entry.id),
+    ]);
+    if (revokedKeyIds.some((id) => !revocableKeyIds.has(id))) {
+        throw new Error("Unraid rotation checkpoint contains an unowned revoked key ID");
+    }
+    const temporary = transactionKeys.filter((entry) => entry.temporary);
+    const canonical = transactionKeys.filter((entry) => !entry.temporary);
+    const allOldRevoked = oldKeys.every((entry) => revokedKeyIds.includes(entry.id));
+    const allTemporaryAttested = temporary.length === desired.length && temporary.every((entry) => entry.attested);
+    const allCanonicalAttested = canonical.length === desired.length && canonical.every((entry) => entry.attested);
+    const revokedTemporary = temporary.filter((entry) => revokedKeyIds.includes(entry.id));
+    if (revokedTemporary.some((entry) => !entry.attested)
+        || (oldKeys.some((entry) => revokedKeyIds.includes(entry.id)) && !allTemporaryAttested)
+        || (revokedTemporary.length > 0 && !allCanonicalAttested)
+        || (canonical.length > 0 && !allOldRevoked)) {
+        throw new Error("Unraid rotation checkpoint revocation order is invalid");
+    }
+    const phase = prior ? text(prior.phase, "Unraid rotation phase") : "preflight";
+    const knownPhases = new Set(["preflight", "temporary_key_created", "temporary_key_attested", "key_revoke_attempted", "key_revoked", "canonical_key_created", "canonical_key_attested", "complete"]);
+    const resumePhase = phase === "failed" ? text(prior?.resumePhase, "Unraid rotation resume phase") : phase;
+    if (!knownPhases.has(resumePhase) || (phase !== "failed" && prior?.resumePhase !== undefined)) {
+        throw new Error("Unraid rotation checkpoint phase is invalid");
+    }
+    const stateMatchesPhase = resumePhase === "preflight" ? transactionKeys.length === 0 && revokedKeyIds.length === 0
+        : resumePhase === "temporary_key_created" ? temporary.length > 0 && canonical.length === 0 && revokedKeyIds.length === 0 && temporary.filter((entry) => !entry.attested).length === 1
+            : resumePhase === "temporary_key_attested" ? temporary.length > 0 && canonical.length === 0 && revokedKeyIds.length === 0 && temporary.every((entry) => entry.attested)
+                : resumePhase === "key_revoke_attempted" ? allTemporaryAttested && (canonical.length === 0 || allCanonicalAttested)
+                    : resumePhase === "key_revoked" ? revokedKeyIds.length > 0 && allTemporaryAttested && (canonical.length === 0 || allCanonicalAttested)
+                        : resumePhase === "canonical_key_created" ? allTemporaryAttested && canonical.length > 0 && sameStrings(revokedKeyIds, oldKeys.map((entry) => entry.id)) && canonical.filter((entry) => !entry.attested).length === 1
+                            : resumePhase === "canonical_key_attested" ? allTemporaryAttested && canonical.length > 0 && sameStrings(revokedKeyIds, oldKeys.map((entry) => entry.id)) && canonical.every((entry) => entry.attested)
+                                : transactionKeys.length === slots.length && allTemporaryAttested && allCanonicalAttested
+                                    && sameStrings(revokedKeyIds, [...oldKeys.map((entry) => entry.id), ...temporary.map((entry) => entry.id)]);
+    if (!stateMatchesPhase)
+        throw new Error("Unraid rotation checkpoint state does not match its phase");
     const base = {
-        schemaVersion: 1,
-        operation: "unraid-key-rotate",
-        phase: "preflight",
-        targetServerId,
-        initialInventoryDigest: digest(initial),
-        createdKeyIds: [],
-        revokedKeyIds: [],
+        schemaVersion: 1, operation: "unraid-key-rotate", targetServerId,
+        initialInventoryDigest: prior ? text(prior.initialInventoryDigest, "initialInventoryDigest") : digest(initial),
+        transactionSuffix: suffix,
     };
-    initializeCheckpoint(root, evidencePath, base);
+    if (!prior)
+        initializeCheckpoint(root, evidencePath, { ...base, phase: "preflight", createdKeyIds, revokedKeyIds, transactionKeys });
+    if (phase === "complete") {
+        const final = inventory(await deps.runAdapter(adapters.inventory, { operation: "inventory_keys", targetServerId }));
+        const canonical = transactionKeys.filter((entry) => !entry.temporary)
+            .map(({ id, name, permissions }) => ({ id, name, permissions, roles: [] }));
+        if (canonical.length !== desired.length || !sameInventory(final, canonical)) {
+            throw new Error("completed Unraid rotation inventory drifted");
+        }
+        return;
+    }
     const created = [];
-    const revokedKeyIds = [];
-    const suffix = deps.randomBytes(8).toString("hex");
+    let activePhase = resumePhase;
+    const checkpoint = (phase) => {
+        activePhase = phase;
+        replaceCheckpoint(root, evidencePath, { ...base, phase, createdKeyIds, revokedKeyIds, transactionKeys });
+    };
     const createAndAttest = async (key, name, temporary) => {
-        const response = object(await deps.runAdapter(adapters.create, {
-            operation: "create_key", targetServerId, name, permissions: key.permissions,
-        }), "created Unraid key");
-        const id = text(response.id, "created key id");
-        const handle = text(response.key, "created key handle");
-        if (response.name !== name || !sameStrings(stringArray(response.permissions, "created permissions"), key.permissions)
-            || !Array.isArray(response.roles) || response.roles.length !== 0
-            || initial.some((entry) => entry.id === id) || created.some((entry) => entry.id === id)) {
-            throw new Error("created Unraid key scope or identity mismatch");
+        const live = inventory(await deps.runAdapter(adapters.inventory, { operation: "inventory_keys", targetServerId }));
+        const matches = live.filter((entry) => entry.name === name);
+        const transaction = transactionKeys.find((entry) => entry.name === name);
+        if (transaction && revokedKeyIds.includes(transaction.id)) {
+            if (matches.length !== 0)
+                throw new Error("revoked Unraid transaction key inventory mismatch");
+            return;
+        }
+        if (transaction && temporary && matches.length === 0)
+            return;
+        let id;
+        let handle;
+        if (matches.length === 1) {
+            const existing = matches[0];
+            if (!sameStrings(existing.permissions, key.permissions) || existing.roles.length !== 0
+                || oldKeys.some((old) => old.id === existing.id)
+                || (transaction && transaction.id !== existing.id)) {
+                throw new Error("resumable Unraid key scope or identity mismatch");
+            }
+            id = existing.id;
+            handle = `unraid-key:${id}:${key.vaultField}`;
+            if (!transaction) {
+                const recovered = object(await deps.runAdapter(adapters.create, {
+                    operation: "create_key", targetServerId, name, permissions: key.permissions,
+                }), "recovered Unraid key");
+                if (recovered.id !== id || recovered.name !== name
+                    || !sameStrings(stringArray(recovered.permissions, "recovered permissions"), key.permissions)
+                    || !Array.isArray(recovered.roles) || recovered.roles.length !== 0) {
+                    throw new Error("recovered Unraid key scope or identity mismatch");
+                }
+                handle = text(recovered.key, "recovered key handle");
+            }
+        }
+        else {
+            const response = object(await deps.runAdapter(adapters.create, {
+                operation: "create_key", targetServerId, name, permissions: key.permissions,
+            }), "created Unraid key");
+            id = text(response.id, "created key id");
+            handle = text(response.key, "created key handle");
+            if (response.name !== name || !sameStrings(stringArray(response.permissions, "created permissions"), key.permissions)
+                || !Array.isArray(response.roles) || response.roles.length !== 0
+                || oldKeys.some((old) => old.id === id) || created.some((entry) => entry.id === id)) {
+                throw new Error("created Unraid key scope or identity mismatch");
+            }
         }
         created.push({ id, name, permissions: key.permissions, roles: [], handle, desired: key, temporary });
-        replaceCheckpoint(root, evidencePath, { ...base, phase: temporary ? "temporary_key_created" : "canonical_key_created", createdKeyIds: created.map((entry) => entry.id), revokedKeyIds });
-        const stored = object(await deps.runAdapter(adapters.store, {
-            operation: "store_key", targetServerId, vaultField: key.vaultField, keyId: id, key: handle,
-        }), "key store result");
-        if (stored.stored !== true || stored.keyId !== id)
-            throw new Error("vault did not attest exact key storage");
+        if (!createdKeyIds.includes(id))
+            createdKeyIds.push(id);
+        const metadata = transaction ?? { id, name, vaultField: key.vaultField, permissions: key.permissions, temporary, attested: false };
+        if (!transaction)
+            transactionKeys.push(metadata);
+        checkpoint(temporary ? "temporary_key_created" : "canonical_key_created");
+        if (!metadata.attested) {
+            const stored = object(await deps.runAdapter(adapters.store, {
+                operation: "store_key", targetServerId, vaultField: key.vaultField, keyId: id, key: handle,
+            }), "key store result");
+            if (stored.stored !== true || stored.keyId !== id)
+                throw new Error("vault did not attest exact key storage");
+        }
         const probe = object(await deps.runAdapter(adapters.probe, {
             operation: "probe_new_key", targetServerId, id, key: handle,
         }), "new key probe");
         if (probe.valid !== true)
             throw new Error("new Unraid key readiness failed");
+        metadata.attested = true;
+        checkpoint(temporary ? "temporary_key_attested" : "canonical_key_attested");
     };
     const revokeAndReject = async (id, handle) => {
-        const revoked = object(await deps.runAdapter(adapters.revoke, { operation: "revoke_key", targetServerId, id }), "key revoke result");
-        if (revoked.revoked !== true || revoked.id !== id)
-            throw new Error("Unraid revoke did not attest exact key id");
+        if (revokedKeyIds.includes(id))
+            return;
+        const live = inventory(await deps.runAdapter(adapters.inventory, { operation: "inventory_keys", targetServerId }));
+        if (live.some((entry) => entry.id === id)) {
+            const revoked = object(await deps.runAdapter(adapters.revoke, { operation: "revoke_key", targetServerId, id }), "key revoke result");
+            if (revoked.revoked !== true || revoked.id !== id)
+                throw new Error("Unraid revoke did not attest exact key id");
+            checkpoint("key_revoke_attempted");
+        }
         const probe = object(await deps.runAdapter(adapters.probe, { operation: "probe_revoked_key", targetServerId, id, key: handle }), "revoked key probe");
         if (probe.valid !== false || (probe.status !== 401 && probe.status !== 403))
             throw new Error("revoked Unraid key still authenticates");
         revokedKeyIds.push(id);
-        replaceCheckpoint(root, evidencePath, { ...base, phase: "key_revoked", createdKeyIds: created.map((entry) => entry.id), revokedKeyIds });
+        checkpoint("key_revoked");
     };
     try {
         for (const key of desired)
             await createAndAttest(key, `${key.name} Rotation ${suffix}`, true);
         const withTemporary = inventory(await deps.runAdapter(adapters.inventory, { operation: "inventory_keys", targetServerId }));
-        if (!sameInventory(withTemporary, [...initial, ...created.map(({ handle: _handle, desired: _desired, temporary: _temporary, ...entry }) => entry)])) {
+        const liveOld = withTemporary.filter((entry) => oldKeys.some((old) => old.id === entry.id));
+        const inventoryAmbiguous = withTemporary.some((entry) => {
+            if (oldKeys.some((old) => old.id === entry.id)) {
+                const expected = desired.find((candidate) => candidate.name === entry.name);
+                return !expected || !sameStrings(expected.permissions, entry.permissions) || entry.roles.length !== 0;
+            }
+            const metadata = transactionKeys.find((candidate) => candidate.id === entry.id);
+            return !metadata || metadata.name !== entry.name || !sameStrings(metadata.permissions, entry.permissions) || entry.roles.length !== 0;
+        });
+        const missingBridge = liveOld.length > 0 && transactionKeys.filter((entry) => entry.temporary && !revokedKeyIds.includes(entry.id))
+            .some((entry) => !withTemporary.some((candidate) => candidate.id === entry.id));
+        if (inventoryAmbiguous || missingBridge) {
             throw new Error("temporary Unraid key inventory changed ambiguously");
         }
         for (const old of oldKeys) {
-            const secret = object(await deps.runAdapter(old.secretAdapter, { operation: "read_old_key", targetServerId, id: old.id }), "old key handle result");
-            await revokeAndReject(old.id, text(secret.key, "old key handle"));
+            const live = inventory(await deps.runAdapter(adapters.inventory, { operation: "inventory_keys", targetServerId }));
+            if (live.some((entry) => entry.id === old.id)) {
+                const secret = object(await deps.runAdapter(old.secretAdapter, { operation: "read_old_key", targetServerId, id: old.id }), "old key handle result");
+                await revokeAndReject(old.id, text(secret.key, "old key handle"));
+            }
+            else {
+                await revokeAndReject(old.id, `unraid-key:${old.id}:legacy`);
+            }
+        }
+        let afterOldRevokes = inventory(await deps.runAdapter(adapters.inventory, { operation: "inventory_keys", targetServerId }));
+        const existingCanonical = transactionKeys.filter((entry) => !entry.temporary);
+        if (existingCanonical.length === desired.length) {
+            for (const key of transactionKeys.filter((entry) => entry.temporary && !revokedKeyIds.includes(entry.id))) {
+                if (!afterOldRevokes.some((entry) => entry.id === key.id)) {
+                    await revokeAndReject(key.id, `unraid-key:${key.id}:${key.vaultField}`);
+                }
+            }
+            afterOldRevokes = inventory(await deps.runAdapter(adapters.inventory, { operation: "inventory_keys", targetServerId }));
         }
-        const temporary = [...created];
-        const afterOldRevokes = inventory(await deps.runAdapter(adapters.inventory, { operation: "inventory_keys", targetServerId }));
-        const expectedTemporary = temporary.map(({ handle: _handle, desired: _desired, temporary: _temporary, ...entry }) => entry);
-        if (!sameInventory(afterOldRevokes, expectedTemporary))
+        const exactPhase = transactionKeys.filter((entry) => !revokedKeyIds.includes(entry.id))
+            .map(({ id, name, permissions }) => ({ id, name, permissions, roles: [] }));
+        if (existingCanonical.length > desired.length || !sameInventory(afterOldRevokes, exactPhase)) {
             throw new Error("old-key revoke inventory is not the exact temporary pair");
+        }
         for (const key of desired)
             await createAndAttest(key, key.name, false);
         const withCanonical = inventory(await deps.runAdapter(adapters.inventory, { operation: "inventory_keys", targetServerId }));
-        const expectedWithCanonical = created.map(({ handle: _handle, desired: _desired, temporary: _temporary, ...entry }) => entry);
-        if (!sameInventory(withCanonical, expectedWithCanonical))
+        if (withCanonical.some((entry) => {
+            const metadata = transactionKeys.find((candidate) => candidate.id === entry.id);
+            return !metadata || metadata.name !== entry.name || !sameStrings(metadata.permissions, entry.permissions) || entry.roles.length !== 0;
+        }) || transactionKeys.filter((entry) => !entry.temporary).some((entry) => !withCanonical.some((candidate) => candidate.id === entry.id))) {
             throw new Error("canonical-key mint inventory changed ambiguously");
-        for (const key of temporary)
-            await revokeAndReject(key.id, key.handle);
+        }
+        for (const key of transactionKeys.filter((entry) => entry.temporary)) {
+            await revokeAndReject(key.id, `unraid-key:${key.id}:${key.vaultField}`);
+        }
         const final = inventory(await deps.runAdapter(adapters.inventory, { operation: "inventory_keys", targetServerId }));
-        const canonical = created.filter((entry) => !entry.temporary).map(({ handle: _handle, desired: _desired, temporary: _temporary, ...entry }) => entry);
+        const canonical = transactionKeys.filter((entry) => !entry.temporary)
+            .map(({ id, name, permissions }) => ({ id, name, permissions, roles: [] }));
         if (!sameInventory(final, canonical))
             throw new Error("final canonical Unraid key inventory mismatch");
         replaceCheckpoint(root, evidencePath, {
             ...base,
             phase: "complete",
-            createdKeyIds: created.map((entry) => entry.id),
+            createdKeyIds,
             revokedKeyIds,
+            transactionKeys,
             finalInventoryDigest: digest(final),
             completedAt: deps.now(),
         });
     }
     catch (error) {
-        failedCheckpoint(root, evidencePath, { ...base, createdKeyIds: created.map((entry) => entry.id), revokedKeyIds }, error);
+        replaceCheckpoint(root, evidencePath, { ...base, phase: "failed", resumePhase: activePhase, createdKeyIds, revokedKeyIds, transactionKeys, errorCategory: safeErrorCategory(error) });
         throw error;
     }
 }
 async function unraidKeyRotate(config, deps) {
     const root = privateAllowedRoot(config);
     const evidencePath = confinedPath(root, config.evidencePath, "evidencePath");
-    refuseExistingCheckpoint(root, evidencePath);
     const targetServerId = text(config.targetServerId, "targetServerId");
     const inventoryAdapter = adapter(config.inventoryAdapter, "inventoryAdapter");
     const createAdapter = adapter(config.createAdapter, "createAdapter");
@@ -1257,12 +1451,13 @@ async function unraidKeyRotate(config, deps) {
         throw new Error("old key IDs must be unique");
     const initial = inventory(await deps.runAdapter(inventoryAdapter, { operation: "inventory_keys", targetServerId }));
     const occupiedCanonicalContract = targetServerId === "sanctuary-unraid"
-        && desired.length === 2 && initial.length === 2 && oldKeys.length === 2
+        && desired.length === 2 && oldKeys.length === 2
         && JSON.stringify(desired.map((key) => key.name).sort()) === JSON.stringify(["Butler RO", "Butler RW"])
-        && desired.every((key) => {
+        && (pathEntryExists(evidencePath) || (initial.length === 2 && desired.every((key) => {
             const existing = initial.find((entry) => entry.name === key.name);
-            return existing !== undefined && oldKeys.some((old) => old.id === existing.id);
-        })
+            return existing !== undefined && oldKeys.some((old) => old.id === existing.id)
+                && existing.roles.length === 0 && sameStrings(existing.permissions, key.permissions);
+        })))
         && [inventoryAdapter, createAdapter, storeAdapter, revokeAdapter, probeAdapter, ...oldKeys.map((old) => old.secretAdapter)]
             .every((value) => value === SANCTUARY_ACCEPTANCE_ADAPTER);
     if (occupiedCanonicalContract) {
@@ -1273,6 +1468,7 @@ async function unraidKeyRotate(config, deps) {
         });
         return;
     }
+    refuseExistingCheckpoint(root, evidencePath);
     for (const key of desired)
         if (initial.some((entry) => entry.name === key.name))
             throw new Error(`Unraid key ${key.name} already exists`);

package/npm-shrinkwrap.json

@@ -1,12 +1,12 @@
 {
   "name": "@ouro.bot/cli",
-  "version": "0.1.0-alpha.771",
+  "version": "0.1.0-alpha.772",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@ouro.bot/cli",
-      "version": "0.1.0-alpha.771",
+      "version": "0.1.0-alpha.772",
       "dependencies": {
         "@anthropic-ai/sdk": "^0.78.0",
         "@azure/identity": "^4.13.0",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ouro.bot/cli",
-  "version": "0.1.0-alpha.771",
+  "version": "0.1.0-alpha.772",
   "engines": {
     "node": ">=22"
   },