@or3/cloud 0.1.82 → 0.1.85

package/dist/cli.mjs

@@ -125,7 +125,7 @@ import { createReadStream as createReadStream3 } from "node:fs";
 import { statfs } from "node:fs/promises";
 import { Writable } from "node:stream";
 import { pipeline as pipeline2 } from "node:stream/promises";
-import { dirname as dirname3, join as join7 } from "node:path";
+import { dirname as dirname3, join as join7, posix } from "node:path";
 import { createGunzip } from "node:zlib";
 
 // src/deployment/env.ts
@@ -165,7 +165,7 @@ import { readFileSync } from "node:fs";
 import { join, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 var PACKAGE_ROOT = resolve(fileURLToPath(new URL("../", import.meta.url)));
-var PACKAGE_VERSION = "0.1.82";
+var PACKAGE_VERSION = "0.1.85";
 var IMAGE_REPOSITORY = "ghcr.io/saluana/or3-chat";
 var ASSET_ROOT = resolve(fileURLToPath(new URL("../assets/", import.meta.url)));
 var SHA256_DIGEST = /^sha256:[0-9a-f]{64}$/;
@@ -944,10 +944,11 @@ async function streamCommandToFile(command, args, destination, cwd, secrets = []
     child.once("error", reject);
     child.once("close", resolvePromise);
   });
+  const transfer = pipeline(child.stdout, createWriteStream(temporary, { flags: "wx", mode: 384 }));
   try {
     const [exitCode] = await Promise.all([
       exit,
-      pipeline(child.stdout, createWriteStream(temporary, { flags: "wx", mode: 384 }))
+      transfer
     ]);
     if (hasTimedOut())
       throw new Error(`${command} exceeded the ${STREAM_COMMAND_TIMEOUT_MS / 1000}-second archive deadline.`);
@@ -958,12 +959,18 @@ async function streamCommandToFile(command, args, destination, cwd, secrets = []
     await durableRename(temporary, destination);
   } catch (error) {
     terminateChildProcess(child);
-    await rm2(temporary, { force: true }).catch(() => {
-      return;
-    });
-    const detail = error instanceof Error ? error.message : String(error);
-    throw new Error(`${detail}${stderr && !detail.includes(stderr) ? `
-${redact(stderr, secrets)}` : ""}`);
+    await Promise.allSettled([exit, transfer]);
+    const detail = redact(error instanceof Error ? error.message : String(error), secrets);
+    const message = `${detail}${stderr && !detail.includes(stderr) ? `
+${redact(stderr, secrets)}` : ""}`;
+    try {
+      await lifecycleFaults.beforeTemporaryArchiveDelete?.();
+      await rm2(temporary, { force: true });
+    } catch (cleanupError) {
+      throw new AggregateError([error, cleanupError], `${message}
+Temporary archive cleanup failed: ${redact(cleanupError instanceof Error ? cleanupError.message : String(cleanupError), secrets)}. Removal is unverified for ${temporary}; inspect this path and its permissions before retrying.`);
+    }
+    throw new Error(message, { cause: error });
   } finally {
     clearTimeout(timeout);
   }
@@ -1356,14 +1363,15 @@ ${existing.stderr}`)) {
     throw new Error(`Volume ${state.volumeName} does not carry the expected managed deployment labels.`);
   }
 }
-async function dataVolumeSize(directory, mode, env) {
+async function dataVolumeSize(directory, mode, env, size = "apparent") {
+  const script = size === "allocated" ? "du -sk /data 2>/dev/null" : "du -sb /data 2>/dev/null";
   const parseDuOutput = (stdout) => {
     const match = stdout.trim().match(/^(\d+)/);
     if (!match)
       throw new Error(`Could not parse the data volume size from "du" output.`);
-    return Number(match[1]);
+    return Number(match[1]) * (size === "allocated" ? 1024 : 1);
   };
-  const exec = await run("docker", [...composeArgs(directory, mode, ["exec", "-T", "or3", "sh", "-c", "du -sb /data 2>/dev/null"])], directory);
+  const exec = await run("docker", [...composeArgs(directory, mode, ["exec", "-T", "or3", "sh", "-c", script])], directory);
   if (exec.ok)
     return parseDuOutput(exec.stdout);
   const fallback = await run("docker", [
@@ -1380,7 +1388,7 @@ async function dataVolumeSize(directory, mode, env) {
     "sh",
     env.OR3_IMAGE,
     "-c",
-    "du -sb /data 2>/dev/null"
+    script
   ], directory);
   if (fallback.ok)
     return parseDuOutput(fallback.stdout);
@@ -1536,7 +1544,9 @@ async function restoreVolumeArchive(directory, mode, env, backupPath) {
     "tar xzf - -C /data"
   ], join7(backupPath, "data.tgz"), directory, secretValues(env));
 }
-async function validateVolumeArchive(directory, _mode, env, backupPath) {
+async function validateVolumeArchive(directory, _mode, env, backupPath, archivedStorageRoot) {
+  const storageRoot = posix.relative("/data", archivedStorageRoot ?? env.OR3_STORAGE_FS_ROOT ?? "/data/storage");
+  const storagePrefix = storageRoot === ".." || storageRoot.startsWith("../") || posix.isAbsolute(storageRoot) ? "" : `${storageRoot ? `${storageRoot}/` : ""}workspaces/`;
   await streamFileToCommand("docker", [
     "run",
     "--rm",
@@ -1552,7 +1562,46 @@ async function validateVolumeArchive(directory, _mode, env, backupPath) {
     "sh",
     env.OR3_IMAGE,
     "-c",
-    "tar tzf - >/dev/null"
+    `{ tar tzf -; printf '\\nOR3_TAR_RESULT:%s\\n' "$?"; } | awk -v prefix="$1" '
+{
+  entry = $0
+  sub(/^\\.\\//, "", entry)
+  if (entry == "auth.sqlite") auth = 1
+  if (entry == "sync.sqlite") sync = 1
+  if (entry ~ /^OR3_TAR_RESULT:[0-9]+$/) {
+    seen_status = 1
+    sub(/^OR3_TAR_RESULT:/, "", entry)
+    tar_status = entry
+    next
+  }
+  if (prefix != "" && index(entry, prefix) == 1) {
+    path = substr(entry, length(prefix) + 1)
+    if (split(path, parts, "/") != 2 || parts[1] !~ /^[a-zA-Z0-9_-]+$/) next
+    key = parts[2]
+    sidecar = sub(/[.]meta[.]json$/, "", key)
+    if ((key ~ /^sha256_[a-fA-F0-9]+$/ && length(key) == 71) || (key ~ /^md5_[a-fA-F0-9]+$/ && length(key) == 36)) {
+      if (sidecar) metadata[entry] = 1
+      else payloads[entry] = 1
+    }
+  }
+}
+END {
+  if (!seen_status || tar_status != "0") exit 1
+  if (!auth || !sync) {
+    print "Backup archive is missing required managed database payloads (auth.sqlite and sync.sqlite)." > "/dev/stderr"
+    exit 1
+  }
+  for (entry in metadata) {
+    payload = entry
+    sub(/[.]meta[.]json$/, "", payload)
+    if (!(payload in payloads)) {
+      print "Backup archive contains a committed filesystem metadata sidecar without its required blob payload." > "/dev/stderr"
+      exit 1
+    }
+  }
+}'`,
+    "or3-archive-validation",
+    storagePrefix
   ], join7(backupPath, "data.tgz"), directory, secretValues(env));
 }
 
@@ -3190,12 +3239,82 @@ ${sourceRecoveryError ? "The original V1 deployment needs manual recovery. " : "
 }
 
 // src/backup/create.ts
-import { chmod as chmod6, mkdir as mkdir6 } from "node:fs/promises";
-import { join as join14, resolve as resolve9 } from "node:path";
+import { chmod as chmod6, lstat as lstat2, mkdir as mkdir6 } from "node:fs/promises";
+import { join as join15, resolve as resolve9 } from "node:path";
+
+// src/backup/references.ts
+import { randomUUID as randomUUID2 } from "node:crypto";
+import { join as join13 } from "node:path";
+var INSPECTOR = "/app/scripts/docker/backup-inspect.mjs";
+var NODE = 'if [ -x /nodejs/bin/node ]; then exec /nodejs/bin/node "$@"; else exec node "$@"; fi';
+async function validateCanonicalBackupReferences(directory, env, backupPath) {
+  if (env.OR3_SYNC_PROVIDER !== "sqlite" || env.NUXT_PUBLIC_STORAGE_PROVIDER !== "fs" || env.OR3_SQLITE_DRIVER && env.OR3_SQLITE_DRIVER !== "better-sqlite3") {
+    throw new Error("Unsupported offline backup inspection provider profile; SQLite/FS is required.");
+  }
+  const helper = `or3-backup-inspection-${randomUUID2()}`;
+  const checked = async (args) => {
+    const result = await run("docker", args, directory);
+    if (!result.ok)
+      throw new Error(`Offline backup reference inspection failed or is unsupported. ${redact(result.stderr, secretValues(env))}`);
+    return result.stdout.trim();
+  };
+  let primary;
+  try {
+    await checked([
+      "run",
+      "--pull",
+      "never",
+      "--rm",
+      "--detach",
+      "--name",
+      helper,
+      "--network",
+      "none",
+      "--read-only",
+      "--security-opt",
+      "no-new-privileges:true",
+      "--cap-drop",
+      "ALL",
+      "--memory",
+      "256m",
+      "--cpus",
+      "0.5",
+      "--pids-limit",
+      "32",
+      "--tmpfs",
+      "/scratch:rw,noexec,nosuid,nodev,size=512m,mode=700,uid=65532,gid=65532",
+      "--entrypoint",
+      "sh",
+      env.OR3_IMAGE,
+      "-c",
+      "sleep 240"
+    ]);
+    await streamFileToCommand("docker", ["exec", "-i", helper, "sh", "-c", "ulimit -f 16384; tar tvzf - > /scratch/inventory"], join13(backupPath, "data.tgz"), directory, secretValues(env));
+    const inventory = JSON.parse(await checked(["exec", helper, "sh", "-c", NODE, "or3-backup-inspection", INSPECTOR, "inventory"]));
+    if (!Array.isArray(inventory.selected) || !inventory.selected.includes("./sync.sqlite") && !inventory.selected.includes("sync.sqlite") || inventory.selected.some((name) => !/^(?:\.\/)?sync[.]sqlite(?:-wal)?$/.test(name)) || inventory.selected.length > 2 || !Number.isSafeInteger(inventory.bytes) || inventory.bytes < 0)
+      throw new Error("Invalid offline backup inspection inventory response.");
+    const freeKiB = Number(await checked(["exec", helper, "sh", "-c", "df -Pk /scratch | awk 'NR == 2 { print $4 }'"]));
+    if (!Number.isFinite(freeKiB) || freeKiB < 0)
+      throw new Error("Could not verify offline backup scratch capacity.");
+    assertEnoughFreeSpace(freeKiB * 1024, inventory.bytes, "Offline backup inspection scratch");
+    await streamFileToCommand("docker", ["exec", "-i", helper, "tar", "-xz", "-f", "-", "-C", "/scratch", ...inventory.selected], join13(backupPath, "data.tgz"), directory, secretValues(env));
+    const result = JSON.parse(await checked(["exec", helper, "sh", "-c", NODE, "or3-backup-inspection", INSPECTOR, "references", env.OR3_STORAGE_FS_ROOT ?? "/data/storage"]));
+    if (result.contract !== 1 || result.complete !== true || !Number.isSafeInteger(result.liveAttachments) || result.liveAttachments < 0)
+      throw new Error("Offline canonical backup reference inspection did not complete.");
+  } catch (error) {
+    primary = error;
+  }
+  const cleanup = await run("docker", ["rm", "--force", helper], directory);
+  if (!cleanup.ok && !/No such container/i.test(cleanup.stderr)) {
+    throw new AggregateError(primary ? [primary] : [], `Offline backup inspection helper cleanup failed. ${redact(cleanup.stderr, secretValues(env))}`);
+  }
+  if (primary)
+    throw primary;
+}
 
 // src/backup/retention.ts
 import { rm as rm5 } from "node:fs/promises";
-import { basename as basename4, dirname as dirname4, join as join13, resolve as resolve8 } from "node:path";
+import { basename as basename4, dirname as dirname4, join as join14, resolve as resolve8 } from "node:path";
 var BACKUP_RETENTION_KEEP = 5;
 function assertRemovableArtifactName(name) {
   if (!BACKUP_ID_PATTERN.test(name)) {
@@ -3211,7 +3330,7 @@ async function removeNamedBackupArtifact(directory, backupId) {
     throw new Error(`Refusing to remove a path outside the backups directory: ${target}.`);
   }
   await rm5(target, { recursive: true, force: true });
-  await rm5(join13(deploymentPaths(directory).exports, `${backupId}.json`), { force: true });
+  await rm5(join14(deploymentPaths(directory).exports, `${backupId}.json`), { force: true });
 }
 async function removeEnumeratedBackupArtifact(directory, backup) {
   await lifecycleFaults.beforeArtifactDelete?.();
@@ -3221,7 +3340,7 @@ async function removeEnumeratedBackupArtifact(directory, backup) {
     throw new Error(`Refusing to prune an untrusted backup path ${backup.path}.`);
   }
   await rm5(target, { recursive: true, force: true });
-  await rm5(join13(deploymentPaths(directory).exports, `${backup.backupId}.json`), { force: true });
+  await rm5(join14(deploymentPaths(directory).exports, `${backup.backupId}.json`), { force: true });
 }
 function planRetention(entries, keep, protectedIds, options = {}) {
   if (!Number.isInteger(keep) || keep < 1)
@@ -3307,6 +3426,35 @@ async function pruneBackups(directory, state, keep, force, options = {}) {
 }
 
 // src/backup/create.ts
+var asError = (error) => error instanceof Error ? error : new Error(String(error));
+var RECOVERY_STEPS = 'Keep the operation record. Inspect the named paths and their permissions, then run "npx @or3/cloud recover --dry-run" and "npx @or3/cloud recover" from the deployment directory. Do not restore an unverified artifact or remove recovery locks manually.';
+async function artifactStatus(path) {
+  try {
+    await lstat2(path);
+    return { absent: false, detail: `Still present: ${path}.` };
+  } catch (error) {
+    if (error.code === "ENOENT")
+      return { absent: true, detail: `Confirmed absent: ${path}.` };
+    return { absent: false, detail: `Could not inspect ${path}: ${asError(error).message}. Its presence or absence is unverified.` };
+  }
+}
+async function cleanupPartialBackup(directory, backupId) {
+  let error;
+  try {
+    await removeNamedBackupArtifact(directory, backupId);
+  } catch (failure) {
+    error = asError(failure);
+  }
+  const statuses = await Promise.all([
+    artifactStatus(backupDirectory(directory, backupId)),
+    artifactStatus(join15(deploymentPaths(directory).exports, `${backupId}.json`))
+  ]);
+  const removed = statuses.every((status) => status.absent);
+  const detail = statuses.map((status) => status.detail).join(" ");
+  if (!removed && !error)
+    error = new Error("Backup cleanup did not confirm removal of every artifact.");
+  return { error, removed, detail };
+}
 async function createBackup(directory, state, env, options = {}) {
   const restartAfter = options.restartAfter ?? true;
   await requireImageDigest(state.image, state.imageDigest, "Current deployment");
@@ -3315,90 +3463,172 @@ async function createBackup(directory, state, env, options = {}) {
     throw new Error(`Backup ID ${backupId} is invalid.`);
   const backupDir = backupDirectory(directory, backupId);
   const initiallyRunning = options.initiallyRunning ?? await projectServiceRunning(directory, state.mode);
+  let service = initiallyRunning ? "running" : "stopped";
+  let artifact = "not-created";
+  let downtimeMs;
+  const report = async (stage, message) => {
+    await options.onProgress?.({ stage, service, artifact, message, downtimeMs });
+  };
+  await report("preflight", "Preflight: checking image identity and backup disk space; service state is unchanged.");
   const volumeSize = await dataVolumeSize(directory, state.mode, env);
   await assertFreeSpaceForArchive(backupDir, volumeSize, "Backup");
-  let manifestWritten = false;
+  let created = false;
   let stopAttempted = false;
-  let backupFailure;
+  let stopStartedAt = 0;
+  let capturedAt = now();
+  let primaryFailure;
+  let restartFailure;
+  const progressFailures = [];
+  let managedAssetSha256;
+  let verifiedManifest;
   try {
-    await mkdir6(backupDir, { recursive: true, mode: 448 });
+    await mkdir6(backupDir, { mode: 448 });
+    created = true;
+    artifact = "partial";
     await chmod6(backupDir, 448);
+    await report("maintenance", "Maintenance: stopping OR3 to capture the databases and files together.");
     stopAttempted = true;
+    stopStartedAt = Date.now();
+    service = "unknown";
     await stopProject(directory, state.mode);
+    service = "stopped";
+    capturedAt = now();
+    await report("capturing", `Backup capture: OR3 is stopped; writing ${backupDir}.`);
     await volumeArchive(directory, state.mode, env, backupDir);
-    await copySecure(deploymentPaths(directory).env, join14(backupDir, "config.env"));
-    const managedAssetSha256 = await snapshotManagedAssets(directory, state.mode, backupDir);
-    const manifest2 = {
-      schemaVersion: 1,
-      backupId,
-      createdAt: now(),
-      appVersion: state.appVersion,
-      image: state.image,
-      imageDigest: state.imageDigest,
-      dataSha256: await sha256File(join14(backupDir, "data.tgz")),
-      dataBytes: volumeSize,
-      configSha256: await sha256File(join14(backupDir, "config.env")),
-      managedAssetSha256,
-      managedAssetInventoryVersion: managedAssetSha256["compose.operator.yaml"] ? MANAGED_ASSET_INVENTORY_VERSION : managedAssetSha256["dashboard-operator.mjs"] ? 2 : undefined,
-      mode: state.mode,
-      domain: state.domain,
-      composeProject: state.composeProject,
-      volumeName: state.volumeName,
-      caddyDataVolume: state.caddyDataVolume,
-      caddyConfigVolume: state.caddyConfigVolume,
-      deploymentId: state.deploymentId,
-      port: state.port
-    };
-    const manifestContents = `${JSON.stringify(manifest2, null, 2)}
-`;
-    await writeSecure(join14(backupDir, "manifest.json"), manifestContents);
-    await writeBackupAuthentication(directory, backupDir, manifestContents);
-    const verifiedManifest = await readManifest(backupDir, directory);
-    manifestWritten = true;
-    return { backupId, backupDir, manifest: verifiedManifest };
+    await copySecure(deploymentPaths(directory).env, join15(backupDir, "config.env"));
+    managedAssetSha256 = await snapshotManagedAssets(directory, state.mode, backupDir);
+    artifact = "captured";
   } catch (error) {
-    if (!manifestWritten) {
-      await removeNamedBackupArtifact(directory, backupId).catch(() => {
-        return;
-      });
-      backupFailure = new Error(`${error instanceof Error ? error.message : String(error)} The partial backup artifact at ${backupDir} was removed.`);
-      throw backupFailure;
-    }
-    backupFailure = error instanceof Error ? error : new Error(String(error));
-    throw error;
+    primaryFailure = asError(error);
   } finally {
     if (stopAttempted && restartAfter && initiallyRunning) {
+      service = "restarting";
+      try {
+        await report("restarting", "Restart: bringing OR3 back and checking deep health; backup verification is still pending.");
+      } catch (error) {
+        progressFailures.push(asError(error));
+      }
       try {
         await startProject(directory, state.mode, env);
+        service = "healthy";
+        downtimeMs = Date.now() - stopStartedAt;
       } catch (error) {
-        const restartFailure = error instanceof Error ? error : new Error(String(error));
-        if (backupFailure) {
-          throw new AggregateError([backupFailure, restartFailure], `Backup failed and OR3 could not restart. Primary failure: ${backupFailure.message}. Restart failure: ${restartFailure.message}`);
-        }
-        throw new Error(`Backup was created but OR3 could not restart: ${error instanceof Error ? error.message : String(error)}`);
+        service = "unknown";
+        restartFailure = asError(error);
       }
     }
   }
+  if (!primaryFailure) {
+    try {
+      await report("verifying", `Verification: checking the captured backup; ${service === "healthy" ? "OR3 is deeply healthy" : service === "stopped" ? "OR3 remains stopped" : "service recovery is unverified"}.`);
+      const manifest2 = {
+        schemaVersion: 1,
+        backupId,
+        createdAt: capturedAt,
+        appVersion: state.appVersion,
+        image: state.image,
+        imageDigest: state.imageDigest,
+        dataSha256: await sha256File(join15(backupDir, "data.tgz")),
+        dataBytes: volumeSize,
+        configSha256: await sha256File(join15(backupDir, "config.env")),
+        managedAssetSha256,
+        managedAssetInventoryVersion: managedAssetSha256?.["compose.operator.yaml"] ? MANAGED_ASSET_INVENTORY_VERSION : managedAssetSha256?.["dashboard-operator.mjs"] ? 2 : undefined,
+        mode: state.mode,
+        domain: state.domain,
+        composeProject: state.composeProject,
+        volumeName: state.volumeName,
+        caddyDataVolume: state.caddyDataVolume,
+        caddyConfigVolume: state.caddyConfigVolume,
+        deploymentId: state.deploymentId,
+        port: state.port
+      };
+      const manifestContents = `${JSON.stringify(manifest2, null, 2)}
+`;
+      await writeSecure(join15(backupDir, "manifest.json"), manifestContents);
+      await writeBackupAuthentication(directory, backupDir, manifestContents);
+      if (!options.safetySnapshot)
+        await validateCanonicalBackupReferences(directory, { ...env, OR3_IMAGE: imageAtDigest(state.image, state.imageDigest) }, backupDir);
+      verifiedManifest = await readManifest(backupDir, directory);
+      artifact = "verified";
+    } catch (error) {
+      primaryFailure = asError(error);
+    }
+  }
+  let cleanupFailure;
+  let cleanupDetail = "";
+  if (primaryFailure && created && !verifiedManifest) {
+    try {
+      await report("cleanup", "Cleanup: removing only this operation’s unverified backup artifacts.");
+    } catch (error) {
+      progressFailures.push(asError(error));
+    }
+    const cleanup = await cleanupPartialBackup(directory, backupId);
+    cleanupFailure = cleanup.error;
+    cleanupDetail = cleanup.detail;
+    artifact = cleanup.removed ? "removed" : "unknown";
+  }
+  const failures = [primaryFailure, cleanupFailure, restartFailure, ...progressFailures].filter((error) => Boolean(error));
+  const serviceDetail = service === "healthy" ? `OR3 restarted and is deeply healthy (maintenance interval ${downtimeMs} ms).` : service === "stopped" ? "OR3 remains stopped as requested." : service === "running" ? "OR3 was not stopped; deep health has not been checked." : "Service recovery is unverified; OR3 may be unavailable.";
+  if (failures.length > 0) {
+    const message = [
+      primaryFailure ? `Backup failed: ${primaryFailure.message}` : `Verified backup retained at ${backupDir}.`,
+      cleanupFailure ? `Cleanup failed: ${cleanupFailure.message}` : "",
+      cleanupDetail,
+      restartFailure ? `Restart failed: ${restartFailure.message}` : "",
+      ...progressFailures.map((error) => `Progress recording failed: ${error.message}`),
+      serviceDetail,
+      RECOVERY_STEPS
+    ].filter(Boolean).join(" ");
+    try {
+      await report("failed", message);
+    } catch (error) {
+      failures.push(asError(error));
+    }
+    throw new AggregateError(failures, message);
+  }
+  try {
+    await report("complete", `Backup verified at ${backupDir}. ${serviceDetail}`);
+  } catch (error) {
+    throw new Error(`Verified backup retained at ${backupDir}. ${serviceDetail} Could not record backup completion: ${asError(error).message}. ${RECOVERY_STEPS}`, { cause: error });
+  }
+  return { backupId, backupDir, manifest: verifiedManifest };
+}
+async function inspectUnclaimedBackupArtifacts(directory, backupId, backupPath) {
+  if (!backupId || !backupPath || !BACKUP_ID_PATTERN.test(backupId))
+    return;
+  const expectedPath = resolve9(backupDirectory(directory, backupId));
+  if (resolve9(backupPath) !== expectedPath)
+    return;
+  const statuses = await Promise.all([
+    artifactStatus(expectedPath),
+    artifactStatus(join15(deploymentPaths(directory).exports, `${backupId}.json`))
+  ]);
+  const detail = statuses.map((status) => status.detail).join(" ");
+  return statuses.every((status) => status.absent) ? detail : `Maintenance warning: backup artifact ownership was not recorded. ${detail} This recovery did not verify or remove these paths. Inspect them with "npx @or3/cloud backup list".`;
 }
 async function cleanupJournaledPartialBackup(directory, backupId, backupPath) {
   if (!backupId || !backupPath || !BACKUP_ID_PATTERN.test(backupId))
     return;
   const expectedPath = resolve9(backupDirectory(directory, backupId));
-  if (resolve9(backupPath) !== expectedPath || !await fileExists(expectedPath))
+  if (resolve9(backupPath) !== expectedPath)
     return;
   try {
     await readManifest(expectedPath, directory);
+    return { artifact: "verified", detail: `Verified backup retained at ${expectedPath}.` };
   } catch {
-    await removeNamedBackupArtifact(directory, backupId);
+    const cleanup = await cleanupPartialBackup(directory, backupId);
+    if (cleanup.error)
+      throw new Error(`Partial backup cleanup failed: ${cleanup.error.message} ${cleanup.detail} ${RECOVERY_STEPS}`, { cause: cleanup.error });
+    return { artifact: "removed", detail: cleanup.detail };
   }
 }
 async function createPreMutationSnapshot(directory, state, env, backupId) {
-  return await createBackup(directory, state, env, { restartAfter: false, backupId });
+  return await createBackup(directory, state, env, { restartAfter: false, backupId, safetySnapshot: true });
 }
 
 // src/backup/export.ts
 import { chmod as chmod7, mkdir as mkdir7, realpath, rm as rm6, stat as stat3 } from "node:fs/promises";
-import { basename as basename5, dirname as dirname5, isAbsolute as isAbsolute3, join as join15, resolve as resolve10, sep } from "node:path";
+import { basename as basename5, dirname as dirname5, isAbsolute as isAbsolute3, join as join16, resolve as resolve10, sep } from "node:path";
 var PURGE_REQUIRES_BACKUP_WITHIN_MS = 24 * 60 * 60 * 1000;
 async function exportBackup(directory, backupId, destination) {
   const backupPath = await resolveBackup(directory, backupId);
@@ -3412,7 +3642,7 @@ async function exportBackup(directory, backupId, destination) {
   if (await fileExists(dest))
     throw new Error(`Destination ${dest} already exists. Choose a new empty destination so an export can never merge with unrelated files.`);
   await mkdir7(dirname5(dest), { recursive: true, mode: 448 });
-  const canonicalDestination = join15(await realpath(dirname5(dest)), basename5(dest));
+  const canonicalDestination = join16(await realpath(dirname5(dest)), basename5(dest));
   if (canonicalDestination === deploymentRoot || canonicalDestination.startsWith(`${deploymentRoot}${sep}`)) {
     throw new Error("Backup exports must live outside the managed deployment directory so `remove --purge-data` can never delete the remaining copy.");
   }
@@ -3423,19 +3653,19 @@ async function exportBackup(directory, backupId, destination) {
     }
     await chmod7(dest, 448);
     for (const file of ["data.tgz", "config.env", "manifest.json", "manifest.auth"]) {
-      await copySecure(join15(backupPath, file), join15(dest, file));
+      await copySecure(join16(backupPath, file), join16(dest, file));
     }
     if (manifest2.managedAssetSha256) {
       for (const name of managedAssetNamesForInventory(manifest2.mode, manifest2.managedAssetInventoryVersion)) {
-        await copySecure(join15(backupPath, "managed-assets", name), join15(dest, "managed-assets", name));
+        await copySecure(join16(backupPath, "managed-assets", name), join16(dest, "managed-assets", name));
       }
     }
-    const dataSha = await sha256File(join15(dest, "data.tgz"));
+    const dataSha = await sha256File(join16(dest, "data.tgz"));
     if (dataSha !== manifest2.dataSha256) {
       throw new Error(`Exported data.tgz checksum mismatch for ${manifest2.backupId}. Expected ${manifest2.dataSha256}, got ${dataSha}.`);
     }
     if (manifest2.configSha256) {
-      const configSha = await sha256File(join15(dest, "config.env"));
+      const configSha = await sha256File(join16(dest, "config.env"));
       if (configSha !== manifest2.configSha256) {
         throw new Error(`Exported config.env checksum mismatch for ${manifest2.backupId}.`);
       }
@@ -3452,12 +3682,12 @@ async function exportBackup(directory, backupId, destination) {
       dataSha256: manifest2.dataSha256,
       configSha256: manifest2.configSha256
     };
-    await writeSecure(join15(deploymentPaths(directory).exports, `${manifest2.backupId}.json`), `${JSON.stringify(receipt, null, 2)}
+    await writeSecure(join16(deploymentPaths(directory).exports, `${manifest2.backupId}.json`), `${JSON.stringify(receipt, null, 2)}
 `);
-    let bytes = (await stat3(join15(dest, "data.tgz"))).size + (await stat3(join15(dest, "config.env"))).size + (await stat3(join15(dest, "manifest.json"))).size + (await stat3(join15(dest, "manifest.auth"))).size;
+    let bytes = (await stat3(join16(dest, "data.tgz"))).size + (await stat3(join16(dest, "config.env"))).size + (await stat3(join16(dest, "manifest.json"))).size + (await stat3(join16(dest, "manifest.auth"))).size;
     if (manifest2.managedAssetSha256) {
       for (const name of managedAssetNamesForInventory(manifest2.mode, manifest2.managedAssetInventoryVersion)) {
-        bytes += (await stat3(join15(dest, "managed-assets", name))).size;
+        bytes += (await stat3(join16(dest, "managed-assets", name))).size;
       }
     }
     console.log(`Exported backup ${manifest2.backupId} (${bytes} bytes) to ${dest}`);
@@ -3480,7 +3710,7 @@ function assertPurgeBackupFreshness(backups, nowMs) {
 }
 async function readBackupExportReceipt(directory, backupId) {
   try {
-    const receipt = JSON.parse(await readText(join15(deploymentPaths(directory).exports, `${backupId}.json`)));
+    const receipt = JSON.parse(await readText(join16(deploymentPaths(directory).exports, `${backupId}.json`)));
     if (receipt.schemaVersion !== 1 || receipt.backupId !== backupId || typeof receipt.destination !== "string" || !isAbsolute3(receipt.destination) || typeof receipt.destinationDevice !== "number" || !receipt.dataSha256)
       return;
     return receipt;
@@ -3530,26 +3760,57 @@ async function backupCreateCommand(directory) {
     id: id("backup-operation"),
     operation: "backup",
     startedAt: now(),
-    message: "Creating a stopped-volume backup",
+    message: "Preflight: preparing a stopped-volume backup",
     backupId,
     backupPath: backupDirectory(loaded.directory, backupId),
     initialAppRunning,
     phase: "prepared"
   };
   await markPending(loaded.directory, loaded.state, pending);
+  let result;
+  try {
+    result = await createBackup(loaded.directory, loaded.state, loaded.env, {
+      backupId,
+      initiallyRunning: initialAppRunning,
+      onProgress: async (progress) => {
+        const message = redact(progress.message, secretValues(loaded.env));
+        console.error(message);
+        await updatePending(loaded.directory, loaded.state, { message, backupProgress: { ...progress, message } });
+      }
+    });
+    const completed = structuredClone(loaded.state);
+    completed.lastError = undefined;
+    await clearPending(loaded.directory, completed);
+    loaded.state = completed;
+  } catch (error) {
+    const detail = error instanceof Error ? error.message : String(error);
+    loaded.state.lastError = redact(result ? `Verified backup retained at ${result.backupDir}. Completion recording failed: ${detail}. Keep the operation record and run "npx @or3/cloud recover --dry-run", then "npx @or3/cloud recover".` : detail, secretValues(loaded.env));
+    if (result && loaded.state.incompleteOperation?.backupProgress) {
+      loaded.state.incompleteOperation.backupProgress = {
+        ...loaded.state.incompleteOperation.backupProgress,
+        stage: "failed",
+        message: loaded.state.lastError
+      };
+    }
+    try {
+      await writeState(loaded.directory, loaded.state);
+    } catch (stateError) {
+      throw new AggregateError([error, stateError], `${loaded.state.lastError} Could not record the backup failure: ${redact(stateError instanceof Error ? stateError.message : String(stateError), secretValues(loaded.env))}. Keep the existing operation record and run "npx @or3/cloud recover --dry-run".`);
+    }
+    if (result)
+      throw new Error(loaded.state.lastError, { cause: error });
+    throw error;
+  }
+  console.log(`Backup ${result.backupId} created at ${result.backupDir}`);
+  console.log(`SHA-256: ${result.manifest.dataSha256}`);
+  console.error("Maintenance: checking backup retention after snapshot verification and service recovery.");
   try {
-    const result = await createBackup(loaded.directory, loaded.state, loaded.env, { backupId, initiallyRunning: initialAppRunning });
-    await clearPending(loaded.directory, loaded.state);
     const prune = await pruneBackups(loaded.directory, loaded.state, BACKUP_RETENTION_KEEP, false, { automatic: true });
-    console.log(`Backup ${result.backupId} created at ${result.backupDir}`);
-    console.log(`SHA-256: ${result.manifest.dataSha256}`);
     if (prune.deferred.length > 0) {
       console.warn(`Maintenance warning: ${prune.deferred.length} backup entr${prune.deferred.length === 1 ? "y" : "ies"} need inspection; all backups were preserved. Run "npx @or3/cloud backup list" for details.`);
     }
   } catch (error) {
-    loaded.state.lastError = redact(error instanceof Error ? error.message : String(error), secretValues(loaded.env));
-    await writeState(loaded.directory, loaded.state);
-    throw error;
+    console.warn(`Maintenance warning: backup ${result.backupId} is verified and committed, but retention did not complete: ${redact(error instanceof Error ? error.message : String(error), secretValues(loaded.env))}. Run "npx @or3/cloud backup list" to inspect the remaining backups.`);
   }
 }
 async function backupListCommand(directory, flags = {}) {
@@ -3653,7 +3914,7 @@ async function backupCommand(directory, positionals, flags) {
 
 // src/credentials/reset.ts
 import { rm as rm7 } from "node:fs/promises";
-import { join as join16 } from "node:path";
+import { join as join17 } from "node:path";
 
 // src/credentials/reset-script.ts
 function buildCredentialsResetScript(input2) {
@@ -3846,7 +4107,7 @@ async function applyCredentialReset(directory, state, nextEnv) {
   await runCredentialsResetScript(directory, state.mode, values);
   const runtimeEnv = withoutProvisioningCredentials(nextEnv);
   await writeSecure(deploymentPaths(directory).env, serializeEnv(runtimeEnv));
-  const initialCredentials = join16(directory, ".or3-initial-credentials");
+  const initialCredentials = join17(directory, ".or3-initial-credentials");
   await rm7(initialCredentials, { force: true });
   await stopProject(directory, state.mode);
   await startProject(directory, state.mode, runtimeEnv);
@@ -3908,7 +4169,7 @@ async function credentialsCommand(directory, positionals, flags) {
 // src/commands/doctor.ts
 import { existsSync as existsSync3 } from "node:fs";
 import { stat as stat4 } from "node:fs/promises";
-import { basename as basename6, join as join17, resolve as resolve11 } from "node:path";
+import { basename as basename6, join as join18, resolve as resolve11 } from "node:path";
 async function doctorCommand(directory) {
   const resolved = resolve11(directory);
   const paths = deploymentPaths(resolved);
@@ -3933,7 +4194,7 @@ async function doctorCommand(directory) {
     failures.push(error instanceof Error ? error.message : String(error));
     console.log("✗ Managed state or .env is missing/invalid");
   }
-  for (const file of [paths.env, paths.state, join17(resolved, ".or3-initial-credentials")]) {
+  for (const file of [paths.env, paths.state, join18(resolved, ".or3-initial-credentials")]) {
     if (!existsSync3(file))
       continue;
     const mode = (await stat4(file)).mode & 511;
@@ -4005,7 +4266,7 @@ async function doctorCommand(directory) {
           }
           assertDashboardOperatorMounts(container.Mounts ?? [], resolved);
           const ipcDirectory = await stat4(paths.operatorIpc);
-          const operatorSocket = await stat4(join17(paths.operatorIpc, "operator.sock"));
+          const operatorSocket = await stat4(join18(paths.operatorIpc, "operator.sock"));
           const operatorUid = Number(env.OR3_OPERATOR_UID);
           const operatorGid = Number(env.OR3_OPERATOR_GID);
           if ((ipcDirectory.mode & 511) !== 456 || ipcDirectory.uid !== operatorUid || ipcDirectory.gid !== operatorGid || !operatorSocket.isSocket() || (operatorSocket.mode & 511) !== 432 || operatorSocket.uid !== operatorUid || operatorSocket.gid !== operatorGid) {
@@ -4064,7 +4325,7 @@ async function doctorCommand(directory) {
 
 // src/commands/init.ts
 import { chmod as chmod8, mkdir as mkdir8 } from "node:fs/promises";
-import { join as join18, resolve as resolve12 } from "node:path";
+import { join as join19, resolve as resolve12 } from "node:path";
 async function initCommand(positionals, flags) {
   const hasLocal = boolFlag(flags, "local");
   const hasPublic = boolFlag(flags, "public");
@@ -4072,7 +4333,7 @@ async function initCommand(positionals, flags) {
     throw new Error("Choose exactly one of --local or --public.");
   const mode = hasPublic ? "public" : "local";
   const directory = resolve12(process.cwd(), positionals[0] ?? "or3-cloud");
-  if (await fileExists(join18(directory, ".or3-cloud", "state.json"))) {
+  if (await fileExists(join19(directory, ".or3-cloud", "state.json"))) {
     throw new Error(`${directory} is already a managed deployment. Use update, doctor, or adopt.`);
   }
   await readDirectoryEmpty(directory);
@@ -4123,7 +4384,7 @@ ${existing.stderr}`)) {
   await copyAssets(directory, mode);
   const env = buildEnv({ mode, version, directory, image, email, password, domain, port, dashboardOperator: operator });
   await writeSecure(deploymentPaths(directory).env, serializeEnv(env));
-  await writeSecure(join18(directory, ".or3-initial-credentials"), serializeInitialCredentials({
+  await writeSecure(join19(directory, ".or3-initial-credentials"), serializeInitialCredentials({
     bootstrapEmail: email,
     bootstrapPassword: password,
     adminUsername: email,
@@ -4146,7 +4407,7 @@ ${existing.stderr}`)) {
     await clearPending(directory, state);
     console.log(`
 OR3 Cloud ${version} is running at ${mode === "public" ? `https://${domain}` : `http://127.0.0.1:${port}`}`);
-    console.log(`Credentials were written to ${join18(directory, ".or3-initial-credentials")} (mode 0600). Move them to protected storage (a password manager), then remove that file.`);
+    console.log(`Credentials were written to ${join19(directory, ".or3-initial-credentials")} (mode 0600). Move them to protected storage (a password manager), then remove that file.`);
     console.log('The bootstrap owner signs in to the app; the admin account manages the deployment. Rotate either later with "npx @or3/cloud credentials reset --yes".');
     console.log(`
 Check: cd ${quote(directory)} && npx @or3/cloud doctor`);
@@ -4191,28 +4452,93 @@ async function restartCommand(directory) {
 }
 
 // src/commands/recover.ts
-import { join as join21 } from "node:path";
+import { join as join22 } from "node:path";
 
 // src/backup/restore.ts
-import { join as join19 } from "node:path";
+import { join as join20 } from "node:path";
+function assertFilesystemPayloadHash(key, readChunk, createHash3) {
+  const match = /^(sha256|md5)_([a-fA-F0-9]+)$/.exec(key);
+  const algorithm = match?.[1];
+  const expected = match?.[2];
+  if (algorithm !== "sha256" && algorithm !== "md5" || expected?.length !== (algorithm === "sha256" ? 64 : 32)) {
+    throw new Error("Invalid canonical filesystem payload key");
+  }
+  const hash = createHash3(algorithm);
+  for (let chunk = readChunk();chunk !== undefined; chunk = readChunk())
+    hash.update(chunk);
+  if (hash.digest("hex") !== expected.toLowerCase())
+    throw new Error("Filesystem payload hash mismatch");
+}
+var VERIFY_RESTORED_DATA_SCRIPT = `
+const fs = require('node:fs');
+const Database = require('/app/.output/server/node_modules/better-sqlite3');
+const validFilesystemPayload = ${assertFilesystemPayloadHash.toString()};
+const paths = [process.env.OR3_BASIC_AUTH_DB_PATH, process.env.OR3_SQLITE_DB_PATH].filter(Boolean);
+if (paths.length !== 2) throw new Error('Both managed database paths must be configured');
+for (const path of paths) {
+  const databasePath = fs.realpathSync(path);
+  if (!fs.statSync(databasePath).isFile()) throw new Error('Restored database payload must be a regular file');
+  const walPath = databasePath + '-wal';
+  if (fs.existsSync(walPath)) {
+    const info = fs.statSync(walPath);
+    if (!info.isFile()) throw new Error('Restored WAL payload must be a regular file');
+
+  }
+  const db = new Database(databasePath, { readonly: true, fileMustExist: true });
+  try {
+    if (db.pragma('quick_check', { simple: true }) !== 'ok') throw new Error('SQLite quick_check failed');
+    const tables = db.prepare("select count(*) as count from sqlite_master where type = 'table'").get().count;
+    if (tables < 1) throw new Error('Restored database has no tables');
+  } finally { db.close(); }
+}
+const path = require('node:path');
+const crypto = require('node:crypto');
+const storageRoot = path.resolve(process.env.OR3_STORAGE_FS_ROOT || '/data/storage');
+const workspacesRoot = path.join(storageRoot, 'workspaces');
+if ((storageRoot === '/data' || storageRoot.startsWith('/data/')) && fs.existsSync(workspacesRoot)) {
+  const chunk = Buffer.alloc(65536);
+  for (const workspace of fs.readdirSync(workspacesRoot, { withFileTypes: true }).filter(entry => /^[a-zA-Z0-9_-]+$/.test(entry.name))) {
+    const directory = path.join(workspacesRoot, workspace.name);
+    if (!fs.statSync(directory).isDirectory()) continue;
+    for (const entry of fs.readdirSync(directory)) {
+      if (!/^(sha256_[a-fA-F0-9]{64}|md5_[a-fA-F0-9]{32})[.]meta[.]json$/.test(entry)) continue;
+      const key = entry.slice(0, -10);
+      const file = path.join(directory, key);
+      if (!fs.statSync(file).isFile()) throw new Error('Restored filesystem payload must be a regular file');
+      const fd = fs.openSync(file, 'r');
+      try {
+        validFilesystemPayload(key, () => {
+          const count = fs.readSync(fd, chunk, 0, chunk.length, null);
+          return count ? chunk.subarray(0, count) : undefined;
+        }, algorithm => crypto.createHash(algorithm));
+      } finally { fs.closeSync(fd); }
+    }
+  }
+}
+`;
 async function assertRestoreFreeSpace(directory, state, env, manifest2, backupPath) {
-  const archiveBytes = await gzipUncompressedBytes(join19(backupPath, "data.tgz"));
+  const archiveBytes = await gzipUncompressedBytes(join20(backupPath, "data.tgz"));
   const requiredBytes = Math.max(manifest2.dataBytes ?? 0, archiveBytes);
   const freeBytes = await dataVolumeFreeBytes(directory, state.mode, env);
-  const reclaimableBytes = await dataVolumeSize(directory, state.mode, env);
+  const reclaimableBytes = await dataVolumeSize(directory, state.mode, env, "allocated");
   assertEnoughFreeSpace(freeBytes + reclaimableBytes, requiredBytes, "Restore");
 }
 async function restoreBackupData(directory, state, env, backupPath, options = {}) {
   const manifest2 = await readManifest(backupPath, directory);
   assertRestorableManagedAssets(manifest2);
-  const backupEnv = parseEnv(await readText(join19(backupPath, "config.env")));
+  const backupEnv = parseEnv(await readText(join20(backupPath, "config.env")));
   if (backupEnv.OR3_VERSION !== manifest2.appVersion || backupEnv.OR3_IMAGE !== manifest2.image) {
     throw new Error(`Backup ${manifest2.backupId} configuration does not match its manifest.`);
   }
   assertBackupMatchesDeployment(manifest2, backupEnv, state, env);
   if (manifest2.imageDigest)
     await requireImageDigest(manifest2.image, manifest2.imageDigest, `Backup ${manifest2.backupId}`);
-  await validateVolumeArchive(directory, state.mode, env, backupPath);
+  await validateVolumeArchive(directory, state.mode, env, backupPath, backupEnv.OR3_STORAGE_FS_ROOT);
+  if (!options.previousSafetySnapshot) {
+    await validateCanonicalBackupReferences(directory, { ...backupEnv, OR3_IMAGE: imageAtDigest(manifest2.image, manifest2.imageDigest) }, backupPath);
+  } else {
+    console.warn("Restoring the recorded byte-preservation safety snapshot; canonical attachment completeness is not certified. Verify previously uploaded files.");
+  }
   await ensureManagedDataVolume(directory, state.mode, state, env);
   await assertRestoreFreeSpace(directory, state, env, manifest2, backupPath);
   const restoresOperatorOverlay = manifest2.managedAssetInventoryVersion === MANAGED_ASSET_INVENTORY_VERSION && Boolean(manifest2.managedAssetSha256?.["compose.operator.yaml"]);
@@ -4233,6 +4559,33 @@ async function restoreBackupData(directory, state, env, backupPath, options = {}
   await writeSecure(deploymentPaths(directory).env, serializeEnv(restoredEnv));
   await restoreManagedAssets(directory, backupPath, manifest2);
   await restoreVolumeArchive(directory, state.mode, restoredEnv, backupPath);
+  const snapshotCheck = await run("docker", [
+    "run",
+    "--rm",
+    "--network",
+    "none",
+    "--read-only",
+    "--security-opt",
+    "no-new-privileges:true",
+    "--cap-drop",
+    "ALL",
+    "-v",
+    `${state.volumeName}:/data`,
+    "--env",
+    `OR3_BASIC_AUTH_DB_PATH=${restoredEnv.OR3_BASIC_AUTH_DB_PATH ?? ""}`,
+    "--env",
+    `OR3_SQLITE_DB_PATH=${restoredEnv.OR3_SQLITE_DB_PATH ?? ""}`,
+    "--env",
+    `OR3_STORAGE_FS_ROOT=${restoredEnv.OR3_STORAGE_FS_ROOT ?? "/data/storage"}`,
+    "--entrypoint",
+    "sh",
+    restoredEnv.OR3_IMAGE,
+    ...containerNodeCommand(VERIFY_RESTORED_DATA_SCRIPT).slice(1)
+  ], directory);
+  if (!snapshotCheck.ok) {
+    throw new Error(`Restored snapshot integrity verification failed. ${redact(snapshotCheck.stderr, secretValues(restoredEnv))}`);
+  }
+  await options.beforeStart?.();
   await startProject(directory, state.mode, restoredEnv);
   return manifest2;
 }
@@ -4243,14 +4596,14 @@ async function restorePreMutationSnapshot(directory, state, env) {
   const previous = await recordedBackupPath(directory, pending);
   await updatePending(directory, state, { phase: "restoring-previous" });
   if (pending.recreateDataVolume) {
-    await restoreBackupData(directory, state, env, previous.path, { recreateDataVolume: true });
+    await restoreBackupData(directory, state, env, previous.path, { recreateDataVolume: true, previousSafetySnapshot: true });
     return previous;
   }
   if (pending.previousRootOwnership) {
     await pullAndRequireImage(state.image, state.imageDigest, "Previous deployment");
     await setManagedVolumeRootOwnership(directory, state, env, state.image, pending.previousRootOwnership);
   }
-  await restoreBackupData(directory, state, env, previous.path);
+  await restoreBackupData(directory, state, env, previous.path, { previousSafetySnapshot: true });
   return previous;
 }
 
@@ -4261,11 +4614,11 @@ import { resolve as resolve13 } from "node:path";
 // src/deployment/lease.ts
 import { randomBytes as randomBytes8 } from "node:crypto";
 import { mkdir as mkdir9, rename as rename2, rm as rm8 } from "node:fs/promises";
-import { dirname as dirname6, join as join20 } from "node:path";
+import { dirname as dirname6, join as join21 } from "node:path";
 var DASHBOARD_LEASE_STALE_MS = 30000;
 async function readLeaseOwner(leasePath) {
   try {
-    const owner = JSON.parse(await readText(join20(leasePath, "owner.json")));
+    const owner = JSON.parse(await readText(join21(leasePath, "owner.json")));
     if (owner.schemaVersion !== 1 || typeof owner.nonce !== "string" || typeof owner.command !== "string" || owner.origin !== "cli" && owner.origin !== "dashboard" || !Number.isSafeInteger(owner.pid) || typeof owner.acquiredAt !== "string" || typeof owner.heartbeatAt !== "string")
       return;
     return owner;
@@ -4324,7 +4677,7 @@ async function acquireDeploymentLease(directory, command) {
     }
   }
   try {
-    await writeSecure(join20(leasePath, "owner.json"), `${JSON.stringify(owner, null, 2)}
+    await writeSecure(join21(leasePath, "owner.json"), `${JSON.stringify(owner, null, 2)}
 `);
   } catch (error) {
     await rm8(leasePath, { recursive: true, force: true }).catch(() => {
@@ -4334,7 +4687,7 @@ async function acquireDeploymentLease(directory, command) {
   }
   const heartbeat = setInterval(() => {
     owner.heartbeatAt = now();
-    writeSecure(join20(leasePath, "owner.json"), `${JSON.stringify(owner, null, 2)}
+    writeSecure(join21(leasePath, "owner.json"), `${JSON.stringify(owner, null, 2)}
 `).catch(() => {
       return;
     });
@@ -4484,7 +4837,14 @@ function publicStateProjection(state) {
       origin: pending.origin ?? null,
       dashboardJobId: pending.dashboardJobId ?? null,
       targetVersion: pending.targetVersion ?? null,
-      targetImageDigest: pending.targetImageDigest ?? null
+      targetImageDigest: pending.targetImageDigest ?? null,
+      backupProgress: pending.backupProgress ? {
+        stage: pending.backupProgress.stage,
+        service: pending.backupProgress.service,
+        artifact: pending.backupProgress.artifact,
+        downtimeMs: pending.backupProgress.downtimeMs ?? null,
+        message: redact(pending.backupProgress.message)
+      } : null
     } : null,
     lastReceipt: state.lastReceipt ?? null,
     lastError: state.lastError ? redact(state.lastError) : null
@@ -4827,8 +5187,51 @@ async function runRecovery(directory, context) {
   }
   const requiresExplicitRestore = decision.action === "require-explicit-restore";
   try {
+    if (pending.operation === "backup") {
+      const failures = [];
+      let service = pending.initialAppRunning === false ? "stopped" : "unknown";
+      let artifact = pending.backupProgress?.artifact ?? "not-created";
+      let artifactDetail = "";
+      if (pending.initialAppRunning !== false) {
+        report("Restart: recovering OR3 and checking deep health before backup cleanup.");
+        try {
+          await pullAndRequireImage(loaded.state.image, loaded.state.imageDigest, "Current deployment");
+          await startProject(loaded.directory, loaded.state.mode, loaded.env);
+          service = "healthy";
+        } catch (error) {
+          failures.push(error instanceof Error ? error : new Error(String(error)));
+        }
+      }
+      try {
+        if (artifact === "not-created") {
+          artifactDetail = await inspectUnclaimedBackupArtifacts(loaded.directory, pending.backupId, pending.backupPath) ?? "";
+        } else {
+          const cleanup = await cleanupJournaledPartialBackup(loaded.directory, pending.backupId, pending.backupPath);
+          if (cleanup) {
+            artifact = cleanup.artifact;
+            report(cleanup.detail);
+          }
+        }
+      } catch (error) {
+        if (artifact !== "not-created")
+          artifact = "unknown";
+        failures.push(error instanceof Error ? error : new Error(String(error)));
+      }
+      const serviceDetail = service === "healthy" ? "OR3 is deeply healthy." : service === "stopped" ? "The intentionally stopped OR3 service was preserved." : "Service recovery is unverified; OR3 may be unavailable.";
+      if (failures.length > 0) {
+        const message = [failures.map((error) => error.message).join(" "), serviceDetail, artifactDetail].filter(Boolean).join(" ");
+        pending.backupProgress = { stage: "failed", service, artifact, message: redact(message, secretValues(loaded.env)) };
+        throw new AggregateError(failures, message);
+      }
+      const completed = structuredClone(loaded.state);
+      completed.lastError = undefined;
+      await clearPending(loaded.directory, completed);
+      const detail2 = [`Recovered the incomplete backup operation. ${serviceDetail}`, artifactDetail].filter(Boolean).join(" ");
+      report(detail2);
+      return { kind: "recovered", operation: "backup", detail: detail2 };
+    }
     if (pending.phase === "prepared") {
-      if (pending.operation === "backup" || pending.operation === "update" || pending.operation === "adopt") {
+      if (pending.operation === "update" || pending.operation === "adopt") {
         await cleanupJournaledPartialBackup(loaded.directory, pending.backupId, pending.backupPath);
       }
       if (pending.operation === "restore" || pending.operation === "rollback") {
@@ -4906,17 +5309,10 @@ async function runRecovery(directory, context) {
       await stopProject(loaded.directory, loaded.state.mode);
       await restoreVolumeArchive(loaded.directory, loaded.state.mode, loaded.env, sourceBackupPath);
     }
-    if (pending.operation === "backup" && pending.initialAppRunning === false) {
-      loaded.state.lastError = undefined;
-      await clearPending(loaded.directory, loaded.state);
-      const detail2 = "Recovered the incomplete backup operation and preserved the intentionally stopped OR3 service.";
-      report(detail2);
-      return { kind: "recovered", operation: "backup", detail: detail2 };
-    }
     await pullAndRequireImage(loaded.state.image, loaded.state.imageDigest, "Current deployment");
     await startProject(loaded.directory, loaded.state.mode, loaded.env);
     let recoveredEnv = loaded.env;
-    if ((pending.operation === "init" || pending.operation === "adopt") && loaded.env.OR3_BASIC_AUTH_BOOTSTRAP_EMAIL && loaded.env.OR3_BASIC_AUTH_BOOTSTRAP_PASSWORD && loaded.env.OR3_ADMIN_PASSWORD) {
+    if (loaded.env.OR3_BASIC_AUTH_BOOTSTRAP_EMAIL && loaded.env.OR3_BASIC_AUTH_BOOTSTRAP_PASSWORD && loaded.env.OR3_ADMIN_PASSWORD) {
       await provisionManagedCredentials(loaded.state.mode === "public" ? new URL(`https://${loaded.env.OR3_PUBLIC_DOMAIN}`) : new URL(`http://127.0.0.1:${loaded.env.OR3_PORT}`), loaded.env.OR3_BASIC_AUTH_BOOTSTRAP_EMAIL, loaded.env.OR3_BASIC_AUTH_BOOTSTRAP_PASSWORD, loaded.env.OR3_ADMIN_USERNAME, loaded.env.OR3_ADMIN_PASSWORD);
       recoveredEnv = withoutProvisioningCredentials(loaded.env);
       await writeSecure(deploymentPaths(loaded.directory).env, serializeEnv(recoveredEnv));
@@ -4939,11 +5335,11 @@ async function runRecovery(directory, context) {
         await compose(loaded.directory, loaded.state.mode, ["down"]).catch(() => {
           return;
         });
-        const sourceEnv = parseEnv(await readText(join21(pending.sourceDirectory, ".env")));
-        const sourceFiles = ["-f", join21(pending.sourceDirectory, "compose.yaml")];
+        const sourceEnv = parseEnv(await readText(join22(pending.sourceDirectory, ".env")));
+        const sourceFiles = ["-f", join22(pending.sourceDirectory, "compose.yaml")];
         const sourceModeValue = sourceMode(pending.sourceDirectory, sourceEnv);
         if (sourceModeValue === "public")
-          sourceFiles.push("-f", join21(pending.sourceDirectory, "compose.public.yaml"));
+          sourceFiles.push("-f", join22(pending.sourceDirectory, "compose.public.yaml"));
         if (pending.sourceInitiallyRunning !== false) {
           await restartSource(pending.sourceDirectory, sourceFiles, secretValues(sourceEnv));
         }
@@ -4958,7 +5354,7 @@ async function runRecovery(directory, context) {
 
 // src/commands/remove.ts
 import { rm as rm9 } from "node:fs/promises";
-import { join as join22 } from "node:path";
+import { join as join23 } from "node:path";
 function purgeVolumesFromState(state) {
   const volumes = [state.volumeName];
   if (state.mode === "public") {
@@ -4995,7 +5391,7 @@ async function removeCommand(directory, flags) {
   for (const volume of volumes)
     console.log(`  docker volume ${volume}`);
   for (const file of managedFiles)
-    console.log(`  ${join22(directory, file)}`);
+    console.log(`  ${join23(directory, file)}`);
   await compose(directory, state.mode, ["down"]);
   await assertPurgeHasVerifiedExport(directory, await enumerateBackups(directory), Date.now());
   for (const volume of volumes) {
@@ -5005,13 +5401,13 @@ async function removeCommand(directory, flags) {
 ${result.stderr}`);
   }
   for (const file of managedFiles) {
-    await rm9(join22(directory, file), { recursive: true, force: true });
+    await rm9(join23(directory, file), { recursive: true, force: true });
   }
   console.log(`Purged: data volumes, backups, .env, managed state, and compose files were deleted. The verified export at ${exportReceipt.destination} is the remaining copy.`);
 }
 
 // src/commands/restore.ts
-import { join as join23 } from "node:path";
+import { join as join24 } from "node:path";
 async function restoreCommand(directory, flags, positionals) {
   if (!boolFlag(flags, "yes"))
     throw new Error("Restore replaces live data. Re-run with --yes after confirming the backup and data-loss boundary.");
@@ -5024,7 +5420,7 @@ async function restoreCommand(directory, flags, positionals) {
   const backupPath = await resolveBackup(loaded.directory, backupValue);
   const manifest2 = await readManifest(backupPath, loaded.directory);
   assertRestorableManagedAssets(manifest2);
-  const backupEnv = parseEnv(await readText(join23(backupPath, "config.env")));
+  const backupEnv = parseEnv(await readText(join24(backupPath, "config.env")));
   if (backupEnv.OR3_VERSION !== manifest2.appVersion || backupEnv.OR3_IMAGE !== manifest2.image) {
     throw new Error(`Backup ${manifest2.backupId} configuration does not match its manifest.`);
   }
@@ -5034,6 +5430,8 @@ async function restoreCommand(directory, flags, positionals) {
     await pullAndRequireImage(manifest2.image, manifest2.imageDigest, `Backup ${manifest2.backupId}`);
   await assertSupportedHostArchitecture(manifest2.image);
   await assertRestoreFreeSpace(loaded.directory, loaded.state, loaded.env, manifest2, backupPath);
+  await validateVolumeArchive(loaded.directory, loaded.state.mode, loaded.env, backupPath, backupEnv.OR3_STORAGE_FS_ROOT);
+  await validateCanonicalBackupReferences(loaded.directory, { ...backupEnv, OR3_IMAGE: imageAtDigest(manifest2.image, manifest2.imageDigest) }, backupPath);
   const pending = {
     id: id("restore"),
     operation: "restore",
@@ -5063,7 +5461,10 @@ async function restoreCommand(directory, flags, positionals) {
       phase: "snapshot-created"
     });
     await updatePending(loaded.directory, loaded.state, { phase: "target-mutating" });
-    await restoreBackupData(loaded.directory, loaded.state, loaded.env, backupPath, { recreateDataVolume });
+    await restoreBackupData(loaded.directory, loaded.state, loaded.env, backupPath, {
+      recreateDataVolume,
+      beforeStart: () => updatePending(loaded.directory, loaded.state, { phase: "starting-target" })
+    });
     const restoredEnv = parseEnv(await readText(deploymentPaths(loaded.directory).env));
     const digest = await imageDigest(restoredEnv.OR3_IMAGE);
     const nextState = stateFromEnv(loaded.directory, restoredEnv, loaded.state.mode, "restore", digest);
@@ -5074,7 +5475,12 @@ async function restoreCommand(directory, flags, positionals) {
   } catch (error) {
     const original = redact(error instanceof Error ? error.message : String(error), secretValues(loaded.env));
     const phase = loaded.state.incompleteOperation?.phase;
-    const targetMayHaveMutated = phase === "target-mutating" || phase === "restoring-previous" || phase === "starting-target";
+    if (phase === "starting-target") {
+      loaded.state.lastError = `Restore of ${manifest2.backupId} started the target but did not complete: ${original}. Writes may have been accepted, so the deployment was NOT restored automatically. Keep the operation record and run "npx @or3/cloud recover --dry-run", then "npx @or3/cloud recover --restore --yes" only after confirming loss of post-replacement writes.`;
+      await writeState(loaded.directory, loaded.state);
+      throw new Error(loaded.state.lastError, { cause: error });
+    }
+    const targetMayHaveMutated = phase === "target-mutating" || phase === "restoring-previous";
     if (targetMayHaveMutated && (loaded.state.incompleteOperation?.previousBackupPath || loaded.state.incompleteOperation?.previousBackupId)) {
       let recoveredMessage;
       try {
@@ -5104,7 +5510,7 @@ async function restoreCommand(directory, flags, positionals) {
 }
 
 // src/commands/rollback.ts
-import { join as join24 } from "node:path";
+import { join as join25 } from "node:path";
 async function rollbackCommand(directory, flags) {
   if (!boolFlag(flags, "yes"))
     throw new Error("Rollback restores the previous image and data snapshot. Re-run with --yes after confirming post-update data loss.");
@@ -5120,7 +5526,7 @@ async function rollbackCommand(directory, flags) {
   if (manifest2.appVersion !== point.appVersion || manifest2.image !== point.image || manifest2.imageDigest !== point.imageDigest) {
     throw new Error(`Rollback point ${point.backupId} no longer matches its recorded image/version. Refusing to mutate the deployment.`);
   }
-  const backupEnv = parseEnv(await readText(join24(backupPath, "config.env")));
+  const backupEnv = parseEnv(await readText(join25(backupPath, "config.env")));
   if (backupEnv.OR3_VERSION !== manifest2.appVersion || backupEnv.OR3_IMAGE !== manifest2.image) {
     throw new Error(`Backup ${manifest2.backupId} configuration does not match its manifest.`);
   }
@@ -5129,6 +5535,7 @@ async function rollbackCommand(directory, flags) {
   await pullAndRequireImage(point.image, point.imageDigest, "Rollback");
   await assertSupportedHostArchitecture(point.image);
   await assertRestoreFreeSpace(loaded.directory, loaded.state, loaded.env, manifest2, backupPath);
+  await validateVolumeArchive(loaded.directory, loaded.state.mode, loaded.env, backupPath, backupEnv.OR3_STORAGE_FS_ROOT);
   const pending = {
     id: id("rollback"),
     operation: "rollback",
@@ -5158,7 +5565,11 @@ async function rollbackCommand(directory, flags) {
       phase: "snapshot-created"
     });
     await updatePending(loaded.directory, loaded.state, { phase: "target-mutating" });
-    await restoreBackupData(loaded.directory, loaded.state, loaded.env, backupPath, { recreateDataVolume });
+    await restoreBackupData(loaded.directory, loaded.state, loaded.env, backupPath, {
+      recreateDataVolume,
+      previousSafetySnapshot: true,
+      beforeStart: () => updatePending(loaded.directory, loaded.state, { phase: "starting-target" })
+    });
     const restoredEnv = parseEnv(await readText(deploymentPaths(loaded.directory).env));
     const nextState = stateFromEnv(loaded.directory, restoredEnv, loaded.state.mode, "restore", await imageDigest(restoredEnv.OR3_IMAGE));
     nextState.rollback = undefined;
@@ -5168,7 +5579,12 @@ async function rollbackCommand(directory, flags) {
   } catch (error) {
     const original = redact(error instanceof Error ? error.message : String(error), secretValues(loaded.env));
     const phase = loaded.state.incompleteOperation?.phase;
-    const targetMayHaveMutated = phase === "target-mutating" || phase === "restoring-previous" || phase === "starting-target";
+    if (phase === "starting-target") {
+      loaded.state.lastError = `Rollback to ${point.appVersion} started the target but did not complete: ${original}. Writes may have been accepted, so the deployment was NOT restored automatically. Keep the operation record and run "npx @or3/cloud recover --dry-run", then "npx @or3/cloud recover --restore --yes" only after confirming loss of post-replacement writes.`;
+      await writeState(loaded.directory, loaded.state);
+      throw new Error(loaded.state.lastError, { cause: error });
+    }
+    const targetMayHaveMutated = phase === "target-mutating" || phase === "restoring-previous";
     if (targetMayHaveMutated && (loaded.state.incompleteOperation?.previousBackupPath || loaded.state.incompleteOperation?.previousBackupId)) {
       let recoveredMessage;
       try {
@@ -5240,6 +5656,8 @@ async function statusCommand(directory, flags = {}) {
     console.log(`  last successful operation: ${state.lastSuccessfulOperation} (${state.updatedAt})`);
     if (state.incompleteOperation)
       console.log(`  incomplete operation: ${state.incompleteOperation.operation} (${state.incompleteOperation.id})`);
+    if (state.incompleteOperation?.backupProgress)
+      console.log(`  last backup milestone: ${redact(state.incompleteOperation.backupProgress.message)}`);
     if (state.lastError)
       console.log(`  last error: ${state.lastError}`);
   }
@@ -5319,7 +5737,7 @@ ${result.stderr}`);
 
 // src/commands/update.ts
 import { readFile as readFile3, statfs as statfs2 } from "node:fs/promises";
-import { join as join25, resolve as resolve14 } from "node:path";
+import { join as join26, resolve as resolve14 } from "node:path";
 async function assessUpdate(directory, targetVersion, options = {}) {
   const checkDocker = options.checkDocker ?? false;
   const observation = await observeDeployment(directory, { checkDocker, checkImage: false });
@@ -5384,7 +5802,7 @@ async function assessUpdate(directory, targetVersion, options = {}) {
     for (const name of managedAssetNames(state?.mode ?? "local")) {
       let desired;
       try {
-        desired = await readFile3(join25(ASSET_ROOT, name));
+        desired = await readFile3(join26(ASSET_ROOT, name));
       } catch (error) {
         findings.push({
           code: "managed-asset-unreadable",
@@ -5395,7 +5813,7 @@ async function assessUpdate(directory, targetVersion, options = {}) {
         continue;
       }
       try {
-        const installedPath = join25(directory, name);
+        const installedPath = join26(directory, name);
         if (!await fileExists(installedPath)) {
           changed.push(`${name} (add)`);
           continue;
@@ -5537,7 +5955,7 @@ async function runUpdateCommand(directory, flags, asJson) {
   const startedAtMs = Date.now();
   try {
     const operator = await prepareVerifiedDashboardOperator(loaded.directory, targetVersion);
-    const backup = await createBackup(loaded.directory, state, env, { restartAfter: false, backupId });
+    const backup = await createBackup(loaded.directory, state, env, { restartAfter: false, backupId, safetySnapshot: true });
     const writeSchema = writeStateSchema();
     if (writeSchema >= 2) {
       const minimumSource = packagedMinimumSourceVersion();
@@ -5605,7 +6023,10 @@ async function runUpdateCommand(directory, flags, asJson) {
         if (migrateLegacyVolume && !recreateDataVolume) {
           await setManagedVolumeRootOwnership(loaded.directory, state, oldEnv, targetImage, previousRootOwnership);
         }
-        await restoreBackupData(loaded.directory, state, oldEnv, backup.backupDir, { recreateDataVolume });
+        await restoreBackupData(loaded.directory, state, oldEnv, backup.backupDir, {
+          recreateDataVolume,
+          previousSafetySnapshot: true
+        });
       } catch (restoreError) {
         throw new Error(`Update to ${targetVersion} failed, and automatic backup restoration also failed: ${restoreError instanceof Error ? restoreError.message : String(restoreError)}. Original update error: ${error instanceof Error ? error.message : String(error)}`);
       }
@@ -5747,7 +6168,7 @@ async function runUpdateCommand(directory, flags, asJson) {
 }
 
 // src/commands/verify.ts
-import { join as join26, resolve as resolve15 } from "node:path";
+import { join as join27, resolve as resolve15 } from "node:path";
 async function verificationCredentials(directory, flags, env) {
   const passwordFile = stringFlag(flags, "verification-password-file");
   if (flags["verification-email"] !== undefined && !stringFlag(flags, "verification-email")) {
@@ -5759,7 +6180,7 @@ async function verificationCredentials(directory, flags, env) {
   const configuredEmail = stringFlag(flags, "verification-email")?.trim() || env.OR3_MANAGED_OWNER_EMAIL || env.OR3_BASIC_AUTH_BOOTSTRAP_EMAIL;
   let initialCredentials = {};
   if (!configuredEmail || !passwordFile && !env.OR3_BASIC_AUTH_BOOTSTRAP_PASSWORD) {
-    const initialCredentialsPath = join26(directory, ".or3-initial-credentials");
+    const initialCredentialsPath = join27(directory, ".or3-initial-credentials");
     if (await fileExists(initialCredentialsPath)) {
       initialCredentials = parseEnv(await readOwnerOnlyText(initialCredentialsPath, "The initial credentials file"));
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@or3/cloud",
-  "version": "0.1.82",
+  "version": "0.1.85",
   "description": "Install and operate the OR3 Cloud container distribution",
   "type": "module",
   "license": "GPL-3.0",
@@ -19,9 +19,9 @@
     "dashboardUpdateProtocol": 1,
     "dashboardUpdateMinimumSourceVersion": "0.1.40",
     "stateSchema": 1,
-    "imageDigest": "sha256:22a0dad8e014c7eba9e295643eb8e99c497b3dae6303ef462db44dbda986255f",
-    "operatorImageDigest": "sha256:50e1c9ccb51f867f9daf02a00891d70faa36556b6b3026c38de3f1855d8b01bb",
-    "sourceRevision": "3768e7f43b67b574c4eecb3aea196849b2c0f16e"
+    "imageDigest": "sha256:306702ef2d4aad247d2308f09e0a11e9343008bc28207dd08202e40020bd4da7",
+    "operatorImageDigest": "sha256:fd70c4fb96272387200fb9ba9e9a888246b62cfd492030084212ab928821ede8",
+    "sourceRevision": "33c76a7551045fbfb3e2aa2e1412f117118ddb9e"
   },
   "files": [
     "dist",