@or3/cloud 0.1.72 → 0.1.75

package/README.md

@@ -69,8 +69,8 @@ never a development image.
 
 Run `verify` after every update and before declaring a deployment healthy. It
 checks the managed image digest, deep provider health, Basic Auth sign-in,
-session hydration, SQLite sync, a disposable filesystem upload/download/delete
-cycle, both SQLite databases, volume ownership, proxy runtime settings, and a
+session hydration, SQLite sync, a disposable filesystem upload/download and
+canonical metadata cleanup cycle, both SQLite databases, volume ownership, proxy runtime settings, and a
 bounded window of serious container logs. On a public VPS, require the real
 HTTPS path (with no redirects) using `npx @or3/cloud verify --public`.
 After changing the owner password in the app, pass the current credential only
@@ -80,8 +80,11 @@ Until the one-time handoff file is deleted, `verify` may read its initial owner
 credential from that protected file; it is never restored to `.env` or Docker
 container metadata.
 Verification rejects cross-origin filesystem grants and unexpected methods or
-headers, validates the storage ID before uploading, deletes its probe, and
-revokes its temporary session even when a later check fails.
+headers, validates the storage ID before uploading, tombstones its probe before
+requesting physical deletion, and revokes its temporary session even when a later
+check fails. When the provider explicitly requires deletion coordination,
+verification reports physical cleanup as deferred; retained probe bytes remain
+subject to the provider retention policy. Other deletion failures still fail verification.
 
 Do not run `docker compose down --volumes` on a normal deployment: it deletes
 the application data. Use `backup` before an update or any destructive action.

package/dist/cli.mjs

@@ -31,7 +31,7 @@ import { fileURLToPath } from "node:url";
 import { createGunzip } from "node:zlib";
 var execFile = promisify(execFileCallback);
 var PACKAGE_ROOT = resolve(fileURLToPath(new URL("../", import.meta.url)));
-var PACKAGE_VERSION = "0.1.72";
+var PACKAGE_VERSION = "0.1.75";
 var IMAGE_REPOSITORY = "ghcr.io/saluana/or3-chat";
 var ASSET_ROOT = resolve(fileURLToPath(new URL("../assets/", import.meta.url)));
 var LEGACY_STATE_SCHEMA_VERSION = 1;
@@ -968,10 +968,12 @@ async function managedVolumeRootOwnership(image, volume) {
     throw new Error(`Managed volume ${volume} returned an invalid root ownership value.`);
   return { uid: Number(match[1]), gid: Number(match[2]) };
 }
-async function setManagedVolumeRootOwnership(image, volume, ownership) {
+async function setManagedVolumeRootOwnership(directory, state, env, image, ownership) {
   if (!Number.isSafeInteger(ownership.uid) || ownership.uid < 0 || !Number.isSafeInteger(ownership.gid) || ownership.gid < 0) {
     throw new Error("Refusing an invalid managed volume root UID/GID.");
   }
+  await ensureManagedDataVolume(directory, state.mode, state, env);
+  const volume = state.volumeName;
   const result = await run("docker", [
     "run",
     "--rm",
@@ -1045,35 +1047,42 @@ async function removeManagedDataVolumeForRecreation(directory, state) {
 }
 async function ensureManagedDataVolume(directory, mode, state, env) {
   const existing = await run("docker", ["volume", "inspect", state.volumeName, "--format", "{{.Name}}"], directory);
-  if (existing.ok && existing.stdout.trim() === state.volumeName)
-    return;
-  const created = await run("docker", composeArgs(directory, mode, [
-    "run",
-    "--rm",
-    "-T",
-    "--no-deps",
-    "--entrypoint",
-    "sh",
-    "or3",
-    "-c",
-    "true"
-  ]), directory);
-  if (!created.ok)
-    throw new Error(`Could not create the managed data volume. ${redact(created.stderr, secretValues(env))}`);
+  if (existing.ok && existing.stdout.trim() !== state.volumeName) {
+    throw new Error(`Docker returned an unexpected name for managed volume ${state.volumeName}.`);
+  }
+  if (!existing.ok) {
+    if (existing.exitCode !== 1 || !/no such volume/i.test(`${existing.stdout}
+${existing.stderr}`)) {
+      throw new Error(`Could not inspect managed volume ${state.volumeName}. Refusing to assume it is missing. ${redact(existing.stderr, secretValues(env))}`);
+    }
+    const created = await run("docker", composeArgs(directory, mode, [
+      "run",
+      "--rm",
+      "-T",
+      "--no-deps",
+      "--entrypoint",
+      "sh",
+      "or3",
+      "-c",
+      "true"
+    ]), directory);
+    if (!created.ok)
+      throw new Error(`Could not create the managed data volume. ${redact(created.stderr, secretValues(env))}`);
+  }
   const inspected = await run("docker", ["volume", "inspect", state.volumeName, "--format", "{{json .Labels}}"], directory);
   if (!inspected.ok)
-    throw new Error(`Could not verify the recreated managed data volume ${state.volumeName}.`);
+    throw new Error(`Could not verify managed data volume ${state.volumeName}.`);
   let labels;
   try {
     labels = JSON.parse(inspected.stdout.trim());
   } catch {
-    throw new Error(`Docker returned unreadable labels for recreated managed volume ${state.volumeName}.`);
+    throw new Error(`Docker returned unreadable labels for managed volume ${state.volumeName}.`);
   }
   if (!labels || typeof labels !== "object") {
-    throw new Error(`Recreated volume ${state.volumeName} has no managed deployment labels.`);
+    throw new Error(`Volume ${state.volumeName} has no managed deployment labels.`);
   }
   if (labels["com.docker.compose.project"] !== state.composeProject || labels["com.docker.compose.volume"] !== "or3-data" || env.OR3_DEPLOYMENT_ID && labels["io.or3.cloud.deployment-id"] !== env.OR3_DEPLOYMENT_ID) {
-    throw new Error(`Recreated volume ${state.volumeName} does not carry the expected managed deployment labels.`);
+    throw new Error(`Volume ${state.volumeName} does not carry the expected managed deployment labels.`);
   }
 }
 async function portAvailable(port) {
@@ -2754,7 +2763,7 @@ Options:
   --verification-password-file <path>  Read the current owner password for this verification
   --port <port>                  Local OR3 port (default: 3000)
   --keep <n>                     Backups to retain when pruning (default: 5)
-  --force                        Prune backups even when referenced by the rollback point
+  --force                        Bypass suspect-entry deferral; protected recovery backups remain protected
   --tail <n>                     Log lines to show (default: 200)
   --public                       Require verification through the public HTTPS origin
   --read-only                    Verify without the lease, login, storage, or database writes
@@ -3959,7 +3968,7 @@ async function runUpdateCommand(directory, flags, asJson) {
         await stopProject(loaded.directory, state.mode);
       }
       if (migrateLegacyVolume && !recreateDataVolume) {
-        await setManagedVolumeRootOwnership(targetImage, state.volumeName, {
+        await setManagedVolumeRootOwnership(loaded.directory, state, oldEnv, targetImage, {
           uid: MANAGED_RUNTIME_UID,
           gid: MANAGED_RUNTIME_GID
         });
@@ -3977,7 +3986,7 @@ async function runUpdateCommand(directory, flags, asJson) {
             return;
           });
         if (migrateLegacyVolume && !recreateDataVolume) {
-          await setManagedVolumeRootOwnership(targetImage, state.volumeName, previousRootOwnership);
+          await setManagedVolumeRootOwnership(loaded.directory, state, oldEnv, targetImage, previousRootOwnership);
         }
         await restoreBackupData(loaded.directory, state, oldEnv, backup.backupDir, { recreateDataVolume });
       } catch (restoreError) {
@@ -4169,7 +4178,7 @@ async function restorePreMutationSnapshot(directory, state, env) {
   }
   if (pending.previousRootOwnership) {
     await pullAndRequireImage(state.image, state.imageDigest, "Previous deployment");
-    await setManagedVolumeRootOwnership(state.image, state.volumeName, pending.previousRootOwnership);
+    await setManagedVolumeRootOwnership(directory, state, env, state.image, pending.previousRootOwnership);
   }
   await restoreBackupData(directory, state, env, previous.path);
   return previous;
@@ -4534,7 +4543,17 @@ async function verificationJson(baseUrl, path, options = {}) {
     ...options.body === undefined ? {} : { body: JSON.stringify(options.body) }
   });
   if (response.status !== 200) {
-    const detail = (await response.text().catch(() => "")).slice(0, 300);
+    const responseText = await response.text().catch(() => "");
+    const detail = responseText.slice(0, 300);
+    if (options.allowStorageDeletionDeferral && path === "/api/storage/delete" && response.status === 503) {
+      let failure;
+      try {
+        failure = JSON.parse(responseText);
+      } catch {}
+      if (failure && typeof failure === "object" && "statusMessage" in failure && failure.statusMessage === "Provider-owned deletion coordination is required") {
+        return { deletionDeferred: true };
+      }
+    }
     throw new Error(`${response.url} returned HTTP ${response.status}${detail ? `: ${detail}` : ""}`);
   }
   const value = await response.json();
@@ -4626,6 +4645,7 @@ async function verifyPublicApplication(baseUrl, credentials) {
     const hash = `sha256:${createHash("sha256").update(probeBytes).digest("hex")}`;
     let storageId;
     let metadataAttempted = false;
+    let physicalCleanupDeferred = false;
     try {
       const presign = await verificationJson(baseUrl, "/api/storage/presign-upload", {
         cookie,
@@ -4648,6 +4668,7 @@ async function verifyPublicApplication(baseUrl, credentials) {
       });
       if (!upload.ok)
         throw new Error(`Filesystem verification upload returned HTTP ${upload.status}.`);
+      metadataAttempted = true;
       await verificationJson(baseUrl, "/api/storage/commit", {
         cookie,
         body: {
@@ -4663,7 +4684,6 @@ async function verifyPublicApplication(baseUrl, credentials) {
       });
       const metadataCreatedAt = Date.now();
       const metadataOpId = randomUUID();
-      metadataAttempted = true;
       const pushed = await verificationJson(baseUrl, "/api/sync/push", {
         cookie,
         body: {
@@ -4711,45 +4731,44 @@ async function verifyPublicApplication(baseUrl, credentials) {
       }
     } finally {
       if (storageId) {
-        try {
-          await verificationJson(baseUrl, "/api/storage/delete", {
+        if (metadataAttempted) {
+          const metadataDeleteAt = Date.now();
+          const metadataDeleteOpId = randomUUID();
+          const deleted = await verificationJson(baseUrl, "/api/sync/push", {
             cookie,
-            body: { workspace_id: workspaceId, hash, storage_id: storageId }
-          });
-        } finally {
-          if (metadataAttempted) {
-            const metadataDeleteAt = Date.now();
-            const metadataDeleteOpId = randomUUID();
-            const deleted = await verificationJson(baseUrl, "/api/sync/push", {
-              cookie,
-              body: {
-                scope: { workspaceId },
-                ops: [{
-                  id: `or3-verification-${metadataDeleteOpId}`,
-                  tableName: "file_meta",
-                  operation: "delete",
-                  pk: hash,
-                  payload: { hash },
-                  stamp: {
-                    deviceId: "or3-cloud-verification",
-                    opId: metadataDeleteOpId,
-                    hlc: `${String(metadataDeleteAt).padStart(13, "0")}:0000:or3-cloud-verification`,
-                    clock: metadataDeleteAt
-                  },
-                  createdAt: metadataDeleteAt,
-                  attempts: 0,
-                  status: "pending"
-                }]
-              }
-            });
-            if (deleted.results?.[0]?.success !== true) {
-              throw new Error(`Filesystem verification metadata cleanup failed: ${JSON.stringify(deleted)}`);
+            body: {
+              scope: { workspaceId },
+              ops: [{
+                id: `or3-verification-${metadataDeleteOpId}`,
+                tableName: "file_meta",
+                operation: "delete",
+                pk: hash,
+                payload: { hash },
+                stamp: {
+                  deviceId: "or3-cloud-verification",
+                  opId: metadataDeleteOpId,
+                  hlc: `${String(metadataDeleteAt).padStart(13, "0")}:0000:or3-cloud-verification`,
+                  clock: metadataDeleteAt
+                },
+                createdAt: metadataDeleteAt,
+                attempts: 0,
+                status: "pending"
+              }]
             }
+          });
+          if (deleted.results?.[0]?.success !== true) {
+            throw new Error(`Filesystem verification metadata cleanup failed: ${JSON.stringify(deleted)}`);
           }
         }
+        const deletion = await verificationJson(baseUrl, "/api/storage/delete", {
+          cookie,
+          body: { workspace_id: workspaceId, hash, storage_id: storageId },
+          allowStorageDeletionDeferral: true
+        });
+        physicalCleanupDeferred = deletion.deletionDeferred === true;
       }
     }
-    return health;
+    return { ...health, physicalCleanupDeferred };
   } finally {
     await verificationJson(baseUrl, "/api/basic-auth/sign-out", {
       cookie,
@@ -4827,7 +4846,7 @@ async function verifyCommand(directory, flags, positionals) {
   }
   await waitForDeepHealth(loaded.directory, loaded.state.mode, secretValues(loaded.env));
   const baseUrl = new URL(loaded.state.mode === "public" ? `https://${loaded.state.domain}` : `http://127.0.0.1:${loaded.state.port}`);
-  await verifyPublicApplication(baseUrl, await verificationCredentials(loaded.directory, flags, loaded.env));
+  const verification = await verifyPublicApplication(baseUrl, await verificationCredentials(loaded.directory, flags, loaded.env));
   const databaseCheck = await run("docker", [
     ...composeArgs(loaded.directory, loaded.state.mode, ["exec", "-T", "or3", ...containerNodeCommand(VERIFY_DATABASES_SCRIPT)])
   ], loaded.directory);
@@ -4858,7 +4877,7 @@ ${redact(serious.slice(-20).join(`
   console.log(`✓ OR3 ${loaded.state.appVersion} image digest matches managed state`);
   console.log(`✓ ${baseUrl.origin} root and deep health return HTTP 200 without redirects`);
   console.log("✓ Basic Auth sign-in, session hydration, and SQLite sync pull passed");
-  console.log("✓ Filesystem storage write/read/delete probe passed");
+  console.log(verification.physicalCleanupDeferred ? "✓ Filesystem storage write/read and canonical metadata cleanup passed; physical cleanup deferred by provider retention policy" : "✓ Filesystem storage write/read and canonical/physical cleanup passed");
   console.log("✓ auth.sqlite and sync.sqlite quick_check passed with managed ownership");
   console.log("✓ Recent bounded logs contain no fatal, panic, unhandled, or OOM events");
   console.log("OR3 production verification passed.");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@or3/cloud",
-  "version": "0.1.72",
+  "version": "0.1.75",
   "description": "Install and operate the OR3 Cloud container distribution",
   "type": "module",
   "license": "GPL-3.0",
@@ -19,9 +19,9 @@
     "dashboardUpdateProtocol": 1,
     "dashboardUpdateMinimumSourceVersion": "0.1.40",
     "stateSchema": 1,
-    "imageDigest": "sha256:c4b408ea3a5fbd647b26d6efe89ddad58f90be7528aa2b5b6ed20d6d41c928ac",
-    "operatorImageDigest": "sha256:05b03560594da28e436977b82a905aa9a7702668b5e4ac4daea8b54376a75813",
-    "sourceRevision": "90c5756fa313331df7be656fcbc1bce5a4164d6e"
+    "imageDigest": "sha256:daa27fb6f4d3ee7b2fbdd0ab7727d114f3ad042e8330174338d44d720f872ece",
+    "operatorImageDigest": "sha256:cc5c0873085a6efae51252cd96046c526b976584cf69c9829403647433cf9710",
+    "sourceRevision": "988310e28ebbbb06795195e159fe8257feb8e395"
   },
   "files": [
     "dist",