@or3/cloud 0.1.70 → 0.1.74

package/README.md

@@ -69,8 +69,8 @@ never a development image.
 
 Run `verify` after every update and before declaring a deployment healthy. It
 checks the managed image digest, deep provider health, Basic Auth sign-in,
-session hydration, SQLite sync, a disposable filesystem upload/download/delete
-cycle, both SQLite databases, volume ownership, proxy runtime settings, and a
+session hydration, SQLite sync, a disposable filesystem upload/download and
+canonical metadata cleanup cycle, both SQLite databases, volume ownership, proxy runtime settings, and a
 bounded window of serious container logs. On a public VPS, require the real
 HTTPS path (with no redirects) using `npx @or3/cloud verify --public`.
 After changing the owner password in the app, pass the current credential only
@@ -80,8 +80,11 @@ Until the one-time handoff file is deleted, `verify` may read its initial owner
 credential from that protected file; it is never restored to `.env` or Docker
 container metadata.
 Verification rejects cross-origin filesystem grants and unexpected methods or
-headers, validates the storage ID before uploading, deletes its probe, and
-revokes its temporary session even when a later check fails.
+headers, validates the storage ID before uploading, tombstones its probe before
+requesting physical deletion, and revokes its temporary session even when a later
+check fails. When the provider explicitly requires deletion coordination,
+verification reports physical cleanup as deferred; retained probe bytes remain
+subject to the provider retention policy. Other deletion failures still fail verification.
 
 Do not run `docker compose down --volumes` on a normal deployment: it deletes
 the application data. Use `backup` before an update or any destructive action.

package/dist/cli.mjs

@@ -31,7 +31,7 @@ import { fileURLToPath } from "node:url";
 import { createGunzip } from "node:zlib";
 var execFile = promisify(execFileCallback);
 var PACKAGE_ROOT = resolve(fileURLToPath(new URL("../", import.meta.url)));
-var PACKAGE_VERSION = "0.1.70";
+var PACKAGE_VERSION = "0.1.74";
 var IMAGE_REPOSITORY = "ghcr.io/saluana/or3-chat";
 var ASSET_ROOT = resolve(fileURLToPath(new URL("../assets/", import.meta.url)));
 var LEGACY_STATE_SCHEMA_VERSION = 1;
@@ -368,8 +368,8 @@ async function run(command, args, cwd, environment) {
     });
     return {
       ok: true,
-      stdout: result.stdout ?? "",
-      stderr: result.stderr ?? "",
+      stdout: result.stdout,
+      stderr: result.stderr,
       command: printable,
       exitCode: 0
     };
@@ -494,7 +494,7 @@ function serializeCredentialValue(value) {
   return serializeEnvValue(value);
 }
 function serializeEnv(values) {
-  return `${Object.entries(values).filter(([, value]) => value !== undefined).map(([key, value]) => `${key}=${serializeEnvValue(value)}`).join(`
+  return `${Object.entries(values).filter((entry) => entry[1] !== undefined).map(([key, value]) => `${key}=${serializeEnvValue(value)}`).join(`
 `)}
 `;
 }
@@ -968,10 +968,12 @@ async function managedVolumeRootOwnership(image, volume) {
     throw new Error(`Managed volume ${volume} returned an invalid root ownership value.`);
   return { uid: Number(match[1]), gid: Number(match[2]) };
 }
-async function setManagedVolumeRootOwnership(image, volume, ownership) {
+async function setManagedVolumeRootOwnership(directory, state, env, image, ownership) {
   if (!Number.isSafeInteger(ownership.uid) || ownership.uid < 0 || !Number.isSafeInteger(ownership.gid) || ownership.gid < 0) {
     throw new Error("Refusing an invalid managed volume root UID/GID.");
   }
+  await ensureManagedDataVolume(directory, state.mode, state, env);
+  const volume = state.volumeName;
   const result = await run("docker", [
     "run",
     "--rm",
@@ -1036,7 +1038,7 @@ async function removeManagedDataVolumeForRecreation(directory, state) {
   } catch {
     throw new Error(`Docker returned unreadable metadata for managed volume ${state.volumeName}.`);
   }
-  if (volume.Name !== state.volumeName || volume.Labels?.["com.docker.compose.project"] !== state.composeProject || volume.Labels?.["com.docker.compose.volume"] !== "or3-data") {
+  if (volume.Name !== state.volumeName || volume.Labels?.["com.docker.compose.project"] !== state.composeProject || volume.Labels["com.docker.compose.volume"] !== "or3-data") {
     throw new Error(`Volume ${state.volumeName} is not bound to this managed Compose deployment. Refusing to recreate it.`);
   }
   const removedVolume = await run("docker", ["volume", "rm", state.volumeName], directory);
@@ -1045,35 +1047,42 @@ async function removeManagedDataVolumeForRecreation(directory, state) {
 }
 async function ensureManagedDataVolume(directory, mode, state, env) {
   const existing = await run("docker", ["volume", "inspect", state.volumeName, "--format", "{{.Name}}"], directory);
-  if (existing.ok && existing.stdout.trim() === state.volumeName)
-    return;
-  const created = await run("docker", composeArgs(directory, mode, [
-    "run",
-    "--rm",
-    "-T",
-    "--no-deps",
-    "--entrypoint",
-    "sh",
-    "or3",
-    "-c",
-    "true"
-  ]), directory);
-  if (!created.ok)
-    throw new Error(`Could not create the managed data volume. ${redact(created.stderr, secretValues(env))}`);
+  if (existing.ok && existing.stdout.trim() !== state.volumeName) {
+    throw new Error(`Docker returned an unexpected name for managed volume ${state.volumeName}.`);
+  }
+  if (!existing.ok) {
+    if (existing.exitCode !== 1 || !/no such volume/i.test(`${existing.stdout}
+${existing.stderr}`)) {
+      throw new Error(`Could not inspect managed volume ${state.volumeName}. Refusing to assume it is missing. ${redact(existing.stderr, secretValues(env))}`);
+    }
+    const created = await run("docker", composeArgs(directory, mode, [
+      "run",
+      "--rm",
+      "-T",
+      "--no-deps",
+      "--entrypoint",
+      "sh",
+      "or3",
+      "-c",
+      "true"
+    ]), directory);
+    if (!created.ok)
+      throw new Error(`Could not create the managed data volume. ${redact(created.stderr, secretValues(env))}`);
+  }
   const inspected = await run("docker", ["volume", "inspect", state.volumeName, "--format", "{{json .Labels}}"], directory);
   if (!inspected.ok)
-    throw new Error(`Could not verify the recreated managed data volume ${state.volumeName}.`);
+    throw new Error(`Could not verify managed data volume ${state.volumeName}.`);
   let labels;
   try {
     labels = JSON.parse(inspected.stdout.trim());
   } catch {
-    throw new Error(`Docker returned unreadable labels for recreated managed volume ${state.volumeName}.`);
+    throw new Error(`Docker returned unreadable labels for managed volume ${state.volumeName}.`);
   }
   if (!labels || typeof labels !== "object") {
-    throw new Error(`Recreated volume ${state.volumeName} has no managed deployment labels.`);
+    throw new Error(`Volume ${state.volumeName} has no managed deployment labels.`);
   }
   if (labels["com.docker.compose.project"] !== state.composeProject || labels["com.docker.compose.volume"] !== "or3-data" || env.OR3_DEPLOYMENT_ID && labels["io.or3.cloud.deployment-id"] !== env.OR3_DEPLOYMENT_ID) {
-    throw new Error(`Recreated volume ${state.volumeName} does not carry the expected managed deployment labels.`);
+    throw new Error(`Volume ${state.volumeName} does not carry the expected managed deployment labels.`);
   }
 }
 async function portAvailable(port) {
@@ -1269,6 +1278,8 @@ async function assertImageReleaseIdentity(image, version) {
   } catch {
     throw new Error(`OR3 image ${image} has no readable release labels.`);
   }
+  if (!labels || typeof labels !== "object")
+    throw new Error(`OR3 image ${image} has no readable release labels.`);
   assertImageReleaseLabels(image, labels, version, packagedSourceRevision(version));
 }
 async function imageDigest(image) {
@@ -1636,6 +1647,7 @@ async function installManagedAssets(directory, assets) {
   const token = randomBytes(4).toString("hex");
   const staged = [];
   let retainRollbackCopies = false;
+  const canCleanRollback = () => !retainRollbackCopies;
   try {
     for (const [name, contents] of assets) {
       const destination = join(directory, name);
@@ -1687,7 +1699,7 @@ async function installManagedAssets(directory, assets) {
       await rm(entry.replacement, { force: true }).catch(() => {
         return;
       });
-      if (entry.rollback && !retainRollbackCopies)
+      if (entry.rollback && canCleanRollback())
         await rm(entry.rollback, { force: true }).catch(() => {
           return;
         });
@@ -2330,6 +2342,7 @@ async function streamCommandToFile(command, args, destination, cwd, secrets = []
   });
   let stderr = "";
   let timedOut = false;
+  const hasTimedOut = () => timedOut;
   const timeout = setTimeout(() => {
     timedOut = true;
     terminateChildProcess(child);
@@ -2347,7 +2360,7 @@ async function streamCommandToFile(command, args, destination, cwd, secrets = []
       exit,
       pipeline(child.stdout, createWriteStream(temporary, { flags: "wx", mode: 384 }))
     ]);
-    if (timedOut)
+    if (hasTimedOut())
       throw new Error(`${command} exceeded the ${STREAM_COMMAND_TIMEOUT_MS / 1000}-second archive deadline.`);
     if (exitCode !== 0) {
       throw new Error(`${command} ${args.join(" ")} exited with ${exitCode}. ${redact(stderr, secrets)}`.trim());
@@ -2375,6 +2388,7 @@ async function streamFileToCommand(command, args, source, cwd, secrets = []) {
   });
   let stderr = "";
   let timedOut = false;
+  const hasTimedOut = () => timedOut;
   const timeout = setTimeout(() => {
     timedOut = true;
     terminateChildProcess(child);
@@ -2392,7 +2406,7 @@ async function streamFileToCommand(command, args, source, cwd, secrets = []) {
       exit,
       pipeline(createReadStream(source), child.stdin)
     ]);
-    if (timedOut)
+    if (hasTimedOut())
       throw new Error(`${command} exceeded the ${STREAM_COMMAND_TIMEOUT_MS / 1000}-second archive deadline.`);
     if (exitCode !== 0) {
       throw new Error(`${command} ${args.join(" ")} exited with ${exitCode}. ${redact(stderr, secrets)}`.trim());
@@ -3785,7 +3799,7 @@ async function assessUpdate(directory, targetVersion, options = {}) {
   }
   try {
     assertKnownStateSchema(state?.schemaVersion);
-    checks.push({ code: "state-schema", status: "passed", detail: `Managed state schema ${String(state?.schemaVersion)} is supported.` });
+    checks.push({ code: "state-schema", status: "passed", detail: `Managed state schema ${String(state.schemaVersion)} is supported.` });
   } catch (error) {
     findings.push({ code: "state-schema-unsupported", severity: "blocker", message: error instanceof Error ? error.message : String(error) });
   }
@@ -3954,7 +3968,7 @@ async function runUpdateCommand(directory, flags, asJson) {
         await stopProject(loaded.directory, state.mode);
       }
       if (migrateLegacyVolume && !recreateDataVolume) {
-        await setManagedVolumeRootOwnership(targetImage, state.volumeName, {
+        await setManagedVolumeRootOwnership(loaded.directory, state, oldEnv, targetImage, {
           uid: MANAGED_RUNTIME_UID,
           gid: MANAGED_RUNTIME_GID
         });
@@ -3972,7 +3986,7 @@ async function runUpdateCommand(directory, flags, asJson) {
             return;
           });
         if (migrateLegacyVolume && !recreateDataVolume) {
-          await setManagedVolumeRootOwnership(targetImage, state.volumeName, previousRootOwnership);
+          await setManagedVolumeRootOwnership(loaded.directory, state, oldEnv, targetImage, previousRootOwnership);
         }
         await restoreBackupData(loaded.directory, state, oldEnv, backup.backupDir, { recreateDataVolume });
       } catch (restoreError) {
@@ -4164,7 +4178,7 @@ async function restorePreMutationSnapshot(directory, state, env) {
   }
   if (pending.previousRootOwnership) {
     await pullAndRequireImage(state.image, state.imageDigest, "Previous deployment");
-    await setManagedVolumeRootOwnership(state.image, state.volumeName, pending.previousRootOwnership);
+    await setManagedVolumeRootOwnership(directory, state, env, state.image, pending.previousRootOwnership);
   }
   await restoreBackupData(directory, state, env, previous.path);
   return previous;
@@ -4504,7 +4518,7 @@ async function doctorCommand(directory) {
 }
 function validateVerificationHealth(value) {
   const health = value;
-  if (!health || health.status !== "ok" || health.providers?.auth?.provider !== "basic-auth" || health.providers?.sync?.provider !== "sqlite" || health.providers?.storage?.provider !== "fs") {
+  if (!health || health.status !== "ok" || health.providers?.auth?.provider !== "basic-auth" || health.providers.sync?.provider !== "sqlite" || health.providers.storage?.provider !== "fs") {
     throw new Error("Public deep health does not report the managed Basic Auth + SQLite + filesystem profile.");
   }
   return health;
@@ -4524,15 +4538,29 @@ async function verificationJson(baseUrl, path, options = {}) {
       accept: "application/json",
       ...options.body === undefined ? {} : { "content-type": "application/json" },
       ...options.cookie ? { cookie: options.cookie } : {},
-      ...method === "GET" ? {} : { origin: baseUrl.origin }
+      ...method === "GET" ? {} : { origin: baseUrl.origin, "x-or3-cloud-intent": "mutation" }
     },
     ...options.body === undefined ? {} : { body: JSON.stringify(options.body) }
   });
   if (response.status !== 200) {
-    const detail = (await response.text().catch(() => "")).slice(0, 300);
+    const responseText = await response.text().catch(() => "");
+    const detail = responseText.slice(0, 300);
+    if (options.allowStorageDeletionDeferral && path === "/api/storage/delete" && response.status === 503) {
+      let failure;
+      try {
+        failure = JSON.parse(responseText);
+      } catch {}
+      if (failure && typeof failure === "object" && "statusMessage" in failure && failure.statusMessage === "Provider-owned deletion coordination is required") {
+        return { deletionDeferred: true };
+      }
+    }
     throw new Error(`${response.url} returned HTTP ${response.status}${detail ? `: ${detail}` : ""}`);
   }
-  return response.json();
+  const value = await response.json();
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    throw new Error(`${response.url} returned an invalid JSON object.`);
+  }
+  return value;
 }
 function sameOriginVerificationUrl(baseUrl, value, label) {
   if (typeof value !== "string")
@@ -4599,7 +4627,7 @@ async function verifyPublicApplication(baseUrl, credentials) {
     throw new Error("Public Basic Auth sign-in did not set a session cookie.");
   try {
     const session = await verificationJson(baseUrl, "/api/auth/session", { cookie });
-    if (session.session?.user?.email !== email || !session.session?.workspace?.id) {
+    if (session.session?.user?.email !== email || !session.session.workspace?.id) {
       throw new Error("Public session hydration did not return the verification user and workspace.");
     }
     const workspaceId = String(session.session.workspace.id);
@@ -4617,6 +4645,7 @@ async function verifyPublicApplication(baseUrl, credentials) {
     const hash = `sha256:${createHash("sha256").update(probeBytes).digest("hex")}`;
     let storageId;
     let metadataAttempted = false;
+    let physicalCleanupDeferred = false;
     try {
       const presign = await verificationJson(baseUrl, "/api/storage/presign-upload", {
         cookie,
@@ -4639,6 +4668,7 @@ async function verifyPublicApplication(baseUrl, credentials) {
       });
       if (!upload.ok)
         throw new Error(`Filesystem verification upload returned HTTP ${upload.status}.`);
+      metadataAttempted = true;
       await verificationJson(baseUrl, "/api/storage/commit", {
         cookie,
         body: {
@@ -4654,7 +4684,6 @@ async function verifyPublicApplication(baseUrl, credentials) {
       });
       const metadataCreatedAt = Date.now();
       const metadataOpId = randomUUID();
-      metadataAttempted = true;
       const pushed = await verificationJson(baseUrl, "/api/sync/push", {
         cookie,
         body: {
@@ -4702,45 +4731,44 @@ async function verifyPublicApplication(baseUrl, credentials) {
       }
     } finally {
       if (storageId) {
-        try {
-          await verificationJson(baseUrl, "/api/storage/delete", {
+        if (metadataAttempted) {
+          const metadataDeleteAt = Date.now();
+          const metadataDeleteOpId = randomUUID();
+          const deleted = await verificationJson(baseUrl, "/api/sync/push", {
             cookie,
-            body: { workspace_id: workspaceId, hash, storage_id: storageId }
-          });
-        } finally {
-          if (metadataAttempted) {
-            const metadataDeleteAt = Date.now();
-            const metadataDeleteOpId = randomUUID();
-            const deleted = await verificationJson(baseUrl, "/api/sync/push", {
-              cookie,
-              body: {
-                scope: { workspaceId },
-                ops: [{
-                  id: `or3-verification-${metadataDeleteOpId}`,
-                  tableName: "file_meta",
-                  operation: "delete",
-                  pk: hash,
-                  payload: { hash },
-                  stamp: {
-                    deviceId: "or3-cloud-verification",
-                    opId: metadataDeleteOpId,
-                    hlc: `${String(metadataDeleteAt).padStart(13, "0")}:0000:or3-cloud-verification`,
-                    clock: metadataDeleteAt
-                  },
-                  createdAt: metadataDeleteAt,
-                  attempts: 0,
-                  status: "pending"
-                }]
-              }
-            });
-            if (deleted.results?.[0]?.success !== true) {
-              throw new Error(`Filesystem verification metadata cleanup failed: ${JSON.stringify(deleted)}`);
+            body: {
+              scope: { workspaceId },
+              ops: [{
+                id: `or3-verification-${metadataDeleteOpId}`,
+                tableName: "file_meta",
+                operation: "delete",
+                pk: hash,
+                payload: { hash },
+                stamp: {
+                  deviceId: "or3-cloud-verification",
+                  opId: metadataDeleteOpId,
+                  hlc: `${String(metadataDeleteAt).padStart(13, "0")}:0000:or3-cloud-verification`,
+                  clock: metadataDeleteAt
+                },
+                createdAt: metadataDeleteAt,
+                attempts: 0,
+                status: "pending"
+              }]
             }
+          });
+          if (deleted.results?.[0]?.success !== true) {
+            throw new Error(`Filesystem verification metadata cleanup failed: ${JSON.stringify(deleted)}`);
           }
         }
+        const deletion = await verificationJson(baseUrl, "/api/storage/delete", {
+          cookie,
+          body: { workspace_id: workspaceId, hash, storage_id: storageId },
+          allowStorageDeletionDeferral: true
+        });
+        physicalCleanupDeferred = deletion.deletionDeferred === true;
       }
     }
-    return health;
+    return { ...health, physicalCleanupDeferred };
   } finally {
     await verificationJson(baseUrl, "/api/basic-auth/sign-out", {
       cookie,
@@ -4767,7 +4795,7 @@ async function provisionManagedCredentialsOnce(baseUrl, email, password, adminUs
   const appCookie = responseCookie(appSignIn, "Managed owner provisioning");
   try {
     const session = await verificationJson(baseUrl, "/api/auth/session", { cookie: appCookie });
-    if (session.session?.user?.email !== email || !session.session?.workspace?.id) {
+    if (session.session?.user?.email !== email || !session.session.workspace?.id) {
       throw new Error("Managed owner provisioning did not create the expected account and workspace.");
     }
   } finally {
@@ -4818,7 +4846,7 @@ async function verifyCommand(directory, flags, positionals) {
   }
   await waitForDeepHealth(loaded.directory, loaded.state.mode, secretValues(loaded.env));
   const baseUrl = new URL(loaded.state.mode === "public" ? `https://${loaded.state.domain}` : `http://127.0.0.1:${loaded.state.port}`);
-  await verifyPublicApplication(baseUrl, await verificationCredentials(loaded.directory, flags, loaded.env));
+  const verification = await verifyPublicApplication(baseUrl, await verificationCredentials(loaded.directory, flags, loaded.env));
   const databaseCheck = await run("docker", [
     ...composeArgs(loaded.directory, loaded.state.mode, ["exec", "-T", "or3", ...containerNodeCommand(VERIFY_DATABASES_SCRIPT)])
   ], loaded.directory);
@@ -4849,7 +4877,7 @@ ${redact(serious.slice(-20).join(`
   console.log(`✓ OR3 ${loaded.state.appVersion} image digest matches managed state`);
   console.log(`✓ ${baseUrl.origin} root and deep health return HTTP 200 without redirects`);
   console.log("✓ Basic Auth sign-in, session hydration, and SQLite sync pull passed");
-  console.log("✓ Filesystem storage write/read/delete probe passed");
+  console.log(verification.physicalCleanupDeferred ? "✓ Filesystem storage write/read and canonical metadata cleanup passed; physical cleanup deferred by provider retention policy" : "✓ Filesystem storage write/read and canonical/physical cleanup passed");
   console.log("✓ auth.sqlite and sync.sqlite quick_check passed with managed ownership");
   console.log("✓ Recent bounded logs contain no fatal, panic, unhandled, or OOM events");
   console.log("OR3 production verification passed.");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@or3/cloud",
-  "version": "0.1.70",
+  "version": "0.1.74",
   "description": "Install and operate the OR3 Cloud container distribution",
   "type": "module",
   "license": "GPL-3.0",
@@ -19,9 +19,9 @@
     "dashboardUpdateProtocol": 1,
     "dashboardUpdateMinimumSourceVersion": "0.1.40",
     "stateSchema": 1,
-    "imageDigest": "sha256:98bf853b1f7f4e482ce705b9819c2921be648b30e9ad52980e1f12641fbe8064",
-    "operatorImageDigest": "sha256:bd585210d63c09d98073d19bb4c4f6ba598059fe6c91d6b8db7c8f646cd86161",
-    "sourceRevision": "bd1f1310af3c1ef9e66de6d438057eed48a9e049"
+    "imageDigest": "sha256:d01c7513cb26858b23c279ab0cfe39a28bb73691e69e566be29460ed385d42ca",
+    "operatorImageDigest": "sha256:bd7162734324f40daa85861c314771d7437cbde258f0b45ae7d6a878810e5fdf",
+    "sourceRevision": "77049d904cf405b3f5a33ef85b622d91c9250008"
   },
   "files": [
     "dist",