@or3/cloud 0.1.67 → 0.1.68

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/cli.mjs CHANGED
@@ -31,10 +31,10 @@ import { fileURLToPath } from "node:url";
31
31
  import { createGunzip } from "node:zlib";
32
32
  var execFile = promisify(execFileCallback);
33
33
  var PACKAGE_ROOT = resolve(fileURLToPath(new URL("../", import.meta.url)));
34
- var PACKAGE_VERSION = "0.1.67";
34
+ var PACKAGE_VERSION = "0.1.68";
35
35
  var IMAGE_REPOSITORY = "ghcr.io/saluana/or3-chat";
36
36
  var ASSET_ROOT = resolve(fileURLToPath(new URL("../assets/", import.meta.url)));
37
- var STATE_SCHEMA_VERSION = 1;
37
+ var LEGACY_STATE_SCHEMA_VERSION = 1;
38
38
  var DEFAULT_PORT = 3000;
39
39
  var DEEP_HEALTH_TIMEOUT_MS = 180000;
40
40
  var COMMAND_TIMEOUT_MS = 180000;
@@ -70,6 +70,21 @@ var DEPLOYMENT_ENV_KEYS = [
70
70
  "OR3_PUBLIC_DOMAIN"
71
71
  ];
72
72
  var DEPLOYMENT_ID_ENV_KEY = "OR3_DEPLOYMENT_ID";
73
+ function emitOperationResult(outcome) {
74
+ console.log(JSON.stringify({ schemaVersion: 1, kind: "or3-operation-result", outcome }, null, 2));
75
+ }
76
+ var STATE_SCHEMA_COMPATIBILITY = {
77
+ 1: { readers: ["bridge", "schema-2"], writer: "migrating", mutable: false },
78
+ 2: { readers: ["bridge", "schema-2"], writer: "schema-2", mutable: true }
79
+ };
80
+ var MAX_SUPPORTED_STATE_SCHEMA = 2;
81
+ var READER_SUPPORTED_STATE_SCHEMAS = [1, 2];
82
+ function assertKnownStateSchema(schemaVersion) {
83
+ if (schemaVersion !== 1 && schemaVersion !== 2) {
84
+ throw new Error(`Unsupported managed state schema ${String(schemaVersion)}. This CLI reads schemas 1 and 2. Run a compatible exact-version @or3/cloud CLI for this deployment instead of editing managed state.`);
85
+ }
86
+ }
87
+ var lifecycleFaults = {};
73
88
  var PROCESS_ENV_PASSTHROUGH = [
74
89
  "PATH",
75
90
  "HOME",
@@ -201,22 +216,25 @@ function parseFlags(argv) {
201
216
  }
202
217
  var COMMAND_FLAGS = {
203
218
  init: ["local", "public", "domain", "admin-email", "admin-password", "admin-password-file", "port"],
204
- update: ["to"],
205
- backup: ["keep", "force", "yes"],
219
+ update: ["to", "dry-run", "json"],
220
+ backup: ["keep", "force", "yes", "json"],
206
221
  restore: ["yes"],
207
222
  rollback: ["yes"],
208
223
  doctor: [],
209
- verify: ["public", "verification-email", "verification-password-file"],
210
- recover: [],
224
+ verify: ["public", "verification-email", "verification-password-file", "read-only", "json"],
225
+ recover: ["dry-run", "finish", "restore", "yes", "json"],
211
226
  adopt: ["from"],
212
227
  credentials: ["yes", "owner-password", "owner-password-file", "admin-password", "admin-password-file"],
213
- status: [],
228
+ status: ["json"],
214
229
  logs: ["tail"],
215
230
  start: [],
216
231
  stop: [],
217
232
  restart: [],
218
233
  remove: ["purge-data", "yes"]
219
234
  };
235
+ function verifyIsReadOnly(flags) {
236
+ return flags["read-only"] === true;
237
+ }
220
238
  function assertCommandFlags(command, flags) {
221
239
  const allowed = new Set(["help", ...COMMAND_FLAGS[command] ?? []]);
222
240
  const unknown = Object.keys(flags).filter((flag) => !allowed.has(flag));
@@ -262,6 +280,15 @@ function requireStringFlag(flags, key) {
262
280
  function isVersion(value) {
263
281
  return /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/.test(value);
264
282
  }
283
+ function compareReleaseVersions(left, right) {
284
+ const leftParts = left.split("-", 1)[0].split(".").map(Number);
285
+ const rightParts = right.split("-", 1)[0].split(".").map(Number);
286
+ for (let index = 0;index < 3; index += 1) {
287
+ if (leftParts[index] !== rightParts[index])
288
+ return leftParts[index] > rightParts[index] ? 1 : -1;
289
+ }
290
+ return 0;
291
+ }
265
292
  function imageFor(version) {
266
293
  if (!isVersion(version))
267
294
  throw new Error(`Invalid release version "${version}".`);
@@ -329,6 +356,7 @@ function commandEnvironment(command, args, cwd, supplied) {
329
356
  }
330
357
  async function run(command, args, cwd, environment) {
331
358
  const printable = `${command} ${args.map(quote).join(" ")}`;
359
+ await lifecycleFaults.beforeCommand?.(command, args);
332
360
  try {
333
361
  const result = await execFile(command, args, {
334
362
  cwd,
@@ -478,6 +506,7 @@ function deploymentPaths(directory) {
478
506
  operations: join(cloud, "operations"),
479
507
  backups: join(cloud, "backups"),
480
508
  exports: join(cloud, "exports"),
509
+ lastOperation: join(cloud, "last-operation.json"),
481
510
  lease: join(cloud, "operation-lease"),
482
511
  backupAuthKey: join(cloud, "backup-auth.key"),
483
512
  operatorIpc: join(cloud, "operator-ipc")
@@ -530,14 +559,17 @@ async function assertBackupAuthentication(directory, backupPath, manifestContent
530
559
  async function readState(directory) {
531
560
  const paths = deploymentPaths(directory);
532
561
  const parsed = JSON.parse(await readText(paths.state));
533
- if (parsed.schemaVersion !== STATE_SCHEMA_VERSION || !parsed.appVersion || !parsed.image || !parsed.composeProject) {
562
+ assertKnownStateSchema(parsed.schemaVersion);
563
+ if (!parsed.appVersion || !parsed.image || !parsed.composeProject || !parsed.mode) {
534
564
  throw new Error(`Invalid managed state at ${paths.state}. Run "npx @or3/cloud doctor" for diagnostics.`);
535
565
  }
536
566
  return parsed;
537
567
  }
538
568
  async function writeState(directory, state) {
569
+ await lifecycleFaults.beforeStateWrite?.();
539
570
  await writeSecure(deploymentPaths(directory).state, `${JSON.stringify(state, null, 2)}
540
571
  `);
572
+ await lifecycleFaults.afterStateWrite?.();
541
573
  }
542
574
  async function readDirectoryEmpty(directory) {
543
575
  try {
@@ -883,8 +915,10 @@ function containerNodeCommand(script) {
883
915
  return ["sh", "-c", CONTAINER_NODE_SHELL, "or3-node", script];
884
916
  }
885
917
  async function waitForDeepHealthWithArgs(composeCommand, directory, secrets = []) {
886
- const deadline = Date.now() + DEEP_HEALTH_TIMEOUT_MS;
918
+ const startedAt = Date.now();
919
+ const deadline = startedAt + DEEP_HEALTH_TIMEOUT_MS;
887
920
  let lastError = "health check did not complete";
921
+ let lastProgressAt = startedAt;
888
922
  while (Date.now() < deadline) {
889
923
  const result = await run("docker", [
890
924
  ...composeCommand,
@@ -896,6 +930,10 @@ async function waitForDeepHealthWithArgs(composeCommand, directory, secrets = []
896
930
  if (result.ok)
897
931
  return;
898
932
  lastError = redact(result.stderr, secrets);
933
+ if (Date.now() - lastProgressAt >= 15000) {
934
+ console.error(`Waiting for OR3 deep health (${Math.round((Date.now() - startedAt) / 1000)}s elapsed)…`);
935
+ lastProgressAt = Date.now();
936
+ }
899
937
  await new Promise((resolvePromise) => setTimeout(resolvePromise, 1000));
900
938
  }
901
939
  throw new Error(`OR3 deep health did not pass within ${DEEP_HEALTH_TIMEOUT_MS / 1000} seconds. Last error: ${lastError}`);
@@ -1112,6 +1150,26 @@ function packagedSourceRevision(version) {
1112
1150
  }
1113
1151
  return revision;
1114
1152
  }
1153
+ function packagedOr3CloudMetadata() {
1154
+ try {
1155
+ const manifest = JSON.parse(readFileSync(join(PACKAGE_ROOT, "package.json"), "utf8"));
1156
+ return manifest.or3Cloud ?? {};
1157
+ } catch {
1158
+ return {};
1159
+ }
1160
+ }
1161
+ function writeStateSchema() {
1162
+ const value = packagedOr3CloudMetadata().stateSchema;
1163
+ if (value === undefined)
1164
+ return LEGACY_STATE_SCHEMA_VERSION;
1165
+ if (value === 1 || value === 2)
1166
+ return value;
1167
+ throw new Error("This @or3/cloud package declares an unsupported managed state schema. Refusing to write managed state.");
1168
+ }
1169
+ function packagedMinimumSourceVersion() {
1170
+ const value = packagedOr3CloudMetadata().dashboardUpdateMinimumSourceVersion;
1171
+ return typeof value === "string" && /^\d+\.\d+\.\d+$/.test(value) ? value : undefined;
1172
+ }
1115
1173
  function expectedImageDigest(version) {
1116
1174
  const packaged = packagedImageDigest(version);
1117
1175
  const supplied = process.env.OR3_EXPECTED_IMAGE_DIGEST?.trim();
@@ -1438,10 +1496,11 @@ function dashboardOperatorHandoffArgs(directory, env, jobId) {
1438
1496
  project
1439
1497
  ];
1440
1498
  }
1441
- async function scheduleDashboardOperatorHandoff(directory, env) {
1442
- const jobId = process.env.OR3_DASHBOARD_UPDATE_JOB_ID?.trim();
1499
+ async function scheduleDashboardOperatorHandoff(directory, env, jobIdOverride) {
1500
+ const jobId = jobIdOverride ?? process.env.OR3_DASHBOARD_UPDATE_JOB_ID?.trim();
1443
1501
  if (!jobId || env.OR3_DASHBOARD_UPDATES_ENABLED !== "true")
1444
1502
  return;
1503
+ await lifecycleFaults.beforeHandoff?.();
1445
1504
  const scheduled = await run("docker", dashboardOperatorHandoffArgs(directory, env, jobId), directory);
1446
1505
  if (!scheduled.ok || !/^[0-9a-f]{12,64}$/i.test(scheduled.stdout.trim())) {
1447
1506
  throw new Error(`Could not schedule the dashboard operator handoff. ${scheduled.stderr.trim()}`);
@@ -1642,6 +1701,15 @@ async function copyAssets(directory, mode) {
1642
1701
  }
1643
1702
  await installManagedAssets(directory, assets);
1644
1703
  }
1704
+ async function installedManagedAssetChecksums(directory, mode) {
1705
+ const checksums = {};
1706
+ for (const name of managedAssetNames(mode)) {
1707
+ if (!await fileExists(join(directory, name)))
1708
+ continue;
1709
+ checksums[name] = await sha256File(join(directory, name));
1710
+ }
1711
+ return checksums;
1712
+ }
1645
1713
  async function snapshotManagedAssets(directory, mode, backupDir) {
1646
1714
  const assetDir = join(backupDir, "managed-assets");
1647
1715
  await mkdir(assetDir, { recursive: true, mode: 448 });
@@ -1731,14 +1799,61 @@ async function updatePending(directory, state, patch) {
1731
1799
  async function removeOperationRecord(directory, operationId) {
1732
1800
  if (!operationId)
1733
1801
  return;
1802
+ await lifecycleFaults.beforeMirrorDelete?.();
1734
1803
  await rm(join(deploymentPaths(directory).operations, `${operationId}.json`), { force: true });
1735
1804
  }
1736
- async function clearPending(directory, state) {
1805
+ async function removeOperationRecordSafely(directory, operationId) {
1806
+ try {
1807
+ await removeOperationRecord(directory, operationId);
1808
+ } catch (error) {
1809
+ console.warn(`Could not remove the redundant operation mirror ${operationId ?? "(none)"}: ${redact(error instanceof Error ? error.message : String(error))}`);
1810
+ }
1811
+ }
1812
+ async function commitTerminalState(directory, state, receipt) {
1737
1813
  const operationId = state.incompleteOperation?.id;
1738
1814
  delete state.incompleteOperation;
1815
+ if (receipt)
1816
+ state.lastReceipt = receipt;
1739
1817
  state.updatedAt = now();
1740
- await removeOperationRecord(directory, operationId);
1741
1818
  await writeState(directory, state);
1819
+ if (receipt) {
1820
+ try {
1821
+ await exportTerminalReceipt(directory, receipt);
1822
+ } catch (error) {
1823
+ console.warn(`Could not export the latest operation receipt: ${redact(error instanceof Error ? error.message : String(error))}`);
1824
+ }
1825
+ }
1826
+ return { committed: true, operationId };
1827
+ }
1828
+ async function exportTerminalReceipt(directory, receipt) {
1829
+ const payload = `${JSON.stringify(receipt, null, 2)}
1830
+ `;
1831
+ await writeSecure(deploymentPaths(directory).lastOperation, payload);
1832
+ }
1833
+ async function persistReceiptWarnings(directory, state, warnings) {
1834
+ if (warnings.length === 0 || !state.lastReceipt)
1835
+ return;
1836
+ const existing = state.lastReceipt.warnings;
1837
+ const additions = warnings.filter((warning) => !existing.some((current) => current.code === warning.code && current.message === warning.message));
1838
+ if (additions.length === 0)
1839
+ return;
1840
+ const merged = { ...state.lastReceipt, warnings: [...existing, ...additions] };
1841
+ try {
1842
+ if (state.incompleteOperation)
1843
+ return;
1844
+ state.lastReceipt = merged;
1845
+ state.updatedAt = now();
1846
+ await writeState(directory, state);
1847
+ await exportTerminalReceipt(directory, merged).catch((error) => {
1848
+ console.warn(`Could not re-export the operation receipt: ${redact(error instanceof Error ? error.message : String(error))}`);
1849
+ });
1850
+ } catch (error) {
1851
+ console.warn(`Could not persist maintenance warnings to the receipt: ${redact(error instanceof Error ? error.message : String(error))}`);
1852
+ }
1853
+ }
1854
+ async function clearPending(directory, state, receipt) {
1855
+ const { operationId } = await commitTerminalState(directory, state, receipt);
1856
+ await removeOperationRecordSafely(directory, operationId);
1742
1857
  }
1743
1858
  async function stopProject(directory, mode) {
1744
1859
  await compose(directory, mode, ["stop", "or3"]);
@@ -1807,6 +1922,7 @@ function assertRemovableArtifactName(name) {
1807
1922
  return name;
1808
1923
  }
1809
1924
  async function removeNamedBackupArtifact(directory, backupId) {
1925
+ await lifecycleFaults.beforeArtifactDelete?.();
1810
1926
  assertRemovableArtifactName(backupId);
1811
1927
  const target = backupDirectory(directory, backupId);
1812
1928
  if (dirname(target) !== deploymentPaths(directory).backups) {
@@ -1816,6 +1932,7 @@ async function removeNamedBackupArtifact(directory, backupId) {
1816
1932
  await rm(join(deploymentPaths(directory).exports, `${backupId}.json`), { force: true });
1817
1933
  }
1818
1934
  async function removeEnumeratedBackupArtifact(directory, backup) {
1935
+ await lifecycleFaults.beforeArtifactDelete?.();
1819
1936
  const backupsRoot = resolve(deploymentPaths(directory).backups);
1820
1937
  const target = resolve(backup.path);
1821
1938
  if (dirname(target) !== backupsRoot || basename(target) !== backup.backupId || !BACKUP_ID_PATTERN.test(backup.backupId)) {
@@ -1824,55 +1941,159 @@ async function removeEnumeratedBackupArtifact(directory, backup) {
1824
1941
  await rm(target, { recursive: true, force: true });
1825
1942
  await rm(join(deploymentPaths(directory).exports, `${backup.backupId}.json`), { force: true });
1826
1943
  }
1827
- async function enumerateBackups(directory) {
1828
- const backupsRoot = deploymentPaths(directory).backups;
1829
- let entries = [];
1944
+ function classifyBackupError(error) {
1945
+ const message = error instanceof Error ? error.message : String(error);
1946
+ if (/authentication failed|no valid deployment authentication tag|invalid authentication tag/i.test(message)) {
1947
+ return { code: "backup-authentication-failed", message };
1948
+ }
1949
+ if (/checksum mismatch/i.test(message))
1950
+ return { code: "backup-checksum-mismatch", message };
1951
+ if (/invalid managed asset inventory|managed asset checksum mismatch/i.test(message)) {
1952
+ return { code: "backup-assets-invalid", message };
1953
+ }
1954
+ if (/Invalid backup manifest/i.test(message))
1955
+ return { code: "backup-manifest-invalid", message };
1956
+ if (error?.code === "ENOENT")
1957
+ return { code: "backup-entry-incomplete", message };
1958
+ return { code: "backup-entry-invalid", message };
1959
+ }
1960
+ async function inspectBackupEntry(backupsRoot, entry, directory) {
1961
+ const path = join(backupsRoot, entry);
1962
+ let info;
1830
1963
  try {
1831
- entries = await readdir(backupsRoot);
1964
+ info = await lstat(path);
1832
1965
  } catch (error) {
1833
- if (error.code === "ENOENT")
1834
- return [];
1835
- throw new Error(`Could not enumerate managed backups at ${backupsRoot}: ${error instanceof Error ? error.message : String(error)}`);
1966
+ return { kind: "unreadable", entryName: entry, ...classifyBackupError(error) };
1836
1967
  }
1837
- const result = [];
1838
- for (const entry of entries) {
1839
- if (/^adopt-source-[0-9A-Za-z-]+$/.test(entry) && (await lstat(join(backupsRoot, entry))).isDirectory()) {
1840
- continue;
1841
- }
1842
- if (!BACKUP_ID_PATTERN.test(entry)) {
1843
- throw new Error(`Backup store contains an unexpected artifact ${join(backupsRoot, entry)}. Refusing to use it for retention; inspect or remove it explicitly.`);
1968
+ if (info.isSymbolicLink() || !info.isDirectory()) {
1969
+ return {
1970
+ kind: "invalid",
1971
+ entryName: entry,
1972
+ code: "backup-entry-not-directory",
1973
+ message: `Backup store entry ${entry} is not a regular directory; symlinks and non-directories are never trusted.`
1974
+ };
1975
+ }
1976
+ if (/^adopt-source-[0-9A-Za-z-]+$/.test(entry)) {
1977
+ return {
1978
+ kind: "legacy-adoption",
1979
+ entryName: entry,
1980
+ code: "backup-legacy-adoption",
1981
+ message: `Preserved legacy adoption source ${entry}; it is not an authenticated managed backup.`
1982
+ };
1983
+ }
1984
+ if (!BACKUP_ID_PATTERN.test(entry)) {
1985
+ return {
1986
+ kind: "invalid",
1987
+ entryName: entry,
1988
+ code: "backup-entry-unexpected",
1989
+ message: `Backup store contains an unexpected artifact ${path}; inspect or remove it explicitly.`
1990
+ };
1991
+ }
1992
+ try {
1993
+ await lstat(join(path, "manifest.auth"));
1994
+ } catch (error) {
1995
+ if (error.code === "ENOENT") {
1996
+ return {
1997
+ kind: "legacy-unsigned",
1998
+ entryName: entry,
1999
+ code: "backup-unsigned",
2000
+ message: `Preserved unauthenticated backup ${entry} (no manifest.auth); never trusted for restore.`
2001
+ };
1844
2002
  }
1845
- const path = join(backupsRoot, entry);
1846
- try {
1847
- await lstat(join(path, "manifest.auth"));
1848
- } catch (error) {
1849
- if (error.code !== "ENOENT")
1850
- throw error;
1851
- console.warn(`Preserving unauthenticated backup ${entry}: no manifest.auth; excluded from managed backup listing and retention.`);
1852
- continue;
2003
+ return { kind: "unreadable", entryName: entry, ...classifyBackupError(error) };
2004
+ }
2005
+ try {
2006
+ const raw = JSON.parse(await readText(join(path, "manifest.json")));
2007
+ if (typeof raw.schemaVersion === "number" && raw.schemaVersion > 1) {
2008
+ return {
2009
+ kind: "unsupported",
2010
+ entryName: entry,
2011
+ code: "backup-format-unsupported",
2012
+ message: `Backup ${entry} uses manifest schema ${raw.schemaVersion}; preserved for a newer reader.`
2013
+ };
1853
2014
  }
2015
+ } catch (error) {
2016
+ return { kind: "invalid", entryName: entry, ...classifyBackupError(error) };
2017
+ }
2018
+ try {
1854
2019
  const manifest = await readManifest(path, directory);
1855
2020
  if (manifest.backupId !== entry) {
1856
- throw new Error(`Backup directory ${path} does not match manifest ID ${manifest.backupId}. Refusing to use it for retention.`);
2021
+ return {
2022
+ kind: "invalid",
2023
+ entryName: entry,
2024
+ code: "backup-id-mismatch",
2025
+ message: `Backup directory ${path} does not match manifest ID ${manifest.backupId}.`
2026
+ };
1857
2027
  }
1858
2028
  if (!Number.isFinite(Date.parse(manifest.createdAt))) {
1859
- throw new Error(`Backup ${manifest.backupId} has an invalid creation time.`);
2029
+ return {
2030
+ kind: "invalid",
2031
+ entryName: entry,
2032
+ code: "backup-created-at-invalid",
2033
+ message: `Backup ${manifest.backupId} has an invalid creation time.`
2034
+ };
1860
2035
  }
1861
2036
  let bytes = 0;
1862
2037
  for (const file of ["data.tgz", "config.env", "manifest.json", "manifest.auth"]) {
1863
- bytes += (await stat(join(path, file))).size;
2038
+ bytes += (await lstat(join(path, file))).size;
1864
2039
  }
1865
- result.push({
1866
- backupId: manifest.backupId,
1867
- createdAt: manifest.createdAt,
1868
- appVersion: manifest.appVersion,
1869
- path,
1870
- bytes,
1871
- dataSha256: manifest.dataSha256
1872
- });
2040
+ return {
2041
+ kind: "verified",
2042
+ backup: {
2043
+ backupId: manifest.backupId,
2044
+ createdAt: manifest.createdAt,
2045
+ appVersion: manifest.appVersion,
2046
+ path,
2047
+ bytes,
2048
+ dataSha256: manifest.dataSha256
2049
+ }
2050
+ };
2051
+ } catch (error) {
2052
+ return { kind: "invalid", entryName: entry, ...classifyBackupError(error) };
1873
2053
  }
1874
- result.sort((a, b) => a.createdAt < b.createdAt ? 1 : a.createdAt > b.createdAt ? -1 : 0);
1875
- return result;
2054
+ }
2055
+ var BACKUP_ENTRY_SEVERITY = {
2056
+ verified: 0,
2057
+ "legacy-unsigned": 1,
2058
+ "legacy-adoption": 2,
2059
+ unsupported: 3,
2060
+ invalid: 4,
2061
+ unreadable: 5
2062
+ };
2063
+ async function inventoryBackups(directory) {
2064
+ const backupsRoot = deploymentPaths(directory).backups;
2065
+ let names;
2066
+ try {
2067
+ names = await readdir(backupsRoot);
2068
+ } catch (error) {
2069
+ if (error.code === "ENOENT")
2070
+ return { entries: [], storeErrors: [] };
2071
+ return {
2072
+ entries: [],
2073
+ storeErrors: [{
2074
+ code: "backup-store-unreadable",
2075
+ severity: "blocker",
2076
+ resource: backupsRoot,
2077
+ message: `Could not enumerate managed backups at ${backupsRoot}: ${error instanceof Error ? error.message : String(error)}`
2078
+ }]
2079
+ };
2080
+ }
2081
+ const entries = [];
2082
+ for (const name of [...names].sort())
2083
+ entries.push(await inspectBackupEntry(backupsRoot, name, directory));
2084
+ entries.sort((a, b) => {
2085
+ const severity = BACKUP_ENTRY_SEVERITY[a.kind] - BACKUP_ENTRY_SEVERITY[b.kind];
2086
+ if (severity !== 0)
2087
+ return severity;
2088
+ const aName = a.kind === "verified" ? a.backup.backupId : a.entryName;
2089
+ const bName = b.kind === "verified" ? b.backup.backupId : b.entryName;
2090
+ return aName < bName ? -1 : aName > bName ? 1 : 0;
2091
+ });
2092
+ return { entries, storeErrors: [] };
2093
+ }
2094
+ async function enumerateBackups(directory) {
2095
+ const inventory = await inventoryBackups(directory);
2096
+ return inventory.entries.filter((entry) => entry.kind === "verified").map((entry) => entry.backup).sort((a, b) => a.createdAt < b.createdAt ? 1 : a.createdAt > b.createdAt ? -1 : 0);
1876
2097
  }
1877
2098
  function selectPruneTargets(backups, keep, protectedIds, force = false) {
1878
2099
  if (!Number.isInteger(keep) || keep < 1)
@@ -1881,6 +2102,40 @@ function selectPruneTargets(backups, keep, protectedIds, force = false) {
1881
2102
  const deletable = force ? sorted : sorted.filter((backup) => !protectedIds.has(backup.backupId));
1882
2103
  return deletable.slice(keep).map((backup) => backup.backupId);
1883
2104
  }
2105
+ function planRetention(entries, keep, protectedIds, options = {}) {
2106
+ if (!Number.isInteger(keep) || keep < 1)
2107
+ throw new Error("Backup retention must be an integer of at least 1.");
2108
+ const automatic = options.automatic ?? true;
2109
+ const warnings = [];
2110
+ const preserve = [];
2111
+ let canPrune = true;
2112
+ for (const entry of entries) {
2113
+ if (entry.kind === "verified")
2114
+ continue;
2115
+ preserve.push({ entryName: entry.entryName, reason: entry.message });
2116
+ if (entry.kind === "legacy-unsigned" || entry.kind === "legacy-adoption")
2117
+ continue;
2118
+ warnings.push({
2119
+ code: entry.code,
2120
+ severity: entry.kind === "unreadable" ? "blocker" : "warning",
2121
+ resource: entry.entryName,
2122
+ message: entry.message
2123
+ });
2124
+ if (automatic)
2125
+ canPrune = false;
2126
+ }
2127
+ const verified = entries.filter((entry) => entry.kind === "verified").map((entry) => entry.backup).sort((a, b) => a.createdAt < b.createdAt ? 1 : a.createdAt > b.createdAt ? -1 : 0);
2128
+ const removable = verified.filter((backup) => !protectedIds.has(backup.backupId));
2129
+ const remove = removable.slice(keep).map((backup) => backup.backupId);
2130
+ const removeSet = new Set(remove);
2131
+ return {
2132
+ keep: verified.filter((backup) => !removeSet.has(backup.backupId)).map((backup) => backup.backupId),
2133
+ remove,
2134
+ preserve,
2135
+ warnings,
2136
+ canPrune
2137
+ };
2138
+ }
1884
2139
  function parseKeep(flags) {
1885
2140
  const value = stringFlag(flags, "keep") ?? String(BACKUP_RETENTION_KEEP);
1886
2141
  const keep = Number(value);
@@ -1888,25 +2143,53 @@ function parseKeep(flags) {
1888
2143
  throw new Error("--keep must be an integer of at least 1.");
1889
2144
  return keep;
1890
2145
  }
1891
- async function pruneBackups(directory, state, keep, force) {
1892
- const backups = await enumerateBackups(directory);
2146
+ function retentionProtectedIds(state) {
1893
2147
  const protectedIds = new Set;
1894
2148
  if (state.rollback?.backupId)
1895
2149
  protectedIds.add(state.rollback.backupId);
1896
- if (state.incompleteOperation?.backupId)
1897
- protectedIds.add(state.incompleteOperation.backupId);
1898
- const targets = selectPruneTargets(backups, keep, protectedIds, force);
1899
- if (force && targets.length > 0) {
1900
- console.log(`--force --yes will permanently delete: ${targets.join(", ")}`);
1901
- }
1902
- for (const backupId of targets) {
1903
- const backup = backups.find((entry) => entry.backupId === backupId);
2150
+ const pending = state.incompleteOperation;
2151
+ if (pending?.backupId)
2152
+ protectedIds.add(pending.backupId);
2153
+ if (pending?.previousBackupId)
2154
+ protectedIds.add(pending.previousBackupId);
2155
+ return protectedIds;
2156
+ }
2157
+ async function pruneBackups(directory, state, keep, force, options = {}) {
2158
+ const automatic = options.automatic ?? !force;
2159
+ const log = options.log ?? ((message) => console.log(message));
2160
+ const warn = options.warn ?? ((message) => console.warn(message));
2161
+ const inventory = await inventoryBackups(directory);
2162
+ if (inventory.storeErrors.length > 0) {
2163
+ if (automatic)
2164
+ return { removed: 0, deferred: inventory.storeErrors };
2165
+ throw new Error(inventory.storeErrors.map((diagnostic) => diagnostic.message).join(" "));
2166
+ }
2167
+ const plan = planRetention(inventory.entries, keep, retentionProtectedIds(state), { automatic });
2168
+ if (!plan.canPrune && !force) {
2169
+ warn("Automatic backup pruning was deferred because the store contains entries that need inspection.");
2170
+ for (const warning of plan.warnings)
2171
+ warn(` ${warning.code}: ${warning.message}`);
2172
+ return { removed: 0, deferred: plan.warnings };
2173
+ }
2174
+ const verified = new Map(inventory.entries.filter((entry) => entry.kind === "verified").map((entry) => [entry.backup.backupId, entry.backup]));
2175
+ if (force && plan.remove.length > 0) {
2176
+ log(`--force --yes will permanently delete: ${plan.remove.join(", ")}`);
2177
+ }
2178
+ let removed = 0;
2179
+ for (const backupId of plan.remove) {
2180
+ const backup = verified.get(backupId);
1904
2181
  if (!backup)
1905
2182
  throw new Error(`Retention selected backup ${backupId}, but its verified path disappeared before deletion.`);
1906
- await removeEnumeratedBackupArtifact(directory, backup);
1907
- console.log(`Deleted backup ${backupId} at ${backup.path}`);
2183
+ const revalidated = await inspectBackupEntry(deploymentPaths(directory).backups, backupId, directory);
2184
+ const current = revalidated.kind === "verified" ? revalidated.backup : undefined;
2185
+ if (!current || current.path !== backup.path || current.dataSha256 !== backup.dataSha256) {
2186
+ throw new Error(`Backup ${backupId} changed while pruning was in progress. Aborting retention before deletion.`);
2187
+ }
2188
+ await removeEnumeratedBackupArtifact(directory, current);
2189
+ log(`Deleted backup ${backupId} at ${current.path}`);
2190
+ removed += 1;
1908
2191
  }
1909
- return targets.length;
2192
+ return { removed, deferred: plan.warnings };
1910
2193
  }
1911
2194
  function assertDeploymentIdentity(state, env) {
1912
2195
  const expected = {
@@ -2314,6 +2597,7 @@ async function createBackup(directory, state, env, options = {}) {
2314
2597
  }
2315
2598
  }
2316
2599
  async function readManifest(backupPath, authenticatedForDirectory) {
2600
+ await lifecycleFaults.beforeArchiveRead?.();
2317
2601
  const manifestContents = await readText(join(backupPath, "manifest.json"));
2318
2602
  if (authenticatedForDirectory)
2319
2603
  await assertBackupAuthentication(authenticatedForDirectory, backupPath, manifestContents);
@@ -2418,7 +2702,7 @@ async function restoreBackupData(directory, state, env, backupPath, options = {}
2418
2702
  }
2419
2703
  function stateFromEnv(directory, env, mode, operation, digest) {
2420
2704
  return {
2421
- schemaVersion: 1,
2705
+ schemaVersion: writeStateSchema(),
2422
2706
  mode,
2423
2707
  composeProject: env.OR3_COMPOSE_PROJECT,
2424
2708
  volumeName: env.OR3_VOLUME_NAME,
@@ -2441,16 +2725,16 @@ function help() {
2441
2725
  Usage:
2442
2726
  npx @or3/cloud init [directory] --local
2443
2727
  npx @or3/cloud init [directory] --public --domain <hostname>
2444
- npx @or3/cloud update [--to <exact-version>]
2445
- npx @or3/cloud backup [list|prune [--keep <n>]|export <backup-id> <destination-dir>]
2728
+ npx @or3/cloud update [--to <exact-version>] [--dry-run] [--json]
2729
+ npx @or3/cloud backup [list [--json]|prune [--keep <n>]|export <backup-id> <destination-dir>]
2446
2730
  npx @or3/cloud restore <backup-id-or-path> --yes
2447
2731
  npx @or3/cloud rollback --yes
2448
2732
  npx @or3/cloud credentials reset --yes [--owner-password-file <path> --admin-password-file <path>]
2449
2733
  npx @or3/cloud doctor
2450
- npx @or3/cloud verify [--public] [--verification-email <email> --verification-password-file <path>]
2451
- npx @or3/cloud recover
2734
+ npx @or3/cloud verify [--read-only] [--public] [--verification-email <email> --verification-password-file <path>]
2735
+ npx @or3/cloud recover [--dry-run | --finish | --restore --yes]
2452
2736
  npx @or3/cloud adopt --from <v1-directory> [directory]
2453
- npx @or3/cloud status
2737
+ npx @or3/cloud status [--json]
2454
2738
  npx @or3/cloud logs [--tail <n>] [service]
2455
2739
  npx @or3/cloud start | stop | restart
2456
2740
  npx @or3/cloud remove [--purge-data --yes]
@@ -2468,6 +2752,11 @@ Options:
2468
2752
  --force Prune backups even when referenced by the rollback point
2469
2753
  --tail <n> Log lines to show (default: 200)
2470
2754
  --public Require verification through the public HTTPS origin
2755
+ --read-only Verify without the lease, login, storage, or database writes
2756
+ --dry-run Preview an update or recovery without changing anything
2757
+ --finish Commit a proven completed replacement without restoring data
2758
+ --restore Explicitly restore the recorded snapshot (requires --yes)
2759
+ --json Emit one machine-readable result object on stdout
2471
2760
  --purge-data Remove data volumes and managed files (with remove)
2472
2761
  --yes Confirm a destructive restore, rollback, credentials reset, or purge
2473
2762
  --help Show this help
@@ -2557,7 +2846,8 @@ ${existing.stderr}`)) {
2557
2846
  await clearPending(directory, state);
2558
2847
  console.log(`
2559
2848
  OR3 Cloud ${version} is running at ${mode === "public" ? `https://${domain}` : `http://127.0.0.1:${port}`}`);
2560
- console.log(`Credentials were written to ${join(directory, ".or3-initial-credentials")} (mode 0600). Save them, then remove that file.`);
2849
+ console.log(`Credentials were written to ${join(directory, ".or3-initial-credentials")} (mode 0600). Move them to protected storage (a password manager), then remove that file.`);
2850
+ console.log('The bootstrap owner signs in to the app; the admin account manages the deployment. Rotate either later with "npx @or3/cloud credentials reset --yes".');
2561
2851
  console.log(`
2562
2852
  Check: cd ${quote(directory)} && npx @or3/cloud doctor`);
2563
2853
  if (operator)
@@ -2572,22 +2862,179 @@ Check: cd ${quote(directory)} && npx @or3/cloud doctor`);
2572
2862
  The new deployment files were preserved at ${directory}.`);
2573
2863
  }
2574
2864
  }
2575
- async function loadManaged(directory = process.cwd()) {
2865
+ function assertStateSchemaWritable(state) {
2866
+ const writer = writeStateSchema();
2867
+ if (state.schemaVersion > writer) {
2868
+ throw new Error(`This CLI reads managed state schema ${state.schemaVersion} but is qualified to write schema ${writer}. Run the compatible exact-version @or3/cloud CLI for this deployment; do not edit or hand-migrate managed state.`);
2869
+ }
2870
+ }
2871
+ async function loadManaged(directory = process.cwd(), options = {}) {
2576
2872
  const resolved = resolve(directory);
2577
2873
  const state = await readState(resolved);
2874
+ if (options.writable !== false)
2875
+ assertStateSchemaWritable(state);
2578
2876
  const env = parseEnv(await readText(deploymentPaths(resolved).env));
2579
2877
  assertDeploymentIdentity(state, env);
2580
2878
  assertDeploymentDirectoryIdentity(resolved, state);
2581
2879
  return { directory: resolved, state, env };
2582
2880
  }
2881
+ async function observeDeployment(directory, options = {}) {
2882
+ const resolved = resolve(directory);
2883
+ const observedAt = now();
2884
+ let state = null;
2885
+ let stateError = null;
2886
+ try {
2887
+ state = await readState(resolved);
2888
+ } catch (error) {
2889
+ stateError = redact(error instanceof Error ? error.message : String(error));
2890
+ }
2891
+ let env = null;
2892
+ let envError = null;
2893
+ try {
2894
+ env = parseEnv(await readText(deploymentPaths(resolved).env));
2895
+ } catch (error) {
2896
+ envError = redact(error instanceof Error ? error.message : String(error));
2897
+ }
2898
+ const leasePath = deploymentPaths(resolved).lease;
2899
+ let lease = { status: "none" };
2900
+ if (await fileExists(leasePath)) {
2901
+ const owner = await readLeaseOwner(leasePath);
2902
+ if (!owner)
2903
+ lease = { status: "unreadable" };
2904
+ else if (cliLeaseOwnerIsGone(owner) || dashboardLeaseOwnerIsStale(owner))
2905
+ lease = { status: "stale", owner };
2906
+ else
2907
+ lease = { status: "active", owner };
2908
+ }
2909
+ let backups = null;
2910
+ const backupErrors = [];
2911
+ try {
2912
+ backups = await inventoryBackups(resolved);
2913
+ backupErrors.push(...backups.storeErrors);
2914
+ } catch (error) {
2915
+ backupErrors.push({ code: "backup-store-unreadable", severity: "blocker", message: redact(error instanceof Error ? error.message : String(error)) });
2916
+ }
2917
+ const checkDocker = options.checkDocker ?? true;
2918
+ const checkImage = options.checkImage ?? true;
2919
+ let docker = false;
2920
+ let recordedImageDigest = state?.imageDigest ?? null;
2921
+ let actualImageDigest = null;
2922
+ let identityMatches = null;
2923
+ if (checkDocker) {
2924
+ try {
2925
+ await ensureDocker();
2926
+ docker = true;
2927
+ } catch {
2928
+ docker = false;
2929
+ }
2930
+ }
2931
+ if (checkImage && docker && state?.image) {
2932
+ try {
2933
+ actualImageDigest = await runningContainerImageDigest(resolved, state.mode, state.image);
2934
+ identityMatches = actualImageDigest === state.imageDigest;
2935
+ } catch {
2936
+ actualImageDigest = null;
2937
+ identityMatches = null;
2938
+ }
2939
+ }
2940
+ const partial = Boolean(stateError || envError || backupErrors.length > 0 || checkDocker && !docker);
2941
+ return {
2942
+ directory: resolved,
2943
+ observedAt,
2944
+ state,
2945
+ stateError,
2946
+ env,
2947
+ envError,
2948
+ lease,
2949
+ docker,
2950
+ recordedImageDigest,
2951
+ actualImageDigest,
2952
+ identityMatches,
2953
+ backups,
2954
+ backupErrors,
2955
+ changing: lease.status === "active",
2956
+ partial
2957
+ };
2958
+ }
2959
+ async function runningContainerImageDigest(directory, mode, fallbackImage) {
2960
+ let containerId = "";
2961
+ try {
2962
+ const ps = await run("docker", composeArgs(directory, mode, ["ps", "-q", "or3"]), directory);
2963
+ containerId = ps.ok ? ps.stdout.trim() : "";
2964
+ } catch {
2965
+ containerId = "";
2966
+ }
2967
+ if (!containerId)
2968
+ return await imageDigest(fallbackImage);
2969
+ const image = await run("docker", ["inspect", "--format", "{{.Image}}", containerId], directory);
2970
+ const imageId = image.ok ? image.stdout.trim() : "";
2971
+ if (!imageId)
2972
+ throw new Error("Could not inspect the running OR3 container image.");
2973
+ const digests = await run("docker", ["image", "inspect", "--format", "{{json .RepoDigests}}", imageId], directory);
2974
+ if (!digests.ok)
2975
+ throw new Error("Could not inspect repository digests for the running OR3 container.");
2976
+ let values;
2977
+ try {
2978
+ values = JSON.parse(digests.stdout.trim());
2979
+ } catch {
2980
+ throw new Error("The running OR3 container has no readable repository digests.");
2981
+ }
2982
+ const match = values.map((value) => value.match(/@(sha256:[0-9a-f]{64})$/i)?.[1]).find((value) => Boolean(value));
2983
+ if (!match)
2984
+ throw new Error("The running OR3 container image is not digest-qualified.");
2985
+ return match;
2986
+ }
2987
+ function publicStateProjection(state) {
2988
+ const pending = state.incompleteOperation;
2989
+ return {
2990
+ schemaVersion: state.schemaVersion,
2991
+ mode: state.mode,
2992
+ appVersion: state.appVersion,
2993
+ image: state.image,
2994
+ imageDigest: state.imageDigest,
2995
+ domain: state.domain ?? null,
2996
+ port: state.port,
2997
+ deploymentId: state.deploymentId ?? null,
2998
+ lastSuccessfulOperation: state.lastSuccessfulOperation,
2999
+ updatedAt: state.updatedAt,
3000
+ rollback: state.rollback ? { appVersion: state.rollback.appVersion, imageDigest: state.rollback.imageDigest, backupId: state.rollback.backupId, createdAt: state.rollback.createdAt } : null,
3001
+ incompleteOperation: pending ? {
3002
+ id: pending.id,
3003
+ operation: pending.operation,
3004
+ phase: pending.phase ?? null,
3005
+ origin: pending.origin ?? null,
3006
+ dashboardJobId: pending.dashboardJobId ?? null,
3007
+ targetVersion: pending.targetVersion ?? null,
3008
+ targetImageDigest: pending.targetImageDigest ?? null
3009
+ } : null,
3010
+ lastReceipt: state.lastReceipt ?? null,
3011
+ lastError: state.lastError ? redact(state.lastError) : null
3012
+ };
3013
+ }
3014
+ function observationFindings(observation) {
3015
+ const findings = [];
3016
+ if (observation.stateError)
3017
+ findings.push({ code: "state-unreadable", severity: "blocker", resource: deploymentPaths(observation.directory).state, message: observation.stateError });
3018
+ if (observation.envError)
3019
+ findings.push({ code: "env-unreadable", severity: "blocker", resource: deploymentPaths(observation.directory).env, message: observation.envError });
3020
+ if (observation.lease.status === "active") {
3021
+ findings.push({ code: "operation-in-progress", severity: "info", message: `A ${observation.lease.owner.origin} operation (${observation.lease.owner.command}) is active; this observation is not authoritative.` });
3022
+ }
3023
+ if (observation.lease.status === "unreadable")
3024
+ findings.push({ code: "lease-unreadable", severity: "warning", message: "The deployment lease owner record is unreadable; run doctor before any mutation." });
3025
+ if (observation.identityMatches === false) {
3026
+ findings.push({ code: "image-digest-mismatch", severity: "blocker", resource: observation.state?.image, message: `Recorded digest ${observation.recordedImageDigest} does not match the local image ${observation.actualImageDigest}.` });
3027
+ }
3028
+ return findings;
3029
+ }
2583
3030
  function operationToStateOperation(operation, fallback) {
2584
3031
  if (operation === "init" || operation === "update" || operation === "adopt")
2585
3032
  return operation;
2586
3033
  return fallback;
2587
3034
  }
2588
3035
  async function commitRecoveredState(directory, previous, next) {
2589
- await removeOperationRecord(directory, previous.incompleteOperation?.id);
2590
3036
  await writeState(directory, next);
3037
+ await removeOperationRecordSafely(directory, previous.incompleteOperation?.id);
2591
3038
  }
2592
3039
  async function commitPreMutationRecovery(directory, state, message) {
2593
3040
  const restoredEnv = parseEnv(await readText(deploymentPaths(directory).env));
@@ -2597,14 +3044,293 @@ async function commitPreMutationRecovery(directory, state, message) {
2597
3044
  await commitRecoveredState(directory, state, recovered);
2598
3045
  return recovered;
2599
3046
  }
2600
- async function recoverCommand(directory) {
3047
+ function decideRecoveryAction(state) {
3048
+ const pending = state.incompleteOperation;
3049
+ if (!pending) {
3050
+ const handoff = state.lastReceipt?.operatorHandoff;
3051
+ if (handoff === "pending" || handoff === "needs-attention") {
3052
+ return { action: "reconcile-handoff", detail: `The application is complete but the dashboard operator handoff is ${handoff}. Reconcile only the recorded handoff.` };
3053
+ }
3054
+ return { action: "none", detail: "No incomplete operation is recorded." };
3055
+ }
3056
+ if (pending.operation === "update" && pending.phase === "target-ready" && pending.evidence) {
3057
+ return { action: "finish", detail: "A completed replacement carries durable target-ready evidence and can be committed without restoring data." };
3058
+ }
3059
+ if (pending.phase === "prepared" || pending.phase === "snapshot-created") {
3060
+ return { action: "resume", detail: `The ${pending.operation} had not replaced data; resume the known-good source.` };
3061
+ }
3062
+ if (pending.operation === "update" || pending.operation === "restore" || pending.operation === "rollback") {
3063
+ const snapshotId = pending.operation === "update" ? pending.backupId : pending.previousBackupId ?? pending.backupId;
3064
+ return {
3065
+ action: "require-explicit-restore",
3066
+ detail: `The ${pending.operation} may have changed data and has no completion proof. Health alone cannot authorize adoption.`,
3067
+ snapshotId
3068
+ };
3069
+ }
3070
+ return { action: "resume", detail: `Resume the ${pending.operation} using its durable evidence.` };
3071
+ }
3072
+ function assertRecoverFlags(flags) {
3073
+ const dryRun = boolFlag(flags, "dry-run");
3074
+ const finish = boolFlag(flags, "finish");
3075
+ const restore = boolFlag(flags, "restore");
3076
+ if (finish && restore)
3077
+ throw new Error("Use either --finish or --restore, not both.");
3078
+ if (dryRun && (finish || restore))
3079
+ throw new Error("--dry-run cannot be combined with --finish or --restore.");
3080
+ if (restore && !boolFlag(flags, "yes")) {
3081
+ throw new Error("recover --restore replaces live data. Review `npx @or3/cloud recover --dry-run` first, then re-run with `recover --restore --yes`.");
3082
+ }
3083
+ }
3084
+ async function reconcileOperatorHandoff(directory, state, report = console.log) {
3085
+ const env = parseEnv(await readText(deploymentPaths(directory).env));
3086
+ if (env.OR3_DASHBOARD_UPDATES_ENABLED !== "true") {
3087
+ const detail2 = "Dashboard updates are disabled in this deployment; no operator handoff needs reconciliation.";
3088
+ report(detail2);
3089
+ return detail2;
3090
+ }
3091
+ const receipt = state.lastReceipt;
3092
+ const jobId = receipt?.dashboardJobId ?? process.env.OR3_DASHBOARD_UPDATE_JOB_ID?.trim();
3093
+ if (!jobId) {
3094
+ throw new Error("The completed update recorded no dashboard job for its operator handoff. Re-run the host CLI update, or disable dashboard updates until the operator is restored.");
3095
+ }
3096
+ await scheduleDashboardOperatorHandoff(directory, env, jobId);
3097
+ if (receipt) {
3098
+ receipt.operatorHandoff = "pending";
3099
+ await writeState(directory, state);
3100
+ await exportTerminalReceipt(directory, receipt).catch(() => {
3101
+ return;
3102
+ });
3103
+ }
3104
+ const detail = `Rescheduled the dashboard operator handoff for job ${jobId.slice(0, 8)}. The successor operator marks it verified once it is running.`;
3105
+ report(detail);
3106
+ return detail;
3107
+ }
3108
+ async function collectTargetReadyChecks(directory, mode, env) {
3109
+ const checks = [];
3110
+ let containerId;
3111
+ const container = await run("docker", composeArgs(directory, mode, ["ps", "-q", "or3"]), directory);
3112
+ containerId = container.ok ? container.stdout.trim() || undefined : undefined;
3113
+ checks.push(containerId ? { code: "container-binding", status: "passed", detail: `Observed target container ${containerId}.` } : { code: "container-binding", status: "failed", detail: "Could not resolve the running target container." });
3114
+ const databaseCheck = await run("docker", [
3115
+ ...composeArgs(directory, mode, ["exec", "-T", "or3", ...containerNodeCommand(VERIFY_DATABASES_SCRIPT)])
3116
+ ], directory);
3117
+ let databasesOk = false;
3118
+ if (databaseCheck.ok) {
3119
+ try {
3120
+ const databases = JSON.parse(databaseCheck.stdout.trim());
3121
+ databasesOk = databases.length === 2 && databases.every((entry) => entry.quickCheck === "ok" && entry.tables >= 1);
3122
+ } catch {
3123
+ databasesOk = false;
3124
+ }
3125
+ }
3126
+ checks.push(databasesOk ? { code: "database-integrity", status: "passed", detail: "auth.sqlite and sync.sqlite quick_check passed with managed ownership." } : { code: "database-integrity", status: "failed", detail: `SQLite integrity or ownership verification failed. ${redact(databaseCheck.stderr.trim(), secretValues(env))}` });
3127
+ const baseUrl = new URL(mode === "public" ? `https://${env.OR3_PUBLIC_DOMAIN}` : `http://127.0.0.1:${env.OR3_PORT}`);
3128
+ try {
3129
+ validateVerificationHealth(await verificationJson(baseUrl, "/api/health?deep=true"));
3130
+ checks.push({ code: "public-health", status: "passed", detail: `${baseUrl.origin} deep health reports the managed profile.` });
3131
+ } catch (error) {
3132
+ checks.push({ code: "public-health", status: "failed", detail: redact(error instanceof Error ? error.message : String(error)) });
3133
+ }
3134
+ return { containerId, checks };
3135
+ }
3136
+ async function finishTargetReadyUpdate(directory, state, env, report = console.log) {
3137
+ const pending = state.incompleteOperation;
3138
+ if (!pending?.evidence || !pending.targetVersion || !pending.targetImage) {
3139
+ throw new Error("recover --finish requires a recorded target-ready milestone; none is present.");
3140
+ }
3141
+ const targetEnv = parseEnv(await readText(deploymentPaths(directory).env));
3142
+ if (targetEnv.OR3_VERSION !== pending.targetVersion || targetEnv.OR3_IMAGE !== pending.targetImage) {
3143
+ throw new Error("The live .env no longer matches the recorded target-ready identities; refusing to finish.");
3144
+ }
3145
+ const expectedTargetDigest = pending.targetImageDigest ?? await imageDigest(targetEnv.OR3_IMAGE);
3146
+ await pullAndRequireImage(targetEnv.OR3_IMAGE, expectedTargetDigest, "Target deployment");
3147
+ await assertRunningAppImage(directory, state.mode, targetEnv.OR3_IMAGE);
3148
+ const actualDigest = await imageDigest(targetEnv.OR3_IMAGE);
3149
+ if (actualDigest !== pending.evidence.imageDigest) {
3150
+ throw new Error(`The running target image ${actualDigest} does not match the recorded target-ready proof ${pending.evidence.imageDigest}.`);
3151
+ }
3152
+ const configurationSha256 = await sha256File(deploymentPaths(directory).env);
3153
+ if (configurationSha256 !== pending.evidence.configurationSha256) {
3154
+ throw new Error("The managed configuration changed after the target-ready milestone; refusing to finish. Inspect the change, or run `recover --restore --yes` to return to the recorded snapshot.");
3155
+ }
3156
+ const assets = await installedManagedAssetChecksums(directory, state.mode);
3157
+ for (const [name, expected] of Object.entries(pending.evidence.managedAssetSha256)) {
3158
+ if (assets[name] !== expected) {
3159
+ throw new Error(`Managed asset ${name} changed after the target-ready milestone; refusing to finish.`);
3160
+ }
3161
+ }
3162
+ const expectedDeploymentId = targetEnv.OR3_DEPLOYMENT_ID ?? pending.targetDeploymentId;
3163
+ if (pending.evidence.deploymentId && expectedDeploymentId && pending.evidence.deploymentId !== expectedDeploymentId) {
3164
+ throw new Error("The recorded target-ready deployment identity no longer matches the live deployment.");
3165
+ }
3166
+ await recordedBackupPath(directory, pending);
3167
+ for (const required of ["container-binding", "database-integrity", "public-health"]) {
3168
+ const recorded = pending.evidence.checks.find((check) => check.code === required);
3169
+ if (!recorded || recorded.status !== "passed") {
3170
+ throw new Error(`The recorded target-ready proof is missing ${required}; refusing to finish. Run "npx @or3/cloud recover --restore --yes" to return to the recorded snapshot.`);
3171
+ }
3172
+ }
3173
+ await waitForDeepHealth(directory, state.mode, secretValues(targetEnv));
3174
+ const revalidated = await collectTargetReadyChecks(directory, state.mode, targetEnv);
3175
+ if (!revalidated.containerId || revalidated.containerId !== pending.evidence.containerId) {
3176
+ throw new Error("The running target container no longer matches the recorded target-ready proof; refusing to finish.");
3177
+ }
3178
+ const failedChecks = revalidated.checks.filter((check) => check.status !== "passed");
3179
+ if (failedChecks.length > 0) {
3180
+ throw new Error(`Target verification failed at finish (${failedChecks.map((check) => check.code).join(", ")}); refusing to finish.`);
3181
+ }
3182
+ const sourceVersion = state.appVersion;
3183
+ const sourceImage = state.image;
3184
+ const sourceDigest = state.imageDigest;
3185
+ const digest = actualDigest;
3186
+ const recordedDashboardJobId = pending.dashboardJobId ?? (process.env.OR3_DASHBOARD_UPDATE_JOB_ID?.trim() || undefined);
3187
+ const receipt = {
3188
+ schemaVersion: 1,
3189
+ operationId: pending.id,
3190
+ dashboardJobId: recordedDashboardJobId,
3191
+ cliVersion: PACKAGE_VERSION,
3192
+ source: { appVersion: sourceVersion, image: sourceImage, imageDigest: sourceDigest, sourceRevision: packagedSourceRevision(sourceVersion), operatorImageDigest: expectedOperatorImageDigest(sourceVersion) },
3193
+ target: { appVersion: pending.targetVersion, image: pending.targetImage, imageDigest: digest, sourceRevision: packagedSourceRevision(pending.targetVersion), operatorImageDigest: expectedOperatorImageDigest(pending.targetVersion) },
3194
+ observed: { appVersion: pending.targetVersion, image: pending.targetImage, imageDigest: digest },
3195
+ completedAt: now(),
3196
+ rollbackBackupId: pending.backupId ?? "",
3197
+ checks: revalidated.checks,
3198
+ warnings: [],
3199
+ phaseDurationsMs: {},
3200
+ operatorHandoff: "not-required"
3201
+ };
3202
+ const next = stateFromEnv(directory, targetEnv, state.mode, "update", digest);
3203
+ next.rollback = {
3204
+ appVersion: sourceVersion,
3205
+ image: sourceImage,
3206
+ imageDigest: sourceDigest,
3207
+ backupId: pending.backupId ?? "",
3208
+ createdAt: now()
3209
+ };
3210
+ next.lastError = undefined;
3211
+ if (targetEnv.OR3_DASHBOARD_UPDATES_ENABLED === "true" && recordedDashboardJobId) {
3212
+ try {
3213
+ await scheduleDashboardOperatorHandoff(directory, targetEnv, recordedDashboardJobId);
3214
+ receipt.operatorHandoff = "pending";
3215
+ } catch (error) {
3216
+ receipt.operatorHandoff = "needs-attention";
3217
+ receipt.warnings.push({ code: "operator-handoff-schedule-failed", severity: "warning", message: redact(error instanceof Error ? error.message : String(error), secretValues(targetEnv)) });
3218
+ }
3219
+ }
3220
+ const commit = await commitTerminalState(directory, next, receipt);
3221
+ await removeOperationRecordSafely(directory, commit.operationId);
3222
+ const maintenance = [];
3223
+ const prune = await pruneBackups(directory, next, BACKUP_RETENTION_KEEP, false, {
3224
+ automatic: true,
3225
+ log: (message) => report(message),
3226
+ warn: (message) => report(message)
3227
+ }).catch((error) => {
3228
+ maintenance.push({ code: "retention-failed", severity: "warning", message: redact(error instanceof Error ? error.message : String(error), secretValues(targetEnv)) });
3229
+ return { removed: 0, deferred: [] };
3230
+ });
3231
+ maintenance.push(...prune.deferred);
3232
+ await persistReceiptWarnings(directory, next, maintenance);
3233
+ const detail = `Finished the recorded update as OR3 ${pending.targetVersion}. Post-replacement writes were preserved.`;
3234
+ report(detail);
3235
+ for (const warning of [...receipt.warnings, ...maintenance])
3236
+ report(` ⚠ ${warning.code}: ${warning.message}`);
3237
+ return detail;
3238
+ }
3239
+ async function recoverCommand(directory, flags = {}) {
3240
+ assertRecoverFlags(flags);
3241
+ const explicitRestore = boolFlag(flags, "restore");
3242
+ const finishRequested = boolFlag(flags, "finish");
3243
+ const dryRun = boolFlag(flags, "dry-run");
3244
+ if (dryRun) {
3245
+ const observation = await observeDeployment(directory, { checkDocker: true, checkImage: true });
3246
+ const state = observation.state;
3247
+ const decision = state ? decideRecoveryAction(state) : { action: "none", detail: "Managed state is unreadable; only diagnosis is available." };
3248
+ const pending = state?.incompleteOperation;
3249
+ const payload = {
3250
+ schemaVersion: 1,
3251
+ kind: "or3-recover-preview",
3252
+ observedAt: observation.observedAt,
3253
+ directory: observation.directory,
3254
+ partial: observation.partial,
3255
+ inProgress: observation.changing,
3256
+ source: state ? { appVersion: state.appVersion, image: state.image, imageDigest: state.imageDigest } : null,
3257
+ target: pending?.targetVersion ? { appVersion: pending.targetVersion, image: pending.targetImage ?? null, imageDigest: pending.targetImageDigest ?? null } : null,
3258
+ phase: pending?.phase ?? null,
3259
+ operation: pending?.operation ?? null,
3260
+ decision,
3261
+ dataLoss: decision.action === "require-explicit-restore",
3262
+ findings: observationFindings(observation)
3263
+ };
3264
+ if (boolFlag(flags, "json"))
3265
+ console.log(JSON.stringify(payload, null, 2));
3266
+ else {
3267
+ console.log(`OR3 recovery preview for ${observation.directory}`);
3268
+ console.log(` source: ${payload.source ? `OR3 ${payload.source.appVersion} (${payload.source.imageDigest})` : "unknown"}`);
3269
+ console.log(` target: ${payload.target ? `OR3 ${payload.target.appVersion}` : "none"}`);
3270
+ console.log(` operation: ${payload.operation ?? "none"}${payload.phase ? ` (${payload.phase})` : ""}`);
3271
+ console.log(` action: ${decision.action} — ${decision.detail}`);
3272
+ if (payload.dataLoss)
3273
+ console.log(" data loss: yes — restoring the recorded snapshot discards writes made after it.");
3274
+ for (const finding of payload.findings)
3275
+ console.log(` [${finding.severity}] ${finding.code}: ${finding.message}`);
3276
+ console.log(`
3277
+ Preview only: nothing was changed.`);
3278
+ }
3279
+ return;
3280
+ }
3281
+ const asJson = boolFlag(flags, "json");
3282
+ const report = (...args) => {
3283
+ if (asJson)
3284
+ console.error(...args);
3285
+ else
3286
+ console.log(...args);
3287
+ };
2601
3288
  await ensureDocker();
3289
+ try {
3290
+ const outcome = await runRecovery(directory, { explicitRestore, finishRequested, report });
3291
+ if (asJson)
3292
+ emitOperationResult(outcome);
3293
+ } catch (error) {
3294
+ if (!asJson)
3295
+ throw error;
3296
+ const message = redact(error instanceof Error ? error.message : String(error));
3297
+ console.error(`OR3 Cloud failed: ${message}`);
3298
+ emitOperationResult({ kind: "blocked", findings: [{ code: "recovery-failed", severity: "blocker", message, nextCommand: "npx @or3/cloud recover --dry-run" }] });
3299
+ process.exitCode = 1;
3300
+ }
3301
+ }
3302
+ async function runRecovery(directory, context) {
3303
+ const { explicitRestore, finishRequested, report } = context;
2602
3304
  const loaded = await loadManaged(directory);
2603
3305
  const pending = loaded.state.incompleteOperation;
3306
+ const decision = decideRecoveryAction(loaded.state);
3307
+ if (decision.action === "reconcile-handoff") {
3308
+ if (explicitRestore)
3309
+ throw new Error("This deployment completed its application update; only the dashboard operator handoff needs reconciliation. Run `npx @or3/cloud recover --finish` without --restore.");
3310
+ const detail = await reconcileOperatorHandoff(loaded.directory, loaded.state, report);
3311
+ return { kind: "recovered", operation: "update", detail };
3312
+ }
2604
3313
  if (!pending) {
2605
- console.log("No incomplete OR3 Cloud operation is recorded.");
2606
- return;
3314
+ if (finishRequested) {
3315
+ if (loaded.state.lastReceipt) {
3316
+ const detail2 = `No incomplete operation is recorded; the last ${loaded.state.lastReceipt.operationId} completion remains authoritative. Nothing to finish.`;
3317
+ report(detail2);
3318
+ return { kind: "no-op", detail: detail2 };
3319
+ }
3320
+ throw new Error("recover --finish requires an incomplete operation, but none is recorded.");
3321
+ }
3322
+ const detail = "No incomplete OR3 Cloud operation is recorded.";
3323
+ report(detail);
3324
+ return { kind: "no-op", detail };
3325
+ }
3326
+ if (finishRequested) {
3327
+ if (decision.action !== "finish") {
3328
+ throw new Error(`recover --finish is not admissible: ${decision.detail} Run "npx @or3/cloud recover --dry-run" to inspect the supported actions.`);
3329
+ }
3330
+ const detail = await finishTargetReadyUpdate(loaded.directory, loaded.state, loaded.env, report);
3331
+ return { kind: "recovered", operation: "update", detail };
2607
3332
  }
3333
+ const requiresExplicitRestore = decision.action === "require-explicit-restore";
2608
3334
  try {
2609
3335
  if (pending.phase === "prepared") {
2610
3336
  if (pending.operation === "backup" || pending.operation === "update" || pending.operation === "adopt") {
@@ -2621,24 +3347,30 @@ async function recoverCommand(directory) {
2621
3347
  await applyCredentialReset(loaded.directory, loaded.state, nextEnv);
2622
3348
  loaded.state.lastError = undefined;
2623
3349
  await clearPending(loaded.directory, loaded.state);
2624
- console.log("Recovered the incomplete credential reset. Owner and admin credentials were verified inside the OR3 container.");
2625
- return;
3350
+ const detail2 = "Recovered the incomplete credential reset. Owner and admin credentials were verified inside the OR3 container.";
3351
+ report(detail2);
3352
+ return { kind: "recovered", operation: "credentials-reset", detail: detail2 };
2626
3353
  }
2627
3354
  if (pending.operation === "restore" || pending.operation === "rollback") {
2628
3355
  if (pending.phase === "prepared" || pending.phase === "snapshot-created") {
2629
3356
  await startProject(loaded.directory, loaded.state.mode, loaded.env);
2630
3357
  loaded.state.lastError = undefined;
2631
3358
  await clearPending(loaded.directory, loaded.state);
2632
- console.log(`Recovered the incomplete ${pending.operation} before data replacement. OR3 ${loaded.state.appVersion} is deeply healthy.`);
2633
- return;
3359
+ const detail3 = `Recovered the incomplete ${pending.operation} before data replacement. OR3 ${loaded.state.appVersion} is deeply healthy.`;
3360
+ report(detail3);
3361
+ return { kind: "recovered", operation: pending.operation, detail: detail3 };
3362
+ }
3363
+ if (requiresExplicitRestore && !explicitRestore) {
3364
+ throw new Error(`The incomplete ${pending.operation} may have replaced data and has no completion proof. Nothing was changed. Review "npx @or3/cloud recover --dry-run", then restore deliberately with "npx @or3/cloud recover --restore --yes" (this discards writes made after the recorded snapshot).`);
2634
3365
  }
2635
3366
  if (!pending.previousBackupPath && !pending.previousBackupId) {
2636
3367
  throw new Error(`The incomplete ${pending.operation} has no verified pre-mutation snapshot. Refusing to replay a target that may have been partially restored; inspect the deployment and restore an authenticated backup explicitly.`);
2637
3368
  }
2638
3369
  const previous = await restorePreMutationSnapshot(loaded.directory, loaded.state, loaded.env);
2639
3370
  const recovered2 = await commitPreMutationRecovery(loaded.directory, loaded.state, `Interrupted ${pending.operation} was rolled back to pre-mutation snapshot ${previous.manifest.backupId}.`);
2640
- console.log(`Recovered the incomplete ${pending.operation} by restoring pre-mutation snapshot ${previous.manifest.backupId}. OR3 ${recovered2.appVersion} is deeply healthy.`);
2641
- return;
3371
+ const detail2 = `Recovered the incomplete ${pending.operation} by restoring pre-mutation snapshot ${previous.manifest.backupId}. OR3 ${recovered2.appVersion} is deeply healthy.`;
3372
+ report(detail2);
3373
+ return { kind: "recovered", operation: pending.operation, detail: detail2 };
2642
3374
  }
2643
3375
  if (pending.operation === "update") {
2644
3376
  if (pending.phase === "prepared" || pending.phase === "snapshot-created") {
@@ -2648,16 +3380,25 @@ async function recoverCommand(directory) {
2648
3380
  recovered3.rollback = loaded.state.rollback;
2649
3381
  recovered3.lastError = undefined;
2650
3382
  await commitRecoveredState(loaded.directory, loaded.state, recovered3);
2651
- console.log(`Recovered the incomplete update before the replacement was applied. OR3 ${recovered3.appVersion} is deeply healthy.`);
2652
- return;
3383
+ const detail3 = `Recovered the incomplete update before the replacement was applied. OR3 ${recovered3.appVersion} is deeply healthy.`;
3384
+ report(detail3);
3385
+ return { kind: "recovered", operation: "update", detail: detail3 };
3386
+ }
3387
+ if (decision.action === "finish" && !explicitRestore) {
3388
+ const detail3 = await finishTargetReadyUpdate(loaded.directory, loaded.state, loaded.env, report);
3389
+ return { kind: "recovered", operation: "update", detail: detail3 };
3390
+ }
3391
+ if (requiresExplicitRestore && !explicitRestore) {
3392
+ throw new Error(`The incomplete update may have replaced data and has no completion proof. Nothing was changed. Review "npx @or3/cloud recover --dry-run", then restore deliberately with "npx @or3/cloud recover --restore --yes" (this discards writes made after the recorded pre-update snapshot).`);
2653
3393
  }
2654
3394
  if (!pending.backupPath && !pending.backupId) {
2655
3395
  throw new Error("The incomplete update has no verified pre-update snapshot. Refusing to guess which image or data should be live.");
2656
3396
  }
2657
3397
  const previous = await restorePreMutationSnapshot(loaded.directory, loaded.state, loaded.env);
2658
3398
  const recovered2 = await commitPreMutationRecovery(loaded.directory, loaded.state, `Interrupted update was rolled back to pre-update snapshot ${previous.manifest.backupId}.`);
2659
- console.log(`Recovered the incomplete update by restoring pre-update snapshot ${previous.manifest.backupId}. OR3 ${recovered2.appVersion} is deeply healthy.`);
2660
- return;
3399
+ const detail2 = `Recovered the incomplete update by restoring pre-update snapshot ${previous.manifest.backupId}. OR3 ${recovered2.appVersion} is deeply healthy.`;
3400
+ report(detail2);
3401
+ return { kind: "recovered", operation: "update", detail: detail2 };
2661
3402
  }
2662
3403
  if (pending.operation === "adopt") {
2663
3404
  if (!pending.backupId)
@@ -2673,8 +3414,9 @@ async function recoverCommand(directory) {
2673
3414
  if (pending.operation === "backup" && pending.initialAppRunning === false) {
2674
3415
  loaded.state.lastError = undefined;
2675
3416
  await clearPending(loaded.directory, loaded.state);
2676
- console.log("Recovered the incomplete backup operation and preserved the intentionally stopped OR3 service.");
2677
- return;
3417
+ const detail2 = "Recovered the incomplete backup operation and preserved the intentionally stopped OR3 service.";
3418
+ report(detail2);
3419
+ return { kind: "recovered", operation: "backup", detail: detail2 };
2678
3420
  }
2679
3421
  await pullAndRequireImage(loaded.state.image, loaded.state.imageDigest, "Current deployment");
2680
3422
  await startProject(loaded.directory, loaded.state.mode, loaded.env);
@@ -2691,7 +3433,9 @@ async function recoverCommand(directory) {
2691
3433
  recovered.rollback = loaded.state.rollback;
2692
3434
  recovered.lastError = undefined;
2693
3435
  await commitRecoveredState(loaded.directory, loaded.state, recovered);
2694
- console.log(`Recovered the incomplete ${pending.operation} operation. OR3 ${recovered.appVersion} is deeply healthy.`);
3436
+ const detail = `Recovered the incomplete ${pending.operation} operation. OR3 ${recovered.appVersion} is deeply healthy.`;
3437
+ report(detail);
3438
+ return { kind: "recovered", operation: pending.operation, detail };
2695
3439
  } catch (error) {
2696
3440
  const recoverySecrets = pending.operation === "credentials-reset" ? secretValues(pending.credentialReset?.nextEnv ?? loaded.env) : secretValues(loaded.env);
2697
3441
  loaded.state.lastError = redact(error instanceof Error ? error.message : String(error), recoverySecrets);
@@ -2721,6 +3465,12 @@ function assertNoPending(state) {
2721
3465
  throw new Error(`An incomplete ${state.incompleteOperation.operation} is recorded. Run "npx @or3/cloud recover" before starting another operation.`);
2722
3466
  }
2723
3467
  }
3468
+ function assertPhaseAllowsCommand(command, state) {
3469
+ const pending = state.incompleteOperation;
3470
+ if (!pending || command === "stop")
3471
+ return;
3472
+ throw new Error(`An incomplete ${pending.operation}${pending.phase ? ` (${pending.phase})` : ""} is recorded. Refusing to ${command} an ambiguous deployment. Run "npx @or3/cloud recover --dry-run" to inspect the supported actions, then "npx @or3/cloud recover --finish" or "npx @or3/cloud recover --restore --yes".`);
3473
+ }
2724
3474
  async function backupCreateCommand(directory) {
2725
3475
  await ensureDocker();
2726
3476
  const loaded = await loadManaged(directory);
@@ -2741,26 +3491,66 @@ async function backupCreateCommand(directory) {
2741
3491
  try {
2742
3492
  const result = await createBackup(loaded.directory, loaded.state, loaded.env, { backupId, initiallyRunning: initialAppRunning });
2743
3493
  await clearPending(loaded.directory, loaded.state);
2744
- await pruneBackups(loaded.directory, loaded.state, BACKUP_RETENTION_KEEP, false);
3494
+ const prune = await pruneBackups(loaded.directory, loaded.state, BACKUP_RETENTION_KEEP, false, { automatic: true });
2745
3495
  console.log(`Backup ${result.backupId} created at ${result.backupDir}`);
2746
3496
  console.log(`SHA-256: ${result.manifest.dataSha256}`);
3497
+ if (prune.deferred.length > 0) {
3498
+ console.warn(`Maintenance warning: ${prune.deferred.length} backup entr${prune.deferred.length === 1 ? "y" : "ies"} need inspection; all backups were preserved. Run "npx @or3/cloud backup list" for details.`);
3499
+ }
2747
3500
  } catch (error) {
2748
3501
  loaded.state.lastError = redact(error instanceof Error ? error.message : String(error), secretValues(loaded.env));
2749
3502
  await writeState(loaded.directory, loaded.state);
2750
3503
  throw error;
2751
3504
  }
2752
3505
  }
2753
- async function backupListCommand(directory) {
2754
- const loaded = await loadManaged(directory);
2755
- const backups = await enumerateBackups(loaded.directory);
2756
- if (backups.length === 0) {
2757
- console.log(`No authenticated backups are available for ${loaded.directory}. Run "npx @or3/cloud backup" to create one.`);
3506
+ async function backupListCommand(directory, flags = {}) {
3507
+ const loaded = await loadManaged(directory, { writable: false });
3508
+ const inventory = await inventoryBackups(loaded.directory);
3509
+ const verified = inventory.entries.filter((entry) => entry.kind === "verified").map((entry) => entry.backup);
3510
+ const findings = inventory.entries.filter((entry) => entry.kind !== "verified");
3511
+ if (boolFlag(flags, "json")) {
3512
+ console.log(JSON.stringify({
3513
+ schemaVersion: 1,
3514
+ directory: loaded.directory,
3515
+ backups: verified.map((backup) => ({
3516
+ backupId: backup.backupId,
3517
+ createdAt: backup.createdAt,
3518
+ appVersion: backup.appVersion,
3519
+ bytes: backup.bytes,
3520
+ dataSha256: backup.dataSha256,
3521
+ trust: "verified"
3522
+ })),
3523
+ findings: findings.map((entry) => ({ entryName: entry.entryName, kind: entry.kind, code: entry.code, message: entry.message })),
3524
+ storeErrors: inventory.storeErrors
3525
+ }, null, 2));
2758
3526
  return;
2759
3527
  }
2760
- console.log(`OR3 Cloud backups for ${loaded.directory} (${backups.length}):`);
2761
- console.log("backupId createdAt version bytes checksum");
2762
- for (const backup of backups) {
2763
- console.log(`${backup.backupId.padEnd(30)} ${backup.createdAt.padEnd(30)} ${backup.appVersion.padEnd(8)} ${String(backup.bytes).padStart(9)} ${backup.dataSha256.slice(0, 12)}`);
3528
+ if (verified.length === 0 && findings.length === 0) {
3529
+ console.log(`No backups are available for ${loaded.directory}. Run "npx @or3/cloud backup" to create one.`);
3530
+ return;
3531
+ }
3532
+ if (verified.length === 0) {
3533
+ console.log(`No authenticated backups are available for ${loaded.directory}.`);
3534
+ } else {
3535
+ console.log(`OR3 Cloud backups for ${loaded.directory} (${verified.length} verified):`);
3536
+ console.log("backupId createdAt version bytes trust checksum");
3537
+ for (const backup of verified) {
3538
+ console.log(`${backup.backupId.padEnd(30)} ${backup.createdAt.padEnd(30)} ${backup.appVersion.padEnd(8)} ${String(backup.bytes).padStart(9)} ${"verified".padEnd(9)} ${backup.dataSha256.slice(0, 12)}`);
3539
+ }
3540
+ }
3541
+ if (findings.length > 0) {
3542
+ console.log(`
3543
+ Preserved history (${findings.length}; never trusted for restore):`);
3544
+ for (const entry of findings) {
3545
+ console.log(` ${entry.entryName.padEnd(30)} ${entry.kind.padEnd(16)} ${entry.code}`);
3546
+ console.log(` ${entry.message}`);
3547
+ }
3548
+ console.log(`
3549
+ To obtain a trusted restore point, create a new backup of the current healthy deployment: npx @or3/cloud backup`);
3550
+ }
3551
+ for (const diagnostic of inventory.storeErrors) {
3552
+ console.log(`
3553
+ Store error (${diagnostic.severity}): ${diagnostic.message}`);
2764
3554
  }
2765
3555
  console.log(`
2766
3556
  Backups contain credentials and secrets; keep them owner-only and export off-host.`);
@@ -2771,9 +3561,18 @@ async function backupPruneCommand(directory, flags) {
2771
3561
  const keep = parseKeep(flags);
2772
3562
  const force = boolFlag(flags, "force");
2773
3563
  if (force && !boolFlag(flags, "yes"))
2774
- throw new Error("--force may delete the only rollback or recovery backup. Re-run with --force --yes after confirming the exact backups shown by `backup list`.");
2775
- const deleted = await pruneBackups(loaded.directory, loaded.state, keep, force);
2776
- console.log(deleted > 0 ? `Pruned ${deleted} backup(s); keeping the newest ${keep}.` : `Nothing to prune: keeping all backups (newest ${keep}).`);
3564
+ throw new Error("--force may delete backups beyond the retention count. Re-run with --force --yes after confirming the exact backups shown by `backup list`.");
3565
+ const result = await pruneBackups(loaded.directory, loaded.state, keep, force, { automatic: !force });
3566
+ if (!force && result.deferred.length > 0 && result.removed === 0) {
3567
+ throw new Error(`Backup pruning is blocked because the store contains entries that need inspection:
3568
+ ${result.deferred.map((diagnostic) => ` ${diagnostic.code}: ${diagnostic.message}`).join(`
3569
+ `)}
3570
+ No backups were deleted. Run "npx @or3/cloud backup list" to inspect them, or re-run with --force --yes to prune only verified backups while preserving protected recovery sources.`);
3571
+ }
3572
+ console.log(result.removed > 0 ? `Pruned ${result.removed} backup(s); keeping the newest ${keep}.` : `Nothing to prune: keeping all backups (newest ${keep}).`);
3573
+ if (result.deferred.length > 0) {
3574
+ console.warn(`Preserved ${result.deferred.length} suspect or legacy entr${result.deferred.length === 1 ? "y" : "ies"} without deletion.`);
3575
+ }
2777
3576
  }
2778
3577
  async function backupExportCommand(directory, backupId, destination) {
2779
3578
  const backupPath = await resolveBackup(directory, backupId);
@@ -2854,8 +3653,10 @@ async function backupCommand(directory, positionals, flags) {
2854
3653
  if (subcommand === "list") {
2855
3654
  if (positionals.length > 1)
2856
3655
  throw new Error("backup list accepts no arguments.");
2857
- return await backupListCommand(directory);
3656
+ return await backupListCommand(directory, flags);
2858
3657
  }
3658
+ if (boolFlag(flags, "json"))
3659
+ throw new Error("--json is supported only by `backup list`.");
2859
3660
  if (subcommand === "prune") {
2860
3661
  if (positionals.length > 1)
2861
3662
  throw new Error("backup prune accepts no arguments.");
@@ -2872,7 +3673,184 @@ async function backupCommand(directory, positionals, flags) {
2872
3673
  }
2873
3674
  throw new Error(`Unknown backup subcommand "${subcommand}". Use list, prune, export, or no subcommand to create a backup.`);
2874
3675
  }
3676
+ function stateFingerprint(state) {
3677
+ return createHash("sha256").update(JSON.stringify({
3678
+ schema: state.schemaVersion,
3679
+ appVersion: state.appVersion,
3680
+ image: state.image,
3681
+ imageDigest: state.imageDigest,
3682
+ pending: state.incompleteOperation?.id ?? null
3683
+ })).digest("hex").slice(0, 16);
3684
+ }
3685
+ async function assessUpdate(directory, targetVersion, options = {}) {
3686
+ const checkDocker = options.checkDocker ?? false;
3687
+ const observation = await observeDeployment(directory, { checkDocker, checkImage: false });
3688
+ const state = observation.state;
3689
+ const checks = [];
3690
+ const findings = [...observationFindings(observation)];
3691
+ const targetImage = imageFor(targetVersion);
3692
+ if (!state) {
3693
+ findings.push({ code: "state-unreadable", severity: "blocker", message: "Managed state must be readable to preview an update." });
3694
+ } else if (state.incompleteOperation) {
3695
+ findings.push({
3696
+ code: "operation-incomplete",
3697
+ severity: "blocker",
3698
+ message: `An incomplete ${state.incompleteOperation.operation} is recorded. Run "npx @or3/cloud recover" before updating.`,
3699
+ nextCommand: "npx @or3/cloud recover"
3700
+ });
3701
+ }
3702
+ if (observation.changing && !options.underLease) {
3703
+ findings.push({ code: "operation-in-progress", severity: "blocker", message: "A mutation lease is active; preview is non-executable until it settles." });
3704
+ }
3705
+ let noOp = false;
3706
+ if (state && targetVersion === state.appVersion && !state.incompleteOperation) {
3707
+ noOp = true;
3708
+ checks.push({ code: "already-installed", status: "passed", detail: `OR3 ${targetVersion} is already the installed release; execution is a no-op.` });
3709
+ } else {
3710
+ checks.push({ code: "target-version", status: "passed", detail: `Target OR3 ${targetVersion} (${targetImage}).` });
3711
+ }
3712
+ try {
3713
+ const stats = await statfs(deploymentPaths(directory).backups);
3714
+ const freeBytes = Number(stats.bavail) * Number(stats.bsize);
3715
+ checks.push({ code: "backup-filesystem-space", status: "passed", detail: `${freeBytes} bytes available on the backup filesystem.` });
3716
+ } catch (error) {
3717
+ checks.push({ code: "backup-filesystem-space", status: "unknown", detail: redact(error instanceof Error ? error.message : String(error)) });
3718
+ }
3719
+ checks.push({ code: "data-volume-space", status: "deferred", detail: "The live data-volume requirement is validated after the snapshot, before downtime." });
3720
+ if (checkDocker) {
3721
+ if (observation.docker) {
3722
+ try {
3723
+ const architecture = await dockerDaemonArchitecture();
3724
+ checks.push({ code: "docker-availability", status: "passed", detail: `Docker daemon is reachable (${architecture}).` });
3725
+ } catch (error) {
3726
+ checks.push({ code: "docker-availability", status: "failed", detail: redact(error instanceof Error ? error.message : String(error)) });
3727
+ }
3728
+ } else {
3729
+ checks.push({ code: "docker-availability", status: "unknown", detail: "The Docker daemon is not reachable from this host; it is revalidated at execution time." });
3730
+ }
3731
+ } else {
3732
+ checks.push({ code: "docker-availability", status: "deferred", detail: "Docker availability and daemon architecture are revalidated at execution time." });
3733
+ }
3734
+ checks.push({ code: "image-pull", status: "deferred", detail: "The target image pull and architecture/provenance checks run at execution time." });
3735
+ checks.push({ code: "fresh-snapshot", status: "deferred", detail: "A fresh authenticated snapshot is created at execution time." });
3736
+ const plan = planRetention(observation.backups?.entries ?? [], BACKUP_RETENTION_KEEP, state ? retentionProtectedIds(state) : new Set, { automatic: true });
3737
+ checks.push({
3738
+ code: "retention",
3739
+ status: plan.canPrune ? "passed" : "deferred",
3740
+ detail: plan.canPrune ? `Would keep ${plan.keep.length} verified backup(s) and prune ${plan.remove.length}.` : `${plan.warnings.length} entry(ies) need inspection; automatic pruning is deferred and all backups are preserved.`
3741
+ });
3742
+ for (const warning of plan.warnings)
3743
+ findings.push(warning);
3744
+ if (options.inspectAssets !== false) {
3745
+ const changed = [];
3746
+ for (const name of managedAssetNames(state?.mode ?? "local")) {
3747
+ let desired;
3748
+ try {
3749
+ desired = await readFile(join(ASSET_ROOT, name));
3750
+ } catch (error) {
3751
+ findings.push({
3752
+ code: "managed-asset-unreadable",
3753
+ severity: "blocker",
3754
+ resource: name,
3755
+ message: `Required generated asset ${name} could not be read from this CLI: ${redact(error instanceof Error ? error.message : String(error))}`
3756
+ });
3757
+ continue;
3758
+ }
3759
+ try {
3760
+ const installedPath = join(directory, name);
3761
+ if (!await fileExists(installedPath)) {
3762
+ changed.push(`${name} (add)`);
3763
+ continue;
3764
+ }
3765
+ const installed = await readFile(installedPath);
3766
+ if (!installed.equals(desired))
3767
+ changed.push(`${name} (update)`);
3768
+ } catch (error) {
3769
+ findings.push({
3770
+ code: "managed-asset-unreadable",
3771
+ severity: "blocker",
3772
+ resource: name,
3773
+ message: `Installed generated asset ${name} could not be read from ${directory}: ${redact(error instanceof Error ? error.message : String(error))}`
3774
+ });
3775
+ }
3776
+ }
3777
+ checks.push(changed.length ? { code: "managed-assets", status: "passed", detail: `Generated assets change: ${changed.join(", ")}.` } : { code: "managed-assets", status: "passed", detail: "Generated assets are unchanged." });
3778
+ }
3779
+ try {
3780
+ assertKnownStateSchema(state?.schemaVersion);
3781
+ checks.push({ code: "state-schema", status: "passed", detail: `Managed state schema ${String(state?.schemaVersion)} is supported.` });
3782
+ } catch (error) {
3783
+ findings.push({ code: "state-schema-unsupported", severity: "blocker", message: error instanceof Error ? error.message : String(error) });
3784
+ }
3785
+ const blockers = findings.filter((finding) => finding.severity === "blocker");
3786
+ blockers.sort((a, b) => a.code < b.code ? -1 : a.code > b.code ? 1 : 0);
3787
+ return {
3788
+ schemaVersion: 1,
3789
+ observedAt: observation.observedAt,
3790
+ source: state ? { appVersion: state.appVersion, image: state.image, imageDigest: state.imageDigest } : null,
3791
+ target: { appVersion: targetVersion, image: targetImage, imageDigest: expectedImageDigest(targetVersion) ?? "unknown" },
3792
+ checks,
3793
+ findings: [...blockers, ...findings.filter((finding) => finding.severity !== "blocker")],
3794
+ retention: plan,
3795
+ stateFingerprint: state ? stateFingerprint(state) : "unavailable"
3796
+ };
3797
+ }
3798
+ async function updatePreviewCommand(directory, flags) {
3799
+ const targetVersion = stringFlag(flags, "to")?.trim() ?? PACKAGE_VERSION;
3800
+ if (!isVersion(targetVersion))
3801
+ throw new Error(`--to must be a complete semantic version such as ${PACKAGE_VERSION}.`);
3802
+ if (targetVersion !== PACKAGE_VERSION) {
3803
+ throw new Error(`This CLI contains deployment assets for OR3 ${PACKAGE_VERSION}. Preview the matching CLI with \`npx --yes @or3/cloud@${targetVersion} update --to ${targetVersion} --dry-run\`.`);
3804
+ }
3805
+ const assessment = await assessUpdate(directory, targetVersion, { checkDocker: true });
3806
+ if (boolFlag(flags, "json")) {
3807
+ console.log(JSON.stringify({ ...assessment, kind: "or3-update-preview" }, null, 2));
3808
+ } else {
3809
+ console.log(`OR3 update preview for ${directory}`);
3810
+ console.log(` source: ${assessment.source ? `OR3 ${assessment.source.appVersion} (${assessment.source.imageDigest})` : "unknown"}`);
3811
+ console.log(` target: OR3 ${assessment.target.appVersion} (${assessment.target.imageDigest})`);
3812
+ console.log(` state fingerprint: ${assessment.stateFingerprint}`);
3813
+ console.log();
3814
+ for (const check of assessment.checks)
3815
+ console.log(` [${check.status}] ${check.code}: ${check.detail}`);
3816
+ console.log();
3817
+ for (const finding of assessment.findings)
3818
+ console.log(` [${finding.severity}] ${finding.code}: ${finding.message}`);
3819
+ if (assessment.retention.preserve.length) {
3820
+ console.log(`
3821
+ Preserved backups:`);
3822
+ for (const entry of assessment.retention.preserve)
3823
+ console.log(` ${entry.entryName}: ${entry.reason}`);
3824
+ }
3825
+ console.log(`
3826
+ Dry run only: no state, assets, lease, images, services, snapshots, or backups were changed.`);
3827
+ }
3828
+ if (assessment.findings.some((finding) => finding.severity === "blocker"))
3829
+ process.exitCode = 1;
3830
+ }
2875
3831
  async function updateCommand(directory, flags) {
3832
+ const asJson = boolFlag(flags, "json");
3833
+ try {
3834
+ return await runUpdateCommand(directory, flags, asJson);
3835
+ } catch (error) {
3836
+ if (!asJson)
3837
+ throw error;
3838
+ const message = redact(error instanceof Error ? error.message : String(error));
3839
+ console.error(`OR3 Cloud failed: ${message}`);
3840
+ emitOperationResult({
3841
+ kind: "blocked",
3842
+ findings: [{ code: "update-failed", severity: "blocker", message, nextCommand: "npx @or3/cloud recover --dry-run" }]
3843
+ });
3844
+ process.exitCode = 1;
3845
+ }
3846
+ }
3847
+ async function runUpdateCommand(directory, flags, asJson) {
3848
+ const progress = (...args) => {
3849
+ if (asJson)
3850
+ console.error(...args);
3851
+ else
3852
+ console.log(...args);
3853
+ };
2876
3854
  await ensureDocker();
2877
3855
  const loaded = await loadManaged(directory);
2878
3856
  const { state, env } = loaded;
@@ -2884,8 +3862,18 @@ async function updateCommand(directory, flags) {
2884
3862
  if (targetVersion !== PACKAGE_VERSION) {
2885
3863
  throw new Error(`This CLI contains deployment assets for OR3 ${PACKAGE_VERSION}. Run \`npx --yes @or3/cloud@${targetVersion} update --to ${targetVersion}\` so the image and generated assets match.`);
2886
3864
  }
2887
- if (targetVersion === state.appVersion)
2888
- throw new Error(`The deployment is already on OR3 ${targetVersion}.`);
3865
+ if (targetVersion === state.appVersion) {
3866
+ if (asJson)
3867
+ emitOperationResult({ kind: "no-op", detail: `OR3 ${targetVersion} is already installed.`, currentVersion: state.appVersion, targetVersion });
3868
+ else
3869
+ progress(`OR3 ${targetVersion} is already installed. Nothing to do.`);
3870
+ return;
3871
+ }
3872
+ const preflight = await assessUpdate(loaded.directory, targetVersion, { underLease: true, checkDocker: true });
3873
+ const preflightBlockers = preflight.findings.filter((finding) => finding.severity === "blocker");
3874
+ if (preflightBlockers.length > 0) {
3875
+ throw new Error(`Update to ${targetVersion} is blocked by preflight: ${preflightBlockers.map((finding) => `${finding.code} (${finding.message})`).join(" ")}`);
3876
+ }
2889
3877
  const targetImageTag = imageFor(targetVersion);
2890
3878
  const targetDigest = await pullImage(targetImageTag, expectedImageDigest(targetVersion));
2891
3879
  await assertSupportedHostArchitecture(targetImageTag);
@@ -2908,16 +3896,32 @@ async function updateCommand(directory, flags) {
2908
3896
  phase: "prepared"
2909
3897
  };
2910
3898
  await markPending(loaded.directory, state, pending);
3899
+ const startedAtMs = Date.now();
2911
3900
  try {
3901
+ const operator = await prepareVerifiedDashboardOperator(loaded.directory, targetVersion);
2912
3902
  const backup = await createBackup(loaded.directory, state, env, { restartAfter: false, backupId });
3903
+ const writeSchema = writeStateSchema();
3904
+ if (writeSchema >= 2) {
3905
+ const minimumSource = packagedMinimumSourceVersion();
3906
+ if (minimumSource && compareReleaseVersions(state.appVersion, minimumSource) < 0) {
3907
+ throw new Error(`OR3 ${targetVersion} writes managed state schema 2, which requires the compatibility bridge (source ${minimumSource} or newer). Run the host CLI bridge update first, then retry this schema-2 release.`);
3908
+ }
3909
+ state.schemaVersion = writeSchema;
3910
+ }
2913
3911
  await updatePending(loaded.directory, state, {
2914
3912
  backupId: backup.backupId,
2915
3913
  backupPath: backup.backupDir,
2916
3914
  backupDataSha256: backup.manifest.dataSha256,
2917
3915
  backupConfigSha256: backup.manifest.configSha256,
2918
- phase: "snapshot-created"
3916
+ phase: "snapshot-created",
3917
+ verifiedSnapshot: {
3918
+ backupId: backup.backupId,
3919
+ path: backup.backupDir,
3920
+ dataSha256: backup.manifest.dataSha256,
3921
+ configSha256: backup.manifest.configSha256 ?? "",
3922
+ createdAt: backup.manifest.createdAt
3923
+ }
2919
3924
  });
2920
- const operator = await prepareVerifiedDashboardOperator(loaded.directory, targetVersion);
2921
3925
  if (env.OR3_DASHBOARD_UPDATES_ENABLED === "true" && !operator && !process.env.OR3_DASHBOARD_UPDATE_JOB_ID) {
2922
3926
  await removeDashboardOperator(loaded.directory, state.mode);
2923
3927
  }
@@ -2953,7 +3957,6 @@ async function updateCommand(directory, flags) {
2953
3957
  if (migrateLegacyVolume || recreateDataVolume) {
2954
3958
  await restoreVolumeArchive(loaded.directory, state.mode, nextEnv, backup.backupDir);
2955
3959
  }
2956
- await startProject(loaded.directory, state.mode, nextEnv);
2957
3960
  } catch (error) {
2958
3961
  await updatePending(loaded.directory, state, { phase: "restoring-previous" });
2959
3962
  try {
@@ -2971,11 +3974,88 @@ async function updateCommand(directory, flags) {
2971
3974
  const recovered = await commitPreMutationRecovery(loaded.directory, state, `Update to ${targetVersion} failed and was restored: ${redact(error instanceof Error ? error.message : String(error), secretValues(oldEnv))}`);
2972
3975
  throw new Error(recovered.lastError);
2973
3976
  }
3977
+ await updatePending(loaded.directory, state, { phase: "starting-target" });
3978
+ try {
3979
+ await startProject(loaded.directory, state.mode, nextEnv);
3980
+ } catch (error) {
3981
+ throw new Error(`Update to ${targetVersion} started the target but it did not pass verification: ${redact(error instanceof Error ? error.message : String(error), secretValues(nextEnv))}. Writes may have been accepted, so the deployment was NOT restored automatically. Run "npx @or3/cloud recover --dry-run", then "npx @or3/cloud recover --finish" if the target is proven, or "npx @or3/cloud recover --restore --yes" to return to the pre-update snapshot.`);
3982
+ }
2974
3983
  const digest = await requireImageDigest(targetImage, targetDigest, `OR3 ${targetVersion}`);
3984
+ const targetChecks = await collectTargetReadyChecks(loaded.directory, state.mode, nextEnv);
3985
+ const failedTargetChecks = targetChecks.checks.filter((check) => check.status !== "passed");
3986
+ if (!targetChecks.containerId || failedTargetChecks.length > 0) {
3987
+ throw new Error(`Update to ${targetVersion} did not pass the required target checks (${failedTargetChecks.map((check) => check.code).join(", ") || "container-binding"}). Writes may have been accepted, so the deployment was NOT restored automatically. Run "npx @or3/cloud recover --dry-run", then "npx @or3/cloud recover --restore --yes" to return to the pre-update snapshot.`);
3988
+ }
3989
+ const targetReadyEvidence = {
3990
+ checkedAt: now(),
3991
+ deploymentId: targetDeploymentId,
3992
+ deploymentRoot: resolve(loaded.directory),
3993
+ containerId: targetChecks.containerId,
3994
+ imageDigest: digest,
3995
+ configurationSha256: await sha256File(deploymentPaths(loaded.directory).env),
3996
+ managedAssetSha256: await installedManagedAssetChecksums(loaded.directory, state.mode),
3997
+ dataReplacementCompleted: true,
3998
+ checks: [
3999
+ { code: "image-digest", status: "passed", detail: `${targetImage}@${digest}` },
4000
+ { code: "deep-health", status: "passed", detail: "OR3 deep health and running-image identity verified" },
4001
+ ...targetChecks.checks
4002
+ ]
4003
+ };
4004
+ if (writeSchema >= 2) {
4005
+ await updatePending(loaded.directory, state, { phase: "target-ready", evidence: targetReadyEvidence });
4006
+ }
4007
+ const warnings = [];
4008
+ const hadDashboardJob = Boolean(process.env.OR3_DASHBOARD_UPDATE_JOB_ID);
4009
+ let operatorHandoff = "not-required";
4010
+ if (nextEnv.OR3_DASHBOARD_UPDATES_ENABLED === "true" && hadDashboardJob) {
4011
+ try {
4012
+ await scheduleDashboardOperatorHandoff(loaded.directory, nextEnv);
4013
+ operatorHandoff = "pending";
4014
+ } catch (error) {
4015
+ operatorHandoff = "needs-attention";
4016
+ warnings.push({
4017
+ code: "operator-handoff-schedule-failed",
4018
+ severity: "warning",
4019
+ resource: pending.id,
4020
+ message: `The application is upgraded, but the dashboard operator handoff could not be scheduled: ${redact(error instanceof Error ? error.message : String(error), secretValues(nextEnv))}. Run "npx @or3/cloud recover --finish" on the host.`,
4021
+ nextCommand: "npx @or3/cloud recover --finish"
4022
+ });
4023
+ }
4024
+ }
4025
+ const sourceVersion = state.appVersion;
4026
+ const sourceImage = state.image;
4027
+ const sourceDigest = state.imageDigest;
4028
+ const receipt = {
4029
+ schemaVersion: 1,
4030
+ operationId: pending.id,
4031
+ dashboardJobId: process.env.OR3_DASHBOARD_UPDATE_JOB_ID?.trim() || undefined,
4032
+ cliVersion: PACKAGE_VERSION,
4033
+ source: {
4034
+ appVersion: sourceVersion,
4035
+ image: sourceImage,
4036
+ imageDigest: sourceDigest,
4037
+ sourceRevision: packagedSourceRevision(sourceVersion),
4038
+ operatorImageDigest: expectedOperatorImageDigest(sourceVersion)
4039
+ },
4040
+ target: {
4041
+ appVersion: targetVersion,
4042
+ image: targetImage,
4043
+ imageDigest: digest,
4044
+ sourceRevision: packagedSourceRevision(targetVersion),
4045
+ operatorImageDigest: expectedOperatorImageDigest(targetVersion)
4046
+ },
4047
+ observed: { appVersion: targetVersion, image: targetImage, imageDigest: digest },
4048
+ completedAt: now(),
4049
+ rollbackBackupId: backup.backupId,
4050
+ checks: targetReadyEvidence.checks,
4051
+ warnings,
4052
+ phaseDurationsMs: { "replace-and-verify": Date.now() - startedAtMs },
4053
+ operatorHandoff
4054
+ };
2975
4055
  state.rollback = {
2976
- appVersion: state.appVersion,
2977
- image: state.image,
2978
- imageDigest: state.imageDigest,
4056
+ appVersion: sourceVersion,
4057
+ image: sourceImage,
4058
+ imageDigest: sourceDigest,
2979
4059
  backupId: backup.backupId,
2980
4060
  createdAt: now()
2981
4061
  };
@@ -2986,11 +4066,39 @@ async function updateCommand(directory, flags) {
2986
4066
  state.deploymentRoot = resolve(loaded.directory);
2987
4067
  state.lastSuccessfulOperation = "update";
2988
4068
  state.lastError = undefined;
2989
- await pruneBackups(loaded.directory, state, BACKUP_RETENTION_KEEP, false);
2990
- await scheduleDashboardOperatorHandoff(loaded.directory, nextEnv);
2991
- await clearPending(loaded.directory, state);
2992
- console.log(`OR3 updated to ${targetVersion}. Image digest: ${digest}`);
2993
- console.log(`Rollback point: ${backup.backupId}. Keep it until login, chat, and file checks pass.`);
4069
+ let commit;
4070
+ try {
4071
+ commit = await commitTerminalState(loaded.directory, state, receipt);
4072
+ } catch (error) {
4073
+ throw new Error(`OR3 ${targetVersion} replaced the running release, but the terminal state write could not be confirmed. Recovery evidence was preserved; re-run status/doctor and "npx @or3/cloud recover" to resolve the durable outcome before any further mutation. ${redact(error instanceof Error ? error.message : String(error), secretValues(nextEnv))}`);
4074
+ }
4075
+ try {
4076
+ await removeOperationRecord(loaded.directory, commit.operationId);
4077
+ } catch (error) {
4078
+ warnings.push({ code: "operation-mirror-delete-failed", severity: "warning", resource: commit.operationId, message: redact(error instanceof Error ? error.message : String(error), secretValues(nextEnv)) });
4079
+ }
4080
+ const prune = await pruneBackups(loaded.directory, state, BACKUP_RETENTION_KEEP, false, { automatic: true, log: progress, warn: progress }).catch((error) => {
4081
+ warnings.push({ code: "retention-failed", severity: "warning", message: redact(error instanceof Error ? error.message : String(error), secretValues(nextEnv)) });
4082
+ return { removed: 0, deferred: [] };
4083
+ });
4084
+ for (const diagnostic of prune.deferred)
4085
+ warnings.push(diagnostic);
4086
+ progress(`OR3 updated to ${targetVersion}. Image digest: ${digest}`);
4087
+ progress(`Rollback point: ${backup.backupId}. Keep it until login, chat, and file checks pass.`);
4088
+ progress("Not checked: live OpenRouter authorization (complete it in the browser).");
4089
+ if (warnings.length > 0) {
4090
+ warnings.forEach((warning) => console.warn(`⚠ ${warning.code}: ${warning.message}`));
4091
+ }
4092
+ await persistReceiptWarnings(loaded.directory, state, warnings);
4093
+ if (asJson) {
4094
+ const outcome = warnings.length > 0 ? { kind: "completed-with-warnings", receipt, warnings } : { kind: "completed", receipt };
4095
+ emitOperationResult(outcome);
4096
+ } else if (warnings.length > 0) {
4097
+ console.log(`Update complete with ${warnings.length} maintenance warning(s).`);
4098
+ console.log("Next: npx @or3/cloud backup list");
4099
+ } else {
4100
+ console.log("Update complete. No maintenance warnings.");
4101
+ }
2994
4102
  } catch (error) {
2995
4103
  if (state.incompleteOperation) {
2996
4104
  state.lastError = redact(error instanceof Error ? error.message : String(error), secretValues(env));
@@ -3110,8 +4218,8 @@ async function restoreCommand(directory, flags, positionals) {
3110
4218
  const digest = await imageDigest(restoredEnv.OR3_IMAGE);
3111
4219
  const nextState = stateFromEnv(loaded.directory, restoredEnv, loaded.state.mode, "restore", digest);
3112
4220
  nextState.lastError = undefined;
3113
- await removeOperationRecord(loaded.directory, pending.id);
3114
4221
  await writeState(loaded.directory, nextState);
4222
+ await removeOperationRecordSafely(loaded.directory, pending.id);
3115
4223
  console.log(`Restored ${manifest.backupId}. Verify sign-in, a conversation, and a previously uploaded file.`);
3116
4224
  } catch (error) {
3117
4225
  const original = redact(error instanceof Error ? error.message : String(error), secretValues(loaded.env));
@@ -3201,8 +4309,8 @@ async function rollbackCommand(directory, flags) {
3201
4309
  const restoredEnv = parseEnv(await readText(deploymentPaths(loaded.directory).env));
3202
4310
  const nextState = stateFromEnv(loaded.directory, restoredEnv, loaded.state.mode, "restore", await imageDigest(restoredEnv.OR3_IMAGE));
3203
4311
  nextState.rollback = undefined;
3204
- await removeOperationRecord(loaded.directory, pending.id);
3205
4312
  await writeState(loaded.directory, nextState);
4313
+ await removeOperationRecordSafely(loaded.directory, pending.id);
3206
4314
  console.log(`Rolled back to OR3 ${nextState.appVersion}. Verify sign-in, chat, and file access.`);
3207
4315
  } catch (error) {
3208
4316
  const original = redact(error instanceof Error ? error.message : String(error), secretValues(loaded.env));
@@ -3739,6 +4847,53 @@ ${redact(serious.slice(-20).join(`
3739
4847
  console.log("✓ Recent bounded logs contain no fatal, panic, unhandled, or OOM events");
3740
4848
  console.log("OR3 production verification passed.");
3741
4849
  }
4850
+ async function verifyReadOnlyCommand(directory, flags) {
4851
+ const observation = await observeDeployment(directory, { checkDocker: true, checkImage: true });
4852
+ const state = observation.state;
4853
+ if (!state) {
4854
+ throw new Error(`Read-only verification needs readable managed state.${observation.stateError ? ` ${observation.stateError}` : ""} Run "npx @or3/cloud doctor" for diagnostics.`);
4855
+ }
4856
+ if (boolFlag(flags, "public") && state.mode !== "public")
4857
+ throw new Error("--public requires a managed public deployment.");
4858
+ const checks = [];
4859
+ const findings = observationFindings(observation);
4860
+ checks.push(observation.identityMatches === true ? { code: "image-digest", status: "passed", detail: `Recorded digest matches the local image (${state.imageDigest}).` } : observation.identityMatches === false ? { code: "image-digest", status: "failed", detail: `Recorded ${observation.recordedImageDigest} does not match local ${observation.actualImageDigest}.` } : { code: "image-digest", status: "unknown", detail: "The managed image could not be inspected safely." });
4861
+ const baseUrl = new URL(state.mode === "public" ? `https://${state.domain}` : `http://127.0.0.1:${state.port}`);
4862
+ try {
4863
+ const root = await verificationFetch(baseUrl);
4864
+ checks.push(root.status === 200 ? { code: "http-root", status: "passed", detail: `${baseUrl.origin} returned HTTP 200 without redirects.` } : { code: "http-root", status: "failed", detail: `${baseUrl.origin} returned HTTP ${root.status}.` });
4865
+ } catch (error) {
4866
+ checks.push({ code: "http-root", status: "failed", detail: redact(error instanceof Error ? error.message : String(error)) });
4867
+ }
4868
+ try {
4869
+ validateVerificationHealth(await verificationJson(baseUrl, "/api/health?deep=true"));
4870
+ checks.push({ code: "deep-health", status: "passed", detail: "Deep health reports the managed Basic Auth + SQLite + filesystem profile." });
4871
+ } catch (error) {
4872
+ checks.push({ code: "deep-health", status: "failed", detail: redact(error instanceof Error ? error.message : String(error)) });
4873
+ }
4874
+ checks.push({ code: "auth-and-storage-journey", status: "deferred", detail: "Sign-in, sync, and storage probes are excluded from read-only verification." });
4875
+ checks.push({ code: "sqlite-integrity", status: "deferred", detail: "SQLite inspection is deferred because a safe read-only mechanism is unavailable." });
4876
+ checks.push({ code: "bounded-logs", status: "deferred", detail: "Log scanning is deliberately skipped by read-only verification; run full verify for it." });
4877
+ if (boolFlag(flags, "json")) {
4878
+ console.log(JSON.stringify({
4879
+ schemaVersion: 1,
4880
+ kind: "or3-verify-read-only",
4881
+ observedAt: observation.observedAt,
4882
+ directory: observation.directory,
4883
+ readOnly: true,
4884
+ checks,
4885
+ findings
4886
+ }, null, 2));
4887
+ } else {
4888
+ for (const check of checks)
4889
+ console.log(` [${check.status}] ${check.code}: ${check.detail}`);
4890
+ for (const finding of findings)
4891
+ console.log(` [${finding.severity}] ${finding.code}: ${finding.message}`);
4892
+ }
4893
+ if (checks.some((check) => check.status === "failed")) {
4894
+ process.exitCode = 1;
4895
+ }
4896
+ }
3742
4897
  async function readSourceVolume(sourceDirectory) {
3743
4898
  const composeResult = await run("docker", [
3744
4899
  "compose",
@@ -4353,37 +5508,79 @@ async function credentialsCommand(directory, positionals, flags) {
4353
5508
  throw new Error("credentials reset accepts no arguments.");
4354
5509
  return await credentialsResetCommand(directory, flags);
4355
5510
  }
4356
- async function statusCommand(directory) {
4357
- const loaded = await loadManaged(directory);
4358
- assertDeploymentDirectoryIdentity(directory, loaded.state);
4359
- const { state } = loaded;
4360
- console.log(`OR3 Cloud deployment: ${loaded.directory}`);
4361
- console.log(` mode: ${state.mode}`);
4362
- console.log(` version: ${state.appVersion}`);
4363
- if (state.domain)
4364
- console.log(` domain: ${state.domain}`);
4365
- console.log(` port: ${state.port}`);
4366
- console.log(` image digest: ${state.imageDigest}`);
4367
- console.log(` last successful operation: ${state.lastSuccessfulOperation} (${state.updatedAt})`);
4368
- if (state.incompleteOperation)
4369
- console.log(` incomplete operation: ${state.incompleteOperation.operation} (${state.incompleteOperation.id})`);
4370
- if (state.lastError)
4371
- console.log(` last error: ${state.lastError}`);
4372
- console.log();
4373
- const ps = await run("docker", [...composeArgs(directory, state.mode, ["ps"])], directory);
4374
- if (ps.ok) {
4375
- console.log(ps.stdout.trim());
5511
+ async function statusCommand(directory, flags = {}) {
5512
+ const observation = await observeDeployment(directory, { checkDocker: true, checkImage: true });
5513
+ const findings = observationFindings(observation);
5514
+ const state = observation.state;
5515
+ if (boolFlag(flags, "json")) {
5516
+ console.log(JSON.stringify({
5517
+ schemaVersion: 1,
5518
+ kind: "or3-deployment-status",
5519
+ observedAt: observation.observedAt,
5520
+ partial: observation.partial,
5521
+ inProgress: observation.changing,
5522
+ directory: observation.directory,
5523
+ state: state ? publicStateProjection(state) : null,
5524
+ stateError: observation.stateError,
5525
+ envError: observation.envError,
5526
+ lease: observation.lease.status,
5527
+ docker: observation.docker,
5528
+ recordedImageDigest: observation.recordedImageDigest,
5529
+ actualImageDigest: observation.actualImageDigest,
5530
+ identityMatches: observation.identityMatches,
5531
+ lastReceipt: state?.lastReceipt ?? null,
5532
+ backups: observation.backups,
5533
+ findings
5534
+ }, null, 2));
5535
+ return;
5536
+ }
5537
+ console.log(`OR3 Cloud deployment: ${observation.directory}`);
5538
+ if (observation.changing)
5539
+ console.log(" status: in progress (a mutation lease is active; this observation is not authoritative)");
5540
+ if (!state) {
5541
+ console.log(` state: unreadable${observation.stateError ? ` — ${observation.stateError}` : ""}`);
4376
5542
  } else {
4377
- console.log(`Could not list containers: ${ps.stderr}`);
5543
+ console.log(` mode: ${state.mode}`);
5544
+ console.log(` version: ${state.appVersion}`);
5545
+ if (state.domain)
5546
+ console.log(` domain: ${state.domain}`);
5547
+ console.log(` port: ${state.port}`);
5548
+ console.log(` image digest: ${state.imageDigest}`);
5549
+ console.log(` last successful operation: ${state.lastSuccessfulOperation} (${state.updatedAt})`);
5550
+ if (state.incompleteOperation)
5551
+ console.log(` incomplete operation: ${state.incompleteOperation.operation} (${state.incompleteOperation.id})`);
5552
+ if (state.lastError)
5553
+ console.log(` last error: ${state.lastError}`);
5554
+ }
5555
+ if (observation.envError)
5556
+ console.log(` environment: unreadable — ${observation.envError}`);
5557
+ if (observation.identityMatches === false)
5558
+ console.log(` image: MISMATCH (recorded ${observation.recordedImageDigest}, local ${observation.actualImageDigest})`);
5559
+ else if (observation.identityMatches === true)
5560
+ console.log(" image: digest matches managed state");
5561
+ console.log();
5562
+ if (state && observation.docker && observation.env) {
5563
+ try {
5564
+ const ps = await run("docker", [...composeArgs(directory, state.mode, ["ps"])], directory);
5565
+ console.log(ps.ok ? ps.stdout.trim() : `Could not list containers: ${ps.stderr}`);
5566
+ const health = await probeDeepHealth(directory, state.mode);
5567
+ if (health === "ok")
5568
+ console.log("Deep health: OK");
5569
+ else if (health === "degraded")
5570
+ console.log("Deep health: DEGRADED (the container is running but /api/health?deep=true is failing).");
5571
+ else
5572
+ console.log("Deep health: unreachable (the or3 container is not running).");
5573
+ await printMaintenanceSummary(directory, state.mode);
5574
+ } catch (error) {
5575
+ console.log(`Container and health checks were skipped: ${redact(error instanceof Error ? error.message : String(error))}`);
5576
+ }
5577
+ } else if (state && !observation.docker) {
5578
+ console.log("Docker is unavailable; container and health checks were skipped.");
5579
+ } else if (state && !observation.env) {
5580
+ console.log("The managed environment is unreadable; container and health checks were skipped.");
4378
5581
  }
4379
- const health = await probeDeepHealth(directory, state.mode);
4380
- if (health === "ok")
4381
- console.log("Deep health: OK");
4382
- else if (health === "degraded")
4383
- console.log("Deep health: DEGRADED (the container is running but /api/health?deep=true is failing).");
4384
- else
4385
- console.log("Deep health: unreachable (the or3 container is not running).");
4386
- await printMaintenanceSummary(directory, state.mode);
5582
+ for (const finding of findings)
5583
+ console.log(` [${finding.severity}] ${finding.code}: ${finding.message}`);
4387
5584
  }
4388
5585
  async function printMaintenanceSummary(directory, mode) {
4389
5586
  const result = await run("docker", [...composeArgs(directory, mode, ["exec", "-T", "or3", ...containerNodeCommand(MAINTENANCE_SCRIPT)])], directory);
@@ -4403,7 +5600,7 @@ async function printMaintenanceSummary(directory, mode) {
4403
5600
  } catch {}
4404
5601
  }
4405
5602
  async function logsCommand(directory, flags, positionals) {
4406
- const loaded = await loadManaged(directory);
5603
+ const loaded = await loadManaged(directory, { writable: false });
4407
5604
  assertDeploymentDirectoryIdentity(directory, loaded.state);
4408
5605
  if (positionals.length > 1)
4409
5606
  throw new Error("logs accepts at most one service name.");
@@ -4432,6 +5629,7 @@ async function startCommand(directory) {
4432
5629
  await ensureDocker();
4433
5630
  const loaded = await loadManaged(directory);
4434
5631
  assertDeploymentDirectoryIdentity(directory, loaded.state);
5632
+ assertPhaseAllowsCommand("start", loaded.state);
4435
5633
  await startProject(directory, loaded.state.mode, loaded.env);
4436
5634
  const url = loaded.state.mode === "public" ? `https://${loaded.state.domain}` : `http://127.0.0.1:${loaded.state.port}`;
4437
5635
  console.log(`OR3 started and is deeply healthy at ${url}.`);
@@ -4447,6 +5645,7 @@ async function restartCommand(directory) {
4447
5645
  await ensureDocker();
4448
5646
  const loaded = await loadManaged(directory);
4449
5647
  assertDeploymentDirectoryIdentity(directory, loaded.state);
5648
+ assertPhaseAllowsCommand("restart", loaded.state);
4450
5649
  await compose(directory, loaded.state.mode, ["restart", "or3"]);
4451
5650
  await waitForDeepHealth(directory, loaded.state.mode, secretValues(loaded.env));
4452
5651
  console.log("OR3 restarted and is deeply healthy.");
@@ -4588,8 +5787,11 @@ async function main(argv = process.argv.slice(2)) {
4588
5787
  const dispatch = async () => {
4589
5788
  if (command === "init")
4590
5789
  return await initCommand(parsed.positionals, parsed.flags);
4591
- if (command === "update")
5790
+ if (command === "update") {
5791
+ if (boolFlag(parsed.flags, "dry-run"))
5792
+ return await updatePreviewCommand(process.cwd(), parsed.flags);
4592
5793
  return await updateCommand(process.cwd(), parsed.flags);
5794
+ }
4593
5795
  if (command === "backup")
4594
5796
  return await backupCommand(process.cwd(), parsed.positionals, parsed.flags);
4595
5797
  if (command === "restore")
@@ -4600,14 +5802,17 @@ async function main(argv = process.argv.slice(2)) {
4600
5802
  return await credentialsCommand(process.cwd(), parsed.positionals, parsed.flags);
4601
5803
  if (command === "doctor")
4602
5804
  return await doctorCommand(process.cwd());
4603
- if (command === "verify")
5805
+ if (command === "verify") {
5806
+ if (verifyIsReadOnly(parsed.flags))
5807
+ return await verifyReadOnlyCommand(process.cwd(), parsed.flags);
4604
5808
  return await verifyCommand(process.cwd(), parsed.flags, parsed.positionals);
5809
+ }
4605
5810
  if (command === "recover")
4606
- return await recoverCommand(process.cwd());
5811
+ return await recoverCommand(process.cwd(), parsed.flags);
4607
5812
  if (command === "adopt")
4608
5813
  return await adoptCommand(parsed.positionals, parsed.flags);
4609
5814
  if (command === "status")
4610
- return await statusCommand(process.cwd());
5815
+ return await statusCommand(process.cwd(), parsed.flags);
4611
5816
  if (command === "logs")
4612
5817
  return await logsCommand(process.cwd(), parsed.flags, parsed.positionals);
4613
5818
  if (command === "start")
@@ -4620,7 +5825,8 @@ async function main(argv = process.argv.slice(2)) {
4620
5825
  return await removeCommand(process.cwd(), parsed.flags);
4621
5826
  throw new Error(`Unknown command "${command}". Run npx @or3/cloud --help.`);
4622
5827
  };
4623
- if (MUTATING_COMMANDS.has(command)) {
5828
+ const readOnlyInvocation = command === "verify" && verifyIsReadOnly(parsed.flags) || command === "update" && boolFlag(parsed.flags, "dry-run") || command === "recover" && boolFlag(parsed.flags, "dry-run") || command === "backup" && parsed.positionals[0] === "list";
5829
+ if (!readOnlyInvocation && MUTATING_COMMANDS.has(command)) {
4624
5830
  return await withDeploymentLease(mutationDirectory(command, parsed.positionals, parsed.flags), command, dispatch);
4625
5831
  }
4626
5832
  return await dispatch();
@@ -4635,12 +5841,15 @@ if (invokedAsCli || process.argv[1] && resolve(process.argv[1]) === resolve(file
4635
5841
  await main();
4636
5842
  }
4637
5843
  export {
5844
+ writeStateSchema,
4638
5845
  withoutProvisioningCredentials,
5846
+ verifyIsReadOnly,
4639
5847
  validateVerificationHealth,
4640
5848
  validatePassword,
4641
5849
  updateRequiresVolumeRecreation,
4642
5850
  supportedImageArchitectures,
4643
5851
  stateFromEnv,
5852
+ stateFingerprint,
4644
5853
  snapshotManagedAssets,
4645
5854
  serializeInitialCredentials,
4646
5855
  serializeEnv,
@@ -4652,28 +5861,45 @@ export {
4652
5861
  redact,
4653
5862
  recordedBackupPath,
4654
5863
  purgeVolumesFromState,
5864
+ publicStateProjection,
5865
+ planRetention,
4655
5866
  parseFlags,
4656
5867
  parseEnv,
5868
+ packagedMinimumSourceVersion,
5869
+ observeDeployment,
5870
+ observationFindings,
4657
5871
  managedAssetNames,
5872
+ lifecycleFaults,
4658
5873
  isVersion,
5874
+ inventoryBackups,
5875
+ installedManagedAssetChecksums,
4659
5876
  enumerateBackups,
5877
+ decideRecoveryAction,
4660
5878
  dashboardOperatorHandoffArgs,
4661
5879
  copyAssets,
5880
+ compareReleaseVersions,
4662
5881
  checkResolvedLoopbackBinding,
4663
5882
  buildEnv,
4664
5883
  buildCredentialsResetScript,
5884
+ assessUpdate,
4665
5885
  assertVerificationGrant,
4666
5886
  assertSupportedSourceCompose,
4667
5887
  assertSupportedSource,
4668
5888
  assertSupportedArchitecture,
5889
+ assertStateSchemaWritable,
4669
5890
  assertRemovableArtifactName,
4670
5891
  assertPurgeBackupFreshness,
5892
+ assertPhaseAllowsCommand,
5893
+ assertKnownStateSchema,
4671
5894
  assertImageReleaseLabels,
4672
5895
  assertEnoughFreeSpace,
4673
5896
  assertDashboardOperatorMounts,
4674
5897
  assertCommandPositionals,
4675
5898
  assertCommandFlags,
4676
5899
  assertBackupMatchesDeployment,
5900
+ STATE_SCHEMA_COMPATIBILITY,
5901
+ READER_SUPPORTED_STATE_SCHEMAS,
4677
5902
  PACKAGE_VERSION,
5903
+ MAX_SUPPORTED_STATE_SCHEMA,
4678
5904
  IMAGE_REPOSITORY
4679
5905
  };