@or3/cloud 0.1.67 → 0.1.68
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +85 -9
- package/assets/dashboard-operator.mjs +211 -25
- package/dist/cli.mjs +1376 -150
- package/package.json +6 -5
package/dist/cli.mjs
CHANGED
|
@@ -31,10 +31,10 @@ import { fileURLToPath } from "node:url";
|
|
|
31
31
|
import { createGunzip } from "node:zlib";
|
|
32
32
|
var execFile = promisify(execFileCallback);
|
|
33
33
|
var PACKAGE_ROOT = resolve(fileURLToPath(new URL("../", import.meta.url)));
|
|
34
|
-
var PACKAGE_VERSION = "0.1.
|
|
34
|
+
var PACKAGE_VERSION = "0.1.68";
|
|
35
35
|
var IMAGE_REPOSITORY = "ghcr.io/saluana/or3-chat";
|
|
36
36
|
var ASSET_ROOT = resolve(fileURLToPath(new URL("../assets/", import.meta.url)));
|
|
37
|
-
var
|
|
37
|
+
var LEGACY_STATE_SCHEMA_VERSION = 1;
|
|
38
38
|
var DEFAULT_PORT = 3000;
|
|
39
39
|
var DEEP_HEALTH_TIMEOUT_MS = 180000;
|
|
40
40
|
var COMMAND_TIMEOUT_MS = 180000;
|
|
@@ -70,6 +70,21 @@ var DEPLOYMENT_ENV_KEYS = [
|
|
|
70
70
|
"OR3_PUBLIC_DOMAIN"
|
|
71
71
|
];
|
|
72
72
|
var DEPLOYMENT_ID_ENV_KEY = "OR3_DEPLOYMENT_ID";
|
|
73
|
+
function emitOperationResult(outcome) {
|
|
74
|
+
console.log(JSON.stringify({ schemaVersion: 1, kind: "or3-operation-result", outcome }, null, 2));
|
|
75
|
+
}
|
|
76
|
+
var STATE_SCHEMA_COMPATIBILITY = {
|
|
77
|
+
1: { readers: ["bridge", "schema-2"], writer: "migrating", mutable: false },
|
|
78
|
+
2: { readers: ["bridge", "schema-2"], writer: "schema-2", mutable: true }
|
|
79
|
+
};
|
|
80
|
+
var MAX_SUPPORTED_STATE_SCHEMA = 2;
|
|
81
|
+
var READER_SUPPORTED_STATE_SCHEMAS = [1, 2];
|
|
82
|
+
function assertKnownStateSchema(schemaVersion) {
|
|
83
|
+
if (schemaVersion !== 1 && schemaVersion !== 2) {
|
|
84
|
+
throw new Error(`Unsupported managed state schema ${String(schemaVersion)}. This CLI reads schemas 1 and 2. Run a compatible exact-version @or3/cloud CLI for this deployment instead of editing managed state.`);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
var lifecycleFaults = {};
|
|
73
88
|
var PROCESS_ENV_PASSTHROUGH = [
|
|
74
89
|
"PATH",
|
|
75
90
|
"HOME",
|
|
@@ -201,22 +216,25 @@ function parseFlags(argv) {
|
|
|
201
216
|
}
|
|
202
217
|
var COMMAND_FLAGS = {
|
|
203
218
|
init: ["local", "public", "domain", "admin-email", "admin-password", "admin-password-file", "port"],
|
|
204
|
-
update: ["to"],
|
|
205
|
-
backup: ["keep", "force", "yes"],
|
|
219
|
+
update: ["to", "dry-run", "json"],
|
|
220
|
+
backup: ["keep", "force", "yes", "json"],
|
|
206
221
|
restore: ["yes"],
|
|
207
222
|
rollback: ["yes"],
|
|
208
223
|
doctor: [],
|
|
209
|
-
verify: ["public", "verification-email", "verification-password-file"],
|
|
210
|
-
recover: [],
|
|
224
|
+
verify: ["public", "verification-email", "verification-password-file", "read-only", "json"],
|
|
225
|
+
recover: ["dry-run", "finish", "restore", "yes", "json"],
|
|
211
226
|
adopt: ["from"],
|
|
212
227
|
credentials: ["yes", "owner-password", "owner-password-file", "admin-password", "admin-password-file"],
|
|
213
|
-
status: [],
|
|
228
|
+
status: ["json"],
|
|
214
229
|
logs: ["tail"],
|
|
215
230
|
start: [],
|
|
216
231
|
stop: [],
|
|
217
232
|
restart: [],
|
|
218
233
|
remove: ["purge-data", "yes"]
|
|
219
234
|
};
|
|
235
|
+
function verifyIsReadOnly(flags) {
|
|
236
|
+
return flags["read-only"] === true;
|
|
237
|
+
}
|
|
220
238
|
function assertCommandFlags(command, flags) {
|
|
221
239
|
const allowed = new Set(["help", ...COMMAND_FLAGS[command] ?? []]);
|
|
222
240
|
const unknown = Object.keys(flags).filter((flag) => !allowed.has(flag));
|
|
@@ -262,6 +280,15 @@ function requireStringFlag(flags, key) {
|
|
|
262
280
|
function isVersion(value) {
|
|
263
281
|
return /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/.test(value);
|
|
264
282
|
}
|
|
283
|
+
function compareReleaseVersions(left, right) {
|
|
284
|
+
const leftParts = left.split("-", 1)[0].split(".").map(Number);
|
|
285
|
+
const rightParts = right.split("-", 1)[0].split(".").map(Number);
|
|
286
|
+
for (let index = 0;index < 3; index += 1) {
|
|
287
|
+
if (leftParts[index] !== rightParts[index])
|
|
288
|
+
return leftParts[index] > rightParts[index] ? 1 : -1;
|
|
289
|
+
}
|
|
290
|
+
return 0;
|
|
291
|
+
}
|
|
265
292
|
function imageFor(version) {
|
|
266
293
|
if (!isVersion(version))
|
|
267
294
|
throw new Error(`Invalid release version "${version}".`);
|
|
@@ -329,6 +356,7 @@ function commandEnvironment(command, args, cwd, supplied) {
|
|
|
329
356
|
}
|
|
330
357
|
async function run(command, args, cwd, environment) {
|
|
331
358
|
const printable = `${command} ${args.map(quote).join(" ")}`;
|
|
359
|
+
await lifecycleFaults.beforeCommand?.(command, args);
|
|
332
360
|
try {
|
|
333
361
|
const result = await execFile(command, args, {
|
|
334
362
|
cwd,
|
|
@@ -478,6 +506,7 @@ function deploymentPaths(directory) {
|
|
|
478
506
|
operations: join(cloud, "operations"),
|
|
479
507
|
backups: join(cloud, "backups"),
|
|
480
508
|
exports: join(cloud, "exports"),
|
|
509
|
+
lastOperation: join(cloud, "last-operation.json"),
|
|
481
510
|
lease: join(cloud, "operation-lease"),
|
|
482
511
|
backupAuthKey: join(cloud, "backup-auth.key"),
|
|
483
512
|
operatorIpc: join(cloud, "operator-ipc")
|
|
@@ -530,14 +559,17 @@ async function assertBackupAuthentication(directory, backupPath, manifestContent
|
|
|
530
559
|
async function readState(directory) {
|
|
531
560
|
const paths = deploymentPaths(directory);
|
|
532
561
|
const parsed = JSON.parse(await readText(paths.state));
|
|
533
|
-
|
|
562
|
+
assertKnownStateSchema(parsed.schemaVersion);
|
|
563
|
+
if (!parsed.appVersion || !parsed.image || !parsed.composeProject || !parsed.mode) {
|
|
534
564
|
throw new Error(`Invalid managed state at ${paths.state}. Run "npx @or3/cloud doctor" for diagnostics.`);
|
|
535
565
|
}
|
|
536
566
|
return parsed;
|
|
537
567
|
}
|
|
538
568
|
async function writeState(directory, state) {
|
|
569
|
+
await lifecycleFaults.beforeStateWrite?.();
|
|
539
570
|
await writeSecure(deploymentPaths(directory).state, `${JSON.stringify(state, null, 2)}
|
|
540
571
|
`);
|
|
572
|
+
await lifecycleFaults.afterStateWrite?.();
|
|
541
573
|
}
|
|
542
574
|
async function readDirectoryEmpty(directory) {
|
|
543
575
|
try {
|
|
@@ -883,8 +915,10 @@ function containerNodeCommand(script) {
|
|
|
883
915
|
return ["sh", "-c", CONTAINER_NODE_SHELL, "or3-node", script];
|
|
884
916
|
}
|
|
885
917
|
async function waitForDeepHealthWithArgs(composeCommand, directory, secrets = []) {
|
|
886
|
-
const
|
|
918
|
+
const startedAt = Date.now();
|
|
919
|
+
const deadline = startedAt + DEEP_HEALTH_TIMEOUT_MS;
|
|
887
920
|
let lastError = "health check did not complete";
|
|
921
|
+
let lastProgressAt = startedAt;
|
|
888
922
|
while (Date.now() < deadline) {
|
|
889
923
|
const result = await run("docker", [
|
|
890
924
|
...composeCommand,
|
|
@@ -896,6 +930,10 @@ async function waitForDeepHealthWithArgs(composeCommand, directory, secrets = []
|
|
|
896
930
|
if (result.ok)
|
|
897
931
|
return;
|
|
898
932
|
lastError = redact(result.stderr, secrets);
|
|
933
|
+
if (Date.now() - lastProgressAt >= 15000) {
|
|
934
|
+
console.error(`Waiting for OR3 deep health (${Math.round((Date.now() - startedAt) / 1000)}s elapsed)…`);
|
|
935
|
+
lastProgressAt = Date.now();
|
|
936
|
+
}
|
|
899
937
|
await new Promise((resolvePromise) => setTimeout(resolvePromise, 1000));
|
|
900
938
|
}
|
|
901
939
|
throw new Error(`OR3 deep health did not pass within ${DEEP_HEALTH_TIMEOUT_MS / 1000} seconds. Last error: ${lastError}`);
|
|
@@ -1112,6 +1150,26 @@ function packagedSourceRevision(version) {
|
|
|
1112
1150
|
}
|
|
1113
1151
|
return revision;
|
|
1114
1152
|
}
|
|
1153
|
+
function packagedOr3CloudMetadata() {
|
|
1154
|
+
try {
|
|
1155
|
+
const manifest = JSON.parse(readFileSync(join(PACKAGE_ROOT, "package.json"), "utf8"));
|
|
1156
|
+
return manifest.or3Cloud ?? {};
|
|
1157
|
+
} catch {
|
|
1158
|
+
return {};
|
|
1159
|
+
}
|
|
1160
|
+
}
|
|
1161
|
+
function writeStateSchema() {
|
|
1162
|
+
const value = packagedOr3CloudMetadata().stateSchema;
|
|
1163
|
+
if (value === undefined)
|
|
1164
|
+
return LEGACY_STATE_SCHEMA_VERSION;
|
|
1165
|
+
if (value === 1 || value === 2)
|
|
1166
|
+
return value;
|
|
1167
|
+
throw new Error("This @or3/cloud package declares an unsupported managed state schema. Refusing to write managed state.");
|
|
1168
|
+
}
|
|
1169
|
+
function packagedMinimumSourceVersion() {
|
|
1170
|
+
const value = packagedOr3CloudMetadata().dashboardUpdateMinimumSourceVersion;
|
|
1171
|
+
return typeof value === "string" && /^\d+\.\d+\.\d+$/.test(value) ? value : undefined;
|
|
1172
|
+
}
|
|
1115
1173
|
function expectedImageDigest(version) {
|
|
1116
1174
|
const packaged = packagedImageDigest(version);
|
|
1117
1175
|
const supplied = process.env.OR3_EXPECTED_IMAGE_DIGEST?.trim();
|
|
@@ -1438,10 +1496,11 @@ function dashboardOperatorHandoffArgs(directory, env, jobId) {
|
|
|
1438
1496
|
project
|
|
1439
1497
|
];
|
|
1440
1498
|
}
|
|
1441
|
-
async function scheduleDashboardOperatorHandoff(directory, env) {
|
|
1442
|
-
const jobId = process.env.OR3_DASHBOARD_UPDATE_JOB_ID?.trim();
|
|
1499
|
+
async function scheduleDashboardOperatorHandoff(directory, env, jobIdOverride) {
|
|
1500
|
+
const jobId = jobIdOverride ?? process.env.OR3_DASHBOARD_UPDATE_JOB_ID?.trim();
|
|
1443
1501
|
if (!jobId || env.OR3_DASHBOARD_UPDATES_ENABLED !== "true")
|
|
1444
1502
|
return;
|
|
1503
|
+
await lifecycleFaults.beforeHandoff?.();
|
|
1445
1504
|
const scheduled = await run("docker", dashboardOperatorHandoffArgs(directory, env, jobId), directory);
|
|
1446
1505
|
if (!scheduled.ok || !/^[0-9a-f]{12,64}$/i.test(scheduled.stdout.trim())) {
|
|
1447
1506
|
throw new Error(`Could not schedule the dashboard operator handoff. ${scheduled.stderr.trim()}`);
|
|
@@ -1642,6 +1701,15 @@ async function copyAssets(directory, mode) {
|
|
|
1642
1701
|
}
|
|
1643
1702
|
await installManagedAssets(directory, assets);
|
|
1644
1703
|
}
|
|
1704
|
+
async function installedManagedAssetChecksums(directory, mode) {
|
|
1705
|
+
const checksums = {};
|
|
1706
|
+
for (const name of managedAssetNames(mode)) {
|
|
1707
|
+
if (!await fileExists(join(directory, name)))
|
|
1708
|
+
continue;
|
|
1709
|
+
checksums[name] = await sha256File(join(directory, name));
|
|
1710
|
+
}
|
|
1711
|
+
return checksums;
|
|
1712
|
+
}
|
|
1645
1713
|
async function snapshotManagedAssets(directory, mode, backupDir) {
|
|
1646
1714
|
const assetDir = join(backupDir, "managed-assets");
|
|
1647
1715
|
await mkdir(assetDir, { recursive: true, mode: 448 });
|
|
@@ -1731,14 +1799,61 @@ async function updatePending(directory, state, patch) {
|
|
|
1731
1799
|
async function removeOperationRecord(directory, operationId) {
|
|
1732
1800
|
if (!operationId)
|
|
1733
1801
|
return;
|
|
1802
|
+
await lifecycleFaults.beforeMirrorDelete?.();
|
|
1734
1803
|
await rm(join(deploymentPaths(directory).operations, `${operationId}.json`), { force: true });
|
|
1735
1804
|
}
|
|
1736
|
-
async function
|
|
1805
|
+
async function removeOperationRecordSafely(directory, operationId) {
|
|
1806
|
+
try {
|
|
1807
|
+
await removeOperationRecord(directory, operationId);
|
|
1808
|
+
} catch (error) {
|
|
1809
|
+
console.warn(`Could not remove the redundant operation mirror ${operationId ?? "(none)"}: ${redact(error instanceof Error ? error.message : String(error))}`);
|
|
1810
|
+
}
|
|
1811
|
+
}
|
|
1812
|
+
async function commitTerminalState(directory, state, receipt) {
|
|
1737
1813
|
const operationId = state.incompleteOperation?.id;
|
|
1738
1814
|
delete state.incompleteOperation;
|
|
1815
|
+
if (receipt)
|
|
1816
|
+
state.lastReceipt = receipt;
|
|
1739
1817
|
state.updatedAt = now();
|
|
1740
|
-
await removeOperationRecord(directory, operationId);
|
|
1741
1818
|
await writeState(directory, state);
|
|
1819
|
+
if (receipt) {
|
|
1820
|
+
try {
|
|
1821
|
+
await exportTerminalReceipt(directory, receipt);
|
|
1822
|
+
} catch (error) {
|
|
1823
|
+
console.warn(`Could not export the latest operation receipt: ${redact(error instanceof Error ? error.message : String(error))}`);
|
|
1824
|
+
}
|
|
1825
|
+
}
|
|
1826
|
+
return { committed: true, operationId };
|
|
1827
|
+
}
|
|
1828
|
+
async function exportTerminalReceipt(directory, receipt) {
|
|
1829
|
+
const payload = `${JSON.stringify(receipt, null, 2)}
|
|
1830
|
+
`;
|
|
1831
|
+
await writeSecure(deploymentPaths(directory).lastOperation, payload);
|
|
1832
|
+
}
|
|
1833
|
+
async function persistReceiptWarnings(directory, state, warnings) {
|
|
1834
|
+
if (warnings.length === 0 || !state.lastReceipt)
|
|
1835
|
+
return;
|
|
1836
|
+
const existing = state.lastReceipt.warnings;
|
|
1837
|
+
const additions = warnings.filter((warning) => !existing.some((current) => current.code === warning.code && current.message === warning.message));
|
|
1838
|
+
if (additions.length === 0)
|
|
1839
|
+
return;
|
|
1840
|
+
const merged = { ...state.lastReceipt, warnings: [...existing, ...additions] };
|
|
1841
|
+
try {
|
|
1842
|
+
if (state.incompleteOperation)
|
|
1843
|
+
return;
|
|
1844
|
+
state.lastReceipt = merged;
|
|
1845
|
+
state.updatedAt = now();
|
|
1846
|
+
await writeState(directory, state);
|
|
1847
|
+
await exportTerminalReceipt(directory, merged).catch((error) => {
|
|
1848
|
+
console.warn(`Could not re-export the operation receipt: ${redact(error instanceof Error ? error.message : String(error))}`);
|
|
1849
|
+
});
|
|
1850
|
+
} catch (error) {
|
|
1851
|
+
console.warn(`Could not persist maintenance warnings to the receipt: ${redact(error instanceof Error ? error.message : String(error))}`);
|
|
1852
|
+
}
|
|
1853
|
+
}
|
|
1854
|
+
async function clearPending(directory, state, receipt) {
|
|
1855
|
+
const { operationId } = await commitTerminalState(directory, state, receipt);
|
|
1856
|
+
await removeOperationRecordSafely(directory, operationId);
|
|
1742
1857
|
}
|
|
1743
1858
|
async function stopProject(directory, mode) {
|
|
1744
1859
|
await compose(directory, mode, ["stop", "or3"]);
|
|
@@ -1807,6 +1922,7 @@ function assertRemovableArtifactName(name) {
|
|
|
1807
1922
|
return name;
|
|
1808
1923
|
}
|
|
1809
1924
|
async function removeNamedBackupArtifact(directory, backupId) {
|
|
1925
|
+
await lifecycleFaults.beforeArtifactDelete?.();
|
|
1810
1926
|
assertRemovableArtifactName(backupId);
|
|
1811
1927
|
const target = backupDirectory(directory, backupId);
|
|
1812
1928
|
if (dirname(target) !== deploymentPaths(directory).backups) {
|
|
@@ -1816,6 +1932,7 @@ async function removeNamedBackupArtifact(directory, backupId) {
|
|
|
1816
1932
|
await rm(join(deploymentPaths(directory).exports, `${backupId}.json`), { force: true });
|
|
1817
1933
|
}
|
|
1818
1934
|
async function removeEnumeratedBackupArtifact(directory, backup) {
|
|
1935
|
+
await lifecycleFaults.beforeArtifactDelete?.();
|
|
1819
1936
|
const backupsRoot = resolve(deploymentPaths(directory).backups);
|
|
1820
1937
|
const target = resolve(backup.path);
|
|
1821
1938
|
if (dirname(target) !== backupsRoot || basename(target) !== backup.backupId || !BACKUP_ID_PATTERN.test(backup.backupId)) {
|
|
@@ -1824,55 +1941,159 @@ async function removeEnumeratedBackupArtifact(directory, backup) {
|
|
|
1824
1941
|
await rm(target, { recursive: true, force: true });
|
|
1825
1942
|
await rm(join(deploymentPaths(directory).exports, `${backup.backupId}.json`), { force: true });
|
|
1826
1943
|
}
|
|
1827
|
-
|
|
1828
|
-
const
|
|
1829
|
-
|
|
1944
|
+
function classifyBackupError(error) {
|
|
1945
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1946
|
+
if (/authentication failed|no valid deployment authentication tag|invalid authentication tag/i.test(message)) {
|
|
1947
|
+
return { code: "backup-authentication-failed", message };
|
|
1948
|
+
}
|
|
1949
|
+
if (/checksum mismatch/i.test(message))
|
|
1950
|
+
return { code: "backup-checksum-mismatch", message };
|
|
1951
|
+
if (/invalid managed asset inventory|managed asset checksum mismatch/i.test(message)) {
|
|
1952
|
+
return { code: "backup-assets-invalid", message };
|
|
1953
|
+
}
|
|
1954
|
+
if (/Invalid backup manifest/i.test(message))
|
|
1955
|
+
return { code: "backup-manifest-invalid", message };
|
|
1956
|
+
if (error?.code === "ENOENT")
|
|
1957
|
+
return { code: "backup-entry-incomplete", message };
|
|
1958
|
+
return { code: "backup-entry-invalid", message };
|
|
1959
|
+
}
|
|
1960
|
+
async function inspectBackupEntry(backupsRoot, entry, directory) {
|
|
1961
|
+
const path = join(backupsRoot, entry);
|
|
1962
|
+
let info;
|
|
1830
1963
|
try {
|
|
1831
|
-
|
|
1964
|
+
info = await lstat(path);
|
|
1832
1965
|
} catch (error) {
|
|
1833
|
-
|
|
1834
|
-
return [];
|
|
1835
|
-
throw new Error(`Could not enumerate managed backups at ${backupsRoot}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1966
|
+
return { kind: "unreadable", entryName: entry, ...classifyBackupError(error) };
|
|
1836
1967
|
}
|
|
1837
|
-
|
|
1838
|
-
|
|
1839
|
-
|
|
1840
|
-
|
|
1841
|
-
|
|
1842
|
-
|
|
1843
|
-
|
|
1968
|
+
if (info.isSymbolicLink() || !info.isDirectory()) {
|
|
1969
|
+
return {
|
|
1970
|
+
kind: "invalid",
|
|
1971
|
+
entryName: entry,
|
|
1972
|
+
code: "backup-entry-not-directory",
|
|
1973
|
+
message: `Backup store entry ${entry} is not a regular directory; symlinks and non-directories are never trusted.`
|
|
1974
|
+
};
|
|
1975
|
+
}
|
|
1976
|
+
if (/^adopt-source-[0-9A-Za-z-]+$/.test(entry)) {
|
|
1977
|
+
return {
|
|
1978
|
+
kind: "legacy-adoption",
|
|
1979
|
+
entryName: entry,
|
|
1980
|
+
code: "backup-legacy-adoption",
|
|
1981
|
+
message: `Preserved legacy adoption source ${entry}; it is not an authenticated managed backup.`
|
|
1982
|
+
};
|
|
1983
|
+
}
|
|
1984
|
+
if (!BACKUP_ID_PATTERN.test(entry)) {
|
|
1985
|
+
return {
|
|
1986
|
+
kind: "invalid",
|
|
1987
|
+
entryName: entry,
|
|
1988
|
+
code: "backup-entry-unexpected",
|
|
1989
|
+
message: `Backup store contains an unexpected artifact ${path}; inspect or remove it explicitly.`
|
|
1990
|
+
};
|
|
1991
|
+
}
|
|
1992
|
+
try {
|
|
1993
|
+
await lstat(join(path, "manifest.auth"));
|
|
1994
|
+
} catch (error) {
|
|
1995
|
+
if (error.code === "ENOENT") {
|
|
1996
|
+
return {
|
|
1997
|
+
kind: "legacy-unsigned",
|
|
1998
|
+
entryName: entry,
|
|
1999
|
+
code: "backup-unsigned",
|
|
2000
|
+
message: `Preserved unauthenticated backup ${entry} (no manifest.auth); never trusted for restore.`
|
|
2001
|
+
};
|
|
1844
2002
|
}
|
|
1845
|
-
|
|
1846
|
-
|
|
1847
|
-
|
|
1848
|
-
|
|
1849
|
-
|
|
1850
|
-
|
|
1851
|
-
|
|
1852
|
-
|
|
2003
|
+
return { kind: "unreadable", entryName: entry, ...classifyBackupError(error) };
|
|
2004
|
+
}
|
|
2005
|
+
try {
|
|
2006
|
+
const raw = JSON.parse(await readText(join(path, "manifest.json")));
|
|
2007
|
+
if (typeof raw.schemaVersion === "number" && raw.schemaVersion > 1) {
|
|
2008
|
+
return {
|
|
2009
|
+
kind: "unsupported",
|
|
2010
|
+
entryName: entry,
|
|
2011
|
+
code: "backup-format-unsupported",
|
|
2012
|
+
message: `Backup ${entry} uses manifest schema ${raw.schemaVersion}; preserved for a newer reader.`
|
|
2013
|
+
};
|
|
1853
2014
|
}
|
|
2015
|
+
} catch (error) {
|
|
2016
|
+
return { kind: "invalid", entryName: entry, ...classifyBackupError(error) };
|
|
2017
|
+
}
|
|
2018
|
+
try {
|
|
1854
2019
|
const manifest = await readManifest(path, directory);
|
|
1855
2020
|
if (manifest.backupId !== entry) {
|
|
1856
|
-
|
|
2021
|
+
return {
|
|
2022
|
+
kind: "invalid",
|
|
2023
|
+
entryName: entry,
|
|
2024
|
+
code: "backup-id-mismatch",
|
|
2025
|
+
message: `Backup directory ${path} does not match manifest ID ${manifest.backupId}.`
|
|
2026
|
+
};
|
|
1857
2027
|
}
|
|
1858
2028
|
if (!Number.isFinite(Date.parse(manifest.createdAt))) {
|
|
1859
|
-
|
|
2029
|
+
return {
|
|
2030
|
+
kind: "invalid",
|
|
2031
|
+
entryName: entry,
|
|
2032
|
+
code: "backup-created-at-invalid",
|
|
2033
|
+
message: `Backup ${manifest.backupId} has an invalid creation time.`
|
|
2034
|
+
};
|
|
1860
2035
|
}
|
|
1861
2036
|
let bytes = 0;
|
|
1862
2037
|
for (const file of ["data.tgz", "config.env", "manifest.json", "manifest.auth"]) {
|
|
1863
|
-
bytes += (await
|
|
2038
|
+
bytes += (await lstat(join(path, file))).size;
|
|
1864
2039
|
}
|
|
1865
|
-
|
|
1866
|
-
|
|
1867
|
-
|
|
1868
|
-
|
|
1869
|
-
|
|
1870
|
-
|
|
1871
|
-
|
|
1872
|
-
|
|
2040
|
+
return {
|
|
2041
|
+
kind: "verified",
|
|
2042
|
+
backup: {
|
|
2043
|
+
backupId: manifest.backupId,
|
|
2044
|
+
createdAt: manifest.createdAt,
|
|
2045
|
+
appVersion: manifest.appVersion,
|
|
2046
|
+
path,
|
|
2047
|
+
bytes,
|
|
2048
|
+
dataSha256: manifest.dataSha256
|
|
2049
|
+
}
|
|
2050
|
+
};
|
|
2051
|
+
} catch (error) {
|
|
2052
|
+
return { kind: "invalid", entryName: entry, ...classifyBackupError(error) };
|
|
1873
2053
|
}
|
|
1874
|
-
|
|
1875
|
-
|
|
2054
|
+
}
|
|
2055
|
+
var BACKUP_ENTRY_SEVERITY = {
|
|
2056
|
+
verified: 0,
|
|
2057
|
+
"legacy-unsigned": 1,
|
|
2058
|
+
"legacy-adoption": 2,
|
|
2059
|
+
unsupported: 3,
|
|
2060
|
+
invalid: 4,
|
|
2061
|
+
unreadable: 5
|
|
2062
|
+
};
|
|
2063
|
+
async function inventoryBackups(directory) {
|
|
2064
|
+
const backupsRoot = deploymentPaths(directory).backups;
|
|
2065
|
+
let names;
|
|
2066
|
+
try {
|
|
2067
|
+
names = await readdir(backupsRoot);
|
|
2068
|
+
} catch (error) {
|
|
2069
|
+
if (error.code === "ENOENT")
|
|
2070
|
+
return { entries: [], storeErrors: [] };
|
|
2071
|
+
return {
|
|
2072
|
+
entries: [],
|
|
2073
|
+
storeErrors: [{
|
|
2074
|
+
code: "backup-store-unreadable",
|
|
2075
|
+
severity: "blocker",
|
|
2076
|
+
resource: backupsRoot,
|
|
2077
|
+
message: `Could not enumerate managed backups at ${backupsRoot}: ${error instanceof Error ? error.message : String(error)}`
|
|
2078
|
+
}]
|
|
2079
|
+
};
|
|
2080
|
+
}
|
|
2081
|
+
const entries = [];
|
|
2082
|
+
for (const name of [...names].sort())
|
|
2083
|
+
entries.push(await inspectBackupEntry(backupsRoot, name, directory));
|
|
2084
|
+
entries.sort((a, b) => {
|
|
2085
|
+
const severity = BACKUP_ENTRY_SEVERITY[a.kind] - BACKUP_ENTRY_SEVERITY[b.kind];
|
|
2086
|
+
if (severity !== 0)
|
|
2087
|
+
return severity;
|
|
2088
|
+
const aName = a.kind === "verified" ? a.backup.backupId : a.entryName;
|
|
2089
|
+
const bName = b.kind === "verified" ? b.backup.backupId : b.entryName;
|
|
2090
|
+
return aName < bName ? -1 : aName > bName ? 1 : 0;
|
|
2091
|
+
});
|
|
2092
|
+
return { entries, storeErrors: [] };
|
|
2093
|
+
}
|
|
2094
|
+
async function enumerateBackups(directory) {
|
|
2095
|
+
const inventory = await inventoryBackups(directory);
|
|
2096
|
+
return inventory.entries.filter((entry) => entry.kind === "verified").map((entry) => entry.backup).sort((a, b) => a.createdAt < b.createdAt ? 1 : a.createdAt > b.createdAt ? -1 : 0);
|
|
1876
2097
|
}
|
|
1877
2098
|
function selectPruneTargets(backups, keep, protectedIds, force = false) {
|
|
1878
2099
|
if (!Number.isInteger(keep) || keep < 1)
|
|
@@ -1881,6 +2102,40 @@ function selectPruneTargets(backups, keep, protectedIds, force = false) {
|
|
|
1881
2102
|
const deletable = force ? sorted : sorted.filter((backup) => !protectedIds.has(backup.backupId));
|
|
1882
2103
|
return deletable.slice(keep).map((backup) => backup.backupId);
|
|
1883
2104
|
}
|
|
2105
|
+
function planRetention(entries, keep, protectedIds, options = {}) {
|
|
2106
|
+
if (!Number.isInteger(keep) || keep < 1)
|
|
2107
|
+
throw new Error("Backup retention must be an integer of at least 1.");
|
|
2108
|
+
const automatic = options.automatic ?? true;
|
|
2109
|
+
const warnings = [];
|
|
2110
|
+
const preserve = [];
|
|
2111
|
+
let canPrune = true;
|
|
2112
|
+
for (const entry of entries) {
|
|
2113
|
+
if (entry.kind === "verified")
|
|
2114
|
+
continue;
|
|
2115
|
+
preserve.push({ entryName: entry.entryName, reason: entry.message });
|
|
2116
|
+
if (entry.kind === "legacy-unsigned" || entry.kind === "legacy-adoption")
|
|
2117
|
+
continue;
|
|
2118
|
+
warnings.push({
|
|
2119
|
+
code: entry.code,
|
|
2120
|
+
severity: entry.kind === "unreadable" ? "blocker" : "warning",
|
|
2121
|
+
resource: entry.entryName,
|
|
2122
|
+
message: entry.message
|
|
2123
|
+
});
|
|
2124
|
+
if (automatic)
|
|
2125
|
+
canPrune = false;
|
|
2126
|
+
}
|
|
2127
|
+
const verified = entries.filter((entry) => entry.kind === "verified").map((entry) => entry.backup).sort((a, b) => a.createdAt < b.createdAt ? 1 : a.createdAt > b.createdAt ? -1 : 0);
|
|
2128
|
+
const removable = verified.filter((backup) => !protectedIds.has(backup.backupId));
|
|
2129
|
+
const remove = removable.slice(keep).map((backup) => backup.backupId);
|
|
2130
|
+
const removeSet = new Set(remove);
|
|
2131
|
+
return {
|
|
2132
|
+
keep: verified.filter((backup) => !removeSet.has(backup.backupId)).map((backup) => backup.backupId),
|
|
2133
|
+
remove,
|
|
2134
|
+
preserve,
|
|
2135
|
+
warnings,
|
|
2136
|
+
canPrune
|
|
2137
|
+
};
|
|
2138
|
+
}
|
|
1884
2139
|
function parseKeep(flags) {
|
|
1885
2140
|
const value = stringFlag(flags, "keep") ?? String(BACKUP_RETENTION_KEEP);
|
|
1886
2141
|
const keep = Number(value);
|
|
@@ -1888,25 +2143,53 @@ function parseKeep(flags) {
|
|
|
1888
2143
|
throw new Error("--keep must be an integer of at least 1.");
|
|
1889
2144
|
return keep;
|
|
1890
2145
|
}
|
|
1891
|
-
|
|
1892
|
-
const backups = await enumerateBackups(directory);
|
|
2146
|
+
function retentionProtectedIds(state) {
|
|
1893
2147
|
const protectedIds = new Set;
|
|
1894
2148
|
if (state.rollback?.backupId)
|
|
1895
2149
|
protectedIds.add(state.rollback.backupId);
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
if (
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
|
|
2150
|
+
const pending = state.incompleteOperation;
|
|
2151
|
+
if (pending?.backupId)
|
|
2152
|
+
protectedIds.add(pending.backupId);
|
|
2153
|
+
if (pending?.previousBackupId)
|
|
2154
|
+
protectedIds.add(pending.previousBackupId);
|
|
2155
|
+
return protectedIds;
|
|
2156
|
+
}
|
|
2157
|
+
async function pruneBackups(directory, state, keep, force, options = {}) {
|
|
2158
|
+
const automatic = options.automatic ?? !force;
|
|
2159
|
+
const log = options.log ?? ((message) => console.log(message));
|
|
2160
|
+
const warn = options.warn ?? ((message) => console.warn(message));
|
|
2161
|
+
const inventory = await inventoryBackups(directory);
|
|
2162
|
+
if (inventory.storeErrors.length > 0) {
|
|
2163
|
+
if (automatic)
|
|
2164
|
+
return { removed: 0, deferred: inventory.storeErrors };
|
|
2165
|
+
throw new Error(inventory.storeErrors.map((diagnostic) => diagnostic.message).join(" "));
|
|
2166
|
+
}
|
|
2167
|
+
const plan = planRetention(inventory.entries, keep, retentionProtectedIds(state), { automatic });
|
|
2168
|
+
if (!plan.canPrune && !force) {
|
|
2169
|
+
warn("Automatic backup pruning was deferred because the store contains entries that need inspection.");
|
|
2170
|
+
for (const warning of plan.warnings)
|
|
2171
|
+
warn(` ${warning.code}: ${warning.message}`);
|
|
2172
|
+
return { removed: 0, deferred: plan.warnings };
|
|
2173
|
+
}
|
|
2174
|
+
const verified = new Map(inventory.entries.filter((entry) => entry.kind === "verified").map((entry) => [entry.backup.backupId, entry.backup]));
|
|
2175
|
+
if (force && plan.remove.length > 0) {
|
|
2176
|
+
log(`--force --yes will permanently delete: ${plan.remove.join(", ")}`);
|
|
2177
|
+
}
|
|
2178
|
+
let removed = 0;
|
|
2179
|
+
for (const backupId of plan.remove) {
|
|
2180
|
+
const backup = verified.get(backupId);
|
|
1904
2181
|
if (!backup)
|
|
1905
2182
|
throw new Error(`Retention selected backup ${backupId}, but its verified path disappeared before deletion.`);
|
|
1906
|
-
await
|
|
1907
|
-
|
|
2183
|
+
const revalidated = await inspectBackupEntry(deploymentPaths(directory).backups, backupId, directory);
|
|
2184
|
+
const current = revalidated.kind === "verified" ? revalidated.backup : undefined;
|
|
2185
|
+
if (!current || current.path !== backup.path || current.dataSha256 !== backup.dataSha256) {
|
|
2186
|
+
throw new Error(`Backup ${backupId} changed while pruning was in progress. Aborting retention before deletion.`);
|
|
2187
|
+
}
|
|
2188
|
+
await removeEnumeratedBackupArtifact(directory, current);
|
|
2189
|
+
log(`Deleted backup ${backupId} at ${current.path}`);
|
|
2190
|
+
removed += 1;
|
|
1908
2191
|
}
|
|
1909
|
-
return
|
|
2192
|
+
return { removed, deferred: plan.warnings };
|
|
1910
2193
|
}
|
|
1911
2194
|
function assertDeploymentIdentity(state, env) {
|
|
1912
2195
|
const expected = {
|
|
@@ -2314,6 +2597,7 @@ async function createBackup(directory, state, env, options = {}) {
|
|
|
2314
2597
|
}
|
|
2315
2598
|
}
|
|
2316
2599
|
async function readManifest(backupPath, authenticatedForDirectory) {
|
|
2600
|
+
await lifecycleFaults.beforeArchiveRead?.();
|
|
2317
2601
|
const manifestContents = await readText(join(backupPath, "manifest.json"));
|
|
2318
2602
|
if (authenticatedForDirectory)
|
|
2319
2603
|
await assertBackupAuthentication(authenticatedForDirectory, backupPath, manifestContents);
|
|
@@ -2418,7 +2702,7 @@ async function restoreBackupData(directory, state, env, backupPath, options = {}
|
|
|
2418
2702
|
}
|
|
2419
2703
|
function stateFromEnv(directory, env, mode, operation, digest) {
|
|
2420
2704
|
return {
|
|
2421
|
-
schemaVersion:
|
|
2705
|
+
schemaVersion: writeStateSchema(),
|
|
2422
2706
|
mode,
|
|
2423
2707
|
composeProject: env.OR3_COMPOSE_PROJECT,
|
|
2424
2708
|
volumeName: env.OR3_VOLUME_NAME,
|
|
@@ -2441,16 +2725,16 @@ function help() {
|
|
|
2441
2725
|
Usage:
|
|
2442
2726
|
npx @or3/cloud init [directory] --local
|
|
2443
2727
|
npx @or3/cloud init [directory] --public --domain <hostname>
|
|
2444
|
-
npx @or3/cloud update [--to <exact-version>]
|
|
2445
|
-
npx @or3/cloud backup [list|prune [--keep <n>]|export <backup-id> <destination-dir>]
|
|
2728
|
+
npx @or3/cloud update [--to <exact-version>] [--dry-run] [--json]
|
|
2729
|
+
npx @or3/cloud backup [list [--json]|prune [--keep <n>]|export <backup-id> <destination-dir>]
|
|
2446
2730
|
npx @or3/cloud restore <backup-id-or-path> --yes
|
|
2447
2731
|
npx @or3/cloud rollback --yes
|
|
2448
2732
|
npx @or3/cloud credentials reset --yes [--owner-password-file <path> --admin-password-file <path>]
|
|
2449
2733
|
npx @or3/cloud doctor
|
|
2450
|
-
npx @or3/cloud verify [--public] [--verification-email <email> --verification-password-file <path>]
|
|
2451
|
-
npx @or3/cloud recover
|
|
2734
|
+
npx @or3/cloud verify [--read-only] [--public] [--verification-email <email> --verification-password-file <path>]
|
|
2735
|
+
npx @or3/cloud recover [--dry-run | --finish | --restore --yes]
|
|
2452
2736
|
npx @or3/cloud adopt --from <v1-directory> [directory]
|
|
2453
|
-
npx @or3/cloud status
|
|
2737
|
+
npx @or3/cloud status [--json]
|
|
2454
2738
|
npx @or3/cloud logs [--tail <n>] [service]
|
|
2455
2739
|
npx @or3/cloud start | stop | restart
|
|
2456
2740
|
npx @or3/cloud remove [--purge-data --yes]
|
|
@@ -2468,6 +2752,11 @@ Options:
|
|
|
2468
2752
|
--force Prune backups even when referenced by the rollback point
|
|
2469
2753
|
--tail <n> Log lines to show (default: 200)
|
|
2470
2754
|
--public Require verification through the public HTTPS origin
|
|
2755
|
+
--read-only Verify without the lease, login, storage, or database writes
|
|
2756
|
+
--dry-run Preview an update or recovery without changing anything
|
|
2757
|
+
--finish Commit a proven completed replacement without restoring data
|
|
2758
|
+
--restore Explicitly restore the recorded snapshot (requires --yes)
|
|
2759
|
+
--json Emit one machine-readable result object on stdout
|
|
2471
2760
|
--purge-data Remove data volumes and managed files (with remove)
|
|
2472
2761
|
--yes Confirm a destructive restore, rollback, credentials reset, or purge
|
|
2473
2762
|
--help Show this help
|
|
@@ -2557,7 +2846,8 @@ ${existing.stderr}`)) {
|
|
|
2557
2846
|
await clearPending(directory, state);
|
|
2558
2847
|
console.log(`
|
|
2559
2848
|
OR3 Cloud ${version} is running at ${mode === "public" ? `https://${domain}` : `http://127.0.0.1:${port}`}`);
|
|
2560
|
-
console.log(`Credentials were written to ${join(directory, ".or3-initial-credentials")} (mode 0600).
|
|
2849
|
+
console.log(`Credentials were written to ${join(directory, ".or3-initial-credentials")} (mode 0600). Move them to protected storage (a password manager), then remove that file.`);
|
|
2850
|
+
console.log('The bootstrap owner signs in to the app; the admin account manages the deployment. Rotate either later with "npx @or3/cloud credentials reset --yes".');
|
|
2561
2851
|
console.log(`
|
|
2562
2852
|
Check: cd ${quote(directory)} && npx @or3/cloud doctor`);
|
|
2563
2853
|
if (operator)
|
|
@@ -2572,22 +2862,179 @@ Check: cd ${quote(directory)} && npx @or3/cloud doctor`);
|
|
|
2572
2862
|
The new deployment files were preserved at ${directory}.`);
|
|
2573
2863
|
}
|
|
2574
2864
|
}
|
|
2575
|
-
|
|
2865
|
+
function assertStateSchemaWritable(state) {
|
|
2866
|
+
const writer = writeStateSchema();
|
|
2867
|
+
if (state.schemaVersion > writer) {
|
|
2868
|
+
throw new Error(`This CLI reads managed state schema ${state.schemaVersion} but is qualified to write schema ${writer}. Run the compatible exact-version @or3/cloud CLI for this deployment; do not edit or hand-migrate managed state.`);
|
|
2869
|
+
}
|
|
2870
|
+
}
|
|
2871
|
+
async function loadManaged(directory = process.cwd(), options = {}) {
|
|
2576
2872
|
const resolved = resolve(directory);
|
|
2577
2873
|
const state = await readState(resolved);
|
|
2874
|
+
if (options.writable !== false)
|
|
2875
|
+
assertStateSchemaWritable(state);
|
|
2578
2876
|
const env = parseEnv(await readText(deploymentPaths(resolved).env));
|
|
2579
2877
|
assertDeploymentIdentity(state, env);
|
|
2580
2878
|
assertDeploymentDirectoryIdentity(resolved, state);
|
|
2581
2879
|
return { directory: resolved, state, env };
|
|
2582
2880
|
}
|
|
2881
|
+
async function observeDeployment(directory, options = {}) {
|
|
2882
|
+
const resolved = resolve(directory);
|
|
2883
|
+
const observedAt = now();
|
|
2884
|
+
let state = null;
|
|
2885
|
+
let stateError = null;
|
|
2886
|
+
try {
|
|
2887
|
+
state = await readState(resolved);
|
|
2888
|
+
} catch (error) {
|
|
2889
|
+
stateError = redact(error instanceof Error ? error.message : String(error));
|
|
2890
|
+
}
|
|
2891
|
+
let env = null;
|
|
2892
|
+
let envError = null;
|
|
2893
|
+
try {
|
|
2894
|
+
env = parseEnv(await readText(deploymentPaths(resolved).env));
|
|
2895
|
+
} catch (error) {
|
|
2896
|
+
envError = redact(error instanceof Error ? error.message : String(error));
|
|
2897
|
+
}
|
|
2898
|
+
const leasePath = deploymentPaths(resolved).lease;
|
|
2899
|
+
let lease = { status: "none" };
|
|
2900
|
+
if (await fileExists(leasePath)) {
|
|
2901
|
+
const owner = await readLeaseOwner(leasePath);
|
|
2902
|
+
if (!owner)
|
|
2903
|
+
lease = { status: "unreadable" };
|
|
2904
|
+
else if (cliLeaseOwnerIsGone(owner) || dashboardLeaseOwnerIsStale(owner))
|
|
2905
|
+
lease = { status: "stale", owner };
|
|
2906
|
+
else
|
|
2907
|
+
lease = { status: "active", owner };
|
|
2908
|
+
}
|
|
2909
|
+
let backups = null;
|
|
2910
|
+
const backupErrors = [];
|
|
2911
|
+
try {
|
|
2912
|
+
backups = await inventoryBackups(resolved);
|
|
2913
|
+
backupErrors.push(...backups.storeErrors);
|
|
2914
|
+
} catch (error) {
|
|
2915
|
+
backupErrors.push({ code: "backup-store-unreadable", severity: "blocker", message: redact(error instanceof Error ? error.message : String(error)) });
|
|
2916
|
+
}
|
|
2917
|
+
const checkDocker = options.checkDocker ?? true;
|
|
2918
|
+
const checkImage = options.checkImage ?? true;
|
|
2919
|
+
let docker = false;
|
|
2920
|
+
let recordedImageDigest = state?.imageDigest ?? null;
|
|
2921
|
+
let actualImageDigest = null;
|
|
2922
|
+
let identityMatches = null;
|
|
2923
|
+
if (checkDocker) {
|
|
2924
|
+
try {
|
|
2925
|
+
await ensureDocker();
|
|
2926
|
+
docker = true;
|
|
2927
|
+
} catch {
|
|
2928
|
+
docker = false;
|
|
2929
|
+
}
|
|
2930
|
+
}
|
|
2931
|
+
if (checkImage && docker && state?.image) {
|
|
2932
|
+
try {
|
|
2933
|
+
actualImageDigest = await runningContainerImageDigest(resolved, state.mode, state.image);
|
|
2934
|
+
identityMatches = actualImageDigest === state.imageDigest;
|
|
2935
|
+
} catch {
|
|
2936
|
+
actualImageDigest = null;
|
|
2937
|
+
identityMatches = null;
|
|
2938
|
+
}
|
|
2939
|
+
}
|
|
2940
|
+
const partial = Boolean(stateError || envError || backupErrors.length > 0 || checkDocker && !docker);
|
|
2941
|
+
return {
|
|
2942
|
+
directory: resolved,
|
|
2943
|
+
observedAt,
|
|
2944
|
+
state,
|
|
2945
|
+
stateError,
|
|
2946
|
+
env,
|
|
2947
|
+
envError,
|
|
2948
|
+
lease,
|
|
2949
|
+
docker,
|
|
2950
|
+
recordedImageDigest,
|
|
2951
|
+
actualImageDigest,
|
|
2952
|
+
identityMatches,
|
|
2953
|
+
backups,
|
|
2954
|
+
backupErrors,
|
|
2955
|
+
changing: lease.status === "active",
|
|
2956
|
+
partial
|
|
2957
|
+
};
|
|
2958
|
+
}
|
|
2959
|
+
async function runningContainerImageDigest(directory, mode, fallbackImage) {
|
|
2960
|
+
let containerId = "";
|
|
2961
|
+
try {
|
|
2962
|
+
const ps = await run("docker", composeArgs(directory, mode, ["ps", "-q", "or3"]), directory);
|
|
2963
|
+
containerId = ps.ok ? ps.stdout.trim() : "";
|
|
2964
|
+
} catch {
|
|
2965
|
+
containerId = "";
|
|
2966
|
+
}
|
|
2967
|
+
if (!containerId)
|
|
2968
|
+
return await imageDigest(fallbackImage);
|
|
2969
|
+
const image = await run("docker", ["inspect", "--format", "{{.Image}}", containerId], directory);
|
|
2970
|
+
const imageId = image.ok ? image.stdout.trim() : "";
|
|
2971
|
+
if (!imageId)
|
|
2972
|
+
throw new Error("Could not inspect the running OR3 container image.");
|
|
2973
|
+
const digests = await run("docker", ["image", "inspect", "--format", "{{json .RepoDigests}}", imageId], directory);
|
|
2974
|
+
if (!digests.ok)
|
|
2975
|
+
throw new Error("Could not inspect repository digests for the running OR3 container.");
|
|
2976
|
+
let values;
|
|
2977
|
+
try {
|
|
2978
|
+
values = JSON.parse(digests.stdout.trim());
|
|
2979
|
+
} catch {
|
|
2980
|
+
throw new Error("The running OR3 container has no readable repository digests.");
|
|
2981
|
+
}
|
|
2982
|
+
const match = values.map((value) => value.match(/@(sha256:[0-9a-f]{64})$/i)?.[1]).find((value) => Boolean(value));
|
|
2983
|
+
if (!match)
|
|
2984
|
+
throw new Error("The running OR3 container image is not digest-qualified.");
|
|
2985
|
+
return match;
|
|
2986
|
+
}
|
|
2987
|
+
function publicStateProjection(state) {
|
|
2988
|
+
const pending = state.incompleteOperation;
|
|
2989
|
+
return {
|
|
2990
|
+
schemaVersion: state.schemaVersion,
|
|
2991
|
+
mode: state.mode,
|
|
2992
|
+
appVersion: state.appVersion,
|
|
2993
|
+
image: state.image,
|
|
2994
|
+
imageDigest: state.imageDigest,
|
|
2995
|
+
domain: state.domain ?? null,
|
|
2996
|
+
port: state.port,
|
|
2997
|
+
deploymentId: state.deploymentId ?? null,
|
|
2998
|
+
lastSuccessfulOperation: state.lastSuccessfulOperation,
|
|
2999
|
+
updatedAt: state.updatedAt,
|
|
3000
|
+
rollback: state.rollback ? { appVersion: state.rollback.appVersion, imageDigest: state.rollback.imageDigest, backupId: state.rollback.backupId, createdAt: state.rollback.createdAt } : null,
|
|
3001
|
+
incompleteOperation: pending ? {
|
|
3002
|
+
id: pending.id,
|
|
3003
|
+
operation: pending.operation,
|
|
3004
|
+
phase: pending.phase ?? null,
|
|
3005
|
+
origin: pending.origin ?? null,
|
|
3006
|
+
dashboardJobId: pending.dashboardJobId ?? null,
|
|
3007
|
+
targetVersion: pending.targetVersion ?? null,
|
|
3008
|
+
targetImageDigest: pending.targetImageDigest ?? null
|
|
3009
|
+
} : null,
|
|
3010
|
+
lastReceipt: state.lastReceipt ?? null,
|
|
3011
|
+
lastError: state.lastError ? redact(state.lastError) : null
|
|
3012
|
+
};
|
|
3013
|
+
}
|
|
3014
|
+
function observationFindings(observation) {
|
|
3015
|
+
const findings = [];
|
|
3016
|
+
if (observation.stateError)
|
|
3017
|
+
findings.push({ code: "state-unreadable", severity: "blocker", resource: deploymentPaths(observation.directory).state, message: observation.stateError });
|
|
3018
|
+
if (observation.envError)
|
|
3019
|
+
findings.push({ code: "env-unreadable", severity: "blocker", resource: deploymentPaths(observation.directory).env, message: observation.envError });
|
|
3020
|
+
if (observation.lease.status === "active") {
|
|
3021
|
+
findings.push({ code: "operation-in-progress", severity: "info", message: `A ${observation.lease.owner.origin} operation (${observation.lease.owner.command}) is active; this observation is not authoritative.` });
|
|
3022
|
+
}
|
|
3023
|
+
if (observation.lease.status === "unreadable")
|
|
3024
|
+
findings.push({ code: "lease-unreadable", severity: "warning", message: "The deployment lease owner record is unreadable; run doctor before any mutation." });
|
|
3025
|
+
if (observation.identityMatches === false) {
|
|
3026
|
+
findings.push({ code: "image-digest-mismatch", severity: "blocker", resource: observation.state?.image, message: `Recorded digest ${observation.recordedImageDigest} does not match the local image ${observation.actualImageDigest}.` });
|
|
3027
|
+
}
|
|
3028
|
+
return findings;
|
|
3029
|
+
}
|
|
2583
3030
|
function operationToStateOperation(operation, fallback) {
|
|
2584
3031
|
if (operation === "init" || operation === "update" || operation === "adopt")
|
|
2585
3032
|
return operation;
|
|
2586
3033
|
return fallback;
|
|
2587
3034
|
}
|
|
2588
3035
|
async function commitRecoveredState(directory, previous, next) {
|
|
2589
|
-
await removeOperationRecord(directory, previous.incompleteOperation?.id);
|
|
2590
3036
|
await writeState(directory, next);
|
|
3037
|
+
await removeOperationRecordSafely(directory, previous.incompleteOperation?.id);
|
|
2591
3038
|
}
|
|
2592
3039
|
async function commitPreMutationRecovery(directory, state, message) {
|
|
2593
3040
|
const restoredEnv = parseEnv(await readText(deploymentPaths(directory).env));
|
|
@@ -2597,14 +3044,293 @@ async function commitPreMutationRecovery(directory, state, message) {
|
|
|
2597
3044
|
await commitRecoveredState(directory, state, recovered);
|
|
2598
3045
|
return recovered;
|
|
2599
3046
|
}
|
|
2600
|
-
|
|
3047
|
+
function decideRecoveryAction(state) {
|
|
3048
|
+
const pending = state.incompleteOperation;
|
|
3049
|
+
if (!pending) {
|
|
3050
|
+
const handoff = state.lastReceipt?.operatorHandoff;
|
|
3051
|
+
if (handoff === "pending" || handoff === "needs-attention") {
|
|
3052
|
+
return { action: "reconcile-handoff", detail: `The application is complete but the dashboard operator handoff is ${handoff}. Reconcile only the recorded handoff.` };
|
|
3053
|
+
}
|
|
3054
|
+
return { action: "none", detail: "No incomplete operation is recorded." };
|
|
3055
|
+
}
|
|
3056
|
+
if (pending.operation === "update" && pending.phase === "target-ready" && pending.evidence) {
|
|
3057
|
+
return { action: "finish", detail: "A completed replacement carries durable target-ready evidence and can be committed without restoring data." };
|
|
3058
|
+
}
|
|
3059
|
+
if (pending.phase === "prepared" || pending.phase === "snapshot-created") {
|
|
3060
|
+
return { action: "resume", detail: `The ${pending.operation} had not replaced data; resume the known-good source.` };
|
|
3061
|
+
}
|
|
3062
|
+
if (pending.operation === "update" || pending.operation === "restore" || pending.operation === "rollback") {
|
|
3063
|
+
const snapshotId = pending.operation === "update" ? pending.backupId : pending.previousBackupId ?? pending.backupId;
|
|
3064
|
+
return {
|
|
3065
|
+
action: "require-explicit-restore",
|
|
3066
|
+
detail: `The ${pending.operation} may have changed data and has no completion proof. Health alone cannot authorize adoption.`,
|
|
3067
|
+
snapshotId
|
|
3068
|
+
};
|
|
3069
|
+
}
|
|
3070
|
+
return { action: "resume", detail: `Resume the ${pending.operation} using its durable evidence.` };
|
|
3071
|
+
}
|
|
3072
|
+
function assertRecoverFlags(flags) {
|
|
3073
|
+
const dryRun = boolFlag(flags, "dry-run");
|
|
3074
|
+
const finish = boolFlag(flags, "finish");
|
|
3075
|
+
const restore = boolFlag(flags, "restore");
|
|
3076
|
+
if (finish && restore)
|
|
3077
|
+
throw new Error("Use either --finish or --restore, not both.");
|
|
3078
|
+
if (dryRun && (finish || restore))
|
|
3079
|
+
throw new Error("--dry-run cannot be combined with --finish or --restore.");
|
|
3080
|
+
if (restore && !boolFlag(flags, "yes")) {
|
|
3081
|
+
throw new Error("recover --restore replaces live data. Review `npx @or3/cloud recover --dry-run` first, then re-run with `recover --restore --yes`.");
|
|
3082
|
+
}
|
|
3083
|
+
}
|
|
3084
|
+
async function reconcileOperatorHandoff(directory, state, report = console.log) {
|
|
3085
|
+
const env = parseEnv(await readText(deploymentPaths(directory).env));
|
|
3086
|
+
if (env.OR3_DASHBOARD_UPDATES_ENABLED !== "true") {
|
|
3087
|
+
const detail2 = "Dashboard updates are disabled in this deployment; no operator handoff needs reconciliation.";
|
|
3088
|
+
report(detail2);
|
|
3089
|
+
return detail2;
|
|
3090
|
+
}
|
|
3091
|
+
const receipt = state.lastReceipt;
|
|
3092
|
+
const jobId = receipt?.dashboardJobId ?? process.env.OR3_DASHBOARD_UPDATE_JOB_ID?.trim();
|
|
3093
|
+
if (!jobId) {
|
|
3094
|
+
throw new Error("The completed update recorded no dashboard job for its operator handoff. Re-run the host CLI update, or disable dashboard updates until the operator is restored.");
|
|
3095
|
+
}
|
|
3096
|
+
await scheduleDashboardOperatorHandoff(directory, env, jobId);
|
|
3097
|
+
if (receipt) {
|
|
3098
|
+
receipt.operatorHandoff = "pending";
|
|
3099
|
+
await writeState(directory, state);
|
|
3100
|
+
await exportTerminalReceipt(directory, receipt).catch(() => {
|
|
3101
|
+
return;
|
|
3102
|
+
});
|
|
3103
|
+
}
|
|
3104
|
+
const detail = `Rescheduled the dashboard operator handoff for job ${jobId.slice(0, 8)}. The successor operator marks it verified once it is running.`;
|
|
3105
|
+
report(detail);
|
|
3106
|
+
return detail;
|
|
3107
|
+
}
|
|
3108
|
+
async function collectTargetReadyChecks(directory, mode, env) {
|
|
3109
|
+
const checks = [];
|
|
3110
|
+
let containerId;
|
|
3111
|
+
const container = await run("docker", composeArgs(directory, mode, ["ps", "-q", "or3"]), directory);
|
|
3112
|
+
containerId = container.ok ? container.stdout.trim() || undefined : undefined;
|
|
3113
|
+
checks.push(containerId ? { code: "container-binding", status: "passed", detail: `Observed target container ${containerId}.` } : { code: "container-binding", status: "failed", detail: "Could not resolve the running target container." });
|
|
3114
|
+
const databaseCheck = await run("docker", [
|
|
3115
|
+
...composeArgs(directory, mode, ["exec", "-T", "or3", ...containerNodeCommand(VERIFY_DATABASES_SCRIPT)])
|
|
3116
|
+
], directory);
|
|
3117
|
+
let databasesOk = false;
|
|
3118
|
+
if (databaseCheck.ok) {
|
|
3119
|
+
try {
|
|
3120
|
+
const databases = JSON.parse(databaseCheck.stdout.trim());
|
|
3121
|
+
databasesOk = databases.length === 2 && databases.every((entry) => entry.quickCheck === "ok" && entry.tables >= 1);
|
|
3122
|
+
} catch {
|
|
3123
|
+
databasesOk = false;
|
|
3124
|
+
}
|
|
3125
|
+
}
|
|
3126
|
+
checks.push(databasesOk ? { code: "database-integrity", status: "passed", detail: "auth.sqlite and sync.sqlite quick_check passed with managed ownership." } : { code: "database-integrity", status: "failed", detail: `SQLite integrity or ownership verification failed. ${redact(databaseCheck.stderr.trim(), secretValues(env))}` });
|
|
3127
|
+
const baseUrl = new URL(mode === "public" ? `https://${env.OR3_PUBLIC_DOMAIN}` : `http://127.0.0.1:${env.OR3_PORT}`);
|
|
3128
|
+
try {
|
|
3129
|
+
validateVerificationHealth(await verificationJson(baseUrl, "/api/health?deep=true"));
|
|
3130
|
+
checks.push({ code: "public-health", status: "passed", detail: `${baseUrl.origin} deep health reports the managed profile.` });
|
|
3131
|
+
} catch (error) {
|
|
3132
|
+
checks.push({ code: "public-health", status: "failed", detail: redact(error instanceof Error ? error.message : String(error)) });
|
|
3133
|
+
}
|
|
3134
|
+
return { containerId, checks };
|
|
3135
|
+
}
|
|
3136
|
+
async function finishTargetReadyUpdate(directory, state, env, report = console.log) {
|
|
3137
|
+
const pending = state.incompleteOperation;
|
|
3138
|
+
if (!pending?.evidence || !pending.targetVersion || !pending.targetImage) {
|
|
3139
|
+
throw new Error("recover --finish requires a recorded target-ready milestone; none is present.");
|
|
3140
|
+
}
|
|
3141
|
+
const targetEnv = parseEnv(await readText(deploymentPaths(directory).env));
|
|
3142
|
+
if (targetEnv.OR3_VERSION !== pending.targetVersion || targetEnv.OR3_IMAGE !== pending.targetImage) {
|
|
3143
|
+
throw new Error("The live .env no longer matches the recorded target-ready identities; refusing to finish.");
|
|
3144
|
+
}
|
|
3145
|
+
const expectedTargetDigest = pending.targetImageDigest ?? await imageDigest(targetEnv.OR3_IMAGE);
|
|
3146
|
+
await pullAndRequireImage(targetEnv.OR3_IMAGE, expectedTargetDigest, "Target deployment");
|
|
3147
|
+
await assertRunningAppImage(directory, state.mode, targetEnv.OR3_IMAGE);
|
|
3148
|
+
const actualDigest = await imageDigest(targetEnv.OR3_IMAGE);
|
|
3149
|
+
if (actualDigest !== pending.evidence.imageDigest) {
|
|
3150
|
+
throw new Error(`The running target image ${actualDigest} does not match the recorded target-ready proof ${pending.evidence.imageDigest}.`);
|
|
3151
|
+
}
|
|
3152
|
+
const configurationSha256 = await sha256File(deploymentPaths(directory).env);
|
|
3153
|
+
if (configurationSha256 !== pending.evidence.configurationSha256) {
|
|
3154
|
+
throw new Error("The managed configuration changed after the target-ready milestone; refusing to finish. Inspect the change, or run `recover --restore --yes` to return to the recorded snapshot.");
|
|
3155
|
+
}
|
|
3156
|
+
const assets = await installedManagedAssetChecksums(directory, state.mode);
|
|
3157
|
+
for (const [name, expected] of Object.entries(pending.evidence.managedAssetSha256)) {
|
|
3158
|
+
if (assets[name] !== expected) {
|
|
3159
|
+
throw new Error(`Managed asset ${name} changed after the target-ready milestone; refusing to finish.`);
|
|
3160
|
+
}
|
|
3161
|
+
}
|
|
3162
|
+
const expectedDeploymentId = targetEnv.OR3_DEPLOYMENT_ID ?? pending.targetDeploymentId;
|
|
3163
|
+
if (pending.evidence.deploymentId && expectedDeploymentId && pending.evidence.deploymentId !== expectedDeploymentId) {
|
|
3164
|
+
throw new Error("The recorded target-ready deployment identity no longer matches the live deployment.");
|
|
3165
|
+
}
|
|
3166
|
+
await recordedBackupPath(directory, pending);
|
|
3167
|
+
for (const required of ["container-binding", "database-integrity", "public-health"]) {
|
|
3168
|
+
const recorded = pending.evidence.checks.find((check) => check.code === required);
|
|
3169
|
+
if (!recorded || recorded.status !== "passed") {
|
|
3170
|
+
throw new Error(`The recorded target-ready proof is missing ${required}; refusing to finish. Run "npx @or3/cloud recover --restore --yes" to return to the recorded snapshot.`);
|
|
3171
|
+
}
|
|
3172
|
+
}
|
|
3173
|
+
await waitForDeepHealth(directory, state.mode, secretValues(targetEnv));
|
|
3174
|
+
const revalidated = await collectTargetReadyChecks(directory, state.mode, targetEnv);
|
|
3175
|
+
if (!revalidated.containerId || revalidated.containerId !== pending.evidence.containerId) {
|
|
3176
|
+
throw new Error("The running target container no longer matches the recorded target-ready proof; refusing to finish.");
|
|
3177
|
+
}
|
|
3178
|
+
const failedChecks = revalidated.checks.filter((check) => check.status !== "passed");
|
|
3179
|
+
if (failedChecks.length > 0) {
|
|
3180
|
+
throw new Error(`Target verification failed at finish (${failedChecks.map((check) => check.code).join(", ")}); refusing to finish.`);
|
|
3181
|
+
}
|
|
3182
|
+
const sourceVersion = state.appVersion;
|
|
3183
|
+
const sourceImage = state.image;
|
|
3184
|
+
const sourceDigest = state.imageDigest;
|
|
3185
|
+
const digest = actualDigest;
|
|
3186
|
+
const recordedDashboardJobId = pending.dashboardJobId ?? (process.env.OR3_DASHBOARD_UPDATE_JOB_ID?.trim() || undefined);
|
|
3187
|
+
const receipt = {
|
|
3188
|
+
schemaVersion: 1,
|
|
3189
|
+
operationId: pending.id,
|
|
3190
|
+
dashboardJobId: recordedDashboardJobId,
|
|
3191
|
+
cliVersion: PACKAGE_VERSION,
|
|
3192
|
+
source: { appVersion: sourceVersion, image: sourceImage, imageDigest: sourceDigest, sourceRevision: packagedSourceRevision(sourceVersion), operatorImageDigest: expectedOperatorImageDigest(sourceVersion) },
|
|
3193
|
+
target: { appVersion: pending.targetVersion, image: pending.targetImage, imageDigest: digest, sourceRevision: packagedSourceRevision(pending.targetVersion), operatorImageDigest: expectedOperatorImageDigest(pending.targetVersion) },
|
|
3194
|
+
observed: { appVersion: pending.targetVersion, image: pending.targetImage, imageDigest: digest },
|
|
3195
|
+
completedAt: now(),
|
|
3196
|
+
rollbackBackupId: pending.backupId ?? "",
|
|
3197
|
+
checks: revalidated.checks,
|
|
3198
|
+
warnings: [],
|
|
3199
|
+
phaseDurationsMs: {},
|
|
3200
|
+
operatorHandoff: "not-required"
|
|
3201
|
+
};
|
|
3202
|
+
const next = stateFromEnv(directory, targetEnv, state.mode, "update", digest);
|
|
3203
|
+
next.rollback = {
|
|
3204
|
+
appVersion: sourceVersion,
|
|
3205
|
+
image: sourceImage,
|
|
3206
|
+
imageDigest: sourceDigest,
|
|
3207
|
+
backupId: pending.backupId ?? "",
|
|
3208
|
+
createdAt: now()
|
|
3209
|
+
};
|
|
3210
|
+
next.lastError = undefined;
|
|
3211
|
+
if (targetEnv.OR3_DASHBOARD_UPDATES_ENABLED === "true" && recordedDashboardJobId) {
|
|
3212
|
+
try {
|
|
3213
|
+
await scheduleDashboardOperatorHandoff(directory, targetEnv, recordedDashboardJobId);
|
|
3214
|
+
receipt.operatorHandoff = "pending";
|
|
3215
|
+
} catch (error) {
|
|
3216
|
+
receipt.operatorHandoff = "needs-attention";
|
|
3217
|
+
receipt.warnings.push({ code: "operator-handoff-schedule-failed", severity: "warning", message: redact(error instanceof Error ? error.message : String(error), secretValues(targetEnv)) });
|
|
3218
|
+
}
|
|
3219
|
+
}
|
|
3220
|
+
const commit = await commitTerminalState(directory, next, receipt);
|
|
3221
|
+
await removeOperationRecordSafely(directory, commit.operationId);
|
|
3222
|
+
const maintenance = [];
|
|
3223
|
+
const prune = await pruneBackups(directory, next, BACKUP_RETENTION_KEEP, false, {
|
|
3224
|
+
automatic: true,
|
|
3225
|
+
log: (message) => report(message),
|
|
3226
|
+
warn: (message) => report(message)
|
|
3227
|
+
}).catch((error) => {
|
|
3228
|
+
maintenance.push({ code: "retention-failed", severity: "warning", message: redact(error instanceof Error ? error.message : String(error), secretValues(targetEnv)) });
|
|
3229
|
+
return { removed: 0, deferred: [] };
|
|
3230
|
+
});
|
|
3231
|
+
maintenance.push(...prune.deferred);
|
|
3232
|
+
await persistReceiptWarnings(directory, next, maintenance);
|
|
3233
|
+
const detail = `Finished the recorded update as OR3 ${pending.targetVersion}. Post-replacement writes were preserved.`;
|
|
3234
|
+
report(detail);
|
|
3235
|
+
for (const warning of [...receipt.warnings, ...maintenance])
|
|
3236
|
+
report(` ⚠ ${warning.code}: ${warning.message}`);
|
|
3237
|
+
return detail;
|
|
3238
|
+
}
|
|
3239
|
+
async function recoverCommand(directory, flags = {}) {
|
|
3240
|
+
assertRecoverFlags(flags);
|
|
3241
|
+
const explicitRestore = boolFlag(flags, "restore");
|
|
3242
|
+
const finishRequested = boolFlag(flags, "finish");
|
|
3243
|
+
const dryRun = boolFlag(flags, "dry-run");
|
|
3244
|
+
if (dryRun) {
|
|
3245
|
+
const observation = await observeDeployment(directory, { checkDocker: true, checkImage: true });
|
|
3246
|
+
const state = observation.state;
|
|
3247
|
+
const decision = state ? decideRecoveryAction(state) : { action: "none", detail: "Managed state is unreadable; only diagnosis is available." };
|
|
3248
|
+
const pending = state?.incompleteOperation;
|
|
3249
|
+
const payload = {
|
|
3250
|
+
schemaVersion: 1,
|
|
3251
|
+
kind: "or3-recover-preview",
|
|
3252
|
+
observedAt: observation.observedAt,
|
|
3253
|
+
directory: observation.directory,
|
|
3254
|
+
partial: observation.partial,
|
|
3255
|
+
inProgress: observation.changing,
|
|
3256
|
+
source: state ? { appVersion: state.appVersion, image: state.image, imageDigest: state.imageDigest } : null,
|
|
3257
|
+
target: pending?.targetVersion ? { appVersion: pending.targetVersion, image: pending.targetImage ?? null, imageDigest: pending.targetImageDigest ?? null } : null,
|
|
3258
|
+
phase: pending?.phase ?? null,
|
|
3259
|
+
operation: pending?.operation ?? null,
|
|
3260
|
+
decision,
|
|
3261
|
+
dataLoss: decision.action === "require-explicit-restore",
|
|
3262
|
+
findings: observationFindings(observation)
|
|
3263
|
+
};
|
|
3264
|
+
if (boolFlag(flags, "json"))
|
|
3265
|
+
console.log(JSON.stringify(payload, null, 2));
|
|
3266
|
+
else {
|
|
3267
|
+
console.log(`OR3 recovery preview for ${observation.directory}`);
|
|
3268
|
+
console.log(` source: ${payload.source ? `OR3 ${payload.source.appVersion} (${payload.source.imageDigest})` : "unknown"}`);
|
|
3269
|
+
console.log(` target: ${payload.target ? `OR3 ${payload.target.appVersion}` : "none"}`);
|
|
3270
|
+
console.log(` operation: ${payload.operation ?? "none"}${payload.phase ? ` (${payload.phase})` : ""}`);
|
|
3271
|
+
console.log(` action: ${decision.action} — ${decision.detail}`);
|
|
3272
|
+
if (payload.dataLoss)
|
|
3273
|
+
console.log(" data loss: yes — restoring the recorded snapshot discards writes made after it.");
|
|
3274
|
+
for (const finding of payload.findings)
|
|
3275
|
+
console.log(` [${finding.severity}] ${finding.code}: ${finding.message}`);
|
|
3276
|
+
console.log(`
|
|
3277
|
+
Preview only: nothing was changed.`);
|
|
3278
|
+
}
|
|
3279
|
+
return;
|
|
3280
|
+
}
|
|
3281
|
+
const asJson = boolFlag(flags, "json");
|
|
3282
|
+
const report = (...args) => {
|
|
3283
|
+
if (asJson)
|
|
3284
|
+
console.error(...args);
|
|
3285
|
+
else
|
|
3286
|
+
console.log(...args);
|
|
3287
|
+
};
|
|
2601
3288
|
await ensureDocker();
|
|
3289
|
+
try {
|
|
3290
|
+
const outcome = await runRecovery(directory, { explicitRestore, finishRequested, report });
|
|
3291
|
+
if (asJson)
|
|
3292
|
+
emitOperationResult(outcome);
|
|
3293
|
+
} catch (error) {
|
|
3294
|
+
if (!asJson)
|
|
3295
|
+
throw error;
|
|
3296
|
+
const message = redact(error instanceof Error ? error.message : String(error));
|
|
3297
|
+
console.error(`OR3 Cloud failed: ${message}`);
|
|
3298
|
+
emitOperationResult({ kind: "blocked", findings: [{ code: "recovery-failed", severity: "blocker", message, nextCommand: "npx @or3/cloud recover --dry-run" }] });
|
|
3299
|
+
process.exitCode = 1;
|
|
3300
|
+
}
|
|
3301
|
+
}
|
|
3302
|
+
async function runRecovery(directory, context) {
|
|
3303
|
+
const { explicitRestore, finishRequested, report } = context;
|
|
2602
3304
|
const loaded = await loadManaged(directory);
|
|
2603
3305
|
const pending = loaded.state.incompleteOperation;
|
|
3306
|
+
const decision = decideRecoveryAction(loaded.state);
|
|
3307
|
+
if (decision.action === "reconcile-handoff") {
|
|
3308
|
+
if (explicitRestore)
|
|
3309
|
+
throw new Error("This deployment completed its application update; only the dashboard operator handoff needs reconciliation. Run `npx @or3/cloud recover --finish` without --restore.");
|
|
3310
|
+
const detail = await reconcileOperatorHandoff(loaded.directory, loaded.state, report);
|
|
3311
|
+
return { kind: "recovered", operation: "update", detail };
|
|
3312
|
+
}
|
|
2604
3313
|
if (!pending) {
|
|
2605
|
-
|
|
2606
|
-
|
|
3314
|
+
if (finishRequested) {
|
|
3315
|
+
if (loaded.state.lastReceipt) {
|
|
3316
|
+
const detail2 = `No incomplete operation is recorded; the last ${loaded.state.lastReceipt.operationId} completion remains authoritative. Nothing to finish.`;
|
|
3317
|
+
report(detail2);
|
|
3318
|
+
return { kind: "no-op", detail: detail2 };
|
|
3319
|
+
}
|
|
3320
|
+
throw new Error("recover --finish requires an incomplete operation, but none is recorded.");
|
|
3321
|
+
}
|
|
3322
|
+
const detail = "No incomplete OR3 Cloud operation is recorded.";
|
|
3323
|
+
report(detail);
|
|
3324
|
+
return { kind: "no-op", detail };
|
|
3325
|
+
}
|
|
3326
|
+
if (finishRequested) {
|
|
3327
|
+
if (decision.action !== "finish") {
|
|
3328
|
+
throw new Error(`recover --finish is not admissible: ${decision.detail} Run "npx @or3/cloud recover --dry-run" to inspect the supported actions.`);
|
|
3329
|
+
}
|
|
3330
|
+
const detail = await finishTargetReadyUpdate(loaded.directory, loaded.state, loaded.env, report);
|
|
3331
|
+
return { kind: "recovered", operation: "update", detail };
|
|
2607
3332
|
}
|
|
3333
|
+
const requiresExplicitRestore = decision.action === "require-explicit-restore";
|
|
2608
3334
|
try {
|
|
2609
3335
|
if (pending.phase === "prepared") {
|
|
2610
3336
|
if (pending.operation === "backup" || pending.operation === "update" || pending.operation === "adopt") {
|
|
@@ -2621,24 +3347,30 @@ async function recoverCommand(directory) {
|
|
|
2621
3347
|
await applyCredentialReset(loaded.directory, loaded.state, nextEnv);
|
|
2622
3348
|
loaded.state.lastError = undefined;
|
|
2623
3349
|
await clearPending(loaded.directory, loaded.state);
|
|
2624
|
-
|
|
2625
|
-
|
|
3350
|
+
const detail2 = "Recovered the incomplete credential reset. Owner and admin credentials were verified inside the OR3 container.";
|
|
3351
|
+
report(detail2);
|
|
3352
|
+
return { kind: "recovered", operation: "credentials-reset", detail: detail2 };
|
|
2626
3353
|
}
|
|
2627
3354
|
if (pending.operation === "restore" || pending.operation === "rollback") {
|
|
2628
3355
|
if (pending.phase === "prepared" || pending.phase === "snapshot-created") {
|
|
2629
3356
|
await startProject(loaded.directory, loaded.state.mode, loaded.env);
|
|
2630
3357
|
loaded.state.lastError = undefined;
|
|
2631
3358
|
await clearPending(loaded.directory, loaded.state);
|
|
2632
|
-
|
|
2633
|
-
|
|
3359
|
+
const detail3 = `Recovered the incomplete ${pending.operation} before data replacement. OR3 ${loaded.state.appVersion} is deeply healthy.`;
|
|
3360
|
+
report(detail3);
|
|
3361
|
+
return { kind: "recovered", operation: pending.operation, detail: detail3 };
|
|
3362
|
+
}
|
|
3363
|
+
if (requiresExplicitRestore && !explicitRestore) {
|
|
3364
|
+
throw new Error(`The incomplete ${pending.operation} may have replaced data and has no completion proof. Nothing was changed. Review "npx @or3/cloud recover --dry-run", then restore deliberately with "npx @or3/cloud recover --restore --yes" (this discards writes made after the recorded snapshot).`);
|
|
2634
3365
|
}
|
|
2635
3366
|
if (!pending.previousBackupPath && !pending.previousBackupId) {
|
|
2636
3367
|
throw new Error(`The incomplete ${pending.operation} has no verified pre-mutation snapshot. Refusing to replay a target that may have been partially restored; inspect the deployment and restore an authenticated backup explicitly.`);
|
|
2637
3368
|
}
|
|
2638
3369
|
const previous = await restorePreMutationSnapshot(loaded.directory, loaded.state, loaded.env);
|
|
2639
3370
|
const recovered2 = await commitPreMutationRecovery(loaded.directory, loaded.state, `Interrupted ${pending.operation} was rolled back to pre-mutation snapshot ${previous.manifest.backupId}.`);
|
|
2640
|
-
|
|
2641
|
-
|
|
3371
|
+
const detail2 = `Recovered the incomplete ${pending.operation} by restoring pre-mutation snapshot ${previous.manifest.backupId}. OR3 ${recovered2.appVersion} is deeply healthy.`;
|
|
3372
|
+
report(detail2);
|
|
3373
|
+
return { kind: "recovered", operation: pending.operation, detail: detail2 };
|
|
2642
3374
|
}
|
|
2643
3375
|
if (pending.operation === "update") {
|
|
2644
3376
|
if (pending.phase === "prepared" || pending.phase === "snapshot-created") {
|
|
@@ -2648,16 +3380,25 @@ async function recoverCommand(directory) {
|
|
|
2648
3380
|
recovered3.rollback = loaded.state.rollback;
|
|
2649
3381
|
recovered3.lastError = undefined;
|
|
2650
3382
|
await commitRecoveredState(loaded.directory, loaded.state, recovered3);
|
|
2651
|
-
|
|
2652
|
-
|
|
3383
|
+
const detail3 = `Recovered the incomplete update before the replacement was applied. OR3 ${recovered3.appVersion} is deeply healthy.`;
|
|
3384
|
+
report(detail3);
|
|
3385
|
+
return { kind: "recovered", operation: "update", detail: detail3 };
|
|
3386
|
+
}
|
|
3387
|
+
if (decision.action === "finish" && !explicitRestore) {
|
|
3388
|
+
const detail3 = await finishTargetReadyUpdate(loaded.directory, loaded.state, loaded.env, report);
|
|
3389
|
+
return { kind: "recovered", operation: "update", detail: detail3 };
|
|
3390
|
+
}
|
|
3391
|
+
if (requiresExplicitRestore && !explicitRestore) {
|
|
3392
|
+
throw new Error(`The incomplete update may have replaced data and has no completion proof. Nothing was changed. Review "npx @or3/cloud recover --dry-run", then restore deliberately with "npx @or3/cloud recover --restore --yes" (this discards writes made after the recorded pre-update snapshot).`);
|
|
2653
3393
|
}
|
|
2654
3394
|
if (!pending.backupPath && !pending.backupId) {
|
|
2655
3395
|
throw new Error("The incomplete update has no verified pre-update snapshot. Refusing to guess which image or data should be live.");
|
|
2656
3396
|
}
|
|
2657
3397
|
const previous = await restorePreMutationSnapshot(loaded.directory, loaded.state, loaded.env);
|
|
2658
3398
|
const recovered2 = await commitPreMutationRecovery(loaded.directory, loaded.state, `Interrupted update was rolled back to pre-update snapshot ${previous.manifest.backupId}.`);
|
|
2659
|
-
|
|
2660
|
-
|
|
3399
|
+
const detail2 = `Recovered the incomplete update by restoring pre-update snapshot ${previous.manifest.backupId}. OR3 ${recovered2.appVersion} is deeply healthy.`;
|
|
3400
|
+
report(detail2);
|
|
3401
|
+
return { kind: "recovered", operation: "update", detail: detail2 };
|
|
2661
3402
|
}
|
|
2662
3403
|
if (pending.operation === "adopt") {
|
|
2663
3404
|
if (!pending.backupId)
|
|
@@ -2673,8 +3414,9 @@ async function recoverCommand(directory) {
|
|
|
2673
3414
|
if (pending.operation === "backup" && pending.initialAppRunning === false) {
|
|
2674
3415
|
loaded.state.lastError = undefined;
|
|
2675
3416
|
await clearPending(loaded.directory, loaded.state);
|
|
2676
|
-
|
|
2677
|
-
|
|
3417
|
+
const detail2 = "Recovered the incomplete backup operation and preserved the intentionally stopped OR3 service.";
|
|
3418
|
+
report(detail2);
|
|
3419
|
+
return { kind: "recovered", operation: "backup", detail: detail2 };
|
|
2678
3420
|
}
|
|
2679
3421
|
await pullAndRequireImage(loaded.state.image, loaded.state.imageDigest, "Current deployment");
|
|
2680
3422
|
await startProject(loaded.directory, loaded.state.mode, loaded.env);
|
|
@@ -2691,7 +3433,9 @@ async function recoverCommand(directory) {
|
|
|
2691
3433
|
recovered.rollback = loaded.state.rollback;
|
|
2692
3434
|
recovered.lastError = undefined;
|
|
2693
3435
|
await commitRecoveredState(loaded.directory, loaded.state, recovered);
|
|
2694
|
-
|
|
3436
|
+
const detail = `Recovered the incomplete ${pending.operation} operation. OR3 ${recovered.appVersion} is deeply healthy.`;
|
|
3437
|
+
report(detail);
|
|
3438
|
+
return { kind: "recovered", operation: pending.operation, detail };
|
|
2695
3439
|
} catch (error) {
|
|
2696
3440
|
const recoverySecrets = pending.operation === "credentials-reset" ? secretValues(pending.credentialReset?.nextEnv ?? loaded.env) : secretValues(loaded.env);
|
|
2697
3441
|
loaded.state.lastError = redact(error instanceof Error ? error.message : String(error), recoverySecrets);
|
|
@@ -2721,6 +3465,12 @@ function assertNoPending(state) {
|
|
|
2721
3465
|
throw new Error(`An incomplete ${state.incompleteOperation.operation} is recorded. Run "npx @or3/cloud recover" before starting another operation.`);
|
|
2722
3466
|
}
|
|
2723
3467
|
}
|
|
3468
|
+
function assertPhaseAllowsCommand(command, state) {
|
|
3469
|
+
const pending = state.incompleteOperation;
|
|
3470
|
+
if (!pending || command === "stop")
|
|
3471
|
+
return;
|
|
3472
|
+
throw new Error(`An incomplete ${pending.operation}${pending.phase ? ` (${pending.phase})` : ""} is recorded. Refusing to ${command} an ambiguous deployment. Run "npx @or3/cloud recover --dry-run" to inspect the supported actions, then "npx @or3/cloud recover --finish" or "npx @or3/cloud recover --restore --yes".`);
|
|
3473
|
+
}
|
|
2724
3474
|
async function backupCreateCommand(directory) {
|
|
2725
3475
|
await ensureDocker();
|
|
2726
3476
|
const loaded = await loadManaged(directory);
|
|
@@ -2741,26 +3491,66 @@ async function backupCreateCommand(directory) {
|
|
|
2741
3491
|
try {
|
|
2742
3492
|
const result = await createBackup(loaded.directory, loaded.state, loaded.env, { backupId, initiallyRunning: initialAppRunning });
|
|
2743
3493
|
await clearPending(loaded.directory, loaded.state);
|
|
2744
|
-
await pruneBackups(loaded.directory, loaded.state, BACKUP_RETENTION_KEEP, false);
|
|
3494
|
+
const prune = await pruneBackups(loaded.directory, loaded.state, BACKUP_RETENTION_KEEP, false, { automatic: true });
|
|
2745
3495
|
console.log(`Backup ${result.backupId} created at ${result.backupDir}`);
|
|
2746
3496
|
console.log(`SHA-256: ${result.manifest.dataSha256}`);
|
|
3497
|
+
if (prune.deferred.length > 0) {
|
|
3498
|
+
console.warn(`Maintenance warning: ${prune.deferred.length} backup entr${prune.deferred.length === 1 ? "y" : "ies"} need inspection; all backups were preserved. Run "npx @or3/cloud backup list" for details.`);
|
|
3499
|
+
}
|
|
2747
3500
|
} catch (error) {
|
|
2748
3501
|
loaded.state.lastError = redact(error instanceof Error ? error.message : String(error), secretValues(loaded.env));
|
|
2749
3502
|
await writeState(loaded.directory, loaded.state);
|
|
2750
3503
|
throw error;
|
|
2751
3504
|
}
|
|
2752
3505
|
}
|
|
2753
|
-
async function backupListCommand(directory) {
|
|
2754
|
-
const loaded = await loadManaged(directory);
|
|
2755
|
-
const
|
|
2756
|
-
|
|
2757
|
-
|
|
3506
|
+
async function backupListCommand(directory, flags = {}) {
|
|
3507
|
+
const loaded = await loadManaged(directory, { writable: false });
|
|
3508
|
+
const inventory = await inventoryBackups(loaded.directory);
|
|
3509
|
+
const verified = inventory.entries.filter((entry) => entry.kind === "verified").map((entry) => entry.backup);
|
|
3510
|
+
const findings = inventory.entries.filter((entry) => entry.kind !== "verified");
|
|
3511
|
+
if (boolFlag(flags, "json")) {
|
|
3512
|
+
console.log(JSON.stringify({
|
|
3513
|
+
schemaVersion: 1,
|
|
3514
|
+
directory: loaded.directory,
|
|
3515
|
+
backups: verified.map((backup) => ({
|
|
3516
|
+
backupId: backup.backupId,
|
|
3517
|
+
createdAt: backup.createdAt,
|
|
3518
|
+
appVersion: backup.appVersion,
|
|
3519
|
+
bytes: backup.bytes,
|
|
3520
|
+
dataSha256: backup.dataSha256,
|
|
3521
|
+
trust: "verified"
|
|
3522
|
+
})),
|
|
3523
|
+
findings: findings.map((entry) => ({ entryName: entry.entryName, kind: entry.kind, code: entry.code, message: entry.message })),
|
|
3524
|
+
storeErrors: inventory.storeErrors
|
|
3525
|
+
}, null, 2));
|
|
2758
3526
|
return;
|
|
2759
3527
|
}
|
|
2760
|
-
|
|
2761
|
-
|
|
2762
|
-
|
|
2763
|
-
|
|
3528
|
+
if (verified.length === 0 && findings.length === 0) {
|
|
3529
|
+
console.log(`No backups are available for ${loaded.directory}. Run "npx @or3/cloud backup" to create one.`);
|
|
3530
|
+
return;
|
|
3531
|
+
}
|
|
3532
|
+
if (verified.length === 0) {
|
|
3533
|
+
console.log(`No authenticated backups are available for ${loaded.directory}.`);
|
|
3534
|
+
} else {
|
|
3535
|
+
console.log(`OR3 Cloud backups for ${loaded.directory} (${verified.length} verified):`);
|
|
3536
|
+
console.log("backupId createdAt version bytes trust checksum");
|
|
3537
|
+
for (const backup of verified) {
|
|
3538
|
+
console.log(`${backup.backupId.padEnd(30)} ${backup.createdAt.padEnd(30)} ${backup.appVersion.padEnd(8)} ${String(backup.bytes).padStart(9)} ${"verified".padEnd(9)} ${backup.dataSha256.slice(0, 12)}`);
|
|
3539
|
+
}
|
|
3540
|
+
}
|
|
3541
|
+
if (findings.length > 0) {
|
|
3542
|
+
console.log(`
|
|
3543
|
+
Preserved history (${findings.length}; never trusted for restore):`);
|
|
3544
|
+
for (const entry of findings) {
|
|
3545
|
+
console.log(` ${entry.entryName.padEnd(30)} ${entry.kind.padEnd(16)} ${entry.code}`);
|
|
3546
|
+
console.log(` ${entry.message}`);
|
|
3547
|
+
}
|
|
3548
|
+
console.log(`
|
|
3549
|
+
To obtain a trusted restore point, create a new backup of the current healthy deployment: npx @or3/cloud backup`);
|
|
3550
|
+
}
|
|
3551
|
+
for (const diagnostic of inventory.storeErrors) {
|
|
3552
|
+
console.log(`
|
|
3553
|
+
Store error (${diagnostic.severity}): ${diagnostic.message}`);
|
|
2764
3554
|
}
|
|
2765
3555
|
console.log(`
|
|
2766
3556
|
Backups contain credentials and secrets; keep them owner-only and export off-host.`);
|
|
@@ -2771,9 +3561,18 @@ async function backupPruneCommand(directory, flags) {
|
|
|
2771
3561
|
const keep = parseKeep(flags);
|
|
2772
3562
|
const force = boolFlag(flags, "force");
|
|
2773
3563
|
if (force && !boolFlag(flags, "yes"))
|
|
2774
|
-
throw new Error("--force may delete
|
|
2775
|
-
const
|
|
2776
|
-
|
|
3564
|
+
throw new Error("--force may delete backups beyond the retention count. Re-run with --force --yes after confirming the exact backups shown by `backup list`.");
|
|
3565
|
+
const result = await pruneBackups(loaded.directory, loaded.state, keep, force, { automatic: !force });
|
|
3566
|
+
if (!force && result.deferred.length > 0 && result.removed === 0) {
|
|
3567
|
+
throw new Error(`Backup pruning is blocked because the store contains entries that need inspection:
|
|
3568
|
+
${result.deferred.map((diagnostic) => ` ${diagnostic.code}: ${diagnostic.message}`).join(`
|
|
3569
|
+
`)}
|
|
3570
|
+
No backups were deleted. Run "npx @or3/cloud backup list" to inspect them, or re-run with --force --yes to prune only verified backups while preserving protected recovery sources.`);
|
|
3571
|
+
}
|
|
3572
|
+
console.log(result.removed > 0 ? `Pruned ${result.removed} backup(s); keeping the newest ${keep}.` : `Nothing to prune: keeping all backups (newest ${keep}).`);
|
|
3573
|
+
if (result.deferred.length > 0) {
|
|
3574
|
+
console.warn(`Preserved ${result.deferred.length} suspect or legacy entr${result.deferred.length === 1 ? "y" : "ies"} without deletion.`);
|
|
3575
|
+
}
|
|
2777
3576
|
}
|
|
2778
3577
|
async function backupExportCommand(directory, backupId, destination) {
|
|
2779
3578
|
const backupPath = await resolveBackup(directory, backupId);
|
|
@@ -2854,8 +3653,10 @@ async function backupCommand(directory, positionals, flags) {
|
|
|
2854
3653
|
if (subcommand === "list") {
|
|
2855
3654
|
if (positionals.length > 1)
|
|
2856
3655
|
throw new Error("backup list accepts no arguments.");
|
|
2857
|
-
return await backupListCommand(directory);
|
|
3656
|
+
return await backupListCommand(directory, flags);
|
|
2858
3657
|
}
|
|
3658
|
+
if (boolFlag(flags, "json"))
|
|
3659
|
+
throw new Error("--json is supported only by `backup list`.");
|
|
2859
3660
|
if (subcommand === "prune") {
|
|
2860
3661
|
if (positionals.length > 1)
|
|
2861
3662
|
throw new Error("backup prune accepts no arguments.");
|
|
@@ -2872,7 +3673,184 @@ async function backupCommand(directory, positionals, flags) {
|
|
|
2872
3673
|
}
|
|
2873
3674
|
throw new Error(`Unknown backup subcommand "${subcommand}". Use list, prune, export, or no subcommand to create a backup.`);
|
|
2874
3675
|
}
|
|
3676
|
+
function stateFingerprint(state) {
|
|
3677
|
+
return createHash("sha256").update(JSON.stringify({
|
|
3678
|
+
schema: state.schemaVersion,
|
|
3679
|
+
appVersion: state.appVersion,
|
|
3680
|
+
image: state.image,
|
|
3681
|
+
imageDigest: state.imageDigest,
|
|
3682
|
+
pending: state.incompleteOperation?.id ?? null
|
|
3683
|
+
})).digest("hex").slice(0, 16);
|
|
3684
|
+
}
|
|
3685
|
+
async function assessUpdate(directory, targetVersion, options = {}) {
|
|
3686
|
+
const checkDocker = options.checkDocker ?? false;
|
|
3687
|
+
const observation = await observeDeployment(directory, { checkDocker, checkImage: false });
|
|
3688
|
+
const state = observation.state;
|
|
3689
|
+
const checks = [];
|
|
3690
|
+
const findings = [...observationFindings(observation)];
|
|
3691
|
+
const targetImage = imageFor(targetVersion);
|
|
3692
|
+
if (!state) {
|
|
3693
|
+
findings.push({ code: "state-unreadable", severity: "blocker", message: "Managed state must be readable to preview an update." });
|
|
3694
|
+
} else if (state.incompleteOperation) {
|
|
3695
|
+
findings.push({
|
|
3696
|
+
code: "operation-incomplete",
|
|
3697
|
+
severity: "blocker",
|
|
3698
|
+
message: `An incomplete ${state.incompleteOperation.operation} is recorded. Run "npx @or3/cloud recover" before updating.`,
|
|
3699
|
+
nextCommand: "npx @or3/cloud recover"
|
|
3700
|
+
});
|
|
3701
|
+
}
|
|
3702
|
+
if (observation.changing && !options.underLease) {
|
|
3703
|
+
findings.push({ code: "operation-in-progress", severity: "blocker", message: "A mutation lease is active; preview is non-executable until it settles." });
|
|
3704
|
+
}
|
|
3705
|
+
let noOp = false;
|
|
3706
|
+
if (state && targetVersion === state.appVersion && !state.incompleteOperation) {
|
|
3707
|
+
noOp = true;
|
|
3708
|
+
checks.push({ code: "already-installed", status: "passed", detail: `OR3 ${targetVersion} is already the installed release; execution is a no-op.` });
|
|
3709
|
+
} else {
|
|
3710
|
+
checks.push({ code: "target-version", status: "passed", detail: `Target OR3 ${targetVersion} (${targetImage}).` });
|
|
3711
|
+
}
|
|
3712
|
+
try {
|
|
3713
|
+
const stats = await statfs(deploymentPaths(directory).backups);
|
|
3714
|
+
const freeBytes = Number(stats.bavail) * Number(stats.bsize);
|
|
3715
|
+
checks.push({ code: "backup-filesystem-space", status: "passed", detail: `${freeBytes} bytes available on the backup filesystem.` });
|
|
3716
|
+
} catch (error) {
|
|
3717
|
+
checks.push({ code: "backup-filesystem-space", status: "unknown", detail: redact(error instanceof Error ? error.message : String(error)) });
|
|
3718
|
+
}
|
|
3719
|
+
checks.push({ code: "data-volume-space", status: "deferred", detail: "The live data-volume requirement is validated after the snapshot, before downtime." });
|
|
3720
|
+
if (checkDocker) {
|
|
3721
|
+
if (observation.docker) {
|
|
3722
|
+
try {
|
|
3723
|
+
const architecture = await dockerDaemonArchitecture();
|
|
3724
|
+
checks.push({ code: "docker-availability", status: "passed", detail: `Docker daemon is reachable (${architecture}).` });
|
|
3725
|
+
} catch (error) {
|
|
3726
|
+
checks.push({ code: "docker-availability", status: "failed", detail: redact(error instanceof Error ? error.message : String(error)) });
|
|
3727
|
+
}
|
|
3728
|
+
} else {
|
|
3729
|
+
checks.push({ code: "docker-availability", status: "unknown", detail: "The Docker daemon is not reachable from this host; it is revalidated at execution time." });
|
|
3730
|
+
}
|
|
3731
|
+
} else {
|
|
3732
|
+
checks.push({ code: "docker-availability", status: "deferred", detail: "Docker availability and daemon architecture are revalidated at execution time." });
|
|
3733
|
+
}
|
|
3734
|
+
checks.push({ code: "image-pull", status: "deferred", detail: "The target image pull and architecture/provenance checks run at execution time." });
|
|
3735
|
+
checks.push({ code: "fresh-snapshot", status: "deferred", detail: "A fresh authenticated snapshot is created at execution time." });
|
|
3736
|
+
const plan = planRetention(observation.backups?.entries ?? [], BACKUP_RETENTION_KEEP, state ? retentionProtectedIds(state) : new Set, { automatic: true });
|
|
3737
|
+
checks.push({
|
|
3738
|
+
code: "retention",
|
|
3739
|
+
status: plan.canPrune ? "passed" : "deferred",
|
|
3740
|
+
detail: plan.canPrune ? `Would keep ${plan.keep.length} verified backup(s) and prune ${plan.remove.length}.` : `${plan.warnings.length} entry(ies) need inspection; automatic pruning is deferred and all backups are preserved.`
|
|
3741
|
+
});
|
|
3742
|
+
for (const warning of plan.warnings)
|
|
3743
|
+
findings.push(warning);
|
|
3744
|
+
if (options.inspectAssets !== false) {
|
|
3745
|
+
const changed = [];
|
|
3746
|
+
for (const name of managedAssetNames(state?.mode ?? "local")) {
|
|
3747
|
+
let desired;
|
|
3748
|
+
try {
|
|
3749
|
+
desired = await readFile(join(ASSET_ROOT, name));
|
|
3750
|
+
} catch (error) {
|
|
3751
|
+
findings.push({
|
|
3752
|
+
code: "managed-asset-unreadable",
|
|
3753
|
+
severity: "blocker",
|
|
3754
|
+
resource: name,
|
|
3755
|
+
message: `Required generated asset ${name} could not be read from this CLI: ${redact(error instanceof Error ? error.message : String(error))}`
|
|
3756
|
+
});
|
|
3757
|
+
continue;
|
|
3758
|
+
}
|
|
3759
|
+
try {
|
|
3760
|
+
const installedPath = join(directory, name);
|
|
3761
|
+
if (!await fileExists(installedPath)) {
|
|
3762
|
+
changed.push(`${name} (add)`);
|
|
3763
|
+
continue;
|
|
3764
|
+
}
|
|
3765
|
+
const installed = await readFile(installedPath);
|
|
3766
|
+
if (!installed.equals(desired))
|
|
3767
|
+
changed.push(`${name} (update)`);
|
|
3768
|
+
} catch (error) {
|
|
3769
|
+
findings.push({
|
|
3770
|
+
code: "managed-asset-unreadable",
|
|
3771
|
+
severity: "blocker",
|
|
3772
|
+
resource: name,
|
|
3773
|
+
message: `Installed generated asset ${name} could not be read from ${directory}: ${redact(error instanceof Error ? error.message : String(error))}`
|
|
3774
|
+
});
|
|
3775
|
+
}
|
|
3776
|
+
}
|
|
3777
|
+
checks.push(changed.length ? { code: "managed-assets", status: "passed", detail: `Generated assets change: ${changed.join(", ")}.` } : { code: "managed-assets", status: "passed", detail: "Generated assets are unchanged." });
|
|
3778
|
+
}
|
|
3779
|
+
try {
|
|
3780
|
+
assertKnownStateSchema(state?.schemaVersion);
|
|
3781
|
+
checks.push({ code: "state-schema", status: "passed", detail: `Managed state schema ${String(state?.schemaVersion)} is supported.` });
|
|
3782
|
+
} catch (error) {
|
|
3783
|
+
findings.push({ code: "state-schema-unsupported", severity: "blocker", message: error instanceof Error ? error.message : String(error) });
|
|
3784
|
+
}
|
|
3785
|
+
const blockers = findings.filter((finding) => finding.severity === "blocker");
|
|
3786
|
+
blockers.sort((a, b) => a.code < b.code ? -1 : a.code > b.code ? 1 : 0);
|
|
3787
|
+
return {
|
|
3788
|
+
schemaVersion: 1,
|
|
3789
|
+
observedAt: observation.observedAt,
|
|
3790
|
+
source: state ? { appVersion: state.appVersion, image: state.image, imageDigest: state.imageDigest } : null,
|
|
3791
|
+
target: { appVersion: targetVersion, image: targetImage, imageDigest: expectedImageDigest(targetVersion) ?? "unknown" },
|
|
3792
|
+
checks,
|
|
3793
|
+
findings: [...blockers, ...findings.filter((finding) => finding.severity !== "blocker")],
|
|
3794
|
+
retention: plan,
|
|
3795
|
+
stateFingerprint: state ? stateFingerprint(state) : "unavailable"
|
|
3796
|
+
};
|
|
3797
|
+
}
|
|
3798
|
+
async function updatePreviewCommand(directory, flags) {
|
|
3799
|
+
const targetVersion = stringFlag(flags, "to")?.trim() ?? PACKAGE_VERSION;
|
|
3800
|
+
if (!isVersion(targetVersion))
|
|
3801
|
+
throw new Error(`--to must be a complete semantic version such as ${PACKAGE_VERSION}.`);
|
|
3802
|
+
if (targetVersion !== PACKAGE_VERSION) {
|
|
3803
|
+
throw new Error(`This CLI contains deployment assets for OR3 ${PACKAGE_VERSION}. Preview the matching CLI with \`npx --yes @or3/cloud@${targetVersion} update --to ${targetVersion} --dry-run\`.`);
|
|
3804
|
+
}
|
|
3805
|
+
const assessment = await assessUpdate(directory, targetVersion, { checkDocker: true });
|
|
3806
|
+
if (boolFlag(flags, "json")) {
|
|
3807
|
+
console.log(JSON.stringify({ ...assessment, kind: "or3-update-preview" }, null, 2));
|
|
3808
|
+
} else {
|
|
3809
|
+
console.log(`OR3 update preview for ${directory}`);
|
|
3810
|
+
console.log(` source: ${assessment.source ? `OR3 ${assessment.source.appVersion} (${assessment.source.imageDigest})` : "unknown"}`);
|
|
3811
|
+
console.log(` target: OR3 ${assessment.target.appVersion} (${assessment.target.imageDigest})`);
|
|
3812
|
+
console.log(` state fingerprint: ${assessment.stateFingerprint}`);
|
|
3813
|
+
console.log();
|
|
3814
|
+
for (const check of assessment.checks)
|
|
3815
|
+
console.log(` [${check.status}] ${check.code}: ${check.detail}`);
|
|
3816
|
+
console.log();
|
|
3817
|
+
for (const finding of assessment.findings)
|
|
3818
|
+
console.log(` [${finding.severity}] ${finding.code}: ${finding.message}`);
|
|
3819
|
+
if (assessment.retention.preserve.length) {
|
|
3820
|
+
console.log(`
|
|
3821
|
+
Preserved backups:`);
|
|
3822
|
+
for (const entry of assessment.retention.preserve)
|
|
3823
|
+
console.log(` ${entry.entryName}: ${entry.reason}`);
|
|
3824
|
+
}
|
|
3825
|
+
console.log(`
|
|
3826
|
+
Dry run only: no state, assets, lease, images, services, snapshots, or backups were changed.`);
|
|
3827
|
+
}
|
|
3828
|
+
if (assessment.findings.some((finding) => finding.severity === "blocker"))
|
|
3829
|
+
process.exitCode = 1;
|
|
3830
|
+
}
|
|
2875
3831
|
async function updateCommand(directory, flags) {
|
|
3832
|
+
const asJson = boolFlag(flags, "json");
|
|
3833
|
+
try {
|
|
3834
|
+
return await runUpdateCommand(directory, flags, asJson);
|
|
3835
|
+
} catch (error) {
|
|
3836
|
+
if (!asJson)
|
|
3837
|
+
throw error;
|
|
3838
|
+
const message = redact(error instanceof Error ? error.message : String(error));
|
|
3839
|
+
console.error(`OR3 Cloud failed: ${message}`);
|
|
3840
|
+
emitOperationResult({
|
|
3841
|
+
kind: "blocked",
|
|
3842
|
+
findings: [{ code: "update-failed", severity: "blocker", message, nextCommand: "npx @or3/cloud recover --dry-run" }]
|
|
3843
|
+
});
|
|
3844
|
+
process.exitCode = 1;
|
|
3845
|
+
}
|
|
3846
|
+
}
|
|
3847
|
+
async function runUpdateCommand(directory, flags, asJson) {
|
|
3848
|
+
const progress = (...args) => {
|
|
3849
|
+
if (asJson)
|
|
3850
|
+
console.error(...args);
|
|
3851
|
+
else
|
|
3852
|
+
console.log(...args);
|
|
3853
|
+
};
|
|
2876
3854
|
await ensureDocker();
|
|
2877
3855
|
const loaded = await loadManaged(directory);
|
|
2878
3856
|
const { state, env } = loaded;
|
|
@@ -2884,8 +3862,18 @@ async function updateCommand(directory, flags) {
|
|
|
2884
3862
|
if (targetVersion !== PACKAGE_VERSION) {
|
|
2885
3863
|
throw new Error(`This CLI contains deployment assets for OR3 ${PACKAGE_VERSION}. Run \`npx --yes @or3/cloud@${targetVersion} update --to ${targetVersion}\` so the image and generated assets match.`);
|
|
2886
3864
|
}
|
|
2887
|
-
if (targetVersion === state.appVersion)
|
|
2888
|
-
|
|
3865
|
+
if (targetVersion === state.appVersion) {
|
|
3866
|
+
if (asJson)
|
|
3867
|
+
emitOperationResult({ kind: "no-op", detail: `OR3 ${targetVersion} is already installed.`, currentVersion: state.appVersion, targetVersion });
|
|
3868
|
+
else
|
|
3869
|
+
progress(`OR3 ${targetVersion} is already installed. Nothing to do.`);
|
|
3870
|
+
return;
|
|
3871
|
+
}
|
|
3872
|
+
const preflight = await assessUpdate(loaded.directory, targetVersion, { underLease: true, checkDocker: true });
|
|
3873
|
+
const preflightBlockers = preflight.findings.filter((finding) => finding.severity === "blocker");
|
|
3874
|
+
if (preflightBlockers.length > 0) {
|
|
3875
|
+
throw new Error(`Update to ${targetVersion} is blocked by preflight: ${preflightBlockers.map((finding) => `${finding.code} (${finding.message})`).join(" ")}`);
|
|
3876
|
+
}
|
|
2889
3877
|
const targetImageTag = imageFor(targetVersion);
|
|
2890
3878
|
const targetDigest = await pullImage(targetImageTag, expectedImageDigest(targetVersion));
|
|
2891
3879
|
await assertSupportedHostArchitecture(targetImageTag);
|
|
@@ -2908,16 +3896,32 @@ async function updateCommand(directory, flags) {
|
|
|
2908
3896
|
phase: "prepared"
|
|
2909
3897
|
};
|
|
2910
3898
|
await markPending(loaded.directory, state, pending);
|
|
3899
|
+
const startedAtMs = Date.now();
|
|
2911
3900
|
try {
|
|
3901
|
+
const operator = await prepareVerifiedDashboardOperator(loaded.directory, targetVersion);
|
|
2912
3902
|
const backup = await createBackup(loaded.directory, state, env, { restartAfter: false, backupId });
|
|
3903
|
+
const writeSchema = writeStateSchema();
|
|
3904
|
+
if (writeSchema >= 2) {
|
|
3905
|
+
const minimumSource = packagedMinimumSourceVersion();
|
|
3906
|
+
if (minimumSource && compareReleaseVersions(state.appVersion, minimumSource) < 0) {
|
|
3907
|
+
throw new Error(`OR3 ${targetVersion} writes managed state schema 2, which requires the compatibility bridge (source ${minimumSource} or newer). Run the host CLI bridge update first, then retry this schema-2 release.`);
|
|
3908
|
+
}
|
|
3909
|
+
state.schemaVersion = writeSchema;
|
|
3910
|
+
}
|
|
2913
3911
|
await updatePending(loaded.directory, state, {
|
|
2914
3912
|
backupId: backup.backupId,
|
|
2915
3913
|
backupPath: backup.backupDir,
|
|
2916
3914
|
backupDataSha256: backup.manifest.dataSha256,
|
|
2917
3915
|
backupConfigSha256: backup.manifest.configSha256,
|
|
2918
|
-
phase: "snapshot-created"
|
|
3916
|
+
phase: "snapshot-created",
|
|
3917
|
+
verifiedSnapshot: {
|
|
3918
|
+
backupId: backup.backupId,
|
|
3919
|
+
path: backup.backupDir,
|
|
3920
|
+
dataSha256: backup.manifest.dataSha256,
|
|
3921
|
+
configSha256: backup.manifest.configSha256 ?? "",
|
|
3922
|
+
createdAt: backup.manifest.createdAt
|
|
3923
|
+
}
|
|
2919
3924
|
});
|
|
2920
|
-
const operator = await prepareVerifiedDashboardOperator(loaded.directory, targetVersion);
|
|
2921
3925
|
if (env.OR3_DASHBOARD_UPDATES_ENABLED === "true" && !operator && !process.env.OR3_DASHBOARD_UPDATE_JOB_ID) {
|
|
2922
3926
|
await removeDashboardOperator(loaded.directory, state.mode);
|
|
2923
3927
|
}
|
|
@@ -2953,7 +3957,6 @@ async function updateCommand(directory, flags) {
|
|
|
2953
3957
|
if (migrateLegacyVolume || recreateDataVolume) {
|
|
2954
3958
|
await restoreVolumeArchive(loaded.directory, state.mode, nextEnv, backup.backupDir);
|
|
2955
3959
|
}
|
|
2956
|
-
await startProject(loaded.directory, state.mode, nextEnv);
|
|
2957
3960
|
} catch (error) {
|
|
2958
3961
|
await updatePending(loaded.directory, state, { phase: "restoring-previous" });
|
|
2959
3962
|
try {
|
|
@@ -2971,11 +3974,88 @@ async function updateCommand(directory, flags) {
|
|
|
2971
3974
|
const recovered = await commitPreMutationRecovery(loaded.directory, state, `Update to ${targetVersion} failed and was restored: ${redact(error instanceof Error ? error.message : String(error), secretValues(oldEnv))}`);
|
|
2972
3975
|
throw new Error(recovered.lastError);
|
|
2973
3976
|
}
|
|
3977
|
+
await updatePending(loaded.directory, state, { phase: "starting-target" });
|
|
3978
|
+
try {
|
|
3979
|
+
await startProject(loaded.directory, state.mode, nextEnv);
|
|
3980
|
+
} catch (error) {
|
|
3981
|
+
throw new Error(`Update to ${targetVersion} started the target but it did not pass verification: ${redact(error instanceof Error ? error.message : String(error), secretValues(nextEnv))}. Writes may have been accepted, so the deployment was NOT restored automatically. Run "npx @or3/cloud recover --dry-run", then "npx @or3/cloud recover --finish" if the target is proven, or "npx @or3/cloud recover --restore --yes" to return to the pre-update snapshot.`);
|
|
3982
|
+
}
|
|
2974
3983
|
const digest = await requireImageDigest(targetImage, targetDigest, `OR3 ${targetVersion}`);
|
|
3984
|
+
const targetChecks = await collectTargetReadyChecks(loaded.directory, state.mode, nextEnv);
|
|
3985
|
+
const failedTargetChecks = targetChecks.checks.filter((check) => check.status !== "passed");
|
|
3986
|
+
if (!targetChecks.containerId || failedTargetChecks.length > 0) {
|
|
3987
|
+
throw new Error(`Update to ${targetVersion} did not pass the required target checks (${failedTargetChecks.map((check) => check.code).join(", ") || "container-binding"}). Writes may have been accepted, so the deployment was NOT restored automatically. Run "npx @or3/cloud recover --dry-run", then "npx @or3/cloud recover --restore --yes" to return to the pre-update snapshot.`);
|
|
3988
|
+
}
|
|
3989
|
+
const targetReadyEvidence = {
|
|
3990
|
+
checkedAt: now(),
|
|
3991
|
+
deploymentId: targetDeploymentId,
|
|
3992
|
+
deploymentRoot: resolve(loaded.directory),
|
|
3993
|
+
containerId: targetChecks.containerId,
|
|
3994
|
+
imageDigest: digest,
|
|
3995
|
+
configurationSha256: await sha256File(deploymentPaths(loaded.directory).env),
|
|
3996
|
+
managedAssetSha256: await installedManagedAssetChecksums(loaded.directory, state.mode),
|
|
3997
|
+
dataReplacementCompleted: true,
|
|
3998
|
+
checks: [
|
|
3999
|
+
{ code: "image-digest", status: "passed", detail: `${targetImage}@${digest}` },
|
|
4000
|
+
{ code: "deep-health", status: "passed", detail: "OR3 deep health and running-image identity verified" },
|
|
4001
|
+
...targetChecks.checks
|
|
4002
|
+
]
|
|
4003
|
+
};
|
|
4004
|
+
if (writeSchema >= 2) {
|
|
4005
|
+
await updatePending(loaded.directory, state, { phase: "target-ready", evidence: targetReadyEvidence });
|
|
4006
|
+
}
|
|
4007
|
+
const warnings = [];
|
|
4008
|
+
const hadDashboardJob = Boolean(process.env.OR3_DASHBOARD_UPDATE_JOB_ID);
|
|
4009
|
+
let operatorHandoff = "not-required";
|
|
4010
|
+
if (nextEnv.OR3_DASHBOARD_UPDATES_ENABLED === "true" && hadDashboardJob) {
|
|
4011
|
+
try {
|
|
4012
|
+
await scheduleDashboardOperatorHandoff(loaded.directory, nextEnv);
|
|
4013
|
+
operatorHandoff = "pending";
|
|
4014
|
+
} catch (error) {
|
|
4015
|
+
operatorHandoff = "needs-attention";
|
|
4016
|
+
warnings.push({
|
|
4017
|
+
code: "operator-handoff-schedule-failed",
|
|
4018
|
+
severity: "warning",
|
|
4019
|
+
resource: pending.id,
|
|
4020
|
+
message: `The application is upgraded, but the dashboard operator handoff could not be scheduled: ${redact(error instanceof Error ? error.message : String(error), secretValues(nextEnv))}. Run "npx @or3/cloud recover --finish" on the host.`,
|
|
4021
|
+
nextCommand: "npx @or3/cloud recover --finish"
|
|
4022
|
+
});
|
|
4023
|
+
}
|
|
4024
|
+
}
|
|
4025
|
+
const sourceVersion = state.appVersion;
|
|
4026
|
+
const sourceImage = state.image;
|
|
4027
|
+
const sourceDigest = state.imageDigest;
|
|
4028
|
+
const receipt = {
|
|
4029
|
+
schemaVersion: 1,
|
|
4030
|
+
operationId: pending.id,
|
|
4031
|
+
dashboardJobId: process.env.OR3_DASHBOARD_UPDATE_JOB_ID?.trim() || undefined,
|
|
4032
|
+
cliVersion: PACKAGE_VERSION,
|
|
4033
|
+
source: {
|
|
4034
|
+
appVersion: sourceVersion,
|
|
4035
|
+
image: sourceImage,
|
|
4036
|
+
imageDigest: sourceDigest,
|
|
4037
|
+
sourceRevision: packagedSourceRevision(sourceVersion),
|
|
4038
|
+
operatorImageDigest: expectedOperatorImageDigest(sourceVersion)
|
|
4039
|
+
},
|
|
4040
|
+
target: {
|
|
4041
|
+
appVersion: targetVersion,
|
|
4042
|
+
image: targetImage,
|
|
4043
|
+
imageDigest: digest,
|
|
4044
|
+
sourceRevision: packagedSourceRevision(targetVersion),
|
|
4045
|
+
operatorImageDigest: expectedOperatorImageDigest(targetVersion)
|
|
4046
|
+
},
|
|
4047
|
+
observed: { appVersion: targetVersion, image: targetImage, imageDigest: digest },
|
|
4048
|
+
completedAt: now(),
|
|
4049
|
+
rollbackBackupId: backup.backupId,
|
|
4050
|
+
checks: targetReadyEvidence.checks,
|
|
4051
|
+
warnings,
|
|
4052
|
+
phaseDurationsMs: { "replace-and-verify": Date.now() - startedAtMs },
|
|
4053
|
+
operatorHandoff
|
|
4054
|
+
};
|
|
2975
4055
|
state.rollback = {
|
|
2976
|
-
appVersion:
|
|
2977
|
-
image:
|
|
2978
|
-
imageDigest:
|
|
4056
|
+
appVersion: sourceVersion,
|
|
4057
|
+
image: sourceImage,
|
|
4058
|
+
imageDigest: sourceDigest,
|
|
2979
4059
|
backupId: backup.backupId,
|
|
2980
4060
|
createdAt: now()
|
|
2981
4061
|
};
|
|
@@ -2986,11 +4066,39 @@ async function updateCommand(directory, flags) {
|
|
|
2986
4066
|
state.deploymentRoot = resolve(loaded.directory);
|
|
2987
4067
|
state.lastSuccessfulOperation = "update";
|
|
2988
4068
|
state.lastError = undefined;
|
|
2989
|
-
|
|
2990
|
-
|
|
2991
|
-
|
|
2992
|
-
|
|
2993
|
-
|
|
4069
|
+
let commit;
|
|
4070
|
+
try {
|
|
4071
|
+
commit = await commitTerminalState(loaded.directory, state, receipt);
|
|
4072
|
+
} catch (error) {
|
|
4073
|
+
throw new Error(`OR3 ${targetVersion} replaced the running release, but the terminal state write could not be confirmed. Recovery evidence was preserved; re-run status/doctor and "npx @or3/cloud recover" to resolve the durable outcome before any further mutation. ${redact(error instanceof Error ? error.message : String(error), secretValues(nextEnv))}`);
|
|
4074
|
+
}
|
|
4075
|
+
try {
|
|
4076
|
+
await removeOperationRecord(loaded.directory, commit.operationId);
|
|
4077
|
+
} catch (error) {
|
|
4078
|
+
warnings.push({ code: "operation-mirror-delete-failed", severity: "warning", resource: commit.operationId, message: redact(error instanceof Error ? error.message : String(error), secretValues(nextEnv)) });
|
|
4079
|
+
}
|
|
4080
|
+
const prune = await pruneBackups(loaded.directory, state, BACKUP_RETENTION_KEEP, false, { automatic: true, log: progress, warn: progress }).catch((error) => {
|
|
4081
|
+
warnings.push({ code: "retention-failed", severity: "warning", message: redact(error instanceof Error ? error.message : String(error), secretValues(nextEnv)) });
|
|
4082
|
+
return { removed: 0, deferred: [] };
|
|
4083
|
+
});
|
|
4084
|
+
for (const diagnostic of prune.deferred)
|
|
4085
|
+
warnings.push(diagnostic);
|
|
4086
|
+
progress(`OR3 updated to ${targetVersion}. Image digest: ${digest}`);
|
|
4087
|
+
progress(`Rollback point: ${backup.backupId}. Keep it until login, chat, and file checks pass.`);
|
|
4088
|
+
progress("Not checked: live OpenRouter authorization (complete it in the browser).");
|
|
4089
|
+
if (warnings.length > 0) {
|
|
4090
|
+
warnings.forEach((warning) => console.warn(`⚠ ${warning.code}: ${warning.message}`));
|
|
4091
|
+
}
|
|
4092
|
+
await persistReceiptWarnings(loaded.directory, state, warnings);
|
|
4093
|
+
if (asJson) {
|
|
4094
|
+
const outcome = warnings.length > 0 ? { kind: "completed-with-warnings", receipt, warnings } : { kind: "completed", receipt };
|
|
4095
|
+
emitOperationResult(outcome);
|
|
4096
|
+
} else if (warnings.length > 0) {
|
|
4097
|
+
console.log(`Update complete with ${warnings.length} maintenance warning(s).`);
|
|
4098
|
+
console.log("Next: npx @or3/cloud backup list");
|
|
4099
|
+
} else {
|
|
4100
|
+
console.log("Update complete. No maintenance warnings.");
|
|
4101
|
+
}
|
|
2994
4102
|
} catch (error) {
|
|
2995
4103
|
if (state.incompleteOperation) {
|
|
2996
4104
|
state.lastError = redact(error instanceof Error ? error.message : String(error), secretValues(env));
|
|
@@ -3110,8 +4218,8 @@ async function restoreCommand(directory, flags, positionals) {
|
|
|
3110
4218
|
const digest = await imageDigest(restoredEnv.OR3_IMAGE);
|
|
3111
4219
|
const nextState = stateFromEnv(loaded.directory, restoredEnv, loaded.state.mode, "restore", digest);
|
|
3112
4220
|
nextState.lastError = undefined;
|
|
3113
|
-
await removeOperationRecord(loaded.directory, pending.id);
|
|
3114
4221
|
await writeState(loaded.directory, nextState);
|
|
4222
|
+
await removeOperationRecordSafely(loaded.directory, pending.id);
|
|
3115
4223
|
console.log(`Restored ${manifest.backupId}. Verify sign-in, a conversation, and a previously uploaded file.`);
|
|
3116
4224
|
} catch (error) {
|
|
3117
4225
|
const original = redact(error instanceof Error ? error.message : String(error), secretValues(loaded.env));
|
|
@@ -3201,8 +4309,8 @@ async function rollbackCommand(directory, flags) {
|
|
|
3201
4309
|
const restoredEnv = parseEnv(await readText(deploymentPaths(loaded.directory).env));
|
|
3202
4310
|
const nextState = stateFromEnv(loaded.directory, restoredEnv, loaded.state.mode, "restore", await imageDigest(restoredEnv.OR3_IMAGE));
|
|
3203
4311
|
nextState.rollback = undefined;
|
|
3204
|
-
await removeOperationRecord(loaded.directory, pending.id);
|
|
3205
4312
|
await writeState(loaded.directory, nextState);
|
|
4313
|
+
await removeOperationRecordSafely(loaded.directory, pending.id);
|
|
3206
4314
|
console.log(`Rolled back to OR3 ${nextState.appVersion}. Verify sign-in, chat, and file access.`);
|
|
3207
4315
|
} catch (error) {
|
|
3208
4316
|
const original = redact(error instanceof Error ? error.message : String(error), secretValues(loaded.env));
|
|
@@ -3739,6 +4847,53 @@ ${redact(serious.slice(-20).join(`
|
|
|
3739
4847
|
console.log("✓ Recent bounded logs contain no fatal, panic, unhandled, or OOM events");
|
|
3740
4848
|
console.log("OR3 production verification passed.");
|
|
3741
4849
|
}
|
|
4850
|
+
async function verifyReadOnlyCommand(directory, flags) {
|
|
4851
|
+
const observation = await observeDeployment(directory, { checkDocker: true, checkImage: true });
|
|
4852
|
+
const state = observation.state;
|
|
4853
|
+
if (!state) {
|
|
4854
|
+
throw new Error(`Read-only verification needs readable managed state.${observation.stateError ? ` ${observation.stateError}` : ""} Run "npx @or3/cloud doctor" for diagnostics.`);
|
|
4855
|
+
}
|
|
4856
|
+
if (boolFlag(flags, "public") && state.mode !== "public")
|
|
4857
|
+
throw new Error("--public requires a managed public deployment.");
|
|
4858
|
+
const checks = [];
|
|
4859
|
+
const findings = observationFindings(observation);
|
|
4860
|
+
checks.push(observation.identityMatches === true ? { code: "image-digest", status: "passed", detail: `Recorded digest matches the local image (${state.imageDigest}).` } : observation.identityMatches === false ? { code: "image-digest", status: "failed", detail: `Recorded ${observation.recordedImageDigest} does not match local ${observation.actualImageDigest}.` } : { code: "image-digest", status: "unknown", detail: "The managed image could not be inspected safely." });
|
|
4861
|
+
const baseUrl = new URL(state.mode === "public" ? `https://${state.domain}` : `http://127.0.0.1:${state.port}`);
|
|
4862
|
+
try {
|
|
4863
|
+
const root = await verificationFetch(baseUrl);
|
|
4864
|
+
checks.push(root.status === 200 ? { code: "http-root", status: "passed", detail: `${baseUrl.origin} returned HTTP 200 without redirects.` } : { code: "http-root", status: "failed", detail: `${baseUrl.origin} returned HTTP ${root.status}.` });
|
|
4865
|
+
} catch (error) {
|
|
4866
|
+
checks.push({ code: "http-root", status: "failed", detail: redact(error instanceof Error ? error.message : String(error)) });
|
|
4867
|
+
}
|
|
4868
|
+
try {
|
|
4869
|
+
validateVerificationHealth(await verificationJson(baseUrl, "/api/health?deep=true"));
|
|
4870
|
+
checks.push({ code: "deep-health", status: "passed", detail: "Deep health reports the managed Basic Auth + SQLite + filesystem profile." });
|
|
4871
|
+
} catch (error) {
|
|
4872
|
+
checks.push({ code: "deep-health", status: "failed", detail: redact(error instanceof Error ? error.message : String(error)) });
|
|
4873
|
+
}
|
|
4874
|
+
checks.push({ code: "auth-and-storage-journey", status: "deferred", detail: "Sign-in, sync, and storage probes are excluded from read-only verification." });
|
|
4875
|
+
checks.push({ code: "sqlite-integrity", status: "deferred", detail: "SQLite inspection is deferred because a safe read-only mechanism is unavailable." });
|
|
4876
|
+
checks.push({ code: "bounded-logs", status: "deferred", detail: "Log scanning is deliberately skipped by read-only verification; run full verify for it." });
|
|
4877
|
+
if (boolFlag(flags, "json")) {
|
|
4878
|
+
console.log(JSON.stringify({
|
|
4879
|
+
schemaVersion: 1,
|
|
4880
|
+
kind: "or3-verify-read-only",
|
|
4881
|
+
observedAt: observation.observedAt,
|
|
4882
|
+
directory: observation.directory,
|
|
4883
|
+
readOnly: true,
|
|
4884
|
+
checks,
|
|
4885
|
+
findings
|
|
4886
|
+
}, null, 2));
|
|
4887
|
+
} else {
|
|
4888
|
+
for (const check of checks)
|
|
4889
|
+
console.log(` [${check.status}] ${check.code}: ${check.detail}`);
|
|
4890
|
+
for (const finding of findings)
|
|
4891
|
+
console.log(` [${finding.severity}] ${finding.code}: ${finding.message}`);
|
|
4892
|
+
}
|
|
4893
|
+
if (checks.some((check) => check.status === "failed")) {
|
|
4894
|
+
process.exitCode = 1;
|
|
4895
|
+
}
|
|
4896
|
+
}
|
|
3742
4897
|
async function readSourceVolume(sourceDirectory) {
|
|
3743
4898
|
const composeResult = await run("docker", [
|
|
3744
4899
|
"compose",
|
|
@@ -4353,37 +5508,79 @@ async function credentialsCommand(directory, positionals, flags) {
|
|
|
4353
5508
|
throw new Error("credentials reset accepts no arguments.");
|
|
4354
5509
|
return await credentialsResetCommand(directory, flags);
|
|
4355
5510
|
}
|
|
4356
|
-
async function statusCommand(directory) {
|
|
4357
|
-
const
|
|
4358
|
-
|
|
4359
|
-
const
|
|
4360
|
-
|
|
4361
|
-
|
|
4362
|
-
|
|
4363
|
-
|
|
4364
|
-
|
|
4365
|
-
|
|
4366
|
-
|
|
4367
|
-
|
|
4368
|
-
|
|
4369
|
-
|
|
4370
|
-
|
|
4371
|
-
|
|
4372
|
-
|
|
4373
|
-
|
|
4374
|
-
|
|
4375
|
-
|
|
5511
|
+
async function statusCommand(directory, flags = {}) {
|
|
5512
|
+
const observation = await observeDeployment(directory, { checkDocker: true, checkImage: true });
|
|
5513
|
+
const findings = observationFindings(observation);
|
|
5514
|
+
const state = observation.state;
|
|
5515
|
+
if (boolFlag(flags, "json")) {
|
|
5516
|
+
console.log(JSON.stringify({
|
|
5517
|
+
schemaVersion: 1,
|
|
5518
|
+
kind: "or3-deployment-status",
|
|
5519
|
+
observedAt: observation.observedAt,
|
|
5520
|
+
partial: observation.partial,
|
|
5521
|
+
inProgress: observation.changing,
|
|
5522
|
+
directory: observation.directory,
|
|
5523
|
+
state: state ? publicStateProjection(state) : null,
|
|
5524
|
+
stateError: observation.stateError,
|
|
5525
|
+
envError: observation.envError,
|
|
5526
|
+
lease: observation.lease.status,
|
|
5527
|
+
docker: observation.docker,
|
|
5528
|
+
recordedImageDigest: observation.recordedImageDigest,
|
|
5529
|
+
actualImageDigest: observation.actualImageDigest,
|
|
5530
|
+
identityMatches: observation.identityMatches,
|
|
5531
|
+
lastReceipt: state?.lastReceipt ?? null,
|
|
5532
|
+
backups: observation.backups,
|
|
5533
|
+
findings
|
|
5534
|
+
}, null, 2));
|
|
5535
|
+
return;
|
|
5536
|
+
}
|
|
5537
|
+
console.log(`OR3 Cloud deployment: ${observation.directory}`);
|
|
5538
|
+
if (observation.changing)
|
|
5539
|
+
console.log(" status: in progress (a mutation lease is active; this observation is not authoritative)");
|
|
5540
|
+
if (!state) {
|
|
5541
|
+
console.log(` state: unreadable${observation.stateError ? ` — ${observation.stateError}` : ""}`);
|
|
4376
5542
|
} else {
|
|
4377
|
-
console.log(`
|
|
5543
|
+
console.log(` mode: ${state.mode}`);
|
|
5544
|
+
console.log(` version: ${state.appVersion}`);
|
|
5545
|
+
if (state.domain)
|
|
5546
|
+
console.log(` domain: ${state.domain}`);
|
|
5547
|
+
console.log(` port: ${state.port}`);
|
|
5548
|
+
console.log(` image digest: ${state.imageDigest}`);
|
|
5549
|
+
console.log(` last successful operation: ${state.lastSuccessfulOperation} (${state.updatedAt})`);
|
|
5550
|
+
if (state.incompleteOperation)
|
|
5551
|
+
console.log(` incomplete operation: ${state.incompleteOperation.operation} (${state.incompleteOperation.id})`);
|
|
5552
|
+
if (state.lastError)
|
|
5553
|
+
console.log(` last error: ${state.lastError}`);
|
|
5554
|
+
}
|
|
5555
|
+
if (observation.envError)
|
|
5556
|
+
console.log(` environment: unreadable — ${observation.envError}`);
|
|
5557
|
+
if (observation.identityMatches === false)
|
|
5558
|
+
console.log(` image: MISMATCH (recorded ${observation.recordedImageDigest}, local ${observation.actualImageDigest})`);
|
|
5559
|
+
else if (observation.identityMatches === true)
|
|
5560
|
+
console.log(" image: digest matches managed state");
|
|
5561
|
+
console.log();
|
|
5562
|
+
if (state && observation.docker && observation.env) {
|
|
5563
|
+
try {
|
|
5564
|
+
const ps = await run("docker", [...composeArgs(directory, state.mode, ["ps"])], directory);
|
|
5565
|
+
console.log(ps.ok ? ps.stdout.trim() : `Could not list containers: ${ps.stderr}`);
|
|
5566
|
+
const health = await probeDeepHealth(directory, state.mode);
|
|
5567
|
+
if (health === "ok")
|
|
5568
|
+
console.log("Deep health: OK");
|
|
5569
|
+
else if (health === "degraded")
|
|
5570
|
+
console.log("Deep health: DEGRADED (the container is running but /api/health?deep=true is failing).");
|
|
5571
|
+
else
|
|
5572
|
+
console.log("Deep health: unreachable (the or3 container is not running).");
|
|
5573
|
+
await printMaintenanceSummary(directory, state.mode);
|
|
5574
|
+
} catch (error) {
|
|
5575
|
+
console.log(`Container and health checks were skipped: ${redact(error instanceof Error ? error.message : String(error))}`);
|
|
5576
|
+
}
|
|
5577
|
+
} else if (state && !observation.docker) {
|
|
5578
|
+
console.log("Docker is unavailable; container and health checks were skipped.");
|
|
5579
|
+
} else if (state && !observation.env) {
|
|
5580
|
+
console.log("The managed environment is unreadable; container and health checks were skipped.");
|
|
4378
5581
|
}
|
|
4379
|
-
const
|
|
4380
|
-
|
|
4381
|
-
console.log("Deep health: OK");
|
|
4382
|
-
else if (health === "degraded")
|
|
4383
|
-
console.log("Deep health: DEGRADED (the container is running but /api/health?deep=true is failing).");
|
|
4384
|
-
else
|
|
4385
|
-
console.log("Deep health: unreachable (the or3 container is not running).");
|
|
4386
|
-
await printMaintenanceSummary(directory, state.mode);
|
|
5582
|
+
for (const finding of findings)
|
|
5583
|
+
console.log(` [${finding.severity}] ${finding.code}: ${finding.message}`);
|
|
4387
5584
|
}
|
|
4388
5585
|
async function printMaintenanceSummary(directory, mode) {
|
|
4389
5586
|
const result = await run("docker", [...composeArgs(directory, mode, ["exec", "-T", "or3", ...containerNodeCommand(MAINTENANCE_SCRIPT)])], directory);
|
|
@@ -4403,7 +5600,7 @@ async function printMaintenanceSummary(directory, mode) {
|
|
|
4403
5600
|
} catch {}
|
|
4404
5601
|
}
|
|
4405
5602
|
async function logsCommand(directory, flags, positionals) {
|
|
4406
|
-
const loaded = await loadManaged(directory);
|
|
5603
|
+
const loaded = await loadManaged(directory, { writable: false });
|
|
4407
5604
|
assertDeploymentDirectoryIdentity(directory, loaded.state);
|
|
4408
5605
|
if (positionals.length > 1)
|
|
4409
5606
|
throw new Error("logs accepts at most one service name.");
|
|
@@ -4432,6 +5629,7 @@ async function startCommand(directory) {
|
|
|
4432
5629
|
await ensureDocker();
|
|
4433
5630
|
const loaded = await loadManaged(directory);
|
|
4434
5631
|
assertDeploymentDirectoryIdentity(directory, loaded.state);
|
|
5632
|
+
assertPhaseAllowsCommand("start", loaded.state);
|
|
4435
5633
|
await startProject(directory, loaded.state.mode, loaded.env);
|
|
4436
5634
|
const url = loaded.state.mode === "public" ? `https://${loaded.state.domain}` : `http://127.0.0.1:${loaded.state.port}`;
|
|
4437
5635
|
console.log(`OR3 started and is deeply healthy at ${url}.`);
|
|
@@ -4447,6 +5645,7 @@ async function restartCommand(directory) {
|
|
|
4447
5645
|
await ensureDocker();
|
|
4448
5646
|
const loaded = await loadManaged(directory);
|
|
4449
5647
|
assertDeploymentDirectoryIdentity(directory, loaded.state);
|
|
5648
|
+
assertPhaseAllowsCommand("restart", loaded.state);
|
|
4450
5649
|
await compose(directory, loaded.state.mode, ["restart", "or3"]);
|
|
4451
5650
|
await waitForDeepHealth(directory, loaded.state.mode, secretValues(loaded.env));
|
|
4452
5651
|
console.log("OR3 restarted and is deeply healthy.");
|
|
@@ -4588,8 +5787,11 @@ async function main(argv = process.argv.slice(2)) {
|
|
|
4588
5787
|
const dispatch = async () => {
|
|
4589
5788
|
if (command === "init")
|
|
4590
5789
|
return await initCommand(parsed.positionals, parsed.flags);
|
|
4591
|
-
if (command === "update")
|
|
5790
|
+
if (command === "update") {
|
|
5791
|
+
if (boolFlag(parsed.flags, "dry-run"))
|
|
5792
|
+
return await updatePreviewCommand(process.cwd(), parsed.flags);
|
|
4592
5793
|
return await updateCommand(process.cwd(), parsed.flags);
|
|
5794
|
+
}
|
|
4593
5795
|
if (command === "backup")
|
|
4594
5796
|
return await backupCommand(process.cwd(), parsed.positionals, parsed.flags);
|
|
4595
5797
|
if (command === "restore")
|
|
@@ -4600,14 +5802,17 @@ async function main(argv = process.argv.slice(2)) {
|
|
|
4600
5802
|
return await credentialsCommand(process.cwd(), parsed.positionals, parsed.flags);
|
|
4601
5803
|
if (command === "doctor")
|
|
4602
5804
|
return await doctorCommand(process.cwd());
|
|
4603
|
-
if (command === "verify")
|
|
5805
|
+
if (command === "verify") {
|
|
5806
|
+
if (verifyIsReadOnly(parsed.flags))
|
|
5807
|
+
return await verifyReadOnlyCommand(process.cwd(), parsed.flags);
|
|
4604
5808
|
return await verifyCommand(process.cwd(), parsed.flags, parsed.positionals);
|
|
5809
|
+
}
|
|
4605
5810
|
if (command === "recover")
|
|
4606
|
-
return await recoverCommand(process.cwd());
|
|
5811
|
+
return await recoverCommand(process.cwd(), parsed.flags);
|
|
4607
5812
|
if (command === "adopt")
|
|
4608
5813
|
return await adoptCommand(parsed.positionals, parsed.flags);
|
|
4609
5814
|
if (command === "status")
|
|
4610
|
-
return await statusCommand(process.cwd());
|
|
5815
|
+
return await statusCommand(process.cwd(), parsed.flags);
|
|
4611
5816
|
if (command === "logs")
|
|
4612
5817
|
return await logsCommand(process.cwd(), parsed.flags, parsed.positionals);
|
|
4613
5818
|
if (command === "start")
|
|
@@ -4620,7 +5825,8 @@ async function main(argv = process.argv.slice(2)) {
|
|
|
4620
5825
|
return await removeCommand(process.cwd(), parsed.flags);
|
|
4621
5826
|
throw new Error(`Unknown command "${command}". Run npx @or3/cloud --help.`);
|
|
4622
5827
|
};
|
|
4623
|
-
|
|
5828
|
+
const readOnlyInvocation = command === "verify" && verifyIsReadOnly(parsed.flags) || command === "update" && boolFlag(parsed.flags, "dry-run") || command === "recover" && boolFlag(parsed.flags, "dry-run") || command === "backup" && parsed.positionals[0] === "list";
|
|
5829
|
+
if (!readOnlyInvocation && MUTATING_COMMANDS.has(command)) {
|
|
4624
5830
|
return await withDeploymentLease(mutationDirectory(command, parsed.positionals, parsed.flags), command, dispatch);
|
|
4625
5831
|
}
|
|
4626
5832
|
return await dispatch();
|
|
@@ -4635,12 +5841,15 @@ if (invokedAsCli || process.argv[1] && resolve(process.argv[1]) === resolve(file
|
|
|
4635
5841
|
await main();
|
|
4636
5842
|
}
|
|
4637
5843
|
export {
|
|
5844
|
+
writeStateSchema,
|
|
4638
5845
|
withoutProvisioningCredentials,
|
|
5846
|
+
verifyIsReadOnly,
|
|
4639
5847
|
validateVerificationHealth,
|
|
4640
5848
|
validatePassword,
|
|
4641
5849
|
updateRequiresVolumeRecreation,
|
|
4642
5850
|
supportedImageArchitectures,
|
|
4643
5851
|
stateFromEnv,
|
|
5852
|
+
stateFingerprint,
|
|
4644
5853
|
snapshotManagedAssets,
|
|
4645
5854
|
serializeInitialCredentials,
|
|
4646
5855
|
serializeEnv,
|
|
@@ -4652,28 +5861,45 @@ export {
|
|
|
4652
5861
|
redact,
|
|
4653
5862
|
recordedBackupPath,
|
|
4654
5863
|
purgeVolumesFromState,
|
|
5864
|
+
publicStateProjection,
|
|
5865
|
+
planRetention,
|
|
4655
5866
|
parseFlags,
|
|
4656
5867
|
parseEnv,
|
|
5868
|
+
packagedMinimumSourceVersion,
|
|
5869
|
+
observeDeployment,
|
|
5870
|
+
observationFindings,
|
|
4657
5871
|
managedAssetNames,
|
|
5872
|
+
lifecycleFaults,
|
|
4658
5873
|
isVersion,
|
|
5874
|
+
inventoryBackups,
|
|
5875
|
+
installedManagedAssetChecksums,
|
|
4659
5876
|
enumerateBackups,
|
|
5877
|
+
decideRecoveryAction,
|
|
4660
5878
|
dashboardOperatorHandoffArgs,
|
|
4661
5879
|
copyAssets,
|
|
5880
|
+
compareReleaseVersions,
|
|
4662
5881
|
checkResolvedLoopbackBinding,
|
|
4663
5882
|
buildEnv,
|
|
4664
5883
|
buildCredentialsResetScript,
|
|
5884
|
+
assessUpdate,
|
|
4665
5885
|
assertVerificationGrant,
|
|
4666
5886
|
assertSupportedSourceCompose,
|
|
4667
5887
|
assertSupportedSource,
|
|
4668
5888
|
assertSupportedArchitecture,
|
|
5889
|
+
assertStateSchemaWritable,
|
|
4669
5890
|
assertRemovableArtifactName,
|
|
4670
5891
|
assertPurgeBackupFreshness,
|
|
5892
|
+
assertPhaseAllowsCommand,
|
|
5893
|
+
assertKnownStateSchema,
|
|
4671
5894
|
assertImageReleaseLabels,
|
|
4672
5895
|
assertEnoughFreeSpace,
|
|
4673
5896
|
assertDashboardOperatorMounts,
|
|
4674
5897
|
assertCommandPositionals,
|
|
4675
5898
|
assertCommandFlags,
|
|
4676
5899
|
assertBackupMatchesDeployment,
|
|
5900
|
+
STATE_SCHEMA_COMPATIBILITY,
|
|
5901
|
+
READER_SUPPORTED_STATE_SCHEMAS,
|
|
4677
5902
|
PACKAGE_VERSION,
|
|
5903
|
+
MAX_SUPPORTED_STATE_SCHEMA,
|
|
4678
5904
|
IMAGE_REPOSITORY
|
|
4679
5905
|
};
|