@indigoai-us/hq-cloud 6.16.51-alpha.0 → 6.16.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backup-prune.d.ts +20 -0
- package/dist/backup-prune.d.ts.map +1 -1
- package/dist/backup-prune.js +62 -3
- package/dist/backup-prune.js.map +1 -1
- package/dist/backup-prune.test.js +25 -0
- package/dist/backup-prune.test.js.map +1 -1
- package/dist/bin/rescue-runner.js +2 -1
- package/dist/bin/rescue-runner.js.map +1 -1
- package/dist/bin/sync-runner-company.d.ts +0 -6
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +1 -30
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-company.test.js +0 -55
- package/dist/bin/sync-runner-company.test.js.map +1 -1
- package/dist/bin/sync-runner-planning.d.ts +3 -12
- package/dist/bin/sync-runner-planning.d.ts.map +1 -1
- package/dist/bin/sync-runner-planning.js +3 -65
- package/dist/bin/sync-runner-planning.js.map +1 -1
- package/dist/bin/sync-runner-planning.test.js +1 -108
- package/dist/bin/sync-runner-planning.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +2 -48
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +113 -281
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +5 -33
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +4 -34
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +78 -555
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/rescue-breaker.test.d.ts +2 -0
- package/dist/cli/rescue-breaker.test.d.ts.map +1 -0
- package/dist/cli/rescue-breaker.test.js +334 -0
- package/dist/cli/rescue-breaker.test.js.map +1 -0
- package/dist/cli/rescue-core.d.ts +40 -0
- package/dist/cli/rescue-core.d.ts.map +1 -1
- package/dist/cli/rescue-core.js +1205 -109
- package/dist/cli/rescue-core.js.map +1 -1
- package/dist/cli/rescue-realpath.test.d.ts +2 -0
- package/dist/cli/rescue-realpath.test.d.ts.map +1 -0
- package/dist/cli/rescue-realpath.test.js +109 -0
- package/dist/cli/rescue-realpath.test.js.map +1 -0
- package/dist/cli/rescue-rsync-preflight.test.js +256 -4
- package/dist/cli/rescue-rsync-preflight.test.js.map +1 -1
- package/dist/cli/rescue-snapshot-copy.test.d.ts +2 -0
- package/dist/cli/rescue-snapshot-copy.test.d.ts.map +1 -0
- package/dist/cli/rescue-snapshot-copy.test.js +435 -0
- package/dist/cli/rescue-snapshot-copy.test.js.map +1 -0
- package/dist/cli/rescue-snapshot-recovery.test.d.ts +2 -0
- package/dist/cli/rescue-snapshot-recovery.test.d.ts.map +1 -0
- package/dist/cli/rescue-snapshot-recovery.test.js +260 -0
- package/dist/cli/rescue-snapshot-recovery.test.js.map +1 -0
- package/dist/cli/rescue-source-switch-floor.test.d.ts +2 -0
- package/dist/cli/rescue-source-switch-floor.test.d.ts.map +1 -0
- package/dist/cli/rescue-source-switch-floor.test.js +210 -0
- package/dist/cli/rescue-source-switch-floor.test.js.map +1 -0
- package/dist/cli/rescue.d.ts +4 -0
- package/dist/cli/rescue.d.ts.map +1 -1
- package/dist/cli/rescue.js +20 -1
- package/dist/cli/rescue.js.map +1 -1
- package/dist/cli/rescue.test.js +16 -1
- package/dist/cli/rescue.test.js.map +1 -1
- package/dist/cli/share-presign-transport.test.js +2 -0
- package/dist/cli/share-presign-transport.test.js.map +1 -1
- package/dist/cli/share.d.ts +0 -19
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +37 -57
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +93 -87
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync-scope.test.js +0 -8
- package/dist/cli/sync-scope.test.js.map +1 -1
- package/dist/cli/sync.d.ts +0 -12
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +118 -226
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +2 -215
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/cli/watch-event-push-conflict.test.js +2 -0
- package/dist/cli/watch-event-push-conflict.test.js.map +1 -1
- package/dist/cognito-auth.d.ts +30 -0
- package/dist/cognito-auth.d.ts.map +1 -1
- package/dist/cognito-auth.js +175 -0
- package/dist/cognito-auth.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/journal-store.test.js +7 -130
- package/dist/journal-store.test.js.map +1 -1
- package/dist/journal.d.ts +1 -9
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +21 -30
- package/dist/journal.js.map +1 -1
- package/dist/lib/exit-codes.d.ts +0 -15
- package/dist/lib/exit-codes.d.ts.map +1 -1
- package/dist/lib/exit-codes.js +0 -15
- package/dist/lib/exit-codes.js.map +1 -1
- package/dist/machine-auth.test.js +295 -0
- package/dist/machine-auth.test.js.map +1 -1
- package/dist/object-io.d.ts +2 -2
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +18 -3
- package/dist/object-io.js.map +1 -1
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +3 -10
- package/dist/personal-vault.js.map +1 -1
- package/dist/personal-vault.test.js +0 -35
- package/dist/personal-vault.test.js.map +1 -1
- package/dist/remote-pull.d.ts.map +1 -1
- package/dist/remote-pull.js +2 -9
- package/dist/remote-pull.js.map +1 -1
- package/dist/remote-pull.test.js +2 -59
- package/dist/remote-pull.test.js.map +1 -1
- package/dist/s3.d.ts +18 -1
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +47 -24
- package/dist/s3.js.map +1 -1
- package/dist/s3.test.js +27 -1
- package/dist/s3.test.js.map +1 -1
- package/dist/sync/realtime-telemetry.d.ts +1 -1
- package/dist/sync/realtime-telemetry.d.ts.map +1 -1
- package/dist/sync/realtime-telemetry.js +6 -14
- package/dist/sync/realtime-telemetry.js.map +1 -1
- package/dist/sync/realtime-telemetry.test.js +16 -136
- package/dist/sync/realtime-telemetry.test.js.map +1 -1
- package/dist/sync-core.d.ts +6 -4
- package/dist/sync-core.d.ts.map +1 -1
- package/dist/sync-core.js +10 -4
- package/dist/sync-core.js.map +1 -1
- package/dist/sync-core.test.js +25 -1
- package/dist/sync-core.test.js.map +1 -1
- package/package.json +1 -2
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts +0 -2
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts.map +0 -1
- package/dist/cli/onepass-pull-planner-regressions.test.js +0 -300
- package/dist/cli/onepass-pull-planner-regressions.test.js.map +0 -1
- package/dist/cli/onepass-pull-planner.test.d.ts +0 -2
- package/dist/cli/onepass-pull-planner.test.d.ts.map +0 -1
- package/dist/cli/onepass-pull-planner.test.js +0 -266
- package/dist/cli/onepass-pull-planner.test.js.map +0 -1
- package/dist/cli/streaming-pull-planner.bench.test.d.ts +0 -15
- package/dist/cli/streaming-pull-planner.bench.test.d.ts.map +0 -1
- package/dist/cli/streaming-pull-planner.bench.test.js +0 -273
- package/dist/cli/streaming-pull-planner.bench.test.js.map +0 -1
- package/dist/s3.list-cost.test.d.ts +0 -2
- package/dist/s3.list-cost.test.d.ts.map +0 -1
- package/dist/s3.list-cost.test.js +0 -51
- package/dist/s3.list-cost.test.js.map +0 -1
- package/dist/sync/remote-file-spool.d.ts +0 -16
- package/dist/sync/remote-file-spool.d.ts.map +0 -1
- package/dist/sync/remote-file-spool.js +0 -91
- package/dist/sync/remote-file-spool.js.map +0 -1
- package/dist/sync/remote-file-spool.test.d.ts +0 -2
- package/dist/sync/remote-file-spool.test.d.ts.map +0 -1
- package/dist/sync/remote-file-spool.test.js +0 -77
- package/dist/sync/remote-file-spool.test.js.map +0 -1
package/dist/cli/rescue-core.js
CHANGED
|
@@ -31,7 +31,7 @@ import * as fs from "fs";
|
|
|
31
31
|
import * as os from "os";
|
|
32
32
|
import * as path from "path";
|
|
33
33
|
import { assertRescuePathTypeUnchanged, isRescueTransientDirectory, } from "./rescue-snapshot.js";
|
|
34
|
-
import { applyBackupPrune, canonicalRootHash, inventoryManagedSnapshots, } from "../backup-prune.js";
|
|
34
|
+
import { applyBackupPrune, canonicalRootHash, inventoryManagedSnapshots, isManagedSnapshotManifest, } from "../backup-prune.js";
|
|
35
35
|
// Paths always preserved across wipe+overlay (shuttled out, restored after).
|
|
36
36
|
const CARVE_OUT_PATHS = [
|
|
37
37
|
"core/packages",
|
|
@@ -148,10 +148,26 @@ function boundedDiagnostic(value, limit) {
|
|
|
148
148
|
function pathEntries(pathValue) {
|
|
149
149
|
return pathValue ? pathValue.split(path.delimiter) : [];
|
|
150
150
|
}
|
|
151
|
-
|
|
151
|
+
const DEFAULT_WINDOWS_EXECUTABLE_EXTENSIONS = [".COM", ".EXE", ".BAT", ".CMD"];
|
|
152
|
+
function rsyncCandidateNames(env) {
|
|
153
|
+
if (process.platform !== "win32")
|
|
154
|
+
return ["rsync", "rsync.exe"];
|
|
155
|
+
const configuredExtensions = envValue(env, "PATHEXT")
|
|
156
|
+
?.split(";")
|
|
157
|
+
.map((extension) => extension.trim())
|
|
158
|
+
.filter((extension) => /^\.[A-Za-z0-9]+$/.test(extension));
|
|
159
|
+
const extensions = [
|
|
160
|
+
...(configuredExtensions?.length
|
|
161
|
+
? configuredExtensions
|
|
162
|
+
: DEFAULT_WINDOWS_EXECUTABLE_EXTENSIONS),
|
|
163
|
+
...DEFAULT_WINDOWS_EXECUTABLE_EXTENSIONS,
|
|
164
|
+
];
|
|
165
|
+
return [...new Set(["rsync", ...extensions.map((extension) => `rsync${extension.toLowerCase()}`)])];
|
|
166
|
+
}
|
|
167
|
+
export function findRsyncOnPath(entries, env) {
|
|
152
168
|
let firstUnexecutable;
|
|
153
169
|
for (const entry of entries) {
|
|
154
|
-
for (const name of
|
|
170
|
+
for (const name of rsyncCandidateNames(env)) {
|
|
155
171
|
const candidate = path.resolve(entry || ".", name);
|
|
156
172
|
try {
|
|
157
173
|
if (!fs.statSync(candidate).isFile())
|
|
@@ -174,14 +190,22 @@ function findRsyncOnPath(entries) {
|
|
|
174
190
|
: undefined;
|
|
175
191
|
}
|
|
176
192
|
/** The overlay requires rsync; do not allocate a recovery copy when it cannot run. */
|
|
177
|
-
function requireWorkingRsync(env, err) {
|
|
193
|
+
export function requireWorkingRsync(env, err) {
|
|
178
194
|
const resolvedPath = envValue(env, "PATH");
|
|
179
195
|
const entries = pathEntries(resolvedPath);
|
|
180
|
-
const foundRsync = findRsyncOnPath(entries);
|
|
181
|
-
|
|
196
|
+
const foundRsync = findRsyncOnPath(entries, env);
|
|
197
|
+
// Node documents that Windows .cmd/.bat files require cmd.exe rather than a
|
|
198
|
+
// direct child-process launch. Keep the path and fixed argument as distinct
|
|
199
|
+
// arguments instead of enabling shell:true for a constructed command string.
|
|
200
|
+
const launcher = process.platform === "win32" && /\.(?:cmd|bat)$/i.test(foundRsync?.path ?? "")
|
|
201
|
+
? { command: "cmd.exe", leadingArgs: ["/d", "/s", "/c", foundRsync.path] }
|
|
202
|
+
: { command: "rsync", leadingArgs: [] };
|
|
203
|
+
const probe = runWithSpawnError(launcher.command, [...launcher.leadingArgs, "--version"], {
|
|
204
|
+
env,
|
|
205
|
+
});
|
|
182
206
|
const output = `${probe.stdout}\n${probe.stderr}`;
|
|
183
207
|
if (probe.status === 0 && /\brsync\b.*\bversion\b/i.test(output))
|
|
184
|
-
return;
|
|
208
|
+
return launcher;
|
|
185
209
|
err("error: rsync preflight failed before any safety snapshot was allocated.\n");
|
|
186
210
|
if (probe.error) {
|
|
187
211
|
const errorDetail = "code" in probe.error && typeof probe.error.code === "string"
|
|
@@ -303,6 +327,9 @@ export function parseRescueArgs(argv) {
|
|
|
303
327
|
backupRoot: process.env.HQ_BACKUP_DIR || path.join(os.homedir(), ".hq", "backups"),
|
|
304
328
|
backupRetentionDays: process.env.HQ_BACKUP_RETENTION_DAYS || "7",
|
|
305
329
|
backupRetentionCount: process.env.HQ_BACKUP_RETENTION_COUNT || "2",
|
|
330
|
+
recoverSnapshots: false,
|
|
331
|
+
verifyRestoredSnapshot: "",
|
|
332
|
+
restoreSnapshot: "",
|
|
306
333
|
};
|
|
307
334
|
const usage = () => {
|
|
308
335
|
// The legacy script printed its comment header here; we emit a concise
|
|
@@ -311,7 +338,9 @@ export function parseRescueArgs(argv) {
|
|
|
311
338
|
" [--preserve PATH]... [--preserve-subpath REL]...\n" +
|
|
312
339
|
" [--hq-root DIR] [--no-history-check] [--floor-sha SHA]\n" +
|
|
313
340
|
" [--no-backup] [--backup-dir DIR] [--cloud-update]\n" +
|
|
314
|
-
" [--dry-run] [--yes]\n"
|
|
341
|
+
" [--dry-run] [--yes] [--recover-snapshots]\n" +
|
|
342
|
+
" [--verify-restored-snapshot SNAPSHOT]\n" +
|
|
343
|
+
" [--restore-snapshot SNAPSHOT]\n");
|
|
315
344
|
throw new ExitError(1);
|
|
316
345
|
};
|
|
317
346
|
let narrowCsv = "";
|
|
@@ -360,6 +389,23 @@ export function parseRescueArgs(argv) {
|
|
|
360
389
|
case "--backup-dir":
|
|
361
390
|
cfg.backupRoot = need(a);
|
|
362
391
|
break;
|
|
392
|
+
case "--recover-snapshots":
|
|
393
|
+
cfg.recoverSnapshots = true;
|
|
394
|
+
break;
|
|
395
|
+
case "--verify-restored-snapshot":
|
|
396
|
+
cfg.verifyRestoredSnapshot = need(a);
|
|
397
|
+
if (!cfg.verifyRestoredSnapshot) {
|
|
398
|
+
process.stderr.write("error: --verify-restored-snapshot requires a snapshot name.\n");
|
|
399
|
+
throw new ExitError(2);
|
|
400
|
+
}
|
|
401
|
+
break;
|
|
402
|
+
case "--restore-snapshot":
|
|
403
|
+
cfg.restoreSnapshot = need(a);
|
|
404
|
+
if (!cfg.restoreSnapshot) {
|
|
405
|
+
process.stderr.write("error: --restore-snapshot requires a snapshot name.\n");
|
|
406
|
+
throw new ExitError(2);
|
|
407
|
+
}
|
|
408
|
+
break;
|
|
363
409
|
case "--dry-run":
|
|
364
410
|
cfg.dryRun = true;
|
|
365
411
|
break;
|
|
@@ -420,6 +466,18 @@ export function parseRescueArgs(argv) {
|
|
|
420
466
|
throw new ExitError(2);
|
|
421
467
|
}
|
|
422
468
|
}
|
|
469
|
+
if (cfg.verifyRestoredSnapshot && !cfg.recoverSnapshots) {
|
|
470
|
+
process.stderr.write("error: --verify-restored-snapshot requires --recover-snapshots.\n");
|
|
471
|
+
throw new ExitError(2);
|
|
472
|
+
}
|
|
473
|
+
if (cfg.restoreSnapshot && !cfg.recoverSnapshots) {
|
|
474
|
+
process.stderr.write("error: --restore-snapshot requires --recover-snapshots.\n");
|
|
475
|
+
throw new ExitError(2);
|
|
476
|
+
}
|
|
477
|
+
if (cfg.restoreSnapshot && cfg.verifyRestoredSnapshot) {
|
|
478
|
+
process.stderr.write("error: --restore-snapshot and --verify-restored-snapshot must be run separately.\n");
|
|
479
|
+
throw new ExitError(2);
|
|
480
|
+
}
|
|
423
481
|
return cfg;
|
|
424
482
|
}
|
|
425
483
|
/**
|
|
@@ -453,6 +511,8 @@ export function runRescue(argv, opts = {}) {
|
|
|
453
511
|
}
|
|
454
512
|
};
|
|
455
513
|
try {
|
|
514
|
+
if (cfg.recoverSnapshots)
|
|
515
|
+
return recoverSnapshots(cfg, out, err);
|
|
456
516
|
return doRescue(cfg, env, out, err, (d) => {
|
|
457
517
|
tmpdir = d;
|
|
458
518
|
});
|
|
@@ -466,38 +526,8 @@ export function runRescue(argv, opts = {}) {
|
|
|
466
526
|
cleanup();
|
|
467
527
|
}
|
|
468
528
|
}
|
|
469
|
-
/**
|
|
470
|
-
|
|
471
|
-
* a bare drive letter. Pure + exported for tests; `platformPath` defaults to
|
|
472
|
-
* the host `path` module so Windows and POSIX runtimes each apply their own
|
|
473
|
-
* absoluteness rules (tests pass `path.win32` / `path.posix` explicitly).
|
|
474
|
-
*
|
|
475
|
-
* Rejecting bare `C:` matters even though `path.win32.isAbsolute("C:")` is
|
|
476
|
-
* already false — the explicit pattern documents the exact field failure and
|
|
477
|
-
* guards against a future helper swap that treats drive-relative paths as
|
|
478
|
-
* absolute.
|
|
479
|
-
*/
|
|
480
|
-
export function isUsableRescueRoot(root, platformPath = path) {
|
|
481
|
-
if (/^[A-Za-z]:$/.test(root))
|
|
482
|
-
return false;
|
|
483
|
-
return platformPath.isAbsolute(root);
|
|
484
|
-
}
|
|
485
|
-
/**
|
|
486
|
-
* `fs.realpathSync` that converts failure into the rescue's clean error path
|
|
487
|
-
* (message + ExitError) instead of an uncaught exception. An uncaught throw
|
|
488
|
-
* here is what turned one bad `--hq-root` into a daily crash-loop on Windows.
|
|
489
|
-
*/
|
|
490
|
-
function realpathOrExit(p, err) {
|
|
491
|
-
try {
|
|
492
|
-
return fs.realpathSync(p);
|
|
493
|
-
}
|
|
494
|
-
catch (e) {
|
|
495
|
-
err(`error: cannot resolve --hq-root ${p}: ${e instanceof Error ? e.message : String(e)}\n`);
|
|
496
|
-
throw new ExitError(1);
|
|
497
|
-
}
|
|
498
|
-
}
|
|
499
|
-
function doRescue(cfg, env, out, err, setTmp) {
|
|
500
|
-
// --- Resolve HQ root ---
|
|
529
|
+
/** Resolve and sanity-check the HQ root before either rescue mode can touch it. */
|
|
530
|
+
function resolveHqRoot(cfg, err) {
|
|
501
531
|
let hqRoot;
|
|
502
532
|
if (cfg.hqRootOverride) {
|
|
503
533
|
// Windows field failure (2026-08-02): the menubar app's daily background
|
|
@@ -544,11 +574,556 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
544
574
|
err(" pass --hq-root <dir> if the script is not at personal/skills/<skill>/.\n");
|
|
545
575
|
throw new ExitError(3);
|
|
546
576
|
}
|
|
577
|
+
return hqRoot;
|
|
578
|
+
}
|
|
579
|
+
/**
|
|
580
|
+
* Explicit recovery surface for a previously interrupted rescue.
|
|
581
|
+
*
|
|
582
|
+
* A recovery-required snapshot is evidence that a rescue started changing the
|
|
583
|
+
* release-managed tree and then failed. It is safe to delete only when the
|
|
584
|
+
* retention model has a later `completed-good` snapshot for this same canonical
|
|
585
|
+
* HQ root. That later manifest is the durable proof that a subsequent rescue
|
|
586
|
+
* completed; an age, an empty current tree, or a manually edited sync stamp is
|
|
587
|
+
* deliberately not accepted as proof.
|
|
588
|
+
*/
|
|
589
|
+
function recoverSnapshots(cfg, out, err) {
|
|
590
|
+
const hqRoot = resolveHqRoot(cfg, err);
|
|
591
|
+
const retention = {
|
|
592
|
+
keepGood: Number(cfg.backupRetentionCount),
|
|
593
|
+
maxAgeDays: Number(cfg.backupRetentionDays),
|
|
594
|
+
};
|
|
595
|
+
const isCopyingOwnerLive = (m) => {
|
|
596
|
+
try {
|
|
597
|
+
process.kill(m.creatorPid, 0);
|
|
598
|
+
return true;
|
|
599
|
+
}
|
|
600
|
+
catch {
|
|
601
|
+
return false;
|
|
602
|
+
}
|
|
603
|
+
};
|
|
604
|
+
const inventory = inventoryManagedSnapshots(cfg.backupRoot, hqRoot, isCopyingOwnerLive, retention);
|
|
605
|
+
if (inventory.unmanaged.some((item) => item.reason === "backup-root-unreadable")) {
|
|
606
|
+
err(`error: cannot inspect safety snapshots: backup root is unreadable: ${cfg.backupRoot}\n`);
|
|
607
|
+
return { status: 1 };
|
|
608
|
+
}
|
|
609
|
+
if (cfg.restoreSnapshot) {
|
|
610
|
+
return restoreRecoverySnapshot(cfg, hqRoot, inventory.entries, inventory.protected, out, err);
|
|
611
|
+
}
|
|
612
|
+
if (cfg.verifyRestoredSnapshot) {
|
|
613
|
+
return verifyRestoredSnapshot(cfg, hqRoot, inventory.entries, inventory.protected, out, err);
|
|
614
|
+
}
|
|
615
|
+
const verified = inventory.candidates.filter((item) => item.entry.manifest.status === "recovery-required" &&
|
|
616
|
+
item.reason === "recovery-required-superseded");
|
|
617
|
+
const blocked = inventory.protected.filter((item) => item.reason === "recovery-required-no-later-good");
|
|
618
|
+
const retainedByGrace = inventory.protected.filter((item) => item.entry.manifest.status === "recovery-required" &&
|
|
619
|
+
item.reason !== "recovery-required-no-later-good");
|
|
620
|
+
const laterGoodFor = (completedMs, name) => inventory.entries.find((entry) => entry.manifest.status === "completed-good" &&
|
|
621
|
+
(entry.completedMs > completedMs ||
|
|
622
|
+
(entry.completedMs === completedMs && entry.name > name)))?.name;
|
|
623
|
+
out(`==> Safety snapshot recovery inspection: ${cfg.backupRoot}\n`);
|
|
624
|
+
if (verified.length === 0 && blocked.length === 0 && retainedByGrace.length === 0) {
|
|
625
|
+
out(" No recovery-required snapshots need action.\n");
|
|
626
|
+
return { status: 0 };
|
|
627
|
+
}
|
|
628
|
+
for (const item of verified) {
|
|
629
|
+
const laterGood = laterGoodFor(item.entry.completedMs, item.entry.name);
|
|
630
|
+
out(` verified ${item.entry.name}: later completed-good snapshot ${laterGood ?? "(revalidated during delete)"} proves this recovery evidence is superseded.\n`);
|
|
631
|
+
}
|
|
632
|
+
for (const item of blocked) {
|
|
633
|
+
out(` retained ${item.entry.name}: no later completed-good snapshot exists for this HQ root.\n`);
|
|
634
|
+
}
|
|
635
|
+
for (const item of retainedByGrace) {
|
|
636
|
+
out(` retained ${item.entry.name}: a later completed-good snapshot exists, but normal retention still keeps this newest recovery snapshot.\n`);
|
|
637
|
+
}
|
|
638
|
+
if (verified.length > 0) {
|
|
639
|
+
if (cfg.dryRun) {
|
|
640
|
+
for (const item of verified) {
|
|
641
|
+
out(` would resolve ${item.entry.name}: verified superseded recovery snapshot would be retained during this dry run.\n`);
|
|
642
|
+
}
|
|
643
|
+
}
|
|
644
|
+
else {
|
|
645
|
+
const result = applyBackupPrune(cfg.backupRoot, hqRoot, { candidates: verified, protected: [], unmanaged: [] }, retention);
|
|
646
|
+
for (const name of result.deleted) {
|
|
647
|
+
out(` resolved ${name}: verified superseded recovery snapshot deleted.\n`);
|
|
648
|
+
}
|
|
649
|
+
for (const failure of result.failures) {
|
|
650
|
+
err(`error: could not resolve verified snapshot (retained): ${failure}\n`);
|
|
651
|
+
}
|
|
652
|
+
if (result.failures.length > 0)
|
|
653
|
+
return { status: 1 };
|
|
654
|
+
}
|
|
655
|
+
}
|
|
656
|
+
if (cfg.dryRun && blocked.length === 0) {
|
|
657
|
+
out("==> Snapshot recovery dry run complete: no snapshot was changed.\n");
|
|
658
|
+
return { status: 0 };
|
|
659
|
+
}
|
|
660
|
+
// Re-inventory after an apply because another process might have created or
|
|
661
|
+
// changed a snapshot while the prune lock was held. Nothing unverified has
|
|
662
|
+
// been deleted, and the updated inventory is the report we return to users.
|
|
663
|
+
const remaining = inventoryManagedSnapshots(cfg.backupRoot, hqRoot, isCopyingOwnerLive, retention);
|
|
664
|
+
const unresolved = remaining.protected.filter((item) => item.reason === "recovery-required-no-later-good");
|
|
665
|
+
if (unresolved.length > 0) {
|
|
666
|
+
err("error: snapshot recovery refused: the interrupted updates are not proven superseded.\n");
|
|
667
|
+
err(" HQ retained every unverified snapshot. A later completed-good rescue for this same HQ root is the only automatic proof.\n");
|
|
668
|
+
err(" Restore one listed snapshot, then verify it before one retry is permitted:\n");
|
|
669
|
+
for (const item of unresolved) {
|
|
670
|
+
err(` restore ${item.entry.name}: ${formatSnapshotRestoreCommand(hqRoot, cfg.backupRoot, item.entry.name)}\n`);
|
|
671
|
+
err(` verify ${item.entry.name}: ${formatRecoveryCommand(hqRoot, cfg.backupRoot, item.entry.name)}\n`);
|
|
672
|
+
}
|
|
673
|
+
return { status: 1 };
|
|
674
|
+
}
|
|
675
|
+
out("==> Snapshot recovery complete: every eligible recovery snapshot was positively verified before deletion.\n");
|
|
676
|
+
return { status: 0 };
|
|
677
|
+
}
|
|
678
|
+
/**
|
|
679
|
+
* Explicitly restore every captured root, including links represented only in
|
|
680
|
+
* the snapshot manifest. Verification remains a separate positive-proof step.
|
|
681
|
+
*/
|
|
682
|
+
function restoreRecoverySnapshot(cfg, hqRoot, entries, protectedEntries, out, err) {
|
|
683
|
+
const entry = entries.find((item) => item.name === cfg.restoreSnapshot);
|
|
684
|
+
const unresolved = protectedEntries.find((item) => item.entry.name === cfg.restoreSnapshot &&
|
|
685
|
+
item.reason === "recovery-required-no-later-good");
|
|
686
|
+
if (!entry || !unresolved || entry.manifest.status !== "recovery-required") {
|
|
687
|
+
err(`error: ${cfg.restoreSnapshot} is not an unresolved recovery-required snapshot for this HQ root.\n`);
|
|
688
|
+
return { status: 1 };
|
|
689
|
+
}
|
|
690
|
+
if (cfg.dryRun) {
|
|
691
|
+
out(`==> Dry run would restore every captured root and recorded symlink from ${entry.name}.\n`);
|
|
692
|
+
return { status: 0 };
|
|
693
|
+
}
|
|
694
|
+
try {
|
|
695
|
+
restoreSnapshotPaths(path.join(cfg.backupRoot, entry.name), hqRoot, entry.manifest.includedRoots, entry.manifest);
|
|
696
|
+
}
|
|
697
|
+
catch (error) {
|
|
698
|
+
err(`error: could not restore safety snapshot ${entry.name}: ${error instanceof Error ? error.message : String(error)}\n`);
|
|
699
|
+
err(" HQ retained the snapshot and did not write a post-restore permit.\n");
|
|
700
|
+
return { status: 1 };
|
|
701
|
+
}
|
|
702
|
+
out(`==> Restored every captured root and recorded symlink from ${entry.name}.\n`);
|
|
703
|
+
out(` Verify the restored tree before retrying: ${formatRecoveryCommand(hqRoot, cfg.backupRoot, entry.name)}\n`);
|
|
704
|
+
return { status: 0 };
|
|
705
|
+
}
|
|
706
|
+
/** A live tree must still match the selected recovery snapshot at permit use. */
|
|
707
|
+
function verifyRestoredSnapshot(cfg, hqRoot, entries, protectedEntries, out, err) {
|
|
708
|
+
const entry = entries.find((item) => item.name === cfg.verifyRestoredSnapshot);
|
|
709
|
+
const unresolved = protectedEntries.find((item) => item.entry.name === cfg.verifyRestoredSnapshot &&
|
|
710
|
+
item.reason === "recovery-required-no-later-good");
|
|
711
|
+
if (!entry || !unresolved || entry.manifest.status !== "recovery-required") {
|
|
712
|
+
err(`error: ${cfg.verifyRestoredSnapshot} is not an unresolved recovery-required snapshot for this HQ root.\n`);
|
|
713
|
+
return { status: 1 };
|
|
714
|
+
}
|
|
715
|
+
const mismatch = snapshotTreeMismatch(path.join(cfg.backupRoot, entry.name), hqRoot, entry.manifest.includedRoots, entry.manifest.symlinks);
|
|
716
|
+
if (mismatch) {
|
|
717
|
+
err(`error: restored tree does not match snapshot ${entry.name}: ${mismatch}\n`);
|
|
718
|
+
err(" No post-restore permit was written; the circuit breaker remains armed.\n");
|
|
719
|
+
return { status: 1 };
|
|
720
|
+
}
|
|
721
|
+
if (cfg.dryRun) {
|
|
722
|
+
out(`==> Dry run verified restored snapshot ${entry.name}; no post-restore permit was written.\n`);
|
|
723
|
+
return { status: 0 };
|
|
724
|
+
}
|
|
725
|
+
try {
|
|
726
|
+
writePostRestorePermit(hqRoot, entry);
|
|
727
|
+
}
|
|
728
|
+
catch (error) {
|
|
729
|
+
err(`error: verified snapshot ${entry.name}, but could not write the post-restore permit: ${error instanceof Error ? error.message : String(error)}\n`);
|
|
730
|
+
return { status: 1 };
|
|
731
|
+
}
|
|
732
|
+
out(`==> Verified restored snapshot ${entry.name}; one rescue attempt is now permitted.\n`);
|
|
733
|
+
out(" That permit is rechecked and consumed before rescue changes files. If the attempt fails, the circuit breaker is armed again.\n");
|
|
734
|
+
return { status: 0 };
|
|
735
|
+
}
|
|
736
|
+
/** Return a precise mismatch instead of treating a partial restore as evidence. */
|
|
737
|
+
function snapshotTreeMismatch(snapshotRoot, hqRoot, includedRoots, symlinks) {
|
|
738
|
+
const roots = snapshotCapturedRoots(includedRoots);
|
|
739
|
+
if (typeof roots === "string")
|
|
740
|
+
return roots;
|
|
741
|
+
const recorded = snapshotSymlinkTargets(hqRoot, roots, symlinks);
|
|
742
|
+
if (typeof recorded === "string")
|
|
743
|
+
return recorded;
|
|
744
|
+
for (const root of roots) {
|
|
745
|
+
if (rescuePathKind(path.join(snapshotRoot, root)) === "missing" &&
|
|
746
|
+
!recorded.has(root)) {
|
|
747
|
+
return `snapshot does not contain captured root ${root}`;
|
|
748
|
+
}
|
|
749
|
+
const mismatch = snapshotPathMismatch(path.join(snapshotRoot, root), path.join(hqRoot, root), root, recorded);
|
|
750
|
+
if (mismatch)
|
|
751
|
+
return mismatch;
|
|
752
|
+
}
|
|
753
|
+
return null;
|
|
754
|
+
}
|
|
755
|
+
function snapshotCapturedRoots(includedRoots) {
|
|
756
|
+
if (!Array.isArray(includedRoots) || includedRoots.length === 0) {
|
|
757
|
+
return "snapshot manifest records no captured roots to verify";
|
|
758
|
+
}
|
|
759
|
+
const seen = new Set();
|
|
760
|
+
for (const root of includedRoots) {
|
|
761
|
+
if (typeof root !== "string" ||
|
|
762
|
+
!root ||
|
|
763
|
+
root === "." ||
|
|
764
|
+
root === ".." ||
|
|
765
|
+
root.includes("/") ||
|
|
766
|
+
root.includes("\\") ||
|
|
767
|
+
isRescueTransientDirectory(root) ||
|
|
768
|
+
seen.has(root)) {
|
|
769
|
+
return `snapshot manifest has an unsafe captured root ${JSON.stringify(root)}`;
|
|
770
|
+
}
|
|
771
|
+
seen.add(root);
|
|
772
|
+
}
|
|
773
|
+
return seen;
|
|
774
|
+
}
|
|
775
|
+
function snapshotSymlinkTargets(hqRoot, includedRoots, symlinks) {
|
|
776
|
+
if (symlinks !== undefined && !Array.isArray(symlinks)) {
|
|
777
|
+
return "snapshot manifest symlinks is not an array";
|
|
778
|
+
}
|
|
779
|
+
const recorded = new Map();
|
|
780
|
+
for (const link of symlinks ?? []) {
|
|
781
|
+
if (!link ||
|
|
782
|
+
typeof link.path !== "string" ||
|
|
783
|
+
(link.representation !== undefined && link.representation !== "file-copy")) {
|
|
784
|
+
return "snapshot manifest has an invalid recorded symlink";
|
|
785
|
+
}
|
|
786
|
+
const problem = validateSnapshotSymlink(hqRoot, link.path, link.target);
|
|
787
|
+
if (problem)
|
|
788
|
+
return `snapshot manifest ${problem}`;
|
|
789
|
+
const root = link.path.split("/")[0];
|
|
790
|
+
if (!includedRoots.has(root)) {
|
|
791
|
+
return `snapshot manifest records symlink ${link.path} outside its captured roots`;
|
|
792
|
+
}
|
|
793
|
+
if (recorded.has(link.path)) {
|
|
794
|
+
return `snapshot manifest records duplicate symlink ${link.path}`;
|
|
795
|
+
}
|
|
796
|
+
recorded.set(link.path, link);
|
|
797
|
+
}
|
|
798
|
+
for (const linkPath of recorded.keys()) {
|
|
799
|
+
for (let parent = path.posix.dirname(linkPath); parent !== "."; parent = path.posix.dirname(parent)) {
|
|
800
|
+
if (recorded.has(parent)) {
|
|
801
|
+
return `snapshot manifest records symlink ${linkPath} beneath recorded symlink ${parent}`;
|
|
802
|
+
}
|
|
803
|
+
}
|
|
804
|
+
}
|
|
805
|
+
return recorded;
|
|
806
|
+
}
|
|
807
|
+
function recordedSymlinkTargetMatches(livePath, recordedTarget, liveTarget) {
|
|
808
|
+
const parent = path.dirname(livePath);
|
|
809
|
+
return (path.resolve(parent, recordedTarget) === path.resolve(parent, liveTarget));
|
|
810
|
+
}
|
|
811
|
+
function snapshotPathMismatch(snapshotPath, livePath, rel, recordedSymlinks) {
|
|
812
|
+
const snapshotKind = rescuePathKind(snapshotPath);
|
|
813
|
+
const liveKind = rescuePathKind(livePath);
|
|
814
|
+
const recordedLink = recordedSymlinks.get(rel);
|
|
815
|
+
if (recordedLink !== undefined) {
|
|
816
|
+
if (recordedLink.representation === "file-copy") {
|
|
817
|
+
if (snapshotKind !== "file") {
|
|
818
|
+
return `${rel} is ${snapshotKind} but the snapshot records a copied file link`;
|
|
819
|
+
}
|
|
820
|
+
if (liveKind !== "file") {
|
|
821
|
+
return `${rel} is ${liveKind} but the snapshot records a copied file link`;
|
|
822
|
+
}
|
|
823
|
+
return bytesEqual(snapshotPath, livePath)
|
|
824
|
+
? null
|
|
825
|
+
: `${rel} has different file contents`;
|
|
826
|
+
}
|
|
827
|
+
if (snapshotKind !== "missing") {
|
|
828
|
+
return `${rel} is stored as both a snapshot entry and recorded symlink`;
|
|
829
|
+
}
|
|
830
|
+
if (liveKind !== "symlink") {
|
|
831
|
+
return `${rel} is ${liveKind} but the snapshot records a symlink`;
|
|
832
|
+
}
|
|
833
|
+
try {
|
|
834
|
+
return recordedSymlinkTargetMatches(livePath, recordedLink.target, fs.readlinkSync(livePath))
|
|
835
|
+
? null
|
|
836
|
+
: `${rel} has a different symlink target`;
|
|
837
|
+
}
|
|
838
|
+
catch (error) {
|
|
839
|
+
return `${rel} could not read a symlink target: ${error instanceof Error ? error.message : String(error)}`;
|
|
840
|
+
}
|
|
841
|
+
}
|
|
842
|
+
if (snapshotKind !== liveKind) {
|
|
843
|
+
return `${rel} is ${liveKind} but the snapshot records ${snapshotKind}`;
|
|
844
|
+
}
|
|
845
|
+
if (snapshotKind === "missing")
|
|
846
|
+
return null;
|
|
847
|
+
if (snapshotKind === "file") {
|
|
848
|
+
return bytesEqual(snapshotPath, livePath)
|
|
849
|
+
? null
|
|
850
|
+
: `${rel} has different file contents`;
|
|
851
|
+
}
|
|
852
|
+
if (snapshotKind === "symlink") {
|
|
853
|
+
try {
|
|
854
|
+
return fs.readlinkSync(snapshotPath) === fs.readlinkSync(livePath)
|
|
855
|
+
? null
|
|
856
|
+
: `${rel} has a different symlink target`;
|
|
857
|
+
}
|
|
858
|
+
catch (error) {
|
|
859
|
+
return `${rel} could not read a symlink target: ${error instanceof Error ? error.message : String(error)}`;
|
|
860
|
+
}
|
|
861
|
+
}
|
|
862
|
+
if (snapshotKind !== "directory") {
|
|
863
|
+
return `${rel} has unsupported path type ${snapshotKind}`;
|
|
864
|
+
}
|
|
865
|
+
let snapshotNames;
|
|
866
|
+
let liveNames;
|
|
867
|
+
try {
|
|
868
|
+
snapshotNames = fs
|
|
869
|
+
.readdirSync(snapshotPath)
|
|
870
|
+
.filter((name) => !isRescueTransientDirectory(path.posix.join(rel, name)))
|
|
871
|
+
.sort();
|
|
872
|
+
liveNames = fs
|
|
873
|
+
.readdirSync(livePath)
|
|
874
|
+
.filter((name) => !isRescueTransientDirectory(path.posix.join(rel, name)))
|
|
875
|
+
.sort();
|
|
876
|
+
}
|
|
877
|
+
catch (error) {
|
|
878
|
+
return `${rel} could not be read: ${error instanceof Error ? error.message : String(error)}`;
|
|
879
|
+
}
|
|
880
|
+
const expectedNames = new Set(snapshotNames);
|
|
881
|
+
for (const recordedRel of recordedSymlinks.keys()) {
|
|
882
|
+
if (path.posix.dirname(recordedRel) === rel) {
|
|
883
|
+
expectedNames.add(path.posix.basename(recordedRel));
|
|
884
|
+
}
|
|
885
|
+
}
|
|
886
|
+
const expected = [...expectedNames].sort();
|
|
887
|
+
if (expected.length !== liveNames.length ||
|
|
888
|
+
expected.some((name, index) => name !== liveNames[index])) {
|
|
889
|
+
return `${rel} has different entries`;
|
|
890
|
+
}
|
|
891
|
+
for (const name of expected) {
|
|
892
|
+
const mismatch = snapshotPathMismatch(path.join(snapshotPath, name), path.join(livePath, name), path.posix.join(rel, name), recordedSymlinks);
|
|
893
|
+
if (mismatch)
|
|
894
|
+
return mismatch;
|
|
895
|
+
}
|
|
896
|
+
return null;
|
|
897
|
+
}
|
|
898
|
+
/**
|
|
899
|
+
* Restore one captured path, then recreate only manifest-recorded symlinks
|
|
900
|
+
* whose locations and targets are both confined to the HQ root.
|
|
901
|
+
*/
|
|
902
|
+
export function restoreSnapshotPath(snapshotDir, hqRoot, rel, manifest) {
|
|
903
|
+
restoreSnapshotPaths(snapshotDir, hqRoot, [rel], manifest);
|
|
904
|
+
}
|
|
905
|
+
/** Restore all requested physical paths before recreating their recorded links. */
|
|
906
|
+
function restoreSnapshotPaths(snapshotDir, hqRoot, rels, manifest) {
|
|
907
|
+
const capturedRoots = snapshotCapturedRoots(manifest.includedRoots);
|
|
908
|
+
if (typeof capturedRoots === "string")
|
|
909
|
+
throw new Error(capturedRoots);
|
|
910
|
+
const recorded = snapshotSymlinkTargets(hqRoot, capturedRoots, manifest.symlinks);
|
|
911
|
+
if (typeof recorded === "string")
|
|
912
|
+
throw new Error(recorded);
|
|
913
|
+
for (const [recordedRel, link] of recorded) {
|
|
914
|
+
const snapshotKind = rescuePathKind(path.join(snapshotDir, recordedRel));
|
|
915
|
+
if (link.representation === "file-copy") {
|
|
916
|
+
if (snapshotKind !== "file") {
|
|
917
|
+
throw new Error(`snapshot records copied file link ${recordedRel} without copied file data`);
|
|
918
|
+
}
|
|
919
|
+
}
|
|
920
|
+
else if (snapshotKind !== "missing") {
|
|
921
|
+
throw new Error(`snapshot records symlink ${recordedRel} alongside copied data`);
|
|
922
|
+
}
|
|
923
|
+
}
|
|
924
|
+
const requested = [...new Set(rels)].sort((left, right) => left.localeCompare(right));
|
|
925
|
+
if (requested.length === 0)
|
|
926
|
+
throw new Error("snapshot restore requested no paths");
|
|
927
|
+
const items = requested.map((rel) => {
|
|
928
|
+
const destination = snapshotRelativePath(hqRoot, rel);
|
|
929
|
+
if (!destination) {
|
|
930
|
+
throw new Error(`invalid snapshot restore path ${JSON.stringify(rel)}`);
|
|
931
|
+
}
|
|
932
|
+
if (!restoreParentIsSafe(hqRoot, destination)) {
|
|
933
|
+
throw new Error(`snapshot restore parent for ${rel} is not a directory inside the HQ root`);
|
|
934
|
+
}
|
|
935
|
+
if (!capturedRoots.has(rel.split("/")[0])) {
|
|
936
|
+
throw new Error(`snapshot does not include ${rel}`);
|
|
937
|
+
}
|
|
938
|
+
const source = path.join(snapshotDir, rel);
|
|
939
|
+
const physicalSourceExists = lexists(source);
|
|
940
|
+
const recordedWithinPath = [...recorded.keys()].some((recordedRel) => recordedRel === rel || recordedRel.startsWith(`${rel}/`));
|
|
941
|
+
if (!physicalSourceExists && !recordedWithinPath) {
|
|
942
|
+
throw new Error(`snapshot does not contain ${rel}`);
|
|
943
|
+
}
|
|
944
|
+
return { rel, source, destination, physicalSourceExists };
|
|
945
|
+
});
|
|
946
|
+
for (const item of items) {
|
|
947
|
+
fs.rmSync(item.destination, { recursive: true, force: true });
|
|
948
|
+
}
|
|
949
|
+
for (const item of items) {
|
|
950
|
+
if (item.physicalSourceExists) {
|
|
951
|
+
copySnapshotPhysicalTree(item.source, item.destination, hqRoot, item.rel);
|
|
952
|
+
}
|
|
953
|
+
}
|
|
954
|
+
const links = orderSnapshotSymlinkRestore(hqRoot, [...recorded.entries()]
|
|
955
|
+
.filter(([recordedRel, link]) => link.representation === undefined &&
|
|
956
|
+
items.some((item) => recordedRel === item.rel || recordedRel.startsWith(`${item.rel}/`)))
|
|
957
|
+
.map(([recordedRel, link]) => [recordedRel, link.target]));
|
|
958
|
+
for (const [linkRel, target] of links) {
|
|
959
|
+
recreateSnapshotSymlink(hqRoot, linkRel, target);
|
|
960
|
+
}
|
|
961
|
+
}
|
|
962
|
+
function orderSnapshotSymlinkRestore(hqRoot, links) {
|
|
963
|
+
const pending = new Map(links);
|
|
964
|
+
const ordered = [];
|
|
965
|
+
while (pending.size > 0) {
|
|
966
|
+
const next = [...pending.entries()]
|
|
967
|
+
.sort(([left], [right]) => left.localeCompare(right))
|
|
968
|
+
.find(([rel, target]) => {
|
|
969
|
+
const linkPath = snapshotRelativePath(hqRoot, rel);
|
|
970
|
+
const resolvedTarget = path.resolve(path.dirname(linkPath), target);
|
|
971
|
+
return ![...pending.keys()].some((candidate) => {
|
|
972
|
+
if (candidate === rel)
|
|
973
|
+
return false;
|
|
974
|
+
const candidatePath = snapshotRelativePath(hqRoot, candidate);
|
|
975
|
+
return (resolvedTarget === candidatePath ||
|
|
976
|
+
resolvedTarget.startsWith(`${candidatePath}${path.sep}`));
|
|
977
|
+
});
|
|
978
|
+
});
|
|
979
|
+
if (!next) {
|
|
980
|
+
ordered.push(...[...pending.entries()].sort(([left], [right]) => left.localeCompare(right)));
|
|
981
|
+
break;
|
|
982
|
+
}
|
|
983
|
+
pending.delete(next[0]);
|
|
984
|
+
ordered.push(next);
|
|
985
|
+
}
|
|
986
|
+
return ordered;
|
|
987
|
+
}
|
|
988
|
+
function copySnapshotPhysicalTree(source, destination, hqRoot, rel) {
|
|
989
|
+
const st = fs.lstatSync(source);
|
|
990
|
+
if (st.isSymbolicLink()) {
|
|
991
|
+
recreateSnapshotSymlink(hqRoot, rel, fs.readlinkSync(source));
|
|
992
|
+
return;
|
|
993
|
+
}
|
|
994
|
+
if (st.isDirectory()) {
|
|
995
|
+
fs.mkdirSync(destination, { recursive: true });
|
|
996
|
+
for (const name of fs.readdirSync(source)) {
|
|
997
|
+
copySnapshotPhysicalTree(path.join(source, name), path.join(destination, name), hqRoot, path.posix.join(rel, name));
|
|
998
|
+
}
|
|
999
|
+
preserveSnapshotMetadata(destination, st);
|
|
1000
|
+
return;
|
|
1001
|
+
}
|
|
1002
|
+
if (!st.isFile())
|
|
1003
|
+
throw new Error(`snapshot contains unsupported entry ${rel}`);
|
|
1004
|
+
fs.mkdirSync(path.dirname(destination), { recursive: true });
|
|
1005
|
+
fs.copyFileSync(source, destination);
|
|
1006
|
+
preserveSnapshotMetadata(destination, st);
|
|
1007
|
+
}
|
|
1008
|
+
/** Data restored successfully remains restored when filesystem metadata is unsupported. */
|
|
1009
|
+
function preserveSnapshotMetadata(destination, st) {
|
|
1010
|
+
try {
|
|
1011
|
+
fs.chmodSync(destination, st.mode);
|
|
1012
|
+
fs.utimesSync(destination, st.atime, st.mtime);
|
|
1013
|
+
}
|
|
1014
|
+
catch {
|
|
1015
|
+
/* metadata preservation is best-effort, matching the prior rollback helper */
|
|
1016
|
+
}
|
|
1017
|
+
}
|
|
1018
|
+
function recreateSnapshotSymlink(hqRoot, rel, target, destination = snapshotRelativePath(hqRoot, rel)) {
|
|
1019
|
+
const problem = validateSnapshotSymlink(hqRoot, rel, target);
|
|
1020
|
+
if (problem)
|
|
1021
|
+
throw new Error(`invalid snapshot symlink: ${problem}`);
|
|
1022
|
+
if (!destination)
|
|
1023
|
+
throw new Error(`invalid snapshot symlink destination ${rel}`);
|
|
1024
|
+
const restoredLinkPath = snapshotRelativePath(hqRoot, rel);
|
|
1025
|
+
const resolvedTarget = path.resolve(path.dirname(restoredLinkPath), target);
|
|
1026
|
+
fs.mkdirSync(path.dirname(destination), { recursive: true });
|
|
1027
|
+
fs.rmSync(destination, { recursive: true, force: true });
|
|
1028
|
+
try {
|
|
1029
|
+
fs.symlinkSync(target, destination);
|
|
1030
|
+
}
|
|
1031
|
+
catch (error) {
|
|
1032
|
+
if (process.platform === "win32" &&
|
|
1033
|
+
error.code === "EPERM" &&
|
|
1034
|
+
isDir(resolvedTarget)) {
|
|
1035
|
+
fs.symlinkSync(resolvedTarget, destination, "junction");
|
|
1036
|
+
return;
|
|
1037
|
+
}
|
|
1038
|
+
throw error;
|
|
1039
|
+
}
|
|
1040
|
+
}
|
|
1041
|
+
/**
|
|
1042
|
+
* Directory links can become junctions on Windows. A file link without link
|
|
1043
|
+
* privilege instead needs copied file data at its original snapshot path.
|
|
1044
|
+
*/
|
|
1045
|
+
function snapshotSymlinkRepresentation(hqRoot, source, destination, rel, target) {
|
|
1046
|
+
if (process.platform !== "win32")
|
|
1047
|
+
return undefined;
|
|
1048
|
+
const probe = `${destination}.hq-restore-probe-${process.pid}-${Date.now().toString(36)}`;
|
|
1049
|
+
try {
|
|
1050
|
+
recreateSnapshotSymlink(hqRoot, rel, target, probe);
|
|
1051
|
+
fs.rmSync(probe, { recursive: true, force: true });
|
|
1052
|
+
return undefined;
|
|
1053
|
+
}
|
|
1054
|
+
catch (error) {
|
|
1055
|
+
try {
|
|
1056
|
+
fs.rmSync(probe, { recursive: true, force: true });
|
|
1057
|
+
}
|
|
1058
|
+
catch (cleanupError) {
|
|
1059
|
+
throw new SnapshotCopyError(rel, "write", cleanupError, probe);
|
|
1060
|
+
}
|
|
1061
|
+
if (process.platform === "win32" &&
|
|
1062
|
+
error.code === "EPERM") {
|
|
1063
|
+
try {
|
|
1064
|
+
if (fs.statSync(source).isFile())
|
|
1065
|
+
return "file-copy";
|
|
1066
|
+
}
|
|
1067
|
+
catch (targetError) {
|
|
1068
|
+
throw new SnapshotCopyError(rel, "read", targetError, source);
|
|
1069
|
+
}
|
|
1070
|
+
}
|
|
1071
|
+
throw new SnapshotCopyError(rel, "metadata", new Error(`recorded symlink cannot be restored on this machine: ${error instanceof Error ? error.message : String(error)}`), source);
|
|
1072
|
+
}
|
|
1073
|
+
}
|
|
1074
|
+
/**
|
|
1075
|
+
* True when `root` is usable as a rescue target: an absolute path that is not
|
|
1076
|
+
* a bare drive letter. Pure + exported for tests; `platformPath` defaults to
|
|
1077
|
+
* the host `path` module so Windows and POSIX runtimes each apply their own
|
|
1078
|
+
* absoluteness rules (tests pass `path.win32` / `path.posix` explicitly).
|
|
1079
|
+
*
|
|
1080
|
+
* Rejecting bare `C:` matters even though `path.win32.isAbsolute("C:")` is
|
|
1081
|
+
* already false — the explicit pattern documents the exact field failure and
|
|
1082
|
+
* guards against a future helper swap that treats drive-relative paths as
|
|
1083
|
+
* absolute.
|
|
1084
|
+
*/
|
|
1085
|
+
export function isUsableRescueRoot(root, platformPath = path) {
|
|
1086
|
+
if (/^[A-Za-z]:$/.test(root))
|
|
1087
|
+
return false;
|
|
1088
|
+
return platformPath.isAbsolute(root);
|
|
1089
|
+
}
|
|
1090
|
+
/**
|
|
1091
|
+
* Canonicalize the rescue root without treating a known-good directory as
|
|
1092
|
+
* unusable when Node's JavaScript realpath walker cannot inspect a Windows
|
|
1093
|
+
* drive prefix. Prefer the native syscall, retain the JS walker as a fallback,
|
|
1094
|
+
* and use an absolute path only after stat confirms it is a directory.
|
|
1095
|
+
*/
|
|
1096
|
+
export function realpathOrExit(p, err) {
|
|
1097
|
+
try {
|
|
1098
|
+
return fs.realpathSync.native(p);
|
|
1099
|
+
}
|
|
1100
|
+
catch {
|
|
1101
|
+
try {
|
|
1102
|
+
return fs.realpathSync(p);
|
|
1103
|
+
}
|
|
1104
|
+
catch (realpathError) {
|
|
1105
|
+
const resolved = path.resolve(p);
|
|
1106
|
+
try {
|
|
1107
|
+
if (!fs.statSync(resolved).isDirectory()) {
|
|
1108
|
+
throw new Error("resolved path is not a directory");
|
|
1109
|
+
}
|
|
1110
|
+
}
|
|
1111
|
+
catch {
|
|
1112
|
+
err(`error: cannot resolve --hq-root ${p}: ${realpathError instanceof Error ? realpathError.message : String(realpathError)}\n`);
|
|
1113
|
+
throw new ExitError(1);
|
|
1114
|
+
}
|
|
1115
|
+
err("warning: could not canonicalize --hq-root; using a normalized absolute path.\n");
|
|
1116
|
+
return resolved;
|
|
1117
|
+
}
|
|
1118
|
+
}
|
|
1119
|
+
}
|
|
1120
|
+
function doRescue(cfg, env, out, err, setTmp) {
|
|
1121
|
+
const hqRoot = resolveHqRoot(cfg, err);
|
|
547
1122
|
const RUN_TS = utcStamp(new Date(), "dash");
|
|
548
1123
|
// Both the release overlay and a restorable snapshot require rsync. A shim
|
|
549
1124
|
// which merely starts but exits non-zero used to trigger an enormous cp -a
|
|
550
1125
|
// fallback, then fail later at overlay. Stop before classification/allocation.
|
|
551
|
-
requireWorkingRsync(env, err);
|
|
1126
|
+
const rsyncLauncher = requireWorkingRsync(env, err);
|
|
552
1127
|
// --- Capture the pre-rescue .claude/settings.json (BEFORE the wipe) ---
|
|
553
1128
|
// This is `ours` for the post-overlay settings reconcile. It must be read now
|
|
554
1129
|
// because the overlay-safe classification deletes/overwrites it before
|
|
@@ -582,8 +1157,9 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
582
1157
|
prevSyncRef = yq('.replaced_from_source.ref // .replaced_from_staging.ref // ""');
|
|
583
1158
|
prevSyncAt = yq('.replaced_from_source.last_sync_at // .replaced_from_staging.last_sync_at // ""');
|
|
584
1159
|
}
|
|
585
|
-
//
|
|
586
|
-
|
|
1160
|
+
// An explicit caller floor is the recovery path for a missing or stale stamp.
|
|
1161
|
+
// It remains safe because the selected release clone must still contain it.
|
|
1162
|
+
if (cfg.floorShaOverride) {
|
|
587
1163
|
prevSyncSha = cfg.floorShaOverride;
|
|
588
1164
|
prevSyncSource = cfg.sourceRepo;
|
|
589
1165
|
prevSyncRef = "(caller-supplied floor)";
|
|
@@ -592,12 +1168,8 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
592
1168
|
out(`==> HQ root: ${hqRoot}\n`);
|
|
593
1169
|
out(`==> Source: https://github.com/${cfg.sourceRepo} @ ${cfg.ref}\n`);
|
|
594
1170
|
if (prevSyncSha) {
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
}
|
|
598
|
-
else {
|
|
599
|
-
out(`==> Prior sync: ${prevSyncSha} from ${prevSyncSource} (different source — ignoring as floor)\n`);
|
|
600
|
-
}
|
|
1171
|
+
const sourceLabel = prevSyncSource || "(source not recorded)";
|
|
1172
|
+
out(`==> Prior sync: ${prevSyncSha} from ${sourceLabel}@${prevSyncRef} (${prevSyncAt}) — checking cloned release history for floor\n`);
|
|
601
1173
|
}
|
|
602
1174
|
if (cfg.narrowPaths.length !== 0) {
|
|
603
1175
|
out("==> Mode: narrow (--paths)\n");
|
|
@@ -753,21 +1325,22 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
753
1325
|
// --- Resolve history floor ---
|
|
754
1326
|
let historyFloor = "";
|
|
755
1327
|
let baselineMode = "head_compare";
|
|
756
|
-
if (cfg.historyCheck && prevSyncSha
|
|
1328
|
+
if (cfg.historyCheck && prevSyncSha) {
|
|
757
1329
|
if (run("git", ["cat-file", "-e", prevSyncSha], { cwd: srcDir, env })
|
|
758
1330
|
.status === 0) {
|
|
759
1331
|
historyFloor = prevSyncSha;
|
|
760
1332
|
baselineMode = "history_floor";
|
|
761
1333
|
}
|
|
762
1334
|
else {
|
|
763
|
-
out(` prior-sync SHA ${prevSyncSha} not reachable in
|
|
1335
|
+
out(` WARN: prior-sync SHA ${prevSyncSha} is not reachable in the cloned release history; falling back to head_compare\n`);
|
|
764
1336
|
}
|
|
765
1337
|
}
|
|
766
1338
|
if (baselineMode === "history_floor") {
|
|
767
1339
|
out(`==> Baseline: ${historyFloor} (last-sync floor reachable)\n`);
|
|
768
1340
|
}
|
|
769
1341
|
else {
|
|
770
|
-
out("==> Baseline: HEAD compare (no usable last-sync
|
|
1342
|
+
out("==> Baseline: HEAD compare (no usable reachable last-sync floor)\n");
|
|
1343
|
+
out("==> WARNING: retired release files cannot be detected or removed in head_compare mode. Re-run with history checking and --floor-sha <sha> for a reachable baseline.\n");
|
|
771
1344
|
}
|
|
772
1345
|
// --- Build wipe/overlay arg sets per mode ---
|
|
773
1346
|
const pruneNames = [];
|
|
@@ -892,7 +1465,7 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
892
1465
|
if (cfg.dryRun) {
|
|
893
1466
|
out("\n");
|
|
894
1467
|
out("==> DRY RUN classification summary:\n");
|
|
895
|
-
out(` user-only (
|
|
1468
|
+
out(` user-only (left in place under release-managed roots): ${counts.userOnly} files\n`);
|
|
896
1469
|
out(` unchanged (preserved/re-laid): ${counts.unchanged} files\n`);
|
|
897
1470
|
out(` user-edit (rescue / diff-append): ${counts.userEdit} files\n`);
|
|
898
1471
|
out(` user-edit (conflict quarantine): ${counts.userConflict} files\n`);
|
|
@@ -968,12 +1541,21 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
968
1541
|
];
|
|
969
1542
|
// Exercise the exact overlay command before any classified delete/move. A
|
|
970
1543
|
// Cygwin path conversion or filter failure therefore leaves the root intact.
|
|
971
|
-
const overlayPreflight = run(
|
|
1544
|
+
const overlayPreflight = run(rsyncLauncher.command, [...rsyncLauncher.leadingArgs, "--dry-run", ...overlayArgs], { env });
|
|
972
1545
|
if (overlayPreflight.status !== 0) {
|
|
973
1546
|
err(overlayPreflight.stderr);
|
|
974
1547
|
throw new ExitError(overlayPreflight.status || 1);
|
|
975
1548
|
}
|
|
976
1549
|
ctx.actions = buildRescueActions(ctx);
|
|
1550
|
+
// Permission preflight is deliberately before snapshot allocation. A
|
|
1551
|
+
// root-owned release tree (EACCES on rename/delete) used to allocate a
|
|
1552
|
+
// snapshot, fail apply twice, and then freeze rescue behind the
|
|
1553
|
+
// recovery-required circuit breaker. Fail closed with no snapshot instead.
|
|
1554
|
+
const unwritable = collectUnwritableRescueMutationPaths(hqRoot, ctx.actions);
|
|
1555
|
+
if (unwritable.length > 0) {
|
|
1556
|
+
reportUnwritableRescuePaths(hqRoot, unwritable, err);
|
|
1557
|
+
throw new ExitError(1);
|
|
1558
|
+
}
|
|
977
1559
|
// Snapshot allocation is deliberately after the entire read-only walk and
|
|
978
1560
|
// action-plan build. A no-op (or a classifier fault above) cannot consume
|
|
979
1561
|
// backup space.
|
|
@@ -987,9 +1569,25 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
987
1569
|
const prePlan = pruneManagedBackups(cfg, hqRoot, out, false);
|
|
988
1570
|
const unresolved = prePlan.protected.filter((x) => x.reason === "recovery-required-no-later-good");
|
|
989
1571
|
if (unresolved.length >= 2) {
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
1572
|
+
const postRestore = consumePostRestorePermit(cfg, hqRoot);
|
|
1573
|
+
if (postRestore.used) {
|
|
1574
|
+
out(`==> Verified post-restore permit for ${postRestore.snapshotName}; allowing one rescue attempt.\n`);
|
|
1575
|
+
out(" The permit was consumed before mutation. If this attempt fails, the circuit breaker is armed again.\n");
|
|
1576
|
+
}
|
|
1577
|
+
else {
|
|
1578
|
+
if (postRestore.problem) {
|
|
1579
|
+
err(`error: post-restore permit is not usable: ${postRestore.problem}\n`);
|
|
1580
|
+
}
|
|
1581
|
+
const snapshotNames = unresolved.map((x) => x.entry.name);
|
|
1582
|
+
writeRescueBlockedMarker(hqRoot, snapshotNames);
|
|
1583
|
+
err("HQ_RESCUE_FAILURE_KIND=snapshot-recovery-circuit-breaker\n");
|
|
1584
|
+
err("error: safety snapshot circuit breaker is open: two interrupted updates still need recovery proof.\n");
|
|
1585
|
+
err(" HQ kept their pre-update snapshots because those updates started changing the release-managed tree but did not finish.\n");
|
|
1586
|
+
err(" It will not overwrite or delete that evidence without proof that a later rescue completed successfully.\n");
|
|
1587
|
+
err(` Inspect safely: ${formatRecoveryCommand(hqRoot, cfg.backupRoot)}\n`);
|
|
1588
|
+
err(" It deletes only snapshots with a later completed-good rescue. For unresolved snapshots, use --restore-snapshot to restore every captured root and recorded link, then run its verification command before retrying.\n");
|
|
1589
|
+
throw new ExitError(1);
|
|
1590
|
+
}
|
|
993
1591
|
}
|
|
994
1592
|
requireBackupCapacity(cfg, hqRoot, wipeToplevel, env, err);
|
|
995
1593
|
fs.mkdirSync(cfg.backupRoot, { recursive: true });
|
|
@@ -997,23 +1595,36 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
997
1595
|
backupDir = allocateSnapshotDir(cfg.backupRoot, RUN_TS, canonicalRootHash(hqRoot), runId);
|
|
998
1596
|
out("\n");
|
|
999
1597
|
out(`==> Safety snapshot -> ${backupDir}\n`);
|
|
1598
|
+
const snapshotSymlinks = [];
|
|
1000
1599
|
try {
|
|
1001
1600
|
fs.mkdirSync(backupDir, { recursive: false });
|
|
1002
1601
|
writeSnapshotManifest(backupDir, snapshotManifest("copying", hqRoot, runId, cfg, srcSha, wipeToplevel));
|
|
1003
1602
|
for (const rootRel of wipeToplevel) {
|
|
1004
1603
|
const srcAbs = path.join(hqRoot, rootRel);
|
|
1005
|
-
if (
|
|
1604
|
+
if (isRescueTransientDirectory(rootRel))
|
|
1006
1605
|
continue;
|
|
1007
|
-
cpAToFiltered(srcAbs, path.join(backupDir, rootRel), rootRel);
|
|
1606
|
+
cpAToFiltered(srcAbs, path.join(backupDir, rootRel), rootRel, hqRoot, snapshotSymlinks);
|
|
1008
1607
|
}
|
|
1009
|
-
updateSnapshotStatus(backupDir, "ready"
|
|
1608
|
+
updateSnapshotStatus(backupDir, "ready", undefined, {
|
|
1609
|
+
symlinks: snapshotSymlinks,
|
|
1610
|
+
});
|
|
1010
1611
|
}
|
|
1011
1612
|
catch (error) {
|
|
1012
|
-
|
|
1613
|
+
if (error instanceof SnapshotCopyError) {
|
|
1614
|
+
updateSnapshotStatus(backupDir, "failed-pre-mutation", undefined, {
|
|
1615
|
+
symlinks: snapshotSymlinks,
|
|
1616
|
+
copyFailure: snapshotCopyFailure(error),
|
|
1617
|
+
});
|
|
1618
|
+
reportSnapshotCopyFailure(error, err);
|
|
1619
|
+
throw new ExitError(1);
|
|
1620
|
+
}
|
|
1621
|
+
updateSnapshotStatus(backupDir, "failed-pre-mutation", undefined, {
|
|
1622
|
+
symlinks: snapshotSymlinks,
|
|
1623
|
+
});
|
|
1013
1624
|
throw error;
|
|
1014
1625
|
}
|
|
1015
1626
|
pruneManagedBackups(cfg, hqRoot, out, true);
|
|
1016
|
-
out(
|
|
1627
|
+
out(" snapshot ready (complete recovery is available through --recover-snapshots).\n");
|
|
1017
1628
|
}
|
|
1018
1629
|
else if (cfg.doBackup && !cfg.dryRun && wipeToplevel.length !== 0) {
|
|
1019
1630
|
out("\n==> Safety snapshot skipped (classification found no destructive paths).\n");
|
|
@@ -1053,7 +1664,7 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
1053
1664
|
toCygwinRsyncPath(hqRoot + "/"),
|
|
1054
1665
|
]
|
|
1055
1666
|
: overlayArgs;
|
|
1056
|
-
const overlay = run(
|
|
1667
|
+
const overlay = run(rsyncLauncher.command, [...rsyncLauncher.leadingArgs, ...overlayRunArgs], { env });
|
|
1057
1668
|
if (overlay.status !== 0) {
|
|
1058
1669
|
// bash runs under set -e: a failed rsync aborts with its status.
|
|
1059
1670
|
err(overlay.stderr);
|
|
@@ -1194,7 +1805,7 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
1194
1805
|
}
|
|
1195
1806
|
out("\n");
|
|
1196
1807
|
out("==> Classification:\n");
|
|
1197
|
-
out(` user-only (left in place):
|
|
1808
|
+
out(` user-only (left in place under release-managed roots): ${counts.userOnly} files\n`);
|
|
1198
1809
|
out(` unchanged (preserved/re-laid): ${counts.unchanged} files\n`);
|
|
1199
1810
|
out(` user-edits (rescued): ${counts.userEdit} files\n`);
|
|
1200
1811
|
out(` user-edits (conflict quarantine): ${counts.userConflict} files\n`);
|
|
@@ -1229,7 +1840,7 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
1229
1840
|
lines.push("Baseline: head_compare (no last-sync stamp; first run or floor unreachable)");
|
|
1230
1841
|
}
|
|
1231
1842
|
lines.push("", "## What the rescue did");
|
|
1232
|
-
lines.push(` user-only (left in place):
|
|
1843
|
+
lines.push(` user-only (left in place under release-managed roots): ${counts.userOnly}`);
|
|
1233
1844
|
lines.push(` unchanged (preserved/re-laid): ${counts.unchanged}`);
|
|
1234
1845
|
lines.push(` user-edit (rescued into personal/): ${counts.userEdit}`);
|
|
1235
1846
|
lines.push(` user-edit (conflict quarantine): ${counts.userConflict}`);
|
|
@@ -1258,10 +1869,11 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
1258
1869
|
}
|
|
1259
1870
|
lines.push("", "## Restore");
|
|
1260
1871
|
lines.push("This directory holds the release-managed wipe set as it was before the update; reproducible worktrees and build caches are intentionally excluded.");
|
|
1261
|
-
|
|
1262
|
-
lines.push(
|
|
1263
|
-
lines.push(
|
|
1264
|
-
lines.push(
|
|
1872
|
+
const snapshotName = path.basename(backupDir);
|
|
1873
|
+
lines.push("Restore every captured root and recorded symlink:");
|
|
1874
|
+
lines.push(` ${formatSnapshotRestoreCommand(hqRoot, cfg.backupRoot, snapshotName)}`);
|
|
1875
|
+
lines.push("Then verify the restored tree before retrying:");
|
|
1876
|
+
lines.push(` ${formatRecoveryCommand(hqRoot, cfg.backupRoot, snapshotName)}`);
|
|
1265
1877
|
lines.push("", `Retention: ${cfg.backupRetentionDays} days and ${cfg.backupRetentionCount} completed snapshots (latest recovery floor retained).`);
|
|
1266
1878
|
try {
|
|
1267
1879
|
fs.writeFileSync(path.join(backupDir, "RECOVERY.md"), lines.join("\n") + "\n");
|
|
@@ -1281,6 +1893,7 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
1281
1893
|
if (cfg.doBackup && backupDir) {
|
|
1282
1894
|
out(` A filtered pre-update snapshot is at ${backupDir} (RECOVERY.md explains restore).\n`);
|
|
1283
1895
|
}
|
|
1896
|
+
clearRescueBlockedMarker(hqRoot);
|
|
1284
1897
|
return { status: 0 };
|
|
1285
1898
|
}
|
|
1286
1899
|
catch (error) {
|
|
@@ -1328,12 +1941,15 @@ function applyClassifiedActions(ctx, backupDir) {
|
|
|
1328
1941
|
completed.push(action);
|
|
1329
1942
|
}
|
|
1330
1943
|
catch (err) {
|
|
1331
|
-
const
|
|
1332
|
-
const manifest = writeApplyFailureManifest(ctx, backupDir, action, completed, restored, err);
|
|
1944
|
+
const rollback = rollbackCompletedActions(ctx, completed, backupDir);
|
|
1945
|
+
const manifest = writeApplyFailureManifest(ctx, backupDir, action, completed, rollback.restored, rollback.failures, err);
|
|
1333
1946
|
ctx.out("\n");
|
|
1334
1947
|
ctx.out(`error: rescue apply failed during '${action.label}'.\n`);
|
|
1335
|
-
if (restored.length > 0) {
|
|
1336
|
-
ctx.out(`==> Rolled back ${restored.length} prior action path(s) from the safety snapshot.\n`);
|
|
1948
|
+
if (rollback.restored.length > 0) {
|
|
1949
|
+
ctx.out(`==> Rolled back ${rollback.restored.length} prior action path(s) from the safety snapshot.\n`);
|
|
1950
|
+
}
|
|
1951
|
+
for (const failure of rollback.failures) {
|
|
1952
|
+
ctx.out(`warning: rollback retained recovery evidence: ${failure}\n`);
|
|
1337
1953
|
}
|
|
1338
1954
|
if (manifest) {
|
|
1339
1955
|
ctx.out(`==> Recovery manifest: ${manifest}\n`);
|
|
@@ -1359,7 +1975,14 @@ function injectApplyPathChangeForTest(ctx) {
|
|
|
1359
1975
|
}
|
|
1360
1976
|
function rollbackCompletedActions(ctx, completed, backupDir) {
|
|
1361
1977
|
if (!backupDir || !isDir(backupDir))
|
|
1362
|
-
return [];
|
|
1978
|
+
return { restored: [], failures: [] };
|
|
1979
|
+
const snapshotManifest = readSnapshotManifest(backupDir, ctx.hqRoot);
|
|
1980
|
+
if (!snapshotManifest) {
|
|
1981
|
+
return {
|
|
1982
|
+
restored: [],
|
|
1983
|
+
failures: [`${backupDir}: snapshot manifest is missing or invalid`],
|
|
1984
|
+
};
|
|
1985
|
+
}
|
|
1363
1986
|
const rels = [];
|
|
1364
1987
|
const seen = new Set();
|
|
1365
1988
|
for (let i = completed.length - 1; i >= 0; i--) {
|
|
@@ -1372,25 +1995,30 @@ function rollbackCompletedActions(ctx, completed, backupDir) {
|
|
|
1372
1995
|
}
|
|
1373
1996
|
}
|
|
1374
1997
|
}
|
|
1375
|
-
const
|
|
1998
|
+
const restoreRels = [];
|
|
1376
1999
|
for (const rel of rels) {
|
|
1377
2000
|
const snapshotPath = path.join(backupDir, rel);
|
|
1378
|
-
|
|
2001
|
+
const hasRecordedLink = (snapshotManifest.symlinks ?? []).some((link) => link.path === rel || link.path.startsWith(`${rel}/`));
|
|
2002
|
+
if (!lexists(snapshotPath) && !hasRecordedLink)
|
|
1379
2003
|
continue;
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
}
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
2004
|
+
restoreRels.push(rel);
|
|
2005
|
+
}
|
|
2006
|
+
if (restoreRels.length === 0)
|
|
2007
|
+
return { restored: [], failures: [] };
|
|
2008
|
+
try {
|
|
2009
|
+
restoreSnapshotPaths(backupDir, ctx.hqRoot, restoreRels, snapshotManifest);
|
|
2010
|
+
return { restored: restoreRels, failures: [] };
|
|
2011
|
+
}
|
|
2012
|
+
catch (error) {
|
|
2013
|
+
return {
|
|
2014
|
+
restored: [],
|
|
2015
|
+
failures: [
|
|
2016
|
+
`${restoreRels.join(", ")}: ${error instanceof Error ? error.message : String(error)}`,
|
|
2017
|
+
],
|
|
2018
|
+
};
|
|
1390
2019
|
}
|
|
1391
|
-
return restored;
|
|
1392
2020
|
}
|
|
1393
|
-
function writeApplyFailureManifest(ctx, backupDir, failedAction, completed, restored, err) {
|
|
2021
|
+
function writeApplyFailureManifest(ctx, backupDir, failedAction, completed, restored, rollbackFailures, err) {
|
|
1394
2022
|
const message = err instanceof Error ? err.message : String(err);
|
|
1395
2023
|
const completedLines = completed.length === 0
|
|
1396
2024
|
? [" (none)"]
|
|
@@ -1440,11 +2068,18 @@ function writeApplyFailureManifest(ctx, backupDir, failedAction, completed, rest
|
|
|
1440
2068
|
for (const rel of unrestored)
|
|
1441
2069
|
lines.push(` - ${rel}`);
|
|
1442
2070
|
}
|
|
2071
|
+
if (rollbackFailures.length > 0) {
|
|
2072
|
+
lines.push("", "Rollback failures:");
|
|
2073
|
+
for (const failure of rollbackFailures)
|
|
2074
|
+
lines.push(` - ${failure}`);
|
|
2075
|
+
}
|
|
1443
2076
|
if (backupDir && isDir(backupDir)) {
|
|
1444
2077
|
lines.push("");
|
|
1445
|
-
|
|
1446
|
-
lines.push(
|
|
1447
|
-
lines.push(`
|
|
2078
|
+
const snapshotName = path.basename(backupDir);
|
|
2079
|
+
lines.push("Restore every captured root and recorded symlink:");
|
|
2080
|
+
lines.push(` ${formatSnapshotRestoreCommand(ctx.hqRoot, ctx.cfg.backupRoot, snapshotName)}`);
|
|
2081
|
+
lines.push("Then verify before retrying:");
|
|
2082
|
+
lines.push(` ${formatRecoveryCommand(ctx.hqRoot, ctx.cfg.backupRoot, snapshotName)}`);
|
|
1448
2083
|
}
|
|
1449
2084
|
else {
|
|
1450
2085
|
lines.push("", "No pre-op safety snapshot was available for automatic restore.");
|
|
@@ -2372,10 +3007,11 @@ function renderDryRunDecision(ctx, decision) {
|
|
|
2372
3007
|
}
|
|
2373
3008
|
function buildRescueActions(ctx) {
|
|
2374
3009
|
const actions = [];
|
|
2375
|
-
const add = (label, affectedRels, runAction) => {
|
|
3010
|
+
const add = (label, affectedRels, runAction, destParentRels = []) => {
|
|
2376
3011
|
actions.push({
|
|
2377
3012
|
label,
|
|
2378
3013
|
affectedRels,
|
|
3014
|
+
destParentRels,
|
|
2379
3015
|
expectedPaths: affectedRels.map((rel) => ({
|
|
2380
3016
|
rel,
|
|
2381
3017
|
kind: rescuePathKind(path.join(ctx.hqRoot, rel)),
|
|
@@ -2414,7 +3050,7 @@ function buildRescueActions(ctx) {
|
|
|
2414
3050
|
});
|
|
2415
3051
|
break;
|
|
2416
3052
|
case "user-edit-diff-append":
|
|
2417
|
-
add(`rescue diff-append: ${decision.rel}`, [decision.rel], () => rescueOne(ctx, decision.rel));
|
|
3053
|
+
add(`rescue diff-append: ${decision.rel}`, [decision.rel], () => rescueOne(ctx, decision.rel), [path.posix.dirname(mapRescueTarget(decision.rel))]);
|
|
2418
3054
|
break;
|
|
2419
3055
|
case "user-edit-overwrite-safe":
|
|
2420
3056
|
add(`overwrite-safe: ${decision.rel}`, [decision.rel], () => {
|
|
@@ -2435,10 +3071,12 @@ function buildRescueActions(ctx) {
|
|
|
2435
3071
|
});
|
|
2436
3072
|
break;
|
|
2437
3073
|
case "user-edit-conflict":
|
|
2438
|
-
add(`conflict quarantine: ${decision.rel}`, [decision.rel], () => conflictOne(ctx, decision.rel)
|
|
3074
|
+
add(`conflict quarantine: ${decision.rel}`, [decision.rel], () => conflictOne(ctx, decision.rel), [
|
|
3075
|
+
path.posix.dirname(`.hq-conflicts/rescue-${ctx.runTs}/${decision.rel}`),
|
|
3076
|
+
]);
|
|
2439
3077
|
break;
|
|
2440
3078
|
case "user-edit-rescue":
|
|
2441
|
-
add(`rescue: ${decision.rel}`, [decision.rel], () => rescueOne(ctx, decision.rel));
|
|
3079
|
+
add(`rescue: ${decision.rel}`, [decision.rel], () => rescueOne(ctx, decision.rel), [path.posix.dirname(mapRescueTarget(decision.rel))]);
|
|
2442
3080
|
break;
|
|
2443
3081
|
case "unchanged-preserve":
|
|
2444
3082
|
add(`preserve unchanged: ${decision.rel}`, [], () => {
|
|
@@ -2711,6 +3349,262 @@ function restoreMtimesFromGit(srcDir, env, out) {
|
|
|
2711
3349
|
}
|
|
2712
3350
|
}
|
|
2713
3351
|
}
|
|
3352
|
+
const RESCUE_BLOCKED_MARKER_REL = ".hq/rescue-blocked.json";
|
|
3353
|
+
const RESCUE_BLOCKED_REASON = "recovery-snapshot-breaker";
|
|
3354
|
+
const POST_RESTORE_PERMIT_REL = ".hq/rescue-post-restore.json";
|
|
3355
|
+
function posixShellQuote(value) {
|
|
3356
|
+
return `'${value.replace(/'/g, `'\\''`)}'`;
|
|
3357
|
+
}
|
|
3358
|
+
/** Format the exact safe command presented in a breaker diagnostic. */
|
|
3359
|
+
export function formatRecoveryCommand(hqRoot, backupRoot, verifyRestoredSnapshot) {
|
|
3360
|
+
const args = [
|
|
3361
|
+
"npx",
|
|
3362
|
+
"-y",
|
|
3363
|
+
"--package=@indigoai-us/hq-cloud@latest",
|
|
3364
|
+
"hq-rescue",
|
|
3365
|
+
"--recover-snapshots",
|
|
3366
|
+
"--hq-root",
|
|
3367
|
+
posixShellQuote(hqRoot),
|
|
3368
|
+
"--backup-dir",
|
|
3369
|
+
posixShellQuote(backupRoot),
|
|
3370
|
+
];
|
|
3371
|
+
if (verifyRestoredSnapshot) {
|
|
3372
|
+
args.push("--verify-restored-snapshot", posixShellQuote(verifyRestoredSnapshot));
|
|
3373
|
+
}
|
|
3374
|
+
return args.join(" ");
|
|
3375
|
+
}
|
|
3376
|
+
/** Format the manifest-aware command that restores links as well as files. */
|
|
3377
|
+
export function formatSnapshotRestoreCommand(hqRoot, backupRoot, snapshotName) {
|
|
3378
|
+
return [
|
|
3379
|
+
"npx",
|
|
3380
|
+
"-y",
|
|
3381
|
+
"--package=@indigoai-us/hq-cloud@latest",
|
|
3382
|
+
"hq-rescue",
|
|
3383
|
+
"--recover-snapshots",
|
|
3384
|
+
"--hq-root",
|
|
3385
|
+
posixShellQuote(hqRoot),
|
|
3386
|
+
"--backup-dir",
|
|
3387
|
+
posixShellQuote(backupRoot),
|
|
3388
|
+
"--restore-snapshot",
|
|
3389
|
+
posixShellQuote(snapshotName),
|
|
3390
|
+
].join(" ");
|
|
3391
|
+
}
|
|
3392
|
+
function posixRelFromRoot(hqRoot, abs) {
|
|
3393
|
+
const root = path.resolve(hqRoot);
|
|
3394
|
+
const resolved = path.resolve(abs);
|
|
3395
|
+
if (resolved === root)
|
|
3396
|
+
return ".";
|
|
3397
|
+
const prefix = root.endsWith(path.sep) ? root : root + path.sep;
|
|
3398
|
+
if (resolved.startsWith(prefix)) {
|
|
3399
|
+
return resolved.slice(prefix.length).split(path.sep).join("/");
|
|
3400
|
+
}
|
|
3401
|
+
return resolved;
|
|
3402
|
+
}
|
|
3403
|
+
function firstExistingAncestor(abs) {
|
|
3404
|
+
let current = abs;
|
|
3405
|
+
for (;;) {
|
|
3406
|
+
if (lexists(current))
|
|
3407
|
+
return current;
|
|
3408
|
+
const parent = path.dirname(current);
|
|
3409
|
+
if (parent === current)
|
|
3410
|
+
return null;
|
|
3411
|
+
current = parent;
|
|
3412
|
+
}
|
|
3413
|
+
}
|
|
3414
|
+
function canWrite(abs) {
|
|
3415
|
+
try {
|
|
3416
|
+
fs.accessSync(abs, fs.constants.W_OK);
|
|
3417
|
+
return true;
|
|
3418
|
+
}
|
|
3419
|
+
catch {
|
|
3420
|
+
return false;
|
|
3421
|
+
}
|
|
3422
|
+
}
|
|
3423
|
+
function topLevelRel(rel) {
|
|
3424
|
+
const normalized = rel.replace(/\\/g, "/").replace(/^\.\//, "");
|
|
3425
|
+
if (normalized === "." || normalized === "")
|
|
3426
|
+
return ".";
|
|
3427
|
+
return normalized.split("/")[0] ?? ".";
|
|
3428
|
+
}
|
|
3429
|
+
/**
|
|
3430
|
+
* Paths the current process cannot write among the action plan's rename/delete
|
|
3431
|
+
* sources (and their parents) plus rename destination parents. Empty means the
|
|
3432
|
+
* plan is writable; a non-empty result must refuse before allocating a snapshot.
|
|
3433
|
+
*/
|
|
3434
|
+
export function collectUnwritableRescueMutationPaths(hqRoot, actions) {
|
|
3435
|
+
const failed = new Set();
|
|
3436
|
+
const note = (abs) => {
|
|
3437
|
+
failed.add(posixRelFromRoot(hqRoot, abs));
|
|
3438
|
+
};
|
|
3439
|
+
const checkParent = (abs) => {
|
|
3440
|
+
const existing = firstExistingAncestor(abs);
|
|
3441
|
+
if (!existing || !canWrite(existing))
|
|
3442
|
+
note(existing ?? abs);
|
|
3443
|
+
};
|
|
3444
|
+
for (const action of actions) {
|
|
3445
|
+
for (const rel of action.affectedRels) {
|
|
3446
|
+
if (!rel)
|
|
3447
|
+
continue;
|
|
3448
|
+
const abs = path.join(hqRoot, rel);
|
|
3449
|
+
const st = lstatOrNull(abs);
|
|
3450
|
+
if (st && !st.isSymbolicLink() && !canWrite(abs))
|
|
3451
|
+
note(abs);
|
|
3452
|
+
checkParent(path.dirname(abs));
|
|
3453
|
+
}
|
|
3454
|
+
for (const rel of action.destParentRels ?? []) {
|
|
3455
|
+
if (!rel || rel === ".")
|
|
3456
|
+
continue;
|
|
3457
|
+
checkParent(path.join(hqRoot, rel));
|
|
3458
|
+
}
|
|
3459
|
+
}
|
|
3460
|
+
return [...failed].sort();
|
|
3461
|
+
}
|
|
3462
|
+
function reportUnwritableRescuePaths(hqRoot, unwritable, err) {
|
|
3463
|
+
const shown = unwritable.slice(0, 10);
|
|
3464
|
+
err("error: cannot write the paths rescue would rename or delete (permission denied).\n");
|
|
3465
|
+
err(" Refusing before a safety snapshot is allocated so this cannot trip the recovery-snapshot circuit breaker.\n");
|
|
3466
|
+
err(unwritable.length > 10
|
|
3467
|
+
? ` Unwritable paths (showing 10 of ${unwritable.length}):\n`
|
|
3468
|
+
: " Unwritable paths:\n");
|
|
3469
|
+
for (const rel of shown)
|
|
3470
|
+
err(` ${rel}\n`);
|
|
3471
|
+
const uid = typeof process.getuid === "function" ? String(process.getuid()) : "<uid>";
|
|
3472
|
+
const gid = typeof process.getgid === "function" ? String(process.getgid()) : "<gid>";
|
|
3473
|
+
const tops = [
|
|
3474
|
+
...new Set(unwritable.map((rel) => topLevelRel(rel)).filter((top) => top !== ".")),
|
|
3475
|
+
].sort();
|
|
3476
|
+
err(" Repair ownership, then retry:\n");
|
|
3477
|
+
if (tops.length === 0) {
|
|
3478
|
+
err(` chown -R ${uid}:${gid} ${hqRoot}\n`);
|
|
3479
|
+
}
|
|
3480
|
+
else {
|
|
3481
|
+
for (const top of tops) {
|
|
3482
|
+
err(` chown -R ${uid}:${gid} ${path.join(hqRoot, top)}\n`);
|
|
3483
|
+
}
|
|
3484
|
+
}
|
|
3485
|
+
}
|
|
3486
|
+
function rescueBlockedMarkerPath(hqRoot) {
|
|
3487
|
+
return path.join(hqRoot, RESCUE_BLOCKED_MARKER_REL);
|
|
3488
|
+
}
|
|
3489
|
+
function postRestorePermitPath(hqRoot) {
|
|
3490
|
+
return path.join(hqRoot, POST_RESTORE_PERMIT_REL);
|
|
3491
|
+
}
|
|
3492
|
+
function writePostRestorePermit(hqRoot, entry) {
|
|
3493
|
+
const permit = {
|
|
3494
|
+
schemaVersion: 1,
|
|
3495
|
+
canonicalHqRoot: hqRoot,
|
|
3496
|
+
rootHash: canonicalRootHash(hqRoot),
|
|
3497
|
+
snapshotName: entry.name,
|
|
3498
|
+
snapshotRunId: entry.manifest.runId,
|
|
3499
|
+
verifiedAt: new Date().toISOString(),
|
|
3500
|
+
};
|
|
3501
|
+
const target = postRestorePermitPath(hqRoot);
|
|
3502
|
+
const temporary = `${target}.${process.pid}.tmp`;
|
|
3503
|
+
fs.mkdirSync(path.dirname(target), { recursive: true });
|
|
3504
|
+
fs.writeFileSync(temporary, JSON.stringify(permit, null, 2) + "\n", {
|
|
3505
|
+
mode: 0o600,
|
|
3506
|
+
});
|
|
3507
|
+
fs.renameSync(temporary, target);
|
|
3508
|
+
}
|
|
3509
|
+
function readPostRestorePermit(hqRoot) {
|
|
3510
|
+
let raw;
|
|
3511
|
+
try {
|
|
3512
|
+
raw = fs.readFileSync(postRestorePermitPath(hqRoot), "utf8");
|
|
3513
|
+
}
|
|
3514
|
+
catch (error) {
|
|
3515
|
+
if (error.code === "ENOENT") {
|
|
3516
|
+
return { permit: null };
|
|
3517
|
+
}
|
|
3518
|
+
return {
|
|
3519
|
+
permit: null,
|
|
3520
|
+
problem: `could not read ${POST_RESTORE_PERMIT_REL}: ${error instanceof Error ? error.message : String(error)}`,
|
|
3521
|
+
};
|
|
3522
|
+
}
|
|
3523
|
+
try {
|
|
3524
|
+
const value = JSON.parse(raw);
|
|
3525
|
+
if (value.schemaVersion !== 1 ||
|
|
3526
|
+
typeof value.canonicalHqRoot !== "string" ||
|
|
3527
|
+
typeof value.rootHash !== "string" ||
|
|
3528
|
+
typeof value.snapshotName !== "string" ||
|
|
3529
|
+
typeof value.snapshotRunId !== "string" ||
|
|
3530
|
+
typeof value.verifiedAt !== "string") {
|
|
3531
|
+
return { permit: null, problem: `${POST_RESTORE_PERMIT_REL} is malformed` };
|
|
3532
|
+
}
|
|
3533
|
+
return { permit: value };
|
|
3534
|
+
}
|
|
3535
|
+
catch (error) {
|
|
3536
|
+
return {
|
|
3537
|
+
permit: null,
|
|
3538
|
+
problem: `${POST_RESTORE_PERMIT_REL} is not valid JSON: ${error instanceof Error ? error.message : String(error)}`,
|
|
3539
|
+
};
|
|
3540
|
+
}
|
|
3541
|
+
}
|
|
3542
|
+
function consumePostRestorePermit(cfg, hqRoot) {
|
|
3543
|
+
const read = readPostRestorePermit(hqRoot);
|
|
3544
|
+
if (!read.permit) {
|
|
3545
|
+
return { used: false, ...(read.problem ? { problem: read.problem } : {}) };
|
|
3546
|
+
}
|
|
3547
|
+
const permit = read.permit;
|
|
3548
|
+
if (permit.canonicalHqRoot !== hqRoot ||
|
|
3549
|
+
permit.rootHash !== canonicalRootHash(hqRoot)) {
|
|
3550
|
+
return { used: false, problem: "permit belongs to a different HQ root" };
|
|
3551
|
+
}
|
|
3552
|
+
const inventory = inventoryManagedSnapshots(cfg.backupRoot, hqRoot, () => false, {
|
|
3553
|
+
keepGood: Number(cfg.backupRetentionCount),
|
|
3554
|
+
maxAgeDays: Number(cfg.backupRetentionDays),
|
|
3555
|
+
});
|
|
3556
|
+
if (inventory.unmanaged.some((item) => item.reason === "backup-root-unreadable")) {
|
|
3557
|
+
return { used: false, problem: `backup root is unreadable: ${cfg.backupRoot}` };
|
|
3558
|
+
}
|
|
3559
|
+
const entry = inventory.entries.find((item) => item.name === permit.snapshotName &&
|
|
3560
|
+
item.manifest.runId === permit.snapshotRunId &&
|
|
3561
|
+
item.manifest.status === "recovery-required");
|
|
3562
|
+
const unresolved = inventory.protected.some((item) => item.entry.name === permit.snapshotName &&
|
|
3563
|
+
item.reason === "recovery-required-no-later-good");
|
|
3564
|
+
if (!entry || !unresolved) {
|
|
3565
|
+
return { used: false, problem: "referenced snapshot is no longer unresolved recovery evidence" };
|
|
3566
|
+
}
|
|
3567
|
+
const mismatch = snapshotTreeMismatch(path.join(cfg.backupRoot, entry.name), hqRoot, entry.manifest.includedRoots, entry.manifest.symlinks);
|
|
3568
|
+
if (mismatch) {
|
|
3569
|
+
return { used: false, problem: `live tree no longer matches ${entry.name}: ${mismatch}` };
|
|
3570
|
+
}
|
|
3571
|
+
try {
|
|
3572
|
+
fs.unlinkSync(postRestorePermitPath(hqRoot));
|
|
3573
|
+
}
|
|
3574
|
+
catch (error) {
|
|
3575
|
+
return {
|
|
3576
|
+
used: false,
|
|
3577
|
+
problem: `could not consume the single-use permit: ${error instanceof Error ? error.message : String(error)}`,
|
|
3578
|
+
};
|
|
3579
|
+
}
|
|
3580
|
+
return { used: true, snapshotName: entry.name };
|
|
3581
|
+
}
|
|
3582
|
+
function writeRescueBlockedMarker(hqRoot, snapshots) {
|
|
3583
|
+
const body = {
|
|
3584
|
+
at: new Date().toISOString(),
|
|
3585
|
+
reason: RESCUE_BLOCKED_REASON,
|
|
3586
|
+
failureKind: "snapshot-recovery-circuit-breaker",
|
|
3587
|
+
snapshots,
|
|
3588
|
+
hint: "Run hq-rescue --recover-snapshots to inspect the snapshots. It deletes only snapshots proved superseded by a later completed-good rescue; otherwise restore the needed files, repair the release source, and retry.",
|
|
3589
|
+
};
|
|
3590
|
+
try {
|
|
3591
|
+
fs.mkdirSync(path.dirname(rescueBlockedMarkerPath(hqRoot)), {
|
|
3592
|
+
recursive: true,
|
|
3593
|
+
});
|
|
3594
|
+
fs.writeFileSync(rescueBlockedMarkerPath(hqRoot), JSON.stringify(body, null, 2) + "\n");
|
|
3595
|
+
}
|
|
3596
|
+
catch {
|
|
3597
|
+
/* stderr remains the operator-facing signal if .hq is unwritable */
|
|
3598
|
+
}
|
|
3599
|
+
}
|
|
3600
|
+
function clearRescueBlockedMarker(hqRoot) {
|
|
3601
|
+
try {
|
|
3602
|
+
fs.unlinkSync(rescueBlockedMarkerPath(hqRoot));
|
|
3603
|
+
}
|
|
3604
|
+
catch {
|
|
3605
|
+
/* absent is the success state */
|
|
3606
|
+
}
|
|
3607
|
+
}
|
|
2714
3608
|
function allocateSnapshotDir(backupRoot, runTs, rootHash, runId) {
|
|
2715
3609
|
return path.join(backupRoot, `pre-update-${runTs}-${rootHash}-${runId}`);
|
|
2716
3610
|
}
|
|
@@ -2729,13 +3623,22 @@ function snapshotManifest(status, hqRoot, runId, cfg, sourceSha, includedRoots)
|
|
|
2729
3623
|
sourceSha,
|
|
2730
3624
|
createdAt: now,
|
|
2731
3625
|
includedRoots,
|
|
3626
|
+
symlinks: [],
|
|
2732
3627
|
fileCount: 0,
|
|
2733
3628
|
byteCount: 0,
|
|
2734
3629
|
};
|
|
2735
3630
|
}
|
|
2736
|
-
function readSnapshotManifest(backupDir) {
|
|
3631
|
+
function readSnapshotManifest(backupDir, expectedHqRoot) {
|
|
2737
3632
|
try {
|
|
2738
|
-
|
|
3633
|
+
const parsed = JSON.parse(fs.readFileSync(path.join(backupDir, "SNAPSHOT.json"), "utf8"));
|
|
3634
|
+
const manifestRoot = expectedHqRoot ??
|
|
3635
|
+
(parsed && typeof parsed === "object"
|
|
3636
|
+
? parsed.canonicalHqRoot
|
|
3637
|
+
: undefined);
|
|
3638
|
+
return typeof manifestRoot === "string" &&
|
|
3639
|
+
isManagedSnapshotManifest(parsed, manifestRoot)
|
|
3640
|
+
? parsed
|
|
3641
|
+
: null;
|
|
2739
3642
|
}
|
|
2740
3643
|
catch {
|
|
2741
3644
|
return null;
|
|
@@ -2748,7 +3651,7 @@ function writeSnapshotManifest(backupDir, manifest) {
|
|
|
2748
3651
|
fs.writeFileSync(temp, JSON.stringify(manifest, null, 2) + "\n");
|
|
2749
3652
|
fs.renameSync(temp, target);
|
|
2750
3653
|
}
|
|
2751
|
-
function updateSnapshotStatus(backupDir, status, completedAt) {
|
|
3654
|
+
function updateSnapshotStatus(backupDir, status, completedAt, details = {}) {
|
|
2752
3655
|
const manifest = readSnapshotManifest(backupDir);
|
|
2753
3656
|
if (!manifest)
|
|
2754
3657
|
return;
|
|
@@ -2756,6 +3659,7 @@ function updateSnapshotStatus(backupDir, status, completedAt) {
|
|
|
2756
3659
|
writeSnapshotManifest(backupDir, {
|
|
2757
3660
|
...manifest,
|
|
2758
3661
|
status,
|
|
3662
|
+
...details,
|
|
2759
3663
|
fileCount: stats.files,
|
|
2760
3664
|
byteCount: stats.bytes,
|
|
2761
3665
|
...(completedAt ? { completedAt: new Date().toISOString() } : {}),
|
|
@@ -2912,22 +3816,206 @@ function cpATo(src, dest) {
|
|
|
2912
3816
|
/* ignore */
|
|
2913
3817
|
}
|
|
2914
3818
|
}
|
|
3819
|
+
class SnapshotCopyError extends Error {
|
|
3820
|
+
relativePath;
|
|
3821
|
+
operation;
|
|
3822
|
+
path;
|
|
3823
|
+
code;
|
|
3824
|
+
constructor(relativePath, operation, error, path = relativePath) {
|
|
3825
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
3826
|
+
super(message);
|
|
3827
|
+
this.relativePath = relativePath;
|
|
3828
|
+
this.operation = operation;
|
|
3829
|
+
this.path = path;
|
|
3830
|
+
this.name = "SnapshotCopyError";
|
|
3831
|
+
this.code =
|
|
3832
|
+
error?.code ?? "UNKNOWN";
|
|
3833
|
+
}
|
|
3834
|
+
}
|
|
3835
|
+
function snapshotCopyFailure(error) {
|
|
3836
|
+
return {
|
|
3837
|
+
relativePath: error.relativePath,
|
|
3838
|
+
operation: error.operation,
|
|
3839
|
+
code: error.code,
|
|
3840
|
+
message: error.message,
|
|
3841
|
+
occurredAt: new Date().toISOString(),
|
|
3842
|
+
};
|
|
3843
|
+
}
|
|
3844
|
+
function reportSnapshotCopyFailure(error, err) {
|
|
3845
|
+
const kind = error.operation === "read"
|
|
3846
|
+
? "snapshot-copy-unreadable"
|
|
3847
|
+
: "snapshot-copy-failed";
|
|
3848
|
+
err(`HQ_RESCUE_FAILURE_KIND=${kind}\n`);
|
|
3849
|
+
err(`HQ_RESCUE_SNAPSHOT_COPY_CODE=${error.code}\n`);
|
|
3850
|
+
const verb = error.operation === "metadata" ? "safely record" : error.operation;
|
|
3851
|
+
const detail = error.operation === "metadata" ? `: ${error.message}` : "";
|
|
3852
|
+
err(`error: safety snapshot could not ${verb} ${error.path} (${error.code})${detail}.\n`);
|
|
3853
|
+
if (error.operation === "read") {
|
|
3854
|
+
err(" HQ did not begin the update. If this file is stored in iCloud Drive or another cloud provider, download it locally, then retry.\n");
|
|
3855
|
+
}
|
|
3856
|
+
else {
|
|
3857
|
+
err(" HQ did not begin the update. Repair access to the reported path or backup location, then retry.\n");
|
|
3858
|
+
}
|
|
3859
|
+
}
|
|
3860
|
+
function pathIsInside(root, candidate, allowRoot = false) {
|
|
3861
|
+
const relative = path.relative(root, candidate);
|
|
3862
|
+
return ((allowRoot || relative !== "") &&
|
|
3863
|
+
!relative.startsWith(`..${path.sep}`) &&
|
|
3864
|
+
relative !== ".." &&
|
|
3865
|
+
!path.isAbsolute(relative));
|
|
3866
|
+
}
|
|
3867
|
+
function snapshotRelativePath(hqRoot, rel) {
|
|
3868
|
+
if (!rel ||
|
|
3869
|
+
rel.includes("\0") ||
|
|
3870
|
+
rel.includes("\\") ||
|
|
3871
|
+
path.posix.isAbsolute(rel) ||
|
|
3872
|
+
path.win32.isAbsolute(rel) ||
|
|
3873
|
+
rel.split("/").some((segment) => !segment || segment === "." || segment === "..")) {
|
|
3874
|
+
return null;
|
|
3875
|
+
}
|
|
3876
|
+
const absolute = path.resolve(hqRoot, rel);
|
|
3877
|
+
return pathIsInside(hqRoot, absolute) ? absolute : null;
|
|
3878
|
+
}
|
|
3879
|
+
function restoreParentIsSafe(hqRoot, destination) {
|
|
3880
|
+
for (let current = path.dirname(destination); current !== hqRoot; current = path.dirname(current)) {
|
|
3881
|
+
const st = lstatOrNull(current);
|
|
3882
|
+
if (st?.isSymbolicLink() || (st && !st.isDirectory()))
|
|
3883
|
+
return false;
|
|
3884
|
+
const parent = path.dirname(current);
|
|
3885
|
+
if (parent === current)
|
|
3886
|
+
return false;
|
|
3887
|
+
}
|
|
3888
|
+
return true;
|
|
3889
|
+
}
|
|
3890
|
+
function validateSnapshotSymlink(hqRoot, rel, target) {
|
|
3891
|
+
const linkPath = snapshotRelativePath(hqRoot, rel);
|
|
3892
|
+
if (!linkPath)
|
|
3893
|
+
return `invalid link path ${JSON.stringify(rel)}`;
|
|
3894
|
+
if (typeof target !== "string" || !target || target.includes("\0")) {
|
|
3895
|
+
return `invalid target for ${rel}`;
|
|
3896
|
+
}
|
|
3897
|
+
const resolvedTarget = path.resolve(path.dirname(linkPath), target);
|
|
3898
|
+
return pathIsInside(hqRoot, resolvedTarget, true)
|
|
3899
|
+
? null
|
|
3900
|
+
: `target for ${rel} resolves outside the HQ root`;
|
|
3901
|
+
}
|
|
3902
|
+
function copyRegularFileForSnapshot(src, dest, rel, mode) {
|
|
3903
|
+
let sourceFd;
|
|
3904
|
+
let destinationFd;
|
|
3905
|
+
try {
|
|
3906
|
+
try {
|
|
3907
|
+
sourceFd = fs.openSync(src, "r");
|
|
3908
|
+
}
|
|
3909
|
+
catch (error) {
|
|
3910
|
+
throw new SnapshotCopyError(rel, "read", error, src);
|
|
3911
|
+
}
|
|
3912
|
+
try {
|
|
3913
|
+
destinationFd = fs.openSync(dest, "w", mode);
|
|
3914
|
+
}
|
|
3915
|
+
catch (error) {
|
|
3916
|
+
throw new SnapshotCopyError(rel, "write", error, dest);
|
|
3917
|
+
}
|
|
3918
|
+
const buffer = Buffer.allocUnsafe(64 * 1024);
|
|
3919
|
+
for (;;) {
|
|
3920
|
+
let read;
|
|
3921
|
+
try {
|
|
3922
|
+
read = fs.readSync(sourceFd, buffer, 0, buffer.length, null);
|
|
3923
|
+
}
|
|
3924
|
+
catch (error) {
|
|
3925
|
+
throw new SnapshotCopyError(rel, "read", error, src);
|
|
3926
|
+
}
|
|
3927
|
+
if (read === 0)
|
|
3928
|
+
break;
|
|
3929
|
+
let offset = 0;
|
|
3930
|
+
while (offset < read) {
|
|
3931
|
+
let written;
|
|
3932
|
+
try {
|
|
3933
|
+
written = fs.writeSync(destinationFd, buffer, offset, read - offset);
|
|
3934
|
+
}
|
|
3935
|
+
catch (error) {
|
|
3936
|
+
throw new SnapshotCopyError(rel, "write", error, dest);
|
|
3937
|
+
}
|
|
3938
|
+
if (written === 0) {
|
|
3939
|
+
throw new SnapshotCopyError(rel, "write", new Error("write returned zero bytes"));
|
|
3940
|
+
}
|
|
3941
|
+
offset += written;
|
|
3942
|
+
}
|
|
3943
|
+
}
|
|
3944
|
+
}
|
|
3945
|
+
finally {
|
|
3946
|
+
if (destinationFd !== undefined)
|
|
3947
|
+
fs.closeSync(destinationFd);
|
|
3948
|
+
if (sourceFd !== undefined)
|
|
3949
|
+
fs.closeSync(sourceFd);
|
|
3950
|
+
}
|
|
3951
|
+
}
|
|
3952
|
+
/** Preserve a Windows file link as ordinary file data when links are forbidden. */
|
|
3953
|
+
function copySymlinkFileRepresentationForSnapshot(src, dest, rel) {
|
|
3954
|
+
let targetStats;
|
|
3955
|
+
try {
|
|
3956
|
+
targetStats = fs.statSync(src);
|
|
3957
|
+
}
|
|
3958
|
+
catch (error) {
|
|
3959
|
+
throw new SnapshotCopyError(rel, "read", error, src);
|
|
3960
|
+
}
|
|
3961
|
+
if (!targetStats.isFile()) {
|
|
3962
|
+
throw new SnapshotCopyError(rel, "metadata", new Error("file-link fallback requires a regular file target"), src);
|
|
3963
|
+
}
|
|
3964
|
+
try {
|
|
3965
|
+
fs.mkdirSync(path.dirname(dest), { recursive: true });
|
|
3966
|
+
}
|
|
3967
|
+
catch (error) {
|
|
3968
|
+
throw new SnapshotCopyError(rel, "write", error, dest);
|
|
3969
|
+
}
|
|
3970
|
+
copyRegularFileForSnapshot(src, dest, rel, targetStats.mode);
|
|
3971
|
+
preserveSnapshotMetadata(dest, targetStats);
|
|
3972
|
+
}
|
|
2915
3973
|
/** Snapshot copy that preserves metadata while pruning canonical transient directories. */
|
|
2916
|
-
function cpAToFiltered(src, dest, rel) {
|
|
2917
|
-
|
|
2918
|
-
|
|
2919
|
-
|
|
3974
|
+
function cpAToFiltered(src, dest, rel, hqRoot, symlinks) {
|
|
3975
|
+
let st;
|
|
3976
|
+
try {
|
|
3977
|
+
st = fs.lstatSync(src);
|
|
3978
|
+
}
|
|
3979
|
+
catch (error) {
|
|
3980
|
+
throw new SnapshotCopyError(rel, "read", error, src);
|
|
3981
|
+
}
|
|
2920
3982
|
if (st.isDirectory() && isRescueTransientDirectory(rel))
|
|
2921
3983
|
return;
|
|
2922
3984
|
if (st.isSymbolicLink()) {
|
|
2923
|
-
|
|
2924
|
-
|
|
3985
|
+
let target;
|
|
3986
|
+
try {
|
|
3987
|
+
target = fs.readlinkSync(src);
|
|
3988
|
+
}
|
|
3989
|
+
catch (error) {
|
|
3990
|
+
throw new SnapshotCopyError(rel, "read", error, src);
|
|
3991
|
+
}
|
|
3992
|
+
const problem = validateSnapshotSymlink(hqRoot, rel, target);
|
|
3993
|
+
if (problem) {
|
|
3994
|
+
throw new SnapshotCopyError(rel, "metadata", new Error(problem), src);
|
|
3995
|
+
}
|
|
3996
|
+
const representation = snapshotSymlinkRepresentation(hqRoot, src, dest, rel, target);
|
|
3997
|
+
if (representation === "file-copy") {
|
|
3998
|
+
copySymlinkFileRepresentationForSnapshot(src, dest, rel);
|
|
3999
|
+
}
|
|
4000
|
+
symlinks.push({ path: rel, target, ...(representation ? { representation } : {}) });
|
|
2925
4001
|
return;
|
|
2926
4002
|
}
|
|
2927
4003
|
if (st.isDirectory()) {
|
|
2928
|
-
|
|
2929
|
-
|
|
2930
|
-
|
|
4004
|
+
try {
|
|
4005
|
+
fs.mkdirSync(dest, { recursive: true });
|
|
4006
|
+
}
|
|
4007
|
+
catch (error) {
|
|
4008
|
+
throw new SnapshotCopyError(rel, "write", error, dest);
|
|
4009
|
+
}
|
|
4010
|
+
let names;
|
|
4011
|
+
try {
|
|
4012
|
+
names = fs.readdirSync(src);
|
|
4013
|
+
}
|
|
4014
|
+
catch (error) {
|
|
4015
|
+
throw new SnapshotCopyError(rel, "read", error, src);
|
|
4016
|
+
}
|
|
4017
|
+
for (const name of names) {
|
|
4018
|
+
cpAToFiltered(path.join(src, name), path.join(dest, name), path.posix.join(rel, name), hqRoot, symlinks);
|
|
2931
4019
|
}
|
|
2932
4020
|
try {
|
|
2933
4021
|
fs.chmodSync(dest, st.mode);
|
|
@@ -2938,8 +4026,16 @@ function cpAToFiltered(src, dest, rel) {
|
|
|
2938
4026
|
}
|
|
2939
4027
|
return;
|
|
2940
4028
|
}
|
|
2941
|
-
|
|
2942
|
-
|
|
4029
|
+
if (!st.isFile()) {
|
|
4030
|
+
throw new SnapshotCopyError(rel, "metadata", new Error("unsupported filesystem entry"));
|
|
4031
|
+
}
|
|
4032
|
+
try {
|
|
4033
|
+
fs.mkdirSync(path.dirname(dest), { recursive: true });
|
|
4034
|
+
}
|
|
4035
|
+
catch (error) {
|
|
4036
|
+
throw new SnapshotCopyError(rel, "write", error, dest);
|
|
4037
|
+
}
|
|
4038
|
+
copyRegularFileForSnapshot(src, dest, rel, st.mode);
|
|
2943
4039
|
try {
|
|
2944
4040
|
fs.chmodSync(dest, st.mode);
|
|
2945
4041
|
fs.utimesSync(dest, st.atime, st.mtime);
|