@indigoai-us/hq-cloud 6.16.52-alpha.0 → 6.16.52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backup-prune.d.ts +23 -1
- package/dist/backup-prune.d.ts.map +1 -1
- package/dist/backup-prune.js +92 -8
- package/dist/backup-prune.js.map +1 -1
- package/dist/backup-prune.test.js +36 -0
- package/dist/backup-prune.test.js.map +1 -1
- package/dist/bin/rescue-runner.js +2 -1
- package/dist/bin/rescue-runner.js.map +1 -1
- package/dist/bin/sync-runner-company.d.ts +0 -6
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +1 -30
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-company.test.js +0 -55
- package/dist/bin/sync-runner-company.test.js.map +1 -1
- package/dist/bin/sync-runner-planning.d.ts +3 -12
- package/dist/bin/sync-runner-planning.d.ts.map +1 -1
- package/dist/bin/sync-runner-planning.js +3 -65
- package/dist/bin/sync-runner-planning.js.map +1 -1
- package/dist/bin/sync-runner-planning.test.js +1 -108
- package/dist/bin/sync-runner-planning.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +2 -48
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +116 -311
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +5 -33
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +4 -34
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +79 -802
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/rescue-breaker.test.d.ts +2 -0
- package/dist/cli/rescue-breaker.test.d.ts.map +1 -0
- package/dist/cli/rescue-breaker.test.js +334 -0
- package/dist/cli/rescue-breaker.test.js.map +1 -0
- package/dist/cli/rescue-core.d.ts +40 -0
- package/dist/cli/rescue-core.d.ts.map +1 -1
- package/dist/cli/rescue-core.js +1454 -162
- package/dist/cli/rescue-core.js.map +1 -1
- package/dist/cli/rescue-realpath.test.d.ts +2 -0
- package/dist/cli/rescue-realpath.test.d.ts.map +1 -0
- package/dist/cli/rescue-realpath.test.js +109 -0
- package/dist/cli/rescue-realpath.test.js.map +1 -0
- package/dist/cli/rescue-rsync-preflight.test.js +256 -4
- package/dist/cli/rescue-rsync-preflight.test.js.map +1 -1
- package/dist/cli/rescue-snapshot-copy.test.d.ts +2 -0
- package/dist/cli/rescue-snapshot-copy.test.d.ts.map +1 -0
- package/dist/cli/rescue-snapshot-copy.test.js +627 -0
- package/dist/cli/rescue-snapshot-copy.test.js.map +1 -0
- package/dist/cli/rescue-snapshot-recovery.test.d.ts +2 -0
- package/dist/cli/rescue-snapshot-recovery.test.d.ts.map +1 -0
- package/dist/cli/rescue-snapshot-recovery.test.js +260 -0
- package/dist/cli/rescue-snapshot-recovery.test.js.map +1 -0
- package/dist/cli/rescue-source-switch-floor.test.d.ts +2 -0
- package/dist/cli/rescue-source-switch-floor.test.d.ts.map +1 -0
- package/dist/cli/rescue-source-switch-floor.test.js +210 -0
- package/dist/cli/rescue-source-switch-floor.test.js.map +1 -0
- package/dist/cli/rescue.d.ts +4 -0
- package/dist/cli/rescue.d.ts.map +1 -1
- package/dist/cli/rescue.js +20 -1
- package/dist/cli/rescue.js.map +1 -1
- package/dist/cli/rescue.test.js +16 -1
- package/dist/cli/rescue.test.js.map +1 -1
- package/dist/cli/share-presign-transport.test.js +2 -0
- package/dist/cli/share-presign-transport.test.js.map +1 -1
- package/dist/cli/share.d.ts +0 -19
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +37 -57
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +93 -87
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync-scope.test.js +0 -8
- package/dist/cli/sync-scope.test.js.map +1 -1
- package/dist/cli/sync.d.ts +0 -12
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +118 -226
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +2 -215
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/cli/watch-event-push-conflict.test.js +2 -0
- package/dist/cli/watch-event-push-conflict.test.js.map +1 -1
- package/dist/cognito-auth.d.ts +30 -0
- package/dist/cognito-auth.d.ts.map +1 -1
- package/dist/cognito-auth.js +175 -0
- package/dist/cognito-auth.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/journal-row-store.d.ts +5 -9
- package/dist/journal-row-store.d.ts.map +1 -1
- package/dist/journal-row-store.js +10 -70
- package/dist/journal-row-store.js.map +1 -1
- package/dist/journal-snapshot-v4.d.ts.map +1 -1
- package/dist/journal-snapshot-v4.js +6 -11
- package/dist/journal-snapshot-v4.js.map +1 -1
- package/dist/journal-store.test.js +7 -259
- package/dist/journal-store.test.js.map +1 -1
- package/dist/journal.d.ts +12 -37
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +76 -291
- package/dist/journal.js.map +1 -1
- package/dist/lib/exit-codes.d.ts +0 -15
- package/dist/lib/exit-codes.d.ts.map +1 -1
- package/dist/lib/exit-codes.js +0 -15
- package/dist/lib/exit-codes.js.map +1 -1
- package/dist/machine-auth.test.js +295 -0
- package/dist/machine-auth.test.js.map +1 -1
- package/dist/object-io.d.ts +2 -2
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +18 -3
- package/dist/object-io.js.map +1 -1
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +3 -10
- package/dist/personal-vault.js.map +1 -1
- package/dist/personal-vault.test.js +0 -35
- package/dist/personal-vault.test.js.map +1 -1
- package/dist/remote-pull.d.ts.map +1 -1
- package/dist/remote-pull.js +2 -9
- package/dist/remote-pull.js.map +1 -1
- package/dist/remote-pull.test.js +2 -59
- package/dist/remote-pull.test.js.map +1 -1
- package/dist/s3.d.ts +18 -1
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +47 -24
- package/dist/s3.js.map +1 -1
- package/dist/s3.test.js +27 -1
- package/dist/s3.test.js.map +1 -1
- package/dist/sync/area-ledger.js +4 -4
- package/dist/sync/area-ledger.js.map +1 -1
- package/dist/sync/parallel-sync-coordinator.test.js +0 -29
- package/dist/sync/parallel-sync-coordinator.test.js.map +1 -1
- package/dist/sync/realtime-telemetry.d.ts +1 -1
- package/dist/sync/realtime-telemetry.d.ts.map +1 -1
- package/dist/sync/realtime-telemetry.js +6 -14
- package/dist/sync/realtime-telemetry.js.map +1 -1
- package/dist/sync/realtime-telemetry.test.js +16 -136
- package/dist/sync/realtime-telemetry.test.js.map +1 -1
- package/dist/sync-core.d.ts +6 -4
- package/dist/sync-core.d.ts.map +1 -1
- package/dist/sync-core.js +10 -4
- package/dist/sync-core.js.map +1 -1
- package/dist/sync-core.test.js +25 -1
- package/dist/sync-core.test.js.map +1 -1
- package/package.json +1 -2
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts +0 -2
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts.map +0 -1
- package/dist/cli/onepass-pull-planner-regressions.test.js +0 -300
- package/dist/cli/onepass-pull-planner-regressions.test.js.map +0 -1
- package/dist/cli/onepass-pull-planner.test.d.ts +0 -2
- package/dist/cli/onepass-pull-planner.test.d.ts.map +0 -1
- package/dist/cli/onepass-pull-planner.test.js +0 -266
- package/dist/cli/onepass-pull-planner.test.js.map +0 -1
- package/dist/cli/streaming-pull-planner.bench.test.d.ts +0 -15
- package/dist/cli/streaming-pull-planner.bench.test.d.ts.map +0 -1
- package/dist/cli/streaming-pull-planner.bench.test.js +0 -273
- package/dist/cli/streaming-pull-planner.bench.test.js.map +0 -1
- package/dist/s3.list-cost.test.d.ts +0 -2
- package/dist/s3.list-cost.test.d.ts.map +0 -1
- package/dist/s3.list-cost.test.js +0 -51
- package/dist/s3.list-cost.test.js.map +0 -1
- package/dist/sync/remote-file-spool.d.ts +0 -16
- package/dist/sync/remote-file-spool.d.ts.map +0 -1
- package/dist/sync/remote-file-spool.js +0 -91
- package/dist/sync/remote-file-spool.js.map +0 -1
- package/dist/sync/remote-file-spool.test.d.ts +0 -2
- package/dist/sync/remote-file-spool.test.d.ts.map +0 -1
- package/dist/sync/remote-file-spool.test.js +0 -77
- package/dist/sync/remote-file-spool.test.js.map +0 -1
package/dist/cli/rescue-core.js
CHANGED
|
@@ -31,7 +31,7 @@ import * as fs from "fs";
|
|
|
31
31
|
import * as os from "os";
|
|
32
32
|
import * as path from "path";
|
|
33
33
|
import { assertRescuePathTypeUnchanged, isRescueTransientDirectory, } from "./rescue-snapshot.js";
|
|
34
|
-
import { applyBackupPrune, canonicalRootHash, inventoryManagedSnapshots, } from "../backup-prune.js";
|
|
34
|
+
import { applyBackupPrune, canonicalRootHash, inventoryManagedSnapshots, isManagedSnapshotManifest, } from "../backup-prune.js";
|
|
35
35
|
// Paths always preserved across wipe+overlay (shuttled out, restored after).
|
|
36
36
|
const CARVE_OUT_PATHS = [
|
|
37
37
|
"core/packages",
|
|
@@ -148,10 +148,26 @@ function boundedDiagnostic(value, limit) {
|
|
|
148
148
|
function pathEntries(pathValue) {
|
|
149
149
|
return pathValue ? pathValue.split(path.delimiter) : [];
|
|
150
150
|
}
|
|
151
|
-
|
|
151
|
+
const DEFAULT_WINDOWS_EXECUTABLE_EXTENSIONS = [".COM", ".EXE", ".BAT", ".CMD"];
|
|
152
|
+
function rsyncCandidateNames(env) {
|
|
153
|
+
if (process.platform !== "win32")
|
|
154
|
+
return ["rsync", "rsync.exe"];
|
|
155
|
+
const configuredExtensions = envValue(env, "PATHEXT")
|
|
156
|
+
?.split(";")
|
|
157
|
+
.map((extension) => extension.trim())
|
|
158
|
+
.filter((extension) => /^\.[A-Za-z0-9]+$/.test(extension));
|
|
159
|
+
const extensions = [
|
|
160
|
+
...(configuredExtensions?.length
|
|
161
|
+
? configuredExtensions
|
|
162
|
+
: DEFAULT_WINDOWS_EXECUTABLE_EXTENSIONS),
|
|
163
|
+
...DEFAULT_WINDOWS_EXECUTABLE_EXTENSIONS,
|
|
164
|
+
];
|
|
165
|
+
return [...new Set(["rsync", ...extensions.map((extension) => `rsync${extension.toLowerCase()}`)])];
|
|
166
|
+
}
|
|
167
|
+
export function findRsyncOnPath(entries, env) {
|
|
152
168
|
let firstUnexecutable;
|
|
153
169
|
for (const entry of entries) {
|
|
154
|
-
for (const name of
|
|
170
|
+
for (const name of rsyncCandidateNames(env)) {
|
|
155
171
|
const candidate = path.resolve(entry || ".", name);
|
|
156
172
|
try {
|
|
157
173
|
if (!fs.statSync(candidate).isFile())
|
|
@@ -174,14 +190,22 @@ function findRsyncOnPath(entries) {
|
|
|
174
190
|
: undefined;
|
|
175
191
|
}
|
|
176
192
|
/** The overlay requires rsync; do not allocate a recovery copy when it cannot run. */
|
|
177
|
-
function requireWorkingRsync(env, err) {
|
|
193
|
+
export function requireWorkingRsync(env, err) {
|
|
178
194
|
const resolvedPath = envValue(env, "PATH");
|
|
179
195
|
const entries = pathEntries(resolvedPath);
|
|
180
|
-
const foundRsync = findRsyncOnPath(entries);
|
|
181
|
-
|
|
196
|
+
const foundRsync = findRsyncOnPath(entries, env);
|
|
197
|
+
// Node documents that Windows .cmd/.bat files require cmd.exe rather than a
|
|
198
|
+
// direct child-process launch. Keep the path and fixed argument as distinct
|
|
199
|
+
// arguments instead of enabling shell:true for a constructed command string.
|
|
200
|
+
const launcher = process.platform === "win32" && /\.(?:cmd|bat)$/i.test(foundRsync?.path ?? "")
|
|
201
|
+
? { command: "cmd.exe", leadingArgs: ["/d", "/s", "/c", foundRsync.path] }
|
|
202
|
+
: { command: "rsync", leadingArgs: [] };
|
|
203
|
+
const probe = runWithSpawnError(launcher.command, [...launcher.leadingArgs, "--version"], {
|
|
204
|
+
env,
|
|
205
|
+
});
|
|
182
206
|
const output = `${probe.stdout}\n${probe.stderr}`;
|
|
183
207
|
if (probe.status === 0 && /\brsync\b.*\bversion\b/i.test(output))
|
|
184
|
-
return;
|
|
208
|
+
return launcher;
|
|
185
209
|
err("error: rsync preflight failed before any safety snapshot was allocated.\n");
|
|
186
210
|
if (probe.error) {
|
|
187
211
|
const errorDetail = "code" in probe.error && typeof probe.error.code === "string"
|
|
@@ -303,6 +327,9 @@ export function parseRescueArgs(argv) {
|
|
|
303
327
|
backupRoot: process.env.HQ_BACKUP_DIR || path.join(os.homedir(), ".hq", "backups"),
|
|
304
328
|
backupRetentionDays: process.env.HQ_BACKUP_RETENTION_DAYS || "7",
|
|
305
329
|
backupRetentionCount: process.env.HQ_BACKUP_RETENTION_COUNT || "2",
|
|
330
|
+
recoverSnapshots: false,
|
|
331
|
+
verifyRestoredSnapshot: "",
|
|
332
|
+
restoreSnapshot: "",
|
|
306
333
|
};
|
|
307
334
|
const usage = () => {
|
|
308
335
|
// The legacy script printed its comment header here; we emit a concise
|
|
@@ -311,7 +338,9 @@ export function parseRescueArgs(argv) {
|
|
|
311
338
|
" [--preserve PATH]... [--preserve-subpath REL]...\n" +
|
|
312
339
|
" [--hq-root DIR] [--no-history-check] [--floor-sha SHA]\n" +
|
|
313
340
|
" [--no-backup] [--backup-dir DIR] [--cloud-update]\n" +
|
|
314
|
-
" [--dry-run] [--yes]\n"
|
|
341
|
+
" [--dry-run] [--yes] [--recover-snapshots]\n" +
|
|
342
|
+
" [--verify-restored-snapshot SNAPSHOT]\n" +
|
|
343
|
+
" [--restore-snapshot SNAPSHOT]\n");
|
|
315
344
|
throw new ExitError(1);
|
|
316
345
|
};
|
|
317
346
|
let narrowCsv = "";
|
|
@@ -360,6 +389,23 @@ export function parseRescueArgs(argv) {
|
|
|
360
389
|
case "--backup-dir":
|
|
361
390
|
cfg.backupRoot = need(a);
|
|
362
391
|
break;
|
|
392
|
+
case "--recover-snapshots":
|
|
393
|
+
cfg.recoverSnapshots = true;
|
|
394
|
+
break;
|
|
395
|
+
case "--verify-restored-snapshot":
|
|
396
|
+
cfg.verifyRestoredSnapshot = need(a);
|
|
397
|
+
if (!cfg.verifyRestoredSnapshot) {
|
|
398
|
+
process.stderr.write("error: --verify-restored-snapshot requires a snapshot name.\n");
|
|
399
|
+
throw new ExitError(2);
|
|
400
|
+
}
|
|
401
|
+
break;
|
|
402
|
+
case "--restore-snapshot":
|
|
403
|
+
cfg.restoreSnapshot = need(a);
|
|
404
|
+
if (!cfg.restoreSnapshot) {
|
|
405
|
+
process.stderr.write("error: --restore-snapshot requires a snapshot name.\n");
|
|
406
|
+
throw new ExitError(2);
|
|
407
|
+
}
|
|
408
|
+
break;
|
|
363
409
|
case "--dry-run":
|
|
364
410
|
cfg.dryRun = true;
|
|
365
411
|
break;
|
|
@@ -420,6 +466,18 @@ export function parseRescueArgs(argv) {
|
|
|
420
466
|
throw new ExitError(2);
|
|
421
467
|
}
|
|
422
468
|
}
|
|
469
|
+
if (cfg.verifyRestoredSnapshot && !cfg.recoverSnapshots) {
|
|
470
|
+
process.stderr.write("error: --verify-restored-snapshot requires --recover-snapshots.\n");
|
|
471
|
+
throw new ExitError(2);
|
|
472
|
+
}
|
|
473
|
+
if (cfg.restoreSnapshot && !cfg.recoverSnapshots) {
|
|
474
|
+
process.stderr.write("error: --restore-snapshot requires --recover-snapshots.\n");
|
|
475
|
+
throw new ExitError(2);
|
|
476
|
+
}
|
|
477
|
+
if (cfg.restoreSnapshot && cfg.verifyRestoredSnapshot) {
|
|
478
|
+
process.stderr.write("error: --restore-snapshot and --verify-restored-snapshot must be run separately.\n");
|
|
479
|
+
throw new ExitError(2);
|
|
480
|
+
}
|
|
423
481
|
return cfg;
|
|
424
482
|
}
|
|
425
483
|
/**
|
|
@@ -453,6 +511,8 @@ export function runRescue(argv, opts = {}) {
|
|
|
453
511
|
}
|
|
454
512
|
};
|
|
455
513
|
try {
|
|
514
|
+
if (cfg.recoverSnapshots)
|
|
515
|
+
return recoverSnapshots(cfg, out, err);
|
|
456
516
|
return doRescue(cfg, env, out, err, (d) => {
|
|
457
517
|
tmpdir = d;
|
|
458
518
|
});
|
|
@@ -466,38 +526,8 @@ export function runRescue(argv, opts = {}) {
|
|
|
466
526
|
cleanup();
|
|
467
527
|
}
|
|
468
528
|
}
|
|
469
|
-
/**
|
|
470
|
-
|
|
471
|
-
* a bare drive letter. Pure + exported for tests; `platformPath` defaults to
|
|
472
|
-
* the host `path` module so Windows and POSIX runtimes each apply their own
|
|
473
|
-
* absoluteness rules (tests pass `path.win32` / `path.posix` explicitly).
|
|
474
|
-
*
|
|
475
|
-
* Rejecting bare `C:` matters even though `path.win32.isAbsolute("C:")` is
|
|
476
|
-
* already false — the explicit pattern documents the exact field failure and
|
|
477
|
-
* guards against a future helper swap that treats drive-relative paths as
|
|
478
|
-
* absolute.
|
|
479
|
-
*/
|
|
480
|
-
export function isUsableRescueRoot(root, platformPath = path) {
|
|
481
|
-
if (/^[A-Za-z]:$/.test(root))
|
|
482
|
-
return false;
|
|
483
|
-
return platformPath.isAbsolute(root);
|
|
484
|
-
}
|
|
485
|
-
/**
|
|
486
|
-
* `fs.realpathSync` that converts failure into the rescue's clean error path
|
|
487
|
-
* (message + ExitError) instead of an uncaught exception. An uncaught throw
|
|
488
|
-
* here is what turned one bad `--hq-root` into a daily crash-loop on Windows.
|
|
489
|
-
*/
|
|
490
|
-
function realpathOrExit(p, err) {
|
|
491
|
-
try {
|
|
492
|
-
return fs.realpathSync(p);
|
|
493
|
-
}
|
|
494
|
-
catch (e) {
|
|
495
|
-
err(`error: cannot resolve --hq-root ${p}: ${e instanceof Error ? e.message : String(e)}\n`);
|
|
496
|
-
throw new ExitError(1);
|
|
497
|
-
}
|
|
498
|
-
}
|
|
499
|
-
function doRescue(cfg, env, out, err, setTmp) {
|
|
500
|
-
// --- Resolve HQ root ---
|
|
529
|
+
/** Resolve and sanity-check the HQ root before either rescue mode can touch it. */
|
|
530
|
+
function resolveHqRoot(cfg, err) {
|
|
501
531
|
let hqRoot;
|
|
502
532
|
if (cfg.hqRootOverride) {
|
|
503
533
|
// Windows field failure (2026-08-02): the menubar app's daily background
|
|
@@ -544,11 +574,582 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
544
574
|
err(" pass --hq-root <dir> if the script is not at personal/skills/<skill>/.\n");
|
|
545
575
|
throw new ExitError(3);
|
|
546
576
|
}
|
|
577
|
+
return hqRoot;
|
|
578
|
+
}
|
|
579
|
+
/**
|
|
580
|
+
* Explicit recovery surface for a previously interrupted rescue.
|
|
581
|
+
*
|
|
582
|
+
* A recovery-required snapshot is evidence that a rescue started changing the
|
|
583
|
+
* release-managed tree and then failed. It is safe to delete only when the
|
|
584
|
+
* retention model has a later `completed-good` snapshot for this same canonical
|
|
585
|
+
* HQ root. That later manifest is the durable proof that a subsequent rescue
|
|
586
|
+
* completed; an age, an empty current tree, or a manually edited sync stamp is
|
|
587
|
+
* deliberately not accepted as proof.
|
|
588
|
+
*/
|
|
589
|
+
function recoverSnapshots(cfg, out, err) {
|
|
590
|
+
const hqRoot = resolveHqRoot(cfg, err);
|
|
591
|
+
const retention = {
|
|
592
|
+
keepGood: Number(cfg.backupRetentionCount),
|
|
593
|
+
maxAgeDays: Number(cfg.backupRetentionDays),
|
|
594
|
+
};
|
|
595
|
+
const isCopyingOwnerLive = (m) => {
|
|
596
|
+
try {
|
|
597
|
+
process.kill(m.creatorPid, 0);
|
|
598
|
+
return true;
|
|
599
|
+
}
|
|
600
|
+
catch {
|
|
601
|
+
return false;
|
|
602
|
+
}
|
|
603
|
+
};
|
|
604
|
+
const inventory = inventoryManagedSnapshots(cfg.backupRoot, hqRoot, isCopyingOwnerLive, retention);
|
|
605
|
+
if (inventory.unmanaged.some((item) => item.reason === "backup-root-unreadable")) {
|
|
606
|
+
err(`error: cannot inspect safety snapshots: backup root is unreadable: ${cfg.backupRoot}\n`);
|
|
607
|
+
return { status: 1 };
|
|
608
|
+
}
|
|
609
|
+
if (cfg.restoreSnapshot) {
|
|
610
|
+
return restoreRecoverySnapshot(cfg, hqRoot, inventory.entries, inventory.protected, out, err);
|
|
611
|
+
}
|
|
612
|
+
if (cfg.verifyRestoredSnapshot) {
|
|
613
|
+
return verifyRestoredSnapshot(cfg, hqRoot, inventory.entries, inventory.protected, out, err);
|
|
614
|
+
}
|
|
615
|
+
const verified = inventory.candidates.filter((item) => item.entry.manifest.status === "recovery-required" &&
|
|
616
|
+
item.reason === "recovery-required-superseded");
|
|
617
|
+
const blocked = inventory.protected.filter((item) => item.reason === "recovery-required-no-later-good");
|
|
618
|
+
const retainedByGrace = inventory.protected.filter((item) => item.entry.manifest.status === "recovery-required" &&
|
|
619
|
+
item.reason !== "recovery-required-no-later-good");
|
|
620
|
+
const incomplete = inventory.entries.filter((entry) => entry.manifest.status === "incomplete");
|
|
621
|
+
const laterGoodFor = (completedMs, name) => inventory.entries.find((entry) => entry.manifest.status === "completed-good" &&
|
|
622
|
+
(entry.completedMs > completedMs ||
|
|
623
|
+
(entry.completedMs === completedMs && entry.name > name)))?.name;
|
|
624
|
+
out(`==> Safety snapshot recovery inspection: ${cfg.backupRoot}\n`);
|
|
625
|
+
if (verified.length === 0 &&
|
|
626
|
+
blocked.length === 0 &&
|
|
627
|
+
retainedByGrace.length === 0 &&
|
|
628
|
+
incomplete.length === 0) {
|
|
629
|
+
out(" No recovery-required snapshots need action.\n");
|
|
630
|
+
return { status: 0 };
|
|
631
|
+
}
|
|
632
|
+
for (const item of verified) {
|
|
633
|
+
const laterGood = laterGoodFor(item.entry.completedMs, item.entry.name);
|
|
634
|
+
out(` verified ${item.entry.name}: later completed-good snapshot ${laterGood ?? "(revalidated during delete)"} proves this recovery evidence is superseded.\n`);
|
|
635
|
+
}
|
|
636
|
+
for (const item of blocked) {
|
|
637
|
+
out(` retained ${item.entry.name}: no later completed-good snapshot exists for this HQ root.\n`);
|
|
638
|
+
}
|
|
639
|
+
for (const item of retainedByGrace) {
|
|
640
|
+
out(` retained ${item.entry.name}: a later completed-good snapshot exists, but normal retention still keeps this newest recovery snapshot.\n`);
|
|
641
|
+
}
|
|
642
|
+
for (const entry of incomplete) {
|
|
643
|
+
out(` incomplete ${entry.name}: skipped ${formatSkippedSnapshotPaths(entry.manifest)}; it cannot restore every captured root.\n`);
|
|
644
|
+
}
|
|
645
|
+
if (verified.length === 0 &&
|
|
646
|
+
blocked.length === 0 &&
|
|
647
|
+
retainedByGrace.length === 0 &&
|
|
648
|
+
incomplete.length > 0) {
|
|
649
|
+
out("==> Snapshot recovery inspection complete: incomplete snapshots were retained and cannot restore every captured root.\n");
|
|
650
|
+
return { status: 0 };
|
|
651
|
+
}
|
|
652
|
+
if (verified.length > 0) {
|
|
653
|
+
if (cfg.dryRun) {
|
|
654
|
+
for (const item of verified) {
|
|
655
|
+
out(` would resolve ${item.entry.name}: verified superseded recovery snapshot would be retained during this dry run.\n`);
|
|
656
|
+
}
|
|
657
|
+
}
|
|
658
|
+
else {
|
|
659
|
+
const result = applyBackupPrune(cfg.backupRoot, hqRoot, { candidates: verified, protected: [], unmanaged: [] }, retention);
|
|
660
|
+
for (const name of result.deleted) {
|
|
661
|
+
out(` resolved ${name}: verified superseded recovery snapshot deleted.\n`);
|
|
662
|
+
}
|
|
663
|
+
for (const failure of result.failures) {
|
|
664
|
+
err(`error: could not resolve verified snapshot (retained): ${failure}\n`);
|
|
665
|
+
}
|
|
666
|
+
if (result.failures.length > 0)
|
|
667
|
+
return { status: 1 };
|
|
668
|
+
}
|
|
669
|
+
}
|
|
670
|
+
if (cfg.dryRun && blocked.length === 0) {
|
|
671
|
+
out("==> Snapshot recovery dry run complete: no snapshot was changed.\n");
|
|
672
|
+
return { status: 0 };
|
|
673
|
+
}
|
|
674
|
+
// Re-inventory after an apply because another process might have created or
|
|
675
|
+
// changed a snapshot while the prune lock was held. Nothing unverified has
|
|
676
|
+
// been deleted, and the updated inventory is the report we return to users.
|
|
677
|
+
const remaining = inventoryManagedSnapshots(cfg.backupRoot, hqRoot, isCopyingOwnerLive, retention);
|
|
678
|
+
const unresolved = remaining.protected.filter((item) => item.reason === "recovery-required-no-later-good");
|
|
679
|
+
if (unresolved.length > 0) {
|
|
680
|
+
err("error: snapshot recovery refused: the interrupted updates are not proven superseded.\n");
|
|
681
|
+
err(" HQ retained every unverified snapshot. A later completed-good rescue for this same HQ root is the only automatic proof.\n");
|
|
682
|
+
err(" Restore one listed complete snapshot, then verify it before one retry is permitted:\n");
|
|
683
|
+
for (const item of unresolved) {
|
|
684
|
+
if (hasSkippedSnapshotEntries(item.entry.manifest)) {
|
|
685
|
+
err(` ${item.entry.name} is incomplete: skipped ${formatSkippedSnapshotPaths(item.entry.manifest)} and cannot restore every captured root.\n`);
|
|
686
|
+
continue;
|
|
687
|
+
}
|
|
688
|
+
err(` restore ${item.entry.name}: ${formatSnapshotRestoreCommand(hqRoot, cfg.backupRoot, item.entry.name)}\n`);
|
|
689
|
+
err(` verify ${item.entry.name}: ${formatRecoveryCommand(hqRoot, cfg.backupRoot, item.entry.name)}\n`);
|
|
690
|
+
}
|
|
691
|
+
return { status: 1 };
|
|
692
|
+
}
|
|
693
|
+
out("==> Snapshot recovery complete: every eligible recovery snapshot was positively verified before deletion.\n");
|
|
694
|
+
return { status: 0 };
|
|
695
|
+
}
|
|
696
|
+
/**
|
|
697
|
+
* Explicitly restore every captured root, including links represented only in
|
|
698
|
+
* the snapshot manifest. Verification remains a separate positive-proof step.
|
|
699
|
+
*/
|
|
700
|
+
function restoreRecoverySnapshot(cfg, hqRoot, entries, protectedEntries, out, err) {
|
|
701
|
+
const entry = entries.find((item) => item.name === cfg.restoreSnapshot);
|
|
702
|
+
const unresolved = protectedEntries.find((item) => item.entry.name === cfg.restoreSnapshot &&
|
|
703
|
+
item.reason === "recovery-required-no-later-good");
|
|
704
|
+
if (!entry || !unresolved || entry.manifest.status !== "recovery-required") {
|
|
705
|
+
err(`error: ${cfg.restoreSnapshot} is not an unresolved recovery-required snapshot for this HQ root.\n`);
|
|
706
|
+
return { status: 1 };
|
|
707
|
+
}
|
|
708
|
+
if (hasSkippedSnapshotEntries(entry.manifest)) {
|
|
709
|
+
err(`error: ${cfg.restoreSnapshot} is incomplete: skipped ${formatSkippedSnapshotPaths(entry.manifest)} and cannot restore every captured root.\n`);
|
|
710
|
+
return { status: 1 };
|
|
711
|
+
}
|
|
712
|
+
if (cfg.dryRun) {
|
|
713
|
+
out(`==> Dry run would restore every captured root and recorded symlink from ${entry.name}.\n`);
|
|
714
|
+
return { status: 0 };
|
|
715
|
+
}
|
|
716
|
+
try {
|
|
717
|
+
restoreSnapshotPaths(path.join(cfg.backupRoot, entry.name), hqRoot, entry.manifest.includedRoots, entry.manifest);
|
|
718
|
+
}
|
|
719
|
+
catch (error) {
|
|
720
|
+
err(`error: could not restore safety snapshot ${entry.name}: ${error instanceof Error ? error.message : String(error)}\n`);
|
|
721
|
+
err(" HQ retained the snapshot and did not write a post-restore permit.\n");
|
|
722
|
+
return { status: 1 };
|
|
723
|
+
}
|
|
724
|
+
out(`==> Restored every captured root and recorded symlink from ${entry.name}.\n`);
|
|
725
|
+
out(` Verify the restored tree before retrying: ${formatRecoveryCommand(hqRoot, cfg.backupRoot, entry.name)}\n`);
|
|
726
|
+
return { status: 0 };
|
|
727
|
+
}
|
|
728
|
+
/** A live tree must still match the selected recovery snapshot at permit use. */
|
|
729
|
+
function verifyRestoredSnapshot(cfg, hqRoot, entries, protectedEntries, out, err) {
|
|
730
|
+
const entry = entries.find((item) => item.name === cfg.verifyRestoredSnapshot);
|
|
731
|
+
const unresolved = protectedEntries.find((item) => item.entry.name === cfg.verifyRestoredSnapshot &&
|
|
732
|
+
item.reason === "recovery-required-no-later-good");
|
|
733
|
+
if (!entry || !unresolved || entry.manifest.status !== "recovery-required") {
|
|
734
|
+
err(`error: ${cfg.verifyRestoredSnapshot} is not an unresolved recovery-required snapshot for this HQ root.\n`);
|
|
735
|
+
return { status: 1 };
|
|
736
|
+
}
|
|
737
|
+
if (hasSkippedSnapshotEntries(entry.manifest)) {
|
|
738
|
+
err(`error: ${cfg.verifyRestoredSnapshot} is incomplete: skipped ${formatSkippedSnapshotPaths(entry.manifest)} and cannot verify every captured root.\n`);
|
|
739
|
+
return { status: 1 };
|
|
740
|
+
}
|
|
741
|
+
const mismatch = snapshotTreeMismatch(path.join(cfg.backupRoot, entry.name), hqRoot, entry.manifest.includedRoots, entry.manifest.symlinks);
|
|
742
|
+
if (mismatch) {
|
|
743
|
+
err(`error: restored tree does not match snapshot ${entry.name}: ${mismatch}\n`);
|
|
744
|
+
err(" No post-restore permit was written; the circuit breaker remains armed.\n");
|
|
745
|
+
return { status: 1 };
|
|
746
|
+
}
|
|
747
|
+
if (cfg.dryRun) {
|
|
748
|
+
out(`==> Dry run verified restored snapshot ${entry.name}; no post-restore permit was written.\n`);
|
|
749
|
+
return { status: 0 };
|
|
750
|
+
}
|
|
751
|
+
try {
|
|
752
|
+
writePostRestorePermit(hqRoot, entry);
|
|
753
|
+
}
|
|
754
|
+
catch (error) {
|
|
755
|
+
err(`error: verified snapshot ${entry.name}, but could not write the post-restore permit: ${error instanceof Error ? error.message : String(error)}\n`);
|
|
756
|
+
return { status: 1 };
|
|
757
|
+
}
|
|
758
|
+
out(`==> Verified restored snapshot ${entry.name}; one rescue attempt is now permitted.\n`);
|
|
759
|
+
out(" That permit is rechecked and consumed before rescue changes files. If the attempt fails, the circuit breaker is armed again.\n");
|
|
760
|
+
return { status: 0 };
|
|
761
|
+
}
|
|
762
|
+
/** Return a precise mismatch instead of treating a partial restore as evidence. */
|
|
763
|
+
function snapshotTreeMismatch(snapshotRoot, hqRoot, includedRoots, symlinks) {
|
|
764
|
+
const roots = snapshotCapturedRoots(includedRoots);
|
|
765
|
+
if (typeof roots === "string")
|
|
766
|
+
return roots;
|
|
767
|
+
const recorded = snapshotSymlinkTargets(hqRoot, roots, symlinks);
|
|
768
|
+
if (typeof recorded === "string")
|
|
769
|
+
return recorded;
|
|
770
|
+
for (const root of roots) {
|
|
771
|
+
if (rescuePathKind(path.join(snapshotRoot, root)) === "missing" &&
|
|
772
|
+
!recorded.has(root)) {
|
|
773
|
+
return `snapshot does not contain captured root ${root}`;
|
|
774
|
+
}
|
|
775
|
+
const mismatch = snapshotPathMismatch(path.join(snapshotRoot, root), path.join(hqRoot, root), root, recorded);
|
|
776
|
+
if (mismatch)
|
|
777
|
+
return mismatch;
|
|
778
|
+
}
|
|
779
|
+
return null;
|
|
780
|
+
}
|
|
781
|
+
function snapshotCapturedRoots(includedRoots) {
|
|
782
|
+
if (!Array.isArray(includedRoots) || includedRoots.length === 0) {
|
|
783
|
+
return "snapshot manifest records no captured roots to verify";
|
|
784
|
+
}
|
|
785
|
+
const seen = new Set();
|
|
786
|
+
for (const root of includedRoots) {
|
|
787
|
+
if (typeof root !== "string" ||
|
|
788
|
+
!root ||
|
|
789
|
+
root === "." ||
|
|
790
|
+
root === ".." ||
|
|
791
|
+
root.includes("/") ||
|
|
792
|
+
root.includes("\\") ||
|
|
793
|
+
isRescueTransientDirectory(root) ||
|
|
794
|
+
seen.has(root)) {
|
|
795
|
+
return `snapshot manifest has an unsafe captured root ${JSON.stringify(root)}`;
|
|
796
|
+
}
|
|
797
|
+
seen.add(root);
|
|
798
|
+
}
|
|
799
|
+
return seen;
|
|
800
|
+
}
|
|
801
|
+
function snapshotSymlinkTargets(hqRoot, includedRoots, symlinks) {
|
|
802
|
+
if (symlinks !== undefined && !Array.isArray(symlinks)) {
|
|
803
|
+
return "snapshot manifest symlinks is not an array";
|
|
804
|
+
}
|
|
805
|
+
const recorded = new Map();
|
|
806
|
+
for (const link of symlinks ?? []) {
|
|
807
|
+
if (!link ||
|
|
808
|
+
typeof link.path !== "string" ||
|
|
809
|
+
(link.representation !== undefined && link.representation !== "file-copy")) {
|
|
810
|
+
return "snapshot manifest has an invalid recorded symlink";
|
|
811
|
+
}
|
|
812
|
+
const problem = validateSnapshotSymlink(hqRoot, link.path, link.target);
|
|
813
|
+
if (problem)
|
|
814
|
+
return `snapshot manifest ${problem}`;
|
|
815
|
+
const root = link.path.split("/")[0];
|
|
816
|
+
if (!includedRoots.has(root)) {
|
|
817
|
+
return `snapshot manifest records symlink ${link.path} outside its captured roots`;
|
|
818
|
+
}
|
|
819
|
+
if (recorded.has(link.path)) {
|
|
820
|
+
return `snapshot manifest records duplicate symlink ${link.path}`;
|
|
821
|
+
}
|
|
822
|
+
recorded.set(link.path, link);
|
|
823
|
+
}
|
|
824
|
+
for (const linkPath of recorded.keys()) {
|
|
825
|
+
for (let parent = path.posix.dirname(linkPath); parent !== "."; parent = path.posix.dirname(parent)) {
|
|
826
|
+
if (recorded.has(parent)) {
|
|
827
|
+
return `snapshot manifest records symlink ${linkPath} beneath recorded symlink ${parent}`;
|
|
828
|
+
}
|
|
829
|
+
}
|
|
830
|
+
}
|
|
831
|
+
return recorded;
|
|
832
|
+
}
|
|
833
|
+
function recordedSymlinkTargetMatches(livePath, recordedTarget, liveTarget) {
|
|
834
|
+
const parent = path.dirname(livePath);
|
|
835
|
+
return (path.resolve(parent, recordedTarget) === path.resolve(parent, liveTarget));
|
|
836
|
+
}
|
|
837
|
+
function snapshotPathMismatch(snapshotPath, livePath, rel, recordedSymlinks) {
|
|
838
|
+
const snapshotKind = rescuePathKind(snapshotPath);
|
|
839
|
+
const liveKind = rescuePathKind(livePath);
|
|
840
|
+
const recordedLink = recordedSymlinks.get(rel);
|
|
841
|
+
if (recordedLink !== undefined) {
|
|
842
|
+
if (recordedLink.representation === "file-copy") {
|
|
843
|
+
if (snapshotKind !== "file") {
|
|
844
|
+
return `${rel} is ${snapshotKind} but the snapshot records a copied file link`;
|
|
845
|
+
}
|
|
846
|
+
if (liveKind !== "file") {
|
|
847
|
+
return `${rel} is ${liveKind} but the snapshot records a copied file link`;
|
|
848
|
+
}
|
|
849
|
+
return bytesEqual(snapshotPath, livePath)
|
|
850
|
+
? null
|
|
851
|
+
: `${rel} has different file contents`;
|
|
852
|
+
}
|
|
853
|
+
if (snapshotKind !== "missing") {
|
|
854
|
+
return `${rel} is stored as both a snapshot entry and recorded symlink`;
|
|
855
|
+
}
|
|
856
|
+
if (liveKind !== "symlink") {
|
|
857
|
+
return `${rel} is ${liveKind} but the snapshot records a symlink`;
|
|
858
|
+
}
|
|
859
|
+
try {
|
|
860
|
+
return recordedSymlinkTargetMatches(livePath, recordedLink.target, fs.readlinkSync(livePath))
|
|
861
|
+
? null
|
|
862
|
+
: `${rel} has a different symlink target`;
|
|
863
|
+
}
|
|
864
|
+
catch (error) {
|
|
865
|
+
return `${rel} could not read a symlink target: ${error instanceof Error ? error.message : String(error)}`;
|
|
866
|
+
}
|
|
867
|
+
}
|
|
868
|
+
if (snapshotKind !== liveKind) {
|
|
869
|
+
return `${rel} is ${liveKind} but the snapshot records ${snapshotKind}`;
|
|
870
|
+
}
|
|
871
|
+
if (snapshotKind === "missing")
|
|
872
|
+
return null;
|
|
873
|
+
if (snapshotKind === "file") {
|
|
874
|
+
return bytesEqual(snapshotPath, livePath)
|
|
875
|
+
? null
|
|
876
|
+
: `${rel} has different file contents`;
|
|
877
|
+
}
|
|
878
|
+
if (snapshotKind === "symlink") {
|
|
879
|
+
try {
|
|
880
|
+
return fs.readlinkSync(snapshotPath) === fs.readlinkSync(livePath)
|
|
881
|
+
? null
|
|
882
|
+
: `${rel} has a different symlink target`;
|
|
883
|
+
}
|
|
884
|
+
catch (error) {
|
|
885
|
+
return `${rel} could not read a symlink target: ${error instanceof Error ? error.message : String(error)}`;
|
|
886
|
+
}
|
|
887
|
+
}
|
|
888
|
+
if (snapshotKind !== "directory") {
|
|
889
|
+
return `${rel} has unsupported path type ${snapshotKind}`;
|
|
890
|
+
}
|
|
891
|
+
let snapshotNames;
|
|
892
|
+
let liveNames;
|
|
893
|
+
try {
|
|
894
|
+
snapshotNames = fs
|
|
895
|
+
.readdirSync(snapshotPath)
|
|
896
|
+
.filter((name) => !isRescueTransientDirectory(path.posix.join(rel, name)))
|
|
897
|
+
.sort();
|
|
898
|
+
liveNames = fs
|
|
899
|
+
.readdirSync(livePath)
|
|
900
|
+
.filter((name) => !isRescueTransientDirectory(path.posix.join(rel, name)))
|
|
901
|
+
.sort();
|
|
902
|
+
}
|
|
903
|
+
catch (error) {
|
|
904
|
+
return `${rel} could not be read: ${error instanceof Error ? error.message : String(error)}`;
|
|
905
|
+
}
|
|
906
|
+
const expectedNames = new Set(snapshotNames);
|
|
907
|
+
for (const recordedRel of recordedSymlinks.keys()) {
|
|
908
|
+
if (path.posix.dirname(recordedRel) === rel) {
|
|
909
|
+
expectedNames.add(path.posix.basename(recordedRel));
|
|
910
|
+
}
|
|
911
|
+
}
|
|
912
|
+
const expected = [...expectedNames].sort();
|
|
913
|
+
if (expected.length !== liveNames.length ||
|
|
914
|
+
expected.some((name, index) => name !== liveNames[index])) {
|
|
915
|
+
return `${rel} has different entries`;
|
|
916
|
+
}
|
|
917
|
+
for (const name of expected) {
|
|
918
|
+
const mismatch = snapshotPathMismatch(path.join(snapshotPath, name), path.join(livePath, name), path.posix.join(rel, name), recordedSymlinks);
|
|
919
|
+
if (mismatch)
|
|
920
|
+
return mismatch;
|
|
921
|
+
}
|
|
922
|
+
return null;
|
|
923
|
+
}
|
|
924
|
+
/**
|
|
925
|
+
* Restore one captured path, then recreate only manifest-recorded symlinks
|
|
926
|
+
* whose locations and targets are both confined to the HQ root.
|
|
927
|
+
*/
|
|
928
|
+
export function restoreSnapshotPath(snapshotDir, hqRoot, rel, manifest) {
|
|
929
|
+
restoreSnapshotPaths(snapshotDir, hqRoot, [rel], manifest);
|
|
930
|
+
}
|
|
931
|
+
/** Restore all requested physical paths before recreating their recorded links. */
|
|
932
|
+
function restoreSnapshotPaths(snapshotDir, hqRoot, rels, manifest) {
|
|
933
|
+
const capturedRoots = snapshotCapturedRoots(manifest.includedRoots);
|
|
934
|
+
if (typeof capturedRoots === "string")
|
|
935
|
+
throw new Error(capturedRoots);
|
|
936
|
+
const recorded = snapshotSymlinkTargets(hqRoot, capturedRoots, manifest.symlinks);
|
|
937
|
+
if (typeof recorded === "string")
|
|
938
|
+
throw new Error(recorded);
|
|
939
|
+
for (const [recordedRel, link] of recorded) {
|
|
940
|
+
const snapshotKind = rescuePathKind(path.join(snapshotDir, recordedRel));
|
|
941
|
+
if (link.representation === "file-copy") {
|
|
942
|
+
if (snapshotKind !== "file") {
|
|
943
|
+
throw new Error(`snapshot records copied file link ${recordedRel} without copied file data`);
|
|
944
|
+
}
|
|
945
|
+
}
|
|
946
|
+
else if (snapshotKind !== "missing") {
|
|
947
|
+
throw new Error(`snapshot records symlink ${recordedRel} alongside copied data`);
|
|
948
|
+
}
|
|
949
|
+
}
|
|
950
|
+
const requested = [...new Set(rels)].sort((left, right) => left.localeCompare(right));
|
|
951
|
+
if (requested.length === 0)
|
|
952
|
+
throw new Error("snapshot restore requested no paths");
|
|
953
|
+
const items = requested.map((rel) => {
|
|
954
|
+
const destination = snapshotRelativePath(hqRoot, rel);
|
|
955
|
+
if (!destination) {
|
|
956
|
+
throw new Error(`invalid snapshot restore path ${JSON.stringify(rel)}`);
|
|
957
|
+
}
|
|
958
|
+
if (!restoreParentIsSafe(hqRoot, destination)) {
|
|
959
|
+
throw new Error(`snapshot restore parent for ${rel} is not a directory inside the HQ root`);
|
|
960
|
+
}
|
|
961
|
+
if (!capturedRoots.has(rel.split("/")[0])) {
|
|
962
|
+
throw new Error(`snapshot does not include ${rel}`);
|
|
963
|
+
}
|
|
964
|
+
const source = path.join(snapshotDir, rel);
|
|
965
|
+
const physicalSourceExists = lexists(source);
|
|
966
|
+
const recordedWithinPath = [...recorded.keys()].some((recordedRel) => recordedRel === rel || recordedRel.startsWith(`${rel}/`));
|
|
967
|
+
if (!physicalSourceExists && !recordedWithinPath) {
|
|
968
|
+
throw new Error(`snapshot does not contain ${rel}`);
|
|
969
|
+
}
|
|
970
|
+
return { rel, source, destination, physicalSourceExists };
|
|
971
|
+
});
|
|
972
|
+
for (const item of items) {
|
|
973
|
+
fs.rmSync(item.destination, { recursive: true, force: true });
|
|
974
|
+
}
|
|
975
|
+
for (const item of items) {
|
|
976
|
+
if (item.physicalSourceExists) {
|
|
977
|
+
copySnapshotPhysicalTree(item.source, item.destination, hqRoot, item.rel);
|
|
978
|
+
}
|
|
979
|
+
}
|
|
980
|
+
const links = orderSnapshotSymlinkRestore(hqRoot, [...recorded.entries()]
|
|
981
|
+
.filter(([recordedRel, link]) => link.representation === undefined &&
|
|
982
|
+
items.some((item) => recordedRel === item.rel || recordedRel.startsWith(`${item.rel}/`)))
|
|
983
|
+
.map(([recordedRel, link]) => [recordedRel, link.target]));
|
|
984
|
+
for (const [linkRel, target] of links) {
|
|
985
|
+
recreateSnapshotSymlink(hqRoot, linkRel, target);
|
|
986
|
+
}
|
|
987
|
+
}
|
|
988
|
+
function orderSnapshotSymlinkRestore(hqRoot, links) {
|
|
989
|
+
const pending = new Map(links);
|
|
990
|
+
const ordered = [];
|
|
991
|
+
while (pending.size > 0) {
|
|
992
|
+
const next = [...pending.entries()]
|
|
993
|
+
.sort(([left], [right]) => left.localeCompare(right))
|
|
994
|
+
.find(([rel, target]) => {
|
|
995
|
+
const linkPath = snapshotRelativePath(hqRoot, rel);
|
|
996
|
+
const resolvedTarget = path.resolve(path.dirname(linkPath), target);
|
|
997
|
+
return ![...pending.keys()].some((candidate) => {
|
|
998
|
+
if (candidate === rel)
|
|
999
|
+
return false;
|
|
1000
|
+
const candidatePath = snapshotRelativePath(hqRoot, candidate);
|
|
1001
|
+
return (resolvedTarget === candidatePath ||
|
|
1002
|
+
resolvedTarget.startsWith(`${candidatePath}${path.sep}`));
|
|
1003
|
+
});
|
|
1004
|
+
});
|
|
1005
|
+
if (!next) {
|
|
1006
|
+
ordered.push(...[...pending.entries()].sort(([left], [right]) => left.localeCompare(right)));
|
|
1007
|
+
break;
|
|
1008
|
+
}
|
|
1009
|
+
pending.delete(next[0]);
|
|
1010
|
+
ordered.push(next);
|
|
1011
|
+
}
|
|
1012
|
+
return ordered;
|
|
1013
|
+
}
|
|
1014
|
+
function copySnapshotPhysicalTree(source, destination, hqRoot, rel) {
|
|
1015
|
+
const st = fs.lstatSync(source);
|
|
1016
|
+
if (st.isSymbolicLink()) {
|
|
1017
|
+
recreateSnapshotSymlink(hqRoot, rel, fs.readlinkSync(source));
|
|
1018
|
+
return;
|
|
1019
|
+
}
|
|
1020
|
+
if (st.isDirectory()) {
|
|
1021
|
+
fs.mkdirSync(destination, { recursive: true });
|
|
1022
|
+
for (const name of fs.readdirSync(source)) {
|
|
1023
|
+
copySnapshotPhysicalTree(path.join(source, name), path.join(destination, name), hqRoot, path.posix.join(rel, name));
|
|
1024
|
+
}
|
|
1025
|
+
preserveSnapshotMetadata(destination, st);
|
|
1026
|
+
return;
|
|
1027
|
+
}
|
|
1028
|
+
if (!st.isFile())
|
|
1029
|
+
throw new Error(`snapshot contains unsupported entry ${rel}`);
|
|
1030
|
+
fs.mkdirSync(path.dirname(destination), { recursive: true });
|
|
1031
|
+
fs.copyFileSync(source, destination);
|
|
1032
|
+
preserveSnapshotMetadata(destination, st);
|
|
1033
|
+
}
|
|
1034
|
+
/** Data restored successfully remains restored when filesystem metadata is unsupported. */
|
|
1035
|
+
function preserveSnapshotMetadata(destination, st) {
|
|
1036
|
+
try {
|
|
1037
|
+
fs.chmodSync(destination, st.mode);
|
|
1038
|
+
fs.utimesSync(destination, st.atime, st.mtime);
|
|
1039
|
+
}
|
|
1040
|
+
catch {
|
|
1041
|
+
/* metadata preservation is best-effort, matching the prior rollback helper */
|
|
1042
|
+
}
|
|
1043
|
+
}
|
|
1044
|
+
function recreateSnapshotSymlink(hqRoot, rel, target, destination = snapshotRelativePath(hqRoot, rel)) {
|
|
1045
|
+
const problem = validateSnapshotSymlink(hqRoot, rel, target);
|
|
1046
|
+
if (problem)
|
|
1047
|
+
throw new Error(`invalid snapshot symlink: ${problem}`);
|
|
1048
|
+
if (!destination)
|
|
1049
|
+
throw new Error(`invalid snapshot symlink destination ${rel}`);
|
|
1050
|
+
const restoredLinkPath = snapshotRelativePath(hqRoot, rel);
|
|
1051
|
+
const resolvedTarget = path.resolve(path.dirname(restoredLinkPath), target);
|
|
1052
|
+
fs.mkdirSync(path.dirname(destination), { recursive: true });
|
|
1053
|
+
fs.rmSync(destination, { recursive: true, force: true });
|
|
1054
|
+
try {
|
|
1055
|
+
fs.symlinkSync(target, destination);
|
|
1056
|
+
}
|
|
1057
|
+
catch (error) {
|
|
1058
|
+
if (process.platform === "win32" &&
|
|
1059
|
+
error.code === "EPERM" &&
|
|
1060
|
+
isDir(resolvedTarget)) {
|
|
1061
|
+
fs.symlinkSync(resolvedTarget, destination, "junction");
|
|
1062
|
+
return;
|
|
1063
|
+
}
|
|
1064
|
+
throw error;
|
|
1065
|
+
}
|
|
1066
|
+
}
|
|
1067
|
+
/**
|
|
1068
|
+
* Directory links can become junctions on Windows. A file link without link
|
|
1069
|
+
* privilege instead needs copied file data at its original snapshot path.
|
|
1070
|
+
*/
|
|
1071
|
+
function snapshotSymlinkRepresentation(hqRoot, source, destination, rel, target) {
|
|
1072
|
+
if (process.platform !== "win32")
|
|
1073
|
+
return undefined;
|
|
1074
|
+
const probe = `${destination}.hq-restore-probe-${process.pid}-${Date.now().toString(36)}`;
|
|
1075
|
+
try {
|
|
1076
|
+
recreateSnapshotSymlink(hqRoot, rel, target, probe);
|
|
1077
|
+
fs.rmSync(probe, { recursive: true, force: true });
|
|
1078
|
+
return undefined;
|
|
1079
|
+
}
|
|
1080
|
+
catch (error) {
|
|
1081
|
+
try {
|
|
1082
|
+
fs.rmSync(probe, { recursive: true, force: true });
|
|
1083
|
+
}
|
|
1084
|
+
catch (cleanupError) {
|
|
1085
|
+
throw new SnapshotCopyError(rel, "write", cleanupError, probe);
|
|
1086
|
+
}
|
|
1087
|
+
if (process.platform === "win32" &&
|
|
1088
|
+
error.code === "EPERM") {
|
|
1089
|
+
try {
|
|
1090
|
+
if (fs.statSync(source).isFile())
|
|
1091
|
+
return "file-copy";
|
|
1092
|
+
}
|
|
1093
|
+
catch (targetError) {
|
|
1094
|
+
throw new SnapshotCopyError(rel, "read", targetError, source);
|
|
1095
|
+
}
|
|
1096
|
+
}
|
|
1097
|
+
throw new SnapshotCopyError(rel, "metadata", new Error(`recorded symlink cannot be restored on this machine: ${error instanceof Error ? error.message : String(error)}`), source);
|
|
1098
|
+
}
|
|
1099
|
+
}
|
|
1100
|
+
/**
|
|
1101
|
+
* True when `root` is usable as a rescue target: an absolute path that is not
|
|
1102
|
+
* a bare drive letter. Pure + exported for tests; `platformPath` defaults to
|
|
1103
|
+
* the host `path` module so Windows and POSIX runtimes each apply their own
|
|
1104
|
+
* absoluteness rules (tests pass `path.win32` / `path.posix` explicitly).
|
|
1105
|
+
*
|
|
1106
|
+
* Rejecting bare `C:` matters even though `path.win32.isAbsolute("C:")` is
|
|
1107
|
+
* already false — the explicit pattern documents the exact field failure and
|
|
1108
|
+
* guards against a future helper swap that treats drive-relative paths as
|
|
1109
|
+
* absolute.
|
|
1110
|
+
*/
|
|
1111
|
+
export function isUsableRescueRoot(root, platformPath = path) {
|
|
1112
|
+
if (/^[A-Za-z]:$/.test(root))
|
|
1113
|
+
return false;
|
|
1114
|
+
return platformPath.isAbsolute(root);
|
|
1115
|
+
}
|
|
1116
|
+
/**
|
|
1117
|
+
* Canonicalize the rescue root without treating a known-good directory as
|
|
1118
|
+
* unusable when Node's JavaScript realpath walker cannot inspect a Windows
|
|
1119
|
+
* drive prefix. Prefer the native syscall, retain the JS walker as a fallback,
|
|
1120
|
+
* and use an absolute path only after stat confirms it is a directory.
|
|
1121
|
+
*/
|
|
1122
|
+
export function realpathOrExit(p, err) {
|
|
1123
|
+
try {
|
|
1124
|
+
return fs.realpathSync.native(p);
|
|
1125
|
+
}
|
|
1126
|
+
catch {
|
|
1127
|
+
try {
|
|
1128
|
+
return fs.realpathSync(p);
|
|
1129
|
+
}
|
|
1130
|
+
catch (realpathError) {
|
|
1131
|
+
const resolved = path.resolve(p);
|
|
1132
|
+
try {
|
|
1133
|
+
if (!fs.statSync(resolved).isDirectory()) {
|
|
1134
|
+
throw new Error("resolved path is not a directory");
|
|
1135
|
+
}
|
|
1136
|
+
}
|
|
1137
|
+
catch {
|
|
1138
|
+
err(`error: cannot resolve --hq-root ${p}: ${realpathError instanceof Error ? realpathError.message : String(realpathError)}\n`);
|
|
1139
|
+
throw new ExitError(1);
|
|
1140
|
+
}
|
|
1141
|
+
err("warning: could not canonicalize --hq-root; using a normalized absolute path.\n");
|
|
1142
|
+
return resolved;
|
|
1143
|
+
}
|
|
1144
|
+
}
|
|
1145
|
+
}
|
|
1146
|
+
function doRescue(cfg, env, out, err, setTmp) {
|
|
1147
|
+
const hqRoot = resolveHqRoot(cfg, err);
|
|
547
1148
|
const RUN_TS = utcStamp(new Date(), "dash");
|
|
548
1149
|
// Both the release overlay and a restorable snapshot require rsync. A shim
|
|
549
1150
|
// which merely starts but exits non-zero used to trigger an enormous cp -a
|
|
550
1151
|
// fallback, then fail later at overlay. Stop before classification/allocation.
|
|
551
|
-
requireWorkingRsync(env, err);
|
|
1152
|
+
const rsyncLauncher = requireWorkingRsync(env, err);
|
|
552
1153
|
// --- Capture the pre-rescue .claude/settings.json (BEFORE the wipe) ---
|
|
553
1154
|
// This is `ours` for the post-overlay settings reconcile. It must be read now
|
|
554
1155
|
// because the overlay-safe classification deletes/overwrites it before
|
|
@@ -582,8 +1183,9 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
582
1183
|
prevSyncRef = yq('.replaced_from_source.ref // .replaced_from_staging.ref // ""');
|
|
583
1184
|
prevSyncAt = yq('.replaced_from_source.last_sync_at // .replaced_from_staging.last_sync_at // ""');
|
|
584
1185
|
}
|
|
585
|
-
//
|
|
586
|
-
|
|
1186
|
+
// An explicit caller floor is the recovery path for a missing or stale stamp.
|
|
1187
|
+
// It remains safe because the selected release clone must still contain it.
|
|
1188
|
+
if (cfg.floorShaOverride) {
|
|
587
1189
|
prevSyncSha = cfg.floorShaOverride;
|
|
588
1190
|
prevSyncSource = cfg.sourceRepo;
|
|
589
1191
|
prevSyncRef = "(caller-supplied floor)";
|
|
@@ -592,12 +1194,8 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
592
1194
|
out(`==> HQ root: ${hqRoot}\n`);
|
|
593
1195
|
out(`==> Source: https://github.com/${cfg.sourceRepo} @ ${cfg.ref}\n`);
|
|
594
1196
|
if (prevSyncSha) {
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
}
|
|
598
|
-
else {
|
|
599
|
-
out(`==> Prior sync: ${prevSyncSha} from ${prevSyncSource} (different source — ignoring as floor)\n`);
|
|
600
|
-
}
|
|
1197
|
+
const sourceLabel = prevSyncSource || "(source not recorded)";
|
|
1198
|
+
out(`==> Prior sync: ${prevSyncSha} from ${sourceLabel}@${prevSyncRef} (${prevSyncAt}) — checking cloned release history for floor\n`);
|
|
601
1199
|
}
|
|
602
1200
|
if (cfg.narrowPaths.length !== 0) {
|
|
603
1201
|
out("==> Mode: narrow (--paths)\n");
|
|
@@ -753,21 +1351,22 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
753
1351
|
// --- Resolve history floor ---
|
|
754
1352
|
let historyFloor = "";
|
|
755
1353
|
let baselineMode = "head_compare";
|
|
756
|
-
if (cfg.historyCheck && prevSyncSha
|
|
1354
|
+
if (cfg.historyCheck && prevSyncSha) {
|
|
757
1355
|
if (run("git", ["cat-file", "-e", prevSyncSha], { cwd: srcDir, env })
|
|
758
1356
|
.status === 0) {
|
|
759
1357
|
historyFloor = prevSyncSha;
|
|
760
1358
|
baselineMode = "history_floor";
|
|
761
1359
|
}
|
|
762
1360
|
else {
|
|
763
|
-
out(` prior-sync SHA ${prevSyncSha} not reachable in
|
|
1361
|
+
out(` WARN: prior-sync SHA ${prevSyncSha} is not reachable in the cloned release history; falling back to head_compare\n`);
|
|
764
1362
|
}
|
|
765
1363
|
}
|
|
766
1364
|
if (baselineMode === "history_floor") {
|
|
767
1365
|
out(`==> Baseline: ${historyFloor} (last-sync floor reachable)\n`);
|
|
768
1366
|
}
|
|
769
1367
|
else {
|
|
770
|
-
out("==> Baseline: HEAD compare (no usable last-sync
|
|
1368
|
+
out("==> Baseline: HEAD compare (no usable reachable last-sync floor)\n");
|
|
1369
|
+
out("==> WARNING: retired release files cannot be detected or removed in head_compare mode. Re-run with history checking and --floor-sha <sha> for a reachable baseline.\n");
|
|
771
1370
|
}
|
|
772
1371
|
// --- Build wipe/overlay arg sets per mode ---
|
|
773
1372
|
const pruneNames = [];
|
|
@@ -860,6 +1459,7 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
860
1459
|
out,
|
|
861
1460
|
decisions: [],
|
|
862
1461
|
actions: [],
|
|
1462
|
+
skippedSnapshotRels: [],
|
|
863
1463
|
};
|
|
864
1464
|
// --- Walk + classify (PHASE 1: read-only) ---
|
|
865
1465
|
// Classification mutates nothing on disk: every outcome is recorded as a
|
|
@@ -892,7 +1492,7 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
892
1492
|
if (cfg.dryRun) {
|
|
893
1493
|
out("\n");
|
|
894
1494
|
out("==> DRY RUN classification summary:\n");
|
|
895
|
-
out(` user-only (
|
|
1495
|
+
out(` user-only (left in place under release-managed roots): ${counts.userOnly} files\n`);
|
|
896
1496
|
out(` unchanged (preserved/re-laid): ${counts.unchanged} files\n`);
|
|
897
1497
|
out(` user-edit (rescue / diff-append): ${counts.userEdit} files\n`);
|
|
898
1498
|
out(` user-edit (conflict quarantine): ${counts.userConflict} files\n`);
|
|
@@ -957,7 +1557,7 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
957
1557
|
// unchanged versus before the two-phase split.
|
|
958
1558
|
const overlayProtect = [];
|
|
959
1559
|
if (fs.statSync(unchangedList).size > 0) {
|
|
960
|
-
overlayProtect.push(`--exclude-from=${unchangedList}`);
|
|
1560
|
+
overlayProtect.push("--from0", `--exclude-from=${unchangedList}`);
|
|
961
1561
|
}
|
|
962
1562
|
const overlayArgs = [
|
|
963
1563
|
"-a",
|
|
@@ -968,73 +1568,164 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
968
1568
|
];
|
|
969
1569
|
// Exercise the exact overlay command before any classified delete/move. A
|
|
970
1570
|
// Cygwin path conversion or filter failure therefore leaves the root intact.
|
|
971
|
-
const overlayPreflight = run(
|
|
1571
|
+
const overlayPreflight = run(rsyncLauncher.command, [...rsyncLauncher.leadingArgs, "--dry-run", ...overlayArgs], { env });
|
|
972
1572
|
if (overlayPreflight.status !== 0) {
|
|
973
1573
|
err(overlayPreflight.stderr);
|
|
974
1574
|
throw new ExitError(overlayPreflight.status || 1);
|
|
975
1575
|
}
|
|
976
1576
|
ctx.actions = buildRescueActions(ctx);
|
|
1577
|
+
// Permission preflight is deliberately before snapshot allocation. A
|
|
1578
|
+
// root-owned release tree (EACCES on rename/delete) used to allocate a
|
|
1579
|
+
// snapshot, fail apply twice, and then freeze rescue behind the
|
|
1580
|
+
// recovery-required circuit breaker. Fail closed with no snapshot instead.
|
|
1581
|
+
const unwritable = collectUnwritableRescueMutationPaths(hqRoot, ctx.actions);
|
|
1582
|
+
if (unwritable.length > 0) {
|
|
1583
|
+
reportUnwritableRescuePaths(hqRoot, unwritable, err);
|
|
1584
|
+
throw new ExitError(1);
|
|
1585
|
+
}
|
|
977
1586
|
// Snapshot allocation is deliberately after the entire read-only walk and
|
|
978
1587
|
// action-plan build. A no-op (or a classifier fault above) cannot consume
|
|
979
1588
|
// backup space.
|
|
980
1589
|
let backupDir = "";
|
|
981
1590
|
let mutationStarted = false;
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1591
|
+
let snapshotSkippedEntries = [];
|
|
1592
|
+
try {
|
|
1593
|
+
const hasDestructivePaths = ctx.actions.some((action) => action.affectedRels.length > 0);
|
|
1594
|
+
if (cfg.doBackup &&
|
|
1595
|
+
!cfg.dryRun &&
|
|
1596
|
+
wipeToplevel.length !== 0 &&
|
|
1597
|
+
hasDestructivePaths) {
|
|
1598
|
+
const prePlan = pruneManagedBackups(cfg, hqRoot, out, false);
|
|
1599
|
+
const unresolved = prePlan.protected.filter((x) => x.reason === "recovery-required-no-later-good");
|
|
1600
|
+
if (unresolved.length >= 2) {
|
|
1601
|
+
const postRestore = consumePostRestorePermit(cfg, hqRoot);
|
|
1602
|
+
if (postRestore.used) {
|
|
1603
|
+
out(`==> Verified post-restore permit for ${postRestore.snapshotName}; allowing one rescue attempt.\n`);
|
|
1604
|
+
out(" The permit was consumed before mutation. If this attempt fails, the circuit breaker is armed again.\n");
|
|
1605
|
+
}
|
|
1606
|
+
else {
|
|
1607
|
+
if (postRestore.problem) {
|
|
1608
|
+
err(`error: post-restore permit is not usable: ${postRestore.problem}\n`);
|
|
1609
|
+
}
|
|
1610
|
+
const snapshotNames = unresolved.map((x) => x.entry.name);
|
|
1611
|
+
writeRescueBlockedMarker(hqRoot, snapshotNames);
|
|
1612
|
+
err("HQ_RESCUE_FAILURE_KIND=snapshot-recovery-circuit-breaker\n");
|
|
1613
|
+
err("error: safety snapshot circuit breaker is open: two interrupted updates still need recovery proof.\n");
|
|
1614
|
+
err(" HQ kept their pre-update snapshots because those updates started changing the release-managed tree but did not finish.\n");
|
|
1615
|
+
err(" It will not overwrite or delete that evidence without proof that a later rescue completed successfully.\n");
|
|
1616
|
+
err(` Inspect safely: ${formatRecoveryCommand(hqRoot, cfg.backupRoot)}\n`);
|
|
1617
|
+
err(" It deletes only snapshots with a later completed-good rescue. For unresolved snapshots, use --restore-snapshot to restore every captured root and recorded link, then run its verification command before retrying.\n");
|
|
1618
|
+
throw new ExitError(1);
|
|
1619
|
+
}
|
|
1620
|
+
}
|
|
1621
|
+
requireBackupCapacity(cfg, hqRoot, wipeToplevel, env, err);
|
|
1622
|
+
fs.mkdirSync(cfg.backupRoot, { recursive: true });
|
|
1623
|
+
const runId = `${process.pid}-${Date.now().toString(36)}`;
|
|
1624
|
+
backupDir = allocateSnapshotDir(cfg.backupRoot, RUN_TS, canonicalRootHash(hqRoot), runId);
|
|
1625
|
+
out("\n");
|
|
1626
|
+
out(`==> Safety snapshot -> ${backupDir}\n`);
|
|
1627
|
+
const snapshotSymlinks = [];
|
|
1628
|
+
const snapshotSkippedErrors = [];
|
|
1629
|
+
try {
|
|
1630
|
+
fs.mkdirSync(backupDir, { recursive: false });
|
|
1631
|
+
writeSnapshotManifest(backupDir, snapshotManifest("copying", hqRoot, runId, cfg, srcSha, wipeToplevel));
|
|
1632
|
+
const attemptedRoots = wipeToplevel.filter((rootRel) => !isRescueTransientDirectory(rootRel));
|
|
1633
|
+
const failedRoots = [];
|
|
1634
|
+
for (const rootRel of wipeToplevel) {
|
|
1635
|
+
const srcAbs = path.join(hqRoot, rootRel);
|
|
1636
|
+
if (isRescueTransientDirectory(rootRel))
|
|
1637
|
+
continue;
|
|
1638
|
+
try {
|
|
1639
|
+
const copy = cpAToFiltered(srcAbs, path.join(backupDir, rootRel), rootRel, hqRoot, snapshotSymlinks, snapshotSkippedErrors);
|
|
1640
|
+
if (!copy.captured) {
|
|
1641
|
+
const failure = copy.firstFailure ??
|
|
1642
|
+
new SnapshotCopyError(rootRel, "metadata", new Error("snapshot root contains no capturable entries"), srcAbs);
|
|
1643
|
+
if (!copy.firstFailure)
|
|
1644
|
+
snapshotSkippedErrors.push(failure);
|
|
1645
|
+
failedRoots.push(failure);
|
|
1646
|
+
}
|
|
1647
|
+
}
|
|
1648
|
+
catch (error) {
|
|
1649
|
+
if (error instanceof SnapshotCopyError) {
|
|
1650
|
+
snapshotSkippedErrors.push(error);
|
|
1651
|
+
failedRoots.push(error);
|
|
1652
|
+
continue;
|
|
1653
|
+
}
|
|
1654
|
+
throw error;
|
|
1655
|
+
}
|
|
1656
|
+
}
|
|
1657
|
+
snapshotSkippedEntries = snapshotSkippedErrors.map(snapshotCopyFailure);
|
|
1658
|
+
if (attemptedRoots.length > 0 &&
|
|
1659
|
+
failedRoots.length === attemptedRoots.length) {
|
|
1660
|
+
const failure = failedRoots[0];
|
|
1661
|
+
updateSnapshotStatus(backupDir, "failed-pre-mutation", undefined, {
|
|
1662
|
+
symlinks: snapshotSymlinks,
|
|
1663
|
+
copyFailure: snapshotCopyFailure(failure),
|
|
1664
|
+
skippedEntries: snapshotSkippedEntries,
|
|
1665
|
+
});
|
|
1666
|
+
reportSnapshotCopyFailure(failure, err);
|
|
1667
|
+
throw new ExitError(1);
|
|
1668
|
+
}
|
|
1669
|
+
for (const error of snapshotSkippedErrors) {
|
|
1670
|
+
reportSkippedSnapshotCopy(error, err);
|
|
1671
|
+
}
|
|
1672
|
+
protectSkippedSnapshotPaths(ctx, snapshotSkippedEntries);
|
|
1673
|
+
updateSnapshotStatus(backupDir, snapshotSkippedEntries.length > 0 ? "incomplete" : "ready", undefined, {
|
|
1674
|
+
symlinks: snapshotSymlinks,
|
|
1675
|
+
skippedEntries: snapshotSkippedEntries,
|
|
1676
|
+
});
|
|
1677
|
+
}
|
|
1678
|
+
catch (error) {
|
|
1679
|
+
if (error instanceof SnapshotCopyError) {
|
|
1680
|
+
updateSnapshotStatus(backupDir, "failed-pre-mutation", undefined, {
|
|
1681
|
+
symlinks: snapshotSymlinks,
|
|
1682
|
+
copyFailure: snapshotCopyFailure(error),
|
|
1683
|
+
skippedEntries: snapshotSkippedEntries,
|
|
1684
|
+
});
|
|
1685
|
+
reportSnapshotCopyFailure(error, err);
|
|
1686
|
+
throw new ExitError(1);
|
|
1687
|
+
}
|
|
1688
|
+
updateSnapshotStatus(backupDir, "failed-pre-mutation", undefined, {
|
|
1689
|
+
symlinks: snapshotSymlinks,
|
|
1690
|
+
skippedEntries: snapshotSkippedEntries,
|
|
1691
|
+
});
|
|
1692
|
+
throw error;
|
|
1693
|
+
}
|
|
1694
|
+
pruneManagedBackups(cfg, hqRoot, out, true);
|
|
1695
|
+
if (snapshotSkippedEntries.length > 0) {
|
|
1696
|
+
out(" snapshot incomplete: skipped files were left untouched and this snapshot cannot restore every captured root.\n");
|
|
1697
|
+
}
|
|
1698
|
+
else {
|
|
1699
|
+
out(" snapshot ready (complete recovery is available through --recover-snapshots).\n");
|
|
1008
1700
|
}
|
|
1009
|
-
updateSnapshotStatus(backupDir, "ready");
|
|
1010
1701
|
}
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
throw error;
|
|
1702
|
+
else if (cfg.doBackup && !cfg.dryRun && wipeToplevel.length !== 0) {
|
|
1703
|
+
out("\n==> Safety snapshot skipped (classification found no destructive paths).\n");
|
|
1014
1704
|
}
|
|
1015
|
-
pruneManagedBackups(cfg, hqRoot, out, true);
|
|
1016
|
-
out(` snapshot ready (restore any file: cp "${backupDir}/<relpath>" "${hqRoot}/<relpath>")\n`);
|
|
1017
|
-
}
|
|
1018
|
-
else if (cfg.doBackup && !cfg.dryRun && wipeToplevel.length !== 0) {
|
|
1019
|
-
out("\n==> Safety snapshot skipped (classification found no destructive paths).\n");
|
|
1020
|
-
}
|
|
1021
|
-
try {
|
|
1022
|
-
mutationStarted = true;
|
|
1023
|
-
applyClassifiedActions(ctx, backupDir);
|
|
1024
1705
|
// --- Back up preserve-subpaths to a mktemp shuttle ---
|
|
1706
|
+
// This runs after the snapshot but before classified mutations, so a normal
|
|
1707
|
+
// preserve-subpath remains available even if an action removes its parent.
|
|
1025
1708
|
const shuttle = path.join(tmpdir, "preserve");
|
|
1026
|
-
fs.mkdirSync(shuttle, { recursive: true });
|
|
1027
1709
|
const preserveMap = [];
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1710
|
+
if (cfg.preserveSubpaths.length > 0) {
|
|
1711
|
+
fs.mkdirSync(shuttle, { recursive: true });
|
|
1712
|
+
let shuttleId = 0;
|
|
1713
|
+
for (const sp of cfg.preserveSubpaths) {
|
|
1714
|
+
if (snapshotSkippedEntries.some((entry) => rescuePathsOverlap(sp, entry.relativePath))) {
|
|
1715
|
+
out(`==> Left ${sp} in place because it contains a skipped snapshot path\n`);
|
|
1716
|
+
continue;
|
|
1717
|
+
}
|
|
1718
|
+
const src = path.join(hqRoot, sp);
|
|
1719
|
+
if (lexists(src)) {
|
|
1720
|
+
shuttleId += 1;
|
|
1721
|
+
cpA(src, shuttle, String(shuttleId));
|
|
1722
|
+
preserveMap.push({ id: shuttleId, rel: sp });
|
|
1723
|
+
out(`==> Backed up ${sp} -> shuttle/${shuttleId}\n`);
|
|
1724
|
+
}
|
|
1036
1725
|
}
|
|
1037
1726
|
}
|
|
1727
|
+
mutationStarted = true;
|
|
1728
|
+
applyClassifiedActions(ctx, backupDir);
|
|
1038
1729
|
out("\n");
|
|
1039
1730
|
out(`==> Walk complete (user-only: ${counts.userOnly}, unchanged: ${counts.unchanged}, user-edit-rescued: ${counts.userEdit}, conflict-quarantined: ${counts.userConflict}, overwrite-safe: ${counts.userOverwrite}, kept-local: ${counts.userKeptLocal}, cloud-symlink-reconciled: ${counts.cloudSymlinkReconciled}, symlinks-dropped: ${counts.symlinkDropped})\n`);
|
|
1040
1731
|
// --- Overlay source onto HQ root ---
|
|
@@ -1047,13 +1738,14 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
1047
1738
|
const overlayRunArgs = fs.statSync(unchangedList).size > 0
|
|
1048
1739
|
? [
|
|
1049
1740
|
"-a",
|
|
1741
|
+
"--from0",
|
|
1050
1742
|
`--exclude-from=${unchangedList}`,
|
|
1051
1743
|
...rsyncExcludes,
|
|
1052
1744
|
toCygwinRsyncPath(srcDir + "/"),
|
|
1053
1745
|
toCygwinRsyncPath(hqRoot + "/"),
|
|
1054
1746
|
]
|
|
1055
1747
|
: overlayArgs;
|
|
1056
|
-
const overlay = run(
|
|
1748
|
+
const overlay = run(rsyncLauncher.command, [...rsyncLauncher.leadingArgs, ...overlayRunArgs], { env });
|
|
1057
1749
|
if (overlay.status !== 0) {
|
|
1058
1750
|
// bash runs under set -e: a failed rsync aborts with its status.
|
|
1059
1751
|
err(overlay.stderr);
|
|
@@ -1083,7 +1775,7 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
1083
1775
|
// machine env.PATH onto the release baseline and relocates genuine user drift
|
|
1084
1776
|
// (vs the last-sync floor) into settings.local.json so the effective merged
|
|
1085
1777
|
// settings are preserved. No-op in narrow mode that excludes .claude.
|
|
1086
|
-
if (claudeSettingsInScope) {
|
|
1778
|
+
if (claudeSettingsInScope && !isSkippedSnapshotPath(ctx, claudeSettingsRel)) {
|
|
1087
1779
|
reconcileClaudeSettings({
|
|
1088
1780
|
hqRoot,
|
|
1089
1781
|
srcDir,
|
|
@@ -1093,12 +1785,15 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
1093
1785
|
preserveSubpaths: cfg.preserveSubpaths,
|
|
1094
1786
|
}, preRescueSettingsJson, out, err);
|
|
1095
1787
|
}
|
|
1788
|
+
else if (claudeSettingsInScope) {
|
|
1789
|
+
out("==> Skipped .claude/settings.json reconcile because its safety snapshot entry was skipped\n");
|
|
1790
|
+
}
|
|
1096
1791
|
// --- Restore executable bit on shipped shebang scripts (defense-in-depth) ---
|
|
1097
1792
|
// rsync -a already propagated each file's upstream git tree mode; this guards
|
|
1098
1793
|
// the case where upstream itself committed an executable script as 0644 -- a
|
|
1099
1794
|
// hook the runtime exec's then dies with EACCES and is silently disabled. A
|
|
1100
1795
|
// shebang is an unambiguous "meant to be run" marker. See restoreShebangExecBits.
|
|
1101
|
-
const execBitFixes = restoreShebangExecBits(srcDir, hqRoot, env);
|
|
1796
|
+
const execBitFixes = restoreShebangExecBits(srcDir, hqRoot, env, ctx.skippedSnapshotRels);
|
|
1102
1797
|
if (execBitFixes > 0) {
|
|
1103
1798
|
out(`==> Restored executable bit on ${execBitFixes} shipped shebang script(s) lacking +x\n`);
|
|
1104
1799
|
}
|
|
@@ -1109,13 +1804,19 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
1109
1804
|
// unparseable commit timestamp skips the stamp entirely — a nondeterministic
|
|
1110
1805
|
// stamp is worse than no stamp (it re-creates cross-machine conflict mirrors).
|
|
1111
1806
|
const srcCommitDate = new Date(srcCommitIsoRaw);
|
|
1807
|
+
const coreYamlSkipped = isSkippedSnapshotPath(ctx, "core/core.yaml");
|
|
1808
|
+
if (coreYamlSkipped) {
|
|
1809
|
+
out("==> Skipped core/core.yaml stamp because its safety snapshot entry was skipped\n");
|
|
1810
|
+
}
|
|
1112
1811
|
if (cfg.narrowPaths.length === 0 &&
|
|
1113
1812
|
isFileFollow(coreYaml) &&
|
|
1813
|
+
!coreYamlSkipped &&
|
|
1114
1814
|
Number.isNaN(srcCommitDate.getTime())) {
|
|
1115
1815
|
err(` WARN: could not parse source commit timestamp (${JSON.stringify(srcCommitIsoRaw)}); skipping core/core.yaml stamp to keep it deterministic\n`);
|
|
1116
1816
|
}
|
|
1117
1817
|
if (cfg.narrowPaths.length === 0 &&
|
|
1118
1818
|
isFileFollow(coreYaml) &&
|
|
1819
|
+
!coreYamlSkipped &&
|
|
1119
1820
|
!Number.isNaN(srcCommitDate.getTime())) {
|
|
1120
1821
|
const nowUtc = utcStamp(srcCommitDate, "colon");
|
|
1121
1822
|
if (yqAvailable) {
|
|
@@ -1194,7 +1895,7 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
1194
1895
|
}
|
|
1195
1896
|
out("\n");
|
|
1196
1897
|
out("==> Classification:\n");
|
|
1197
|
-
out(` user-only (left in place):
|
|
1898
|
+
out(` user-only (left in place under release-managed roots): ${counts.userOnly} files\n`);
|
|
1198
1899
|
out(` unchanged (preserved/re-laid): ${counts.unchanged} files\n`);
|
|
1199
1900
|
out(` user-edits (rescued): ${counts.userEdit} files\n`);
|
|
1200
1901
|
out(` user-edits (conflict quarantine): ${counts.userConflict} files\n`);
|
|
@@ -1229,7 +1930,7 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
1229
1930
|
lines.push("Baseline: head_compare (no last-sync stamp; first run or floor unreachable)");
|
|
1230
1931
|
}
|
|
1231
1932
|
lines.push("", "## What the rescue did");
|
|
1232
|
-
lines.push(` user-only (left in place):
|
|
1933
|
+
lines.push(` user-only (left in place under release-managed roots): ${counts.userOnly}`);
|
|
1233
1934
|
lines.push(` unchanged (preserved/re-laid): ${counts.unchanged}`);
|
|
1234
1935
|
lines.push(` user-edit (rescued into personal/): ${counts.userEdit}`);
|
|
1235
1936
|
lines.push(` user-edit (conflict quarantine): ${counts.userConflict}`);
|
|
@@ -1258,10 +1959,20 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
1258
1959
|
}
|
|
1259
1960
|
lines.push("", "## Restore");
|
|
1260
1961
|
lines.push("This directory holds the release-managed wipe set as it was before the update; reproducible worktrees and build caches are intentionally excluded.");
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1962
|
+
const snapshotName = path.basename(backupDir);
|
|
1963
|
+
if (snapshotSkippedEntries.length > 0) {
|
|
1964
|
+
lines.push("This snapshot is incomplete and cannot restore every captured root.");
|
|
1965
|
+
lines.push("The following paths were not backed up and were left untouched by the update:");
|
|
1966
|
+
for (const entry of snapshotSkippedEntries) {
|
|
1967
|
+
lines.push(` ${entry.relativePath} (${entry.operation}, ${entry.code})`);
|
|
1968
|
+
}
|
|
1969
|
+
}
|
|
1970
|
+
else {
|
|
1971
|
+
lines.push("Restore every captured root and recorded symlink:");
|
|
1972
|
+
lines.push(` ${formatSnapshotRestoreCommand(hqRoot, cfg.backupRoot, snapshotName)}`);
|
|
1973
|
+
lines.push("Then verify the restored tree before retrying:");
|
|
1974
|
+
lines.push(` ${formatRecoveryCommand(hqRoot, cfg.backupRoot, snapshotName)}`);
|
|
1975
|
+
}
|
|
1265
1976
|
lines.push("", `Retention: ${cfg.backupRetentionDays} days and ${cfg.backupRetentionCount} completed snapshots (latest recovery floor retained).`);
|
|
1266
1977
|
try {
|
|
1267
1978
|
fs.writeFileSync(path.join(backupDir, "RECOVERY.md"), lines.join("\n") + "\n");
|
|
@@ -1269,9 +1980,9 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
1269
1980
|
catch {
|
|
1270
1981
|
/* best-effort, matches `|| true` */
|
|
1271
1982
|
}
|
|
1272
|
-
updateSnapshotStatus(backupDir, "completed-good", RUN_TS);
|
|
1983
|
+
updateSnapshotStatus(backupDir, snapshotSkippedEntries.length > 0 ? "incomplete" : "completed-good", RUN_TS);
|
|
1273
1984
|
out("\n");
|
|
1274
|
-
out(`==> Pre-update snapshot + recovery manifest: ${backupDir}\n`);
|
|
1985
|
+
out(`==> Pre-update ${snapshotSkippedEntries.length > 0 ? "incomplete " : ""}snapshot + recovery manifest: ${backupDir}\n`);
|
|
1275
1986
|
pruneManagedBackups(cfg, hqRoot, out, true);
|
|
1276
1987
|
}
|
|
1277
1988
|
out("\n");
|
|
@@ -1279,7 +1990,18 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
1279
1990
|
out(" User-edited files were rescued under personal/ (see scan output above).\n");
|
|
1280
1991
|
out(" User-only files (created by you, unknown to upstream) were left untouched.\n");
|
|
1281
1992
|
if (cfg.doBackup && backupDir) {
|
|
1282
|
-
|
|
1993
|
+
if (snapshotSkippedEntries.length > 0) {
|
|
1994
|
+
out(` An incomplete pre-update snapshot is at ${backupDir}; skipped files were left untouched (RECOVERY.md explains its limits).\n`);
|
|
1995
|
+
}
|
|
1996
|
+
else {
|
|
1997
|
+
out(` A filtered pre-update snapshot is at ${backupDir} (RECOVERY.md explains restore).\n`);
|
|
1998
|
+
}
|
|
1999
|
+
}
|
|
2000
|
+
if (isSkippedSnapshotPath(ctx, RESCUE_BLOCKED_MARKER_REL)) {
|
|
2001
|
+
out("==> Left the rescue recovery marker in place because its safety snapshot entry was skipped\n");
|
|
2002
|
+
}
|
|
2003
|
+
else {
|
|
2004
|
+
clearRescueBlockedMarker(hqRoot);
|
|
1283
2005
|
}
|
|
1284
2006
|
return { status: 0 };
|
|
1285
2007
|
}
|
|
@@ -1328,12 +2050,15 @@ function applyClassifiedActions(ctx, backupDir) {
|
|
|
1328
2050
|
completed.push(action);
|
|
1329
2051
|
}
|
|
1330
2052
|
catch (err) {
|
|
1331
|
-
const
|
|
1332
|
-
const manifest = writeApplyFailureManifest(ctx, backupDir, action, completed, restored, err);
|
|
2053
|
+
const rollback = rollbackCompletedActions(ctx, completed, backupDir);
|
|
2054
|
+
const manifest = writeApplyFailureManifest(ctx, backupDir, action, completed, rollback.restored, rollback.failures, err);
|
|
1333
2055
|
ctx.out("\n");
|
|
1334
2056
|
ctx.out(`error: rescue apply failed during '${action.label}'.\n`);
|
|
1335
|
-
if (restored.length > 0) {
|
|
1336
|
-
ctx.out(`==> Rolled back ${restored.length} prior action path(s) from the safety snapshot.\n`);
|
|
2057
|
+
if (rollback.restored.length > 0) {
|
|
2058
|
+
ctx.out(`==> Rolled back ${rollback.restored.length} prior action path(s) from the safety snapshot.\n`);
|
|
2059
|
+
}
|
|
2060
|
+
for (const failure of rollback.failures) {
|
|
2061
|
+
ctx.out(`warning: rollback retained recovery evidence: ${failure}\n`);
|
|
1337
2062
|
}
|
|
1338
2063
|
if (manifest) {
|
|
1339
2064
|
ctx.out(`==> Recovery manifest: ${manifest}\n`);
|
|
@@ -1359,7 +2084,14 @@ function injectApplyPathChangeForTest(ctx) {
|
|
|
1359
2084
|
}
|
|
1360
2085
|
function rollbackCompletedActions(ctx, completed, backupDir) {
|
|
1361
2086
|
if (!backupDir || !isDir(backupDir))
|
|
1362
|
-
return [];
|
|
2087
|
+
return { restored: [], failures: [] };
|
|
2088
|
+
const snapshotManifest = readSnapshotManifest(backupDir, ctx.hqRoot);
|
|
2089
|
+
if (!snapshotManifest) {
|
|
2090
|
+
return {
|
|
2091
|
+
restored: [],
|
|
2092
|
+
failures: [`${backupDir}: snapshot manifest is missing or invalid`],
|
|
2093
|
+
};
|
|
2094
|
+
}
|
|
1363
2095
|
const rels = [];
|
|
1364
2096
|
const seen = new Set();
|
|
1365
2097
|
for (let i = completed.length - 1; i >= 0; i--) {
|
|
@@ -1372,25 +2104,34 @@ function rollbackCompletedActions(ctx, completed, backupDir) {
|
|
|
1372
2104
|
}
|
|
1373
2105
|
}
|
|
1374
2106
|
}
|
|
1375
|
-
const
|
|
2107
|
+
const restoreRels = [];
|
|
1376
2108
|
for (const rel of rels) {
|
|
1377
2109
|
const snapshotPath = path.join(backupDir, rel);
|
|
1378
|
-
|
|
2110
|
+
const hasRecordedLink = (snapshotManifest.symlinks ?? []).some((link) => link.path === rel || link.path.startsWith(`${rel}/`));
|
|
2111
|
+
if (!lexists(snapshotPath) && !hasRecordedLink)
|
|
1379
2112
|
continue;
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
}
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
2113
|
+
restoreRels.push(rel);
|
|
2114
|
+
}
|
|
2115
|
+
if (restoreRels.length === 0)
|
|
2116
|
+
return { restored: [], failures: [] };
|
|
2117
|
+
try {
|
|
2118
|
+
restoreSnapshotPaths(backupDir, ctx.hqRoot, restoreRels, snapshotManifest);
|
|
2119
|
+
return { restored: restoreRels, failures: [] };
|
|
2120
|
+
}
|
|
2121
|
+
catch (error) {
|
|
2122
|
+
return {
|
|
2123
|
+
restored: [],
|
|
2124
|
+
failures: [
|
|
2125
|
+
`${restoreRels.join(", ")}: ${error instanceof Error ? error.message : String(error)}`,
|
|
2126
|
+
],
|
|
2127
|
+
};
|
|
1390
2128
|
}
|
|
1391
|
-
return restored;
|
|
1392
2129
|
}
|
|
1393
|
-
function writeApplyFailureManifest(ctx, backupDir, failedAction, completed, restored, err) {
|
|
2130
|
+
function writeApplyFailureManifest(ctx, backupDir, failedAction, completed, restored, rollbackFailures, err) {
|
|
2131
|
+
if (isSkippedSnapshotPath(ctx, ".hq/rescue-recovery")) {
|
|
2132
|
+
ctx.out("warning: did not write the rescue recovery manifest because its safety snapshot entry was skipped\n");
|
|
2133
|
+
return null;
|
|
2134
|
+
}
|
|
1394
2135
|
const message = err instanceof Error ? err.message : String(err);
|
|
1395
2136
|
const completedLines = completed.length === 0
|
|
1396
2137
|
? [" (none)"]
|
|
@@ -1440,11 +2181,18 @@ function writeApplyFailureManifest(ctx, backupDir, failedAction, completed, rest
|
|
|
1440
2181
|
for (const rel of unrestored)
|
|
1441
2182
|
lines.push(` - ${rel}`);
|
|
1442
2183
|
}
|
|
2184
|
+
if (rollbackFailures.length > 0) {
|
|
2185
|
+
lines.push("", "Rollback failures:");
|
|
2186
|
+
for (const failure of rollbackFailures)
|
|
2187
|
+
lines.push(` - ${failure}`);
|
|
2188
|
+
}
|
|
1443
2189
|
if (backupDir && isDir(backupDir)) {
|
|
1444
2190
|
lines.push("");
|
|
1445
|
-
|
|
1446
|
-
lines.push(
|
|
1447
|
-
lines.push(`
|
|
2191
|
+
const snapshotName = path.basename(backupDir);
|
|
2192
|
+
lines.push("Restore every captured root and recorded symlink:");
|
|
2193
|
+
lines.push(` ${formatSnapshotRestoreCommand(ctx.hqRoot, ctx.cfg.backupRoot, snapshotName)}`);
|
|
2194
|
+
lines.push("Then verify before retrying:");
|
|
2195
|
+
lines.push(` ${formatRecoveryCommand(ctx.hqRoot, ctx.cfg.backupRoot, snapshotName)}`);
|
|
1448
2196
|
}
|
|
1449
2197
|
else {
|
|
1450
2198
|
lines.push("", "No pre-op safety snapshot was available for automatic restore.");
|
|
@@ -2078,7 +2826,7 @@ export function reconcileClaudeSettings(ctx, preRescueSettingsJson, out, err) {
|
|
|
2078
2826
|
* EPERM) is swallowed, matching the rest of rescue's posture. Returns the count
|
|
2079
2827
|
* of files whose mode was changed, for the run summary.
|
|
2080
2828
|
*/
|
|
2081
|
-
function restoreShebangExecBits(srcDir, hqRoot, env) {
|
|
2829
|
+
function restoreShebangExecBits(srcDir, hqRoot, env, skippedSnapshotRels) {
|
|
2082
2830
|
let listing;
|
|
2083
2831
|
try {
|
|
2084
2832
|
const r = spawnSync("git", ["-C", srcDir, "ls-files", "-z"], {
|
|
@@ -2097,6 +2845,9 @@ function restoreShebangExecBits(srcDir, hqRoot, env) {
|
|
|
2097
2845
|
for (const rel of listing.split("\0")) {
|
|
2098
2846
|
if (!rel)
|
|
2099
2847
|
continue;
|
|
2848
|
+
if (skippedSnapshotRels.some((skippedRel) => rescuePathsOverlap(rel, skippedRel))) {
|
|
2849
|
+
continue;
|
|
2850
|
+
}
|
|
2100
2851
|
const dest = path.join(hqRoot, rel);
|
|
2101
2852
|
let st;
|
|
2102
2853
|
try {
|
|
@@ -2372,10 +3123,11 @@ function renderDryRunDecision(ctx, decision) {
|
|
|
2372
3123
|
}
|
|
2373
3124
|
function buildRescueActions(ctx) {
|
|
2374
3125
|
const actions = [];
|
|
2375
|
-
const add = (label, affectedRels, runAction) => {
|
|
3126
|
+
const add = (label, affectedRels, runAction, destParentRels = []) => {
|
|
2376
3127
|
actions.push({
|
|
2377
3128
|
label,
|
|
2378
3129
|
affectedRels,
|
|
3130
|
+
destParentRels,
|
|
2379
3131
|
expectedPaths: affectedRels.map((rel) => ({
|
|
2380
3132
|
rel,
|
|
2381
3133
|
kind: rescuePathKind(path.join(ctx.hqRoot, rel)),
|
|
@@ -2414,7 +3166,7 @@ function buildRescueActions(ctx) {
|
|
|
2414
3166
|
});
|
|
2415
3167
|
break;
|
|
2416
3168
|
case "user-edit-diff-append":
|
|
2417
|
-
add(`rescue diff-append: ${decision.rel}`, [decision.rel], () => rescueOne(ctx, decision.rel));
|
|
3169
|
+
add(`rescue diff-append: ${decision.rel}`, [decision.rel], () => rescueOne(ctx, decision.rel), [path.posix.dirname(mapRescueTarget(decision.rel))]);
|
|
2418
3170
|
break;
|
|
2419
3171
|
case "user-edit-overwrite-safe":
|
|
2420
3172
|
add(`overwrite-safe: ${decision.rel}`, [decision.rel], () => {
|
|
@@ -2428,21 +3180,23 @@ function buildRescueActions(ctx) {
|
|
|
2428
3180
|
// `unchangedList` makes it an `--exclude-from` entry for the overlay
|
|
2429
3181
|
// rsync, so the release copy is never laid over the local one.
|
|
2430
3182
|
add(`keep local: ${decision.rel}`, [], () => {
|
|
2431
|
-
|
|
3183
|
+
appendLiteralRsyncExclude(ctx.unchangedList, decision.rel);
|
|
2432
3184
|
ctx.counts.userKeptLocal += 1;
|
|
2433
3185
|
ctx.out(` kept local: ${decision.rel} (machine-local ignore file; your edits win, overlay skipped)\n`);
|
|
2434
3186
|
ctx.appendLog(`kept-local\t${decision.rel}\t(machine-local ignore file; user edit wins, overlay excluded)\n`);
|
|
2435
3187
|
});
|
|
2436
3188
|
break;
|
|
2437
3189
|
case "user-edit-conflict":
|
|
2438
|
-
add(`conflict quarantine: ${decision.rel}`, [decision.rel], () => conflictOne(ctx, decision.rel)
|
|
3190
|
+
add(`conflict quarantine: ${decision.rel}`, [decision.rel], () => conflictOne(ctx, decision.rel), [
|
|
3191
|
+
path.posix.dirname(`.hq-conflicts/rescue-${ctx.runTs}/${decision.rel}`),
|
|
3192
|
+
]);
|
|
2439
3193
|
break;
|
|
2440
3194
|
case "user-edit-rescue":
|
|
2441
|
-
add(`rescue: ${decision.rel}`, [decision.rel], () => rescueOne(ctx, decision.rel));
|
|
3195
|
+
add(`rescue: ${decision.rel}`, [decision.rel], () => rescueOne(ctx, decision.rel), [path.posix.dirname(mapRescueTarget(decision.rel))]);
|
|
2442
3196
|
break;
|
|
2443
3197
|
case "unchanged-preserve":
|
|
2444
3198
|
add(`preserve unchanged: ${decision.rel}`, [], () => {
|
|
2445
|
-
|
|
3199
|
+
appendLiteralRsyncExclude(ctx.unchangedList, decision.rel);
|
|
2446
3200
|
ctx.appendLog(`unchanged\t${decision.rel}\t(identical to upstream; left in place, mtime preserved)\n`);
|
|
2447
3201
|
});
|
|
2448
3202
|
break;
|
|
@@ -2711,6 +3465,262 @@ function restoreMtimesFromGit(srcDir, env, out) {
|
|
|
2711
3465
|
}
|
|
2712
3466
|
}
|
|
2713
3467
|
}
|
|
3468
|
+
const RESCUE_BLOCKED_MARKER_REL = ".hq/rescue-blocked.json";
|
|
3469
|
+
const RESCUE_BLOCKED_REASON = "recovery-snapshot-breaker";
|
|
3470
|
+
const POST_RESTORE_PERMIT_REL = ".hq/rescue-post-restore.json";
|
|
3471
|
+
function posixShellQuote(value) {
|
|
3472
|
+
return `'${value.replace(/'/g, `'\\''`)}'`;
|
|
3473
|
+
}
|
|
3474
|
+
/** Format the exact safe command presented in a breaker diagnostic. */
|
|
3475
|
+
export function formatRecoveryCommand(hqRoot, backupRoot, verifyRestoredSnapshot) {
|
|
3476
|
+
const args = [
|
|
3477
|
+
"npx",
|
|
3478
|
+
"-y",
|
|
3479
|
+
"--package=@indigoai-us/hq-cloud@latest",
|
|
3480
|
+
"hq-rescue",
|
|
3481
|
+
"--recover-snapshots",
|
|
3482
|
+
"--hq-root",
|
|
3483
|
+
posixShellQuote(hqRoot),
|
|
3484
|
+
"--backup-dir",
|
|
3485
|
+
posixShellQuote(backupRoot),
|
|
3486
|
+
];
|
|
3487
|
+
if (verifyRestoredSnapshot) {
|
|
3488
|
+
args.push("--verify-restored-snapshot", posixShellQuote(verifyRestoredSnapshot));
|
|
3489
|
+
}
|
|
3490
|
+
return args.join(" ");
|
|
3491
|
+
}
|
|
3492
|
+
/** Format the manifest-aware command that restores links as well as files. */
|
|
3493
|
+
export function formatSnapshotRestoreCommand(hqRoot, backupRoot, snapshotName) {
|
|
3494
|
+
return [
|
|
3495
|
+
"npx",
|
|
3496
|
+
"-y",
|
|
3497
|
+
"--package=@indigoai-us/hq-cloud@latest",
|
|
3498
|
+
"hq-rescue",
|
|
3499
|
+
"--recover-snapshots",
|
|
3500
|
+
"--hq-root",
|
|
3501
|
+
posixShellQuote(hqRoot),
|
|
3502
|
+
"--backup-dir",
|
|
3503
|
+
posixShellQuote(backupRoot),
|
|
3504
|
+
"--restore-snapshot",
|
|
3505
|
+
posixShellQuote(snapshotName),
|
|
3506
|
+
].join(" ");
|
|
3507
|
+
}
|
|
3508
|
+
function posixRelFromRoot(hqRoot, abs) {
|
|
3509
|
+
const root = path.resolve(hqRoot);
|
|
3510
|
+
const resolved = path.resolve(abs);
|
|
3511
|
+
if (resolved === root)
|
|
3512
|
+
return ".";
|
|
3513
|
+
const prefix = root.endsWith(path.sep) ? root : root + path.sep;
|
|
3514
|
+
if (resolved.startsWith(prefix)) {
|
|
3515
|
+
return resolved.slice(prefix.length).split(path.sep).join("/");
|
|
3516
|
+
}
|
|
3517
|
+
return resolved;
|
|
3518
|
+
}
|
|
3519
|
+
function firstExistingAncestor(abs) {
|
|
3520
|
+
let current = abs;
|
|
3521
|
+
for (;;) {
|
|
3522
|
+
if (lexists(current))
|
|
3523
|
+
return current;
|
|
3524
|
+
const parent = path.dirname(current);
|
|
3525
|
+
if (parent === current)
|
|
3526
|
+
return null;
|
|
3527
|
+
current = parent;
|
|
3528
|
+
}
|
|
3529
|
+
}
|
|
3530
|
+
function canWrite(abs) {
|
|
3531
|
+
try {
|
|
3532
|
+
fs.accessSync(abs, fs.constants.W_OK);
|
|
3533
|
+
return true;
|
|
3534
|
+
}
|
|
3535
|
+
catch {
|
|
3536
|
+
return false;
|
|
3537
|
+
}
|
|
3538
|
+
}
|
|
3539
|
+
function topLevelRel(rel) {
|
|
3540
|
+
const normalized = rel.replace(/\\/g, "/").replace(/^\.\//, "");
|
|
3541
|
+
if (normalized === "." || normalized === "")
|
|
3542
|
+
return ".";
|
|
3543
|
+
return normalized.split("/")[0] ?? ".";
|
|
3544
|
+
}
|
|
3545
|
+
/**
|
|
3546
|
+
* Paths the current process cannot write among the action plan's rename/delete
|
|
3547
|
+
* sources (and their parents) plus rename destination parents. Empty means the
|
|
3548
|
+
* plan is writable; a non-empty result must refuse before allocating a snapshot.
|
|
3549
|
+
*/
|
|
3550
|
+
export function collectUnwritableRescueMutationPaths(hqRoot, actions) {
|
|
3551
|
+
const failed = new Set();
|
|
3552
|
+
const note = (abs) => {
|
|
3553
|
+
failed.add(posixRelFromRoot(hqRoot, abs));
|
|
3554
|
+
};
|
|
3555
|
+
const checkParent = (abs) => {
|
|
3556
|
+
const existing = firstExistingAncestor(abs);
|
|
3557
|
+
if (!existing || !canWrite(existing))
|
|
3558
|
+
note(existing ?? abs);
|
|
3559
|
+
};
|
|
3560
|
+
for (const action of actions) {
|
|
3561
|
+
for (const rel of action.affectedRels) {
|
|
3562
|
+
if (!rel)
|
|
3563
|
+
continue;
|
|
3564
|
+
const abs = path.join(hqRoot, rel);
|
|
3565
|
+
const st = lstatOrNull(abs);
|
|
3566
|
+
if (st && !st.isSymbolicLink() && !canWrite(abs))
|
|
3567
|
+
note(abs);
|
|
3568
|
+
checkParent(path.dirname(abs));
|
|
3569
|
+
}
|
|
3570
|
+
for (const rel of action.destParentRels ?? []) {
|
|
3571
|
+
if (!rel || rel === ".")
|
|
3572
|
+
continue;
|
|
3573
|
+
checkParent(path.join(hqRoot, rel));
|
|
3574
|
+
}
|
|
3575
|
+
}
|
|
3576
|
+
return [...failed].sort();
|
|
3577
|
+
}
|
|
3578
|
+
function reportUnwritableRescuePaths(hqRoot, unwritable, err) {
|
|
3579
|
+
const shown = unwritable.slice(0, 10);
|
|
3580
|
+
err("error: cannot write the paths rescue would rename or delete (permission denied).\n");
|
|
3581
|
+
err(" Refusing before a safety snapshot is allocated so this cannot trip the recovery-snapshot circuit breaker.\n");
|
|
3582
|
+
err(unwritable.length > 10
|
|
3583
|
+
? ` Unwritable paths (showing 10 of ${unwritable.length}):\n`
|
|
3584
|
+
: " Unwritable paths:\n");
|
|
3585
|
+
for (const rel of shown)
|
|
3586
|
+
err(` ${rel}\n`);
|
|
3587
|
+
const uid = typeof process.getuid === "function" ? String(process.getuid()) : "<uid>";
|
|
3588
|
+
const gid = typeof process.getgid === "function" ? String(process.getgid()) : "<gid>";
|
|
3589
|
+
const tops = [
|
|
3590
|
+
...new Set(unwritable.map((rel) => topLevelRel(rel)).filter((top) => top !== ".")),
|
|
3591
|
+
].sort();
|
|
3592
|
+
err(" Repair ownership, then retry:\n");
|
|
3593
|
+
if (tops.length === 0) {
|
|
3594
|
+
err(` chown -R ${uid}:${gid} ${hqRoot}\n`);
|
|
3595
|
+
}
|
|
3596
|
+
else {
|
|
3597
|
+
for (const top of tops) {
|
|
3598
|
+
err(` chown -R ${uid}:${gid} ${path.join(hqRoot, top)}\n`);
|
|
3599
|
+
}
|
|
3600
|
+
}
|
|
3601
|
+
}
|
|
3602
|
+
function rescueBlockedMarkerPath(hqRoot) {
|
|
3603
|
+
return path.join(hqRoot, RESCUE_BLOCKED_MARKER_REL);
|
|
3604
|
+
}
|
|
3605
|
+
function postRestorePermitPath(hqRoot) {
|
|
3606
|
+
return path.join(hqRoot, POST_RESTORE_PERMIT_REL);
|
|
3607
|
+
}
|
|
3608
|
+
function writePostRestorePermit(hqRoot, entry) {
|
|
3609
|
+
const permit = {
|
|
3610
|
+
schemaVersion: 1,
|
|
3611
|
+
canonicalHqRoot: hqRoot,
|
|
3612
|
+
rootHash: canonicalRootHash(hqRoot),
|
|
3613
|
+
snapshotName: entry.name,
|
|
3614
|
+
snapshotRunId: entry.manifest.runId,
|
|
3615
|
+
verifiedAt: new Date().toISOString(),
|
|
3616
|
+
};
|
|
3617
|
+
const target = postRestorePermitPath(hqRoot);
|
|
3618
|
+
const temporary = `${target}.${process.pid}.tmp`;
|
|
3619
|
+
fs.mkdirSync(path.dirname(target), { recursive: true });
|
|
3620
|
+
fs.writeFileSync(temporary, JSON.stringify(permit, null, 2) + "\n", {
|
|
3621
|
+
mode: 0o600,
|
|
3622
|
+
});
|
|
3623
|
+
fs.renameSync(temporary, target);
|
|
3624
|
+
}
|
|
3625
|
+
function readPostRestorePermit(hqRoot) {
|
|
3626
|
+
let raw;
|
|
3627
|
+
try {
|
|
3628
|
+
raw = fs.readFileSync(postRestorePermitPath(hqRoot), "utf8");
|
|
3629
|
+
}
|
|
3630
|
+
catch (error) {
|
|
3631
|
+
if (error.code === "ENOENT") {
|
|
3632
|
+
return { permit: null };
|
|
3633
|
+
}
|
|
3634
|
+
return {
|
|
3635
|
+
permit: null,
|
|
3636
|
+
problem: `could not read ${POST_RESTORE_PERMIT_REL}: ${error instanceof Error ? error.message : String(error)}`,
|
|
3637
|
+
};
|
|
3638
|
+
}
|
|
3639
|
+
try {
|
|
3640
|
+
const value = JSON.parse(raw);
|
|
3641
|
+
if (value.schemaVersion !== 1 ||
|
|
3642
|
+
typeof value.canonicalHqRoot !== "string" ||
|
|
3643
|
+
typeof value.rootHash !== "string" ||
|
|
3644
|
+
typeof value.snapshotName !== "string" ||
|
|
3645
|
+
typeof value.snapshotRunId !== "string" ||
|
|
3646
|
+
typeof value.verifiedAt !== "string") {
|
|
3647
|
+
return { permit: null, problem: `${POST_RESTORE_PERMIT_REL} is malformed` };
|
|
3648
|
+
}
|
|
3649
|
+
return { permit: value };
|
|
3650
|
+
}
|
|
3651
|
+
catch (error) {
|
|
3652
|
+
return {
|
|
3653
|
+
permit: null,
|
|
3654
|
+
problem: `${POST_RESTORE_PERMIT_REL} is not valid JSON: ${error instanceof Error ? error.message : String(error)}`,
|
|
3655
|
+
};
|
|
3656
|
+
}
|
|
3657
|
+
}
|
|
3658
|
+
function consumePostRestorePermit(cfg, hqRoot) {
|
|
3659
|
+
const read = readPostRestorePermit(hqRoot);
|
|
3660
|
+
if (!read.permit) {
|
|
3661
|
+
return { used: false, ...(read.problem ? { problem: read.problem } : {}) };
|
|
3662
|
+
}
|
|
3663
|
+
const permit = read.permit;
|
|
3664
|
+
if (permit.canonicalHqRoot !== hqRoot ||
|
|
3665
|
+
permit.rootHash !== canonicalRootHash(hqRoot)) {
|
|
3666
|
+
return { used: false, problem: "permit belongs to a different HQ root" };
|
|
3667
|
+
}
|
|
3668
|
+
const inventory = inventoryManagedSnapshots(cfg.backupRoot, hqRoot, () => false, {
|
|
3669
|
+
keepGood: Number(cfg.backupRetentionCount),
|
|
3670
|
+
maxAgeDays: Number(cfg.backupRetentionDays),
|
|
3671
|
+
});
|
|
3672
|
+
if (inventory.unmanaged.some((item) => item.reason === "backup-root-unreadable")) {
|
|
3673
|
+
return { used: false, problem: `backup root is unreadable: ${cfg.backupRoot}` };
|
|
3674
|
+
}
|
|
3675
|
+
const entry = inventory.entries.find((item) => item.name === permit.snapshotName &&
|
|
3676
|
+
item.manifest.runId === permit.snapshotRunId &&
|
|
3677
|
+
item.manifest.status === "recovery-required");
|
|
3678
|
+
const unresolved = inventory.protected.some((item) => item.entry.name === permit.snapshotName &&
|
|
3679
|
+
item.reason === "recovery-required-no-later-good");
|
|
3680
|
+
if (!entry || !unresolved) {
|
|
3681
|
+
return { used: false, problem: "referenced snapshot is no longer unresolved recovery evidence" };
|
|
3682
|
+
}
|
|
3683
|
+
const mismatch = snapshotTreeMismatch(path.join(cfg.backupRoot, entry.name), hqRoot, entry.manifest.includedRoots, entry.manifest.symlinks);
|
|
3684
|
+
if (mismatch) {
|
|
3685
|
+
return { used: false, problem: `live tree no longer matches ${entry.name}: ${mismatch}` };
|
|
3686
|
+
}
|
|
3687
|
+
try {
|
|
3688
|
+
fs.unlinkSync(postRestorePermitPath(hqRoot));
|
|
3689
|
+
}
|
|
3690
|
+
catch (error) {
|
|
3691
|
+
return {
|
|
3692
|
+
used: false,
|
|
3693
|
+
problem: `could not consume the single-use permit: ${error instanceof Error ? error.message : String(error)}`,
|
|
3694
|
+
};
|
|
3695
|
+
}
|
|
3696
|
+
return { used: true, snapshotName: entry.name };
|
|
3697
|
+
}
|
|
3698
|
+
function writeRescueBlockedMarker(hqRoot, snapshots) {
|
|
3699
|
+
const body = {
|
|
3700
|
+
at: new Date().toISOString(),
|
|
3701
|
+
reason: RESCUE_BLOCKED_REASON,
|
|
3702
|
+
failureKind: "snapshot-recovery-circuit-breaker",
|
|
3703
|
+
snapshots,
|
|
3704
|
+
hint: "Run hq-rescue --recover-snapshots to inspect the snapshots. It deletes only snapshots proved superseded by a later completed-good rescue; otherwise restore the needed files, repair the release source, and retry.",
|
|
3705
|
+
};
|
|
3706
|
+
try {
|
|
3707
|
+
fs.mkdirSync(path.dirname(rescueBlockedMarkerPath(hqRoot)), {
|
|
3708
|
+
recursive: true,
|
|
3709
|
+
});
|
|
3710
|
+
fs.writeFileSync(rescueBlockedMarkerPath(hqRoot), JSON.stringify(body, null, 2) + "\n");
|
|
3711
|
+
}
|
|
3712
|
+
catch {
|
|
3713
|
+
/* stderr remains the operator-facing signal if .hq is unwritable */
|
|
3714
|
+
}
|
|
3715
|
+
}
|
|
3716
|
+
function clearRescueBlockedMarker(hqRoot) {
|
|
3717
|
+
try {
|
|
3718
|
+
fs.unlinkSync(rescueBlockedMarkerPath(hqRoot));
|
|
3719
|
+
}
|
|
3720
|
+
catch {
|
|
3721
|
+
/* absent is the success state */
|
|
3722
|
+
}
|
|
3723
|
+
}
|
|
2714
3724
|
function allocateSnapshotDir(backupRoot, runTs, rootHash, runId) {
|
|
2715
3725
|
return path.join(backupRoot, `pre-update-${runTs}-${rootHash}-${runId}`);
|
|
2716
3726
|
}
|
|
@@ -2729,13 +3739,22 @@ function snapshotManifest(status, hqRoot, runId, cfg, sourceSha, includedRoots)
|
|
|
2729
3739
|
sourceSha,
|
|
2730
3740
|
createdAt: now,
|
|
2731
3741
|
includedRoots,
|
|
3742
|
+
symlinks: [],
|
|
2732
3743
|
fileCount: 0,
|
|
2733
3744
|
byteCount: 0,
|
|
2734
3745
|
};
|
|
2735
3746
|
}
|
|
2736
|
-
function readSnapshotManifest(backupDir) {
|
|
3747
|
+
function readSnapshotManifest(backupDir, expectedHqRoot) {
|
|
2737
3748
|
try {
|
|
2738
|
-
|
|
3749
|
+
const parsed = JSON.parse(fs.readFileSync(path.join(backupDir, "SNAPSHOT.json"), "utf8"));
|
|
3750
|
+
const manifestRoot = expectedHqRoot ??
|
|
3751
|
+
(parsed && typeof parsed === "object"
|
|
3752
|
+
? parsed.canonicalHqRoot
|
|
3753
|
+
: undefined);
|
|
3754
|
+
return typeof manifestRoot === "string" &&
|
|
3755
|
+
isManagedSnapshotManifest(parsed, manifestRoot)
|
|
3756
|
+
? parsed
|
|
3757
|
+
: null;
|
|
2739
3758
|
}
|
|
2740
3759
|
catch {
|
|
2741
3760
|
return null;
|
|
@@ -2748,7 +3767,7 @@ function writeSnapshotManifest(backupDir, manifest) {
|
|
|
2748
3767
|
fs.writeFileSync(temp, JSON.stringify(manifest, null, 2) + "\n");
|
|
2749
3768
|
fs.renameSync(temp, target);
|
|
2750
3769
|
}
|
|
2751
|
-
function updateSnapshotStatus(backupDir, status, completedAt) {
|
|
3770
|
+
function updateSnapshotStatus(backupDir, status, completedAt, details = {}) {
|
|
2752
3771
|
const manifest = readSnapshotManifest(backupDir);
|
|
2753
3772
|
if (!manifest)
|
|
2754
3773
|
return;
|
|
@@ -2756,6 +3775,7 @@ function updateSnapshotStatus(backupDir, status, completedAt) {
|
|
|
2756
3775
|
writeSnapshotManifest(backupDir, {
|
|
2757
3776
|
...manifest,
|
|
2758
3777
|
status,
|
|
3778
|
+
...details,
|
|
2759
3779
|
fileCount: stats.files,
|
|
2760
3780
|
byteCount: stats.bytes,
|
|
2761
3781
|
...(completedAt ? { completedAt: new Date().toISOString() } : {}),
|
|
@@ -2912,22 +3932,285 @@ function cpATo(src, dest) {
|
|
|
2912
3932
|
/* ignore */
|
|
2913
3933
|
}
|
|
2914
3934
|
}
|
|
2915
|
-
|
|
2916
|
-
|
|
2917
|
-
|
|
2918
|
-
|
|
2919
|
-
|
|
2920
|
-
|
|
3935
|
+
class SnapshotCopyError extends Error {
|
|
3936
|
+
relativePath;
|
|
3937
|
+
operation;
|
|
3938
|
+
path;
|
|
3939
|
+
code;
|
|
3940
|
+
constructor(relativePath, operation, error, path = relativePath) {
|
|
3941
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
3942
|
+
super(message);
|
|
3943
|
+
this.relativePath = relativePath;
|
|
3944
|
+
this.operation = operation;
|
|
3945
|
+
this.path = path;
|
|
3946
|
+
this.name = "SnapshotCopyError";
|
|
3947
|
+
this.code =
|
|
3948
|
+
error?.code ?? "UNKNOWN";
|
|
3949
|
+
}
|
|
3950
|
+
}
|
|
3951
|
+
function snapshotCopyFailure(error) {
|
|
3952
|
+
return {
|
|
3953
|
+
relativePath: error.relativePath,
|
|
3954
|
+
operation: error.operation,
|
|
3955
|
+
code: error.code,
|
|
3956
|
+
message: error.message,
|
|
3957
|
+
occurredAt: new Date().toISOString(),
|
|
3958
|
+
};
|
|
3959
|
+
}
|
|
3960
|
+
function hasSkippedSnapshotEntries(manifest) {
|
|
3961
|
+
return (manifest.skippedEntries?.length ?? 0) > 0;
|
|
3962
|
+
}
|
|
3963
|
+
function formatSkippedSnapshotPaths(manifest) {
|
|
3964
|
+
return (manifest.skippedEntries ?? [])
|
|
3965
|
+
.map((entry) => entry.relativePath)
|
|
3966
|
+
.join(", ");
|
|
3967
|
+
}
|
|
3968
|
+
function reportSnapshotCopyFailure(error, err) {
|
|
3969
|
+
const kind = error.operation === "read"
|
|
3970
|
+
? "snapshot-copy-unreadable"
|
|
3971
|
+
: "snapshot-copy-failed";
|
|
3972
|
+
err(`HQ_RESCUE_FAILURE_KIND=${kind}\n`);
|
|
3973
|
+
err(`HQ_RESCUE_SNAPSHOT_COPY_CODE=${error.code}\n`);
|
|
3974
|
+
const verb = error.operation === "metadata" ? "safely record" : error.operation;
|
|
3975
|
+
const detail = error.operation === "metadata" ? `: ${error.message}` : "";
|
|
3976
|
+
err(`error: safety snapshot could not ${verb} ${error.path} (${error.code})${detail}.\n`);
|
|
3977
|
+
if (error.operation === "read") {
|
|
3978
|
+
err(" HQ did not begin the update. If this file is stored in iCloud Drive or another cloud provider, download it locally, then retry.\n");
|
|
3979
|
+
}
|
|
3980
|
+
else {
|
|
3981
|
+
err(" HQ did not begin the update. Repair access to the reported path or backup location, then retry.\n");
|
|
3982
|
+
}
|
|
3983
|
+
}
|
|
3984
|
+
function reportSkippedSnapshotCopy(error, err) {
|
|
3985
|
+
const kind = error.operation === "read"
|
|
3986
|
+
? "snapshot-copy-unreadable"
|
|
3987
|
+
: "snapshot-copy-failed";
|
|
3988
|
+
err(`HQ_RESCUE_SKIPPED_KIND=${kind}\n`);
|
|
3989
|
+
err(`HQ_RESCUE_SNAPSHOT_COPY_CODE=${error.code}\n`);
|
|
3990
|
+
err(`warning: snapshot skipped ${error.path}. It was not backed up and was left untouched. The update continued.\n`);
|
|
3991
|
+
if (error.operation === "read") {
|
|
3992
|
+
err("warning: If this file is stored in iCloud Drive or another cloud provider, download it locally, then retry later.\n");
|
|
3993
|
+
}
|
|
3994
|
+
}
|
|
3995
|
+
function rescuePathsOverlap(first, second) {
|
|
3996
|
+
return (first === second ||
|
|
3997
|
+
first.startsWith(`${second}/`) ||
|
|
3998
|
+
second.startsWith(`${first}/`));
|
|
3999
|
+
}
|
|
4000
|
+
function isSkippedSnapshotPath(ctx, rel) {
|
|
4001
|
+
return ctx.skippedSnapshotRels.some((skippedRel) => rescuePathsOverlap(rel, skippedRel));
|
|
4002
|
+
}
|
|
4003
|
+
function appendLiteralRsyncExclude(excludeList, rel) {
|
|
4004
|
+
let pattern = rel;
|
|
4005
|
+
if (/[\\*?[]/.test(pattern)) {
|
|
4006
|
+
pattern = pattern
|
|
4007
|
+
.replace(/\\/g, "\\\\")
|
|
4008
|
+
.replace(/([*?[])/g, "\\$1");
|
|
4009
|
+
}
|
|
4010
|
+
fs.appendFileSync(excludeList, `/${pattern}\0`);
|
|
4011
|
+
}
|
|
4012
|
+
/**
|
|
4013
|
+
* A skipped snapshot path cannot enter the preserve shuttle because copying it
|
|
4014
|
+
* is exactly what failed. Reuse the existing overlay exclusion and action plan
|
|
4015
|
+
* instead: leave it in place, keep rsync off it, and remove every planned
|
|
4016
|
+
* mutation that could touch it or one of its parents.
|
|
4017
|
+
*/
|
|
4018
|
+
function protectSkippedSnapshotPaths(ctx, skippedEntries) {
|
|
4019
|
+
const skippedRels = skippedEntries.map((entry) => entry.relativePath);
|
|
4020
|
+
ctx.skippedSnapshotRels = skippedRels;
|
|
4021
|
+
const protectedRels = [
|
|
4022
|
+
...new Set([
|
|
4023
|
+
...skippedRels,
|
|
4024
|
+
...ctx.cfg.preserveSubpaths.filter((preserveRel) => skippedRels.some((skippedRel) => rescuePathsOverlap(preserveRel, skippedRel))),
|
|
4025
|
+
]),
|
|
4026
|
+
];
|
|
4027
|
+
if (protectedRels.length === 0)
|
|
2921
4028
|
return;
|
|
2922
|
-
|
|
4029
|
+
for (const rel of protectedRels) {
|
|
4030
|
+
appendLiteralRsyncExclude(ctx.unchangedList, rel);
|
|
4031
|
+
}
|
|
4032
|
+
ctx.actions = ctx.actions.filter((action) => {
|
|
4033
|
+
const protectedAction = action.affectedRels.some((rel) => protectedRels.some((protectedRel) => rescuePathsOverlap(rel, protectedRel)));
|
|
4034
|
+
if (protectedAction) {
|
|
4035
|
+
ctx.out(` left untouched after incomplete snapshot: ${action.affectedRels.join(", ")}\n`);
|
|
4036
|
+
}
|
|
4037
|
+
return !protectedAction;
|
|
4038
|
+
});
|
|
4039
|
+
}
|
|
4040
|
+
function pathIsInside(root, candidate, allowRoot = false) {
|
|
4041
|
+
const relative = path.relative(root, candidate);
|
|
4042
|
+
return ((allowRoot || relative !== "") &&
|
|
4043
|
+
!relative.startsWith(`..${path.sep}`) &&
|
|
4044
|
+
relative !== ".." &&
|
|
4045
|
+
!path.isAbsolute(relative));
|
|
4046
|
+
}
|
|
4047
|
+
function snapshotRelativePath(hqRoot, rel) {
|
|
4048
|
+
if (!rel ||
|
|
4049
|
+
rel.includes("\0") ||
|
|
4050
|
+
rel.includes("\\") ||
|
|
4051
|
+
path.posix.isAbsolute(rel) ||
|
|
4052
|
+
path.win32.isAbsolute(rel) ||
|
|
4053
|
+
rel.split("/").some((segment) => !segment || segment === "." || segment === "..")) {
|
|
4054
|
+
return null;
|
|
4055
|
+
}
|
|
4056
|
+
const absolute = path.resolve(hqRoot, rel);
|
|
4057
|
+
return pathIsInside(hqRoot, absolute) ? absolute : null;
|
|
4058
|
+
}
|
|
4059
|
+
function restoreParentIsSafe(hqRoot, destination) {
|
|
4060
|
+
for (let current = path.dirname(destination); current !== hqRoot; current = path.dirname(current)) {
|
|
4061
|
+
const st = lstatOrNull(current);
|
|
4062
|
+
if (st?.isSymbolicLink() || (st && !st.isDirectory()))
|
|
4063
|
+
return false;
|
|
4064
|
+
const parent = path.dirname(current);
|
|
4065
|
+
if (parent === current)
|
|
4066
|
+
return false;
|
|
4067
|
+
}
|
|
4068
|
+
return true;
|
|
4069
|
+
}
|
|
4070
|
+
function validateSnapshotSymlink(hqRoot, rel, target) {
|
|
4071
|
+
const linkPath = snapshotRelativePath(hqRoot, rel);
|
|
4072
|
+
if (!linkPath)
|
|
4073
|
+
return `invalid link path ${JSON.stringify(rel)}`;
|
|
4074
|
+
if (typeof target !== "string" || !target || target.includes("\0")) {
|
|
4075
|
+
return `invalid target for ${rel}`;
|
|
4076
|
+
}
|
|
4077
|
+
const resolvedTarget = path.resolve(path.dirname(linkPath), target);
|
|
4078
|
+
return pathIsInside(hqRoot, resolvedTarget, true)
|
|
4079
|
+
? null
|
|
4080
|
+
: `target for ${rel} resolves outside the HQ root`;
|
|
4081
|
+
}
|
|
4082
|
+
function copyRegularFileForSnapshot(src, dest, rel, mode) {
|
|
4083
|
+
let sourceFd;
|
|
4084
|
+
let destinationFd;
|
|
4085
|
+
try {
|
|
4086
|
+
try {
|
|
4087
|
+
sourceFd = fs.openSync(src, "r");
|
|
4088
|
+
}
|
|
4089
|
+
catch (error) {
|
|
4090
|
+
throw new SnapshotCopyError(rel, "read", error, src);
|
|
4091
|
+
}
|
|
4092
|
+
try {
|
|
4093
|
+
destinationFd = fs.openSync(dest, "w", mode);
|
|
4094
|
+
}
|
|
4095
|
+
catch (error) {
|
|
4096
|
+
throw new SnapshotCopyError(rel, "write", error, dest);
|
|
4097
|
+
}
|
|
4098
|
+
const buffer = Buffer.allocUnsafe(64 * 1024);
|
|
4099
|
+
for (;;) {
|
|
4100
|
+
let read;
|
|
4101
|
+
try {
|
|
4102
|
+
read = fs.readSync(sourceFd, buffer, 0, buffer.length, null);
|
|
4103
|
+
}
|
|
4104
|
+
catch (error) {
|
|
4105
|
+
throw new SnapshotCopyError(rel, "read", error, src);
|
|
4106
|
+
}
|
|
4107
|
+
if (read === 0)
|
|
4108
|
+
break;
|
|
4109
|
+
let offset = 0;
|
|
4110
|
+
while (offset < read) {
|
|
4111
|
+
let written;
|
|
4112
|
+
try {
|
|
4113
|
+
written = fs.writeSync(destinationFd, buffer, offset, read - offset);
|
|
4114
|
+
}
|
|
4115
|
+
catch (error) {
|
|
4116
|
+
throw new SnapshotCopyError(rel, "write", error, dest);
|
|
4117
|
+
}
|
|
4118
|
+
if (written === 0) {
|
|
4119
|
+
throw new SnapshotCopyError(rel, "write", new Error("write returned zero bytes"));
|
|
4120
|
+
}
|
|
4121
|
+
offset += written;
|
|
4122
|
+
}
|
|
4123
|
+
}
|
|
4124
|
+
}
|
|
4125
|
+
finally {
|
|
4126
|
+
if (destinationFd !== undefined)
|
|
4127
|
+
fs.closeSync(destinationFd);
|
|
4128
|
+
if (sourceFd !== undefined)
|
|
4129
|
+
fs.closeSync(sourceFd);
|
|
4130
|
+
}
|
|
4131
|
+
}
|
|
4132
|
+
/** Preserve a Windows file link as ordinary file data when links are forbidden. */
|
|
4133
|
+
function copySymlinkFileRepresentationForSnapshot(src, dest, rel) {
|
|
4134
|
+
let targetStats;
|
|
4135
|
+
try {
|
|
4136
|
+
targetStats = fs.statSync(src);
|
|
4137
|
+
}
|
|
4138
|
+
catch (error) {
|
|
4139
|
+
throw new SnapshotCopyError(rel, "read", error, src);
|
|
4140
|
+
}
|
|
4141
|
+
if (!targetStats.isFile()) {
|
|
4142
|
+
throw new SnapshotCopyError(rel, "metadata", new Error("file-link fallback requires a regular file target"), src);
|
|
4143
|
+
}
|
|
4144
|
+
try {
|
|
2923
4145
|
fs.mkdirSync(path.dirname(dest), { recursive: true });
|
|
2924
|
-
|
|
2925
|
-
|
|
4146
|
+
}
|
|
4147
|
+
catch (error) {
|
|
4148
|
+
throw new SnapshotCopyError(rel, "write", error, dest);
|
|
4149
|
+
}
|
|
4150
|
+
copyRegularFileForSnapshot(src, dest, rel, targetStats.mode);
|
|
4151
|
+
preserveSnapshotMetadata(dest, targetStats);
|
|
4152
|
+
}
|
|
4153
|
+
/** Snapshot copy that preserves metadata while pruning canonical transient directories. */
|
|
4154
|
+
function cpAToFiltered(src, dest, rel, hqRoot, symlinks, skippedEntries) {
|
|
4155
|
+
let st;
|
|
4156
|
+
try {
|
|
4157
|
+
st = fs.lstatSync(src);
|
|
4158
|
+
}
|
|
4159
|
+
catch (error) {
|
|
4160
|
+
throw new SnapshotCopyError(rel, "read", error, src);
|
|
4161
|
+
}
|
|
4162
|
+
if (st.isDirectory() && isRescueTransientDirectory(rel)) {
|
|
4163
|
+
return { captured: true };
|
|
4164
|
+
}
|
|
4165
|
+
if (st.isSymbolicLink()) {
|
|
4166
|
+
let target;
|
|
4167
|
+
try {
|
|
4168
|
+
target = fs.readlinkSync(src);
|
|
4169
|
+
}
|
|
4170
|
+
catch (error) {
|
|
4171
|
+
throw new SnapshotCopyError(rel, "read", error, src);
|
|
4172
|
+
}
|
|
4173
|
+
const problem = validateSnapshotSymlink(hqRoot, rel, target);
|
|
4174
|
+
if (problem) {
|
|
4175
|
+
throw new SnapshotCopyError(rel, "metadata", new Error(problem), src);
|
|
4176
|
+
}
|
|
4177
|
+
const representation = snapshotSymlinkRepresentation(hqRoot, src, dest, rel, target);
|
|
4178
|
+
if (representation === "file-copy") {
|
|
4179
|
+
copySymlinkFileRepresentationForSnapshot(src, dest, rel);
|
|
4180
|
+
}
|
|
4181
|
+
symlinks.push({ path: rel, target, ...(representation ? { representation } : {}) });
|
|
4182
|
+
return { captured: true };
|
|
2926
4183
|
}
|
|
2927
4184
|
if (st.isDirectory()) {
|
|
2928
|
-
|
|
2929
|
-
|
|
2930
|
-
|
|
4185
|
+
try {
|
|
4186
|
+
fs.mkdirSync(dest, { recursive: true });
|
|
4187
|
+
}
|
|
4188
|
+
catch (error) {
|
|
4189
|
+
throw new SnapshotCopyError(rel, "write", error, dest);
|
|
4190
|
+
}
|
|
4191
|
+
let names;
|
|
4192
|
+
try {
|
|
4193
|
+
names = fs.readdirSync(src);
|
|
4194
|
+
}
|
|
4195
|
+
catch (error) {
|
|
4196
|
+
throw new SnapshotCopyError(rel, "read", error, src);
|
|
4197
|
+
}
|
|
4198
|
+
let captured = names.length === 0;
|
|
4199
|
+
let firstFailure;
|
|
4200
|
+
for (const name of names) {
|
|
4201
|
+
try {
|
|
4202
|
+
const child = cpAToFiltered(path.join(src, name), path.join(dest, name), path.posix.join(rel, name), hqRoot, symlinks, skippedEntries);
|
|
4203
|
+
captured ||= child.captured;
|
|
4204
|
+
firstFailure ??= child.firstFailure;
|
|
4205
|
+
}
|
|
4206
|
+
catch (error) {
|
|
4207
|
+
if (error instanceof SnapshotCopyError) {
|
|
4208
|
+
skippedEntries.push(error);
|
|
4209
|
+
firstFailure ??= error;
|
|
4210
|
+
continue;
|
|
4211
|
+
}
|
|
4212
|
+
throw error;
|
|
4213
|
+
}
|
|
2931
4214
|
}
|
|
2932
4215
|
try {
|
|
2933
4216
|
fs.chmodSync(dest, st.mode);
|
|
@@ -2936,10 +4219,18 @@ function cpAToFiltered(src, dest, rel) {
|
|
|
2936
4219
|
catch {
|
|
2937
4220
|
/* metadata preservation is best-effort */
|
|
2938
4221
|
}
|
|
2939
|
-
return;
|
|
4222
|
+
return { captured, ...(firstFailure ? { firstFailure } : {}) };
|
|
2940
4223
|
}
|
|
2941
|
-
|
|
2942
|
-
|
|
4224
|
+
if (!st.isFile()) {
|
|
4225
|
+
throw new SnapshotCopyError(rel, "metadata", new Error("unsupported filesystem entry"));
|
|
4226
|
+
}
|
|
4227
|
+
try {
|
|
4228
|
+
fs.mkdirSync(path.dirname(dest), { recursive: true });
|
|
4229
|
+
}
|
|
4230
|
+
catch (error) {
|
|
4231
|
+
throw new SnapshotCopyError(rel, "write", error, dest);
|
|
4232
|
+
}
|
|
4233
|
+
copyRegularFileForSnapshot(src, dest, rel, st.mode);
|
|
2943
4234
|
try {
|
|
2944
4235
|
fs.chmodSync(dest, st.mode);
|
|
2945
4236
|
fs.utimesSync(dest, st.atime, st.mtime);
|
|
@@ -2947,6 +4238,7 @@ function cpAToFiltered(src, dest, rel) {
|
|
|
2947
4238
|
catch {
|
|
2948
4239
|
/* metadata preservation is best-effort */
|
|
2949
4240
|
}
|
|
4241
|
+
return { captured: true };
|
|
2950
4242
|
}
|
|
2951
4243
|
/** Synchronous single-line read from stdin (mirrors bash `read -r`). */
|
|
2952
4244
|
function readLineSync() {
|