@indigoai-us/hq-cloud 6.16.50 → 6.16.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backup-prune.d.ts +20 -0
- package/dist/backup-prune.d.ts.map +1 -1
- package/dist/backup-prune.js +62 -3
- package/dist/backup-prune.js.map +1 -1
- package/dist/backup-prune.test.js +25 -0
- package/dist/backup-prune.test.js.map +1 -1
- package/dist/cli/rescue-core.d.ts +9 -0
- package/dist/cli/rescue-core.d.ts.map +1 -1
- package/dist/cli/rescue-core.js +629 -66
- package/dist/cli/rescue-core.js.map +1 -1
- package/dist/cli/rescue-snapshot-copy.test.d.ts +2 -0
- package/dist/cli/rescue-snapshot-copy.test.d.ts.map +1 -0
- package/dist/cli/rescue-snapshot-copy.test.js +435 -0
- package/dist/cli/rescue-snapshot-copy.test.js.map +1 -0
- package/package.json +1 -1
package/dist/cli/rescue-core.js
CHANGED
|
@@ -31,7 +31,7 @@ import * as fs from "fs";
|
|
|
31
31
|
import * as os from "os";
|
|
32
32
|
import * as path from "path";
|
|
33
33
|
import { assertRescuePathTypeUnchanged, isRescueTransientDirectory, } from "./rescue-snapshot.js";
|
|
34
|
-
import { applyBackupPrune, canonicalRootHash, inventoryManagedSnapshots, } from "../backup-prune.js";
|
|
34
|
+
import { applyBackupPrune, canonicalRootHash, inventoryManagedSnapshots, isManagedSnapshotManifest, } from "../backup-prune.js";
|
|
35
35
|
// Paths always preserved across wipe+overlay (shuttled out, restored after).
|
|
36
36
|
const CARVE_OUT_PATHS = [
|
|
37
37
|
"core/packages",
|
|
@@ -329,6 +329,7 @@ export function parseRescueArgs(argv) {
|
|
|
329
329
|
backupRetentionCount: process.env.HQ_BACKUP_RETENTION_COUNT || "2",
|
|
330
330
|
recoverSnapshots: false,
|
|
331
331
|
verifyRestoredSnapshot: "",
|
|
332
|
+
restoreSnapshot: "",
|
|
332
333
|
};
|
|
333
334
|
const usage = () => {
|
|
334
335
|
// The legacy script printed its comment header here; we emit a concise
|
|
@@ -338,7 +339,8 @@ export function parseRescueArgs(argv) {
|
|
|
338
339
|
" [--hq-root DIR] [--no-history-check] [--floor-sha SHA]\n" +
|
|
339
340
|
" [--no-backup] [--backup-dir DIR] [--cloud-update]\n" +
|
|
340
341
|
" [--dry-run] [--yes] [--recover-snapshots]\n" +
|
|
341
|
-
" [--verify-restored-snapshot SNAPSHOT]\n"
|
|
342
|
+
" [--verify-restored-snapshot SNAPSHOT]\n" +
|
|
343
|
+
" [--restore-snapshot SNAPSHOT]\n");
|
|
342
344
|
throw new ExitError(1);
|
|
343
345
|
};
|
|
344
346
|
let narrowCsv = "";
|
|
@@ -397,6 +399,13 @@ export function parseRescueArgs(argv) {
|
|
|
397
399
|
throw new ExitError(2);
|
|
398
400
|
}
|
|
399
401
|
break;
|
|
402
|
+
case "--restore-snapshot":
|
|
403
|
+
cfg.restoreSnapshot = need(a);
|
|
404
|
+
if (!cfg.restoreSnapshot) {
|
|
405
|
+
process.stderr.write("error: --restore-snapshot requires a snapshot name.\n");
|
|
406
|
+
throw new ExitError(2);
|
|
407
|
+
}
|
|
408
|
+
break;
|
|
400
409
|
case "--dry-run":
|
|
401
410
|
cfg.dryRun = true;
|
|
402
411
|
break;
|
|
@@ -461,6 +470,14 @@ export function parseRescueArgs(argv) {
|
|
|
461
470
|
process.stderr.write("error: --verify-restored-snapshot requires --recover-snapshots.\n");
|
|
462
471
|
throw new ExitError(2);
|
|
463
472
|
}
|
|
473
|
+
if (cfg.restoreSnapshot && !cfg.recoverSnapshots) {
|
|
474
|
+
process.stderr.write("error: --restore-snapshot requires --recover-snapshots.\n");
|
|
475
|
+
throw new ExitError(2);
|
|
476
|
+
}
|
|
477
|
+
if (cfg.restoreSnapshot && cfg.verifyRestoredSnapshot) {
|
|
478
|
+
process.stderr.write("error: --restore-snapshot and --verify-restored-snapshot must be run separately.\n");
|
|
479
|
+
throw new ExitError(2);
|
|
480
|
+
}
|
|
464
481
|
return cfg;
|
|
465
482
|
}
|
|
466
483
|
/**
|
|
@@ -589,6 +606,9 @@ function recoverSnapshots(cfg, out, err) {
|
|
|
589
606
|
err(`error: cannot inspect safety snapshots: backup root is unreadable: ${cfg.backupRoot}\n`);
|
|
590
607
|
return { status: 1 };
|
|
591
608
|
}
|
|
609
|
+
if (cfg.restoreSnapshot) {
|
|
610
|
+
return restoreRecoverySnapshot(cfg, hqRoot, inventory.entries, inventory.protected, out, err);
|
|
611
|
+
}
|
|
592
612
|
if (cfg.verifyRestoredSnapshot) {
|
|
593
613
|
return verifyRestoredSnapshot(cfg, hqRoot, inventory.entries, inventory.protected, out, err);
|
|
594
614
|
}
|
|
@@ -645,15 +665,44 @@ function recoverSnapshots(cfg, out, err) {
|
|
|
645
665
|
if (unresolved.length > 0) {
|
|
646
666
|
err("error: snapshot recovery refused: the interrupted updates are not proven superseded.\n");
|
|
647
667
|
err(" HQ retained every unverified snapshot. A later completed-good rescue for this same HQ root is the only automatic proof.\n");
|
|
648
|
-
err(" Restore
|
|
668
|
+
err(" Restore one listed snapshot, then verify it before one retry is permitted:\n");
|
|
649
669
|
for (const item of unresolved) {
|
|
650
|
-
err(` ${item.entry.name}: ${
|
|
670
|
+
err(` restore ${item.entry.name}: ${formatSnapshotRestoreCommand(hqRoot, cfg.backupRoot, item.entry.name)}\n`);
|
|
671
|
+
err(` verify ${item.entry.name}: ${formatRecoveryCommand(hqRoot, cfg.backupRoot, item.entry.name)}\n`);
|
|
651
672
|
}
|
|
652
673
|
return { status: 1 };
|
|
653
674
|
}
|
|
654
675
|
out("==> Snapshot recovery complete: every eligible recovery snapshot was positively verified before deletion.\n");
|
|
655
676
|
return { status: 0 };
|
|
656
677
|
}
|
|
678
|
+
/**
|
|
679
|
+
* Explicitly restore every captured root, including links represented only in
|
|
680
|
+
* the snapshot manifest. Verification remains a separate positive-proof step.
|
|
681
|
+
*/
|
|
682
|
+
function restoreRecoverySnapshot(cfg, hqRoot, entries, protectedEntries, out, err) {
|
|
683
|
+
const entry = entries.find((item) => item.name === cfg.restoreSnapshot);
|
|
684
|
+
const unresolved = protectedEntries.find((item) => item.entry.name === cfg.restoreSnapshot &&
|
|
685
|
+
item.reason === "recovery-required-no-later-good");
|
|
686
|
+
if (!entry || !unresolved || entry.manifest.status !== "recovery-required") {
|
|
687
|
+
err(`error: ${cfg.restoreSnapshot} is not an unresolved recovery-required snapshot for this HQ root.\n`);
|
|
688
|
+
return { status: 1 };
|
|
689
|
+
}
|
|
690
|
+
if (cfg.dryRun) {
|
|
691
|
+
out(`==> Dry run would restore every captured root and recorded symlink from ${entry.name}.\n`);
|
|
692
|
+
return { status: 0 };
|
|
693
|
+
}
|
|
694
|
+
try {
|
|
695
|
+
restoreSnapshotPaths(path.join(cfg.backupRoot, entry.name), hqRoot, entry.manifest.includedRoots, entry.manifest);
|
|
696
|
+
}
|
|
697
|
+
catch (error) {
|
|
698
|
+
err(`error: could not restore safety snapshot ${entry.name}: ${error instanceof Error ? error.message : String(error)}\n`);
|
|
699
|
+
err(" HQ retained the snapshot and did not write a post-restore permit.\n");
|
|
700
|
+
return { status: 1 };
|
|
701
|
+
}
|
|
702
|
+
out(`==> Restored every captured root and recorded symlink from ${entry.name}.\n`);
|
|
703
|
+
out(` Verify the restored tree before retrying: ${formatRecoveryCommand(hqRoot, cfg.backupRoot, entry.name)}\n`);
|
|
704
|
+
return { status: 0 };
|
|
705
|
+
}
|
|
657
706
|
/** A live tree must still match the selected recovery snapshot at permit use. */
|
|
658
707
|
function verifyRestoredSnapshot(cfg, hqRoot, entries, protectedEntries, out, err) {
|
|
659
708
|
const entry = entries.find((item) => item.name === cfg.verifyRestoredSnapshot);
|
|
@@ -663,7 +712,7 @@ function verifyRestoredSnapshot(cfg, hqRoot, entries, protectedEntries, out, err
|
|
|
663
712
|
err(`error: ${cfg.verifyRestoredSnapshot} is not an unresolved recovery-required snapshot for this HQ root.\n`);
|
|
664
713
|
return { status: 1 };
|
|
665
714
|
}
|
|
666
|
-
const mismatch = snapshotTreeMismatch(path.join(cfg.backupRoot, entry.name), hqRoot, entry.manifest.includedRoots);
|
|
715
|
+
const mismatch = snapshotTreeMismatch(path.join(cfg.backupRoot, entry.name), hqRoot, entry.manifest.includedRoots, entry.manifest.symlinks);
|
|
667
716
|
if (mismatch) {
|
|
668
717
|
err(`error: restored tree does not match snapshot ${entry.name}: ${mismatch}\n`);
|
|
669
718
|
err(" No post-restore permit was written; the circuit breaker remains armed.\n");
|
|
@@ -685,13 +734,32 @@ function verifyRestoredSnapshot(cfg, hqRoot, entries, protectedEntries, out, err
|
|
|
685
734
|
return { status: 0 };
|
|
686
735
|
}
|
|
687
736
|
/** Return a precise mismatch instead of treating a partial restore as evidence. */
|
|
688
|
-
function snapshotTreeMismatch(snapshotRoot, hqRoot, includedRoots) {
|
|
689
|
-
|
|
737
|
+
function snapshotTreeMismatch(snapshotRoot, hqRoot, includedRoots, symlinks) {
|
|
738
|
+
const roots = snapshotCapturedRoots(includedRoots);
|
|
739
|
+
if (typeof roots === "string")
|
|
740
|
+
return roots;
|
|
741
|
+
const recorded = snapshotSymlinkTargets(hqRoot, roots, symlinks);
|
|
742
|
+
if (typeof recorded === "string")
|
|
743
|
+
return recorded;
|
|
744
|
+
for (const root of roots) {
|
|
745
|
+
if (rescuePathKind(path.join(snapshotRoot, root)) === "missing" &&
|
|
746
|
+
!recorded.has(root)) {
|
|
747
|
+
return `snapshot does not contain captured root ${root}`;
|
|
748
|
+
}
|
|
749
|
+
const mismatch = snapshotPathMismatch(path.join(snapshotRoot, root), path.join(hqRoot, root), root, recorded);
|
|
750
|
+
if (mismatch)
|
|
751
|
+
return mismatch;
|
|
752
|
+
}
|
|
753
|
+
return null;
|
|
754
|
+
}
|
|
755
|
+
function snapshotCapturedRoots(includedRoots) {
|
|
756
|
+
if (!Array.isArray(includedRoots) || includedRoots.length === 0) {
|
|
690
757
|
return "snapshot manifest records no captured roots to verify";
|
|
691
758
|
}
|
|
692
759
|
const seen = new Set();
|
|
693
760
|
for (const root of includedRoots) {
|
|
694
|
-
if (
|
|
761
|
+
if (typeof root !== "string" ||
|
|
762
|
+
!root ||
|
|
695
763
|
root === "." ||
|
|
696
764
|
root === ".." ||
|
|
697
765
|
root.includes("/") ||
|
|
@@ -701,18 +769,76 @@ function snapshotTreeMismatch(snapshotRoot, hqRoot, includedRoots) {
|
|
|
701
769
|
return `snapshot manifest has an unsafe captured root ${JSON.stringify(root)}`;
|
|
702
770
|
}
|
|
703
771
|
seen.add(root);
|
|
704
|
-
|
|
705
|
-
|
|
772
|
+
}
|
|
773
|
+
return seen;
|
|
774
|
+
}
|
|
775
|
+
function snapshotSymlinkTargets(hqRoot, includedRoots, symlinks) {
|
|
776
|
+
if (symlinks !== undefined && !Array.isArray(symlinks)) {
|
|
777
|
+
return "snapshot manifest symlinks is not an array";
|
|
778
|
+
}
|
|
779
|
+
const recorded = new Map();
|
|
780
|
+
for (const link of symlinks ?? []) {
|
|
781
|
+
if (!link ||
|
|
782
|
+
typeof link.path !== "string" ||
|
|
783
|
+
(link.representation !== undefined && link.representation !== "file-copy")) {
|
|
784
|
+
return "snapshot manifest has an invalid recorded symlink";
|
|
706
785
|
}
|
|
707
|
-
const
|
|
708
|
-
if (
|
|
709
|
-
return
|
|
786
|
+
const problem = validateSnapshotSymlink(hqRoot, link.path, link.target);
|
|
787
|
+
if (problem)
|
|
788
|
+
return `snapshot manifest ${problem}`;
|
|
789
|
+
const root = link.path.split("/")[0];
|
|
790
|
+
if (!includedRoots.has(root)) {
|
|
791
|
+
return `snapshot manifest records symlink ${link.path} outside its captured roots`;
|
|
792
|
+
}
|
|
793
|
+
if (recorded.has(link.path)) {
|
|
794
|
+
return `snapshot manifest records duplicate symlink ${link.path}`;
|
|
795
|
+
}
|
|
796
|
+
recorded.set(link.path, link);
|
|
710
797
|
}
|
|
711
|
-
|
|
798
|
+
for (const linkPath of recorded.keys()) {
|
|
799
|
+
for (let parent = path.posix.dirname(linkPath); parent !== "."; parent = path.posix.dirname(parent)) {
|
|
800
|
+
if (recorded.has(parent)) {
|
|
801
|
+
return `snapshot manifest records symlink ${linkPath} beneath recorded symlink ${parent}`;
|
|
802
|
+
}
|
|
803
|
+
}
|
|
804
|
+
}
|
|
805
|
+
return recorded;
|
|
806
|
+
}
|
|
807
|
+
function recordedSymlinkTargetMatches(livePath, recordedTarget, liveTarget) {
|
|
808
|
+
const parent = path.dirname(livePath);
|
|
809
|
+
return (path.resolve(parent, recordedTarget) === path.resolve(parent, liveTarget));
|
|
712
810
|
}
|
|
713
|
-
function snapshotPathMismatch(snapshotPath, livePath, rel) {
|
|
811
|
+
function snapshotPathMismatch(snapshotPath, livePath, rel, recordedSymlinks) {
|
|
714
812
|
const snapshotKind = rescuePathKind(snapshotPath);
|
|
715
813
|
const liveKind = rescuePathKind(livePath);
|
|
814
|
+
const recordedLink = recordedSymlinks.get(rel);
|
|
815
|
+
if (recordedLink !== undefined) {
|
|
816
|
+
if (recordedLink.representation === "file-copy") {
|
|
817
|
+
if (snapshotKind !== "file") {
|
|
818
|
+
return `${rel} is ${snapshotKind} but the snapshot records a copied file link`;
|
|
819
|
+
}
|
|
820
|
+
if (liveKind !== "file") {
|
|
821
|
+
return `${rel} is ${liveKind} but the snapshot records a copied file link`;
|
|
822
|
+
}
|
|
823
|
+
return bytesEqual(snapshotPath, livePath)
|
|
824
|
+
? null
|
|
825
|
+
: `${rel} has different file contents`;
|
|
826
|
+
}
|
|
827
|
+
if (snapshotKind !== "missing") {
|
|
828
|
+
return `${rel} is stored as both a snapshot entry and recorded symlink`;
|
|
829
|
+
}
|
|
830
|
+
if (liveKind !== "symlink") {
|
|
831
|
+
return `${rel} is ${liveKind} but the snapshot records a symlink`;
|
|
832
|
+
}
|
|
833
|
+
try {
|
|
834
|
+
return recordedSymlinkTargetMatches(livePath, recordedLink.target, fs.readlinkSync(livePath))
|
|
835
|
+
? null
|
|
836
|
+
: `${rel} has a different symlink target`;
|
|
837
|
+
}
|
|
838
|
+
catch (error) {
|
|
839
|
+
return `${rel} could not read a symlink target: ${error instanceof Error ? error.message : String(error)}`;
|
|
840
|
+
}
|
|
841
|
+
}
|
|
716
842
|
if (snapshotKind !== liveKind) {
|
|
717
843
|
return `${rel} is ${liveKind} but the snapshot records ${snapshotKind}`;
|
|
718
844
|
}
|
|
@@ -751,17 +877,200 @@ function snapshotPathMismatch(snapshotPath, livePath, rel) {
|
|
|
751
877
|
catch (error) {
|
|
752
878
|
return `${rel} could not be read: ${error instanceof Error ? error.message : String(error)}`;
|
|
753
879
|
}
|
|
754
|
-
|
|
755
|
-
|
|
880
|
+
const expectedNames = new Set(snapshotNames);
|
|
881
|
+
for (const recordedRel of recordedSymlinks.keys()) {
|
|
882
|
+
if (path.posix.dirname(recordedRel) === rel) {
|
|
883
|
+
expectedNames.add(path.posix.basename(recordedRel));
|
|
884
|
+
}
|
|
885
|
+
}
|
|
886
|
+
const expected = [...expectedNames].sort();
|
|
887
|
+
if (expected.length !== liveNames.length ||
|
|
888
|
+
expected.some((name, index) => name !== liveNames[index])) {
|
|
756
889
|
return `${rel} has different entries`;
|
|
757
890
|
}
|
|
758
|
-
for (const name of
|
|
759
|
-
const mismatch = snapshotPathMismatch(path.join(snapshotPath, name), path.join(livePath, name), path.posix.join(rel, name));
|
|
891
|
+
for (const name of expected) {
|
|
892
|
+
const mismatch = snapshotPathMismatch(path.join(snapshotPath, name), path.join(livePath, name), path.posix.join(rel, name), recordedSymlinks);
|
|
760
893
|
if (mismatch)
|
|
761
894
|
return mismatch;
|
|
762
895
|
}
|
|
763
896
|
return null;
|
|
764
897
|
}
|
|
898
|
+
/**
|
|
899
|
+
* Restore one captured path, then recreate only manifest-recorded symlinks
|
|
900
|
+
* whose locations and targets are both confined to the HQ root.
|
|
901
|
+
*/
|
|
902
|
+
export function restoreSnapshotPath(snapshotDir, hqRoot, rel, manifest) {
|
|
903
|
+
restoreSnapshotPaths(snapshotDir, hqRoot, [rel], manifest);
|
|
904
|
+
}
|
|
905
|
+
/** Restore all requested physical paths before recreating their recorded links. */
|
|
906
|
+
function restoreSnapshotPaths(snapshotDir, hqRoot, rels, manifest) {
|
|
907
|
+
const capturedRoots = snapshotCapturedRoots(manifest.includedRoots);
|
|
908
|
+
if (typeof capturedRoots === "string")
|
|
909
|
+
throw new Error(capturedRoots);
|
|
910
|
+
const recorded = snapshotSymlinkTargets(hqRoot, capturedRoots, manifest.symlinks);
|
|
911
|
+
if (typeof recorded === "string")
|
|
912
|
+
throw new Error(recorded);
|
|
913
|
+
for (const [recordedRel, link] of recorded) {
|
|
914
|
+
const snapshotKind = rescuePathKind(path.join(snapshotDir, recordedRel));
|
|
915
|
+
if (link.representation === "file-copy") {
|
|
916
|
+
if (snapshotKind !== "file") {
|
|
917
|
+
throw new Error(`snapshot records copied file link ${recordedRel} without copied file data`);
|
|
918
|
+
}
|
|
919
|
+
}
|
|
920
|
+
else if (snapshotKind !== "missing") {
|
|
921
|
+
throw new Error(`snapshot records symlink ${recordedRel} alongside copied data`);
|
|
922
|
+
}
|
|
923
|
+
}
|
|
924
|
+
const requested = [...new Set(rels)].sort((left, right) => left.localeCompare(right));
|
|
925
|
+
if (requested.length === 0)
|
|
926
|
+
throw new Error("snapshot restore requested no paths");
|
|
927
|
+
const items = requested.map((rel) => {
|
|
928
|
+
const destination = snapshotRelativePath(hqRoot, rel);
|
|
929
|
+
if (!destination) {
|
|
930
|
+
throw new Error(`invalid snapshot restore path ${JSON.stringify(rel)}`);
|
|
931
|
+
}
|
|
932
|
+
if (!restoreParentIsSafe(hqRoot, destination)) {
|
|
933
|
+
throw new Error(`snapshot restore parent for ${rel} is not a directory inside the HQ root`);
|
|
934
|
+
}
|
|
935
|
+
if (!capturedRoots.has(rel.split("/")[0])) {
|
|
936
|
+
throw new Error(`snapshot does not include ${rel}`);
|
|
937
|
+
}
|
|
938
|
+
const source = path.join(snapshotDir, rel);
|
|
939
|
+
const physicalSourceExists = lexists(source);
|
|
940
|
+
const recordedWithinPath = [...recorded.keys()].some((recordedRel) => recordedRel === rel || recordedRel.startsWith(`${rel}/`));
|
|
941
|
+
if (!physicalSourceExists && !recordedWithinPath) {
|
|
942
|
+
throw new Error(`snapshot does not contain ${rel}`);
|
|
943
|
+
}
|
|
944
|
+
return { rel, source, destination, physicalSourceExists };
|
|
945
|
+
});
|
|
946
|
+
for (const item of items) {
|
|
947
|
+
fs.rmSync(item.destination, { recursive: true, force: true });
|
|
948
|
+
}
|
|
949
|
+
for (const item of items) {
|
|
950
|
+
if (item.physicalSourceExists) {
|
|
951
|
+
copySnapshotPhysicalTree(item.source, item.destination, hqRoot, item.rel);
|
|
952
|
+
}
|
|
953
|
+
}
|
|
954
|
+
const links = orderSnapshotSymlinkRestore(hqRoot, [...recorded.entries()]
|
|
955
|
+
.filter(([recordedRel, link]) => link.representation === undefined &&
|
|
956
|
+
items.some((item) => recordedRel === item.rel || recordedRel.startsWith(`${item.rel}/`)))
|
|
957
|
+
.map(([recordedRel, link]) => [recordedRel, link.target]));
|
|
958
|
+
for (const [linkRel, target] of links) {
|
|
959
|
+
recreateSnapshotSymlink(hqRoot, linkRel, target);
|
|
960
|
+
}
|
|
961
|
+
}
|
|
962
|
+
function orderSnapshotSymlinkRestore(hqRoot, links) {
|
|
963
|
+
const pending = new Map(links);
|
|
964
|
+
const ordered = [];
|
|
965
|
+
while (pending.size > 0) {
|
|
966
|
+
const next = [...pending.entries()]
|
|
967
|
+
.sort(([left], [right]) => left.localeCompare(right))
|
|
968
|
+
.find(([rel, target]) => {
|
|
969
|
+
const linkPath = snapshotRelativePath(hqRoot, rel);
|
|
970
|
+
const resolvedTarget = path.resolve(path.dirname(linkPath), target);
|
|
971
|
+
return ![...pending.keys()].some((candidate) => {
|
|
972
|
+
if (candidate === rel)
|
|
973
|
+
return false;
|
|
974
|
+
const candidatePath = snapshotRelativePath(hqRoot, candidate);
|
|
975
|
+
return (resolvedTarget === candidatePath ||
|
|
976
|
+
resolvedTarget.startsWith(`${candidatePath}${path.sep}`));
|
|
977
|
+
});
|
|
978
|
+
});
|
|
979
|
+
if (!next) {
|
|
980
|
+
ordered.push(...[...pending.entries()].sort(([left], [right]) => left.localeCompare(right)));
|
|
981
|
+
break;
|
|
982
|
+
}
|
|
983
|
+
pending.delete(next[0]);
|
|
984
|
+
ordered.push(next);
|
|
985
|
+
}
|
|
986
|
+
return ordered;
|
|
987
|
+
}
|
|
988
|
+
function copySnapshotPhysicalTree(source, destination, hqRoot, rel) {
|
|
989
|
+
const st = fs.lstatSync(source);
|
|
990
|
+
if (st.isSymbolicLink()) {
|
|
991
|
+
recreateSnapshotSymlink(hqRoot, rel, fs.readlinkSync(source));
|
|
992
|
+
return;
|
|
993
|
+
}
|
|
994
|
+
if (st.isDirectory()) {
|
|
995
|
+
fs.mkdirSync(destination, { recursive: true });
|
|
996
|
+
for (const name of fs.readdirSync(source)) {
|
|
997
|
+
copySnapshotPhysicalTree(path.join(source, name), path.join(destination, name), hqRoot, path.posix.join(rel, name));
|
|
998
|
+
}
|
|
999
|
+
preserveSnapshotMetadata(destination, st);
|
|
1000
|
+
return;
|
|
1001
|
+
}
|
|
1002
|
+
if (!st.isFile())
|
|
1003
|
+
throw new Error(`snapshot contains unsupported entry ${rel}`);
|
|
1004
|
+
fs.mkdirSync(path.dirname(destination), { recursive: true });
|
|
1005
|
+
fs.copyFileSync(source, destination);
|
|
1006
|
+
preserveSnapshotMetadata(destination, st);
|
|
1007
|
+
}
|
|
1008
|
+
/** Data restored successfully remains restored when filesystem metadata is unsupported. */
|
|
1009
|
+
function preserveSnapshotMetadata(destination, st) {
|
|
1010
|
+
try {
|
|
1011
|
+
fs.chmodSync(destination, st.mode);
|
|
1012
|
+
fs.utimesSync(destination, st.atime, st.mtime);
|
|
1013
|
+
}
|
|
1014
|
+
catch {
|
|
1015
|
+
/* metadata preservation is best-effort, matching the prior rollback helper */
|
|
1016
|
+
}
|
|
1017
|
+
}
|
|
1018
|
+
function recreateSnapshotSymlink(hqRoot, rel, target, destination = snapshotRelativePath(hqRoot, rel)) {
|
|
1019
|
+
const problem = validateSnapshotSymlink(hqRoot, rel, target);
|
|
1020
|
+
if (problem)
|
|
1021
|
+
throw new Error(`invalid snapshot symlink: ${problem}`);
|
|
1022
|
+
if (!destination)
|
|
1023
|
+
throw new Error(`invalid snapshot symlink destination ${rel}`);
|
|
1024
|
+
const restoredLinkPath = snapshotRelativePath(hqRoot, rel);
|
|
1025
|
+
const resolvedTarget = path.resolve(path.dirname(restoredLinkPath), target);
|
|
1026
|
+
fs.mkdirSync(path.dirname(destination), { recursive: true });
|
|
1027
|
+
fs.rmSync(destination, { recursive: true, force: true });
|
|
1028
|
+
try {
|
|
1029
|
+
fs.symlinkSync(target, destination);
|
|
1030
|
+
}
|
|
1031
|
+
catch (error) {
|
|
1032
|
+
if (process.platform === "win32" &&
|
|
1033
|
+
error.code === "EPERM" &&
|
|
1034
|
+
isDir(resolvedTarget)) {
|
|
1035
|
+
fs.symlinkSync(resolvedTarget, destination, "junction");
|
|
1036
|
+
return;
|
|
1037
|
+
}
|
|
1038
|
+
throw error;
|
|
1039
|
+
}
|
|
1040
|
+
}
|
|
1041
|
+
/**
|
|
1042
|
+
* Directory links can become junctions on Windows. A file link without link
|
|
1043
|
+
* privilege instead needs copied file data at its original snapshot path.
|
|
1044
|
+
*/
|
|
1045
|
+
function snapshotSymlinkRepresentation(hqRoot, source, destination, rel, target) {
|
|
1046
|
+
if (process.platform !== "win32")
|
|
1047
|
+
return undefined;
|
|
1048
|
+
const probe = `${destination}.hq-restore-probe-${process.pid}-${Date.now().toString(36)}`;
|
|
1049
|
+
try {
|
|
1050
|
+
recreateSnapshotSymlink(hqRoot, rel, target, probe);
|
|
1051
|
+
fs.rmSync(probe, { recursive: true, force: true });
|
|
1052
|
+
return undefined;
|
|
1053
|
+
}
|
|
1054
|
+
catch (error) {
|
|
1055
|
+
try {
|
|
1056
|
+
fs.rmSync(probe, { recursive: true, force: true });
|
|
1057
|
+
}
|
|
1058
|
+
catch (cleanupError) {
|
|
1059
|
+
throw new SnapshotCopyError(rel, "write", cleanupError, probe);
|
|
1060
|
+
}
|
|
1061
|
+
if (process.platform === "win32" &&
|
|
1062
|
+
error.code === "EPERM") {
|
|
1063
|
+
try {
|
|
1064
|
+
if (fs.statSync(source).isFile())
|
|
1065
|
+
return "file-copy";
|
|
1066
|
+
}
|
|
1067
|
+
catch (targetError) {
|
|
1068
|
+
throw new SnapshotCopyError(rel, "read", targetError, source);
|
|
1069
|
+
}
|
|
1070
|
+
}
|
|
1071
|
+
throw new SnapshotCopyError(rel, "metadata", new Error(`recorded symlink cannot be restored on this machine: ${error instanceof Error ? error.message : String(error)}`), source);
|
|
1072
|
+
}
|
|
1073
|
+
}
|
|
765
1074
|
/**
|
|
766
1075
|
* True when `root` is usable as a rescue target: an absolute path that is not
|
|
767
1076
|
* a bare drive letter. Pure + exported for tests; `platformPath` defaults to
|
|
@@ -1276,7 +1585,7 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
1276
1585
|
err(" HQ kept their pre-update snapshots because those updates started changing the release-managed tree but did not finish.\n");
|
|
1277
1586
|
err(" It will not overwrite or delete that evidence without proof that a later rescue completed successfully.\n");
|
|
1278
1587
|
err(` Inspect safely: ${formatRecoveryCommand(hqRoot, cfg.backupRoot)}\n`);
|
|
1279
|
-
err(" It deletes only snapshots with a later completed-good rescue. For unresolved snapshots, restore
|
|
1588
|
+
err(" It deletes only snapshots with a later completed-good rescue. For unresolved snapshots, use --restore-snapshot to restore every captured root and recorded link, then run its verification command before retrying.\n");
|
|
1280
1589
|
throw new ExitError(1);
|
|
1281
1590
|
}
|
|
1282
1591
|
}
|
|
@@ -1286,23 +1595,36 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
1286
1595
|
backupDir = allocateSnapshotDir(cfg.backupRoot, RUN_TS, canonicalRootHash(hqRoot), runId);
|
|
1287
1596
|
out("\n");
|
|
1288
1597
|
out(`==> Safety snapshot -> ${backupDir}\n`);
|
|
1598
|
+
const snapshotSymlinks = [];
|
|
1289
1599
|
try {
|
|
1290
1600
|
fs.mkdirSync(backupDir, { recursive: false });
|
|
1291
1601
|
writeSnapshotManifest(backupDir, snapshotManifest("copying", hqRoot, runId, cfg, srcSha, wipeToplevel));
|
|
1292
1602
|
for (const rootRel of wipeToplevel) {
|
|
1293
1603
|
const srcAbs = path.join(hqRoot, rootRel);
|
|
1294
|
-
if (
|
|
1604
|
+
if (isRescueTransientDirectory(rootRel))
|
|
1295
1605
|
continue;
|
|
1296
|
-
cpAToFiltered(srcAbs, path.join(backupDir, rootRel), rootRel);
|
|
1606
|
+
cpAToFiltered(srcAbs, path.join(backupDir, rootRel), rootRel, hqRoot, snapshotSymlinks);
|
|
1297
1607
|
}
|
|
1298
|
-
updateSnapshotStatus(backupDir, "ready"
|
|
1608
|
+
updateSnapshotStatus(backupDir, "ready", undefined, {
|
|
1609
|
+
symlinks: snapshotSymlinks,
|
|
1610
|
+
});
|
|
1299
1611
|
}
|
|
1300
1612
|
catch (error) {
|
|
1301
|
-
|
|
1613
|
+
if (error instanceof SnapshotCopyError) {
|
|
1614
|
+
updateSnapshotStatus(backupDir, "failed-pre-mutation", undefined, {
|
|
1615
|
+
symlinks: snapshotSymlinks,
|
|
1616
|
+
copyFailure: snapshotCopyFailure(error),
|
|
1617
|
+
});
|
|
1618
|
+
reportSnapshotCopyFailure(error, err);
|
|
1619
|
+
throw new ExitError(1);
|
|
1620
|
+
}
|
|
1621
|
+
updateSnapshotStatus(backupDir, "failed-pre-mutation", undefined, {
|
|
1622
|
+
symlinks: snapshotSymlinks,
|
|
1623
|
+
});
|
|
1302
1624
|
throw error;
|
|
1303
1625
|
}
|
|
1304
1626
|
pruneManagedBackups(cfg, hqRoot, out, true);
|
|
1305
|
-
out(
|
|
1627
|
+
out(" snapshot ready (complete recovery is available through --recover-snapshots).\n");
|
|
1306
1628
|
}
|
|
1307
1629
|
else if (cfg.doBackup && !cfg.dryRun && wipeToplevel.length !== 0) {
|
|
1308
1630
|
out("\n==> Safety snapshot skipped (classification found no destructive paths).\n");
|
|
@@ -1547,10 +1869,11 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
1547
1869
|
}
|
|
1548
1870
|
lines.push("", "## Restore");
|
|
1549
1871
|
lines.push("This directory holds the release-managed wipe set as it was before the update; reproducible worktrees and build caches are intentionally excluded.");
|
|
1550
|
-
|
|
1551
|
-
lines.push(
|
|
1552
|
-
lines.push(
|
|
1553
|
-
lines.push(
|
|
1872
|
+
const snapshotName = path.basename(backupDir);
|
|
1873
|
+
lines.push("Restore every captured root and recorded symlink:");
|
|
1874
|
+
lines.push(` ${formatSnapshotRestoreCommand(hqRoot, cfg.backupRoot, snapshotName)}`);
|
|
1875
|
+
lines.push("Then verify the restored tree before retrying:");
|
|
1876
|
+
lines.push(` ${formatRecoveryCommand(hqRoot, cfg.backupRoot, snapshotName)}`);
|
|
1554
1877
|
lines.push("", `Retention: ${cfg.backupRetentionDays} days and ${cfg.backupRetentionCount} completed snapshots (latest recovery floor retained).`);
|
|
1555
1878
|
try {
|
|
1556
1879
|
fs.writeFileSync(path.join(backupDir, "RECOVERY.md"), lines.join("\n") + "\n");
|
|
@@ -1618,12 +1941,15 @@ function applyClassifiedActions(ctx, backupDir) {
|
|
|
1618
1941
|
completed.push(action);
|
|
1619
1942
|
}
|
|
1620
1943
|
catch (err) {
|
|
1621
|
-
const
|
|
1622
|
-
const manifest = writeApplyFailureManifest(ctx, backupDir, action, completed, restored, err);
|
|
1944
|
+
const rollback = rollbackCompletedActions(ctx, completed, backupDir);
|
|
1945
|
+
const manifest = writeApplyFailureManifest(ctx, backupDir, action, completed, rollback.restored, rollback.failures, err);
|
|
1623
1946
|
ctx.out("\n");
|
|
1624
1947
|
ctx.out(`error: rescue apply failed during '${action.label}'.\n`);
|
|
1625
|
-
if (restored.length > 0) {
|
|
1626
|
-
ctx.out(`==> Rolled back ${restored.length} prior action path(s) from the safety snapshot.\n`);
|
|
1948
|
+
if (rollback.restored.length > 0) {
|
|
1949
|
+
ctx.out(`==> Rolled back ${rollback.restored.length} prior action path(s) from the safety snapshot.\n`);
|
|
1950
|
+
}
|
|
1951
|
+
for (const failure of rollback.failures) {
|
|
1952
|
+
ctx.out(`warning: rollback retained recovery evidence: ${failure}\n`);
|
|
1627
1953
|
}
|
|
1628
1954
|
if (manifest) {
|
|
1629
1955
|
ctx.out(`==> Recovery manifest: ${manifest}\n`);
|
|
@@ -1649,7 +1975,14 @@ function injectApplyPathChangeForTest(ctx) {
|
|
|
1649
1975
|
}
|
|
1650
1976
|
function rollbackCompletedActions(ctx, completed, backupDir) {
|
|
1651
1977
|
if (!backupDir || !isDir(backupDir))
|
|
1652
|
-
return [];
|
|
1978
|
+
return { restored: [], failures: [] };
|
|
1979
|
+
const snapshotManifest = readSnapshotManifest(backupDir, ctx.hqRoot);
|
|
1980
|
+
if (!snapshotManifest) {
|
|
1981
|
+
return {
|
|
1982
|
+
restored: [],
|
|
1983
|
+
failures: [`${backupDir}: snapshot manifest is missing or invalid`],
|
|
1984
|
+
};
|
|
1985
|
+
}
|
|
1653
1986
|
const rels = [];
|
|
1654
1987
|
const seen = new Set();
|
|
1655
1988
|
for (let i = completed.length - 1; i >= 0; i--) {
|
|
@@ -1662,25 +1995,30 @@ function rollbackCompletedActions(ctx, completed, backupDir) {
|
|
|
1662
1995
|
}
|
|
1663
1996
|
}
|
|
1664
1997
|
}
|
|
1665
|
-
const
|
|
1998
|
+
const restoreRels = [];
|
|
1666
1999
|
for (const rel of rels) {
|
|
1667
2000
|
const snapshotPath = path.join(backupDir, rel);
|
|
1668
|
-
|
|
2001
|
+
const hasRecordedLink = (snapshotManifest.symlinks ?? []).some((link) => link.path === rel || link.path.startsWith(`${rel}/`));
|
|
2002
|
+
if (!lexists(snapshotPath) && !hasRecordedLink)
|
|
1669
2003
|
continue;
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
}
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
2004
|
+
restoreRels.push(rel);
|
|
2005
|
+
}
|
|
2006
|
+
if (restoreRels.length === 0)
|
|
2007
|
+
return { restored: [], failures: [] };
|
|
2008
|
+
try {
|
|
2009
|
+
restoreSnapshotPaths(backupDir, ctx.hqRoot, restoreRels, snapshotManifest);
|
|
2010
|
+
return { restored: restoreRels, failures: [] };
|
|
2011
|
+
}
|
|
2012
|
+
catch (error) {
|
|
2013
|
+
return {
|
|
2014
|
+
restored: [],
|
|
2015
|
+
failures: [
|
|
2016
|
+
`${restoreRels.join(", ")}: ${error instanceof Error ? error.message : String(error)}`,
|
|
2017
|
+
],
|
|
2018
|
+
};
|
|
1680
2019
|
}
|
|
1681
|
-
return restored;
|
|
1682
2020
|
}
|
|
1683
|
-
function writeApplyFailureManifest(ctx, backupDir, failedAction, completed, restored, err) {
|
|
2021
|
+
function writeApplyFailureManifest(ctx, backupDir, failedAction, completed, restored, rollbackFailures, err) {
|
|
1684
2022
|
const message = err instanceof Error ? err.message : String(err);
|
|
1685
2023
|
const completedLines = completed.length === 0
|
|
1686
2024
|
? [" (none)"]
|
|
@@ -1730,11 +2068,18 @@ function writeApplyFailureManifest(ctx, backupDir, failedAction, completed, rest
|
|
|
1730
2068
|
for (const rel of unrestored)
|
|
1731
2069
|
lines.push(` - ${rel}`);
|
|
1732
2070
|
}
|
|
2071
|
+
if (rollbackFailures.length > 0) {
|
|
2072
|
+
lines.push("", "Rollback failures:");
|
|
2073
|
+
for (const failure of rollbackFailures)
|
|
2074
|
+
lines.push(` - ${failure}`);
|
|
2075
|
+
}
|
|
1733
2076
|
if (backupDir && isDir(backupDir)) {
|
|
1734
2077
|
lines.push("");
|
|
1735
|
-
|
|
1736
|
-
lines.push(
|
|
1737
|
-
lines.push(`
|
|
2078
|
+
const snapshotName = path.basename(backupDir);
|
|
2079
|
+
lines.push("Restore every captured root and recorded symlink:");
|
|
2080
|
+
lines.push(` ${formatSnapshotRestoreCommand(ctx.hqRoot, ctx.cfg.backupRoot, snapshotName)}`);
|
|
2081
|
+
lines.push("Then verify before retrying:");
|
|
2082
|
+
lines.push(` ${formatRecoveryCommand(ctx.hqRoot, ctx.cfg.backupRoot, snapshotName)}`);
|
|
1738
2083
|
}
|
|
1739
2084
|
else {
|
|
1740
2085
|
lines.push("", "No pre-op safety snapshot was available for automatic restore.");
|
|
@@ -3028,6 +3373,22 @@ export function formatRecoveryCommand(hqRoot, backupRoot, verifyRestoredSnapshot
|
|
|
3028
3373
|
}
|
|
3029
3374
|
return args.join(" ");
|
|
3030
3375
|
}
|
|
3376
|
+
/** Format the manifest-aware command that restores links as well as files. */
|
|
3377
|
+
export function formatSnapshotRestoreCommand(hqRoot, backupRoot, snapshotName) {
|
|
3378
|
+
return [
|
|
3379
|
+
"npx",
|
|
3380
|
+
"-y",
|
|
3381
|
+
"--package=@indigoai-us/hq-cloud@latest",
|
|
3382
|
+
"hq-rescue",
|
|
3383
|
+
"--recover-snapshots",
|
|
3384
|
+
"--hq-root",
|
|
3385
|
+
posixShellQuote(hqRoot),
|
|
3386
|
+
"--backup-dir",
|
|
3387
|
+
posixShellQuote(backupRoot),
|
|
3388
|
+
"--restore-snapshot",
|
|
3389
|
+
posixShellQuote(snapshotName),
|
|
3390
|
+
].join(" ");
|
|
3391
|
+
}
|
|
3031
3392
|
function posixRelFromRoot(hqRoot, abs) {
|
|
3032
3393
|
const root = path.resolve(hqRoot);
|
|
3033
3394
|
const resolved = path.resolve(abs);
|
|
@@ -3203,7 +3564,7 @@ function consumePostRestorePermit(cfg, hqRoot) {
|
|
|
3203
3564
|
if (!entry || !unresolved) {
|
|
3204
3565
|
return { used: false, problem: "referenced snapshot is no longer unresolved recovery evidence" };
|
|
3205
3566
|
}
|
|
3206
|
-
const mismatch = snapshotTreeMismatch(path.join(cfg.backupRoot, entry.name), hqRoot, entry.manifest.includedRoots);
|
|
3567
|
+
const mismatch = snapshotTreeMismatch(path.join(cfg.backupRoot, entry.name), hqRoot, entry.manifest.includedRoots, entry.manifest.symlinks);
|
|
3207
3568
|
if (mismatch) {
|
|
3208
3569
|
return { used: false, problem: `live tree no longer matches ${entry.name}: ${mismatch}` };
|
|
3209
3570
|
}
|
|
@@ -3262,13 +3623,22 @@ function snapshotManifest(status, hqRoot, runId, cfg, sourceSha, includedRoots)
|
|
|
3262
3623
|
sourceSha,
|
|
3263
3624
|
createdAt: now,
|
|
3264
3625
|
includedRoots,
|
|
3626
|
+
symlinks: [],
|
|
3265
3627
|
fileCount: 0,
|
|
3266
3628
|
byteCount: 0,
|
|
3267
3629
|
};
|
|
3268
3630
|
}
|
|
3269
|
-
function readSnapshotManifest(backupDir) {
|
|
3631
|
+
function readSnapshotManifest(backupDir, expectedHqRoot) {
|
|
3270
3632
|
try {
|
|
3271
|
-
|
|
3633
|
+
const parsed = JSON.parse(fs.readFileSync(path.join(backupDir, "SNAPSHOT.json"), "utf8"));
|
|
3634
|
+
const manifestRoot = expectedHqRoot ??
|
|
3635
|
+
(parsed && typeof parsed === "object"
|
|
3636
|
+
? parsed.canonicalHqRoot
|
|
3637
|
+
: undefined);
|
|
3638
|
+
return typeof manifestRoot === "string" &&
|
|
3639
|
+
isManagedSnapshotManifest(parsed, manifestRoot)
|
|
3640
|
+
? parsed
|
|
3641
|
+
: null;
|
|
3272
3642
|
}
|
|
3273
3643
|
catch {
|
|
3274
3644
|
return null;
|
|
@@ -3281,7 +3651,7 @@ function writeSnapshotManifest(backupDir, manifest) {
|
|
|
3281
3651
|
fs.writeFileSync(temp, JSON.stringify(manifest, null, 2) + "\n");
|
|
3282
3652
|
fs.renameSync(temp, target);
|
|
3283
3653
|
}
|
|
3284
|
-
function updateSnapshotStatus(backupDir, status, completedAt) {
|
|
3654
|
+
function updateSnapshotStatus(backupDir, status, completedAt, details = {}) {
|
|
3285
3655
|
const manifest = readSnapshotManifest(backupDir);
|
|
3286
3656
|
if (!manifest)
|
|
3287
3657
|
return;
|
|
@@ -3289,6 +3659,7 @@ function updateSnapshotStatus(backupDir, status, completedAt) {
|
|
|
3289
3659
|
writeSnapshotManifest(backupDir, {
|
|
3290
3660
|
...manifest,
|
|
3291
3661
|
status,
|
|
3662
|
+
...details,
|
|
3292
3663
|
fileCount: stats.files,
|
|
3293
3664
|
byteCount: stats.bytes,
|
|
3294
3665
|
...(completedAt ? { completedAt: new Date().toISOString() } : {}),
|
|
@@ -3445,22 +3816,206 @@ function cpATo(src, dest) {
|
|
|
3445
3816
|
/* ignore */
|
|
3446
3817
|
}
|
|
3447
3818
|
}
|
|
3819
|
+
class SnapshotCopyError extends Error {
|
|
3820
|
+
relativePath;
|
|
3821
|
+
operation;
|
|
3822
|
+
path;
|
|
3823
|
+
code;
|
|
3824
|
+
constructor(relativePath, operation, error, path = relativePath) {
|
|
3825
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
3826
|
+
super(message);
|
|
3827
|
+
this.relativePath = relativePath;
|
|
3828
|
+
this.operation = operation;
|
|
3829
|
+
this.path = path;
|
|
3830
|
+
this.name = "SnapshotCopyError";
|
|
3831
|
+
this.code =
|
|
3832
|
+
error?.code ?? "UNKNOWN";
|
|
3833
|
+
}
|
|
3834
|
+
}
|
|
3835
|
+
function snapshotCopyFailure(error) {
|
|
3836
|
+
return {
|
|
3837
|
+
relativePath: error.relativePath,
|
|
3838
|
+
operation: error.operation,
|
|
3839
|
+
code: error.code,
|
|
3840
|
+
message: error.message,
|
|
3841
|
+
occurredAt: new Date().toISOString(),
|
|
3842
|
+
};
|
|
3843
|
+
}
|
|
3844
|
+
function reportSnapshotCopyFailure(error, err) {
|
|
3845
|
+
const kind = error.operation === "read"
|
|
3846
|
+
? "snapshot-copy-unreadable"
|
|
3847
|
+
: "snapshot-copy-failed";
|
|
3848
|
+
err(`HQ_RESCUE_FAILURE_KIND=${kind}\n`);
|
|
3849
|
+
err(`HQ_RESCUE_SNAPSHOT_COPY_CODE=${error.code}\n`);
|
|
3850
|
+
const verb = error.operation === "metadata" ? "safely record" : error.operation;
|
|
3851
|
+
const detail = error.operation === "metadata" ? `: ${error.message}` : "";
|
|
3852
|
+
err(`error: safety snapshot could not ${verb} ${error.path} (${error.code})${detail}.\n`);
|
|
3853
|
+
if (error.operation === "read") {
|
|
3854
|
+
err(" HQ did not begin the update. If this file is stored in iCloud Drive or another cloud provider, download it locally, then retry.\n");
|
|
3855
|
+
}
|
|
3856
|
+
else {
|
|
3857
|
+
err(" HQ did not begin the update. Repair access to the reported path or backup location, then retry.\n");
|
|
3858
|
+
}
|
|
3859
|
+
}
|
|
3860
|
+
function pathIsInside(root, candidate, allowRoot = false) {
|
|
3861
|
+
const relative = path.relative(root, candidate);
|
|
3862
|
+
return ((allowRoot || relative !== "") &&
|
|
3863
|
+
!relative.startsWith(`..${path.sep}`) &&
|
|
3864
|
+
relative !== ".." &&
|
|
3865
|
+
!path.isAbsolute(relative));
|
|
3866
|
+
}
|
|
3867
|
+
function snapshotRelativePath(hqRoot, rel) {
|
|
3868
|
+
if (!rel ||
|
|
3869
|
+
rel.includes("\0") ||
|
|
3870
|
+
rel.includes("\\") ||
|
|
3871
|
+
path.posix.isAbsolute(rel) ||
|
|
3872
|
+
path.win32.isAbsolute(rel) ||
|
|
3873
|
+
rel.split("/").some((segment) => !segment || segment === "." || segment === "..")) {
|
|
3874
|
+
return null;
|
|
3875
|
+
}
|
|
3876
|
+
const absolute = path.resolve(hqRoot, rel);
|
|
3877
|
+
return pathIsInside(hqRoot, absolute) ? absolute : null;
|
|
3878
|
+
}
|
|
3879
|
+
function restoreParentIsSafe(hqRoot, destination) {
|
|
3880
|
+
for (let current = path.dirname(destination); current !== hqRoot; current = path.dirname(current)) {
|
|
3881
|
+
const st = lstatOrNull(current);
|
|
3882
|
+
if (st?.isSymbolicLink() || (st && !st.isDirectory()))
|
|
3883
|
+
return false;
|
|
3884
|
+
const parent = path.dirname(current);
|
|
3885
|
+
if (parent === current)
|
|
3886
|
+
return false;
|
|
3887
|
+
}
|
|
3888
|
+
return true;
|
|
3889
|
+
}
|
|
3890
|
+
function validateSnapshotSymlink(hqRoot, rel, target) {
|
|
3891
|
+
const linkPath = snapshotRelativePath(hqRoot, rel);
|
|
3892
|
+
if (!linkPath)
|
|
3893
|
+
return `invalid link path ${JSON.stringify(rel)}`;
|
|
3894
|
+
if (typeof target !== "string" || !target || target.includes("\0")) {
|
|
3895
|
+
return `invalid target for ${rel}`;
|
|
3896
|
+
}
|
|
3897
|
+
const resolvedTarget = path.resolve(path.dirname(linkPath), target);
|
|
3898
|
+
return pathIsInside(hqRoot, resolvedTarget, true)
|
|
3899
|
+
? null
|
|
3900
|
+
: `target for ${rel} resolves outside the HQ root`;
|
|
3901
|
+
}
|
|
3902
|
+
function copyRegularFileForSnapshot(src, dest, rel, mode) {
|
|
3903
|
+
let sourceFd;
|
|
3904
|
+
let destinationFd;
|
|
3905
|
+
try {
|
|
3906
|
+
try {
|
|
3907
|
+
sourceFd = fs.openSync(src, "r");
|
|
3908
|
+
}
|
|
3909
|
+
catch (error) {
|
|
3910
|
+
throw new SnapshotCopyError(rel, "read", error, src);
|
|
3911
|
+
}
|
|
3912
|
+
try {
|
|
3913
|
+
destinationFd = fs.openSync(dest, "w", mode);
|
|
3914
|
+
}
|
|
3915
|
+
catch (error) {
|
|
3916
|
+
throw new SnapshotCopyError(rel, "write", error, dest);
|
|
3917
|
+
}
|
|
3918
|
+
const buffer = Buffer.allocUnsafe(64 * 1024);
|
|
3919
|
+
for (;;) {
|
|
3920
|
+
let read;
|
|
3921
|
+
try {
|
|
3922
|
+
read = fs.readSync(sourceFd, buffer, 0, buffer.length, null);
|
|
3923
|
+
}
|
|
3924
|
+
catch (error) {
|
|
3925
|
+
throw new SnapshotCopyError(rel, "read", error, src);
|
|
3926
|
+
}
|
|
3927
|
+
if (read === 0)
|
|
3928
|
+
break;
|
|
3929
|
+
let offset = 0;
|
|
3930
|
+
while (offset < read) {
|
|
3931
|
+
let written;
|
|
3932
|
+
try {
|
|
3933
|
+
written = fs.writeSync(destinationFd, buffer, offset, read - offset);
|
|
3934
|
+
}
|
|
3935
|
+
catch (error) {
|
|
3936
|
+
throw new SnapshotCopyError(rel, "write", error, dest);
|
|
3937
|
+
}
|
|
3938
|
+
if (written === 0) {
|
|
3939
|
+
throw new SnapshotCopyError(rel, "write", new Error("write returned zero bytes"));
|
|
3940
|
+
}
|
|
3941
|
+
offset += written;
|
|
3942
|
+
}
|
|
3943
|
+
}
|
|
3944
|
+
}
|
|
3945
|
+
finally {
|
|
3946
|
+
if (destinationFd !== undefined)
|
|
3947
|
+
fs.closeSync(destinationFd);
|
|
3948
|
+
if (sourceFd !== undefined)
|
|
3949
|
+
fs.closeSync(sourceFd);
|
|
3950
|
+
}
|
|
3951
|
+
}
|
|
3952
|
+
/** Preserve a Windows file link as ordinary file data when links are forbidden. */
|
|
3953
|
+
function copySymlinkFileRepresentationForSnapshot(src, dest, rel) {
|
|
3954
|
+
let targetStats;
|
|
3955
|
+
try {
|
|
3956
|
+
targetStats = fs.statSync(src);
|
|
3957
|
+
}
|
|
3958
|
+
catch (error) {
|
|
3959
|
+
throw new SnapshotCopyError(rel, "read", error, src);
|
|
3960
|
+
}
|
|
3961
|
+
if (!targetStats.isFile()) {
|
|
3962
|
+
throw new SnapshotCopyError(rel, "metadata", new Error("file-link fallback requires a regular file target"), src);
|
|
3963
|
+
}
|
|
3964
|
+
try {
|
|
3965
|
+
fs.mkdirSync(path.dirname(dest), { recursive: true });
|
|
3966
|
+
}
|
|
3967
|
+
catch (error) {
|
|
3968
|
+
throw new SnapshotCopyError(rel, "write", error, dest);
|
|
3969
|
+
}
|
|
3970
|
+
copyRegularFileForSnapshot(src, dest, rel, targetStats.mode);
|
|
3971
|
+
preserveSnapshotMetadata(dest, targetStats);
|
|
3972
|
+
}
|
|
3448
3973
|
/** Snapshot copy that preserves metadata while pruning canonical transient directories. */
|
|
3449
|
-
function cpAToFiltered(src, dest, rel) {
|
|
3450
|
-
|
|
3451
|
-
|
|
3452
|
-
|
|
3974
|
+
function cpAToFiltered(src, dest, rel, hqRoot, symlinks) {
|
|
3975
|
+
let st;
|
|
3976
|
+
try {
|
|
3977
|
+
st = fs.lstatSync(src);
|
|
3978
|
+
}
|
|
3979
|
+
catch (error) {
|
|
3980
|
+
throw new SnapshotCopyError(rel, "read", error, src);
|
|
3981
|
+
}
|
|
3453
3982
|
if (st.isDirectory() && isRescueTransientDirectory(rel))
|
|
3454
3983
|
return;
|
|
3455
3984
|
if (st.isSymbolicLink()) {
|
|
3456
|
-
|
|
3457
|
-
|
|
3985
|
+
let target;
|
|
3986
|
+
try {
|
|
3987
|
+
target = fs.readlinkSync(src);
|
|
3988
|
+
}
|
|
3989
|
+
catch (error) {
|
|
3990
|
+
throw new SnapshotCopyError(rel, "read", error, src);
|
|
3991
|
+
}
|
|
3992
|
+
const problem = validateSnapshotSymlink(hqRoot, rel, target);
|
|
3993
|
+
if (problem) {
|
|
3994
|
+
throw new SnapshotCopyError(rel, "metadata", new Error(problem), src);
|
|
3995
|
+
}
|
|
3996
|
+
const representation = snapshotSymlinkRepresentation(hqRoot, src, dest, rel, target);
|
|
3997
|
+
if (representation === "file-copy") {
|
|
3998
|
+
copySymlinkFileRepresentationForSnapshot(src, dest, rel);
|
|
3999
|
+
}
|
|
4000
|
+
symlinks.push({ path: rel, target, ...(representation ? { representation } : {}) });
|
|
3458
4001
|
return;
|
|
3459
4002
|
}
|
|
3460
4003
|
if (st.isDirectory()) {
|
|
3461
|
-
|
|
3462
|
-
|
|
3463
|
-
|
|
4004
|
+
try {
|
|
4005
|
+
fs.mkdirSync(dest, { recursive: true });
|
|
4006
|
+
}
|
|
4007
|
+
catch (error) {
|
|
4008
|
+
throw new SnapshotCopyError(rel, "write", error, dest);
|
|
4009
|
+
}
|
|
4010
|
+
let names;
|
|
4011
|
+
try {
|
|
4012
|
+
names = fs.readdirSync(src);
|
|
4013
|
+
}
|
|
4014
|
+
catch (error) {
|
|
4015
|
+
throw new SnapshotCopyError(rel, "read", error, src);
|
|
4016
|
+
}
|
|
4017
|
+
for (const name of names) {
|
|
4018
|
+
cpAToFiltered(path.join(src, name), path.join(dest, name), path.posix.join(rel, name), hqRoot, symlinks);
|
|
3464
4019
|
}
|
|
3465
4020
|
try {
|
|
3466
4021
|
fs.chmodSync(dest, st.mode);
|
|
@@ -3471,8 +4026,16 @@ function cpAToFiltered(src, dest, rel) {
|
|
|
3471
4026
|
}
|
|
3472
4027
|
return;
|
|
3473
4028
|
}
|
|
3474
|
-
|
|
3475
|
-
|
|
4029
|
+
if (!st.isFile()) {
|
|
4030
|
+
throw new SnapshotCopyError(rel, "metadata", new Error("unsupported filesystem entry"));
|
|
4031
|
+
}
|
|
4032
|
+
try {
|
|
4033
|
+
fs.mkdirSync(path.dirname(dest), { recursive: true });
|
|
4034
|
+
}
|
|
4035
|
+
catch (error) {
|
|
4036
|
+
throw new SnapshotCopyError(rel, "write", error, dest);
|
|
4037
|
+
}
|
|
4038
|
+
copyRegularFileForSnapshot(src, dest, rel, st.mode);
|
|
3476
4039
|
try {
|
|
3477
4040
|
fs.chmodSync(dest, st.mode);
|
|
3478
4041
|
fs.utimesSync(dest, st.atime, st.mtime);
|