@indigoai-us/hq-cloud 6.16.50 → 6.16.51

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -31,7 +31,7 @@ import * as fs from "fs";
31
31
  import * as os from "os";
32
32
  import * as path from "path";
33
33
  import { assertRescuePathTypeUnchanged, isRescueTransientDirectory, } from "./rescue-snapshot.js";
34
- import { applyBackupPrune, canonicalRootHash, inventoryManagedSnapshots, } from "../backup-prune.js";
34
+ import { applyBackupPrune, canonicalRootHash, inventoryManagedSnapshots, isManagedSnapshotManifest, } from "../backup-prune.js";
35
35
  // Paths always preserved across wipe+overlay (shuttled out, restored after).
36
36
  const CARVE_OUT_PATHS = [
37
37
  "core/packages",
@@ -329,6 +329,7 @@ export function parseRescueArgs(argv) {
329
329
  backupRetentionCount: process.env.HQ_BACKUP_RETENTION_COUNT || "2",
330
330
  recoverSnapshots: false,
331
331
  verifyRestoredSnapshot: "",
332
+ restoreSnapshot: "",
332
333
  };
333
334
  const usage = () => {
334
335
  // The legacy script printed its comment header here; we emit a concise
@@ -338,7 +339,8 @@ export function parseRescueArgs(argv) {
338
339
  " [--hq-root DIR] [--no-history-check] [--floor-sha SHA]\n" +
339
340
  " [--no-backup] [--backup-dir DIR] [--cloud-update]\n" +
340
341
  " [--dry-run] [--yes] [--recover-snapshots]\n" +
341
- " [--verify-restored-snapshot SNAPSHOT]\n");
342
+ " [--verify-restored-snapshot SNAPSHOT]\n" +
343
+ " [--restore-snapshot SNAPSHOT]\n");
342
344
  throw new ExitError(1);
343
345
  };
344
346
  let narrowCsv = "";
@@ -397,6 +399,13 @@ export function parseRescueArgs(argv) {
397
399
  throw new ExitError(2);
398
400
  }
399
401
  break;
402
+ case "--restore-snapshot":
403
+ cfg.restoreSnapshot = need(a);
404
+ if (!cfg.restoreSnapshot) {
405
+ process.stderr.write("error: --restore-snapshot requires a snapshot name.\n");
406
+ throw new ExitError(2);
407
+ }
408
+ break;
400
409
  case "--dry-run":
401
410
  cfg.dryRun = true;
402
411
  break;
@@ -461,6 +470,14 @@ export function parseRescueArgs(argv) {
461
470
  process.stderr.write("error: --verify-restored-snapshot requires --recover-snapshots.\n");
462
471
  throw new ExitError(2);
463
472
  }
473
+ if (cfg.restoreSnapshot && !cfg.recoverSnapshots) {
474
+ process.stderr.write("error: --restore-snapshot requires --recover-snapshots.\n");
475
+ throw new ExitError(2);
476
+ }
477
+ if (cfg.restoreSnapshot && cfg.verifyRestoredSnapshot) {
478
+ process.stderr.write("error: --restore-snapshot and --verify-restored-snapshot must be run separately.\n");
479
+ throw new ExitError(2);
480
+ }
464
481
  return cfg;
465
482
  }
466
483
  /**
@@ -589,6 +606,9 @@ function recoverSnapshots(cfg, out, err) {
589
606
  err(`error: cannot inspect safety snapshots: backup root is unreadable: ${cfg.backupRoot}\n`);
590
607
  return { status: 1 };
591
608
  }
609
+ if (cfg.restoreSnapshot) {
610
+ return restoreRecoverySnapshot(cfg, hqRoot, inventory.entries, inventory.protected, out, err);
611
+ }
592
612
  if (cfg.verifyRestoredSnapshot) {
593
613
  return verifyRestoredSnapshot(cfg, hqRoot, inventory.entries, inventory.protected, out, err);
594
614
  }
@@ -645,15 +665,44 @@ function recoverSnapshots(cfg, out, err) {
645
665
  if (unresolved.length > 0) {
646
666
  err("error: snapshot recovery refused: the interrupted updates are not proven superseded.\n");
647
667
  err(" HQ retained every unverified snapshot. A later completed-good rescue for this same HQ root is the only automatic proof.\n");
648
- err(" Restore every captured root from one snapshot, then verify that restored tree before one retry is permitted:\n");
668
+ err(" Restore one listed snapshot, then verify it before one retry is permitted:\n");
649
669
  for (const item of unresolved) {
650
- err(` ${item.entry.name}: ${formatRecoveryCommand(hqRoot, cfg.backupRoot, item.entry.name)}\n`);
670
+ err(` restore ${item.entry.name}: ${formatSnapshotRestoreCommand(hqRoot, cfg.backupRoot, item.entry.name)}\n`);
671
+ err(` verify ${item.entry.name}: ${formatRecoveryCommand(hqRoot, cfg.backupRoot, item.entry.name)}\n`);
651
672
  }
652
673
  return { status: 1 };
653
674
  }
654
675
  out("==> Snapshot recovery complete: every eligible recovery snapshot was positively verified before deletion.\n");
655
676
  return { status: 0 };
656
677
  }
678
+ /**
679
+ * Explicitly restore every captured root, including links represented only in
680
+ * the snapshot manifest. Verification remains a separate positive-proof step.
681
+ */
682
+ function restoreRecoverySnapshot(cfg, hqRoot, entries, protectedEntries, out, err) {
683
+ const entry = entries.find((item) => item.name === cfg.restoreSnapshot);
684
+ const unresolved = protectedEntries.find((item) => item.entry.name === cfg.restoreSnapshot &&
685
+ item.reason === "recovery-required-no-later-good");
686
+ if (!entry || !unresolved || entry.manifest.status !== "recovery-required") {
687
+ err(`error: ${cfg.restoreSnapshot} is not an unresolved recovery-required snapshot for this HQ root.\n`);
688
+ return { status: 1 };
689
+ }
690
+ if (cfg.dryRun) {
691
+ out(`==> Dry run would restore every captured root and recorded symlink from ${entry.name}.\n`);
692
+ return { status: 0 };
693
+ }
694
+ try {
695
+ restoreSnapshotPaths(path.join(cfg.backupRoot, entry.name), hqRoot, entry.manifest.includedRoots, entry.manifest);
696
+ }
697
+ catch (error) {
698
+ err(`error: could not restore safety snapshot ${entry.name}: ${error instanceof Error ? error.message : String(error)}\n`);
699
+ err(" HQ retained the snapshot and did not write a post-restore permit.\n");
700
+ return { status: 1 };
701
+ }
702
+ out(`==> Restored every captured root and recorded symlink from ${entry.name}.\n`);
703
+ out(` Verify the restored tree before retrying: ${formatRecoveryCommand(hqRoot, cfg.backupRoot, entry.name)}\n`);
704
+ return { status: 0 };
705
+ }
657
706
  /** A live tree must still match the selected recovery snapshot at permit use. */
658
707
  function verifyRestoredSnapshot(cfg, hqRoot, entries, protectedEntries, out, err) {
659
708
  const entry = entries.find((item) => item.name === cfg.verifyRestoredSnapshot);
@@ -663,7 +712,7 @@ function verifyRestoredSnapshot(cfg, hqRoot, entries, protectedEntries, out, err
663
712
  err(`error: ${cfg.verifyRestoredSnapshot} is not an unresolved recovery-required snapshot for this HQ root.\n`);
664
713
  return { status: 1 };
665
714
  }
666
- const mismatch = snapshotTreeMismatch(path.join(cfg.backupRoot, entry.name), hqRoot, entry.manifest.includedRoots);
715
+ const mismatch = snapshotTreeMismatch(path.join(cfg.backupRoot, entry.name), hqRoot, entry.manifest.includedRoots, entry.manifest.symlinks);
667
716
  if (mismatch) {
668
717
  err(`error: restored tree does not match snapshot ${entry.name}: ${mismatch}\n`);
669
718
  err(" No post-restore permit was written; the circuit breaker remains armed.\n");
@@ -685,13 +734,32 @@ function verifyRestoredSnapshot(cfg, hqRoot, entries, protectedEntries, out, err
685
734
  return { status: 0 };
686
735
  }
687
736
  /** Return a precise mismatch instead of treating a partial restore as evidence. */
688
- function snapshotTreeMismatch(snapshotRoot, hqRoot, includedRoots) {
689
- if (includedRoots.length === 0) {
737
+ function snapshotTreeMismatch(snapshotRoot, hqRoot, includedRoots, symlinks) {
738
+ const roots = snapshotCapturedRoots(includedRoots);
739
+ if (typeof roots === "string")
740
+ return roots;
741
+ const recorded = snapshotSymlinkTargets(hqRoot, roots, symlinks);
742
+ if (typeof recorded === "string")
743
+ return recorded;
744
+ for (const root of roots) {
745
+ if (rescuePathKind(path.join(snapshotRoot, root)) === "missing" &&
746
+ !recorded.has(root)) {
747
+ return `snapshot does not contain captured root ${root}`;
748
+ }
749
+ const mismatch = snapshotPathMismatch(path.join(snapshotRoot, root), path.join(hqRoot, root), root, recorded);
750
+ if (mismatch)
751
+ return mismatch;
752
+ }
753
+ return null;
754
+ }
755
+ function snapshotCapturedRoots(includedRoots) {
756
+ if (!Array.isArray(includedRoots) || includedRoots.length === 0) {
690
757
  return "snapshot manifest records no captured roots to verify";
691
758
  }
692
759
  const seen = new Set();
693
760
  for (const root of includedRoots) {
694
- if (!root ||
761
+ if (typeof root !== "string" ||
762
+ !root ||
695
763
  root === "." ||
696
764
  root === ".." ||
697
765
  root.includes("/") ||
@@ -701,18 +769,76 @@ function snapshotTreeMismatch(snapshotRoot, hqRoot, includedRoots) {
701
769
  return `snapshot manifest has an unsafe captured root ${JSON.stringify(root)}`;
702
770
  }
703
771
  seen.add(root);
704
- if (rescuePathKind(path.join(snapshotRoot, root)) === "missing") {
705
- return `snapshot does not contain captured root ${root}`;
772
+ }
773
+ return seen;
774
+ }
775
+ function snapshotSymlinkTargets(hqRoot, includedRoots, symlinks) {
776
+ if (symlinks !== undefined && !Array.isArray(symlinks)) {
777
+ return "snapshot manifest symlinks is not an array";
778
+ }
779
+ const recorded = new Map();
780
+ for (const link of symlinks ?? []) {
781
+ if (!link ||
782
+ typeof link.path !== "string" ||
783
+ (link.representation !== undefined && link.representation !== "file-copy")) {
784
+ return "snapshot manifest has an invalid recorded symlink";
706
785
  }
707
- const mismatch = snapshotPathMismatch(path.join(snapshotRoot, root), path.join(hqRoot, root), root);
708
- if (mismatch)
709
- return mismatch;
786
+ const problem = validateSnapshotSymlink(hqRoot, link.path, link.target);
787
+ if (problem)
788
+ return `snapshot manifest ${problem}`;
789
+ const root = link.path.split("/")[0];
790
+ if (!includedRoots.has(root)) {
791
+ return `snapshot manifest records symlink ${link.path} outside its captured roots`;
792
+ }
793
+ if (recorded.has(link.path)) {
794
+ return `snapshot manifest records duplicate symlink ${link.path}`;
795
+ }
796
+ recorded.set(link.path, link);
710
797
  }
711
- return null;
798
+ for (const linkPath of recorded.keys()) {
799
+ for (let parent = path.posix.dirname(linkPath); parent !== "."; parent = path.posix.dirname(parent)) {
800
+ if (recorded.has(parent)) {
801
+ return `snapshot manifest records symlink ${linkPath} beneath recorded symlink ${parent}`;
802
+ }
803
+ }
804
+ }
805
+ return recorded;
806
+ }
807
+ function recordedSymlinkTargetMatches(livePath, recordedTarget, liveTarget) {
808
+ const parent = path.dirname(livePath);
809
+ return (path.resolve(parent, recordedTarget) === path.resolve(parent, liveTarget));
712
810
  }
713
- function snapshotPathMismatch(snapshotPath, livePath, rel) {
811
+ function snapshotPathMismatch(snapshotPath, livePath, rel, recordedSymlinks) {
714
812
  const snapshotKind = rescuePathKind(snapshotPath);
715
813
  const liveKind = rescuePathKind(livePath);
814
+ const recordedLink = recordedSymlinks.get(rel);
815
+ if (recordedLink !== undefined) {
816
+ if (recordedLink.representation === "file-copy") {
817
+ if (snapshotKind !== "file") {
818
+ return `${rel} is ${snapshotKind} but the snapshot records a copied file link`;
819
+ }
820
+ if (liveKind !== "file") {
821
+ return `${rel} is ${liveKind} but the snapshot records a copied file link`;
822
+ }
823
+ return bytesEqual(snapshotPath, livePath)
824
+ ? null
825
+ : `${rel} has different file contents`;
826
+ }
827
+ if (snapshotKind !== "missing") {
828
+ return `${rel} is stored as both a snapshot entry and recorded symlink`;
829
+ }
830
+ if (liveKind !== "symlink") {
831
+ return `${rel} is ${liveKind} but the snapshot records a symlink`;
832
+ }
833
+ try {
834
+ return recordedSymlinkTargetMatches(livePath, recordedLink.target, fs.readlinkSync(livePath))
835
+ ? null
836
+ : `${rel} has a different symlink target`;
837
+ }
838
+ catch (error) {
839
+ return `${rel} could not read a symlink target: ${error instanceof Error ? error.message : String(error)}`;
840
+ }
841
+ }
716
842
  if (snapshotKind !== liveKind) {
717
843
  return `${rel} is ${liveKind} but the snapshot records ${snapshotKind}`;
718
844
  }
@@ -751,17 +877,200 @@ function snapshotPathMismatch(snapshotPath, livePath, rel) {
751
877
  catch (error) {
752
878
  return `${rel} could not be read: ${error instanceof Error ? error.message : String(error)}`;
753
879
  }
754
- if (snapshotNames.length !== liveNames.length ||
755
- snapshotNames.some((name, index) => name !== liveNames[index])) {
880
+ const expectedNames = new Set(snapshotNames);
881
+ for (const recordedRel of recordedSymlinks.keys()) {
882
+ if (path.posix.dirname(recordedRel) === rel) {
883
+ expectedNames.add(path.posix.basename(recordedRel));
884
+ }
885
+ }
886
+ const expected = [...expectedNames].sort();
887
+ if (expected.length !== liveNames.length ||
888
+ expected.some((name, index) => name !== liveNames[index])) {
756
889
  return `${rel} has different entries`;
757
890
  }
758
- for (const name of snapshotNames) {
759
- const mismatch = snapshotPathMismatch(path.join(snapshotPath, name), path.join(livePath, name), path.posix.join(rel, name));
891
+ for (const name of expected) {
892
+ const mismatch = snapshotPathMismatch(path.join(snapshotPath, name), path.join(livePath, name), path.posix.join(rel, name), recordedSymlinks);
760
893
  if (mismatch)
761
894
  return mismatch;
762
895
  }
763
896
  return null;
764
897
  }
898
+ /**
899
+ * Restore one captured path, then recreate only manifest-recorded symlinks
900
+ * whose locations and targets are both confined to the HQ root.
901
+ */
902
+ export function restoreSnapshotPath(snapshotDir, hqRoot, rel, manifest) {
903
+ restoreSnapshotPaths(snapshotDir, hqRoot, [rel], manifest);
904
+ }
905
+ /** Restore all requested physical paths before recreating their recorded links. */
906
+ function restoreSnapshotPaths(snapshotDir, hqRoot, rels, manifest) {
907
+ const capturedRoots = snapshotCapturedRoots(manifest.includedRoots);
908
+ if (typeof capturedRoots === "string")
909
+ throw new Error(capturedRoots);
910
+ const recorded = snapshotSymlinkTargets(hqRoot, capturedRoots, manifest.symlinks);
911
+ if (typeof recorded === "string")
912
+ throw new Error(recorded);
913
+ for (const [recordedRel, link] of recorded) {
914
+ const snapshotKind = rescuePathKind(path.join(snapshotDir, recordedRel));
915
+ if (link.representation === "file-copy") {
916
+ if (snapshotKind !== "file") {
917
+ throw new Error(`snapshot records copied file link ${recordedRel} without copied file data`);
918
+ }
919
+ }
920
+ else if (snapshotKind !== "missing") {
921
+ throw new Error(`snapshot records symlink ${recordedRel} alongside copied data`);
922
+ }
923
+ }
924
+ const requested = [...new Set(rels)].sort((left, right) => left.localeCompare(right));
925
+ if (requested.length === 0)
926
+ throw new Error("snapshot restore requested no paths");
927
+ const items = requested.map((rel) => {
928
+ const destination = snapshotRelativePath(hqRoot, rel);
929
+ if (!destination) {
930
+ throw new Error(`invalid snapshot restore path ${JSON.stringify(rel)}`);
931
+ }
932
+ if (!restoreParentIsSafe(hqRoot, destination)) {
933
+ throw new Error(`snapshot restore parent for ${rel} is not a directory inside the HQ root`);
934
+ }
935
+ if (!capturedRoots.has(rel.split("/")[0])) {
936
+ throw new Error(`snapshot does not include ${rel}`);
937
+ }
938
+ const source = path.join(snapshotDir, rel);
939
+ const physicalSourceExists = lexists(source);
940
+ const recordedWithinPath = [...recorded.keys()].some((recordedRel) => recordedRel === rel || recordedRel.startsWith(`${rel}/`));
941
+ if (!physicalSourceExists && !recordedWithinPath) {
942
+ throw new Error(`snapshot does not contain ${rel}`);
943
+ }
944
+ return { rel, source, destination, physicalSourceExists };
945
+ });
946
+ for (const item of items) {
947
+ fs.rmSync(item.destination, { recursive: true, force: true });
948
+ }
949
+ for (const item of items) {
950
+ if (item.physicalSourceExists) {
951
+ copySnapshotPhysicalTree(item.source, item.destination, hqRoot, item.rel);
952
+ }
953
+ }
954
+ const links = orderSnapshotSymlinkRestore(hqRoot, [...recorded.entries()]
955
+ .filter(([recordedRel, link]) => link.representation === undefined &&
956
+ items.some((item) => recordedRel === item.rel || recordedRel.startsWith(`${item.rel}/`)))
957
+ .map(([recordedRel, link]) => [recordedRel, link.target]));
958
+ for (const [linkRel, target] of links) {
959
+ recreateSnapshotSymlink(hqRoot, linkRel, target);
960
+ }
961
+ }
962
+ function orderSnapshotSymlinkRestore(hqRoot, links) {
963
+ const pending = new Map(links);
964
+ const ordered = [];
965
+ while (pending.size > 0) {
966
+ const next = [...pending.entries()]
967
+ .sort(([left], [right]) => left.localeCompare(right))
968
+ .find(([rel, target]) => {
969
+ const linkPath = snapshotRelativePath(hqRoot, rel);
970
+ const resolvedTarget = path.resolve(path.dirname(linkPath), target);
971
+ return ![...pending.keys()].some((candidate) => {
972
+ if (candidate === rel)
973
+ return false;
974
+ const candidatePath = snapshotRelativePath(hqRoot, candidate);
975
+ return (resolvedTarget === candidatePath ||
976
+ resolvedTarget.startsWith(`${candidatePath}${path.sep}`));
977
+ });
978
+ });
979
+ if (!next) {
980
+ ordered.push(...[...pending.entries()].sort(([left], [right]) => left.localeCompare(right)));
981
+ break;
982
+ }
983
+ pending.delete(next[0]);
984
+ ordered.push(next);
985
+ }
986
+ return ordered;
987
+ }
988
+ function copySnapshotPhysicalTree(source, destination, hqRoot, rel) {
989
+ const st = fs.lstatSync(source);
990
+ if (st.isSymbolicLink()) {
991
+ recreateSnapshotSymlink(hqRoot, rel, fs.readlinkSync(source));
992
+ return;
993
+ }
994
+ if (st.isDirectory()) {
995
+ fs.mkdirSync(destination, { recursive: true });
996
+ for (const name of fs.readdirSync(source)) {
997
+ copySnapshotPhysicalTree(path.join(source, name), path.join(destination, name), hqRoot, path.posix.join(rel, name));
998
+ }
999
+ preserveSnapshotMetadata(destination, st);
1000
+ return;
1001
+ }
1002
+ if (!st.isFile())
1003
+ throw new Error(`snapshot contains unsupported entry ${rel}`);
1004
+ fs.mkdirSync(path.dirname(destination), { recursive: true });
1005
+ fs.copyFileSync(source, destination);
1006
+ preserveSnapshotMetadata(destination, st);
1007
+ }
1008
+ /** Data restored successfully remains restored when filesystem metadata is unsupported. */
1009
+ function preserveSnapshotMetadata(destination, st) {
1010
+ try {
1011
+ fs.chmodSync(destination, st.mode);
1012
+ fs.utimesSync(destination, st.atime, st.mtime);
1013
+ }
1014
+ catch {
1015
+ /* metadata preservation is best-effort, matching the prior rollback helper */
1016
+ }
1017
+ }
1018
+ function recreateSnapshotSymlink(hqRoot, rel, target, destination = snapshotRelativePath(hqRoot, rel)) {
1019
+ const problem = validateSnapshotSymlink(hqRoot, rel, target);
1020
+ if (problem)
1021
+ throw new Error(`invalid snapshot symlink: ${problem}`);
1022
+ if (!destination)
1023
+ throw new Error(`invalid snapshot symlink destination ${rel}`);
1024
+ const restoredLinkPath = snapshotRelativePath(hqRoot, rel);
1025
+ const resolvedTarget = path.resolve(path.dirname(restoredLinkPath), target);
1026
+ fs.mkdirSync(path.dirname(destination), { recursive: true });
1027
+ fs.rmSync(destination, { recursive: true, force: true });
1028
+ try {
1029
+ fs.symlinkSync(target, destination);
1030
+ }
1031
+ catch (error) {
1032
+ if (process.platform === "win32" &&
1033
+ error.code === "EPERM" &&
1034
+ isDir(resolvedTarget)) {
1035
+ fs.symlinkSync(resolvedTarget, destination, "junction");
1036
+ return;
1037
+ }
1038
+ throw error;
1039
+ }
1040
+ }
1041
+ /**
1042
+ * Directory links can become junctions on Windows. A file link without link
1043
+ * privilege instead needs copied file data at its original snapshot path.
1044
+ */
1045
+ function snapshotSymlinkRepresentation(hqRoot, source, destination, rel, target) {
1046
+ if (process.platform !== "win32")
1047
+ return undefined;
1048
+ const probe = `${destination}.hq-restore-probe-${process.pid}-${Date.now().toString(36)}`;
1049
+ try {
1050
+ recreateSnapshotSymlink(hqRoot, rel, target, probe);
1051
+ fs.rmSync(probe, { recursive: true, force: true });
1052
+ return undefined;
1053
+ }
1054
+ catch (error) {
1055
+ try {
1056
+ fs.rmSync(probe, { recursive: true, force: true });
1057
+ }
1058
+ catch (cleanupError) {
1059
+ throw new SnapshotCopyError(rel, "write", cleanupError, probe);
1060
+ }
1061
+ if (process.platform === "win32" &&
1062
+ error.code === "EPERM") {
1063
+ try {
1064
+ if (fs.statSync(source).isFile())
1065
+ return "file-copy";
1066
+ }
1067
+ catch (targetError) {
1068
+ throw new SnapshotCopyError(rel, "read", targetError, source);
1069
+ }
1070
+ }
1071
+ throw new SnapshotCopyError(rel, "metadata", new Error(`recorded symlink cannot be restored on this machine: ${error instanceof Error ? error.message : String(error)}`), source);
1072
+ }
1073
+ }
765
1074
  /**
766
1075
  * True when `root` is usable as a rescue target: an absolute path that is not
767
1076
  * a bare drive letter. Pure + exported for tests; `platformPath` defaults to
@@ -1276,7 +1585,7 @@ function doRescue(cfg, env, out, err, setTmp) {
1276
1585
  err(" HQ kept their pre-update snapshots because those updates started changing the release-managed tree but did not finish.\n");
1277
1586
  err(" It will not overwrite or delete that evidence without proof that a later rescue completed successfully.\n");
1278
1587
  err(` Inspect safely: ${formatRecoveryCommand(hqRoot, cfg.backupRoot)}\n`);
1279
- err(" It deletes only snapshots with a later completed-good rescue. For unresolved snapshots, restore one listed snapshot and run its verification command before retrying.\n");
1588
+ err(" It deletes only snapshots with a later completed-good rescue. For unresolved snapshots, use --restore-snapshot to restore every captured root and recorded link, then run its verification command before retrying.\n");
1280
1589
  throw new ExitError(1);
1281
1590
  }
1282
1591
  }
@@ -1286,23 +1595,36 @@ function doRescue(cfg, env, out, err, setTmp) {
1286
1595
  backupDir = allocateSnapshotDir(cfg.backupRoot, RUN_TS, canonicalRootHash(hqRoot), runId);
1287
1596
  out("\n");
1288
1597
  out(`==> Safety snapshot -> ${backupDir}\n`);
1598
+ const snapshotSymlinks = [];
1289
1599
  try {
1290
1600
  fs.mkdirSync(backupDir, { recursive: false });
1291
1601
  writeSnapshotManifest(backupDir, snapshotManifest("copying", hqRoot, runId, cfg, srcSha, wipeToplevel));
1292
1602
  for (const rootRel of wipeToplevel) {
1293
1603
  const srcAbs = path.join(hqRoot, rootRel);
1294
- if (!lexists(srcAbs) || isRescueTransientDirectory(rootRel))
1604
+ if (isRescueTransientDirectory(rootRel))
1295
1605
  continue;
1296
- cpAToFiltered(srcAbs, path.join(backupDir, rootRel), rootRel);
1606
+ cpAToFiltered(srcAbs, path.join(backupDir, rootRel), rootRel, hqRoot, snapshotSymlinks);
1297
1607
  }
1298
- updateSnapshotStatus(backupDir, "ready");
1608
+ updateSnapshotStatus(backupDir, "ready", undefined, {
1609
+ symlinks: snapshotSymlinks,
1610
+ });
1299
1611
  }
1300
1612
  catch (error) {
1301
- updateSnapshotStatus(backupDir, "failed-pre-mutation");
1613
+ if (error instanceof SnapshotCopyError) {
1614
+ updateSnapshotStatus(backupDir, "failed-pre-mutation", undefined, {
1615
+ symlinks: snapshotSymlinks,
1616
+ copyFailure: snapshotCopyFailure(error),
1617
+ });
1618
+ reportSnapshotCopyFailure(error, err);
1619
+ throw new ExitError(1);
1620
+ }
1621
+ updateSnapshotStatus(backupDir, "failed-pre-mutation", undefined, {
1622
+ symlinks: snapshotSymlinks,
1623
+ });
1302
1624
  throw error;
1303
1625
  }
1304
1626
  pruneManagedBackups(cfg, hqRoot, out, true);
1305
- out(` snapshot ready (restore any file: cp "${backupDir}/<relpath>" "${hqRoot}/<relpath>")\n`);
1627
+ out(" snapshot ready (complete recovery is available through --recover-snapshots).\n");
1306
1628
  }
1307
1629
  else if (cfg.doBackup && !cfg.dryRun && wipeToplevel.length !== 0) {
1308
1630
  out("\n==> Safety snapshot skipped (classification found no destructive paths).\n");
@@ -1547,10 +1869,11 @@ function doRescue(cfg, env, out, err, setTmp) {
1547
1869
  }
1548
1870
  lines.push("", "## Restore");
1549
1871
  lines.push("This directory holds the release-managed wipe set as it was before the update; reproducible worktrees and build caches are intentionally excluded.");
1550
- lines.push("Restore a single file:");
1551
- lines.push(` cp "${backupDir}/<relpath>" "${hqRoot}/<relpath>"`);
1552
- lines.push("Restore everything (overwrites current scaffold — use with care):");
1553
- lines.push(` rsync -a "${backupDir}/" "${hqRoot}/"`);
1872
+ const snapshotName = path.basename(backupDir);
1873
+ lines.push("Restore every captured root and recorded symlink:");
1874
+ lines.push(` ${formatSnapshotRestoreCommand(hqRoot, cfg.backupRoot, snapshotName)}`);
1875
+ lines.push("Then verify the restored tree before retrying:");
1876
+ lines.push(` ${formatRecoveryCommand(hqRoot, cfg.backupRoot, snapshotName)}`);
1554
1877
  lines.push("", `Retention: ${cfg.backupRetentionDays} days and ${cfg.backupRetentionCount} completed snapshots (latest recovery floor retained).`);
1555
1878
  try {
1556
1879
  fs.writeFileSync(path.join(backupDir, "RECOVERY.md"), lines.join("\n") + "\n");
@@ -1618,12 +1941,15 @@ function applyClassifiedActions(ctx, backupDir) {
1618
1941
  completed.push(action);
1619
1942
  }
1620
1943
  catch (err) {
1621
- const restored = rollbackCompletedActions(ctx, completed, backupDir);
1622
- const manifest = writeApplyFailureManifest(ctx, backupDir, action, completed, restored, err);
1944
+ const rollback = rollbackCompletedActions(ctx, completed, backupDir);
1945
+ const manifest = writeApplyFailureManifest(ctx, backupDir, action, completed, rollback.restored, rollback.failures, err);
1623
1946
  ctx.out("\n");
1624
1947
  ctx.out(`error: rescue apply failed during '${action.label}'.\n`);
1625
- if (restored.length > 0) {
1626
- ctx.out(`==> Rolled back ${restored.length} prior action path(s) from the safety snapshot.\n`);
1948
+ if (rollback.restored.length > 0) {
1949
+ ctx.out(`==> Rolled back ${rollback.restored.length} prior action path(s) from the safety snapshot.\n`);
1950
+ }
1951
+ for (const failure of rollback.failures) {
1952
+ ctx.out(`warning: rollback retained recovery evidence: ${failure}\n`);
1627
1953
  }
1628
1954
  if (manifest) {
1629
1955
  ctx.out(`==> Recovery manifest: ${manifest}\n`);
@@ -1649,7 +1975,14 @@ function injectApplyPathChangeForTest(ctx) {
1649
1975
  }
1650
1976
  function rollbackCompletedActions(ctx, completed, backupDir) {
1651
1977
  if (!backupDir || !isDir(backupDir))
1652
- return [];
1978
+ return { restored: [], failures: [] };
1979
+ const snapshotManifest = readSnapshotManifest(backupDir, ctx.hqRoot);
1980
+ if (!snapshotManifest) {
1981
+ return {
1982
+ restored: [],
1983
+ failures: [`${backupDir}: snapshot manifest is missing or invalid`],
1984
+ };
1985
+ }
1653
1986
  const rels = [];
1654
1987
  const seen = new Set();
1655
1988
  for (let i = completed.length - 1; i >= 0; i--) {
@@ -1662,25 +1995,30 @@ function rollbackCompletedActions(ctx, completed, backupDir) {
1662
1995
  }
1663
1996
  }
1664
1997
  }
1665
- const restored = [];
1998
+ const restoreRels = [];
1666
1999
  for (const rel of rels) {
1667
2000
  const snapshotPath = path.join(backupDir, rel);
1668
- if (!lexists(snapshotPath))
2001
+ const hasRecordedLink = (snapshotManifest.symlinks ?? []).some((link) => link.path === rel || link.path.startsWith(`${rel}/`));
2002
+ if (!lexists(snapshotPath) && !hasRecordedLink)
1669
2003
  continue;
1670
- const dest = path.join(ctx.hqRoot, rel);
1671
- try {
1672
- fs.mkdirSync(path.dirname(dest), { recursive: true });
1673
- fs.rmSync(dest, { recursive: true, force: true });
1674
- cpATo(snapshotPath, dest);
1675
- restored.push(rel);
1676
- }
1677
- catch {
1678
- // Recovery manifest below records any path that could not be restored.
1679
- }
2004
+ restoreRels.push(rel);
2005
+ }
2006
+ if (restoreRels.length === 0)
2007
+ return { restored: [], failures: [] };
2008
+ try {
2009
+ restoreSnapshotPaths(backupDir, ctx.hqRoot, restoreRels, snapshotManifest);
2010
+ return { restored: restoreRels, failures: [] };
2011
+ }
2012
+ catch (error) {
2013
+ return {
2014
+ restored: [],
2015
+ failures: [
2016
+ `${restoreRels.join(", ")}: ${error instanceof Error ? error.message : String(error)}`,
2017
+ ],
2018
+ };
1680
2019
  }
1681
- return restored;
1682
2020
  }
1683
- function writeApplyFailureManifest(ctx, backupDir, failedAction, completed, restored, err) {
2021
+ function writeApplyFailureManifest(ctx, backupDir, failedAction, completed, restored, rollbackFailures, err) {
1684
2022
  const message = err instanceof Error ? err.message : String(err);
1685
2023
  const completedLines = completed.length === 0
1686
2024
  ? [" (none)"]
@@ -1730,11 +2068,18 @@ function writeApplyFailureManifest(ctx, backupDir, failedAction, completed, rest
1730
2068
  for (const rel of unrestored)
1731
2069
  lines.push(` - ${rel}`);
1732
2070
  }
2071
+ if (rollbackFailures.length > 0) {
2072
+ lines.push("", "Rollback failures:");
2073
+ for (const failure of rollbackFailures)
2074
+ lines.push(` - ${failure}`);
2075
+ }
1733
2076
  if (backupDir && isDir(backupDir)) {
1734
2077
  lines.push("");
1735
- lines.push("Manual restore examples:");
1736
- lines.push(` cp "${backupDir}/<relpath>" "${ctx.hqRoot}/<relpath>"`);
1737
- lines.push(` rsync -a "${backupDir}/" "${ctx.hqRoot}/"`);
2078
+ const snapshotName = path.basename(backupDir);
2079
+ lines.push("Restore every captured root and recorded symlink:");
2080
+ lines.push(` ${formatSnapshotRestoreCommand(ctx.hqRoot, ctx.cfg.backupRoot, snapshotName)}`);
2081
+ lines.push("Then verify before retrying:");
2082
+ lines.push(` ${formatRecoveryCommand(ctx.hqRoot, ctx.cfg.backupRoot, snapshotName)}`);
1738
2083
  }
1739
2084
  else {
1740
2085
  lines.push("", "No pre-op safety snapshot was available for automatic restore.");
@@ -3028,6 +3373,22 @@ export function formatRecoveryCommand(hqRoot, backupRoot, verifyRestoredSnapshot
3028
3373
  }
3029
3374
  return args.join(" ");
3030
3375
  }
3376
+ /** Format the manifest-aware command that restores links as well as files. */
3377
+ export function formatSnapshotRestoreCommand(hqRoot, backupRoot, snapshotName) {
3378
+ return [
3379
+ "npx",
3380
+ "-y",
3381
+ "--package=@indigoai-us/hq-cloud@latest",
3382
+ "hq-rescue",
3383
+ "--recover-snapshots",
3384
+ "--hq-root",
3385
+ posixShellQuote(hqRoot),
3386
+ "--backup-dir",
3387
+ posixShellQuote(backupRoot),
3388
+ "--restore-snapshot",
3389
+ posixShellQuote(snapshotName),
3390
+ ].join(" ");
3391
+ }
3031
3392
  function posixRelFromRoot(hqRoot, abs) {
3032
3393
  const root = path.resolve(hqRoot);
3033
3394
  const resolved = path.resolve(abs);
@@ -3203,7 +3564,7 @@ function consumePostRestorePermit(cfg, hqRoot) {
3203
3564
  if (!entry || !unresolved) {
3204
3565
  return { used: false, problem: "referenced snapshot is no longer unresolved recovery evidence" };
3205
3566
  }
3206
- const mismatch = snapshotTreeMismatch(path.join(cfg.backupRoot, entry.name), hqRoot, entry.manifest.includedRoots);
3567
+ const mismatch = snapshotTreeMismatch(path.join(cfg.backupRoot, entry.name), hqRoot, entry.manifest.includedRoots, entry.manifest.symlinks);
3207
3568
  if (mismatch) {
3208
3569
  return { used: false, problem: `live tree no longer matches ${entry.name}: ${mismatch}` };
3209
3570
  }
@@ -3262,13 +3623,22 @@ function snapshotManifest(status, hqRoot, runId, cfg, sourceSha, includedRoots)
3262
3623
  sourceSha,
3263
3624
  createdAt: now,
3264
3625
  includedRoots,
3626
+ symlinks: [],
3265
3627
  fileCount: 0,
3266
3628
  byteCount: 0,
3267
3629
  };
3268
3630
  }
3269
- function readSnapshotManifest(backupDir) {
3631
+ function readSnapshotManifest(backupDir, expectedHqRoot) {
3270
3632
  try {
3271
- return JSON.parse(fs.readFileSync(path.join(backupDir, "SNAPSHOT.json"), "utf8"));
3633
+ const parsed = JSON.parse(fs.readFileSync(path.join(backupDir, "SNAPSHOT.json"), "utf8"));
3634
+ const manifestRoot = expectedHqRoot ??
3635
+ (parsed && typeof parsed === "object"
3636
+ ? parsed.canonicalHqRoot
3637
+ : undefined);
3638
+ return typeof manifestRoot === "string" &&
3639
+ isManagedSnapshotManifest(parsed, manifestRoot)
3640
+ ? parsed
3641
+ : null;
3272
3642
  }
3273
3643
  catch {
3274
3644
  return null;
@@ -3281,7 +3651,7 @@ function writeSnapshotManifest(backupDir, manifest) {
3281
3651
  fs.writeFileSync(temp, JSON.stringify(manifest, null, 2) + "\n");
3282
3652
  fs.renameSync(temp, target);
3283
3653
  }
3284
- function updateSnapshotStatus(backupDir, status, completedAt) {
3654
+ function updateSnapshotStatus(backupDir, status, completedAt, details = {}) {
3285
3655
  const manifest = readSnapshotManifest(backupDir);
3286
3656
  if (!manifest)
3287
3657
  return;
@@ -3289,6 +3659,7 @@ function updateSnapshotStatus(backupDir, status, completedAt) {
3289
3659
  writeSnapshotManifest(backupDir, {
3290
3660
  ...manifest,
3291
3661
  status,
3662
+ ...details,
3292
3663
  fileCount: stats.files,
3293
3664
  byteCount: stats.bytes,
3294
3665
  ...(completedAt ? { completedAt: new Date().toISOString() } : {}),
@@ -3445,22 +3816,206 @@ function cpATo(src, dest) {
3445
3816
  /* ignore */
3446
3817
  }
3447
3818
  }
3819
+ class SnapshotCopyError extends Error {
3820
+ relativePath;
3821
+ operation;
3822
+ path;
3823
+ code;
3824
+ constructor(relativePath, operation, error, path = relativePath) {
3825
+ const message = error instanceof Error ? error.message : String(error);
3826
+ super(message);
3827
+ this.relativePath = relativePath;
3828
+ this.operation = operation;
3829
+ this.path = path;
3830
+ this.name = "SnapshotCopyError";
3831
+ this.code =
3832
+ error?.code ?? "UNKNOWN";
3833
+ }
3834
+ }
3835
+ function snapshotCopyFailure(error) {
3836
+ return {
3837
+ relativePath: error.relativePath,
3838
+ operation: error.operation,
3839
+ code: error.code,
3840
+ message: error.message,
3841
+ occurredAt: new Date().toISOString(),
3842
+ };
3843
+ }
3844
+ function reportSnapshotCopyFailure(error, err) {
3845
+ const kind = error.operation === "read"
3846
+ ? "snapshot-copy-unreadable"
3847
+ : "snapshot-copy-failed";
3848
+ err(`HQ_RESCUE_FAILURE_KIND=${kind}\n`);
3849
+ err(`HQ_RESCUE_SNAPSHOT_COPY_CODE=${error.code}\n`);
3850
+ const verb = error.operation === "metadata" ? "safely record" : error.operation;
3851
+ const detail = error.operation === "metadata" ? `: ${error.message}` : "";
3852
+ err(`error: safety snapshot could not ${verb} ${error.path} (${error.code})${detail}.\n`);
3853
+ if (error.operation === "read") {
3854
+ err(" HQ did not begin the update. If this file is stored in iCloud Drive or another cloud provider, download it locally, then retry.\n");
3855
+ }
3856
+ else {
3857
+ err(" HQ did not begin the update. Repair access to the reported path or backup location, then retry.\n");
3858
+ }
3859
+ }
3860
+ function pathIsInside(root, candidate, allowRoot = false) {
3861
+ const relative = path.relative(root, candidate);
3862
+ return ((allowRoot || relative !== "") &&
3863
+ !relative.startsWith(`..${path.sep}`) &&
3864
+ relative !== ".." &&
3865
+ !path.isAbsolute(relative));
3866
+ }
3867
+ function snapshotRelativePath(hqRoot, rel) {
3868
+ if (!rel ||
3869
+ rel.includes("\0") ||
3870
+ rel.includes("\\") ||
3871
+ path.posix.isAbsolute(rel) ||
3872
+ path.win32.isAbsolute(rel) ||
3873
+ rel.split("/").some((segment) => !segment || segment === "." || segment === "..")) {
3874
+ return null;
3875
+ }
3876
+ const absolute = path.resolve(hqRoot, rel);
3877
+ return pathIsInside(hqRoot, absolute) ? absolute : null;
3878
+ }
3879
+ function restoreParentIsSafe(hqRoot, destination) {
3880
+ for (let current = path.dirname(destination); current !== hqRoot; current = path.dirname(current)) {
3881
+ const st = lstatOrNull(current);
3882
+ if (st?.isSymbolicLink() || (st && !st.isDirectory()))
3883
+ return false;
3884
+ const parent = path.dirname(current);
3885
+ if (parent === current)
3886
+ return false;
3887
+ }
3888
+ return true;
3889
+ }
3890
+ function validateSnapshotSymlink(hqRoot, rel, target) {
3891
+ const linkPath = snapshotRelativePath(hqRoot, rel);
3892
+ if (!linkPath)
3893
+ return `invalid link path ${JSON.stringify(rel)}`;
3894
+ if (typeof target !== "string" || !target || target.includes("\0")) {
3895
+ return `invalid target for ${rel}`;
3896
+ }
3897
+ const resolvedTarget = path.resolve(path.dirname(linkPath), target);
3898
+ return pathIsInside(hqRoot, resolvedTarget, true)
3899
+ ? null
3900
+ : `target for ${rel} resolves outside the HQ root`;
3901
+ }
3902
+ function copyRegularFileForSnapshot(src, dest, rel, mode) {
3903
+ let sourceFd;
3904
+ let destinationFd;
3905
+ try {
3906
+ try {
3907
+ sourceFd = fs.openSync(src, "r");
3908
+ }
3909
+ catch (error) {
3910
+ throw new SnapshotCopyError(rel, "read", error, src);
3911
+ }
3912
+ try {
3913
+ destinationFd = fs.openSync(dest, "w", mode);
3914
+ }
3915
+ catch (error) {
3916
+ throw new SnapshotCopyError(rel, "write", error, dest);
3917
+ }
3918
+ const buffer = Buffer.allocUnsafe(64 * 1024);
3919
+ for (;;) {
3920
+ let read;
3921
+ try {
3922
+ read = fs.readSync(sourceFd, buffer, 0, buffer.length, null);
3923
+ }
3924
+ catch (error) {
3925
+ throw new SnapshotCopyError(rel, "read", error, src);
3926
+ }
3927
+ if (read === 0)
3928
+ break;
3929
+ let offset = 0;
3930
+ while (offset < read) {
3931
+ let written;
3932
+ try {
3933
+ written = fs.writeSync(destinationFd, buffer, offset, read - offset);
3934
+ }
3935
+ catch (error) {
3936
+ throw new SnapshotCopyError(rel, "write", error, dest);
3937
+ }
3938
+ if (written === 0) {
3939
+ throw new SnapshotCopyError(rel, "write", new Error("write returned zero bytes"));
3940
+ }
3941
+ offset += written;
3942
+ }
3943
+ }
3944
+ }
3945
+ finally {
3946
+ if (destinationFd !== undefined)
3947
+ fs.closeSync(destinationFd);
3948
+ if (sourceFd !== undefined)
3949
+ fs.closeSync(sourceFd);
3950
+ }
3951
+ }
3952
+ /** Preserve a Windows file link as ordinary file data when links are forbidden. */
3953
+ function copySymlinkFileRepresentationForSnapshot(src, dest, rel) {
3954
+ let targetStats;
3955
+ try {
3956
+ targetStats = fs.statSync(src);
3957
+ }
3958
+ catch (error) {
3959
+ throw new SnapshotCopyError(rel, "read", error, src);
3960
+ }
3961
+ if (!targetStats.isFile()) {
3962
+ throw new SnapshotCopyError(rel, "metadata", new Error("file-link fallback requires a regular file target"), src);
3963
+ }
3964
+ try {
3965
+ fs.mkdirSync(path.dirname(dest), { recursive: true });
3966
+ }
3967
+ catch (error) {
3968
+ throw new SnapshotCopyError(rel, "write", error, dest);
3969
+ }
3970
+ copyRegularFileForSnapshot(src, dest, rel, targetStats.mode);
3971
+ preserveSnapshotMetadata(dest, targetStats);
3972
+ }
3448
3973
  /** Snapshot copy that preserves metadata while pruning canonical transient directories. */
3449
- function cpAToFiltered(src, dest, rel) {
3450
- const st = lstatOrNull(src);
3451
- if (!st)
3452
- return;
3974
+ function cpAToFiltered(src, dest, rel, hqRoot, symlinks) {
3975
+ let st;
3976
+ try {
3977
+ st = fs.lstatSync(src);
3978
+ }
3979
+ catch (error) {
3980
+ throw new SnapshotCopyError(rel, "read", error, src);
3981
+ }
3453
3982
  if (st.isDirectory() && isRescueTransientDirectory(rel))
3454
3983
  return;
3455
3984
  if (st.isSymbolicLink()) {
3456
- fs.mkdirSync(path.dirname(dest), { recursive: true });
3457
- fs.symlinkSync(fs.readlinkSync(src), dest);
3985
+ let target;
3986
+ try {
3987
+ target = fs.readlinkSync(src);
3988
+ }
3989
+ catch (error) {
3990
+ throw new SnapshotCopyError(rel, "read", error, src);
3991
+ }
3992
+ const problem = validateSnapshotSymlink(hqRoot, rel, target);
3993
+ if (problem) {
3994
+ throw new SnapshotCopyError(rel, "metadata", new Error(problem), src);
3995
+ }
3996
+ const representation = snapshotSymlinkRepresentation(hqRoot, src, dest, rel, target);
3997
+ if (representation === "file-copy") {
3998
+ copySymlinkFileRepresentationForSnapshot(src, dest, rel);
3999
+ }
4000
+ symlinks.push({ path: rel, target, ...(representation ? { representation } : {}) });
3458
4001
  return;
3459
4002
  }
3460
4003
  if (st.isDirectory()) {
3461
- fs.mkdirSync(dest, { recursive: true });
3462
- for (const name of fs.readdirSync(src)) {
3463
- cpAToFiltered(path.join(src, name), path.join(dest, name), path.posix.join(rel, name));
4004
+ try {
4005
+ fs.mkdirSync(dest, { recursive: true });
4006
+ }
4007
+ catch (error) {
4008
+ throw new SnapshotCopyError(rel, "write", error, dest);
4009
+ }
4010
+ let names;
4011
+ try {
4012
+ names = fs.readdirSync(src);
4013
+ }
4014
+ catch (error) {
4015
+ throw new SnapshotCopyError(rel, "read", error, src);
4016
+ }
4017
+ for (const name of names) {
4018
+ cpAToFiltered(path.join(src, name), path.join(dest, name), path.posix.join(rel, name), hqRoot, symlinks);
3464
4019
  }
3465
4020
  try {
3466
4021
  fs.chmodSync(dest, st.mode);
@@ -3471,8 +4026,16 @@ function cpAToFiltered(src, dest, rel) {
3471
4026
  }
3472
4027
  return;
3473
4028
  }
3474
- fs.mkdirSync(path.dirname(dest), { recursive: true });
3475
- fs.copyFileSync(src, dest);
4029
+ if (!st.isFile()) {
4030
+ throw new SnapshotCopyError(rel, "metadata", new Error("unsupported filesystem entry"));
4031
+ }
4032
+ try {
4033
+ fs.mkdirSync(path.dirname(dest), { recursive: true });
4034
+ }
4035
+ catch (error) {
4036
+ throw new SnapshotCopyError(rel, "write", error, dest);
4037
+ }
4038
+ copyRegularFileForSnapshot(src, dest, rel, st.mode);
3476
4039
  try {
3477
4040
  fs.chmodSync(dest, st.mode);
3478
4041
  fs.utimesSync(dest, st.atime, st.mtime);