@indigoai-us/hq-cloud 6.16.49 → 6.16.50

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -327,7 +327,8 @@ export function parseRescueArgs(argv) {
327
327
  backupRoot: process.env.HQ_BACKUP_DIR || path.join(os.homedir(), ".hq", "backups"),
328
328
  backupRetentionDays: process.env.HQ_BACKUP_RETENTION_DAYS || "7",
329
329
  backupRetentionCount: process.env.HQ_BACKUP_RETENTION_COUNT || "2",
330
- resolveRecoverySnapshots: false,
330
+ recoverSnapshots: false,
331
+ verifyRestoredSnapshot: "",
331
332
  };
332
333
  const usage = () => {
333
334
  // The legacy script printed its comment header here; we emit a concise
@@ -336,7 +337,8 @@ export function parseRescueArgs(argv) {
336
337
  " [--preserve PATH]... [--preserve-subpath REL]...\n" +
337
338
  " [--hq-root DIR] [--no-history-check] [--floor-sha SHA]\n" +
338
339
  " [--no-backup] [--backup-dir DIR] [--cloud-update]\n" +
339
- " [--resolve-recovery-snapshots] [--dry-run] [--yes]\n");
340
+ " [--dry-run] [--yes] [--recover-snapshots]\n" +
341
+ " [--verify-restored-snapshot SNAPSHOT]\n");
340
342
  throw new ExitError(1);
341
343
  };
342
344
  let narrowCsv = "";
@@ -385,8 +387,15 @@ export function parseRescueArgs(argv) {
385
387
  case "--backup-dir":
386
388
  cfg.backupRoot = need(a);
387
389
  break;
388
- case "--resolve-recovery-snapshots":
389
- cfg.resolveRecoverySnapshots = true;
390
+ case "--recover-snapshots":
391
+ cfg.recoverSnapshots = true;
392
+ break;
393
+ case "--verify-restored-snapshot":
394
+ cfg.verifyRestoredSnapshot = need(a);
395
+ if (!cfg.verifyRestoredSnapshot) {
396
+ process.stderr.write("error: --verify-restored-snapshot requires a snapshot name.\n");
397
+ throw new ExitError(2);
398
+ }
390
399
  break;
391
400
  case "--dry-run":
392
401
  cfg.dryRun = true;
@@ -448,6 +457,10 @@ export function parseRescueArgs(argv) {
448
457
  throw new ExitError(2);
449
458
  }
450
459
  }
460
+ if (cfg.verifyRestoredSnapshot && !cfg.recoverSnapshots) {
461
+ process.stderr.write("error: --verify-restored-snapshot requires --recover-snapshots.\n");
462
+ throw new ExitError(2);
463
+ }
451
464
  return cfg;
452
465
  }
453
466
  /**
@@ -481,6 +494,8 @@ export function runRescue(argv, opts = {}) {
481
494
  }
482
495
  };
483
496
  try {
497
+ if (cfg.recoverSnapshots)
498
+ return recoverSnapshots(cfg, out, err);
484
499
  return doRescue(cfg, env, out, err, (d) => {
485
500
  tmpdir = d;
486
501
  });
@@ -494,6 +509,259 @@ export function runRescue(argv, opts = {}) {
494
509
  cleanup();
495
510
  }
496
511
  }
512
+ /** Resolve and sanity-check the HQ root before either rescue mode can touch it. */
513
+ function resolveHqRoot(cfg, err) {
514
+ let hqRoot;
515
+ if (cfg.hqRootOverride) {
516
+ // Windows field failure (2026-08-02): the menubar app's daily background
517
+ // core update crash-looped for days because `--hq-root` arrived as the
518
+ // bare drive letter `C:`. `isDir("C:")` is true (drive-relative, resolves
519
+ // against the per-drive cwd) but `fs.realpathSync("C:")` throws EISDIR,
520
+ // so every retry died with an uncaught exception — console-window
521
+ // flicker, git.exe 0xc0000142 dialogs, and no core updates ever landing.
522
+ // Fail closed with a diagnosable message instead: a rescue root must be
523
+ // an absolute directory path, and the error echoes the exact argv this
524
+ // process received so the daily log identifies the caller that mangled
525
+ // the path (the split happens upstream of this process).
526
+ if (!isUsableRescueRoot(cfg.hqRootOverride)) {
527
+ err(`error: --hq-root must be an absolute path, got ${JSON.stringify(cfg.hqRootOverride)}.\n` +
528
+ ` argv: ${JSON.stringify(process.argv.slice(2))}\n`);
529
+ throw new ExitError(1);
530
+ }
531
+ if (!isDir(cfg.hqRootOverride)) {
532
+ // bash `cd` failure under set -e would abort; mirror as a generic error.
533
+ err(`error: --hq-root ${cfg.hqRootOverride} is not a directory.\n`);
534
+ throw new ExitError(1);
535
+ }
536
+ hqRoot = realpathOrExit(cfg.hqRootOverride, err);
537
+ }
538
+ else {
539
+ // Legacy default assumed the script lived at personal/skills/<skill>/.
540
+ // The package's programmatic callers always pass --hq-root; fall back to cwd.
541
+ hqRoot = realpathOrExit(process.cwd(), err);
542
+ }
543
+ // HQ-root sanity gate — guards against wiping a non-HQ directory. `companies/`
544
+ // is the stable anchor (present and preserved on every HQ install), confirmed
545
+ // by at least one core scaffold marker. We accept ANY of `.claude/`, `core/`,
546
+ // or `personal/` because the layout has drifted across releases: v14.0.0 ships
547
+ // NEITHER `personal/` (added v15) NOR `core/` (the v15 scaffold home) — it only
548
+ // has `.claude/`. Requiring `personal/` here left v14.0.0 users with no upgrade
549
+ // path (rescue aborted; DEV-1741). `.claude/` exists on every version v14→v15,
550
+ // so it admits the full upgrade range while still rejecting a bare directory.
551
+ const hasHqMarker = isDir(path.join(hqRoot, ".claude")) ||
552
+ isDir(path.join(hqRoot, "core")) ||
553
+ isDir(path.join(hqRoot, "personal"));
554
+ if (!isDir(path.join(hqRoot, "companies")) || !hasHqMarker) {
555
+ err(`error: ${hqRoot} does not look like an HQ root ` +
556
+ `(need companies/ plus one of .claude/, core/, or personal/). Aborting.\n`);
557
+ err(" pass --hq-root <dir> if the script is not at personal/skills/<skill>/.\n");
558
+ throw new ExitError(3);
559
+ }
560
+ return hqRoot;
561
+ }
562
+ /**
563
+ * Explicit recovery surface for a previously interrupted rescue.
564
+ *
565
+ * A recovery-required snapshot is evidence that a rescue started changing the
566
+ * release-managed tree and then failed. It is safe to delete only when the
567
+ * retention model has a later `completed-good` snapshot for this same canonical
568
+ * HQ root. That later manifest is the durable proof that a subsequent rescue
569
+ * completed; an age, an empty current tree, or a manually edited sync stamp is
570
+ * deliberately not accepted as proof.
571
+ */
572
+ function recoverSnapshots(cfg, out, err) {
573
+ const hqRoot = resolveHqRoot(cfg, err);
574
+ const retention = {
575
+ keepGood: Number(cfg.backupRetentionCount),
576
+ maxAgeDays: Number(cfg.backupRetentionDays),
577
+ };
578
+ const isCopyingOwnerLive = (m) => {
579
+ try {
580
+ process.kill(m.creatorPid, 0);
581
+ return true;
582
+ }
583
+ catch {
584
+ return false;
585
+ }
586
+ };
587
+ const inventory = inventoryManagedSnapshots(cfg.backupRoot, hqRoot, isCopyingOwnerLive, retention);
588
+ if (inventory.unmanaged.some((item) => item.reason === "backup-root-unreadable")) {
589
+ err(`error: cannot inspect safety snapshots: backup root is unreadable: ${cfg.backupRoot}\n`);
590
+ return { status: 1 };
591
+ }
592
+ if (cfg.verifyRestoredSnapshot) {
593
+ return verifyRestoredSnapshot(cfg, hqRoot, inventory.entries, inventory.protected, out, err);
594
+ }
595
+ const verified = inventory.candidates.filter((item) => item.entry.manifest.status === "recovery-required" &&
596
+ item.reason === "recovery-required-superseded");
597
+ const blocked = inventory.protected.filter((item) => item.reason === "recovery-required-no-later-good");
598
+ const retainedByGrace = inventory.protected.filter((item) => item.entry.manifest.status === "recovery-required" &&
599
+ item.reason !== "recovery-required-no-later-good");
600
+ const laterGoodFor = (completedMs, name) => inventory.entries.find((entry) => entry.manifest.status === "completed-good" &&
601
+ (entry.completedMs > completedMs ||
602
+ (entry.completedMs === completedMs && entry.name > name)))?.name;
603
+ out(`==> Safety snapshot recovery inspection: ${cfg.backupRoot}\n`);
604
+ if (verified.length === 0 && blocked.length === 0 && retainedByGrace.length === 0) {
605
+ out(" No recovery-required snapshots need action.\n");
606
+ return { status: 0 };
607
+ }
608
+ for (const item of verified) {
609
+ const laterGood = laterGoodFor(item.entry.completedMs, item.entry.name);
610
+ out(` verified ${item.entry.name}: later completed-good snapshot ${laterGood ?? "(revalidated during delete)"} proves this recovery evidence is superseded.\n`);
611
+ }
612
+ for (const item of blocked) {
613
+ out(` retained ${item.entry.name}: no later completed-good snapshot exists for this HQ root.\n`);
614
+ }
615
+ for (const item of retainedByGrace) {
616
+ out(` retained ${item.entry.name}: a later completed-good snapshot exists, but normal retention still keeps this newest recovery snapshot.\n`);
617
+ }
618
+ if (verified.length > 0) {
619
+ if (cfg.dryRun) {
620
+ for (const item of verified) {
621
+ out(` would resolve ${item.entry.name}: verified superseded recovery snapshot would be retained during this dry run.\n`);
622
+ }
623
+ }
624
+ else {
625
+ const result = applyBackupPrune(cfg.backupRoot, hqRoot, { candidates: verified, protected: [], unmanaged: [] }, retention);
626
+ for (const name of result.deleted) {
627
+ out(` resolved ${name}: verified superseded recovery snapshot deleted.\n`);
628
+ }
629
+ for (const failure of result.failures) {
630
+ err(`error: could not resolve verified snapshot (retained): ${failure}\n`);
631
+ }
632
+ if (result.failures.length > 0)
633
+ return { status: 1 };
634
+ }
635
+ }
636
+ if (cfg.dryRun && blocked.length === 0) {
637
+ out("==> Snapshot recovery dry run complete: no snapshot was changed.\n");
638
+ return { status: 0 };
639
+ }
640
+ // Re-inventory after an apply because another process might have created or
641
+ // changed a snapshot while the prune lock was held. Nothing unverified has
642
+ // been deleted, and the updated inventory is the report we return to users.
643
+ const remaining = inventoryManagedSnapshots(cfg.backupRoot, hqRoot, isCopyingOwnerLive, retention);
644
+ const unresolved = remaining.protected.filter((item) => item.reason === "recovery-required-no-later-good");
645
+ if (unresolved.length > 0) {
646
+ err("error: snapshot recovery refused: the interrupted updates are not proven superseded.\n");
647
+ err(" HQ retained every unverified snapshot. A later completed-good rescue for this same HQ root is the only automatic proof.\n");
648
+ err(" Restore every captured root from one snapshot, then verify that restored tree before one retry is permitted:\n");
649
+ for (const item of unresolved) {
650
+ err(` ${item.entry.name}: ${formatRecoveryCommand(hqRoot, cfg.backupRoot, item.entry.name)}\n`);
651
+ }
652
+ return { status: 1 };
653
+ }
654
+ out("==> Snapshot recovery complete: every eligible recovery snapshot was positively verified before deletion.\n");
655
+ return { status: 0 };
656
+ }
657
+ /** A live tree must still match the selected recovery snapshot at permit use. */
658
+ function verifyRestoredSnapshot(cfg, hqRoot, entries, protectedEntries, out, err) {
659
+ const entry = entries.find((item) => item.name === cfg.verifyRestoredSnapshot);
660
+ const unresolved = protectedEntries.find((item) => item.entry.name === cfg.verifyRestoredSnapshot &&
661
+ item.reason === "recovery-required-no-later-good");
662
+ if (!entry || !unresolved || entry.manifest.status !== "recovery-required") {
663
+ err(`error: ${cfg.verifyRestoredSnapshot} is not an unresolved recovery-required snapshot for this HQ root.\n`);
664
+ return { status: 1 };
665
+ }
666
+ const mismatch = snapshotTreeMismatch(path.join(cfg.backupRoot, entry.name), hqRoot, entry.manifest.includedRoots);
667
+ if (mismatch) {
668
+ err(`error: restored tree does not match snapshot ${entry.name}: ${mismatch}\n`);
669
+ err(" No post-restore permit was written; the circuit breaker remains armed.\n");
670
+ return { status: 1 };
671
+ }
672
+ if (cfg.dryRun) {
673
+ out(`==> Dry run verified restored snapshot ${entry.name}; no post-restore permit was written.\n`);
674
+ return { status: 0 };
675
+ }
676
+ try {
677
+ writePostRestorePermit(hqRoot, entry);
678
+ }
679
+ catch (error) {
680
+ err(`error: verified snapshot ${entry.name}, but could not write the post-restore permit: ${error instanceof Error ? error.message : String(error)}\n`);
681
+ return { status: 1 };
682
+ }
683
+ out(`==> Verified restored snapshot ${entry.name}; one rescue attempt is now permitted.\n`);
684
+ out(" That permit is rechecked and consumed before rescue changes files. If the attempt fails, the circuit breaker is armed again.\n");
685
+ return { status: 0 };
686
+ }
687
+ /** Return a precise mismatch instead of treating a partial restore as evidence. */
688
+ function snapshotTreeMismatch(snapshotRoot, hqRoot, includedRoots) {
689
+ if (includedRoots.length === 0) {
690
+ return "snapshot manifest records no captured roots to verify";
691
+ }
692
+ const seen = new Set();
693
+ for (const root of includedRoots) {
694
+ if (!root ||
695
+ root === "." ||
696
+ root === ".." ||
697
+ root.includes("/") ||
698
+ root.includes("\\") ||
699
+ isRescueTransientDirectory(root) ||
700
+ seen.has(root)) {
701
+ return `snapshot manifest has an unsafe captured root ${JSON.stringify(root)}`;
702
+ }
703
+ seen.add(root);
704
+ if (rescuePathKind(path.join(snapshotRoot, root)) === "missing") {
705
+ return `snapshot does not contain captured root ${root}`;
706
+ }
707
+ const mismatch = snapshotPathMismatch(path.join(snapshotRoot, root), path.join(hqRoot, root), root);
708
+ if (mismatch)
709
+ return mismatch;
710
+ }
711
+ return null;
712
+ }
713
+ function snapshotPathMismatch(snapshotPath, livePath, rel) {
714
+ const snapshotKind = rescuePathKind(snapshotPath);
715
+ const liveKind = rescuePathKind(livePath);
716
+ if (snapshotKind !== liveKind) {
717
+ return `${rel} is ${liveKind} but the snapshot records ${snapshotKind}`;
718
+ }
719
+ if (snapshotKind === "missing")
720
+ return null;
721
+ if (snapshotKind === "file") {
722
+ return bytesEqual(snapshotPath, livePath)
723
+ ? null
724
+ : `${rel} has different file contents`;
725
+ }
726
+ if (snapshotKind === "symlink") {
727
+ try {
728
+ return fs.readlinkSync(snapshotPath) === fs.readlinkSync(livePath)
729
+ ? null
730
+ : `${rel} has a different symlink target`;
731
+ }
732
+ catch (error) {
733
+ return `${rel} could not read a symlink target: ${error instanceof Error ? error.message : String(error)}`;
734
+ }
735
+ }
736
+ if (snapshotKind !== "directory") {
737
+ return `${rel} has unsupported path type ${snapshotKind}`;
738
+ }
739
+ let snapshotNames;
740
+ let liveNames;
741
+ try {
742
+ snapshotNames = fs
743
+ .readdirSync(snapshotPath)
744
+ .filter((name) => !isRescueTransientDirectory(path.posix.join(rel, name)))
745
+ .sort();
746
+ liveNames = fs
747
+ .readdirSync(livePath)
748
+ .filter((name) => !isRescueTransientDirectory(path.posix.join(rel, name)))
749
+ .sort();
750
+ }
751
+ catch (error) {
752
+ return `${rel} could not be read: ${error instanceof Error ? error.message : String(error)}`;
753
+ }
754
+ if (snapshotNames.length !== liveNames.length ||
755
+ snapshotNames.some((name, index) => name !== liveNames[index])) {
756
+ return `${rel} has different entries`;
757
+ }
758
+ for (const name of snapshotNames) {
759
+ const mismatch = snapshotPathMismatch(path.join(snapshotPath, name), path.join(livePath, name), path.posix.join(rel, name));
760
+ if (mismatch)
761
+ return mismatch;
762
+ }
763
+ return null;
764
+ }
497
765
  /**
498
766
  * True when `root` is usable as a rescue target: an absolute path that is not
499
767
  * a bare drive letter. Pure + exported for tests; `platformPath` defaults to
@@ -541,53 +809,7 @@ export function realpathOrExit(p, err) {
541
809
  }
542
810
  }
543
811
  function doRescue(cfg, env, out, err, setTmp) {
544
- // --- Resolve HQ root ---
545
- let hqRoot;
546
- if (cfg.hqRootOverride) {
547
- // Windows field failure (2026-08-02): the menubar app's daily background
548
- // core update crash-looped for days because `--hq-root` arrived as the
549
- // bare drive letter `C:`. `isDir("C:")` is true (drive-relative, resolves
550
- // against the per-drive cwd) but `fs.realpathSync("C:")` throws EISDIR,
551
- // so every retry died with an uncaught exception — console-window
552
- // flicker, git.exe 0xc0000142 dialogs, and no core updates ever landing.
553
- // Fail closed with a diagnosable message instead: a rescue root must be
554
- // an absolute directory path, and the error echoes the exact argv this
555
- // process received so the daily log identifies the caller that mangled
556
- // the path (the split happens upstream of this process).
557
- if (!isUsableRescueRoot(cfg.hqRootOverride)) {
558
- err(`error: --hq-root must be an absolute path, got ${JSON.stringify(cfg.hqRootOverride)}.\n` +
559
- ` argv: ${JSON.stringify(process.argv.slice(2))}\n`);
560
- throw new ExitError(1);
561
- }
562
- if (!isDir(cfg.hqRootOverride)) {
563
- // bash `cd` failure under set -e would abort; mirror as a generic error.
564
- err(`error: --hq-root ${cfg.hqRootOverride} is not a directory.\n`);
565
- throw new ExitError(1);
566
- }
567
- hqRoot = realpathOrExit(cfg.hqRootOverride, err);
568
- }
569
- else {
570
- // Legacy default assumed the script lived at personal/skills/<skill>/.
571
- // The package's programmatic callers always pass --hq-root; fall back to cwd.
572
- hqRoot = realpathOrExit(process.cwd(), err);
573
- }
574
- // HQ-root sanity gate — guards against wiping a non-HQ directory. `companies/`
575
- // is the stable anchor (present and preserved on every HQ install), confirmed
576
- // by at least one core scaffold marker. We accept ANY of `.claude/`, `core/`,
577
- // or `personal/` because the layout has drifted across releases: v14.0.0 ships
578
- // NEITHER `personal/` (added v15) NOR `core/` (the v15 scaffold home) — it only
579
- // has `.claude/`. Requiring `personal/` here left v14.0.0 users with no upgrade
580
- // path (rescue aborted; DEV-1741). `.claude/` exists on every version v14→v15,
581
- // so it admits the full upgrade range while still rejecting a bare directory.
582
- const hasHqMarker = isDir(path.join(hqRoot, ".claude")) ||
583
- isDir(path.join(hqRoot, "core")) ||
584
- isDir(path.join(hqRoot, "personal"));
585
- if (!isDir(path.join(hqRoot, "companies")) || !hasHqMarker) {
586
- err(`error: ${hqRoot} does not look like an HQ root ` +
587
- `(need companies/ plus one of .claude/, core/, or personal/). Aborting.\n`);
588
- err(" pass --hq-root <dir> if the script is not at personal/skills/<skill>/.\n");
589
- throw new ExitError(3);
590
- }
812
+ const hqRoot = resolveHqRoot(cfg, err);
591
813
  const RUN_TS = utcStamp(new Date(), "dash");
592
814
  // Both the release overlay and a restorable snapshot require rsync. A shim
593
815
  // which merely starts but exits non-zero used to trigger an enormous cp -a
@@ -1038,15 +1260,25 @@ function doRescue(cfg, env, out, err, setTmp) {
1038
1260
  const prePlan = pruneManagedBackups(cfg, hqRoot, out, false);
1039
1261
  const unresolved = prePlan.protected.filter((x) => x.reason === "recovery-required-no-later-good");
1040
1262
  if (unresolved.length >= 2) {
1041
- const snapshotNames = unresolved.map((x) => x.entry.name);
1042
- writeRescueBlockedMarker(hqRoot, snapshotNames);
1043
- if (!cfg.resolveRecoverySnapshots) {
1044
- err("error: safety snapshot circuit breaker is open: two unresolved recovery-required snapshots exist.\n");
1045
- err(" Inspect/restore those snapshots and repair the release source before retrying (operator override required).\n");
1046
- err(" Re-run with --resolve-recovery-snapshots to move those snapshot directories aside (never deleted) and continue.\n");
1263
+ const postRestore = consumePostRestorePermit(cfg, hqRoot);
1264
+ if (postRestore.used) {
1265
+ out(`==> Verified post-restore permit for ${postRestore.snapshotName}; allowing one rescue attempt.\n`);
1266
+ out(" The permit was consumed before mutation. If this attempt fails, the circuit breaker is armed again.\n");
1267
+ }
1268
+ else {
1269
+ if (postRestore.problem) {
1270
+ err(`error: post-restore permit is not usable: ${postRestore.problem}\n`);
1271
+ }
1272
+ const snapshotNames = unresolved.map((x) => x.entry.name);
1273
+ writeRescueBlockedMarker(hqRoot, snapshotNames);
1274
+ err("HQ_RESCUE_FAILURE_KIND=snapshot-recovery-circuit-breaker\n");
1275
+ err("error: safety snapshot circuit breaker is open: two interrupted updates still need recovery proof.\n");
1276
+ err(" HQ kept their pre-update snapshots because those updates started changing the release-managed tree but did not finish.\n");
1277
+ err(" It will not overwrite or delete that evidence without proof that a later rescue completed successfully.\n");
1278
+ err(` Inspect safely: ${formatRecoveryCommand(hqRoot, cfg.backupRoot)}\n`);
1279
+ err(" It deletes only snapshots with a later completed-good rescue. For unresolved snapshots, restore one listed snapshot and run its verification command before retrying.\n");
1047
1280
  throw new ExitError(1);
1048
1281
  }
1049
- resolveRecoveryRequiredSnapshots(cfg.backupRoot, RUN_TS, snapshotNames, out);
1050
1282
  }
1051
1283
  requireBackupCapacity(cfg, hqRoot, wipeToplevel, env, err);
1052
1284
  fs.mkdirSync(cfg.backupRoot, { recursive: true });
@@ -2774,6 +3006,28 @@ function restoreMtimesFromGit(srcDir, env, out) {
2774
3006
  }
2775
3007
  const RESCUE_BLOCKED_MARKER_REL = ".hq/rescue-blocked.json";
2776
3008
  const RESCUE_BLOCKED_REASON = "recovery-snapshot-breaker";
3009
+ const POST_RESTORE_PERMIT_REL = ".hq/rescue-post-restore.json";
3010
+ function posixShellQuote(value) {
3011
+ return `'${value.replace(/'/g, `'\\''`)}'`;
3012
+ }
3013
+ /** Format the exact safe command presented in a breaker diagnostic. */
3014
+ export function formatRecoveryCommand(hqRoot, backupRoot, verifyRestoredSnapshot) {
3015
+ const args = [
3016
+ "npx",
3017
+ "-y",
3018
+ "--package=@indigoai-us/hq-cloud@latest",
3019
+ "hq-rescue",
3020
+ "--recover-snapshots",
3021
+ "--hq-root",
3022
+ posixShellQuote(hqRoot),
3023
+ "--backup-dir",
3024
+ posixShellQuote(backupRoot),
3025
+ ];
3026
+ if (verifyRestoredSnapshot) {
3027
+ args.push("--verify-restored-snapshot", posixShellQuote(verifyRestoredSnapshot));
3028
+ }
3029
+ return args.join(" ");
3030
+ }
2777
3031
  function posixRelFromRoot(hqRoot, abs) {
2778
3032
  const root = path.resolve(hqRoot);
2779
3033
  const resolved = path.resolve(abs);
@@ -2871,12 +3125,106 @@ function reportUnwritableRescuePaths(hqRoot, unwritable, err) {
2871
3125
  function rescueBlockedMarkerPath(hqRoot) {
2872
3126
  return path.join(hqRoot, RESCUE_BLOCKED_MARKER_REL);
2873
3127
  }
3128
+ function postRestorePermitPath(hqRoot) {
3129
+ return path.join(hqRoot, POST_RESTORE_PERMIT_REL);
3130
+ }
3131
+ function writePostRestorePermit(hqRoot, entry) {
3132
+ const permit = {
3133
+ schemaVersion: 1,
3134
+ canonicalHqRoot: hqRoot,
3135
+ rootHash: canonicalRootHash(hqRoot),
3136
+ snapshotName: entry.name,
3137
+ snapshotRunId: entry.manifest.runId,
3138
+ verifiedAt: new Date().toISOString(),
3139
+ };
3140
+ const target = postRestorePermitPath(hqRoot);
3141
+ const temporary = `${target}.${process.pid}.tmp`;
3142
+ fs.mkdirSync(path.dirname(target), { recursive: true });
3143
+ fs.writeFileSync(temporary, JSON.stringify(permit, null, 2) + "\n", {
3144
+ mode: 0o600,
3145
+ });
3146
+ fs.renameSync(temporary, target);
3147
+ }
3148
+ function readPostRestorePermit(hqRoot) {
3149
+ let raw;
3150
+ try {
3151
+ raw = fs.readFileSync(postRestorePermitPath(hqRoot), "utf8");
3152
+ }
3153
+ catch (error) {
3154
+ if (error.code === "ENOENT") {
3155
+ return { permit: null };
3156
+ }
3157
+ return {
3158
+ permit: null,
3159
+ problem: `could not read ${POST_RESTORE_PERMIT_REL}: ${error instanceof Error ? error.message : String(error)}`,
3160
+ };
3161
+ }
3162
+ try {
3163
+ const value = JSON.parse(raw);
3164
+ if (value.schemaVersion !== 1 ||
3165
+ typeof value.canonicalHqRoot !== "string" ||
3166
+ typeof value.rootHash !== "string" ||
3167
+ typeof value.snapshotName !== "string" ||
3168
+ typeof value.snapshotRunId !== "string" ||
3169
+ typeof value.verifiedAt !== "string") {
3170
+ return { permit: null, problem: `${POST_RESTORE_PERMIT_REL} is malformed` };
3171
+ }
3172
+ return { permit: value };
3173
+ }
3174
+ catch (error) {
3175
+ return {
3176
+ permit: null,
3177
+ problem: `${POST_RESTORE_PERMIT_REL} is not valid JSON: ${error instanceof Error ? error.message : String(error)}`,
3178
+ };
3179
+ }
3180
+ }
3181
+ function consumePostRestorePermit(cfg, hqRoot) {
3182
+ const read = readPostRestorePermit(hqRoot);
3183
+ if (!read.permit) {
3184
+ return { used: false, ...(read.problem ? { problem: read.problem } : {}) };
3185
+ }
3186
+ const permit = read.permit;
3187
+ if (permit.canonicalHqRoot !== hqRoot ||
3188
+ permit.rootHash !== canonicalRootHash(hqRoot)) {
3189
+ return { used: false, problem: "permit belongs to a different HQ root" };
3190
+ }
3191
+ const inventory = inventoryManagedSnapshots(cfg.backupRoot, hqRoot, () => false, {
3192
+ keepGood: Number(cfg.backupRetentionCount),
3193
+ maxAgeDays: Number(cfg.backupRetentionDays),
3194
+ });
3195
+ if (inventory.unmanaged.some((item) => item.reason === "backup-root-unreadable")) {
3196
+ return { used: false, problem: `backup root is unreadable: ${cfg.backupRoot}` };
3197
+ }
3198
+ const entry = inventory.entries.find((item) => item.name === permit.snapshotName &&
3199
+ item.manifest.runId === permit.snapshotRunId &&
3200
+ item.manifest.status === "recovery-required");
3201
+ const unresolved = inventory.protected.some((item) => item.entry.name === permit.snapshotName &&
3202
+ item.reason === "recovery-required-no-later-good");
3203
+ if (!entry || !unresolved) {
3204
+ return { used: false, problem: "referenced snapshot is no longer unresolved recovery evidence" };
3205
+ }
3206
+ const mismatch = snapshotTreeMismatch(path.join(cfg.backupRoot, entry.name), hqRoot, entry.manifest.includedRoots);
3207
+ if (mismatch) {
3208
+ return { used: false, problem: `live tree no longer matches ${entry.name}: ${mismatch}` };
3209
+ }
3210
+ try {
3211
+ fs.unlinkSync(postRestorePermitPath(hqRoot));
3212
+ }
3213
+ catch (error) {
3214
+ return {
3215
+ used: false,
3216
+ problem: `could not consume the single-use permit: ${error instanceof Error ? error.message : String(error)}`,
3217
+ };
3218
+ }
3219
+ return { used: true, snapshotName: entry.name };
3220
+ }
2874
3221
  function writeRescueBlockedMarker(hqRoot, snapshots) {
2875
3222
  const body = {
2876
3223
  at: new Date().toISOString(),
2877
3224
  reason: RESCUE_BLOCKED_REASON,
3225
+ failureKind: "snapshot-recovery-circuit-breaker",
2878
3226
  snapshots,
2879
- hint: "Inspect/restore those snapshots and repair the release source before retrying, or re-run with --resolve-recovery-snapshots (moves the snapshot directories aside under <backup-dir>-resolved-<timestamp>/; never deletes).",
3227
+ hint: "Run hq-rescue --recover-snapshots to inspect the snapshots. It deletes only snapshots proved superseded by a later completed-good rescue; otherwise restore the needed files, repair the release source, and retry.",
2880
3228
  };
2881
3229
  try {
2882
3230
  fs.mkdirSync(path.dirname(rescueBlockedMarkerPath(hqRoot)), {
@@ -2896,18 +3244,6 @@ function clearRescueBlockedMarker(hqRoot) {
2896
3244
  /* absent is the success state */
2897
3245
  }
2898
3246
  }
2899
- function resolveRecoveryRequiredSnapshots(backupRoot, runTs, names, out) {
2900
- const destRoot = `${backupRoot}-resolved-${runTs}`;
2901
- fs.mkdirSync(destRoot, { recursive: true });
2902
- out(`==> Resolving recovery-required snapshots (--resolve-recovery-snapshots) -> ${destRoot}\n`);
2903
- for (const name of names) {
2904
- const from = path.join(backupRoot, name);
2905
- const to = path.join(destRoot, name);
2906
- fs.renameSync(from, to);
2907
- out(` moved ${name} -> ${to}\n`);
2908
- }
2909
- return destRoot;
2910
- }
2911
3247
  function allocateSnapshotDir(backupRoot, runTs, rootHash, runId) {
2912
3248
  return path.join(backupRoot, `pre-update-${runTs}-${rootHash}-${runId}`);
2913
3249
  }