@indigoai-us/hq-cloud 6.16.48 → 6.16.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/rescue-runner.js +2 -1
- package/dist/bin/rescue-runner.js.map +1 -1
- package/dist/cli/rescue-breaker.test.d.ts +2 -0
- package/dist/cli/rescue-breaker.test.d.ts.map +1 -0
- package/dist/cli/rescue-breaker.test.js +334 -0
- package/dist/cli/rescue-breaker.test.js.map +1 -0
- package/dist/cli/rescue-core.d.ts +13 -0
- package/dist/cli/rescue-core.d.ts.map +1 -1
- package/dist/cli/rescue-core.js +550 -55
- package/dist/cli/rescue-core.js.map +1 -1
- package/dist/cli/rescue-snapshot-recovery.test.d.ts +2 -0
- package/dist/cli/rescue-snapshot-recovery.test.d.ts.map +1 -0
- package/dist/cli/rescue-snapshot-recovery.test.js +260 -0
- package/dist/cli/rescue-snapshot-recovery.test.js.map +1 -0
- package/dist/cli/rescue.d.ts +4 -0
- package/dist/cli/rescue.d.ts.map +1 -1
- package/dist/cli/rescue.js +20 -1
- package/dist/cli/rescue.js.map +1 -1
- package/dist/cli/rescue.test.js +16 -1
- package/dist/cli/rescue.test.js.map +1 -1
- package/dist/cli/share-presign-transport.test.js +2 -0
- package/dist/cli/share-presign-transport.test.js.map +1 -1
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +37 -2
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +92 -0
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/watch-event-push-conflict.test.js +2 -0
- package/dist/cli/watch-event-push-conflict.test.js.map +1 -1
- package/dist/object-io.d.ts +2 -2
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +18 -3
- package/dist/object-io.js.map +1 -1
- package/dist/s3.d.ts +18 -0
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +45 -4
- package/dist/s3.js.map +1 -1
- package/dist/s3.test.js +27 -1
- package/dist/s3.test.js.map +1 -1
- package/dist/sync-core.d.ts +6 -4
- package/dist/sync-core.d.ts.map +1 -1
- package/dist/sync-core.js +10 -4
- package/dist/sync-core.js.map +1 -1
- package/dist/sync-core.test.js +25 -1
- package/dist/sync-core.test.js.map +1 -1
- package/package.json +1 -1
package/dist/cli/rescue-core.js
CHANGED
|
@@ -327,6 +327,8 @@ export function parseRescueArgs(argv) {
|
|
|
327
327
|
backupRoot: process.env.HQ_BACKUP_DIR || path.join(os.homedir(), ".hq", "backups"),
|
|
328
328
|
backupRetentionDays: process.env.HQ_BACKUP_RETENTION_DAYS || "7",
|
|
329
329
|
backupRetentionCount: process.env.HQ_BACKUP_RETENTION_COUNT || "2",
|
|
330
|
+
recoverSnapshots: false,
|
|
331
|
+
verifyRestoredSnapshot: "",
|
|
330
332
|
};
|
|
331
333
|
const usage = () => {
|
|
332
334
|
// The legacy script printed its comment header here; we emit a concise
|
|
@@ -335,7 +337,8 @@ export function parseRescueArgs(argv) {
|
|
|
335
337
|
" [--preserve PATH]... [--preserve-subpath REL]...\n" +
|
|
336
338
|
" [--hq-root DIR] [--no-history-check] [--floor-sha SHA]\n" +
|
|
337
339
|
" [--no-backup] [--backup-dir DIR] [--cloud-update]\n" +
|
|
338
|
-
" [--dry-run] [--yes]\n"
|
|
340
|
+
" [--dry-run] [--yes] [--recover-snapshots]\n" +
|
|
341
|
+
" [--verify-restored-snapshot SNAPSHOT]\n");
|
|
339
342
|
throw new ExitError(1);
|
|
340
343
|
};
|
|
341
344
|
let narrowCsv = "";
|
|
@@ -384,6 +387,16 @@ export function parseRescueArgs(argv) {
|
|
|
384
387
|
case "--backup-dir":
|
|
385
388
|
cfg.backupRoot = need(a);
|
|
386
389
|
break;
|
|
390
|
+
case "--recover-snapshots":
|
|
391
|
+
cfg.recoverSnapshots = true;
|
|
392
|
+
break;
|
|
393
|
+
case "--verify-restored-snapshot":
|
|
394
|
+
cfg.verifyRestoredSnapshot = need(a);
|
|
395
|
+
if (!cfg.verifyRestoredSnapshot) {
|
|
396
|
+
process.stderr.write("error: --verify-restored-snapshot requires a snapshot name.\n");
|
|
397
|
+
throw new ExitError(2);
|
|
398
|
+
}
|
|
399
|
+
break;
|
|
387
400
|
case "--dry-run":
|
|
388
401
|
cfg.dryRun = true;
|
|
389
402
|
break;
|
|
@@ -444,6 +457,10 @@ export function parseRescueArgs(argv) {
|
|
|
444
457
|
throw new ExitError(2);
|
|
445
458
|
}
|
|
446
459
|
}
|
|
460
|
+
if (cfg.verifyRestoredSnapshot && !cfg.recoverSnapshots) {
|
|
461
|
+
process.stderr.write("error: --verify-restored-snapshot requires --recover-snapshots.\n");
|
|
462
|
+
throw new ExitError(2);
|
|
463
|
+
}
|
|
447
464
|
return cfg;
|
|
448
465
|
}
|
|
449
466
|
/**
|
|
@@ -477,6 +494,8 @@ export function runRescue(argv, opts = {}) {
|
|
|
477
494
|
}
|
|
478
495
|
};
|
|
479
496
|
try {
|
|
497
|
+
if (cfg.recoverSnapshots)
|
|
498
|
+
return recoverSnapshots(cfg, out, err);
|
|
480
499
|
return doRescue(cfg, env, out, err, (d) => {
|
|
481
500
|
tmpdir = d;
|
|
482
501
|
});
|
|
@@ -490,6 +509,259 @@ export function runRescue(argv, opts = {}) {
|
|
|
490
509
|
cleanup();
|
|
491
510
|
}
|
|
492
511
|
}
|
|
512
|
+
/** Resolve and sanity-check the HQ root before either rescue mode can touch it. */
|
|
513
|
+
function resolveHqRoot(cfg, err) {
|
|
514
|
+
let hqRoot;
|
|
515
|
+
if (cfg.hqRootOverride) {
|
|
516
|
+
// Windows field failure (2026-08-02): the menubar app's daily background
|
|
517
|
+
// core update crash-looped for days because `--hq-root` arrived as the
|
|
518
|
+
// bare drive letter `C:`. `isDir("C:")` is true (drive-relative, resolves
|
|
519
|
+
// against the per-drive cwd) but `fs.realpathSync("C:")` throws EISDIR,
|
|
520
|
+
// so every retry died with an uncaught exception — console-window
|
|
521
|
+
// flicker, git.exe 0xc0000142 dialogs, and no core updates ever landing.
|
|
522
|
+
// Fail closed with a diagnosable message instead: a rescue root must be
|
|
523
|
+
// an absolute directory path, and the error echoes the exact argv this
|
|
524
|
+
// process received so the daily log identifies the caller that mangled
|
|
525
|
+
// the path (the split happens upstream of this process).
|
|
526
|
+
if (!isUsableRescueRoot(cfg.hqRootOverride)) {
|
|
527
|
+
err(`error: --hq-root must be an absolute path, got ${JSON.stringify(cfg.hqRootOverride)}.\n` +
|
|
528
|
+
` argv: ${JSON.stringify(process.argv.slice(2))}\n`);
|
|
529
|
+
throw new ExitError(1);
|
|
530
|
+
}
|
|
531
|
+
if (!isDir(cfg.hqRootOverride)) {
|
|
532
|
+
// bash `cd` failure under set -e would abort; mirror as a generic error.
|
|
533
|
+
err(`error: --hq-root ${cfg.hqRootOverride} is not a directory.\n`);
|
|
534
|
+
throw new ExitError(1);
|
|
535
|
+
}
|
|
536
|
+
hqRoot = realpathOrExit(cfg.hqRootOverride, err);
|
|
537
|
+
}
|
|
538
|
+
else {
|
|
539
|
+
// Legacy default assumed the script lived at personal/skills/<skill>/.
|
|
540
|
+
// The package's programmatic callers always pass --hq-root; fall back to cwd.
|
|
541
|
+
hqRoot = realpathOrExit(process.cwd(), err);
|
|
542
|
+
}
|
|
543
|
+
// HQ-root sanity gate — guards against wiping a non-HQ directory. `companies/`
|
|
544
|
+
// is the stable anchor (present and preserved on every HQ install), confirmed
|
|
545
|
+
// by at least one core scaffold marker. We accept ANY of `.claude/`, `core/`,
|
|
546
|
+
// or `personal/` because the layout has drifted across releases: v14.0.0 ships
|
|
547
|
+
// NEITHER `personal/` (added v15) NOR `core/` (the v15 scaffold home) — it only
|
|
548
|
+
// has `.claude/`. Requiring `personal/` here left v14.0.0 users with no upgrade
|
|
549
|
+
// path (rescue aborted; DEV-1741). `.claude/` exists on every version v14→v15,
|
|
550
|
+
// so it admits the full upgrade range while still rejecting a bare directory.
|
|
551
|
+
const hasHqMarker = isDir(path.join(hqRoot, ".claude")) ||
|
|
552
|
+
isDir(path.join(hqRoot, "core")) ||
|
|
553
|
+
isDir(path.join(hqRoot, "personal"));
|
|
554
|
+
if (!isDir(path.join(hqRoot, "companies")) || !hasHqMarker) {
|
|
555
|
+
err(`error: ${hqRoot} does not look like an HQ root ` +
|
|
556
|
+
`(need companies/ plus one of .claude/, core/, or personal/). Aborting.\n`);
|
|
557
|
+
err(" pass --hq-root <dir> if the script is not at personal/skills/<skill>/.\n");
|
|
558
|
+
throw new ExitError(3);
|
|
559
|
+
}
|
|
560
|
+
return hqRoot;
|
|
561
|
+
}
|
|
562
|
+
/**
|
|
563
|
+
* Explicit recovery surface for a previously interrupted rescue.
|
|
564
|
+
*
|
|
565
|
+
* A recovery-required snapshot is evidence that a rescue started changing the
|
|
566
|
+
* release-managed tree and then failed. It is safe to delete only when the
|
|
567
|
+
* retention model has a later `completed-good` snapshot for this same canonical
|
|
568
|
+
* HQ root. That later manifest is the durable proof that a subsequent rescue
|
|
569
|
+
* completed; an age, an empty current tree, or a manually edited sync stamp is
|
|
570
|
+
* deliberately not accepted as proof.
|
|
571
|
+
*/
|
|
572
|
+
function recoverSnapshots(cfg, out, err) {
|
|
573
|
+
const hqRoot = resolveHqRoot(cfg, err);
|
|
574
|
+
const retention = {
|
|
575
|
+
keepGood: Number(cfg.backupRetentionCount),
|
|
576
|
+
maxAgeDays: Number(cfg.backupRetentionDays),
|
|
577
|
+
};
|
|
578
|
+
const isCopyingOwnerLive = (m) => {
|
|
579
|
+
try {
|
|
580
|
+
process.kill(m.creatorPid, 0);
|
|
581
|
+
return true;
|
|
582
|
+
}
|
|
583
|
+
catch {
|
|
584
|
+
return false;
|
|
585
|
+
}
|
|
586
|
+
};
|
|
587
|
+
const inventory = inventoryManagedSnapshots(cfg.backupRoot, hqRoot, isCopyingOwnerLive, retention);
|
|
588
|
+
if (inventory.unmanaged.some((item) => item.reason === "backup-root-unreadable")) {
|
|
589
|
+
err(`error: cannot inspect safety snapshots: backup root is unreadable: ${cfg.backupRoot}\n`);
|
|
590
|
+
return { status: 1 };
|
|
591
|
+
}
|
|
592
|
+
if (cfg.verifyRestoredSnapshot) {
|
|
593
|
+
return verifyRestoredSnapshot(cfg, hqRoot, inventory.entries, inventory.protected, out, err);
|
|
594
|
+
}
|
|
595
|
+
const verified = inventory.candidates.filter((item) => item.entry.manifest.status === "recovery-required" &&
|
|
596
|
+
item.reason === "recovery-required-superseded");
|
|
597
|
+
const blocked = inventory.protected.filter((item) => item.reason === "recovery-required-no-later-good");
|
|
598
|
+
const retainedByGrace = inventory.protected.filter((item) => item.entry.manifest.status === "recovery-required" &&
|
|
599
|
+
item.reason !== "recovery-required-no-later-good");
|
|
600
|
+
const laterGoodFor = (completedMs, name) => inventory.entries.find((entry) => entry.manifest.status === "completed-good" &&
|
|
601
|
+
(entry.completedMs > completedMs ||
|
|
602
|
+
(entry.completedMs === completedMs && entry.name > name)))?.name;
|
|
603
|
+
out(`==> Safety snapshot recovery inspection: ${cfg.backupRoot}\n`);
|
|
604
|
+
if (verified.length === 0 && blocked.length === 0 && retainedByGrace.length === 0) {
|
|
605
|
+
out(" No recovery-required snapshots need action.\n");
|
|
606
|
+
return { status: 0 };
|
|
607
|
+
}
|
|
608
|
+
for (const item of verified) {
|
|
609
|
+
const laterGood = laterGoodFor(item.entry.completedMs, item.entry.name);
|
|
610
|
+
out(` verified ${item.entry.name}: later completed-good snapshot ${laterGood ?? "(revalidated during delete)"} proves this recovery evidence is superseded.\n`);
|
|
611
|
+
}
|
|
612
|
+
for (const item of blocked) {
|
|
613
|
+
out(` retained ${item.entry.name}: no later completed-good snapshot exists for this HQ root.\n`);
|
|
614
|
+
}
|
|
615
|
+
for (const item of retainedByGrace) {
|
|
616
|
+
out(` retained ${item.entry.name}: a later completed-good snapshot exists, but normal retention still keeps this newest recovery snapshot.\n`);
|
|
617
|
+
}
|
|
618
|
+
if (verified.length > 0) {
|
|
619
|
+
if (cfg.dryRun) {
|
|
620
|
+
for (const item of verified) {
|
|
621
|
+
out(` would resolve ${item.entry.name}: verified superseded recovery snapshot would be retained during this dry run.\n`);
|
|
622
|
+
}
|
|
623
|
+
}
|
|
624
|
+
else {
|
|
625
|
+
const result = applyBackupPrune(cfg.backupRoot, hqRoot, { candidates: verified, protected: [], unmanaged: [] }, retention);
|
|
626
|
+
for (const name of result.deleted) {
|
|
627
|
+
out(` resolved ${name}: verified superseded recovery snapshot deleted.\n`);
|
|
628
|
+
}
|
|
629
|
+
for (const failure of result.failures) {
|
|
630
|
+
err(`error: could not resolve verified snapshot (retained): ${failure}\n`);
|
|
631
|
+
}
|
|
632
|
+
if (result.failures.length > 0)
|
|
633
|
+
return { status: 1 };
|
|
634
|
+
}
|
|
635
|
+
}
|
|
636
|
+
if (cfg.dryRun && blocked.length === 0) {
|
|
637
|
+
out("==> Snapshot recovery dry run complete: no snapshot was changed.\n");
|
|
638
|
+
return { status: 0 };
|
|
639
|
+
}
|
|
640
|
+
// Re-inventory after an apply because another process might have created or
|
|
641
|
+
// changed a snapshot while the prune lock was held. Nothing unverified has
|
|
642
|
+
// been deleted, and the updated inventory is the report we return to users.
|
|
643
|
+
const remaining = inventoryManagedSnapshots(cfg.backupRoot, hqRoot, isCopyingOwnerLive, retention);
|
|
644
|
+
const unresolved = remaining.protected.filter((item) => item.reason === "recovery-required-no-later-good");
|
|
645
|
+
if (unresolved.length > 0) {
|
|
646
|
+
err("error: snapshot recovery refused: the interrupted updates are not proven superseded.\n");
|
|
647
|
+
err(" HQ retained every unverified snapshot. A later completed-good rescue for this same HQ root is the only automatic proof.\n");
|
|
648
|
+
err(" Restore every captured root from one snapshot, then verify that restored tree before one retry is permitted:\n");
|
|
649
|
+
for (const item of unresolved) {
|
|
650
|
+
err(` ${item.entry.name}: ${formatRecoveryCommand(hqRoot, cfg.backupRoot, item.entry.name)}\n`);
|
|
651
|
+
}
|
|
652
|
+
return { status: 1 };
|
|
653
|
+
}
|
|
654
|
+
out("==> Snapshot recovery complete: every eligible recovery snapshot was positively verified before deletion.\n");
|
|
655
|
+
return { status: 0 };
|
|
656
|
+
}
|
|
657
|
+
/** A live tree must still match the selected recovery snapshot at permit use. */
|
|
658
|
+
function verifyRestoredSnapshot(cfg, hqRoot, entries, protectedEntries, out, err) {
|
|
659
|
+
const entry = entries.find((item) => item.name === cfg.verifyRestoredSnapshot);
|
|
660
|
+
const unresolved = protectedEntries.find((item) => item.entry.name === cfg.verifyRestoredSnapshot &&
|
|
661
|
+
item.reason === "recovery-required-no-later-good");
|
|
662
|
+
if (!entry || !unresolved || entry.manifest.status !== "recovery-required") {
|
|
663
|
+
err(`error: ${cfg.verifyRestoredSnapshot} is not an unresolved recovery-required snapshot for this HQ root.\n`);
|
|
664
|
+
return { status: 1 };
|
|
665
|
+
}
|
|
666
|
+
const mismatch = snapshotTreeMismatch(path.join(cfg.backupRoot, entry.name), hqRoot, entry.manifest.includedRoots);
|
|
667
|
+
if (mismatch) {
|
|
668
|
+
err(`error: restored tree does not match snapshot ${entry.name}: ${mismatch}\n`);
|
|
669
|
+
err(" No post-restore permit was written; the circuit breaker remains armed.\n");
|
|
670
|
+
return { status: 1 };
|
|
671
|
+
}
|
|
672
|
+
if (cfg.dryRun) {
|
|
673
|
+
out(`==> Dry run verified restored snapshot ${entry.name}; no post-restore permit was written.\n`);
|
|
674
|
+
return { status: 0 };
|
|
675
|
+
}
|
|
676
|
+
try {
|
|
677
|
+
writePostRestorePermit(hqRoot, entry);
|
|
678
|
+
}
|
|
679
|
+
catch (error) {
|
|
680
|
+
err(`error: verified snapshot ${entry.name}, but could not write the post-restore permit: ${error instanceof Error ? error.message : String(error)}\n`);
|
|
681
|
+
return { status: 1 };
|
|
682
|
+
}
|
|
683
|
+
out(`==> Verified restored snapshot ${entry.name}; one rescue attempt is now permitted.\n`);
|
|
684
|
+
out(" That permit is rechecked and consumed before rescue changes files. If the attempt fails, the circuit breaker is armed again.\n");
|
|
685
|
+
return { status: 0 };
|
|
686
|
+
}
|
|
687
|
+
/** Return a precise mismatch instead of treating a partial restore as evidence. */
|
|
688
|
+
function snapshotTreeMismatch(snapshotRoot, hqRoot, includedRoots) {
|
|
689
|
+
if (includedRoots.length === 0) {
|
|
690
|
+
return "snapshot manifest records no captured roots to verify";
|
|
691
|
+
}
|
|
692
|
+
const seen = new Set();
|
|
693
|
+
for (const root of includedRoots) {
|
|
694
|
+
if (!root ||
|
|
695
|
+
root === "." ||
|
|
696
|
+
root === ".." ||
|
|
697
|
+
root.includes("/") ||
|
|
698
|
+
root.includes("\\") ||
|
|
699
|
+
isRescueTransientDirectory(root) ||
|
|
700
|
+
seen.has(root)) {
|
|
701
|
+
return `snapshot manifest has an unsafe captured root ${JSON.stringify(root)}`;
|
|
702
|
+
}
|
|
703
|
+
seen.add(root);
|
|
704
|
+
if (rescuePathKind(path.join(snapshotRoot, root)) === "missing") {
|
|
705
|
+
return `snapshot does not contain captured root ${root}`;
|
|
706
|
+
}
|
|
707
|
+
const mismatch = snapshotPathMismatch(path.join(snapshotRoot, root), path.join(hqRoot, root), root);
|
|
708
|
+
if (mismatch)
|
|
709
|
+
return mismatch;
|
|
710
|
+
}
|
|
711
|
+
return null;
|
|
712
|
+
}
|
|
713
|
+
function snapshotPathMismatch(snapshotPath, livePath, rel) {
|
|
714
|
+
const snapshotKind = rescuePathKind(snapshotPath);
|
|
715
|
+
const liveKind = rescuePathKind(livePath);
|
|
716
|
+
if (snapshotKind !== liveKind) {
|
|
717
|
+
return `${rel} is ${liveKind} but the snapshot records ${snapshotKind}`;
|
|
718
|
+
}
|
|
719
|
+
if (snapshotKind === "missing")
|
|
720
|
+
return null;
|
|
721
|
+
if (snapshotKind === "file") {
|
|
722
|
+
return bytesEqual(snapshotPath, livePath)
|
|
723
|
+
? null
|
|
724
|
+
: `${rel} has different file contents`;
|
|
725
|
+
}
|
|
726
|
+
if (snapshotKind === "symlink") {
|
|
727
|
+
try {
|
|
728
|
+
return fs.readlinkSync(snapshotPath) === fs.readlinkSync(livePath)
|
|
729
|
+
? null
|
|
730
|
+
: `${rel} has a different symlink target`;
|
|
731
|
+
}
|
|
732
|
+
catch (error) {
|
|
733
|
+
return `${rel} could not read a symlink target: ${error instanceof Error ? error.message : String(error)}`;
|
|
734
|
+
}
|
|
735
|
+
}
|
|
736
|
+
if (snapshotKind !== "directory") {
|
|
737
|
+
return `${rel} has unsupported path type ${snapshotKind}`;
|
|
738
|
+
}
|
|
739
|
+
let snapshotNames;
|
|
740
|
+
let liveNames;
|
|
741
|
+
try {
|
|
742
|
+
snapshotNames = fs
|
|
743
|
+
.readdirSync(snapshotPath)
|
|
744
|
+
.filter((name) => !isRescueTransientDirectory(path.posix.join(rel, name)))
|
|
745
|
+
.sort();
|
|
746
|
+
liveNames = fs
|
|
747
|
+
.readdirSync(livePath)
|
|
748
|
+
.filter((name) => !isRescueTransientDirectory(path.posix.join(rel, name)))
|
|
749
|
+
.sort();
|
|
750
|
+
}
|
|
751
|
+
catch (error) {
|
|
752
|
+
return `${rel} could not be read: ${error instanceof Error ? error.message : String(error)}`;
|
|
753
|
+
}
|
|
754
|
+
if (snapshotNames.length !== liveNames.length ||
|
|
755
|
+
snapshotNames.some((name, index) => name !== liveNames[index])) {
|
|
756
|
+
return `${rel} has different entries`;
|
|
757
|
+
}
|
|
758
|
+
for (const name of snapshotNames) {
|
|
759
|
+
const mismatch = snapshotPathMismatch(path.join(snapshotPath, name), path.join(livePath, name), path.posix.join(rel, name));
|
|
760
|
+
if (mismatch)
|
|
761
|
+
return mismatch;
|
|
762
|
+
}
|
|
763
|
+
return null;
|
|
764
|
+
}
|
|
493
765
|
/**
|
|
494
766
|
* True when `root` is usable as a rescue target: an absolute path that is not
|
|
495
767
|
* a bare drive letter. Pure + exported for tests; `platformPath` defaults to
|
|
@@ -537,53 +809,7 @@ export function realpathOrExit(p, err) {
|
|
|
537
809
|
}
|
|
538
810
|
}
|
|
539
811
|
function doRescue(cfg, env, out, err, setTmp) {
|
|
540
|
-
|
|
541
|
-
let hqRoot;
|
|
542
|
-
if (cfg.hqRootOverride) {
|
|
543
|
-
// Windows field failure (2026-08-02): the menubar app's daily background
|
|
544
|
-
// core update crash-looped for days because `--hq-root` arrived as the
|
|
545
|
-
// bare drive letter `C:`. `isDir("C:")` is true (drive-relative, resolves
|
|
546
|
-
// against the per-drive cwd) but `fs.realpathSync("C:")` throws EISDIR,
|
|
547
|
-
// so every retry died with an uncaught exception — console-window
|
|
548
|
-
// flicker, git.exe 0xc0000142 dialogs, and no core updates ever landing.
|
|
549
|
-
// Fail closed with a diagnosable message instead: a rescue root must be
|
|
550
|
-
// an absolute directory path, and the error echoes the exact argv this
|
|
551
|
-
// process received so the daily log identifies the caller that mangled
|
|
552
|
-
// the path (the split happens upstream of this process).
|
|
553
|
-
if (!isUsableRescueRoot(cfg.hqRootOverride)) {
|
|
554
|
-
err(`error: --hq-root must be an absolute path, got ${JSON.stringify(cfg.hqRootOverride)}.\n` +
|
|
555
|
-
` argv: ${JSON.stringify(process.argv.slice(2))}\n`);
|
|
556
|
-
throw new ExitError(1);
|
|
557
|
-
}
|
|
558
|
-
if (!isDir(cfg.hqRootOverride)) {
|
|
559
|
-
// bash `cd` failure under set -e would abort; mirror as a generic error.
|
|
560
|
-
err(`error: --hq-root ${cfg.hqRootOverride} is not a directory.\n`);
|
|
561
|
-
throw new ExitError(1);
|
|
562
|
-
}
|
|
563
|
-
hqRoot = realpathOrExit(cfg.hqRootOverride, err);
|
|
564
|
-
}
|
|
565
|
-
else {
|
|
566
|
-
// Legacy default assumed the script lived at personal/skills/<skill>/.
|
|
567
|
-
// The package's programmatic callers always pass --hq-root; fall back to cwd.
|
|
568
|
-
hqRoot = realpathOrExit(process.cwd(), err);
|
|
569
|
-
}
|
|
570
|
-
// HQ-root sanity gate — guards against wiping a non-HQ directory. `companies/`
|
|
571
|
-
// is the stable anchor (present and preserved on every HQ install), confirmed
|
|
572
|
-
// by at least one core scaffold marker. We accept ANY of `.claude/`, `core/`,
|
|
573
|
-
// or `personal/` because the layout has drifted across releases: v14.0.0 ships
|
|
574
|
-
// NEITHER `personal/` (added v15) NOR `core/` (the v15 scaffold home) — it only
|
|
575
|
-
// has `.claude/`. Requiring `personal/` here left v14.0.0 users with no upgrade
|
|
576
|
-
// path (rescue aborted; DEV-1741). `.claude/` exists on every version v14→v15,
|
|
577
|
-
// so it admits the full upgrade range while still rejecting a bare directory.
|
|
578
|
-
const hasHqMarker = isDir(path.join(hqRoot, ".claude")) ||
|
|
579
|
-
isDir(path.join(hqRoot, "core")) ||
|
|
580
|
-
isDir(path.join(hqRoot, "personal"));
|
|
581
|
-
if (!isDir(path.join(hqRoot, "companies")) || !hasHqMarker) {
|
|
582
|
-
err(`error: ${hqRoot} does not look like an HQ root ` +
|
|
583
|
-
`(need companies/ plus one of .claude/, core/, or personal/). Aborting.\n`);
|
|
584
|
-
err(" pass --hq-root <dir> if the script is not at personal/skills/<skill>/.\n");
|
|
585
|
-
throw new ExitError(3);
|
|
586
|
-
}
|
|
812
|
+
const hqRoot = resolveHqRoot(cfg, err);
|
|
587
813
|
const RUN_TS = utcStamp(new Date(), "dash");
|
|
588
814
|
// Both the release overlay and a restorable snapshot require rsync. A shim
|
|
589
815
|
// which merely starts but exits non-zero used to trigger an enormous cp -a
|
|
@@ -1012,6 +1238,15 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
1012
1238
|
throw new ExitError(overlayPreflight.status || 1);
|
|
1013
1239
|
}
|
|
1014
1240
|
ctx.actions = buildRescueActions(ctx);
|
|
1241
|
+
// Permission preflight is deliberately before snapshot allocation. A
|
|
1242
|
+
// root-owned release tree (EACCES on rename/delete) used to allocate a
|
|
1243
|
+
// snapshot, fail apply twice, and then freeze rescue behind the
|
|
1244
|
+
// recovery-required circuit breaker. Fail closed with no snapshot instead.
|
|
1245
|
+
const unwritable = collectUnwritableRescueMutationPaths(hqRoot, ctx.actions);
|
|
1246
|
+
if (unwritable.length > 0) {
|
|
1247
|
+
reportUnwritableRescuePaths(hqRoot, unwritable, err);
|
|
1248
|
+
throw new ExitError(1);
|
|
1249
|
+
}
|
|
1015
1250
|
// Snapshot allocation is deliberately after the entire read-only walk and
|
|
1016
1251
|
// action-plan build. A no-op (or a classifier fault above) cannot consume
|
|
1017
1252
|
// backup space.
|
|
@@ -1025,9 +1260,25 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
1025
1260
|
const prePlan = pruneManagedBackups(cfg, hqRoot, out, false);
|
|
1026
1261
|
const unresolved = prePlan.protected.filter((x) => x.reason === "recovery-required-no-later-good");
|
|
1027
1262
|
if (unresolved.length >= 2) {
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1263
|
+
const postRestore = consumePostRestorePermit(cfg, hqRoot);
|
|
1264
|
+
if (postRestore.used) {
|
|
1265
|
+
out(`==> Verified post-restore permit for ${postRestore.snapshotName}; allowing one rescue attempt.\n`);
|
|
1266
|
+
out(" The permit was consumed before mutation. If this attempt fails, the circuit breaker is armed again.\n");
|
|
1267
|
+
}
|
|
1268
|
+
else {
|
|
1269
|
+
if (postRestore.problem) {
|
|
1270
|
+
err(`error: post-restore permit is not usable: ${postRestore.problem}\n`);
|
|
1271
|
+
}
|
|
1272
|
+
const snapshotNames = unresolved.map((x) => x.entry.name);
|
|
1273
|
+
writeRescueBlockedMarker(hqRoot, snapshotNames);
|
|
1274
|
+
err("HQ_RESCUE_FAILURE_KIND=snapshot-recovery-circuit-breaker\n");
|
|
1275
|
+
err("error: safety snapshot circuit breaker is open: two interrupted updates still need recovery proof.\n");
|
|
1276
|
+
err(" HQ kept their pre-update snapshots because those updates started changing the release-managed tree but did not finish.\n");
|
|
1277
|
+
err(" It will not overwrite or delete that evidence without proof that a later rescue completed successfully.\n");
|
|
1278
|
+
err(` Inspect safely: ${formatRecoveryCommand(hqRoot, cfg.backupRoot)}\n`);
|
|
1279
|
+
err(" It deletes only snapshots with a later completed-good rescue. For unresolved snapshots, restore one listed snapshot and run its verification command before retrying.\n");
|
|
1280
|
+
throw new ExitError(1);
|
|
1281
|
+
}
|
|
1031
1282
|
}
|
|
1032
1283
|
requireBackupCapacity(cfg, hqRoot, wipeToplevel, env, err);
|
|
1033
1284
|
fs.mkdirSync(cfg.backupRoot, { recursive: true });
|
|
@@ -1319,6 +1570,7 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
1319
1570
|
if (cfg.doBackup && backupDir) {
|
|
1320
1571
|
out(` A filtered pre-update snapshot is at ${backupDir} (RECOVERY.md explains restore).\n`);
|
|
1321
1572
|
}
|
|
1573
|
+
clearRescueBlockedMarker(hqRoot);
|
|
1322
1574
|
return { status: 0 };
|
|
1323
1575
|
}
|
|
1324
1576
|
catch (error) {
|
|
@@ -2410,10 +2662,11 @@ function renderDryRunDecision(ctx, decision) {
|
|
|
2410
2662
|
}
|
|
2411
2663
|
function buildRescueActions(ctx) {
|
|
2412
2664
|
const actions = [];
|
|
2413
|
-
const add = (label, affectedRels, runAction) => {
|
|
2665
|
+
const add = (label, affectedRels, runAction, destParentRels = []) => {
|
|
2414
2666
|
actions.push({
|
|
2415
2667
|
label,
|
|
2416
2668
|
affectedRels,
|
|
2669
|
+
destParentRels,
|
|
2417
2670
|
expectedPaths: affectedRels.map((rel) => ({
|
|
2418
2671
|
rel,
|
|
2419
2672
|
kind: rescuePathKind(path.join(ctx.hqRoot, rel)),
|
|
@@ -2452,7 +2705,7 @@ function buildRescueActions(ctx) {
|
|
|
2452
2705
|
});
|
|
2453
2706
|
break;
|
|
2454
2707
|
case "user-edit-diff-append":
|
|
2455
|
-
add(`rescue diff-append: ${decision.rel}`, [decision.rel], () => rescueOne(ctx, decision.rel));
|
|
2708
|
+
add(`rescue diff-append: ${decision.rel}`, [decision.rel], () => rescueOne(ctx, decision.rel), [path.posix.dirname(mapRescueTarget(decision.rel))]);
|
|
2456
2709
|
break;
|
|
2457
2710
|
case "user-edit-overwrite-safe":
|
|
2458
2711
|
add(`overwrite-safe: ${decision.rel}`, [decision.rel], () => {
|
|
@@ -2473,10 +2726,12 @@ function buildRescueActions(ctx) {
|
|
|
2473
2726
|
});
|
|
2474
2727
|
break;
|
|
2475
2728
|
case "user-edit-conflict":
|
|
2476
|
-
add(`conflict quarantine: ${decision.rel}`, [decision.rel], () => conflictOne(ctx, decision.rel)
|
|
2729
|
+
add(`conflict quarantine: ${decision.rel}`, [decision.rel], () => conflictOne(ctx, decision.rel), [
|
|
2730
|
+
path.posix.dirname(`.hq-conflicts/rescue-${ctx.runTs}/${decision.rel}`),
|
|
2731
|
+
]);
|
|
2477
2732
|
break;
|
|
2478
2733
|
case "user-edit-rescue":
|
|
2479
|
-
add(`rescue: ${decision.rel}`, [decision.rel], () => rescueOne(ctx, decision.rel));
|
|
2734
|
+
add(`rescue: ${decision.rel}`, [decision.rel], () => rescueOne(ctx, decision.rel), [path.posix.dirname(mapRescueTarget(decision.rel))]);
|
|
2480
2735
|
break;
|
|
2481
2736
|
case "unchanged-preserve":
|
|
2482
2737
|
add(`preserve unchanged: ${decision.rel}`, [], () => {
|
|
@@ -2749,6 +3004,246 @@ function restoreMtimesFromGit(srcDir, env, out) {
|
|
|
2749
3004
|
}
|
|
2750
3005
|
}
|
|
2751
3006
|
}
|
|
3007
|
+
const RESCUE_BLOCKED_MARKER_REL = ".hq/rescue-blocked.json";
|
|
3008
|
+
const RESCUE_BLOCKED_REASON = "recovery-snapshot-breaker";
|
|
3009
|
+
const POST_RESTORE_PERMIT_REL = ".hq/rescue-post-restore.json";
|
|
3010
|
+
function posixShellQuote(value) {
|
|
3011
|
+
return `'${value.replace(/'/g, `'\\''`)}'`;
|
|
3012
|
+
}
|
|
3013
|
+
/** Format the exact safe command presented in a breaker diagnostic. */
|
|
3014
|
+
export function formatRecoveryCommand(hqRoot, backupRoot, verifyRestoredSnapshot) {
|
|
3015
|
+
const args = [
|
|
3016
|
+
"npx",
|
|
3017
|
+
"-y",
|
|
3018
|
+
"--package=@indigoai-us/hq-cloud@latest",
|
|
3019
|
+
"hq-rescue",
|
|
3020
|
+
"--recover-snapshots",
|
|
3021
|
+
"--hq-root",
|
|
3022
|
+
posixShellQuote(hqRoot),
|
|
3023
|
+
"--backup-dir",
|
|
3024
|
+
posixShellQuote(backupRoot),
|
|
3025
|
+
];
|
|
3026
|
+
if (verifyRestoredSnapshot) {
|
|
3027
|
+
args.push("--verify-restored-snapshot", posixShellQuote(verifyRestoredSnapshot));
|
|
3028
|
+
}
|
|
3029
|
+
return args.join(" ");
|
|
3030
|
+
}
|
|
3031
|
+
function posixRelFromRoot(hqRoot, abs) {
|
|
3032
|
+
const root = path.resolve(hqRoot);
|
|
3033
|
+
const resolved = path.resolve(abs);
|
|
3034
|
+
if (resolved === root)
|
|
3035
|
+
return ".";
|
|
3036
|
+
const prefix = root.endsWith(path.sep) ? root : root + path.sep;
|
|
3037
|
+
if (resolved.startsWith(prefix)) {
|
|
3038
|
+
return resolved.slice(prefix.length).split(path.sep).join("/");
|
|
3039
|
+
}
|
|
3040
|
+
return resolved;
|
|
3041
|
+
}
|
|
3042
|
+
function firstExistingAncestor(abs) {
|
|
3043
|
+
let current = abs;
|
|
3044
|
+
for (;;) {
|
|
3045
|
+
if (lexists(current))
|
|
3046
|
+
return current;
|
|
3047
|
+
const parent = path.dirname(current);
|
|
3048
|
+
if (parent === current)
|
|
3049
|
+
return null;
|
|
3050
|
+
current = parent;
|
|
3051
|
+
}
|
|
3052
|
+
}
|
|
3053
|
+
function canWrite(abs) {
|
|
3054
|
+
try {
|
|
3055
|
+
fs.accessSync(abs, fs.constants.W_OK);
|
|
3056
|
+
return true;
|
|
3057
|
+
}
|
|
3058
|
+
catch {
|
|
3059
|
+
return false;
|
|
3060
|
+
}
|
|
3061
|
+
}
|
|
3062
|
+
function topLevelRel(rel) {
|
|
3063
|
+
const normalized = rel.replace(/\\/g, "/").replace(/^\.\//, "");
|
|
3064
|
+
if (normalized === "." || normalized === "")
|
|
3065
|
+
return ".";
|
|
3066
|
+
return normalized.split("/")[0] ?? ".";
|
|
3067
|
+
}
|
|
3068
|
+
/**
|
|
3069
|
+
* Paths the current process cannot write among the action plan's rename/delete
|
|
3070
|
+
* sources (and their parents) plus rename destination parents. Empty means the
|
|
3071
|
+
* plan is writable; a non-empty result must refuse before allocating a snapshot.
|
|
3072
|
+
*/
|
|
3073
|
+
export function collectUnwritableRescueMutationPaths(hqRoot, actions) {
|
|
3074
|
+
const failed = new Set();
|
|
3075
|
+
const note = (abs) => {
|
|
3076
|
+
failed.add(posixRelFromRoot(hqRoot, abs));
|
|
3077
|
+
};
|
|
3078
|
+
const checkParent = (abs) => {
|
|
3079
|
+
const existing = firstExistingAncestor(abs);
|
|
3080
|
+
if (!existing || !canWrite(existing))
|
|
3081
|
+
note(existing ?? abs);
|
|
3082
|
+
};
|
|
3083
|
+
for (const action of actions) {
|
|
3084
|
+
for (const rel of action.affectedRels) {
|
|
3085
|
+
if (!rel)
|
|
3086
|
+
continue;
|
|
3087
|
+
const abs = path.join(hqRoot, rel);
|
|
3088
|
+
const st = lstatOrNull(abs);
|
|
3089
|
+
if (st && !st.isSymbolicLink() && !canWrite(abs))
|
|
3090
|
+
note(abs);
|
|
3091
|
+
checkParent(path.dirname(abs));
|
|
3092
|
+
}
|
|
3093
|
+
for (const rel of action.destParentRels ?? []) {
|
|
3094
|
+
if (!rel || rel === ".")
|
|
3095
|
+
continue;
|
|
3096
|
+
checkParent(path.join(hqRoot, rel));
|
|
3097
|
+
}
|
|
3098
|
+
}
|
|
3099
|
+
return [...failed].sort();
|
|
3100
|
+
}
|
|
3101
|
+
function reportUnwritableRescuePaths(hqRoot, unwritable, err) {
|
|
3102
|
+
const shown = unwritable.slice(0, 10);
|
|
3103
|
+
err("error: cannot write the paths rescue would rename or delete (permission denied).\n");
|
|
3104
|
+
err(" Refusing before a safety snapshot is allocated so this cannot trip the recovery-snapshot circuit breaker.\n");
|
|
3105
|
+
err(unwritable.length > 10
|
|
3106
|
+
? ` Unwritable paths (showing 10 of ${unwritable.length}):\n`
|
|
3107
|
+
: " Unwritable paths:\n");
|
|
3108
|
+
for (const rel of shown)
|
|
3109
|
+
err(` ${rel}\n`);
|
|
3110
|
+
const uid = typeof process.getuid === "function" ? String(process.getuid()) : "<uid>";
|
|
3111
|
+
const gid = typeof process.getgid === "function" ? String(process.getgid()) : "<gid>";
|
|
3112
|
+
const tops = [
|
|
3113
|
+
...new Set(unwritable.map((rel) => topLevelRel(rel)).filter((top) => top !== ".")),
|
|
3114
|
+
].sort();
|
|
3115
|
+
err(" Repair ownership, then retry:\n");
|
|
3116
|
+
if (tops.length === 0) {
|
|
3117
|
+
err(` chown -R ${uid}:${gid} ${hqRoot}\n`);
|
|
3118
|
+
}
|
|
3119
|
+
else {
|
|
3120
|
+
for (const top of tops) {
|
|
3121
|
+
err(` chown -R ${uid}:${gid} ${path.join(hqRoot, top)}\n`);
|
|
3122
|
+
}
|
|
3123
|
+
}
|
|
3124
|
+
}
|
|
3125
|
+
function rescueBlockedMarkerPath(hqRoot) {
|
|
3126
|
+
return path.join(hqRoot, RESCUE_BLOCKED_MARKER_REL);
|
|
3127
|
+
}
|
|
3128
|
+
function postRestorePermitPath(hqRoot) {
|
|
3129
|
+
return path.join(hqRoot, POST_RESTORE_PERMIT_REL);
|
|
3130
|
+
}
|
|
3131
|
+
function writePostRestorePermit(hqRoot, entry) {
|
|
3132
|
+
const permit = {
|
|
3133
|
+
schemaVersion: 1,
|
|
3134
|
+
canonicalHqRoot: hqRoot,
|
|
3135
|
+
rootHash: canonicalRootHash(hqRoot),
|
|
3136
|
+
snapshotName: entry.name,
|
|
3137
|
+
snapshotRunId: entry.manifest.runId,
|
|
3138
|
+
verifiedAt: new Date().toISOString(),
|
|
3139
|
+
};
|
|
3140
|
+
const target = postRestorePermitPath(hqRoot);
|
|
3141
|
+
const temporary = `${target}.${process.pid}.tmp`;
|
|
3142
|
+
fs.mkdirSync(path.dirname(target), { recursive: true });
|
|
3143
|
+
fs.writeFileSync(temporary, JSON.stringify(permit, null, 2) + "\n", {
|
|
3144
|
+
mode: 0o600,
|
|
3145
|
+
});
|
|
3146
|
+
fs.renameSync(temporary, target);
|
|
3147
|
+
}
|
|
3148
|
+
function readPostRestorePermit(hqRoot) {
|
|
3149
|
+
let raw;
|
|
3150
|
+
try {
|
|
3151
|
+
raw = fs.readFileSync(postRestorePermitPath(hqRoot), "utf8");
|
|
3152
|
+
}
|
|
3153
|
+
catch (error) {
|
|
3154
|
+
if (error.code === "ENOENT") {
|
|
3155
|
+
return { permit: null };
|
|
3156
|
+
}
|
|
3157
|
+
return {
|
|
3158
|
+
permit: null,
|
|
3159
|
+
problem: `could not read ${POST_RESTORE_PERMIT_REL}: ${error instanceof Error ? error.message : String(error)}`,
|
|
3160
|
+
};
|
|
3161
|
+
}
|
|
3162
|
+
try {
|
|
3163
|
+
const value = JSON.parse(raw);
|
|
3164
|
+
if (value.schemaVersion !== 1 ||
|
|
3165
|
+
typeof value.canonicalHqRoot !== "string" ||
|
|
3166
|
+
typeof value.rootHash !== "string" ||
|
|
3167
|
+
typeof value.snapshotName !== "string" ||
|
|
3168
|
+
typeof value.snapshotRunId !== "string" ||
|
|
3169
|
+
typeof value.verifiedAt !== "string") {
|
|
3170
|
+
return { permit: null, problem: `${POST_RESTORE_PERMIT_REL} is malformed` };
|
|
3171
|
+
}
|
|
3172
|
+
return { permit: value };
|
|
3173
|
+
}
|
|
3174
|
+
catch (error) {
|
|
3175
|
+
return {
|
|
3176
|
+
permit: null,
|
|
3177
|
+
problem: `${POST_RESTORE_PERMIT_REL} is not valid JSON: ${error instanceof Error ? error.message : String(error)}`,
|
|
3178
|
+
};
|
|
3179
|
+
}
|
|
3180
|
+
}
|
|
3181
|
+
function consumePostRestorePermit(cfg, hqRoot) {
|
|
3182
|
+
const read = readPostRestorePermit(hqRoot);
|
|
3183
|
+
if (!read.permit) {
|
|
3184
|
+
return { used: false, ...(read.problem ? { problem: read.problem } : {}) };
|
|
3185
|
+
}
|
|
3186
|
+
const permit = read.permit;
|
|
3187
|
+
if (permit.canonicalHqRoot !== hqRoot ||
|
|
3188
|
+
permit.rootHash !== canonicalRootHash(hqRoot)) {
|
|
3189
|
+
return { used: false, problem: "permit belongs to a different HQ root" };
|
|
3190
|
+
}
|
|
3191
|
+
const inventory = inventoryManagedSnapshots(cfg.backupRoot, hqRoot, () => false, {
|
|
3192
|
+
keepGood: Number(cfg.backupRetentionCount),
|
|
3193
|
+
maxAgeDays: Number(cfg.backupRetentionDays),
|
|
3194
|
+
});
|
|
3195
|
+
if (inventory.unmanaged.some((item) => item.reason === "backup-root-unreadable")) {
|
|
3196
|
+
return { used: false, problem: `backup root is unreadable: ${cfg.backupRoot}` };
|
|
3197
|
+
}
|
|
3198
|
+
const entry = inventory.entries.find((item) => item.name === permit.snapshotName &&
|
|
3199
|
+
item.manifest.runId === permit.snapshotRunId &&
|
|
3200
|
+
item.manifest.status === "recovery-required");
|
|
3201
|
+
const unresolved = inventory.protected.some((item) => item.entry.name === permit.snapshotName &&
|
|
3202
|
+
item.reason === "recovery-required-no-later-good");
|
|
3203
|
+
if (!entry || !unresolved) {
|
|
3204
|
+
return { used: false, problem: "referenced snapshot is no longer unresolved recovery evidence" };
|
|
3205
|
+
}
|
|
3206
|
+
const mismatch = snapshotTreeMismatch(path.join(cfg.backupRoot, entry.name), hqRoot, entry.manifest.includedRoots);
|
|
3207
|
+
if (mismatch) {
|
|
3208
|
+
return { used: false, problem: `live tree no longer matches ${entry.name}: ${mismatch}` };
|
|
3209
|
+
}
|
|
3210
|
+
try {
|
|
3211
|
+
fs.unlinkSync(postRestorePermitPath(hqRoot));
|
|
3212
|
+
}
|
|
3213
|
+
catch (error) {
|
|
3214
|
+
return {
|
|
3215
|
+
used: false,
|
|
3216
|
+
problem: `could not consume the single-use permit: ${error instanceof Error ? error.message : String(error)}`,
|
|
3217
|
+
};
|
|
3218
|
+
}
|
|
3219
|
+
return { used: true, snapshotName: entry.name };
|
|
3220
|
+
}
|
|
3221
|
+
function writeRescueBlockedMarker(hqRoot, snapshots) {
|
|
3222
|
+
const body = {
|
|
3223
|
+
at: new Date().toISOString(),
|
|
3224
|
+
reason: RESCUE_BLOCKED_REASON,
|
|
3225
|
+
failureKind: "snapshot-recovery-circuit-breaker",
|
|
3226
|
+
snapshots,
|
|
3227
|
+
hint: "Run hq-rescue --recover-snapshots to inspect the snapshots. It deletes only snapshots proved superseded by a later completed-good rescue; otherwise restore the needed files, repair the release source, and retry.",
|
|
3228
|
+
};
|
|
3229
|
+
try {
|
|
3230
|
+
fs.mkdirSync(path.dirname(rescueBlockedMarkerPath(hqRoot)), {
|
|
3231
|
+
recursive: true,
|
|
3232
|
+
});
|
|
3233
|
+
fs.writeFileSync(rescueBlockedMarkerPath(hqRoot), JSON.stringify(body, null, 2) + "\n");
|
|
3234
|
+
}
|
|
3235
|
+
catch {
|
|
3236
|
+
/* stderr remains the operator-facing signal if .hq is unwritable */
|
|
3237
|
+
}
|
|
3238
|
+
}
|
|
3239
|
+
function clearRescueBlockedMarker(hqRoot) {
|
|
3240
|
+
try {
|
|
3241
|
+
fs.unlinkSync(rescueBlockedMarkerPath(hqRoot));
|
|
3242
|
+
}
|
|
3243
|
+
catch {
|
|
3244
|
+
/* absent is the success state */
|
|
3245
|
+
}
|
|
3246
|
+
}
|
|
2752
3247
|
function allocateSnapshotDir(backupRoot, runTs, rootHash, runId) {
|
|
2753
3248
|
return path.join(backupRoot, `pre-update-${runTs}-${rootHash}-${runId}`);
|
|
2754
3249
|
}
|