@indigoai-us/hq-cloud 6.16.49 → 6.16.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/rescue-runner.js +2 -1
- package/dist/bin/rescue-runner.js.map +1 -1
- package/dist/cli/rescue-breaker.test.js +44 -24
- package/dist/cli/rescue-breaker.test.js.map +1 -1
- package/dist/cli/rescue-core.d.ts +4 -2
- package/dist/cli/rescue-core.d.ts.map +1 -1
- package/dist/cli/rescue-core.js +407 -71
- package/dist/cli/rescue-core.js.map +1 -1
- package/dist/cli/rescue-snapshot-recovery.test.d.ts +2 -0
- package/dist/cli/rescue-snapshot-recovery.test.d.ts.map +1 -0
- package/dist/cli/rescue-snapshot-recovery.test.js +260 -0
- package/dist/cli/rescue-snapshot-recovery.test.js.map +1 -0
- package/dist/cli/rescue.d.ts +4 -6
- package/dist/cli/rescue.d.ts.map +1 -1
- package/dist/cli/rescue.js +14 -7
- package/dist/cli/rescue.js.map +1 -1
- package/dist/cli/rescue.test.js +14 -5
- package/dist/cli/rescue.test.js.map +1 -1
- package/package.json +1 -1
package/dist/cli/rescue-core.js
CHANGED
|
@@ -327,7 +327,8 @@ export function parseRescueArgs(argv) {
|
|
|
327
327
|
backupRoot: process.env.HQ_BACKUP_DIR || path.join(os.homedir(), ".hq", "backups"),
|
|
328
328
|
backupRetentionDays: process.env.HQ_BACKUP_RETENTION_DAYS || "7",
|
|
329
329
|
backupRetentionCount: process.env.HQ_BACKUP_RETENTION_COUNT || "2",
|
|
330
|
-
|
|
330
|
+
recoverSnapshots: false,
|
|
331
|
+
verifyRestoredSnapshot: "",
|
|
331
332
|
};
|
|
332
333
|
const usage = () => {
|
|
333
334
|
// The legacy script printed its comment header here; we emit a concise
|
|
@@ -336,7 +337,8 @@ export function parseRescueArgs(argv) {
|
|
|
336
337
|
" [--preserve PATH]... [--preserve-subpath REL]...\n" +
|
|
337
338
|
" [--hq-root DIR] [--no-history-check] [--floor-sha SHA]\n" +
|
|
338
339
|
" [--no-backup] [--backup-dir DIR] [--cloud-update]\n" +
|
|
339
|
-
" [--
|
|
340
|
+
" [--dry-run] [--yes] [--recover-snapshots]\n" +
|
|
341
|
+
" [--verify-restored-snapshot SNAPSHOT]\n");
|
|
340
342
|
throw new ExitError(1);
|
|
341
343
|
};
|
|
342
344
|
let narrowCsv = "";
|
|
@@ -385,8 +387,15 @@ export function parseRescueArgs(argv) {
|
|
|
385
387
|
case "--backup-dir":
|
|
386
388
|
cfg.backupRoot = need(a);
|
|
387
389
|
break;
|
|
388
|
-
case "--
|
|
389
|
-
cfg.
|
|
390
|
+
case "--recover-snapshots":
|
|
391
|
+
cfg.recoverSnapshots = true;
|
|
392
|
+
break;
|
|
393
|
+
case "--verify-restored-snapshot":
|
|
394
|
+
cfg.verifyRestoredSnapshot = need(a);
|
|
395
|
+
if (!cfg.verifyRestoredSnapshot) {
|
|
396
|
+
process.stderr.write("error: --verify-restored-snapshot requires a snapshot name.\n");
|
|
397
|
+
throw new ExitError(2);
|
|
398
|
+
}
|
|
390
399
|
break;
|
|
391
400
|
case "--dry-run":
|
|
392
401
|
cfg.dryRun = true;
|
|
@@ -448,6 +457,10 @@ export function parseRescueArgs(argv) {
|
|
|
448
457
|
throw new ExitError(2);
|
|
449
458
|
}
|
|
450
459
|
}
|
|
460
|
+
if (cfg.verifyRestoredSnapshot && !cfg.recoverSnapshots) {
|
|
461
|
+
process.stderr.write("error: --verify-restored-snapshot requires --recover-snapshots.\n");
|
|
462
|
+
throw new ExitError(2);
|
|
463
|
+
}
|
|
451
464
|
return cfg;
|
|
452
465
|
}
|
|
453
466
|
/**
|
|
@@ -481,6 +494,8 @@ export function runRescue(argv, opts = {}) {
|
|
|
481
494
|
}
|
|
482
495
|
};
|
|
483
496
|
try {
|
|
497
|
+
if (cfg.recoverSnapshots)
|
|
498
|
+
return recoverSnapshots(cfg, out, err);
|
|
484
499
|
return doRescue(cfg, env, out, err, (d) => {
|
|
485
500
|
tmpdir = d;
|
|
486
501
|
});
|
|
@@ -494,6 +509,259 @@ export function runRescue(argv, opts = {}) {
|
|
|
494
509
|
cleanup();
|
|
495
510
|
}
|
|
496
511
|
}
|
|
512
|
+
/** Resolve and sanity-check the HQ root before either rescue mode can touch it. */
|
|
513
|
+
function resolveHqRoot(cfg, err) {
|
|
514
|
+
let hqRoot;
|
|
515
|
+
if (cfg.hqRootOverride) {
|
|
516
|
+
// Windows field failure (2026-08-02): the menubar app's daily background
|
|
517
|
+
// core update crash-looped for days because `--hq-root` arrived as the
|
|
518
|
+
// bare drive letter `C:`. `isDir("C:")` is true (drive-relative, resolves
|
|
519
|
+
// against the per-drive cwd) but `fs.realpathSync("C:")` throws EISDIR,
|
|
520
|
+
// so every retry died with an uncaught exception — console-window
|
|
521
|
+
// flicker, git.exe 0xc0000142 dialogs, and no core updates ever landing.
|
|
522
|
+
// Fail closed with a diagnosable message instead: a rescue root must be
|
|
523
|
+
// an absolute directory path, and the error echoes the exact argv this
|
|
524
|
+
// process received so the daily log identifies the caller that mangled
|
|
525
|
+
// the path (the split happens upstream of this process).
|
|
526
|
+
if (!isUsableRescueRoot(cfg.hqRootOverride)) {
|
|
527
|
+
err(`error: --hq-root must be an absolute path, got ${JSON.stringify(cfg.hqRootOverride)}.\n` +
|
|
528
|
+
` argv: ${JSON.stringify(process.argv.slice(2))}\n`);
|
|
529
|
+
throw new ExitError(1);
|
|
530
|
+
}
|
|
531
|
+
if (!isDir(cfg.hqRootOverride)) {
|
|
532
|
+
// bash `cd` failure under set -e would abort; mirror as a generic error.
|
|
533
|
+
err(`error: --hq-root ${cfg.hqRootOverride} is not a directory.\n`);
|
|
534
|
+
throw new ExitError(1);
|
|
535
|
+
}
|
|
536
|
+
hqRoot = realpathOrExit(cfg.hqRootOverride, err);
|
|
537
|
+
}
|
|
538
|
+
else {
|
|
539
|
+
// Legacy default assumed the script lived at personal/skills/<skill>/.
|
|
540
|
+
// The package's programmatic callers always pass --hq-root; fall back to cwd.
|
|
541
|
+
hqRoot = realpathOrExit(process.cwd(), err);
|
|
542
|
+
}
|
|
543
|
+
// HQ-root sanity gate — guards against wiping a non-HQ directory. `companies/`
|
|
544
|
+
// is the stable anchor (present and preserved on every HQ install), confirmed
|
|
545
|
+
// by at least one core scaffold marker. We accept ANY of `.claude/`, `core/`,
|
|
546
|
+
// or `personal/` because the layout has drifted across releases: v14.0.0 ships
|
|
547
|
+
// NEITHER `personal/` (added v15) NOR `core/` (the v15 scaffold home) — it only
|
|
548
|
+
// has `.claude/`. Requiring `personal/` here left v14.0.0 users with no upgrade
|
|
549
|
+
// path (rescue aborted; DEV-1741). `.claude/` exists on every version v14→v15,
|
|
550
|
+
// so it admits the full upgrade range while still rejecting a bare directory.
|
|
551
|
+
const hasHqMarker = isDir(path.join(hqRoot, ".claude")) ||
|
|
552
|
+
isDir(path.join(hqRoot, "core")) ||
|
|
553
|
+
isDir(path.join(hqRoot, "personal"));
|
|
554
|
+
if (!isDir(path.join(hqRoot, "companies")) || !hasHqMarker) {
|
|
555
|
+
err(`error: ${hqRoot} does not look like an HQ root ` +
|
|
556
|
+
`(need companies/ plus one of .claude/, core/, or personal/). Aborting.\n`);
|
|
557
|
+
err(" pass --hq-root <dir> if the script is not at personal/skills/<skill>/.\n");
|
|
558
|
+
throw new ExitError(3);
|
|
559
|
+
}
|
|
560
|
+
return hqRoot;
|
|
561
|
+
}
|
|
562
|
+
/**
|
|
563
|
+
* Explicit recovery surface for a previously interrupted rescue.
|
|
564
|
+
*
|
|
565
|
+
* A recovery-required snapshot is evidence that a rescue started changing the
|
|
566
|
+
* release-managed tree and then failed. It is safe to delete only when the
|
|
567
|
+
* retention model has a later `completed-good` snapshot for this same canonical
|
|
568
|
+
* HQ root. That later manifest is the durable proof that a subsequent rescue
|
|
569
|
+
* completed; an age, an empty current tree, or a manually edited sync stamp is
|
|
570
|
+
* deliberately not accepted as proof.
|
|
571
|
+
*/
|
|
572
|
+
function recoverSnapshots(cfg, out, err) {
|
|
573
|
+
const hqRoot = resolveHqRoot(cfg, err);
|
|
574
|
+
const retention = {
|
|
575
|
+
keepGood: Number(cfg.backupRetentionCount),
|
|
576
|
+
maxAgeDays: Number(cfg.backupRetentionDays),
|
|
577
|
+
};
|
|
578
|
+
const isCopyingOwnerLive = (m) => {
|
|
579
|
+
try {
|
|
580
|
+
process.kill(m.creatorPid, 0);
|
|
581
|
+
return true;
|
|
582
|
+
}
|
|
583
|
+
catch {
|
|
584
|
+
return false;
|
|
585
|
+
}
|
|
586
|
+
};
|
|
587
|
+
const inventory = inventoryManagedSnapshots(cfg.backupRoot, hqRoot, isCopyingOwnerLive, retention);
|
|
588
|
+
if (inventory.unmanaged.some((item) => item.reason === "backup-root-unreadable")) {
|
|
589
|
+
err(`error: cannot inspect safety snapshots: backup root is unreadable: ${cfg.backupRoot}\n`);
|
|
590
|
+
return { status: 1 };
|
|
591
|
+
}
|
|
592
|
+
if (cfg.verifyRestoredSnapshot) {
|
|
593
|
+
return verifyRestoredSnapshot(cfg, hqRoot, inventory.entries, inventory.protected, out, err);
|
|
594
|
+
}
|
|
595
|
+
const verified = inventory.candidates.filter((item) => item.entry.manifest.status === "recovery-required" &&
|
|
596
|
+
item.reason === "recovery-required-superseded");
|
|
597
|
+
const blocked = inventory.protected.filter((item) => item.reason === "recovery-required-no-later-good");
|
|
598
|
+
const retainedByGrace = inventory.protected.filter((item) => item.entry.manifest.status === "recovery-required" &&
|
|
599
|
+
item.reason !== "recovery-required-no-later-good");
|
|
600
|
+
const laterGoodFor = (completedMs, name) => inventory.entries.find((entry) => entry.manifest.status === "completed-good" &&
|
|
601
|
+
(entry.completedMs > completedMs ||
|
|
602
|
+
(entry.completedMs === completedMs && entry.name > name)))?.name;
|
|
603
|
+
out(`==> Safety snapshot recovery inspection: ${cfg.backupRoot}\n`);
|
|
604
|
+
if (verified.length === 0 && blocked.length === 0 && retainedByGrace.length === 0) {
|
|
605
|
+
out(" No recovery-required snapshots need action.\n");
|
|
606
|
+
return { status: 0 };
|
|
607
|
+
}
|
|
608
|
+
for (const item of verified) {
|
|
609
|
+
const laterGood = laterGoodFor(item.entry.completedMs, item.entry.name);
|
|
610
|
+
out(` verified ${item.entry.name}: later completed-good snapshot ${laterGood ?? "(revalidated during delete)"} proves this recovery evidence is superseded.\n`);
|
|
611
|
+
}
|
|
612
|
+
for (const item of blocked) {
|
|
613
|
+
out(` retained ${item.entry.name}: no later completed-good snapshot exists for this HQ root.\n`);
|
|
614
|
+
}
|
|
615
|
+
for (const item of retainedByGrace) {
|
|
616
|
+
out(` retained ${item.entry.name}: a later completed-good snapshot exists, but normal retention still keeps this newest recovery snapshot.\n`);
|
|
617
|
+
}
|
|
618
|
+
if (verified.length > 0) {
|
|
619
|
+
if (cfg.dryRun) {
|
|
620
|
+
for (const item of verified) {
|
|
621
|
+
out(` would resolve ${item.entry.name}: verified superseded recovery snapshot would be retained during this dry run.\n`);
|
|
622
|
+
}
|
|
623
|
+
}
|
|
624
|
+
else {
|
|
625
|
+
const result = applyBackupPrune(cfg.backupRoot, hqRoot, { candidates: verified, protected: [], unmanaged: [] }, retention);
|
|
626
|
+
for (const name of result.deleted) {
|
|
627
|
+
out(` resolved ${name}: verified superseded recovery snapshot deleted.\n`);
|
|
628
|
+
}
|
|
629
|
+
for (const failure of result.failures) {
|
|
630
|
+
err(`error: could not resolve verified snapshot (retained): ${failure}\n`);
|
|
631
|
+
}
|
|
632
|
+
if (result.failures.length > 0)
|
|
633
|
+
return { status: 1 };
|
|
634
|
+
}
|
|
635
|
+
}
|
|
636
|
+
if (cfg.dryRun && blocked.length === 0) {
|
|
637
|
+
out("==> Snapshot recovery dry run complete: no snapshot was changed.\n");
|
|
638
|
+
return { status: 0 };
|
|
639
|
+
}
|
|
640
|
+
// Re-inventory after an apply because another process might have created or
|
|
641
|
+
// changed a snapshot while the prune lock was held. Nothing unverified has
|
|
642
|
+
// been deleted, and the updated inventory is the report we return to users.
|
|
643
|
+
const remaining = inventoryManagedSnapshots(cfg.backupRoot, hqRoot, isCopyingOwnerLive, retention);
|
|
644
|
+
const unresolved = remaining.protected.filter((item) => item.reason === "recovery-required-no-later-good");
|
|
645
|
+
if (unresolved.length > 0) {
|
|
646
|
+
err("error: snapshot recovery refused: the interrupted updates are not proven superseded.\n");
|
|
647
|
+
err(" HQ retained every unverified snapshot. A later completed-good rescue for this same HQ root is the only automatic proof.\n");
|
|
648
|
+
err(" Restore every captured root from one snapshot, then verify that restored tree before one retry is permitted:\n");
|
|
649
|
+
for (const item of unresolved) {
|
|
650
|
+
err(` ${item.entry.name}: ${formatRecoveryCommand(hqRoot, cfg.backupRoot, item.entry.name)}\n`);
|
|
651
|
+
}
|
|
652
|
+
return { status: 1 };
|
|
653
|
+
}
|
|
654
|
+
out("==> Snapshot recovery complete: every eligible recovery snapshot was positively verified before deletion.\n");
|
|
655
|
+
return { status: 0 };
|
|
656
|
+
}
|
|
657
|
+
/** A live tree must still match the selected recovery snapshot at permit use. */
|
|
658
|
+
function verifyRestoredSnapshot(cfg, hqRoot, entries, protectedEntries, out, err) {
|
|
659
|
+
const entry = entries.find((item) => item.name === cfg.verifyRestoredSnapshot);
|
|
660
|
+
const unresolved = protectedEntries.find((item) => item.entry.name === cfg.verifyRestoredSnapshot &&
|
|
661
|
+
item.reason === "recovery-required-no-later-good");
|
|
662
|
+
if (!entry || !unresolved || entry.manifest.status !== "recovery-required") {
|
|
663
|
+
err(`error: ${cfg.verifyRestoredSnapshot} is not an unresolved recovery-required snapshot for this HQ root.\n`);
|
|
664
|
+
return { status: 1 };
|
|
665
|
+
}
|
|
666
|
+
const mismatch = snapshotTreeMismatch(path.join(cfg.backupRoot, entry.name), hqRoot, entry.manifest.includedRoots);
|
|
667
|
+
if (mismatch) {
|
|
668
|
+
err(`error: restored tree does not match snapshot ${entry.name}: ${mismatch}\n`);
|
|
669
|
+
err(" No post-restore permit was written; the circuit breaker remains armed.\n");
|
|
670
|
+
return { status: 1 };
|
|
671
|
+
}
|
|
672
|
+
if (cfg.dryRun) {
|
|
673
|
+
out(`==> Dry run verified restored snapshot ${entry.name}; no post-restore permit was written.\n`);
|
|
674
|
+
return { status: 0 };
|
|
675
|
+
}
|
|
676
|
+
try {
|
|
677
|
+
writePostRestorePermit(hqRoot, entry);
|
|
678
|
+
}
|
|
679
|
+
catch (error) {
|
|
680
|
+
err(`error: verified snapshot ${entry.name}, but could not write the post-restore permit: ${error instanceof Error ? error.message : String(error)}\n`);
|
|
681
|
+
return { status: 1 };
|
|
682
|
+
}
|
|
683
|
+
out(`==> Verified restored snapshot ${entry.name}; one rescue attempt is now permitted.\n`);
|
|
684
|
+
out(" That permit is rechecked and consumed before rescue changes files. If the attempt fails, the circuit breaker is armed again.\n");
|
|
685
|
+
return { status: 0 };
|
|
686
|
+
}
|
|
687
|
+
/** Return a precise mismatch instead of treating a partial restore as evidence. */
|
|
688
|
+
function snapshotTreeMismatch(snapshotRoot, hqRoot, includedRoots) {
|
|
689
|
+
if (includedRoots.length === 0) {
|
|
690
|
+
return "snapshot manifest records no captured roots to verify";
|
|
691
|
+
}
|
|
692
|
+
const seen = new Set();
|
|
693
|
+
for (const root of includedRoots) {
|
|
694
|
+
if (!root ||
|
|
695
|
+
root === "." ||
|
|
696
|
+
root === ".." ||
|
|
697
|
+
root.includes("/") ||
|
|
698
|
+
root.includes("\\") ||
|
|
699
|
+
isRescueTransientDirectory(root) ||
|
|
700
|
+
seen.has(root)) {
|
|
701
|
+
return `snapshot manifest has an unsafe captured root ${JSON.stringify(root)}`;
|
|
702
|
+
}
|
|
703
|
+
seen.add(root);
|
|
704
|
+
if (rescuePathKind(path.join(snapshotRoot, root)) === "missing") {
|
|
705
|
+
return `snapshot does not contain captured root ${root}`;
|
|
706
|
+
}
|
|
707
|
+
const mismatch = snapshotPathMismatch(path.join(snapshotRoot, root), path.join(hqRoot, root), root);
|
|
708
|
+
if (mismatch)
|
|
709
|
+
return mismatch;
|
|
710
|
+
}
|
|
711
|
+
return null;
|
|
712
|
+
}
|
|
713
|
+
function snapshotPathMismatch(snapshotPath, livePath, rel) {
|
|
714
|
+
const snapshotKind = rescuePathKind(snapshotPath);
|
|
715
|
+
const liveKind = rescuePathKind(livePath);
|
|
716
|
+
if (snapshotKind !== liveKind) {
|
|
717
|
+
return `${rel} is ${liveKind} but the snapshot records ${snapshotKind}`;
|
|
718
|
+
}
|
|
719
|
+
if (snapshotKind === "missing")
|
|
720
|
+
return null;
|
|
721
|
+
if (snapshotKind === "file") {
|
|
722
|
+
return bytesEqual(snapshotPath, livePath)
|
|
723
|
+
? null
|
|
724
|
+
: `${rel} has different file contents`;
|
|
725
|
+
}
|
|
726
|
+
if (snapshotKind === "symlink") {
|
|
727
|
+
try {
|
|
728
|
+
return fs.readlinkSync(snapshotPath) === fs.readlinkSync(livePath)
|
|
729
|
+
? null
|
|
730
|
+
: `${rel} has a different symlink target`;
|
|
731
|
+
}
|
|
732
|
+
catch (error) {
|
|
733
|
+
return `${rel} could not read a symlink target: ${error instanceof Error ? error.message : String(error)}`;
|
|
734
|
+
}
|
|
735
|
+
}
|
|
736
|
+
if (snapshotKind !== "directory") {
|
|
737
|
+
return `${rel} has unsupported path type ${snapshotKind}`;
|
|
738
|
+
}
|
|
739
|
+
let snapshotNames;
|
|
740
|
+
let liveNames;
|
|
741
|
+
try {
|
|
742
|
+
snapshotNames = fs
|
|
743
|
+
.readdirSync(snapshotPath)
|
|
744
|
+
.filter((name) => !isRescueTransientDirectory(path.posix.join(rel, name)))
|
|
745
|
+
.sort();
|
|
746
|
+
liveNames = fs
|
|
747
|
+
.readdirSync(livePath)
|
|
748
|
+
.filter((name) => !isRescueTransientDirectory(path.posix.join(rel, name)))
|
|
749
|
+
.sort();
|
|
750
|
+
}
|
|
751
|
+
catch (error) {
|
|
752
|
+
return `${rel} could not be read: ${error instanceof Error ? error.message : String(error)}`;
|
|
753
|
+
}
|
|
754
|
+
if (snapshotNames.length !== liveNames.length ||
|
|
755
|
+
snapshotNames.some((name, index) => name !== liveNames[index])) {
|
|
756
|
+
return `${rel} has different entries`;
|
|
757
|
+
}
|
|
758
|
+
for (const name of snapshotNames) {
|
|
759
|
+
const mismatch = snapshotPathMismatch(path.join(snapshotPath, name), path.join(livePath, name), path.posix.join(rel, name));
|
|
760
|
+
if (mismatch)
|
|
761
|
+
return mismatch;
|
|
762
|
+
}
|
|
763
|
+
return null;
|
|
764
|
+
}
|
|
497
765
|
/**
|
|
498
766
|
* True when `root` is usable as a rescue target: an absolute path that is not
|
|
499
767
|
* a bare drive letter. Pure + exported for tests; `platformPath` defaults to
|
|
@@ -541,53 +809,7 @@ export function realpathOrExit(p, err) {
|
|
|
541
809
|
}
|
|
542
810
|
}
|
|
543
811
|
function doRescue(cfg, env, out, err, setTmp) {
|
|
544
|
-
|
|
545
|
-
let hqRoot;
|
|
546
|
-
if (cfg.hqRootOverride) {
|
|
547
|
-
// Windows field failure (2026-08-02): the menubar app's daily background
|
|
548
|
-
// core update crash-looped for days because `--hq-root` arrived as the
|
|
549
|
-
// bare drive letter `C:`. `isDir("C:")` is true (drive-relative, resolves
|
|
550
|
-
// against the per-drive cwd) but `fs.realpathSync("C:")` throws EISDIR,
|
|
551
|
-
// so every retry died with an uncaught exception — console-window
|
|
552
|
-
// flicker, git.exe 0xc0000142 dialogs, and no core updates ever landing.
|
|
553
|
-
// Fail closed with a diagnosable message instead: a rescue root must be
|
|
554
|
-
// an absolute directory path, and the error echoes the exact argv this
|
|
555
|
-
// process received so the daily log identifies the caller that mangled
|
|
556
|
-
// the path (the split happens upstream of this process).
|
|
557
|
-
if (!isUsableRescueRoot(cfg.hqRootOverride)) {
|
|
558
|
-
err(`error: --hq-root must be an absolute path, got ${JSON.stringify(cfg.hqRootOverride)}.\n` +
|
|
559
|
-
` argv: ${JSON.stringify(process.argv.slice(2))}\n`);
|
|
560
|
-
throw new ExitError(1);
|
|
561
|
-
}
|
|
562
|
-
if (!isDir(cfg.hqRootOverride)) {
|
|
563
|
-
// bash `cd` failure under set -e would abort; mirror as a generic error.
|
|
564
|
-
err(`error: --hq-root ${cfg.hqRootOverride} is not a directory.\n`);
|
|
565
|
-
throw new ExitError(1);
|
|
566
|
-
}
|
|
567
|
-
hqRoot = realpathOrExit(cfg.hqRootOverride, err);
|
|
568
|
-
}
|
|
569
|
-
else {
|
|
570
|
-
// Legacy default assumed the script lived at personal/skills/<skill>/.
|
|
571
|
-
// The package's programmatic callers always pass --hq-root; fall back to cwd.
|
|
572
|
-
hqRoot = realpathOrExit(process.cwd(), err);
|
|
573
|
-
}
|
|
574
|
-
// HQ-root sanity gate — guards against wiping a non-HQ directory. `companies/`
|
|
575
|
-
// is the stable anchor (present and preserved on every HQ install), confirmed
|
|
576
|
-
// by at least one core scaffold marker. We accept ANY of `.claude/`, `core/`,
|
|
577
|
-
// or `personal/` because the layout has drifted across releases: v14.0.0 ships
|
|
578
|
-
// NEITHER `personal/` (added v15) NOR `core/` (the v15 scaffold home) — it only
|
|
579
|
-
// has `.claude/`. Requiring `personal/` here left v14.0.0 users with no upgrade
|
|
580
|
-
// path (rescue aborted; DEV-1741). `.claude/` exists on every version v14→v15,
|
|
581
|
-
// so it admits the full upgrade range while still rejecting a bare directory.
|
|
582
|
-
const hasHqMarker = isDir(path.join(hqRoot, ".claude")) ||
|
|
583
|
-
isDir(path.join(hqRoot, "core")) ||
|
|
584
|
-
isDir(path.join(hqRoot, "personal"));
|
|
585
|
-
if (!isDir(path.join(hqRoot, "companies")) || !hasHqMarker) {
|
|
586
|
-
err(`error: ${hqRoot} does not look like an HQ root ` +
|
|
587
|
-
`(need companies/ plus one of .claude/, core/, or personal/). Aborting.\n`);
|
|
588
|
-
err(" pass --hq-root <dir> if the script is not at personal/skills/<skill>/.\n");
|
|
589
|
-
throw new ExitError(3);
|
|
590
|
-
}
|
|
812
|
+
const hqRoot = resolveHqRoot(cfg, err);
|
|
591
813
|
const RUN_TS = utcStamp(new Date(), "dash");
|
|
592
814
|
// Both the release overlay and a restorable snapshot require rsync. A shim
|
|
593
815
|
// which merely starts but exits non-zero used to trigger an enormous cp -a
|
|
@@ -1038,15 +1260,25 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
1038
1260
|
const prePlan = pruneManagedBackups(cfg, hqRoot, out, false);
|
|
1039
1261
|
const unresolved = prePlan.protected.filter((x) => x.reason === "recovery-required-no-later-good");
|
|
1040
1262
|
if (unresolved.length >= 2) {
|
|
1041
|
-
const
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1263
|
+
const postRestore = consumePostRestorePermit(cfg, hqRoot);
|
|
1264
|
+
if (postRestore.used) {
|
|
1265
|
+
out(`==> Verified post-restore permit for ${postRestore.snapshotName}; allowing one rescue attempt.\n`);
|
|
1266
|
+
out(" The permit was consumed before mutation. If this attempt fails, the circuit breaker is armed again.\n");
|
|
1267
|
+
}
|
|
1268
|
+
else {
|
|
1269
|
+
if (postRestore.problem) {
|
|
1270
|
+
err(`error: post-restore permit is not usable: ${postRestore.problem}\n`);
|
|
1271
|
+
}
|
|
1272
|
+
const snapshotNames = unresolved.map((x) => x.entry.name);
|
|
1273
|
+
writeRescueBlockedMarker(hqRoot, snapshotNames);
|
|
1274
|
+
err("HQ_RESCUE_FAILURE_KIND=snapshot-recovery-circuit-breaker\n");
|
|
1275
|
+
err("error: safety snapshot circuit breaker is open: two interrupted updates still need recovery proof.\n");
|
|
1276
|
+
err(" HQ kept their pre-update snapshots because those updates started changing the release-managed tree but did not finish.\n");
|
|
1277
|
+
err(" It will not overwrite or delete that evidence without proof that a later rescue completed successfully.\n");
|
|
1278
|
+
err(` Inspect safely: ${formatRecoveryCommand(hqRoot, cfg.backupRoot)}\n`);
|
|
1279
|
+
err(" It deletes only snapshots with a later completed-good rescue. For unresolved snapshots, restore one listed snapshot and run its verification command before retrying.\n");
|
|
1047
1280
|
throw new ExitError(1);
|
|
1048
1281
|
}
|
|
1049
|
-
resolveRecoveryRequiredSnapshots(cfg.backupRoot, RUN_TS, snapshotNames, out);
|
|
1050
1282
|
}
|
|
1051
1283
|
requireBackupCapacity(cfg, hqRoot, wipeToplevel, env, err);
|
|
1052
1284
|
fs.mkdirSync(cfg.backupRoot, { recursive: true });
|
|
@@ -2774,6 +3006,28 @@ function restoreMtimesFromGit(srcDir, env, out) {
|
|
|
2774
3006
|
}
|
|
2775
3007
|
const RESCUE_BLOCKED_MARKER_REL = ".hq/rescue-blocked.json";
|
|
2776
3008
|
const RESCUE_BLOCKED_REASON = "recovery-snapshot-breaker";
|
|
3009
|
+
const POST_RESTORE_PERMIT_REL = ".hq/rescue-post-restore.json";
|
|
3010
|
+
function posixShellQuote(value) {
|
|
3011
|
+
return `'${value.replace(/'/g, `'\\''`)}'`;
|
|
3012
|
+
}
|
|
3013
|
+
/** Format the exact safe command presented in a breaker diagnostic. */
|
|
3014
|
+
export function formatRecoveryCommand(hqRoot, backupRoot, verifyRestoredSnapshot) {
|
|
3015
|
+
const args = [
|
|
3016
|
+
"npx",
|
|
3017
|
+
"-y",
|
|
3018
|
+
"--package=@indigoai-us/hq-cloud@latest",
|
|
3019
|
+
"hq-rescue",
|
|
3020
|
+
"--recover-snapshots",
|
|
3021
|
+
"--hq-root",
|
|
3022
|
+
posixShellQuote(hqRoot),
|
|
3023
|
+
"--backup-dir",
|
|
3024
|
+
posixShellQuote(backupRoot),
|
|
3025
|
+
];
|
|
3026
|
+
if (verifyRestoredSnapshot) {
|
|
3027
|
+
args.push("--verify-restored-snapshot", posixShellQuote(verifyRestoredSnapshot));
|
|
3028
|
+
}
|
|
3029
|
+
return args.join(" ");
|
|
3030
|
+
}
|
|
2777
3031
|
function posixRelFromRoot(hqRoot, abs) {
|
|
2778
3032
|
const root = path.resolve(hqRoot);
|
|
2779
3033
|
const resolved = path.resolve(abs);
|
|
@@ -2871,12 +3125,106 @@ function reportUnwritableRescuePaths(hqRoot, unwritable, err) {
|
|
|
2871
3125
|
function rescueBlockedMarkerPath(hqRoot) {
|
|
2872
3126
|
return path.join(hqRoot, RESCUE_BLOCKED_MARKER_REL);
|
|
2873
3127
|
}
|
|
3128
|
+
function postRestorePermitPath(hqRoot) {
|
|
3129
|
+
return path.join(hqRoot, POST_RESTORE_PERMIT_REL);
|
|
3130
|
+
}
|
|
3131
|
+
function writePostRestorePermit(hqRoot, entry) {
|
|
3132
|
+
const permit = {
|
|
3133
|
+
schemaVersion: 1,
|
|
3134
|
+
canonicalHqRoot: hqRoot,
|
|
3135
|
+
rootHash: canonicalRootHash(hqRoot),
|
|
3136
|
+
snapshotName: entry.name,
|
|
3137
|
+
snapshotRunId: entry.manifest.runId,
|
|
3138
|
+
verifiedAt: new Date().toISOString(),
|
|
3139
|
+
};
|
|
3140
|
+
const target = postRestorePermitPath(hqRoot);
|
|
3141
|
+
const temporary = `${target}.${process.pid}.tmp`;
|
|
3142
|
+
fs.mkdirSync(path.dirname(target), { recursive: true });
|
|
3143
|
+
fs.writeFileSync(temporary, JSON.stringify(permit, null, 2) + "\n", {
|
|
3144
|
+
mode: 0o600,
|
|
3145
|
+
});
|
|
3146
|
+
fs.renameSync(temporary, target);
|
|
3147
|
+
}
|
|
3148
|
+
function readPostRestorePermit(hqRoot) {
|
|
3149
|
+
let raw;
|
|
3150
|
+
try {
|
|
3151
|
+
raw = fs.readFileSync(postRestorePermitPath(hqRoot), "utf8");
|
|
3152
|
+
}
|
|
3153
|
+
catch (error) {
|
|
3154
|
+
if (error.code === "ENOENT") {
|
|
3155
|
+
return { permit: null };
|
|
3156
|
+
}
|
|
3157
|
+
return {
|
|
3158
|
+
permit: null,
|
|
3159
|
+
problem: `could not read ${POST_RESTORE_PERMIT_REL}: ${error instanceof Error ? error.message : String(error)}`,
|
|
3160
|
+
};
|
|
3161
|
+
}
|
|
3162
|
+
try {
|
|
3163
|
+
const value = JSON.parse(raw);
|
|
3164
|
+
if (value.schemaVersion !== 1 ||
|
|
3165
|
+
typeof value.canonicalHqRoot !== "string" ||
|
|
3166
|
+
typeof value.rootHash !== "string" ||
|
|
3167
|
+
typeof value.snapshotName !== "string" ||
|
|
3168
|
+
typeof value.snapshotRunId !== "string" ||
|
|
3169
|
+
typeof value.verifiedAt !== "string") {
|
|
3170
|
+
return { permit: null, problem: `${POST_RESTORE_PERMIT_REL} is malformed` };
|
|
3171
|
+
}
|
|
3172
|
+
return { permit: value };
|
|
3173
|
+
}
|
|
3174
|
+
catch (error) {
|
|
3175
|
+
return {
|
|
3176
|
+
permit: null,
|
|
3177
|
+
problem: `${POST_RESTORE_PERMIT_REL} is not valid JSON: ${error instanceof Error ? error.message : String(error)}`,
|
|
3178
|
+
};
|
|
3179
|
+
}
|
|
3180
|
+
}
|
|
3181
|
+
function consumePostRestorePermit(cfg, hqRoot) {
|
|
3182
|
+
const read = readPostRestorePermit(hqRoot);
|
|
3183
|
+
if (!read.permit) {
|
|
3184
|
+
return { used: false, ...(read.problem ? { problem: read.problem } : {}) };
|
|
3185
|
+
}
|
|
3186
|
+
const permit = read.permit;
|
|
3187
|
+
if (permit.canonicalHqRoot !== hqRoot ||
|
|
3188
|
+
permit.rootHash !== canonicalRootHash(hqRoot)) {
|
|
3189
|
+
return { used: false, problem: "permit belongs to a different HQ root" };
|
|
3190
|
+
}
|
|
3191
|
+
const inventory = inventoryManagedSnapshots(cfg.backupRoot, hqRoot, () => false, {
|
|
3192
|
+
keepGood: Number(cfg.backupRetentionCount),
|
|
3193
|
+
maxAgeDays: Number(cfg.backupRetentionDays),
|
|
3194
|
+
});
|
|
3195
|
+
if (inventory.unmanaged.some((item) => item.reason === "backup-root-unreadable")) {
|
|
3196
|
+
return { used: false, problem: `backup root is unreadable: ${cfg.backupRoot}` };
|
|
3197
|
+
}
|
|
3198
|
+
const entry = inventory.entries.find((item) => item.name === permit.snapshotName &&
|
|
3199
|
+
item.manifest.runId === permit.snapshotRunId &&
|
|
3200
|
+
item.manifest.status === "recovery-required");
|
|
3201
|
+
const unresolved = inventory.protected.some((item) => item.entry.name === permit.snapshotName &&
|
|
3202
|
+
item.reason === "recovery-required-no-later-good");
|
|
3203
|
+
if (!entry || !unresolved) {
|
|
3204
|
+
return { used: false, problem: "referenced snapshot is no longer unresolved recovery evidence" };
|
|
3205
|
+
}
|
|
3206
|
+
const mismatch = snapshotTreeMismatch(path.join(cfg.backupRoot, entry.name), hqRoot, entry.manifest.includedRoots);
|
|
3207
|
+
if (mismatch) {
|
|
3208
|
+
return { used: false, problem: `live tree no longer matches ${entry.name}: ${mismatch}` };
|
|
3209
|
+
}
|
|
3210
|
+
try {
|
|
3211
|
+
fs.unlinkSync(postRestorePermitPath(hqRoot));
|
|
3212
|
+
}
|
|
3213
|
+
catch (error) {
|
|
3214
|
+
return {
|
|
3215
|
+
used: false,
|
|
3216
|
+
problem: `could not consume the single-use permit: ${error instanceof Error ? error.message : String(error)}`,
|
|
3217
|
+
};
|
|
3218
|
+
}
|
|
3219
|
+
return { used: true, snapshotName: entry.name };
|
|
3220
|
+
}
|
|
2874
3221
|
function writeRescueBlockedMarker(hqRoot, snapshots) {
|
|
2875
3222
|
const body = {
|
|
2876
3223
|
at: new Date().toISOString(),
|
|
2877
3224
|
reason: RESCUE_BLOCKED_REASON,
|
|
3225
|
+
failureKind: "snapshot-recovery-circuit-breaker",
|
|
2878
3226
|
snapshots,
|
|
2879
|
-
hint: "
|
|
3227
|
+
hint: "Run hq-rescue --recover-snapshots to inspect the snapshots. It deletes only snapshots proved superseded by a later completed-good rescue; otherwise restore the needed files, repair the release source, and retry.",
|
|
2880
3228
|
};
|
|
2881
3229
|
try {
|
|
2882
3230
|
fs.mkdirSync(path.dirname(rescueBlockedMarkerPath(hqRoot)), {
|
|
@@ -2896,18 +3244,6 @@ function clearRescueBlockedMarker(hqRoot) {
|
|
|
2896
3244
|
/* absent is the success state */
|
|
2897
3245
|
}
|
|
2898
3246
|
}
|
|
2899
|
-
function resolveRecoveryRequiredSnapshots(backupRoot, runTs, names, out) {
|
|
2900
|
-
const destRoot = `${backupRoot}-resolved-${runTs}`;
|
|
2901
|
-
fs.mkdirSync(destRoot, { recursive: true });
|
|
2902
|
-
out(`==> Resolving recovery-required snapshots (--resolve-recovery-snapshots) -> ${destRoot}\n`);
|
|
2903
|
-
for (const name of names) {
|
|
2904
|
-
const from = path.join(backupRoot, name);
|
|
2905
|
-
const to = path.join(destRoot, name);
|
|
2906
|
-
fs.renameSync(from, to);
|
|
2907
|
-
out(` moved ${name} -> ${to}\n`);
|
|
2908
|
-
}
|
|
2909
|
-
return destRoot;
|
|
2910
|
-
}
|
|
2911
3247
|
function allocateSnapshotDir(backupRoot, runTs, rootHash, runId) {
|
|
2912
3248
|
return path.join(backupRoot, `pre-update-${runTs}-${rootHash}-${runId}`);
|
|
2913
3249
|
}
|