@indigoai-us/hq-cloud 6.18.49-alpha.7 → 6.18.49

package/dist/cli/rescue-core.js

@@ -30,6 +30,7 @@
 import { spawnSync } from "child_process";
 import * as crypto from "crypto";
 import * as fs from "fs";
+import { createRequire } from "node:module";
 import * as os from "os";
 import * as path from "path";
 import yaml from "js-yaml";
@@ -37,6 +38,40 @@ import { assertRescuePathTypeUnchanged, isRescueTransientDirectory, } from "./re
 import { applyBackupPrune, canonicalRootHash, inventoryManagedSnapshots, isManagedSnapshotManifest, } from "../backup-prune.js";
 import { makeRestoreProbePath, makeRestoreStagePath, } from "../restore-staging.js";
 import { readCoreSourceStamp, writeCoreSourceStamp } from "./core-source-stamp.js";
+const MAX_CLONE_FILES_PER_BATCH = 200;
+const MAX_CLONE_BATCH_PATH_BYTES = 64 * 1024;
+const require = createRequire(import.meta.url);
+function runDarwinClone(sources, destinationDirectory, destinationPaths) {
+    // Node/libuv does not implement forced APFS clones on macOS. Use the native
+    // clonefile(2) binding, which fails when the volume cannot clone instead of
+    // silently consuming the full-copy budget.
+    const cloneFileSync = require("rclonefile").cloneFileSync;
+    for (const [index, source] of sources.entries()) {
+        const destination = destinationPaths?.[index] ??
+            path.join(destinationDirectory, path.basename(source));
+        cloneFileSync(source, destination);
+    }
+}
+function chunkCloneSources(sources, destinationDirectory) {
+    const batches = [];
+    let batch = [];
+    let batchPathBytes = Buffer.byteLength(destinationDirectory) + 8;
+    for (const source of sources) {
+        const nextBytes = Buffer.byteLength(source) + 1;
+        if (batch.length > 0 &&
+            (batch.length >= MAX_CLONE_FILES_PER_BATCH ||
+                batchPathBytes + nextBytes > MAX_CLONE_BATCH_PATH_BYTES)) {
+            batches.push(batch);
+            batch = [];
+            batchPathBytes = Buffer.byteLength(destinationDirectory) + 8;
+        }
+        batch.push(source);
+        batchPathBytes += nextBytes;
+    }
+    if (batch.length > 0)
+        batches.push(batch);
+    return batches;
+}
 // Paths always preserved across wipe+overlay (shuttled out, restored after).
 const CARVE_OUT_PATHS = [
     "core/packages",
@@ -733,7 +768,7 @@ export function runRescue(argv, opts = {}) {
             tmpdir = d;
         }, () => {
             retainTmpdir = true;
-        }, opts.gitReachabilityProbeTimeoutMs ?? GIT_REACHABILITY_PROBE_TIMEOUT_MS, opts.coreSourceStampWriter, opts.restoreOnRsyncFailure === true, opts.setupPathSettingsLocal === true, opts.backupCapacityStatfsSync ?? fs.statfsSync, opts.backupCapacityCloneProbe ?? probeCopyOnWriteClone, opts.backupCapacityStatSync ?? fs.statSync, opts.pathSepForTest ?? path.sep);
+        }, opts.gitReachabilityProbeTimeoutMs ?? GIT_REACHABILITY_PROBE_TIMEOUT_MS, opts.coreSourceStampWriter, opts.restoreOnRsyncFailure === true, opts.setupPathSettingsLocal === true, opts.backupCapacityStatfsSync ?? fs.statfsSync, opts.backupCapacityCloneProbe, opts.backupCapacityStatSync ?? fs.statSync, opts.pathSepForTest ?? path.sep, opts.snapshotClonePlatform ?? process.platform, opts.snapshotCloneRunner ?? runDarwinClone);
     }
     catch (e) {
         if (e instanceof ExitError)
@@ -1536,7 +1571,7 @@ function recoveryProof(unresolved, cfg, hqRoot, srcDir, srcSha, rsyncLauncher, r
     }
     return { eligible: !hasSkippedSnapshotEntries(newest.entry.manifest), failures };
 }
-function doRescue(cfg, env, out, err, setTmp, retainTmp, gitReachabilityProbeTimeoutMs = GIT_REACHABILITY_PROBE_TIMEOUT_MS, stampWriter = writeCoreSourceStamp, restoreOnRsyncFailure = false, setupPathSettingsLocal = false, backupCapacityStatfsSync = fs.statfsSync, backupCapacityCloneProbe = probeCopyOnWriteClone, backupCapacityStatSync = fs.statSync, pathSep = path.sep) {
+function doRescue(cfg, env, out, err, setTmp, retainTmp, gitReachabilityProbeTimeoutMs = GIT_REACHABILITY_PROBE_TIMEOUT_MS, stampWriter = writeCoreSourceStamp, restoreOnRsyncFailure = false, setupPathSettingsLocal = false, backupCapacityStatfsSync = fs.statfsSync, backupCapacityCloneProbe, backupCapacityStatSync = fs.statSync, pathSep = path.sep, snapshotClonePlatform = process.platform, snapshotCloneRunner = runDarwinClone) {
     const hqRoot = resolveHqRoot(cfg, err);
     const RUN_TS = utcStamp(new Date(), "dash");
     // Both the release overlay and a restorable snapshot require rsync.  A shim
@@ -2052,7 +2087,7 @@ function doRescue(cfg, env, out, err, setTmp, retainTmp, gitReachabilityProbeTim
                     }
                 }
             }
-            const snapshotCapacityAvailable = requireBackupCapacity(cfg, hqRoot, snapshotRoots, srcDir, env, err, backupCapacityStatfsSync, backupCapacityCloneProbe, backupCapacityStatSync);
+            const snapshotCapacityAvailable = requireBackupCapacity(cfg, hqRoot, snapshotRoots, srcDir, env, err, backupCapacityStatfsSync, backupCapacityCloneProbe, backupCapacityStatSync, snapshotClonePlatform, snapshotCloneRunner);
             if (!snapshotCapacityAvailable) {
                 cfg.doBackup = false;
                 out("\nwarning: safety snapshot skipped because available space cannot cover the snapshot allowance; the cloud update continues without a pre-update restore point.\n");
@@ -2076,7 +2111,11 @@ function doRescue(cfg, env, out, err, setTmp, retainTmp, gitReachabilityProbeTim
                         if (isRescueTransientDirectory(rootRel))
                             continue;
                         try {
-                            const copy = cpAToFiltered(srcAbs, path.join(backupDir, rootRel), rootRel, hqRoot, snapshotSymlinks, snapshotSkippedErrors);
+                            const copy = cpAToFiltered(srcAbs, path.join(backupDir, rootRel), rootRel, hqRoot, snapshotSymlinks, snapshotSkippedErrors, {
+                                platform: snapshotClonePlatform,
+                                runner: snapshotCloneRunner,
+                                statfsSync: backupCapacityStatfsSync,
+                            });
                             if (!copy.captured) {
                                 const failure = copy.firstFailure ??
                                     new SnapshotCopyError(rootRel, "metadata", new Error("snapshot root contains no capturable entries"), srcAbs);
@@ -2188,7 +2227,11 @@ function doRescue(cfg, env, out, err, setTmp, retainTmp, gitReachabilityProbeTim
                 if (lexists(src)) {
                     shuttleId += 1;
                     try {
-                        cpA(src, shuttle, String(shuttleId));
+                        cpA(src, shuttle, String(shuttleId), {
+                            platform: snapshotClonePlatform,
+                            runner: snapshotCloneRunner,
+                            statfsSync: backupCapacityStatfsSync,
+                        });
                     }
                     catch (error) {
                         reportPreserveBackupFailure(normalizedSp, error, err);
@@ -4650,6 +4693,10 @@ function estimateFiltered(abs, rel) {
     }
     return { files, bytes };
 }
+function snapshotFreeSpaceReserve(fsStat) {
+    const capacity = Number(fsStat.blocks) * Number(fsStat.bsize);
+    return Math.min(10 * 1024 ** 3, Math.max(5 * 1024 ** 3, Math.ceil(capacity * 0.1)));
+}
 function estimateShuttleBytes(abs) {
     const st = lstatOrNull(abs);
     if (!st)
@@ -4678,8 +4725,9 @@ function estimateShuttleBytes(abs) {
         .readdirSync(abs)
         .reduce((total, name) => total + estimateShuttleBytes(path.join(abs, name)), 0);
 }
-export function requireBackupCapacity(cfg, hqRoot, roots, srcDir, _env, err, statfsSync = fs.statfsSync, cloneProbe = probeCopyOnWriteClone, statSync = fs.statSync) {
+export function requireBackupCapacity(cfg, hqRoot, roots, srcDir, _env, err, statfsSync = fs.statfsSync, cloneProbe = undefined, statSync = fs.statSync, platform = process.platform, cloneRunner = runDarwinClone) {
     try {
+        const probe = cloneProbe ?? ((root, backup, capturedRoots) => probeCopyOnWriteClone(root, backup, capturedRoots, platform, cloneRunner, statSync));
         const estimate = roots.reduce((total, root) => total + estimateFiltered(path.join(hqRoot, root), root).bytes, 0);
         const backupLocation = nearestExistingDirectory(cfg.backupRoot);
         const tempLocation = nearestExistingDirectory(os.tmpdir());
@@ -4688,18 +4736,17 @@ export function requireBackupCapacity(cfg, hqRoot, roots, srcDir, _env, err, sta
         const tempDevice = Number(statSync(tempLocation).dev);
         const backupFs = statfsSync(backupLocation);
         const backupFree = Number(backupFs.bavail) * Number(backupFs.bsize);
-        const backupCapacity = Number(backupFs.blocks) * Number(backupFs.bsize);
         const sameHqBackupFilesystem = hqDevice === backupDevice;
         const sameBackupTempFilesystem = backupDevice === tempDevice;
         const sameHqTempFilesystem = hqDevice === tempDevice;
         const shuttleBytes = (cfg.preserveSubpaths ?? []).reduce((total, rel) => total + estimateShuttleBytes(path.join(hqRoot, rel)), 0);
-        const snapshotCloneSupported = sameHqBackupFilesystem && cloneProbe(hqRoot, cfg.backupRoot, roots);
+        const snapshotCloneSupported = sameHqBackupFilesystem && probe(hqRoot, cfg.backupRoot, roots);
         const shuttleCloneSupported = shuttleBytes > 0 &&
             sameHqTempFilesystem &&
             (sameBackupTempFilesystem && snapshotCloneSupported
                 ? true
-                : cloneProbe(hqRoot, tempLocation, cfg.preserveSubpaths ?? []));
-        const backupMargin = Math.min(10 * 1024 ** 3, Math.max(5 * 1024 ** 3, Math.ceil(backupCapacity * 0.1)));
+                : probe(hqRoot, tempLocation, cfg.preserveSubpaths ?? []));
+        const backupMargin = snapshotFreeSpaceReserve(backupFs);
         const snapshotRequired = snapshotCloneSupported
             ? SNAPSHOT_CLONE_METADATA_ALLOWANCE_BYTES
             : estimate + backupMargin;
@@ -4716,8 +4763,7 @@ export function requireBackupCapacity(cfg, hqRoot, roots, srcDir, _env, err, sta
         if (!sameBackupTempFilesystem && shuttleBytes > 0) {
             const tempFs = statfsSync(tempLocation);
             shuttleFree = Number(tempFs.bavail) * Number(tempFs.bsize);
-            const tempCapacity = Number(tempFs.blocks) * Number(tempFs.bsize);
-            const tempMargin = Math.min(10 * 1024 ** 3, Math.max(5 * 1024 ** 3, Math.ceil(tempCapacity * 0.1)));
+            const tempMargin = snapshotFreeSpaceReserve(tempFs);
             shuttleRequired = shuttleCloneSupported
                 ? SNAPSHOT_CLONE_METADATA_ALLOWANCE_BYTES
                 : shuttleBytes + tempMargin;
@@ -4760,12 +4806,7 @@ export function requireBackupCapacity(cfg, hqRoot, roots, srcDir, _env, err, sta
     }
     return true;
 }
-function probeCopyOnWriteClone(hqRoot, backupRoot, roots) {
-    const source = roots
-        .map((root) => firstRegularFile(path.join(hqRoot, root)))
-        .find((candidate) => candidate !== null);
-    if (!source)
-        return false;
+function probeCopyOnWriteClone(hqRoot, backupRoot, roots, platform = process.platform, cloneRunner = runDarwinClone, statSync = fs.statSync) {
     const parent = nearestExistingDirectory(backupRoot);
     let probeDir;
     try {
@@ -4777,7 +4818,33 @@ function probeCopyOnWriteClone(hqRoot, backupRoot, roots) {
         throw error;
     }
     try {
-        fs.copyFileSync(source, path.join(probeDir, "probe"), fs.constants.COPYFILE_FICLONE_FORCE);
+        if (platform === "darwin") {
+            const hqDevice = Number(statSync(hqRoot).dev);
+            for (let index = 0; index < roots.length; index += 1) {
+                const sample = darwinCloneSampleOnDevice(path.join(hqRoot, roots[index]), hqDevice, statSync);
+                if (!sample.sameDevice)
+                    return false;
+                if (!sample.file)
+                    continue;
+                const destination = path.join(probeDir, `root-${index}`);
+                fs.mkdirSync(destination, { recursive: true });
+                try {
+                    cloneRunner([sample.file], destination);
+                }
+                catch {
+                    return false;
+                }
+            }
+            return true;
+        }
+        else {
+            const source = roots
+                .map((root) => firstRegularFile(path.join(hqRoot, root)))
+                .find((candidate) => candidate !== null);
+            if (!source)
+                return false;
+            fs.copyFileSync(source, path.join(probeDir, "probe"), fs.constants.COPYFILE_FICLONE_FORCE);
+        }
         return true;
     }
     catch (error) {
@@ -4798,6 +4865,33 @@ function probeCopyOnWriteClone(hqRoot, backupRoot, roots) {
         fs.rmSync(probeDir, { recursive: true, force: true });
     }
 }
+function darwinCloneSampleOnDevice(root, hqDevice, statSync) {
+    let sample;
+    const walk = (absolute, relative) => {
+        const lstat = lstatOrNull(absolute);
+        if (!lstat || lstat.isSymbolicLink())
+            return true;
+        if (relative && isRescueTransientDirectory(relative))
+            return true;
+        if (Number(statSync(absolute).dev) !== hqDevice)
+            return false;
+        if (lstat.isFile()) {
+            sample ??= absolute;
+            return true;
+        }
+        if (!lstat.isDirectory())
+            return true;
+        for (const name of fs.readdirSync(absolute)) {
+            const childRelative = path.posix.join(relative, name);
+            if (isRescueTransientDirectory(childRelative))
+                continue;
+            if (!walk(path.join(absolute, name), childRelative))
+                return false;
+        }
+        return true;
+    };
+    return walk(root, "") ? { sameDevice: true, ...(sample ? { file: sample } : {}) } : { sameDevice: false };
+}
 function firstRegularFile(root, rel = "") {
     const st = lstatOrNull(root);
     if (!st || st.isSymbolicLink())
@@ -4871,11 +4965,11 @@ function countFiles(dir) {
  * source basename into destDir) and the shuttle form (`cp -a "$src"
  * "$SHUTTLE/$id"`, an explicit destination name).
  */
-function cpA(src, destDir, asName) {
+function cpA(src, destDir, asName, cloneOptions) {
     const dest = asName
         ? path.join(destDir, asName)
         : path.join(destDir, path.basename(src));
-    cpATo(src, dest);
+    cpATo(src, dest, cloneOptions);
 }
 /** Copy a link, using the same Windows representation choices as snapshots. */
 function copySymlinkWithWindowsFallback(src, dest, link) {
@@ -4902,7 +4996,7 @@ function copySymlinkWithWindowsFallback(src, dest, link) {
     throw linkPrivilegeError;
 }
 /** `cp -a <src> <dest>` with an explicit destination path. */
-function cpATo(src, dest) {
+function cpATo(src, dest, cloneOptions) {
     const st = lstatOrNull(src);
     if (!st)
         return;
@@ -4920,7 +5014,7 @@ function cpATo(src, dest) {
         }
         fs.mkdirSync(dest, { recursive: true });
         for (const name of fs.readdirSync(src)) {
-            cpATo(path.join(src, name), path.join(dest, name));
+            cpATo(path.join(src, name), path.join(dest, name), cloneOptions);
         }
         try {
             fs.utimesSync(dest, st.atime, st.mtime);
@@ -4935,10 +5029,13 @@ function cpATo(src, dest) {
     if (destinationStats && !destinationStats.isFile()) {
         fs.rmSync(dest, { recursive: true, force: true });
     }
-    // The shuttle is on the same filesystem as the HQ root on common installs,
-    // so preserve it with a clone where available. Node falls back to a full
-    // copy on filesystems without reflink support.
-    fs.copyFileSync(src, dest, fs.constants.COPYFILE_FICLONE);
+    if (cloneOptions) {
+        copyRegularFileForSnapshot(src, dest, path.basename(src), cloneOptions);
+    }
+    else {
+        // Restore copies remain independent files and retain their existing path.
+        fs.copyFileSync(src, dest, fs.constants.COPYFILE_FICLONE);
+    }
     try {
         fs.chmodSync(dest, st.mode);
         fs.utimesSync(dest, st.atime, st.mtime);
@@ -5389,7 +5486,25 @@ function validateSnapshotSymlink(hqRoot, rel, target) {
         ? null
         : `target for ${rel} resolves outside the HQ root`;
 }
-function copyRegularFileForSnapshot(src, dest, rel) {
+function copyRegularFileForSnapshot(src, dest, rel, cloneOptions) {
+    if (cloneOptions?.platform === "darwin") {
+        try {
+            cloneOptions.runner([src], path.dirname(dest), [dest]);
+            return;
+        }
+        catch {
+            try {
+                if (lstatOrNull(dest))
+                    fs.rmSync(dest, { force: true });
+                assertFullCopySnapshotCapacity(src, dest, rel, cloneOptions.statfsSync);
+                fs.copyFileSync(src, dest);
+                return;
+            }
+            catch (fallbackError) {
+                throw classifySnapshotCopyFailure(rel, src, dest, fallbackError);
+            }
+        }
+    }
     try {
         // Node falls back to a full copy when the filesystem cannot reflink.
         // Restore paths use ordinary copies, so they remain independent files.
@@ -5408,6 +5523,23 @@ function copyRegularFileForSnapshot(src, dest, rel) {
         throw classifySnapshotCopyFailure(rel, src, dest, error);
     }
 }
+function assertFullCopySnapshotCapacity(src, dest, rel, statfsSync) {
+    try {
+        const bytes = fs.statSync(src).size;
+        const location = nearestExistingDirectory(path.dirname(dest));
+        const fsStat = statfsSync(location);
+        const available = Number(fsStat.bavail) * Number(fsStat.bsize);
+        const required = bytes + snapshotFreeSpaceReserve(fsStat);
+        if (available < required) {
+            const error = new Error(`insufficient free space before full-copy snapshot fallback (need ${required} bytes, have ${available})`);
+            error.code = "ENOSPC";
+            throw error;
+        }
+    }
+    catch (error) {
+        throw classifySnapshotCopyFailure(rel, src, dest, error);
+    }
+}
 function classifySnapshotCopyFailure(rel, src, dest, error) {
     if (error.code === "EDEADLK") {
         return new SnapshotCopyError(rel, "read", error, src);
@@ -5443,7 +5575,7 @@ function copySymlinkFileRepresentationForSnapshot(src, dest, rel) {
     preserveSnapshotMetadata(dest, targetStats);
 }
 /** Snapshot copy that preserves metadata while pruning canonical transient directories. */
-function cpAToFiltered(src, dest, rel, hqRoot, symlinks, skippedEntries) {
+function cpAToFiltered(src, dest, rel, hqRoot, symlinks, skippedEntries, cloneOptions) {
     let st;
     try {
         st = fs.lstatSync(src);
@@ -5489,9 +5621,63 @@ function cpAToFiltered(src, dest, rel, hqRoot, symlinks, skippedEntries) {
         }
         let captured = names.length === 0;
         let firstFailure;
+        const batchClonedNames = new Set();
+        if (cloneOptions.platform === "darwin") {
+            const regularNames = names.filter((name) => {
+                try {
+                    const child = lstatOrNull(path.join(src, name));
+                    return Boolean(child && !child.isSymbolicLink() && child.isFile());
+                }
+                catch {
+                    return false;
+                }
+            });
+            if (regularNames.length > 0) {
+                const sources = regularNames.map((name) => path.join(src, name));
+                for (const batch of chunkCloneSources(sources, dest)) {
+                    const destinations = batch.map((source) => path.join(dest, path.basename(source)));
+                    let batchCloneFailed = false;
+                    try {
+                        cloneOptions.runner(batch, dest, destinations);
+                    }
+                    catch {
+                        batchCloneFailed = true;
+                        for (const destination of destinations) {
+                            fs.rmSync(destination, { force: true });
+                        }
+                    }
+                    for (const [index, source] of batch.entries()) {
+                        const destination = destinations[index];
+                        const name = path.basename(source);
+                        const relative = path.posix.join(rel, name);
+                        try {
+                            if (batchCloneFailed) {
+                                copyRegularFileForSnapshot(source, destination, relative, cloneOptions);
+                            }
+                            const sourceStats = fs.lstatSync(source);
+                            preserveSnapshotMetadata(destination, sourceStats);
+                            captured = true;
+                        }
+                        catch (error) {
+                            if (!(error instanceof SnapshotCopyError))
+                                throw error;
+                            if (error.code === "ENOSPC")
+                                throw error;
+                            skippedEntries.push(error);
+                            firstFailure ??= error;
+                        }
+                        finally {
+                            batchClonedNames.add(name);
+                        }
+                    }
+                }
+            }
+        }
         for (const name of names) {
+            if (batchClonedNames.has(name))
+                continue;
             try {
-                const child = cpAToFiltered(path.join(src, name), path.join(dest, name), path.posix.join(rel, name), hqRoot, symlinks, skippedEntries);
+                const child = cpAToFiltered(path.join(src, name), path.join(dest, name), path.posix.join(rel, name), hqRoot, symlinks, skippedEntries, cloneOptions);
                 captured ||= child.captured;
                 firstFailure ??= child.firstFailure;
             }
@@ -5524,7 +5710,7 @@ function cpAToFiltered(src, dest, rel, hqRoot, symlinks, skippedEntries) {
     catch (error) {
         throw new SnapshotCopyError(rel, "write", error, dest);
     }
-    copyRegularFileForSnapshot(src, dest, rel);
+    copyRegularFileForSnapshot(src, dest, rel, cloneOptions);
     try {
         fs.chmodSync(dest, st.mode);
         fs.utimesSync(dest, st.atime, st.mtime);