@indigoai-us/hq-cloud 6.16.72-alpha.0 → 6.16.72-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-watch-loop.d.ts +5 -35
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +32 -156
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.test.js +12 -96
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/rescue-breaker.test.js +94 -1
- package/dist/cli/rescue-breaker.test.js.map +1 -1
- package/dist/cli/rescue-classify-ordering.test.js +1 -1
- package/dist/cli/rescue-clone-network-retry.test.d.ts +2 -0
- package/dist/cli/rescue-clone-network-retry.test.d.ts.map +1 -0
- package/dist/cli/rescue-clone-network-retry.test.js +415 -0
- package/dist/cli/rescue-clone-network-retry.test.js.map +1 -0
- package/dist/cli/rescue-core.d.ts +12 -5
- package/dist/cli/rescue-core.d.ts.map +1 -1
- package/dist/cli/rescue-core.js +685 -93
- package/dist/cli/rescue-core.js.map +1 -1
- package/dist/cli/rescue-drift-reconcile.test.js +1 -1
- package/dist/cli/rescue-drop-dir-symlink.test.js +1 -1
- package/dist/cli/rescue-exec-bit-preserve.test.js +1 -1
- package/dist/cli/rescue-hq-root-guard.test.js +1 -1
- package/dist/cli/rescue-journal-reconcile.test.js +1 -1
- package/dist/cli/rescue-keep-local-ignore.test.js +2 -2
- package/dist/cli/rescue-move.test.d.ts +2 -0
- package/dist/cli/rescue-move.test.d.ts.map +1 -0
- package/dist/cli/rescue-move.test.js +162 -0
- package/dist/cli/rescue-move.test.js.map +1 -0
- package/dist/cli/rescue-mtime-preserve.test.js +1 -1
- package/dist/cli/rescue-preserve-copy-symlink.test.d.ts +2 -0
- package/dist/cli/rescue-preserve-copy-symlink.test.d.ts.map +1 -0
- package/dist/cli/rescue-preserve-copy-symlink.test.js +338 -0
- package/dist/cli/rescue-preserve-copy-symlink.test.js.map +1 -0
- package/dist/cli/rescue-rsync-preflight.test.js +108 -12
- package/dist/cli/rescue-rsync-preflight.test.js.map +1 -1
- package/dist/cli/rescue-rsync-status.test.d.ts +2 -0
- package/dist/cli/rescue-rsync-status.test.d.ts.map +1 -0
- package/dist/cli/rescue-rsync-status.test.js +199 -0
- package/dist/cli/rescue-rsync-status.test.js.map +1 -0
- package/dist/cli/rescue-settings-reconcile.test.js +2 -2
- package/dist/cli/rescue-snapshot-copy.test.js +1 -1
- package/dist/cli/rescue-snapshot-recovery.test.js +1 -1
- package/dist/cli/rescue-source-switch-floor.test.js +1 -1
- package/package.json +1 -1
package/dist/cli/rescue-core.js
CHANGED
|
@@ -27,6 +27,7 @@
|
|
|
27
27
|
* ignore files — see LOCAL_IGNORE_FILES).
|
|
28
28
|
*/
|
|
29
29
|
import { spawnSync } from "child_process";
|
|
30
|
+
import * as crypto from "crypto";
|
|
30
31
|
import * as fs from "fs";
|
|
31
32
|
import * as os from "os";
|
|
32
33
|
import * as path from "path";
|
|
@@ -79,8 +80,7 @@ class ExitError extends Error {
|
|
|
79
80
|
}
|
|
80
81
|
/** Run a subprocess synchronously, inheriting the parent environment by default. */
|
|
81
82
|
function run(cmd, args, opts = {}) {
|
|
82
|
-
|
|
83
|
-
return result;
|
|
83
|
+
return runWithSpawnError(cmd, args, opts);
|
|
84
84
|
}
|
|
85
85
|
/** Run a subprocess synchronously and retain its spawn error for preflights. */
|
|
86
86
|
function runWithSpawnError(cmd, args, opts = {}) {
|
|
@@ -89,6 +89,7 @@ function runWithSpawnError(cmd, args, opts = {}) {
|
|
|
89
89
|
env: opts.env ?? process.env,
|
|
90
90
|
input: opts.input,
|
|
91
91
|
encoding: "utf-8",
|
|
92
|
+
...(typeof opts.timeout === "number" ? { timeout: opts.timeout } : {}),
|
|
92
93
|
});
|
|
93
94
|
return {
|
|
94
95
|
status: res.status ?? (res.error ? 127 : 1),
|
|
@@ -134,6 +135,139 @@ function redactGitDiagnostic(output, ghToken) {
|
|
|
134
135
|
.replace(/\b([a-z][a-z0-9+.-]*:\/\/)[^\s/@]+(?::[^\s/@]*)?@/gi, "$1***@")
|
|
135
136
|
.replace(/(authorization\s*:\s*)[^\r\n]*/gi, "$1***");
|
|
136
137
|
}
|
|
138
|
+
const GIT_NETWORK_MAX_ATTEMPTS = 3;
|
|
139
|
+
const GIT_NETWORK_RETRY_DELAYS_MS = [1_000, 2_000];
|
|
140
|
+
// Bound the reachability probe only (git ls-remote). That call does DNS, TCP,
|
|
141
|
+
// TLS, and one tiny HTTP exchange: a few kilobytes, independent of repository
|
|
142
|
+
// size. Ninety seconds is past a working handshake, including macOS SYN retries
|
|
143
|
+
// (~75s) on a black hole. Three attempts plus backoff stay under five minutes
|
|
144
|
+
// (90 + 1 + 90 + 2 + 90 = 273s). Clone and checkout have no wall-clock kill:
|
|
145
|
+
// libcurl's low-speed pair (see gitNetworkEnv) aborts a transfer that stalls,
|
|
146
|
+
// and a slow transfer that is still moving is allowed to finish.
|
|
147
|
+
const GIT_REACHABILITY_PROBE_TIMEOUT_MS = 90_000;
|
|
148
|
+
function gitNetworkEnv(env) {
|
|
149
|
+
return {
|
|
150
|
+
...env,
|
|
151
|
+
// Git treats these as overrides for http.lowSpeedLimit and http.lowSpeedTime.
|
|
152
|
+
// One byte per second permits even very slow transfers while a totally stalled
|
|
153
|
+
// HTTP connection fails after one minute instead of holding up an update.
|
|
154
|
+
GIT_HTTP_LOW_SPEED_LIMIT: "1",
|
|
155
|
+
GIT_HTTP_LOW_SPEED_TIME: "60",
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
function gitFailureDiagnostic(result) {
|
|
159
|
+
return `${result.stderr}\n${result.stdout}`.toLowerCase();
|
|
160
|
+
}
|
|
161
|
+
const GIT_PERMANENT_FAILURE_SIGNALS = [
|
|
162
|
+
"authentication failed",
|
|
163
|
+
"invalid username or password",
|
|
164
|
+
"remote: invalid username",
|
|
165
|
+
"could not read username",
|
|
166
|
+
"terminal prompts disabled",
|
|
167
|
+
"http 401",
|
|
168
|
+
"http 403",
|
|
169
|
+
"repository not found",
|
|
170
|
+
"not our ref",
|
|
171
|
+
];
|
|
172
|
+
function isGitPermanentFailure(diagnostic) {
|
|
173
|
+
return GIT_PERMANENT_FAILURE_SIGNALS.some((signal) => diagnostic.includes(signal));
|
|
174
|
+
}
|
|
175
|
+
function classifyGitFailure(result) {
|
|
176
|
+
const diagnostic = gitFailureDiagnostic(result);
|
|
177
|
+
if (isGitPermanentFailure(diagnostic))
|
|
178
|
+
return null;
|
|
179
|
+
if (result.error?.code === "ETIMEDOUT") {
|
|
180
|
+
return "network-unreachable";
|
|
181
|
+
}
|
|
182
|
+
if (isMissingMacCommandLineTools(diagnostic)) {
|
|
183
|
+
return "missing-dependency";
|
|
184
|
+
}
|
|
185
|
+
// Interrupted transfers are retryable. Corrupt packs, checksum mismatches,
|
|
186
|
+
// and other integrity failures are not: retrying those three times hides the
|
|
187
|
+
// real error and can write a bad clone more than once.
|
|
188
|
+
if (isGitIntegrityFailure(diagnostic))
|
|
189
|
+
return null;
|
|
190
|
+
const transientNetworkSignals = [
|
|
191
|
+
"resolving timed out",
|
|
192
|
+
"could not resolve host",
|
|
193
|
+
"could not resolve proxy",
|
|
194
|
+
"temporary failure in name resolution",
|
|
195
|
+
"network is unreachable",
|
|
196
|
+
"connection timed out",
|
|
197
|
+
"connection timeout",
|
|
198
|
+
"operation timed out",
|
|
199
|
+
"failed to connect",
|
|
200
|
+
"connection reset by peer",
|
|
201
|
+
"ssl connection timeout",
|
|
202
|
+
"rpc failed",
|
|
203
|
+
"curl 18",
|
|
204
|
+
"curl 56",
|
|
205
|
+
"unexpected disconnect",
|
|
206
|
+
"early eof",
|
|
207
|
+
"the remote end hung up unexpectedly",
|
|
208
|
+
"transfer closed with outstanding",
|
|
209
|
+
"recv failure",
|
|
210
|
+
"remote helper 'https' aborted session",
|
|
211
|
+
];
|
|
212
|
+
const promisorFetchFailed = diagnostic.includes("could not fetch") &&
|
|
213
|
+
diagnostic.includes("promisor remote");
|
|
214
|
+
return (promisorFetchFailed ||
|
|
215
|
+
transientNetworkSignals.some((signal) => diagnostic.includes(signal)))
|
|
216
|
+
? "network-unreachable"
|
|
217
|
+
: null;
|
|
218
|
+
}
|
|
219
|
+
function isMissingMacCommandLineTools(diagnostic) {
|
|
220
|
+
if (diagnostic.includes("xcrun: error:") &&
|
|
221
|
+
diagnostic.includes("active developer path")) {
|
|
222
|
+
return true;
|
|
223
|
+
}
|
|
224
|
+
if (!diagnostic.includes("xcode-select:"))
|
|
225
|
+
return false;
|
|
226
|
+
return (diagnostic.includes("no developer tools were found") ||
|
|
227
|
+
diagnostic.includes("invalid developer directory") ||
|
|
228
|
+
diagnostic.includes("invalid active developer path"));
|
|
229
|
+
}
|
|
230
|
+
function isGitIntegrityFailure(diagnostic) {
|
|
231
|
+
return [
|
|
232
|
+
"pack is corrupt",
|
|
233
|
+
"checksum mismatch",
|
|
234
|
+
"hash mismatch",
|
|
235
|
+
"sha1 mismatch",
|
|
236
|
+
"sha256 mismatch",
|
|
237
|
+
"bad object",
|
|
238
|
+
"corrupt object",
|
|
239
|
+
"incorrect header check",
|
|
240
|
+
].some((signal) => diagnostic.includes(signal));
|
|
241
|
+
}
|
|
242
|
+
function waitForGitRetry(delayMs) {
|
|
243
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, delayMs);
|
|
244
|
+
}
|
|
245
|
+
function shouldStopAfterShallowCloneFailure(gitResult) {
|
|
246
|
+
if (gitResult.failureKind !== null)
|
|
247
|
+
return true;
|
|
248
|
+
const diagnostic = gitFailureDiagnostic(gitResult.result);
|
|
249
|
+
return isGitIntegrityFailure(diagnostic) || isGitPermanentFailure(diagnostic);
|
|
250
|
+
}
|
|
251
|
+
function runGitWithNetworkRetry(args, opts, out, beforeRetry) {
|
|
252
|
+
for (let attempts = 1; attempts <= GIT_NETWORK_MAX_ATTEMPTS; attempts += 1) {
|
|
253
|
+
const result = run("git", args, {
|
|
254
|
+
cwd: opts.cwd,
|
|
255
|
+
env: gitNetworkEnv(opts.env),
|
|
256
|
+
...(typeof opts.timeout === "number" ? { timeout: opts.timeout } : {}),
|
|
257
|
+
});
|
|
258
|
+
const failureKind = classifyGitFailure(result);
|
|
259
|
+
if (result.status === 0 ||
|
|
260
|
+
failureKind !== "network-unreachable" ||
|
|
261
|
+
attempts === GIT_NETWORK_MAX_ATTEMPTS) {
|
|
262
|
+
return { result, attempts, failureKind };
|
|
263
|
+
}
|
|
264
|
+
const delayMs = GIT_NETWORK_RETRY_DELAYS_MS[attempts - 1];
|
|
265
|
+
out(` GitHub connection failed; retrying (${attempts + 1}/${GIT_NETWORK_MAX_ATTEMPTS}) in ${delayMs / 1_000}s...\n`);
|
|
266
|
+
beforeRetry?.();
|
|
267
|
+
waitForGitRetry(delayMs);
|
|
268
|
+
}
|
|
269
|
+
throw new Error("Git retry loop ended without a result");
|
|
270
|
+
}
|
|
137
271
|
/** `command -v <bin>` — is a binary on PATH? */
|
|
138
272
|
function hasCmd(bin, env) {
|
|
139
273
|
const r = spawnSync(bin, ["--version"], { env, stdio: "ignore" });
|
|
@@ -145,6 +279,10 @@ function boundedDiagnostic(value, limit) {
|
|
|
145
279
|
return value;
|
|
146
280
|
return `${value.slice(0, limit - marker.length)}${marker}`;
|
|
147
281
|
}
|
|
282
|
+
function boundedDiagnosticTail(value, maxLines, limit) {
|
|
283
|
+
const tail = value.split(/\r?\n/).slice(-maxLines).join("\n");
|
|
284
|
+
return boundedDiagnostic(tail, limit);
|
|
285
|
+
}
|
|
148
286
|
function pathEntries(pathValue) {
|
|
149
287
|
return pathValue ? pathValue.split(path.delimiter) : [];
|
|
150
288
|
}
|
|
@@ -168,18 +306,28 @@ function rsyncCandidateNames(env) {
|
|
|
168
306
|
function isWindowsShimPath(candidatePath) {
|
|
169
307
|
return /\.(?:cmd|bat)$/i.test(candidatePath);
|
|
170
308
|
}
|
|
309
|
+
/** PATHEXT never includes an empty extension, so Windows will not launch this name. */
|
|
310
|
+
function isWindowsExtensionlessRsyncName(name) {
|
|
311
|
+
return !/\.[A-Za-z0-9]+$/.test(name);
|
|
312
|
+
}
|
|
171
313
|
/**
|
|
172
314
|
* Scan PATH `entries` for an rsync binary. On win32, a real `rsync.exe` (or
|
|
173
|
-
* any non-shim candidate) found ANYWHERE on PATH always
|
|
174
|
-
* `.cmd`/`.bat` shim, even if the shim appears earlier in PATH
|
|
175
|
-
* must be routed through cmd.exe (see `buildRsyncLauncher`)
|
|
176
|
-
* less reliable, so we only fall back to one when no real
|
|
177
|
-
* anywhere on PATH.
|
|
315
|
+
* any non-shim candidate with an extension) found ANYWHERE on PATH always
|
|
316
|
+
* wins over a `.cmd`/`.bat` shim, even if the shim appears earlier in PATH
|
|
317
|
+
* order — shims must be routed through cmd.exe (see `buildRsyncLauncher`)
|
|
318
|
+
* and are strictly less reliable, so we only fall back to one when no real
|
|
319
|
+
* executable exists anywhere on PATH.
|
|
320
|
+
*
|
|
321
|
+
* An extensionless `rsync` is not a win32 hit: Windows resolves the bare
|
|
322
|
+
* name through PATHEXT and will never start that file. Keep scanning so a
|
|
323
|
+
* later `.exe` or `.cmd` wins. If the extensionless file is the only hit,
|
|
324
|
+
* report it as present but not executable instead of launching it.
|
|
178
325
|
*/
|
|
179
326
|
export function findRsyncOnPath(entries, env) {
|
|
180
327
|
let firstUnexecutable;
|
|
181
328
|
let firstShim;
|
|
182
329
|
let firstUnexecutableShim;
|
|
330
|
+
let firstWindowsExtensionless;
|
|
183
331
|
for (const entry of entries) {
|
|
184
332
|
for (const name of rsyncCandidateNames(env)) {
|
|
185
333
|
const candidate = path.resolve(entry || ".", name);
|
|
@@ -190,6 +338,10 @@ export function findRsyncOnPath(entries, env) {
|
|
|
190
338
|
catch {
|
|
191
339
|
continue;
|
|
192
340
|
}
|
|
341
|
+
if (process.platform === "win32" && isWindowsExtensionlessRsyncName(name)) {
|
|
342
|
+
firstWindowsExtensionless ??= candidate;
|
|
343
|
+
continue;
|
|
344
|
+
}
|
|
193
345
|
const shim = process.platform === "win32" && isWindowsShimPath(candidate);
|
|
194
346
|
try {
|
|
195
347
|
fs.accessSync(candidate, fs.constants.X_OK);
|
|
@@ -209,13 +361,16 @@ export function findRsyncOnPath(entries, env) {
|
|
|
209
361
|
}
|
|
210
362
|
// No real executable anywhere on PATH: fall back to a shim (executable
|
|
211
363
|
// preferred over a present-but-unexecutable one), then to whatever
|
|
212
|
-
// non-executable candidate we saw first.
|
|
364
|
+
// non-executable candidate we saw first. An extensionless file is last,
|
|
365
|
+
// and only so the preflight can name it.
|
|
213
366
|
if (firstShim)
|
|
214
367
|
return firstShim;
|
|
215
368
|
if (firstUnexecutableShim)
|
|
216
369
|
return { path: firstUnexecutableShim, executable: false };
|
|
217
|
-
|
|
218
|
-
|
|
370
|
+
if (firstUnexecutable)
|
|
371
|
+
return { path: firstUnexecutable, executable: false };
|
|
372
|
+
return firstWindowsExtensionless
|
|
373
|
+
? { path: firstWindowsExtensionless, executable: false }
|
|
219
374
|
: undefined;
|
|
220
375
|
}
|
|
221
376
|
/**
|
|
@@ -230,11 +385,22 @@ export function findRsyncOnPath(entries, env) {
|
|
|
230
385
|
* exits 1; `/d /c <path with space> --version` exits 0). Keep the path and
|
|
231
386
|
* fixed argument as distinct array elements instead of enabling shell:true
|
|
232
387
|
* for a constructed command string.
|
|
388
|
+
*
|
|
389
|
+
* A non-shim Windows candidate is launched by the absolute path this scan
|
|
390
|
+
* verified. Bare `rsync` walks PATH and PATHEXT from the first directory, so
|
|
391
|
+
* an earlier `rsync.cmd` would run instead of a later `rsync.exe`. POSIX keeps
|
|
392
|
+
* the bare name: there the scan and the spawn resolve the same path.
|
|
393
|
+
* Extensionless Windows paths are not launched; PATHEXT will not select them.
|
|
233
394
|
*/
|
|
234
395
|
function buildRsyncLauncher(candidatePath) {
|
|
235
396
|
if (process.platform === "win32" && isWindowsShimPath(candidatePath ?? "")) {
|
|
236
397
|
return { command: "cmd.exe", leadingArgs: ["/d", "/c", candidatePath] };
|
|
237
398
|
}
|
|
399
|
+
if (process.platform === "win32" &&
|
|
400
|
+
candidatePath &&
|
|
401
|
+
path.extname(candidatePath)) {
|
|
402
|
+
return { command: candidatePath, leadingArgs: [] };
|
|
403
|
+
}
|
|
238
404
|
return { command: "rsync", leadingArgs: [] };
|
|
239
405
|
}
|
|
240
406
|
/** The overlay requires rsync; do not allocate a recovery copy when it cannot run. */
|
|
@@ -261,6 +427,11 @@ export function requireWorkingRsync(env, err) {
|
|
|
261
427
|
}
|
|
262
428
|
else if (probe.status !== 0) {
|
|
263
429
|
err(` rsync exited with status ${probe.status}.\n`);
|
|
430
|
+
if (foundRsync) {
|
|
431
|
+
const probeOutput = boundedDiagnosticTail(output, 20, 2 * 1024);
|
|
432
|
+
err(` rsync probe output:\n${probeOutput ? `${probeOutput}\n` : " (empty)\n"}`);
|
|
433
|
+
err("HQ_RESCUE_FAILURE_KIND=rsync-found-but-broken\n");
|
|
434
|
+
}
|
|
264
435
|
}
|
|
265
436
|
else {
|
|
266
437
|
const snippet = boundedDiagnostic(output.replace(/\s+/g, " ").trim(), 512);
|
|
@@ -558,7 +729,7 @@ export function runRescue(argv, opts = {}) {
|
|
|
558
729
|
return recoverSnapshots(cfg, out, err);
|
|
559
730
|
return doRescue(cfg, env, out, err, (d) => {
|
|
560
731
|
tmpdir = d;
|
|
561
|
-
});
|
|
732
|
+
}, opts.gitReachabilityProbeTimeoutMs ?? GIT_REACHABILITY_PROBE_TIMEOUT_MS);
|
|
562
733
|
}
|
|
563
734
|
catch (e) {
|
|
564
735
|
if (e instanceof ExitError)
|
|
@@ -1084,16 +1255,11 @@ function preserveSnapshotMetadata(destination, st) {
|
|
|
1084
1255
|
/* metadata preservation is best-effort, matching the prior rollback helper */
|
|
1085
1256
|
}
|
|
1086
1257
|
}
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
throw new Error(`invalid snapshot symlink destination ${rel}`);
|
|
1093
|
-
const restoredLinkPath = snapshotRelativePath(hqRoot, rel);
|
|
1094
|
-
const resolvedTarget = path.resolve(path.dirname(restoredLinkPath), target);
|
|
1095
|
-
fs.mkdirSync(path.dirname(destination), { recursive: true });
|
|
1096
|
-
fs.rmSync(destination, { recursive: true, force: true });
|
|
1258
|
+
/**
|
|
1259
|
+
* Windows denies ordinary symbolic-link creation without Developer Mode or
|
|
1260
|
+
* elevation. Directory targets can be represented by a junction instead.
|
|
1261
|
+
*/
|
|
1262
|
+
function createSymlinkOrJunction(target, destination, resolvedTarget) {
|
|
1097
1263
|
try {
|
|
1098
1264
|
fs.symlinkSync(target, destination);
|
|
1099
1265
|
}
|
|
@@ -1107,6 +1273,18 @@ function recreateSnapshotSymlink(hqRoot, rel, target, destination = snapshotRela
|
|
|
1107
1273
|
throw error;
|
|
1108
1274
|
}
|
|
1109
1275
|
}
|
|
1276
|
+
function recreateSnapshotSymlink(hqRoot, rel, target, destination = snapshotRelativePath(hqRoot, rel)) {
|
|
1277
|
+
const problem = validateSnapshotSymlink(hqRoot, rel, target);
|
|
1278
|
+
if (problem)
|
|
1279
|
+
throw new Error(`invalid snapshot symlink: ${problem}`);
|
|
1280
|
+
if (!destination)
|
|
1281
|
+
throw new Error(`invalid snapshot symlink destination ${rel}`);
|
|
1282
|
+
const restoredLinkPath = snapshotRelativePath(hqRoot, rel);
|
|
1283
|
+
const resolvedTarget = path.resolve(path.dirname(restoredLinkPath), target);
|
|
1284
|
+
fs.mkdirSync(path.dirname(destination), { recursive: true });
|
|
1285
|
+
fs.rmSync(destination, { recursive: true, force: true });
|
|
1286
|
+
createSymlinkOrJunction(target, destination, resolvedTarget);
|
|
1287
|
+
}
|
|
1110
1288
|
/**
|
|
1111
1289
|
* Directory links can become junctions on Windows. A file link without link
|
|
1112
1290
|
* privilege instead needs copied file data at its original snapshot path.
|
|
@@ -1186,7 +1364,159 @@ export function realpathOrExit(p, err) {
|
|
|
1186
1364
|
}
|
|
1187
1365
|
}
|
|
1188
1366
|
}
|
|
1189
|
-
function
|
|
1367
|
+
function reportRsyncFailure(status, phase, stderr, err) {
|
|
1368
|
+
if (stderr)
|
|
1369
|
+
err(stderr);
|
|
1370
|
+
err(`rsync status ${status} during ${phase}\n`);
|
|
1371
|
+
err(`HQ_RESCUE_FAILURE_KIND=${status === 23 || status === 24 ? "rsync-partial" : "rsync-failed"}\n`);
|
|
1372
|
+
}
|
|
1373
|
+
function hashRecoveryTree(root, rootRel) {
|
|
1374
|
+
const entries = [];
|
|
1375
|
+
const visit = (absolute, rel) => {
|
|
1376
|
+
let stat;
|
|
1377
|
+
try {
|
|
1378
|
+
stat = fs.lstatSync(absolute);
|
|
1379
|
+
}
|
|
1380
|
+
catch (error) {
|
|
1381
|
+
if (error.code === "ENOENT") {
|
|
1382
|
+
if (rel === rootRel)
|
|
1383
|
+
return;
|
|
1384
|
+
throw error;
|
|
1385
|
+
}
|
|
1386
|
+
throw error;
|
|
1387
|
+
}
|
|
1388
|
+
if (stat.isDirectory()) {
|
|
1389
|
+
entries.push({ rel, kind: "directory" });
|
|
1390
|
+
let names;
|
|
1391
|
+
try {
|
|
1392
|
+
names = fs.readdirSync(absolute).sort();
|
|
1393
|
+
}
|
|
1394
|
+
catch (error) {
|
|
1395
|
+
throw new Error(`${rel} could not be read: ${error instanceof Error ? error.message : String(error)}`);
|
|
1396
|
+
}
|
|
1397
|
+
for (const name of names) {
|
|
1398
|
+
visit(path.join(absolute, name), path.posix.join(rel, name));
|
|
1399
|
+
}
|
|
1400
|
+
return;
|
|
1401
|
+
}
|
|
1402
|
+
if (stat.isSymbolicLink()) {
|
|
1403
|
+
let target;
|
|
1404
|
+
try {
|
|
1405
|
+
target = fs.readlinkSync(absolute);
|
|
1406
|
+
}
|
|
1407
|
+
catch (error) {
|
|
1408
|
+
throw new Error(`${rel} symlink target could not be read: ${error instanceof Error ? error.message : String(error)}`);
|
|
1409
|
+
}
|
|
1410
|
+
entries.push({ rel, kind: "symlink", target });
|
|
1411
|
+
return;
|
|
1412
|
+
}
|
|
1413
|
+
if (stat.isFile()) {
|
|
1414
|
+
let bytes;
|
|
1415
|
+
try {
|
|
1416
|
+
bytes = fs.readFileSync(absolute);
|
|
1417
|
+
}
|
|
1418
|
+
catch (error) {
|
|
1419
|
+
throw new Error(`${rel} could not be read: ${error instanceof Error ? error.message : String(error)}`);
|
|
1420
|
+
}
|
|
1421
|
+
entries.push({ rel, kind: "file", bytes });
|
|
1422
|
+
return;
|
|
1423
|
+
}
|
|
1424
|
+
throw new Error(`${rel} has unsupported filesystem entry type`);
|
|
1425
|
+
};
|
|
1426
|
+
const absoluteRoot = path.join(root, rootRel);
|
|
1427
|
+
try {
|
|
1428
|
+
try {
|
|
1429
|
+
fs.lstatSync(absoluteRoot);
|
|
1430
|
+
}
|
|
1431
|
+
catch (error) {
|
|
1432
|
+
if (error.code === "ENOENT") {
|
|
1433
|
+
return { kind: "missing" };
|
|
1434
|
+
}
|
|
1435
|
+
throw error;
|
|
1436
|
+
}
|
|
1437
|
+
visit(absoluteRoot, rootRel);
|
|
1438
|
+
}
|
|
1439
|
+
catch (error) {
|
|
1440
|
+
return {
|
|
1441
|
+
kind: "unreadable",
|
|
1442
|
+
message: error instanceof Error ? error.message : String(error),
|
|
1443
|
+
};
|
|
1444
|
+
}
|
|
1445
|
+
const hash = crypto.createHash("sha256");
|
|
1446
|
+
for (const entry of entries.sort((left, right) => left.rel.localeCompare(right.rel))) {
|
|
1447
|
+
hash.update(entry.rel);
|
|
1448
|
+
hash.update("\0");
|
|
1449
|
+
hash.update(entry.kind);
|
|
1450
|
+
hash.update("\0");
|
|
1451
|
+
if (entry.kind === "file")
|
|
1452
|
+
hash.update(entry.bytes);
|
|
1453
|
+
if (entry.kind === "symlink")
|
|
1454
|
+
hash.update(entry.target);
|
|
1455
|
+
hash.update("\0");
|
|
1456
|
+
}
|
|
1457
|
+
return { kind: "present", digest: hash.digest("hex") };
|
|
1458
|
+
}
|
|
1459
|
+
function rsyncOutputHasOnlyDirectoryChanges(output) {
|
|
1460
|
+
for (const rawLine of output.split(/\r?\n/)) {
|
|
1461
|
+
const line = rawLine.trim();
|
|
1462
|
+
if (!line)
|
|
1463
|
+
continue;
|
|
1464
|
+
if (line.startsWith("sending incremental file list") ||
|
|
1465
|
+
line.startsWith("sent ") ||
|
|
1466
|
+
line.startsWith("total size is ") ||
|
|
1467
|
+
line.startsWith("created directory ")) {
|
|
1468
|
+
continue;
|
|
1469
|
+
}
|
|
1470
|
+
if (line.startsWith("*deleting "))
|
|
1471
|
+
return false;
|
|
1472
|
+
const itemized = /^([<>ch.*])([fdLDS])[^ ]*\s+/.exec(line);
|
|
1473
|
+
if (!itemized || itemized[2] !== "d")
|
|
1474
|
+
return false;
|
|
1475
|
+
}
|
|
1476
|
+
return true;
|
|
1477
|
+
}
|
|
1478
|
+
function recoveryProof(unresolved, cfg, hqRoot, srcDir, srcSha, rsyncLauncher, rsyncExcludes, env) {
|
|
1479
|
+
const newest = [...unresolved].sort((left, right) => right.entry.completedMs - left.entry.completedMs ||
|
|
1480
|
+
right.entry.createdMs - left.entry.createdMs ||
|
|
1481
|
+
right.entry.name.localeCompare(left.entry.name))[0];
|
|
1482
|
+
if (!newest)
|
|
1483
|
+
return { eligible: false, failures: [] };
|
|
1484
|
+
const failures = [];
|
|
1485
|
+
if (typeof newest.entry.manifest.sourceSha !== "string" ||
|
|
1486
|
+
newest.entry.manifest.sourceSha !== srcSha) {
|
|
1487
|
+
failures.push("sha");
|
|
1488
|
+
}
|
|
1489
|
+
const releaseProof = run(rsyncLauncher.command, [
|
|
1490
|
+
...rsyncLauncher.leadingArgs,
|
|
1491
|
+
"-a",
|
|
1492
|
+
"--checksum",
|
|
1493
|
+
"--no-times",
|
|
1494
|
+
"--dry-run",
|
|
1495
|
+
"--itemize-changes",
|
|
1496
|
+
...rsyncExcludes,
|
|
1497
|
+
toCygwinRsyncPath(srcDir + "/"),
|
|
1498
|
+
toCygwinRsyncPath(hqRoot + "/"),
|
|
1499
|
+
], { env });
|
|
1500
|
+
if (releaseProof.status !== 0 ||
|
|
1501
|
+
!rsyncOutputHasOnlyDirectoryChanges(`${releaseProof.stdout}\n${releaseProof.stderr}`)) {
|
|
1502
|
+
failures.push("release-bytes");
|
|
1503
|
+
}
|
|
1504
|
+
if (!hasSkippedSnapshotEntries(newest.entry.manifest)) {
|
|
1505
|
+
const snapshotDir = path.join(cfg.backupRoot, newest.entry.name);
|
|
1506
|
+
for (const rel of CARVE_OUT_PATHS) {
|
|
1507
|
+
const live = hashRecoveryTree(hqRoot, rel);
|
|
1508
|
+
const snapshot = hashRecoveryTree(snapshotDir, rel);
|
|
1509
|
+
const equal = (live.kind === "missing" && snapshot.kind === "missing") ||
|
|
1510
|
+
(live.kind === "present" &&
|
|
1511
|
+
snapshot.kind === "present" &&
|
|
1512
|
+
live.digest === snapshot.digest);
|
|
1513
|
+
if (!equal)
|
|
1514
|
+
failures.push(`carve-out:${rel}`);
|
|
1515
|
+
}
|
|
1516
|
+
}
|
|
1517
|
+
return { eligible: !hasSkippedSnapshotEntries(newest.entry.manifest), failures };
|
|
1518
|
+
}
|
|
1519
|
+
function doRescue(cfg, env, out, err, setTmp, gitReachabilityProbeTimeoutMs = GIT_REACHABILITY_PROBE_TIMEOUT_MS) {
|
|
1190
1520
|
const hqRoot = resolveHqRoot(cfg, err);
|
|
1191
1521
|
const RUN_TS = utcStamp(new Date(), "dash");
|
|
1192
1522
|
// Both the release overlay and a restorable snapshot require rsync. A shim
|
|
@@ -1332,45 +1662,59 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
1332
1662
|
if (diagnostic)
|
|
1333
1663
|
err(`${redactGitDiagnostic(diagnostic, ghToken)}\n`);
|
|
1334
1664
|
};
|
|
1665
|
+
const reportFatalGitFailure = (gitResult, fallbackMessage) => {
|
|
1666
|
+
reportGitFailure(gitResult.result);
|
|
1667
|
+
if (gitResult.failureKind === "network-unreachable") {
|
|
1668
|
+
err("HQ_RESCUE_FAILURE_KIND=network-unreachable\n");
|
|
1669
|
+
err("error: HQ could not reach GitHub after retries. Check your Internet connection, then retry. Nothing was changed.\n");
|
|
1670
|
+
}
|
|
1671
|
+
else if (gitResult.failureKind === "missing-dependency") {
|
|
1672
|
+
err("HQ_RESCUE_FAILURE_KIND=missing-dependency\n");
|
|
1673
|
+
err("error: Git needs the macOS command line tools. Install the command line tools, then retry. Nothing was changed.\n");
|
|
1674
|
+
}
|
|
1675
|
+
else {
|
|
1676
|
+
err(fallbackMessage);
|
|
1677
|
+
}
|
|
1678
|
+
throw new ExitError(5);
|
|
1679
|
+
};
|
|
1680
|
+
// One bounded ls-remote per update, not per clone or checkout attempt.
|
|
1681
|
+
// Only network-unreachable and missing-dependency stop the rescue here;
|
|
1682
|
+
// any other probe error is left for the clone to report, so a successful
|
|
1683
|
+
// probe cannot mask a later clone failure.
|
|
1684
|
+
const probeResult = runGitWithNetworkRetry(["ls-remote", "--exit-code", cloneUrl, cfg.ref], { env, timeout: gitReachabilityProbeTimeoutMs }, out);
|
|
1685
|
+
if (probeResult.failureKind === "network-unreachable" ||
|
|
1686
|
+
probeResult.failureKind === "missing-dependency") {
|
|
1687
|
+
reportFatalGitFailure(probeResult, "error: clone failed\n");
|
|
1688
|
+
}
|
|
1335
1689
|
out("\n");
|
|
1336
1690
|
if (cfg.historyCheck) {
|
|
1337
1691
|
out(`==> Cloning ${cloneUrlDisplay} @${cfg.ref} (full history, blob:none filter) ...\n`);
|
|
1338
|
-
const cloneResult =
|
|
1339
|
-
if (cloneResult.status !== 0) {
|
|
1340
|
-
|
|
1341
|
-
err("error: clone failed\n");
|
|
1342
|
-
throw new ExitError(5);
|
|
1692
|
+
const cloneResult = runGitWithNetworkRetry(["clone", "--filter=blob:none", cloneUrl, srcDir], { env }, out, () => fs.rmSync(srcDir, { recursive: true, force: true }));
|
|
1693
|
+
if (cloneResult.result.status !== 0) {
|
|
1694
|
+
reportFatalGitFailure(cloneResult, "error: clone failed\n");
|
|
1343
1695
|
}
|
|
1344
|
-
const checkoutResult =
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
});
|
|
1348
|
-
if (checkoutResult.status !== 0) {
|
|
1349
|
-
reportGitFailure(checkoutResult);
|
|
1350
|
-
err(`error: could not check out ref '${cfg.ref}' from ${cfg.sourceRepo}\n`);
|
|
1351
|
-
throw new ExitError(5);
|
|
1696
|
+
const checkoutResult = runGitWithNetworkRetry(["checkout", cfg.ref], { cwd: srcDir, env }, out);
|
|
1697
|
+
if (checkoutResult.result.status !== 0) {
|
|
1698
|
+
reportFatalGitFailure(checkoutResult, `error: could not check out ref '${cfg.ref}' from ${cfg.sourceRepo}\n`);
|
|
1352
1699
|
}
|
|
1353
1700
|
}
|
|
1354
1701
|
else {
|
|
1355
1702
|
out(`==> Cloning ${cloneUrlDisplay} @${cfg.ref} (shallow) ...\n`);
|
|
1356
|
-
const shallowCloneResult =
|
|
1357
|
-
if (shallowCloneResult.status !== 0) {
|
|
1358
|
-
|
|
1703
|
+
const shallowCloneResult = runGitWithNetworkRetry(["clone", "--depth", "1", "--branch", cfg.ref, cloneUrl, srcDir], { env }, out, () => fs.rmSync(srcDir, { recursive: true, force: true }));
|
|
1704
|
+
if (shallowCloneResult.result.status !== 0) {
|
|
1705
|
+
if (shouldStopAfterShallowCloneFailure(shallowCloneResult)) {
|
|
1706
|
+
reportFatalGitFailure(shallowCloneResult, "error: clone failed\n");
|
|
1707
|
+
}
|
|
1708
|
+
reportGitFailure(shallowCloneResult.result);
|
|
1359
1709
|
out(" (shallow branch clone failed; trying full clone + checkout)\n");
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
throw new ExitError(5);
|
|
1710
|
+
fs.rmSync(srcDir, { recursive: true, force: true });
|
|
1711
|
+
const cloneResult = runGitWithNetworkRetry(["clone", cloneUrl, srcDir], { env }, out, () => fs.rmSync(srcDir, { recursive: true, force: true }));
|
|
1712
|
+
if (cloneResult.result.status !== 0) {
|
|
1713
|
+
reportFatalGitFailure(cloneResult, "error: clone failed\n");
|
|
1365
1714
|
}
|
|
1366
|
-
const checkoutResult =
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
});
|
|
1370
|
-
if (checkoutResult.status !== 0) {
|
|
1371
|
-
reportGitFailure(checkoutResult);
|
|
1372
|
-
err(`error: could not check out ref '${cfg.ref}' from ${cfg.sourceRepo}\n`);
|
|
1373
|
-
throw new ExitError(5);
|
|
1715
|
+
const checkoutResult = runGitWithNetworkRetry(["checkout", cfg.ref], { cwd: srcDir, env }, out);
|
|
1716
|
+
if (checkoutResult.result.status !== 0) {
|
|
1717
|
+
reportFatalGitFailure(checkoutResult, `error: could not check out ref '${cfg.ref}' from ${cfg.sourceRepo}\n`);
|
|
1374
1718
|
}
|
|
1375
1719
|
}
|
|
1376
1720
|
}
|
|
@@ -1500,6 +1844,7 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
1500
1844
|
unchangedList,
|
|
1501
1845
|
appendLog,
|
|
1502
1846
|
out,
|
|
1847
|
+
err,
|
|
1503
1848
|
decisions: [],
|
|
1504
1849
|
actions: [],
|
|
1505
1850
|
skippedSnapshotRels: [],
|
|
@@ -1613,7 +1958,7 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
1613
1958
|
// Cygwin path conversion or filter failure therefore leaves the root intact.
|
|
1614
1959
|
const overlayPreflight = run(rsyncLauncher.command, [...rsyncLauncher.leadingArgs, "--dry-run", ...overlayArgs], { env });
|
|
1615
1960
|
if (overlayPreflight.status !== 0) {
|
|
1616
|
-
|
|
1961
|
+
reportRsyncFailure(overlayPreflight.status, "dry-run", overlayPreflight.stderr, err);
|
|
1617
1962
|
throw new ExitError(overlayPreflight.status || 1);
|
|
1618
1963
|
}
|
|
1619
1964
|
ctx.actions = buildRescueActions(ctx);
|
|
@@ -1650,15 +1995,25 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
1650
1995
|
if (postRestore.problem) {
|
|
1651
1996
|
err(`error: post-restore permit is not usable: ${postRestore.problem}\n`);
|
|
1652
1997
|
}
|
|
1653
|
-
const
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1998
|
+
const proof = recoveryProof(unresolved, cfg, hqRoot, srcDir, srcSha, rsyncLauncher, rsyncExcludes, env);
|
|
1999
|
+
if (proof.eligible && proof.failures.length === 0) {
|
|
2000
|
+
out("==> Recovery proof passed: interrupted update matches the release and preserved paths; continuing.\n");
|
|
2001
|
+
err("HQ_RESCUE_NOTE=recovery-proof-passed\n");
|
|
2002
|
+
}
|
|
2003
|
+
else {
|
|
2004
|
+
for (const failure of proof.failures) {
|
|
2005
|
+
err(`recovery proof failed: ${failure}\n`);
|
|
2006
|
+
}
|
|
2007
|
+
const snapshotNames = unresolved.map((x) => x.entry.name);
|
|
2008
|
+
writeRescueBlockedMarker(hqRoot, snapshotNames);
|
|
2009
|
+
err("error: safety snapshot circuit breaker is open: two interrupted updates still need recovery proof.\n");
|
|
2010
|
+
err(" HQ kept their pre-update snapshots because those updates started changing the release-managed tree but did not finish.\n");
|
|
2011
|
+
err(" It will not overwrite or delete that evidence without proof that a later rescue completed successfully.\n");
|
|
2012
|
+
err(` Inspect safely: ${formatRecoveryCommand(hqRoot, cfg.backupRoot)}\n`);
|
|
2013
|
+
err(" It deletes only snapshots with a later completed-good rescue. For unresolved snapshots, use --restore-snapshot to restore every captured root and recorded link, then run its verification command before retrying.\n");
|
|
2014
|
+
err("HQ_RESCUE_FAILURE_KIND=snapshot-recovery-circuit-breaker\n");
|
|
2015
|
+
throw new ExitError(1);
|
|
2016
|
+
}
|
|
1662
2017
|
}
|
|
1663
2018
|
}
|
|
1664
2019
|
requireBackupCapacity(cfg, hqRoot, wipeToplevel, env, err);
|
|
@@ -1761,7 +2116,13 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
1761
2116
|
const src = path.join(hqRoot, sp);
|
|
1762
2117
|
if (lexists(src)) {
|
|
1763
2118
|
shuttleId += 1;
|
|
1764
|
-
|
|
2119
|
+
try {
|
|
2120
|
+
cpA(src, shuttle, String(shuttleId));
|
|
2121
|
+
}
|
|
2122
|
+
catch (error) {
|
|
2123
|
+
reportPreserveBackupFailure(sp, error, err);
|
|
2124
|
+
throw error;
|
|
2125
|
+
}
|
|
1765
2126
|
preserveMap.push({ id: shuttleId, rel: sp });
|
|
1766
2127
|
out(`==> Backed up ${sp} -> shuttle/${shuttleId}\n`);
|
|
1767
2128
|
}
|
|
@@ -1791,24 +2152,30 @@ function doRescue(cfg, env, out, err, setTmp) {
|
|
|
1791
2152
|
const overlay = run(rsyncLauncher.command, [...rsyncLauncher.leadingArgs, ...overlayRunArgs], { env });
|
|
1792
2153
|
if (overlay.status !== 0) {
|
|
1793
2154
|
// bash runs under set -e: a failed rsync aborts with its status.
|
|
1794
|
-
|
|
2155
|
+
reportRsyncFailure(overlay.status, "overlay", overlay.stderr, err);
|
|
1795
2156
|
throw new ExitError(overlay.status || 1);
|
|
1796
2157
|
}
|
|
1797
2158
|
// --- Restore preserved sub-paths ---
|
|
1798
2159
|
if (preserveMap.length > 0) {
|
|
1799
2160
|
out("==> Restoring preserved sub-paths ...\n");
|
|
2161
|
+
const restoreFailures = [];
|
|
1800
2162
|
for (const { id, rel } of preserveMap) {
|
|
1801
2163
|
const dest = path.join(hqRoot, rel);
|
|
1802
|
-
fs.mkdirSync(path.dirname(dest), { recursive: true });
|
|
1803
2164
|
const shuttleItem = path.join(shuttle, String(id));
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
|
|
2165
|
+
try {
|
|
2166
|
+
restorePreservedPath(shuttleItem, dest);
|
|
2167
|
+
out(` restored ${rel}\n`);
|
|
1807
2168
|
}
|
|
1808
|
-
|
|
1809
|
-
|
|
2169
|
+
catch (error) {
|
|
2170
|
+
restoreFailures.push({ relativePath: rel, error, shuttlePath: shuttleItem });
|
|
1810
2171
|
}
|
|
1811
|
-
|
|
2172
|
+
}
|
|
2173
|
+
if (restoreFailures.length > 0) {
|
|
2174
|
+
for (const failure of restoreFailures) {
|
|
2175
|
+
failure.safetyCopyPath = copyUnrestoredPreserveToSafetySnapshot(failure.shuttlePath, failure.relativePath, cfg.backupRoot, backupDir);
|
|
2176
|
+
}
|
|
2177
|
+
reportPreserveRestoreFailures(restoreFailures, hqRoot, cfg.backupRoot, backupDir, err);
|
|
2178
|
+
throw new ExitError(1);
|
|
1812
2179
|
}
|
|
1813
2180
|
}
|
|
1814
2181
|
// --- Reconcile the generated .claude/settings.json ---
|
|
@@ -2940,6 +3307,84 @@ function readFileOrEmpty(p) {
|
|
|
2940
3307
|
return "";
|
|
2941
3308
|
}
|
|
2942
3309
|
}
|
|
3310
|
+
const RESCUE_MOVE_FALLBACK_CODES = new Set([
|
|
3311
|
+
"ENOTEMPTY",
|
|
3312
|
+
"EEXIST",
|
|
3313
|
+
"EISDIR",
|
|
3314
|
+
"EXDEV",
|
|
3315
|
+
]);
|
|
3316
|
+
function destinationExists(p) {
|
|
3317
|
+
try {
|
|
3318
|
+
fs.lstatSync(p);
|
|
3319
|
+
return true;
|
|
3320
|
+
}
|
|
3321
|
+
catch (error) {
|
|
3322
|
+
if (error.code === "ENOENT")
|
|
3323
|
+
return false;
|
|
3324
|
+
throw error;
|
|
3325
|
+
}
|
|
3326
|
+
}
|
|
3327
|
+
function rescueMoveSibling(dest) {
|
|
3328
|
+
for (let attempt = 0; attempt < 1000; attempt += 1) {
|
|
3329
|
+
const sibling = `${dest}.hq-rescue-${process.pid}-${attempt}`;
|
|
3330
|
+
if (confirmedAbsent(sibling))
|
|
3331
|
+
return sibling;
|
|
3332
|
+
}
|
|
3333
|
+
throw new Error(`could not allocate a rescue move sibling for ${dest}`);
|
|
3334
|
+
}
|
|
3335
|
+
function reportRescueMoveFailure(ctx, sourceRel, destRel, error) {
|
|
3336
|
+
ctx.err("HQ_RESCUE_FAILURE_KIND=rescue-move-failed\n");
|
|
3337
|
+
ctx.err(` source: ${sourceRel}\n`);
|
|
3338
|
+
ctx.err(` destination: ${destRel}\n`);
|
|
3339
|
+
ctx.err(` Could not copy the user-edited file to the rescue sibling: ${preservePathFailureMessage(error)} (${preservePathFailureCode(error)})\n`);
|
|
3340
|
+
}
|
|
3341
|
+
/** Move a rescued file without ever renaming it onto an existing directory. */
|
|
3342
|
+
function moveRescueFile(ctx, sourceRel, dest) {
|
|
3343
|
+
const source = path.join(ctx.hqRoot, sourceRel);
|
|
3344
|
+
const destRel = posixRelFromRoot(ctx.hqRoot, dest);
|
|
3345
|
+
let fallback = destinationExists(dest);
|
|
3346
|
+
if (!fallback) {
|
|
3347
|
+
try {
|
|
3348
|
+
fs.renameSync(source, dest);
|
|
3349
|
+
return destRel;
|
|
3350
|
+
}
|
|
3351
|
+
catch (error) {
|
|
3352
|
+
if (!RESCUE_MOVE_FALLBACK_CODES.has(error.code ?? "")) {
|
|
3353
|
+
throw error;
|
|
3354
|
+
}
|
|
3355
|
+
fallback = true;
|
|
3356
|
+
}
|
|
3357
|
+
}
|
|
3358
|
+
if (!fallback)
|
|
3359
|
+
return destRel;
|
|
3360
|
+
let sibling;
|
|
3361
|
+
try {
|
|
3362
|
+
sibling = rescueMoveSibling(dest);
|
|
3363
|
+
const sourceStat = fs.lstatSync(source);
|
|
3364
|
+
fs.copyFileSync(source, sibling, fs.constants.COPYFILE_EXCL);
|
|
3365
|
+
try {
|
|
3366
|
+
fs.chmodSync(sibling, sourceStat.mode);
|
|
3367
|
+
fs.utimesSync(sibling, sourceStat.atime, sourceStat.mtime);
|
|
3368
|
+
}
|
|
3369
|
+
catch {
|
|
3370
|
+
/* byte copy succeeded; metadata remains best-effort like cp -a */
|
|
3371
|
+
}
|
|
3372
|
+
fs.unlinkSync(source);
|
|
3373
|
+
return posixRelFromRoot(ctx.hqRoot, sibling);
|
|
3374
|
+
}
|
|
3375
|
+
catch (error) {
|
|
3376
|
+
if (sibling && lexists(sibling)) {
|
|
3377
|
+
try {
|
|
3378
|
+
fs.unlinkSync(sibling);
|
|
3379
|
+
}
|
|
3380
|
+
catch {
|
|
3381
|
+
/* retain the primary move failure for the caller */
|
|
3382
|
+
}
|
|
3383
|
+
}
|
|
3384
|
+
reportRescueMoveFailure(ctx, sourceRel, destRel, error);
|
|
3385
|
+
throw new ExitError(1);
|
|
3386
|
+
}
|
|
3387
|
+
}
|
|
2943
3388
|
// --- rescue a single USER-EDIT file into personal/ ---
|
|
2944
3389
|
function rescueOne(ctx, rel) {
|
|
2945
3390
|
const localPath = path.join(ctx.hqRoot, rel);
|
|
@@ -2951,32 +3396,19 @@ function rescueOne(ctx, rel) {
|
|
|
2951
3396
|
return;
|
|
2952
3397
|
}
|
|
2953
3398
|
const target = mapRescueTarget(rel);
|
|
2954
|
-
|
|
3399
|
+
const dest = path.join(ctx.hqRoot, target);
|
|
2955
3400
|
fs.mkdirSync(path.dirname(dest), { recursive: true });
|
|
2956
|
-
|
|
2957
|
-
dest = `${dest}.drift-${Math.floor(Date.now() / 1000)}-${process.pid}`;
|
|
2958
|
-
}
|
|
2959
|
-
fs.renameSync(localPath, dest);
|
|
2960
|
-
const destRel = dest.startsWith(ctx.hqRoot + "/")
|
|
2961
|
-
? dest.slice(ctx.hqRoot.length + 1)
|
|
2962
|
-
: dest;
|
|
3401
|
+
const destRel = moveRescueFile(ctx, rel, dest);
|
|
2963
3402
|
ctx.out(` rescued: ${rel} -> ${destRel}\n`);
|
|
2964
3403
|
ctx.appendLog(`rescued\t${rel}\t-> ${destRel}\n`);
|
|
2965
3404
|
}
|
|
2966
3405
|
// --- quarantine a single USER-EDIT file into .hq-conflicts/ ---
|
|
2967
3406
|
function conflictOne(ctx, rel) {
|
|
2968
|
-
const localPath = path.join(ctx.hqRoot, rel);
|
|
2969
3407
|
ctx.counts.userConflict += 1;
|
|
2970
3408
|
const target = `.hq-conflicts/rescue-${ctx.runTs}/${rel}`;
|
|
2971
|
-
|
|
3409
|
+
const dest = path.join(ctx.hqRoot, target);
|
|
2972
3410
|
fs.mkdirSync(path.dirname(dest), { recursive: true });
|
|
2973
|
-
|
|
2974
|
-
dest = `${dest}.drift-${Math.floor(Date.now() / 1000)}-${process.pid}`;
|
|
2975
|
-
}
|
|
2976
|
-
fs.renameSync(localPath, dest);
|
|
2977
|
-
const destRel = dest.startsWith(ctx.hqRoot + "/")
|
|
2978
|
-
? dest.slice(ctx.hqRoot.length + 1)
|
|
2979
|
-
: dest;
|
|
3411
|
+
const destRel = moveRescueFile(ctx, rel, dest);
|
|
2980
3412
|
ctx.out(` conflicted: ${rel} -> ${destRel}\n`);
|
|
2981
3413
|
ctx.appendLog(`conflicted\t${rel}\t-> ${destRel}\n`);
|
|
2982
3414
|
}
|
|
@@ -3936,6 +4368,30 @@ function cpA(src, destDir, asName) {
|
|
|
3936
4368
|
: path.join(destDir, path.basename(src));
|
|
3937
4369
|
cpATo(src, dest);
|
|
3938
4370
|
}
|
|
4371
|
+
/** Copy a link, using the same Windows representation choices as snapshots. */
|
|
4372
|
+
function copySymlinkWithWindowsFallback(src, dest, link) {
|
|
4373
|
+
const resolvedTarget = path.resolve(path.dirname(src), link);
|
|
4374
|
+
let linkPrivilegeError;
|
|
4375
|
+
try {
|
|
4376
|
+
createSymlinkOrJunction(link, dest, resolvedTarget);
|
|
4377
|
+
return;
|
|
4378
|
+
}
|
|
4379
|
+
catch (error) {
|
|
4380
|
+
if (process.platform !== "win32" ||
|
|
4381
|
+
error.code !== "EPERM") {
|
|
4382
|
+
throw error;
|
|
4383
|
+
}
|
|
4384
|
+
linkPrivilegeError = error;
|
|
4385
|
+
}
|
|
4386
|
+
const targetStats = fs.statSync(src);
|
|
4387
|
+
if (targetStats.isFile()) {
|
|
4388
|
+
fs.mkdirSync(path.dirname(dest), { recursive: true });
|
|
4389
|
+
fs.copyFileSync(src, dest);
|
|
4390
|
+
preserveSnapshotMetadata(dest, targetStats);
|
|
4391
|
+
return;
|
|
4392
|
+
}
|
|
4393
|
+
throw linkPrivilegeError;
|
|
4394
|
+
}
|
|
3939
4395
|
/** `cp -a <src> <dest>` with an explicit destination path. */
|
|
3940
4396
|
function cpATo(src, dest) {
|
|
3941
4397
|
const st = lstatOrNull(src);
|
|
@@ -3943,16 +4399,16 @@ function cpATo(src, dest) {
|
|
|
3943
4399
|
return;
|
|
3944
4400
|
if (st.isSymbolicLink()) {
|
|
3945
4401
|
const link = fs.readlinkSync(src);
|
|
3946
|
-
|
|
3947
|
-
fs.rmSync(dest, { force: true });
|
|
3948
|
-
}
|
|
3949
|
-
catch {
|
|
3950
|
-
/* ignore */
|
|
4402
|
+
if (lstatOrNull(dest)) {
|
|
4403
|
+
fs.rmSync(dest, { recursive: true, force: true });
|
|
3951
4404
|
}
|
|
3952
|
-
|
|
4405
|
+
copySymlinkWithWindowsFallback(src, dest, link);
|
|
3953
4406
|
return;
|
|
3954
4407
|
}
|
|
3955
4408
|
if (st.isDirectory()) {
|
|
4409
|
+
if (lstatOrNull(dest)?.isSymbolicLink()) {
|
|
4410
|
+
fs.rmSync(dest, { recursive: true, force: true });
|
|
4411
|
+
}
|
|
3956
4412
|
fs.mkdirSync(dest, { recursive: true });
|
|
3957
4413
|
for (const name of fs.readdirSync(src)) {
|
|
3958
4414
|
cpATo(path.join(src, name), path.join(dest, name));
|
|
@@ -3966,6 +4422,10 @@ function cpATo(src, dest) {
|
|
|
3966
4422
|
return;
|
|
3967
4423
|
}
|
|
3968
4424
|
// Regular file.
|
|
4425
|
+
const destinationStats = lstatOrNull(dest);
|
|
4426
|
+
if (destinationStats && !destinationStats.isFile()) {
|
|
4427
|
+
fs.rmSync(dest, { recursive: true, force: true });
|
|
4428
|
+
}
|
|
3969
4429
|
fs.copyFileSync(src, dest);
|
|
3970
4430
|
try {
|
|
3971
4431
|
fs.chmodSync(dest, st.mode);
|
|
@@ -3975,6 +4435,138 @@ function cpATo(src, dest) {
|
|
|
3975
4435
|
/* ignore */
|
|
3976
4436
|
}
|
|
3977
4437
|
}
|
|
4438
|
+
function confirmedAbsent(p) {
|
|
4439
|
+
try {
|
|
4440
|
+
fs.lstatSync(p);
|
|
4441
|
+
return false;
|
|
4442
|
+
}
|
|
4443
|
+
catch (error) {
|
|
4444
|
+
if (error.code === "ENOENT")
|
|
4445
|
+
return true;
|
|
4446
|
+
throw error;
|
|
4447
|
+
}
|
|
4448
|
+
}
|
|
4449
|
+
function restoreStagePath(dest) {
|
|
4450
|
+
for (let attempt = 0; attempt < 1000; attempt += 1) {
|
|
4451
|
+
const suffix = attempt === 0 ? "" : `-${attempt}`;
|
|
4452
|
+
const staged = `${dest}.hq-restore-${process.pid}${suffix}`;
|
|
4453
|
+
if (confirmedAbsent(staged))
|
|
4454
|
+
return staged;
|
|
4455
|
+
}
|
|
4456
|
+
throw new Error(`could not allocate a restore staging sibling for ${dest}`);
|
|
4457
|
+
}
|
|
4458
|
+
/** Copy into a sibling before touching the live preserved path. */
|
|
4459
|
+
function restorePreservedPath(shuttleItem, dest) {
|
|
4460
|
+
const staged = restoreStagePath(dest);
|
|
4461
|
+
let completed = false;
|
|
4462
|
+
try {
|
|
4463
|
+
fs.mkdirSync(path.dirname(dest), { recursive: true });
|
|
4464
|
+
cpATo(shuttleItem, staged);
|
|
4465
|
+
fs.rmSync(dest, { recursive: true, force: true });
|
|
4466
|
+
try {
|
|
4467
|
+
fs.renameSync(staged, dest);
|
|
4468
|
+
}
|
|
4469
|
+
catch {
|
|
4470
|
+
cpATo(staged, dest);
|
|
4471
|
+
fs.rmSync(staged, { recursive: true, force: true });
|
|
4472
|
+
}
|
|
4473
|
+
completed = true;
|
|
4474
|
+
}
|
|
4475
|
+
finally {
|
|
4476
|
+
if (!completed && lexists(staged)) {
|
|
4477
|
+
try {
|
|
4478
|
+
fs.rmSync(staged, { recursive: true, force: true });
|
|
4479
|
+
}
|
|
4480
|
+
catch {
|
|
4481
|
+
/* preserve the original restore failure for the caller */
|
|
4482
|
+
}
|
|
4483
|
+
}
|
|
4484
|
+
}
|
|
4485
|
+
}
|
|
4486
|
+
function preservePathFailureCode(error) {
|
|
4487
|
+
return error?.code ?? "UNKNOWN";
|
|
4488
|
+
}
|
|
4489
|
+
function preservePathFailureMessage(error) {
|
|
4490
|
+
return error instanceof Error ? error.message : String(error);
|
|
4491
|
+
}
|
|
4492
|
+
function isWindowsPrivilegeFailure(error) {
|
|
4493
|
+
if (process.platform !== "win32")
|
|
4494
|
+
return false;
|
|
4495
|
+
const err = error;
|
|
4496
|
+
if (err?.code !== "EPERM")
|
|
4497
|
+
return false;
|
|
4498
|
+
// Developer Mode only helps creating a symbolic link. mkdir/rm/copyFile
|
|
4499
|
+
// EPERM is an ACL, lock, or read-only path and the same advice cannot fix it.
|
|
4500
|
+
return err.syscall === "symlink";
|
|
4501
|
+
}
|
|
4502
|
+
function reportWindowsLinkPrivilegeRemedy(err) {
|
|
4503
|
+
err(" On Windows, enable Developer Mode in Windows Settings or run HQ as an administrator, then retry.\n");
|
|
4504
|
+
}
|
|
4505
|
+
function reportPreserveBackupFailure(relativePath, error, err) {
|
|
4506
|
+
err(`==> Cannot preserve ${relativePath} before the update: ${preservePathFailureMessage(error)} (${preservePathFailureCode(error)})\n`);
|
|
4507
|
+
if (isWindowsPrivilegeFailure(error))
|
|
4508
|
+
reportWindowsLinkPrivilegeRemedy(err);
|
|
4509
|
+
err(" No HQ files were changed.\n");
|
|
4510
|
+
}
|
|
4511
|
+
function copyUnrestoredPreserveToSafetySnapshot(shuttlePath, relativePath, backupRoot, backupDir) {
|
|
4512
|
+
const segments = relativePath.split("/");
|
|
4513
|
+
if (!relativePath ||
|
|
4514
|
+
relativePath.startsWith("/") ||
|
|
4515
|
+
segments.some((segment) => !segment || segment === "." || segment === "..")) {
|
|
4516
|
+
return undefined;
|
|
4517
|
+
}
|
|
4518
|
+
const safetyRoot = backupDir && isDir(backupDir)
|
|
4519
|
+
? path.join(backupDir, "preserve-unrestored")
|
|
4520
|
+
: path.join(backupRoot, "preserve-unrestored");
|
|
4521
|
+
const destination = path.join(safetyRoot, ...segments);
|
|
4522
|
+
try {
|
|
4523
|
+
fs.mkdirSync(path.dirname(destination), { recursive: true });
|
|
4524
|
+
cpATo(shuttlePath, destination);
|
|
4525
|
+
return destination;
|
|
4526
|
+
}
|
|
4527
|
+
catch {
|
|
4528
|
+
return undefined;
|
|
4529
|
+
}
|
|
4530
|
+
}
|
|
4531
|
+
function reportPreserveRestoreFailures(failures, hqRoot, backupRoot, backupDir, err) {
|
|
4532
|
+
err("==> Could not restore these preserved paths after the update:\n");
|
|
4533
|
+
for (const { relativePath, error, safetyCopyPath } of failures) {
|
|
4534
|
+
err(` ${relativePath}: ${preservePathFailureMessage(error)} (${preservePathFailureCode(error)})\n`);
|
|
4535
|
+
if (safetyCopyPath) {
|
|
4536
|
+
err(` Preserved bytes for ${relativePath} were kept at ${safetyCopyPath}\n`);
|
|
4537
|
+
}
|
|
4538
|
+
else {
|
|
4539
|
+
err(` Could not keep a safety copy for ${relativePath}\n`);
|
|
4540
|
+
}
|
|
4541
|
+
}
|
|
4542
|
+
if (failures.some(({ error }) => isWindowsPrivilegeFailure(error))) {
|
|
4543
|
+
reportWindowsLinkPrivilegeRemedy(err);
|
|
4544
|
+
}
|
|
4545
|
+
err(" The updated HQ release files are already in place. Each listed path may be missing or contain the release version instead of preserved local data.\n");
|
|
4546
|
+
if (backupDir) {
|
|
4547
|
+
err(` Safety snapshot: ${backupDir}\n`);
|
|
4548
|
+
const manifest = readSnapshotManifest(backupDir);
|
|
4549
|
+
if (!manifest) {
|
|
4550
|
+
err(` Snapshot manifest is unreadable: ${backupDir}.\n`);
|
|
4551
|
+
err(` Recover by copying the files you need from ${backupDir} by hand.\n`);
|
|
4552
|
+
}
|
|
4553
|
+
else if (hasSkippedSnapshotEntries(manifest)) {
|
|
4554
|
+
const captured = manifest.includedRoots.length > 0
|
|
4555
|
+
? manifest.includedRoots.join(", ")
|
|
4556
|
+
: "(none)";
|
|
4557
|
+
err(` This snapshot is incomplete. It captured: ${captured}. It skipped: ${formatSkippedSnapshotPaths(manifest)}.\n`);
|
|
4558
|
+
err(" HQ will not restore it with --restore-snapshot, because that command refuses an incomplete snapshot so it cannot overwrite good files with a partial copy.\n");
|
|
4559
|
+
err(` Recover by copying the files you need from ${backupDir} by hand. The skipped paths were left in HQ as they were.\n`);
|
|
4560
|
+
}
|
|
4561
|
+
else {
|
|
4562
|
+
err(` Restore it with: ${formatSnapshotRestoreCommand(hqRoot, backupRoot, path.basename(backupDir))}\n`);
|
|
4563
|
+
}
|
|
4564
|
+
}
|
|
4565
|
+
else {
|
|
4566
|
+
err(" No safety snapshot was created, so the listed paths need manual recovery.\n");
|
|
4567
|
+
}
|
|
4568
|
+
err("HQ_RESCUE_FAILURE_KIND=preserve-restore-failed\n");
|
|
4569
|
+
}
|
|
3978
4570
|
class SnapshotCopyError extends Error {
|
|
3979
4571
|
relativePath;
|
|
3980
4572
|
operation;
|