@indigoai-us/hq-cloud 6.18.110 → 6.18.112
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-company.d.ts +11 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +58 -0
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +5 -0
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/cli/reconcile-skip-reason.d.ts +12 -2
- package/dist/cli/reconcile-skip-reason.d.ts.map +1 -1
- package/dist/cli/reconcile-skip-reason.js +16 -0
- package/dist/cli/reconcile-skip-reason.js.map +1 -1
- package/dist/cli/reindex.d.ts.map +1 -1
- package/dist/cli/reindex.js +3 -1
- package/dist/cli/reindex.js.map +1 -1
- package/dist/cli/rescue-core.d.ts.map +1 -1
- package/dist/cli/rescue-core.js +16 -4
- package/dist/cli/rescue-core.js.map +1 -1
- package/dist/cli/rescue-hq-registrations.d.ts +45 -0
- package/dist/cli/rescue-hq-registrations.d.ts.map +1 -0
- package/dist/cli/rescue-hq-registrations.js +252 -0
- package/dist/cli/rescue-hq-registrations.js.map +1 -0
- package/dist/cli/rescue.d.ts +8 -0
- package/dist/cli/rescue.d.ts.map +1 -1
- package/dist/cli/rescue.js +27 -8
- package/dist/cli/rescue.js.map +1 -1
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +54 -0
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +6 -2
- package/dist/cli/sync.js.map +1 -1
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +4 -1
- package/dist/object-io.js.map +1 -1
- package/dist/outposts/cli-billing.d.ts +26 -13
- package/dist/outposts/cli-billing.d.ts.map +1 -1
- package/dist/outposts/cli-billing.js +41 -18
- package/dist/outposts/cli-billing.js.map +1 -1
- package/dist/outposts/client.d.ts +20 -3
- package/dist/outposts/client.d.ts.map +1 -1
- package/dist/outposts/client.js +58 -5
- package/dist/outposts/client.js.map +1 -1
- package/dist/outposts/command-company-access.d.ts +14 -0
- package/dist/outposts/command-company-access.d.ts.map +1 -0
- package/dist/outposts/command-company-access.js +328 -0
- package/dist/outposts/command-company-access.js.map +1 -0
- package/dist/outposts/command.d.ts +9 -6
- package/dist/outposts/command.d.ts.map +1 -1
- package/dist/outposts/command.js +435 -172
- package/dist/outposts/command.js.map +1 -1
- package/dist/outposts/company-access.d.ts +92 -0
- package/dist/outposts/company-access.d.ts.map +1 -0
- package/dist/outposts/company-access.js +331 -0
- package/dist/outposts/company-access.js.map +1 -0
- package/dist/outposts/company-outposts.d.ts +34 -2
- package/dist/outposts/company-outposts.d.ts.map +1 -1
- package/dist/outposts/company-outposts.js +75 -5
- package/dist/outposts/company-outposts.js.map +1 -1
- package/dist/outposts/company.d.ts +15 -3
- package/dist/outposts/company.d.ts.map +1 -1
- package/dist/outposts/company.js +41 -8
- package/dist/outposts/company.js.map +1 -1
- package/dist/outposts/errors.d.ts +9 -0
- package/dist/outposts/errors.d.ts.map +1 -1
- package/dist/outposts/errors.js +6 -0
- package/dist/outposts/errors.js.map +1 -1
- package/dist/outposts/host-deps.d.ts +11 -3
- package/dist/outposts/host-deps.d.ts.map +1 -1
- package/dist/outposts/host-deps.js +4 -4
- package/dist/outposts/host-deps.js.map +1 -1
- package/dist/outposts/index.d.ts +1 -1
- package/dist/outposts/index.d.ts.map +1 -1
- package/dist/outposts/index.js +1 -1
- package/dist/outposts/index.js.map +1 -1
- package/dist/outposts/sizes.d.ts +13 -9
- package/dist/outposts/sizes.d.ts.map +1 -1
- package/dist/outposts/sizes.js +19 -11
- package/dist/outposts/sizes.js.map +1 -1
- package/dist/s3.d.ts +79 -0
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +542 -61
- package/dist/s3.js.map +1 -1
- package/dist/session-log-retention.d.ts +86 -0
- package/dist/session-log-retention.d.ts.map +1 -1
- package/dist/session-log-retention.js +127 -19
- package/dist/session-log-retention.js.map +1 -1
- package/dist/sync/idle-full-reconcile.d.ts +13 -3
- package/dist/sync/idle-full-reconcile.d.ts.map +1 -1
- package/dist/sync/idle-full-reconcile.js +15 -3
- package/dist/sync/idle-full-reconcile.js.map +1 -1
- package/package.json +1 -1
package/dist/s3.js
CHANGED
|
@@ -53,7 +53,10 @@ function isRetryableDownloadStreamError(err) {
|
|
|
53
53
|
const code = err.code;
|
|
54
54
|
return (code === "EPIPE" ||
|
|
55
55
|
code === "ECONNRESET" ||
|
|
56
|
-
code === "ERR_HQ_TRUNCATED_RESPONSE"
|
|
56
|
+
code === "ERR_HQ_TRUNCATED_RESPONSE" ||
|
|
57
|
+
// Another sync process took the shared stage; the retry sees it busy and
|
|
58
|
+
// downloads into a private temp file.
|
|
59
|
+
code === "ERR_HQ_STAGE_TAKEN");
|
|
57
60
|
}
|
|
58
61
|
function isRangePreconditionError(err) {
|
|
59
62
|
if (!err || typeof err !== "object")
|
|
@@ -307,12 +310,20 @@ function symlinkBodyContentHash(target) {
|
|
|
307
310
|
*
|
|
308
311
|
* The age threshold far exceeds any live transfer, so a concurrent download's
|
|
309
312
|
* fresh sentinel is never touched. Only non-directories matching the sentinel
|
|
310
|
-
* pattern are candidates.
|
|
311
|
-
* per
|
|
313
|
+
* pattern are candidates. A directory is swept at most once per age window
|
|
314
|
+
* per process. A long-running desktop runner therefore sweeps it again later.
|
|
315
|
+
* When the sweep ran only once per process, orphans created after the first
|
|
316
|
+
* sweep stayed until the runner restarted (feedback_afdc2971, feedback_4912b4fa).
|
|
312
317
|
*/
|
|
313
318
|
const STAGED_SENTINEL_PATTERN = /^\.hq-(tmp|backup)-/;
|
|
314
319
|
const STAGED_SWEEP_MAX_AGE_MS = 60 * 60 * 1000;
|
|
315
|
-
|
|
320
|
+
/**
|
|
321
|
+
* A partial download kept for a ranged resume is dropped after this long. An
|
|
322
|
+
* abandoned partial for a key that is never downloaded again would otherwise
|
|
323
|
+
* stay in the user's folder indefinitely.
|
|
324
|
+
*/
|
|
325
|
+
const RESUMABLE_STAGE_MAX_AGE_MS = 24 * 60 * 60 * 1000;
|
|
326
|
+
const stagedDirSweptAt = new Map();
|
|
316
327
|
function resumableStagePaths(dir, key) {
|
|
317
328
|
const keyHash = crypto.createHash("sha256").update(key).digest("hex");
|
|
318
329
|
const stagePath = path.join(dir, `.hq-tmp-resume-${keyHash}`);
|
|
@@ -331,7 +342,8 @@ function readResumableStageMetadata(metadataPath) {
|
|
|
331
342
|
value.etag.length === 0 ||
|
|
332
343
|
!Number.isSafeInteger(value.expectedSize) ||
|
|
333
344
|
(value.expectedSize ?? 0) <= 0 ||
|
|
334
|
-
(value.versionId !== undefined && typeof value.versionId !== "string")
|
|
345
|
+
(value.versionId !== undefined && typeof value.versionId !== "string") ||
|
|
346
|
+
(value.pid !== undefined && (!Number.isSafeInteger(value.pid) || value.pid <= 0))) {
|
|
335
347
|
return null;
|
|
336
348
|
}
|
|
337
349
|
return value;
|
|
@@ -359,6 +371,116 @@ function hasValidResumableStagePair(stagePath, metadataPath) {
|
|
|
359
371
|
return false;
|
|
360
372
|
}
|
|
361
373
|
}
|
|
374
|
+
/** True when `pid` names a running process (EPERM: running, owned by another user). */
|
|
375
|
+
function processIsRunning(pid) {
|
|
376
|
+
try {
|
|
377
|
+
process.kill(pid, 0);
|
|
378
|
+
return true;
|
|
379
|
+
}
|
|
380
|
+
catch (error) {
|
|
381
|
+
return error.code === "EPERM";
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
/**
|
|
385
|
+
* A sidecar older than this no longer marks its writer as running, even when
|
|
386
|
+
* a process with that pid answers. The pid may have been reused (EPERM from
|
|
387
|
+
* another user's process counts as running), which would otherwise keep a
|
|
388
|
+
* dead writer's stage forever.
|
|
389
|
+
*/
|
|
390
|
+
export const RESUMABLE_STAGE_LIVE_WRITER_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000;
|
|
391
|
+
/**
|
|
392
|
+
* True when a different, still-running process published this stage and may
|
|
393
|
+
* be writing it. Such a stage is not resumed, deleted or renamed. A stage
|
|
394
|
+
* without a pid (older builds), from a process that has exited, or whose
|
|
395
|
+
* sidecar is older than {@link RESUMABLE_STAGE_LIVE_WRITER_MAX_AGE_MS} is not
|
|
396
|
+
* busy.
|
|
397
|
+
*/
|
|
398
|
+
export function resumableStageBusyElsewhere(metadata, selfPid = process.pid, isRunning = processIsRunning, sidecarMtimeMs, nowMs = Date.now()) {
|
|
399
|
+
if (sidecarMtimeMs !== undefined && nowMs - sidecarMtimeMs >= RESUMABLE_STAGE_LIVE_WRITER_MAX_AGE_MS) {
|
|
400
|
+
return false;
|
|
401
|
+
}
|
|
402
|
+
return metadata?.pid !== undefined && metadata.pid !== selfPid && isRunning(metadata.pid);
|
|
403
|
+
}
|
|
404
|
+
function sidecarMtime(metadataPath) {
|
|
405
|
+
try {
|
|
406
|
+
return fs.lstatSync(metadataPath).mtimeMs;
|
|
407
|
+
}
|
|
408
|
+
catch {
|
|
409
|
+
return undefined;
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
/**
|
|
413
|
+
* True when nothing names the writer of a stage that was touched recently: no
|
|
414
|
+
* readable sidecar, or a sidecar from an older build without a pid. Such a
|
|
415
|
+
* stage may belong to a sync process that is still writing it, so it is not
|
|
416
|
+
* resumed, claimed or deleted until it has gone untouched for the stale-file
|
|
417
|
+
* age. `mtimesMs` are the stage's and sidecar's mtimes, for the files present.
|
|
418
|
+
*/
|
|
419
|
+
export function resumableStageOwnerUnknownAndRecent(metadata, mtimesMs, nowMs = Date.now()) {
|
|
420
|
+
if (metadata && metadata.pid !== undefined)
|
|
421
|
+
return false;
|
|
422
|
+
const present = mtimesMs.filter((value) => typeof value === "number");
|
|
423
|
+
if (present.length === 0)
|
|
424
|
+
return false;
|
|
425
|
+
return nowMs - Math.max(...present) < STAGED_SWEEP_MAX_AGE_MS;
|
|
426
|
+
}
|
|
427
|
+
/**
|
|
428
|
+
* Fresh read of the shared stage: true when another running process owns it,
|
|
429
|
+
* or nothing names its writer and it was touched recently. Used right before
|
|
430
|
+
* deleting it, since state read before an awaited call can be stale.
|
|
431
|
+
*/
|
|
432
|
+
function sharedStageInUse(paths, nowMs = Date.now()) {
|
|
433
|
+
let stageMtimeMs;
|
|
434
|
+
let sidecarMtimeMs;
|
|
435
|
+
try {
|
|
436
|
+
stageMtimeMs = fs.lstatSync(paths.stagePath).mtimeMs;
|
|
437
|
+
}
|
|
438
|
+
catch {
|
|
439
|
+
stageMtimeMs = undefined;
|
|
440
|
+
}
|
|
441
|
+
const metadata = readResumableStageMetadata(paths.metadataPath);
|
|
442
|
+
try {
|
|
443
|
+
sidecarMtimeMs = fs.lstatSync(paths.metadataPath).mtimeMs;
|
|
444
|
+
}
|
|
445
|
+
catch {
|
|
446
|
+
sidecarMtimeMs = undefined;
|
|
447
|
+
}
|
|
448
|
+
return (resumableStageBusyElsewhere(metadata, process.pid, processIsRunning, sidecarMtimeMs, nowMs) ||
|
|
449
|
+
resumableStageOwnerUnknownAndRecent(metadata, [stageMtimeMs, sidecarMtimeMs], nowMs));
|
|
450
|
+
}
|
|
451
|
+
/**
|
|
452
|
+
* True when the shared stage path still names the file this process wrote
|
|
453
|
+
* (same inode as the write handle, where the file system reports one) and the
|
|
454
|
+
* sidecar still names this process. Checked right before installing the stage
|
|
455
|
+
* and removing its sidecar.
|
|
456
|
+
*/
|
|
457
|
+
export function sharedStageStillOurs(paths, writtenIno, selfPid = process.pid) {
|
|
458
|
+
if (readResumableStageMetadata(paths.metadataPath)?.pid !== selfPid)
|
|
459
|
+
return false;
|
|
460
|
+
let current;
|
|
461
|
+
try {
|
|
462
|
+
current = fs.lstatSync(paths.stagePath);
|
|
463
|
+
}
|
|
464
|
+
catch {
|
|
465
|
+
return false;
|
|
466
|
+
}
|
|
467
|
+
if (!current.isFile())
|
|
468
|
+
return false;
|
|
469
|
+
if (writtenIno && current.ino && writtenIno !== current.ino)
|
|
470
|
+
return false;
|
|
471
|
+
return true;
|
|
472
|
+
}
|
|
473
|
+
/**
|
|
474
|
+
* True when the file moved by a claim is the stage that was inspected: same
|
|
475
|
+
* size, and the same inode where the file system reports one.
|
|
476
|
+
*/
|
|
477
|
+
export function claimedStageMatches(inspected, moved) {
|
|
478
|
+
if (!moved || moved.size !== inspected.size)
|
|
479
|
+
return false;
|
|
480
|
+
if (inspected.ino && moved.ino && inspected.ino !== moved.ino)
|
|
481
|
+
return false;
|
|
482
|
+
return true;
|
|
483
|
+
}
|
|
362
484
|
export function sweepStaleStagedFiles(dir, maxAgeMs = STAGED_SWEEP_MAX_AGE_MS, nowMs = Date.now()) {
|
|
363
485
|
let entries;
|
|
364
486
|
try {
|
|
@@ -382,14 +504,29 @@ export function sweepStaleStagedFiles(dir, maxAgeMs = STAGED_SWEEP_MAX_AGE_MS, n
|
|
|
382
504
|
const stat = fs.lstatSync(full);
|
|
383
505
|
if (stat.isDirectory())
|
|
384
506
|
continue;
|
|
385
|
-
|
|
507
|
+
const ageMs = nowMs - stat.mtimeMs;
|
|
508
|
+
if (ageMs < maxAgeMs)
|
|
386
509
|
continue;
|
|
387
510
|
const stagePath = name.endsWith(".json")
|
|
388
511
|
? full.slice(0, -".json".length)
|
|
389
512
|
: full;
|
|
390
513
|
const metadataPath = `${stagePath}.json`;
|
|
391
|
-
|
|
514
|
+
// A stage whose writer is still running is kept whatever its age: the
|
|
515
|
+
// sidecar is written once, so its mtime says nothing about a long
|
|
516
|
+
// download. The writer is another live process, or a download of this
|
|
517
|
+
// stage still in flight in this process.
|
|
518
|
+
const sidecar = readResumableStageMetadata(metadataPath);
|
|
519
|
+
const writerLive = resumableStageBusyElsewhere(sidecar, process.pid, processIsRunning, sidecarMtime(metadataPath), nowMs) ||
|
|
520
|
+
(sidecar?.pid === process.pid && downloadStageLocks.has(stagePath));
|
|
521
|
+
if (writerLive)
|
|
522
|
+
continue;
|
|
523
|
+
// Keep a partial that can still be resumed for a bounded time. Past
|
|
524
|
+
// that it is abandoned: nothing resumes it unless the same key is
|
|
525
|
+
// downloaded again.
|
|
526
|
+
if (ageMs < RESUMABLE_STAGE_MAX_AGE_MS &&
|
|
527
|
+
hasValidResumableStagePair(stagePath, metadataPath)) {
|
|
392
528
|
continue;
|
|
529
|
+
}
|
|
393
530
|
fs.unlinkSync(full);
|
|
394
531
|
removed.push(full);
|
|
395
532
|
}
|
|
@@ -408,10 +545,12 @@ export function sweepStaleStagedFiles(dir, maxAgeMs = STAGED_SWEEP_MAX_AGE_MS, n
|
|
|
408
545
|
return removed;
|
|
409
546
|
}
|
|
410
547
|
function sweepStaleStagedFilesOnce(dir) {
|
|
411
|
-
|
|
548
|
+
const nowMs = Date.now();
|
|
549
|
+
const sweptAt = stagedDirSweptAt.get(dir);
|
|
550
|
+
if (sweptAt !== undefined && nowMs - sweptAt < STAGED_SWEEP_MAX_AGE_MS)
|
|
412
551
|
return;
|
|
413
|
-
|
|
414
|
-
sweepStaleStagedFiles(dir);
|
|
552
|
+
stagedDirSweptAt.set(dir, nowMs);
|
|
553
|
+
sweepStaleStagedFiles(dir, STAGED_SWEEP_MAX_AGE_MS, nowMs);
|
|
415
554
|
}
|
|
416
555
|
function downloadTempPath(localPath) {
|
|
417
556
|
const dir = path.dirname(localPath);
|
|
@@ -419,21 +558,174 @@ function downloadTempPath(localPath) {
|
|
|
419
558
|
const random = crypto.randomBytes(10).toString("hex");
|
|
420
559
|
return path.join(dir, `.hq-tmp-${random}`);
|
|
421
560
|
}
|
|
422
|
-
|
|
561
|
+
/**
|
|
562
|
+
* Publish the sidecar and so claim the shared stage path for this process.
|
|
563
|
+
* A hard link publishes the complete file and fails if a sidecar already
|
|
564
|
+
* exists, so two processes cannot both start writing the stage. Returns false
|
|
565
|
+
* when another process holds it; the caller then downloads into a private
|
|
566
|
+
* temp file. Any other link error falls back to creating the sidecar in
|
|
567
|
+
* place with an exclusive create: file systems without hard links report
|
|
568
|
+
* different codes (FAT32 and exFAT on Windows surface ERROR_INVALID_FUNCTION
|
|
569
|
+
* as EISDIR), and a download must not fail there. The exclusive create keeps
|
|
570
|
+
* two processes from both publishing; a reader that catches the file half
|
|
571
|
+
* written sees no readable sidecar on a recent stage and leaves it alone.
|
|
572
|
+
*/
|
|
573
|
+
export function publishResumableStageMetadata(paths, metadata, ops = { link: fs.linkSync }) {
|
|
574
|
+
const body = JSON.stringify(metadata);
|
|
423
575
|
const temporaryPath = `${paths.metadataPath}.${crypto.randomBytes(6).toString("hex")}.tmp`;
|
|
424
576
|
try {
|
|
425
|
-
fs.writeFileSync(temporaryPath,
|
|
426
|
-
|
|
577
|
+
fs.writeFileSync(temporaryPath, body, { flag: "wx" });
|
|
578
|
+
try {
|
|
579
|
+
ops.link(temporaryPath, paths.metadataPath);
|
|
580
|
+
}
|
|
581
|
+
catch (error) {
|
|
582
|
+
if (error.code === "EEXIST")
|
|
583
|
+
return false;
|
|
584
|
+
try {
|
|
585
|
+
fs.writeFileSync(paths.metadataPath, body, { flag: "wx" });
|
|
586
|
+
}
|
|
587
|
+
catch (createError) {
|
|
588
|
+
if (createError.code === "EEXIST")
|
|
589
|
+
return false;
|
|
590
|
+
throw createError;
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
return true;
|
|
427
594
|
}
|
|
428
|
-
|
|
595
|
+
finally {
|
|
429
596
|
removeTempPath(temporaryPath);
|
|
430
|
-
throw error;
|
|
431
597
|
}
|
|
432
598
|
}
|
|
599
|
+
/** True when the sidecar on disk names this process as the stage's writer. */
|
|
600
|
+
function sidecarNamesThisProcess(paths) {
|
|
601
|
+
return readResumableStageMetadata(paths.metadataPath)?.pid === process.pid;
|
|
602
|
+
}
|
|
603
|
+
/** Remove the sidecar only while it still names this process. */
|
|
604
|
+
function removeOwnSidecar(paths) {
|
|
605
|
+
if (sidecarNamesThisProcess(paths))
|
|
606
|
+
removeTempPath(paths.metadataPath);
|
|
607
|
+
}
|
|
608
|
+
/**
|
|
609
|
+
* Before this process resumes its own stage, set the sidecar's mtime to now.
|
|
610
|
+
* Other processes treat a sidecar older than
|
|
611
|
+
* {@link RESUMABLE_STAGE_LIVE_WRITER_MAX_AGE_MS} as naming no live writer, and
|
|
612
|
+
* appending to the stage does not touch the sidecar. Without this, a runner
|
|
613
|
+
* that kept resuming one partial for longer than that could have its stage
|
|
614
|
+
* claimed or deleted by another process while it was still writing it.
|
|
615
|
+
* Returns false when the sidecar no longer names this process.
|
|
616
|
+
*/
|
|
617
|
+
function refreshOwnSidecarAge(paths) {
|
|
618
|
+
if (!sidecarNamesThisProcess(paths))
|
|
619
|
+
return false;
|
|
620
|
+
try {
|
|
621
|
+
const now = new Date();
|
|
622
|
+
fs.utimesSync(paths.metadataPath, now, now);
|
|
623
|
+
}
|
|
624
|
+
catch {
|
|
625
|
+
// Still ours; the age check only applies once it is a week old.
|
|
626
|
+
}
|
|
627
|
+
return true;
|
|
628
|
+
}
|
|
629
|
+
/**
|
|
630
|
+
* Give a private copy of a partial back to the shared stage path, only if
|
|
631
|
+
* nothing has taken that path, and otherwise drop the private copy. A hard
|
|
632
|
+
* link refuses to replace an existing file; where links are unsupported the
|
|
633
|
+
* rename runs only while the path is free. Returns whether it was put back.
|
|
634
|
+
*/
|
|
635
|
+
export function putBackPrivateStage(privatePath, stagePath, ops = { link: fs.linkSync, rename: fs.renameSync }) {
|
|
636
|
+
try {
|
|
637
|
+
ops.link(privatePath, stagePath);
|
|
638
|
+
removeTempPath(privatePath);
|
|
639
|
+
return true;
|
|
640
|
+
}
|
|
641
|
+
catch (error) {
|
|
642
|
+
if (error.code !== "EEXIST" && !fs.existsSync(stagePath)) {
|
|
643
|
+
try {
|
|
644
|
+
ops.rename(privatePath, stagePath);
|
|
645
|
+
return true;
|
|
646
|
+
}
|
|
647
|
+
catch {
|
|
648
|
+
// Fall through: the private copy is dropped below.
|
|
649
|
+
}
|
|
650
|
+
}
|
|
651
|
+
}
|
|
652
|
+
removeTempPath(privatePath);
|
|
653
|
+
return false;
|
|
654
|
+
}
|
|
655
|
+
/**
|
|
656
|
+
* After a failed resume of a partial this process took over, hand the
|
|
657
|
+
* partial back as the shared stage with a sidecar naming this process, so the
|
|
658
|
+
* next attempt continues from where this one stopped instead of byte 0. The
|
|
659
|
+
* sidecar is published first, as a new writer would. When another process
|
|
660
|
+
* holds the sidecar or the stage path, the private copy is dropped.
|
|
661
|
+
*/
|
|
662
|
+
function returnClaimedStage(paths, privatePath, saved) {
|
|
663
|
+
let size;
|
|
664
|
+
try {
|
|
665
|
+
size = fs.lstatSync(privatePath).size;
|
|
666
|
+
}
|
|
667
|
+
catch {
|
|
668
|
+
return;
|
|
669
|
+
}
|
|
670
|
+
if (size <= 0 || size >= saved.expectedSize) {
|
|
671
|
+
removeTempPath(privatePath);
|
|
672
|
+
return;
|
|
673
|
+
}
|
|
674
|
+
if (!publishResumableStageMetadata(paths, { ...saved, pid: process.pid })) {
|
|
675
|
+
removeTempPath(privatePath);
|
|
676
|
+
return;
|
|
677
|
+
}
|
|
678
|
+
if (!putBackPrivateStage(privatePath, paths.stagePath))
|
|
679
|
+
removeOwnSidecar(paths);
|
|
680
|
+
}
|
|
433
681
|
function removeResumableStage(paths) {
|
|
434
682
|
removeTempPath(paths.stagePath);
|
|
435
683
|
removeTempPath(paths.metadataPath);
|
|
436
684
|
}
|
|
685
|
+
/**
|
|
686
|
+
* Take over a partial left by an earlier process before resuming it. The
|
|
687
|
+
* stage is renamed to a private temp name, which only one process can do, so
|
|
688
|
+
* two processes never append to the same file. The sidecar is removed only
|
|
689
|
+
* if it is still the one that was read. Returns the private path, or
|
|
690
|
+
* undefined when another process took the stage first or the file moved is
|
|
691
|
+
* not the one that was inspected (a different file, or it grew because a
|
|
692
|
+
* writer is still appending).
|
|
693
|
+
*/
|
|
694
|
+
function claimResumableStage(paths, saved, localPath, inspected) {
|
|
695
|
+
const privatePath = downloadTempPath(localPath);
|
|
696
|
+
try {
|
|
697
|
+
fs.renameSync(paths.stagePath, privatePath);
|
|
698
|
+
}
|
|
699
|
+
catch {
|
|
700
|
+
return undefined;
|
|
701
|
+
}
|
|
702
|
+
let moved;
|
|
703
|
+
try {
|
|
704
|
+
moved = fs.lstatSync(privatePath);
|
|
705
|
+
}
|
|
706
|
+
catch {
|
|
707
|
+
moved = undefined;
|
|
708
|
+
}
|
|
709
|
+
if (!claimedStageMatches(inspected, moved)) {
|
|
710
|
+
// Put it back only if nothing has taken its place, then let go of it.
|
|
711
|
+
putBackPrivateStage(privatePath, paths.stagePath);
|
|
712
|
+
return undefined;
|
|
713
|
+
}
|
|
714
|
+
try {
|
|
715
|
+
// A rename keeps the old mtime; refresh it so the age sweep in another
|
|
716
|
+
// process does not treat the claimed partial as an orphan.
|
|
717
|
+
const now = new Date();
|
|
718
|
+
fs.utimesSync(privatePath, now, now);
|
|
719
|
+
}
|
|
720
|
+
catch {
|
|
721
|
+
// The size check after the append still rejects a stage that went missing.
|
|
722
|
+
}
|
|
723
|
+
const current = readResumableStageMetadata(paths.metadataPath);
|
|
724
|
+
if (current && current.pid === saved.pid && current.etag === saved.etag) {
|
|
725
|
+
removeTempPath(paths.metadataPath);
|
|
726
|
+
}
|
|
727
|
+
return privatePath;
|
|
728
|
+
}
|
|
437
729
|
/** A same-directory, sync-ignored target for a complete staged download. */
|
|
438
730
|
export function createDownloadStagingPath(localPath) {
|
|
439
731
|
const random = crypto.randomBytes(12).toString("hex");
|
|
@@ -876,14 +1168,31 @@ async function streamRegularFileToTemp(tempPath, initialChunks, iterator, append
|
|
|
876
1168
|
const hashing = new HashingTransform();
|
|
877
1169
|
let chunksToStream = initialChunks;
|
|
878
1170
|
let appendToTemp = append;
|
|
1171
|
+
let createdIno;
|
|
879
1172
|
if (preserveInitialChunks) {
|
|
880
|
-
writeInitialChunksToStage(tempPath, initialChunks);
|
|
1173
|
+
createdIno = writeInitialChunksToStage(tempPath, initialChunks);
|
|
881
1174
|
hashing.seed(initialChunks);
|
|
882
1175
|
chunksToStream = [];
|
|
883
1176
|
appendToTemp = true;
|
|
884
1177
|
}
|
|
885
|
-
|
|
886
|
-
|
|
1178
|
+
const output = fs.createWriteStream(tempPath, { flags: appendToTemp ? "a" : "wx" });
|
|
1179
|
+
// The file the bytes went into, read from the write handle itself, so the
|
|
1180
|
+
// caller can tell whether the path still names that file before installing.
|
|
1181
|
+
let ino;
|
|
1182
|
+
output.once("open", (fd) => {
|
|
1183
|
+
try {
|
|
1184
|
+
ino = fs.fstatSync(fd).ino;
|
|
1185
|
+
}
|
|
1186
|
+
catch {
|
|
1187
|
+
ino = undefined;
|
|
1188
|
+
}
|
|
1189
|
+
});
|
|
1190
|
+
await pipeline(Readable.from(regularDownloadChunks(chunksToStream, iterator)), hashing, output);
|
|
1191
|
+
// The file this call created exclusively wins over the one the append
|
|
1192
|
+
// reopened: if the path was swapped in between, the inode check before the
|
|
1193
|
+
// install then sees the swap.
|
|
1194
|
+
const writtenIno = createdIno ?? ino;
|
|
1195
|
+
return { hash: hashing.digest(), size: hashing.size, ...(writtenIno !== undefined ? { ino: writtenIno } : {}) };
|
|
887
1196
|
}
|
|
888
1197
|
/**
|
|
889
1198
|
* Cheap script-candidate check for a staged download: script extension, or
|
|
@@ -908,6 +1217,27 @@ function stagedFileLooksLikeScript(key, filePath) {
|
|
|
908
1217
|
fs.closeSync(fd);
|
|
909
1218
|
}
|
|
910
1219
|
}
|
|
1220
|
+
/**
|
|
1221
|
+
* Close a GET response that will not be read to the end. Returning the
|
|
1222
|
+
* iterator runs the body wrappers' cleanup (bandwidth slot, idle timer) and
|
|
1223
|
+
* ends the stream without an error; destroy() then closes the response itself
|
|
1224
|
+
* where a wrapper does not pass the return through. Neither injects an error,
|
|
1225
|
+
* since nothing may be listening for one on the body.
|
|
1226
|
+
*/
|
|
1227
|
+
function closeUnreadResponse(response, iterator) {
|
|
1228
|
+
try {
|
|
1229
|
+
void Promise.resolve(iterator.return?.()).catch(() => undefined);
|
|
1230
|
+
}
|
|
1231
|
+
catch {
|
|
1232
|
+
// Best effort: destroy() below still closes the response.
|
|
1233
|
+
}
|
|
1234
|
+
try {
|
|
1235
|
+
response.destroy?.();
|
|
1236
|
+
}
|
|
1237
|
+
catch {
|
|
1238
|
+
// Best effort: the original error is what the caller reports.
|
|
1239
|
+
}
|
|
1240
|
+
}
|
|
911
1241
|
async function hashStagedFile(filePath) {
|
|
912
1242
|
const hash = crypto.createHash("sha256");
|
|
913
1243
|
let size = 0;
|
|
@@ -918,15 +1248,24 @@ async function hashStagedFile(filePath) {
|
|
|
918
1248
|
}
|
|
919
1249
|
return { hash: hash.digest("hex"), size };
|
|
920
1250
|
}
|
|
1251
|
+
/** Create the stage exclusively and write the first chunks; returns its inode. */
|
|
921
1252
|
function writeInitialChunksToStage(filePath, chunks) {
|
|
922
1253
|
const descriptor = fs.openSync(filePath, "wx");
|
|
923
1254
|
try {
|
|
1255
|
+
let ino;
|
|
1256
|
+
try {
|
|
1257
|
+
ino = fs.fstatSync(descriptor).ino;
|
|
1258
|
+
}
|
|
1259
|
+
catch {
|
|
1260
|
+
ino = undefined;
|
|
1261
|
+
}
|
|
924
1262
|
for (const chunk of chunks) {
|
|
925
1263
|
let offset = 0;
|
|
926
1264
|
while (offset < chunk.length) {
|
|
927
1265
|
offset += fs.writeSync(descriptor, chunk, offset, chunk.length - offset);
|
|
928
1266
|
}
|
|
929
1267
|
}
|
|
1268
|
+
return ino;
|
|
930
1269
|
}
|
|
931
1270
|
finally {
|
|
932
1271
|
fs.closeSync(descriptor);
|
|
@@ -1644,8 +1983,17 @@ async function downloadFileAttempt(io, key, localPath, options, allowResume = tr
|
|
|
1644
1983
|
let resumeOffset = 0;
|
|
1645
1984
|
let isResuming = false;
|
|
1646
1985
|
let streamResult;
|
|
1986
|
+
// Where a resumed download appends. The shared stage path when this process
|
|
1987
|
+
// wrote the partial; a private temp after taking over another process's.
|
|
1988
|
+
let resumeStagePath = resumePaths.stagePath;
|
|
1989
|
+
let resumeStageShared = true;
|
|
1990
|
+
// feedback_afdc2971: another running sync process owns the shared stage.
|
|
1991
|
+
// Leave its files alone and download this key into a private temp file.
|
|
1992
|
+
let stageBusy = false;
|
|
1647
1993
|
if (allowResume) {
|
|
1648
|
-
|
|
1994
|
+
// Inspect the stage before its sidecar. A writer publishes the sidecar
|
|
1995
|
+
// first and creates the stage second, so a stage seen here with no
|
|
1996
|
+
// sidecar a moment later was not just started by another process.
|
|
1649
1997
|
let stageStat;
|
|
1650
1998
|
try {
|
|
1651
1999
|
stageStat = fs.lstatSync(resumePaths.stagePath);
|
|
@@ -1653,7 +2001,40 @@ async function downloadFileAttempt(io, key, localPath, options, allowResume = tr
|
|
|
1653
2001
|
catch {
|
|
1654
2002
|
// No stage to resume.
|
|
1655
2003
|
}
|
|
1656
|
-
|
|
2004
|
+
const saved = readResumableStageMetadata(resumePaths.metadataPath);
|
|
2005
|
+
let sidecarMtimeMs;
|
|
2006
|
+
try {
|
|
2007
|
+
sidecarMtimeMs = fs.lstatSync(resumePaths.metadataPath).mtimeMs;
|
|
2008
|
+
}
|
|
2009
|
+
catch {
|
|
2010
|
+
// No sidecar.
|
|
2011
|
+
}
|
|
2012
|
+
// Another running process owns the stage, or nothing names its writer and
|
|
2013
|
+
// it was touched recently (an older build may still be writing it).
|
|
2014
|
+
stageBusy =
|
|
2015
|
+
resumableStageBusyElsewhere(saved, process.pid, processIsRunning, sidecarMtimeMs) ||
|
|
2016
|
+
resumableStageOwnerUnknownAndRecent(saved, [stageStat?.mtimeMs, sidecarMtimeMs]);
|
|
2017
|
+
// Shared-stage removals below run after an awaited network call, so the
|
|
2018
|
+
// state read above may be stale: another runner may have taken the stage
|
|
2019
|
+
// and started writing it. Re-check right before deleting; when it is now
|
|
2020
|
+
// in use, leave it and download privately.
|
|
2021
|
+
const removeSharedStageIfStillStale = () => {
|
|
2022
|
+
if (sharedStageInUse(resumePaths)) {
|
|
2023
|
+
stageBusy = true;
|
|
2024
|
+
return;
|
|
2025
|
+
}
|
|
2026
|
+
removeResumableStage(resumePaths);
|
|
2027
|
+
};
|
|
2028
|
+
const dropResumeStage = () => {
|
|
2029
|
+
if (resumeStageShared)
|
|
2030
|
+
removeSharedStageIfStillStale();
|
|
2031
|
+
else
|
|
2032
|
+
removeTempPath(resumeStagePath);
|
|
2033
|
+
};
|
|
2034
|
+
if (stageBusy) {
|
|
2035
|
+
// Nothing to do here; the other process finishes or abandons its stage.
|
|
2036
|
+
}
|
|
2037
|
+
else if (saved &&
|
|
1657
2038
|
saved.keyHash === resumePaths.keyHash &&
|
|
1658
2039
|
stageStat?.isFile() &&
|
|
1659
2040
|
!stageStat.isSymbolicLink() &&
|
|
@@ -1661,44 +2042,64 @@ async function downloadFileAttempt(io, key, localPath, options, allowResume = tr
|
|
|
1661
2042
|
stageStat.size < saved.expectedSize) {
|
|
1662
2043
|
const head = await io.headObject(key);
|
|
1663
2044
|
if (stageMatchesRemote(saved, head, resumePaths)) {
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
});
|
|
1674
|
-
}
|
|
1675
|
-
catch (error) {
|
|
1676
|
-
if (!isRangePreconditionError(error))
|
|
1677
|
-
throw error;
|
|
1678
|
-
removeResumableStage(resumePaths);
|
|
1679
|
-
resumeMetadata = undefined;
|
|
1680
|
-
resumeOffset = 0;
|
|
2045
|
+
if (saved.pid !== process.pid) {
|
|
2046
|
+
const claimed = claimResumableStage(resumePaths, saved, localPath, stageStat);
|
|
2047
|
+
if (claimed) {
|
|
2048
|
+
resumeStagePath = claimed;
|
|
2049
|
+
resumeStageShared = false;
|
|
2050
|
+
}
|
|
2051
|
+
else {
|
|
2052
|
+
stageBusy = true;
|
|
2053
|
+
}
|
|
1681
2054
|
}
|
|
1682
|
-
if (
|
|
1683
|
-
|
|
2055
|
+
else if (!refreshOwnSidecarAge(resumePaths)) {
|
|
2056
|
+
// Another process took the stage while the remote check ran.
|
|
2057
|
+
stageBusy = true;
|
|
1684
2058
|
}
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
2059
|
+
if (!stageBusy) {
|
|
2060
|
+
resumeOffset = stageStat.size;
|
|
2061
|
+
resumeMetadata = saved;
|
|
2062
|
+
try {
|
|
2063
|
+
streamResult = await getObjectStream(io, key, {
|
|
2064
|
+
...(options.bypassBandwidthGovernor
|
|
2065
|
+
? { bypassBandwidthGovernor: true }
|
|
2066
|
+
: {}),
|
|
2067
|
+
rangeStart: resumeOffset,
|
|
2068
|
+
ifMatch: saved.etag,
|
|
2069
|
+
});
|
|
2070
|
+
}
|
|
2071
|
+
catch (error) {
|
|
2072
|
+
if (!isRangePreconditionError(error)) {
|
|
2073
|
+
// Keep the taken-over partial for the next attempt.
|
|
2074
|
+
if (!resumeStageShared)
|
|
2075
|
+
returnClaimedStage(resumePaths, resumeStagePath, saved);
|
|
2076
|
+
throw error;
|
|
2077
|
+
}
|
|
2078
|
+
dropResumeStage();
|
|
2079
|
+
resumeMetadata = undefined;
|
|
2080
|
+
resumeOffset = 0;
|
|
2081
|
+
}
|
|
2082
|
+
if (streamResult && rangedResponseMatchesStage(streamResult, saved, resumeOffset)) {
|
|
2083
|
+
isResuming = true;
|
|
2084
|
+
}
|
|
2085
|
+
else if (streamResult) {
|
|
2086
|
+
// Close the rejected response without an error: nothing is listening
|
|
2087
|
+
// on its body yet, so an injected error would surface as an
|
|
2088
|
+
// unhandled stream `error` and could terminate the runner.
|
|
2089
|
+
streamResult.destroy?.();
|
|
2090
|
+
dropResumeStage();
|
|
2091
|
+
streamResult = undefined;
|
|
2092
|
+
resumeMetadata = undefined;
|
|
2093
|
+
resumeOffset = 0;
|
|
2094
|
+
}
|
|
1694
2095
|
}
|
|
1695
2096
|
}
|
|
1696
2097
|
else {
|
|
1697
|
-
|
|
2098
|
+
removeSharedStageIfStillStale();
|
|
1698
2099
|
}
|
|
1699
2100
|
}
|
|
1700
2101
|
else if (saved || stageStat) {
|
|
1701
|
-
|
|
2102
|
+
removeSharedStageIfStillStale();
|
|
1702
2103
|
}
|
|
1703
2104
|
}
|
|
1704
2105
|
streamResult ??= await getObjectStream(io, key, {
|
|
@@ -1863,27 +2264,38 @@ async function downloadFileAttempt(io, key, localPath, options, allowResume = tr
|
|
|
1863
2264
|
let tempPath;
|
|
1864
2265
|
let keepResumableStage = false;
|
|
1865
2266
|
if (isResuming && resumeMetadata) {
|
|
1866
|
-
tempPath =
|
|
1867
|
-
keepResumableStage =
|
|
2267
|
+
tempPath = resumeStagePath;
|
|
2268
|
+
keepResumableStage = resumeStageShared;
|
|
1868
2269
|
}
|
|
1869
2270
|
else {
|
|
1870
2271
|
tempPath = downloadTempPath(localPath);
|
|
1871
2272
|
const responseSize = streamResult.contentLength;
|
|
1872
|
-
if (
|
|
2273
|
+
if (!stageBusy &&
|
|
2274
|
+
streamResult.etag &&
|
|
1873
2275
|
responseSize !== undefined &&
|
|
1874
2276
|
Number.isSafeInteger(responseSize) &&
|
|
1875
2277
|
responseSize > initialLength &&
|
|
1876
2278
|
!isSymlinkRecord) {
|
|
1877
|
-
|
|
2279
|
+
const candidate = {
|
|
1878
2280
|
version: 1,
|
|
1879
2281
|
keyHash: resumePaths.keyHash,
|
|
1880
2282
|
etag: streamResult.etag,
|
|
1881
2283
|
...(streamResult.versionId ? { versionId: streamResult.versionId } : {}),
|
|
1882
2284
|
expectedSize: responseSize,
|
|
2285
|
+
pid: process.pid,
|
|
1883
2286
|
};
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
|
|
2287
|
+
if (publishResumableStageMetadata(resumePaths, candidate)) {
|
|
2288
|
+
if (fs.existsSync(resumePaths.stagePath)) {
|
|
2289
|
+
// A stage with no sidecar appeared (an older build, or a stage left
|
|
2290
|
+
// in place by another process). Do not write over or delete it.
|
|
2291
|
+
removeOwnSidecar(resumePaths);
|
|
2292
|
+
}
|
|
2293
|
+
else {
|
|
2294
|
+
resumeMetadata = candidate;
|
|
2295
|
+
tempPath = resumePaths.stagePath;
|
|
2296
|
+
keepResumableStage = true;
|
|
2297
|
+
}
|
|
2298
|
+
}
|
|
1887
2299
|
}
|
|
1888
2300
|
}
|
|
1889
2301
|
// A legacy object has no hq-mode. If it is replacing a regular local file,
|
|
@@ -1900,20 +2312,48 @@ async function downloadFileAttempt(io, key, localPath, options, allowResume = tr
|
|
|
1900
2312
|
// Fresh destination / inaccessible prior path: no local fallback.
|
|
1901
2313
|
}
|
|
1902
2314
|
let tempReady = true;
|
|
2315
|
+
// Set when the shared stage path no longer names this process's file, so
|
|
2316
|
+
// the cleanup below must not touch what is there now.
|
|
2317
|
+
let sharedStageLost = false;
|
|
2318
|
+
let stageHoldsFullObject = false;
|
|
2319
|
+
let writtenIno;
|
|
1903
2320
|
let streamed;
|
|
1904
2321
|
const modeWarnings = [];
|
|
1905
2322
|
try {
|
|
1906
2323
|
if (resumeMetadata && isResuming) {
|
|
1907
|
-
await streamRegularFileToTemp(tempPath, [], iterator, true);
|
|
2324
|
+
writtenIno = (await streamRegularFileToTemp(tempPath, [], iterator, true)).ino;
|
|
1908
2325
|
streamed = await hashStagedFile(tempPath);
|
|
1909
2326
|
}
|
|
1910
2327
|
else {
|
|
1911
|
-
|
|
2328
|
+
try {
|
|
2329
|
+
streamed = await streamRegularFileToTemp(tempPath, initialChunks, iterator, false, resumeMetadata !== undefined);
|
|
2330
|
+
}
|
|
2331
|
+
catch (error) {
|
|
2332
|
+
// The write can fail before the body is read (the exclusive create of
|
|
2333
|
+
// the stage), which leaves the response open. Close it so the retry
|
|
2334
|
+
// does not open a second stream while this one still holds a socket.
|
|
2335
|
+
closeUnreadResponse(streamResult, iterator);
|
|
2336
|
+
if (keepResumableStage && error.code === "EEXIST") {
|
|
2337
|
+
// Another process created the shared stage between the check after
|
|
2338
|
+
// publishing the sidecar and the exclusive create. Withdraw this
|
|
2339
|
+
// process's sidecar so it is not paired with that file, and retry,
|
|
2340
|
+
// which downloads privately.
|
|
2341
|
+
sharedStageLost = true;
|
|
2342
|
+
removeOwnSidecar(resumePaths);
|
|
2343
|
+
throw Object.assign(new Error(`download ${key}: the shared download stage was taken by another sync process`), { code: "ERR_HQ_STAGE_TAKEN" });
|
|
2344
|
+
}
|
|
2345
|
+
throw error;
|
|
2346
|
+
}
|
|
2347
|
+
writtenIno = streamed.ino;
|
|
1912
2348
|
}
|
|
1913
2349
|
if (resumeMetadata &&
|
|
1914
2350
|
streamed.size !== resumeMetadata.expectedSize) {
|
|
1915
2351
|
throw Object.assign(new Error(`download ${key} size mismatch: expected ${resumeMetadata.expectedSize} bytes, received ${streamed.size}`), { code: "ERR_HQ_TRUNCATED_RESPONSE" });
|
|
1916
2352
|
}
|
|
2353
|
+
// From here the stage holds the whole object, possibly rewritten below.
|
|
2354
|
+
// Its length no longer says how much of the remote object it holds, so
|
|
2355
|
+
// it must never be resumed: a failure from here deletes it.
|
|
2356
|
+
stageHoldsFullObject = true;
|
|
1917
2357
|
// Shell scripts land as LF on POSIX receivers (script-eol): an object
|
|
1918
2358
|
// pushed with CRLF by an older or Windows client would otherwise fail
|
|
1919
2359
|
// under bash. The staged file is rewritten in place and re-hashed so the
|
|
@@ -2036,14 +2476,55 @@ async function downloadFileAttempt(io, key, localPath, options, allowResume = tr
|
|
|
2036
2476
|
}
|
|
2037
2477
|
}
|
|
2038
2478
|
options.beforeReplace?.();
|
|
2479
|
+
if (keepResumableStage && !sharedStageStillOurs(resumePaths, writtenIno)) {
|
|
2480
|
+
// Another runner replaced the shared stage or its sidecar while this
|
|
2481
|
+
// one was writing. Installing the path now could put that runner's
|
|
2482
|
+
// partial at localPath under this download's hash. Leave its files
|
|
2483
|
+
// alone; the retry downloads privately. Withdraw this process's sidecar
|
|
2484
|
+
// if it is still there (only the inode changed): otherwise the retry
|
|
2485
|
+
// would read it as its own stage and append to the file now at the path.
|
|
2486
|
+
sharedStageLost = true;
|
|
2487
|
+
removeOwnSidecar(resumePaths);
|
|
2488
|
+
throw Object.assign(new Error(`download ${key}: the shared download stage was taken by another sync process`), { code: "ERR_HQ_STAGE_TAKEN" });
|
|
2489
|
+
}
|
|
2039
2490
|
replaceStagedPath(tempPath, localPath);
|
|
2040
2491
|
tempReady = false;
|
|
2041
2492
|
if (keepResumableStage)
|
|
2042
|
-
|
|
2493
|
+
removeOwnSidecar(resumePaths);
|
|
2043
2494
|
}
|
|
2044
2495
|
finally {
|
|
2045
|
-
if (
|
|
2496
|
+
if (sharedStageLost) {
|
|
2497
|
+
// Not ours any more: nothing to clean up.
|
|
2498
|
+
}
|
|
2499
|
+
else if (tempReady && stageHoldsFullObject) {
|
|
2500
|
+
// A complete (possibly script-rewritten) stage is never resumed.
|
|
2501
|
+
if (!keepResumableStage)
|
|
2502
|
+
removeTempPath(tempPath);
|
|
2503
|
+
else if (sharedStageStillOurs(resumePaths, writtenIno))
|
|
2504
|
+
removeResumableStage(resumePaths);
|
|
2505
|
+
// Only the inode changed: withdraw this process's sidecar so the next
|
|
2506
|
+
// attempt does not append to the file now at the stage path.
|
|
2507
|
+
else
|
|
2508
|
+
removeOwnSidecar(resumePaths);
|
|
2509
|
+
}
|
|
2510
|
+
else if (tempReady && !keepResumableStage && isResuming && !resumeStageShared && resumeMetadata) {
|
|
2511
|
+
// feedback_afdc2971 review: keep a taken-over partial for the next
|
|
2512
|
+
// attempt instead of restarting the object from byte 0.
|
|
2513
|
+
returnClaimedStage(resumePaths, tempPath, resumeMetadata);
|
|
2514
|
+
}
|
|
2515
|
+
else if (tempReady && !keepResumableStage) {
|
|
2046
2516
|
removeTempPath(tempPath);
|
|
2517
|
+
}
|
|
2518
|
+
else if (tempReady &&
|
|
2519
|
+
sidecarNamesThisProcess(resumePaths) &&
|
|
2520
|
+
!hasValidResumableStagePair(resumePaths.stagePath, resumePaths.metadataPath)) {
|
|
2521
|
+
// feedback_4912b4fa: only a partial shorter than the object can be
|
|
2522
|
+
// resumed. A complete stage whose replace step failed (the local file
|
|
2523
|
+
// changed, another process installed it first, or Windows refused the
|
|
2524
|
+
// rename) would otherwise stay next to the file it duplicates, because
|
|
2525
|
+
// nothing downloads that key again.
|
|
2526
|
+
removeResumableStage(resumePaths);
|
|
2527
|
+
}
|
|
2047
2528
|
}
|
|
2048
2529
|
// TODO: stamp hq-btime once Node lands lbirthtime (or birthtimeSync).
|
|
2049
2530
|
// The push side already emits hq-btime when the source FS tracks a
|