@indigoai-us/hq-cloud 6.18.3 → 6.18.4-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bandwidth.d.ts +56 -10
- package/dist/bandwidth.d.ts.map +1 -1
- package/dist/bandwidth.js +253 -17
- package/dist/bandwidth.js.map +1 -1
- package/dist/bandwidth.test.js +103 -4
- package/dist/bandwidth.test.js.map +1 -1
- package/dist/bin/sync-runner-company.d.ts +2 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +15 -0
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-company.test.js +56 -0
- package/dist/bin/sync-runner-company.test.js.map +1 -1
- package/dist/bin/sync-runner-events.test.js +15 -0
- package/dist/bin/sync-runner-events.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +31 -5
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +228 -10
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +31 -2
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +14 -5
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +120 -39
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/share.d.ts +26 -8
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +227 -62
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +252 -3
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts +21 -4
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +140 -19
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +274 -1
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/lib/machine-id.d.ts +7 -0
- package/dist/lib/machine-id.d.ts.map +1 -1
- package/dist/lib/machine-id.js +54 -7
- package/dist/lib/machine-id.js.map +1 -1
- package/dist/lib/machine-id.test.js +20 -1
- package/dist/lib/machine-id.test.js.map +1 -1
- package/dist/lib/ontology-entity-push.d.ts +51 -0
- package/dist/lib/ontology-entity-push.d.ts.map +1 -0
- package/dist/lib/ontology-entity-push.js +125 -0
- package/dist/lib/ontology-entity-push.js.map +1 -0
- package/dist/lib/ontology-entity-push.test.d.ts +2 -0
- package/dist/lib/ontology-entity-push.test.d.ts.map +1 -0
- package/dist/lib/ontology-entity-push.test.js +149 -0
- package/dist/lib/ontology-entity-push.test.js.map +1 -0
- package/dist/object-io.d.ts +38 -7
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +336 -140
- package/dist/object-io.js.map +1 -1
- package/dist/object-io.test.js +423 -1
- package/dist/object-io.test.js.map +1 -1
- package/dist/s3-download-stream-hash.test.d.ts +2 -0
- package/dist/s3-download-stream-hash.test.d.ts.map +1 -0
- package/dist/s3-download-stream-hash.test.js +70 -0
- package/dist/s3-download-stream-hash.test.js.map +1 -0
- package/dist/s3.d.ts +8 -3
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +330 -27
- package/dist/s3.js.map +1 -1
- package/dist/s3.test.js +304 -0
- package/dist/s3.test.js.map +1 -1
- package/dist/sync/event-sync.d.ts +14 -4
- package/dist/sync/event-sync.d.ts.map +1 -1
- package/dist/sync/event-sync.js +145 -14
- package/dist/sync/event-sync.js.map +1 -1
- package/dist/sync/event-sync.test.js +266 -1
- package/dist/sync/event-sync.test.js.map +1 -1
- package/dist/sync/inbound-event-admission.d.ts.map +1 -1
- package/dist/sync/inbound-event-admission.js +3 -0
- package/dist/sync/inbound-event-admission.js.map +1 -1
- package/dist/sync/inbound-event-admission.test.js +6 -0
- package/dist/sync/inbound-event-admission.test.js.map +1 -1
- package/dist/sync/push-transport.d.ts +15 -0
- package/dist/sync/push-transport.d.ts.map +1 -1
- package/dist/sync/push-transport.js +27 -0
- package/dist/sync/push-transport.js.map +1 -1
- package/dist/sync/push-transport.test.js +14 -1
- package/dist/sync/push-transport.test.js.map +1 -1
- package/dist/sync-core.d.ts +2 -1
- package/dist/sync-core.d.ts.map +1 -1
- package/dist/sync-core.js +5 -10
- package/dist/sync-core.js.map +1 -1
- package/dist/sync-core.test.js +16 -0
- package/dist/sync-core.test.js.map +1 -1
- package/dist/vault-client.d.ts +5 -0
- package/dist/vault-client.d.ts.map +1 -1
- package/dist/vault-client.js +7 -0
- package/dist/vault-client.js.map +1 -1
- package/dist/vault-client.test.js +39 -0
- package/dist/vault-client.test.js.map +1 -1
- package/package.json +1 -1
package/dist/s3.js
CHANGED
|
@@ -42,7 +42,22 @@ function isRetryableDownloadStreamError(err) {
|
|
|
42
42
|
if (/\bEPIPE\b/.test(msg))
|
|
43
43
|
return true;
|
|
44
44
|
const code = err.code;
|
|
45
|
-
return code === "EPIPE" ||
|
|
45
|
+
return (code === "EPIPE" ||
|
|
46
|
+
code === "ECONNRESET" ||
|
|
47
|
+
code === "ERR_HQ_TRUNCATED_RESPONSE");
|
|
48
|
+
}
|
|
49
|
+
function isRangePreconditionError(err) {
|
|
50
|
+
if (!err || typeof err !== "object")
|
|
51
|
+
return false;
|
|
52
|
+
const value = err;
|
|
53
|
+
return (value.name === "PreconditionFailed" ||
|
|
54
|
+
value.status === 412 ||
|
|
55
|
+
value.statusCode === 412 ||
|
|
56
|
+
value.statusCode === 416 ||
|
|
57
|
+
value.$metadata?.httpStatusCode === 412 ||
|
|
58
|
+
value.$metadata?.httpStatusCode === 416 ||
|
|
59
|
+
value.$response?.statusCode === 412 ||
|
|
60
|
+
value.$response?.statusCode === 416);
|
|
46
61
|
}
|
|
47
62
|
/**
|
|
48
63
|
* S3 user metadata is ASCII-only (lowercased on read, capped at 2 KB total).
|
|
@@ -287,6 +302,52 @@ export function encodeSymlinkBody(target) {
|
|
|
287
302
|
const STAGED_SENTINEL_PATTERN = /^\.hq-(tmp|backup)-/;
|
|
288
303
|
const STAGED_SWEEP_MAX_AGE_MS = 60 * 60 * 1000;
|
|
289
304
|
const sweptStagedDirs = new Set();
|
|
305
|
+
function resumableStagePaths(dir, key) {
|
|
306
|
+
const keyHash = crypto.createHash("sha256").update(key).digest("hex");
|
|
307
|
+
const stagePath = path.join(dir, `.hq-tmp-resume-${keyHash}`);
|
|
308
|
+
return { stagePath, metadataPath: `${stagePath}.json`, keyHash };
|
|
309
|
+
}
|
|
310
|
+
function readResumableStageMetadata(metadataPath) {
|
|
311
|
+
try {
|
|
312
|
+
const parsed = JSON.parse(fs.readFileSync(metadataPath, "utf8"));
|
|
313
|
+
if (!parsed || typeof parsed !== "object")
|
|
314
|
+
return null;
|
|
315
|
+
const value = parsed;
|
|
316
|
+
if (value.version !== 1 ||
|
|
317
|
+
typeof value.keyHash !== "string" ||
|
|
318
|
+
!/^[a-f0-9]{64}$/.test(value.keyHash) ||
|
|
319
|
+
typeof value.etag !== "string" ||
|
|
320
|
+
value.etag.length === 0 ||
|
|
321
|
+
!Number.isSafeInteger(value.expectedSize) ||
|
|
322
|
+
(value.expectedSize ?? 0) <= 0 ||
|
|
323
|
+
(value.versionId !== undefined && typeof value.versionId !== "string")) {
|
|
324
|
+
return null;
|
|
325
|
+
}
|
|
326
|
+
return value;
|
|
327
|
+
}
|
|
328
|
+
catch {
|
|
329
|
+
return null;
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
function hasValidResumableStagePair(stagePath, metadataPath) {
|
|
333
|
+
const metadata = readResumableStageMetadata(metadataPath);
|
|
334
|
+
if (!metadata)
|
|
335
|
+
return false;
|
|
336
|
+
try {
|
|
337
|
+
const stage = fs.lstatSync(stagePath);
|
|
338
|
+
const meta = fs.lstatSync(metadataPath);
|
|
339
|
+
return (stage.isFile() &&
|
|
340
|
+
!stage.isSymbolicLink() &&
|
|
341
|
+
stage.size > 0 &&
|
|
342
|
+
stage.size < metadata.expectedSize &&
|
|
343
|
+
path.basename(stagePath).endsWith(metadata.keyHash) &&
|
|
344
|
+
meta.isFile() &&
|
|
345
|
+
!meta.isSymbolicLink());
|
|
346
|
+
}
|
|
347
|
+
catch {
|
|
348
|
+
return false;
|
|
349
|
+
}
|
|
350
|
+
}
|
|
290
351
|
export function sweepStaleStagedFiles(dir, maxAgeMs = STAGED_SWEEP_MAX_AGE_MS, nowMs = Date.now()) {
|
|
291
352
|
let entries;
|
|
292
353
|
try {
|
|
@@ -312,6 +373,12 @@ export function sweepStaleStagedFiles(dir, maxAgeMs = STAGED_SWEEP_MAX_AGE_MS, n
|
|
|
312
373
|
continue;
|
|
313
374
|
if (nowMs - stat.mtimeMs < maxAgeMs)
|
|
314
375
|
continue;
|
|
376
|
+
const stagePath = name.endsWith(".json")
|
|
377
|
+
? full.slice(0, -".json".length)
|
|
378
|
+
: full;
|
|
379
|
+
const metadataPath = `${stagePath}.json`;
|
|
380
|
+
if (hasValidResumableStagePair(stagePath, metadataPath))
|
|
381
|
+
continue;
|
|
315
382
|
fs.unlinkSync(full);
|
|
316
383
|
removed.push(full);
|
|
317
384
|
}
|
|
@@ -341,6 +408,28 @@ function downloadTempPath(localPath) {
|
|
|
341
408
|
const random = crypto.randomBytes(10).toString("hex");
|
|
342
409
|
return path.join(dir, `.hq-tmp-${random}`);
|
|
343
410
|
}
|
|
411
|
+
function writeResumableStageMetadata(paths, metadata) {
|
|
412
|
+
const temporaryPath = `${paths.metadataPath}.${crypto.randomBytes(6).toString("hex")}.tmp`;
|
|
413
|
+
try {
|
|
414
|
+
fs.writeFileSync(temporaryPath, JSON.stringify(metadata), { flag: "wx" });
|
|
415
|
+
fs.renameSync(temporaryPath, paths.metadataPath);
|
|
416
|
+
}
|
|
417
|
+
catch (error) {
|
|
418
|
+
removeTempPath(temporaryPath);
|
|
419
|
+
throw error;
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
function removeResumableStage(paths) {
|
|
423
|
+
removeTempPath(paths.stagePath);
|
|
424
|
+
removeTempPath(paths.metadataPath);
|
|
425
|
+
}
|
|
426
|
+
/** A same-directory, sync-ignored target for a complete staged download. */
|
|
427
|
+
export function createDownloadStagingPath(localPath) {
|
|
428
|
+
const random = crypto.randomBytes(12).toString("hex");
|
|
429
|
+
// Match the stale-temp sweep sentinel so an interrupted complete stage is
|
|
430
|
+
// eventually reclaimed after a restart.
|
|
431
|
+
return path.join(path.dirname(localPath), `.hq-tmp-stage-${random}`);
|
|
432
|
+
}
|
|
344
433
|
function removeTempPath(tempPath) {
|
|
345
434
|
try {
|
|
346
435
|
fs.unlinkSync(tempPath);
|
|
@@ -673,10 +762,10 @@ export function replaceStagedPath(stagedPath, localPath, ops = DEFAULT_REPLACE_S
|
|
|
673
762
|
throw primaryError;
|
|
674
763
|
}
|
|
675
764
|
}
|
|
676
|
-
async function getObjectStream(io, key) {
|
|
765
|
+
async function getObjectStream(io, key, options = {}) {
|
|
677
766
|
if (io.getObjectStream)
|
|
678
|
-
return io.getObjectStream(key);
|
|
679
|
-
const res = await io.getObject(key);
|
|
767
|
+
return io.getObjectStream(key, options);
|
|
768
|
+
const res = await io.getObject(key, options);
|
|
680
769
|
return {
|
|
681
770
|
body: (async function* () {
|
|
682
771
|
yield res.body;
|
|
@@ -741,12 +830,85 @@ class HashingTransform extends Transform {
|
|
|
741
830
|
digest() {
|
|
742
831
|
return this.hash.digest("hex");
|
|
743
832
|
}
|
|
833
|
+
seed(chunks) {
|
|
834
|
+
for (const chunk of chunks) {
|
|
835
|
+
this.hash.update(chunk);
|
|
836
|
+
this.size += chunk.length;
|
|
837
|
+
}
|
|
838
|
+
}
|
|
744
839
|
}
|
|
745
|
-
async function streamRegularFileToTemp(tempPath, initialChunks, iterator) {
|
|
840
|
+
async function streamRegularFileToTemp(tempPath, initialChunks, iterator, append = false, preserveInitialChunks = false) {
|
|
746
841
|
const hashing = new HashingTransform();
|
|
747
|
-
|
|
842
|
+
let chunksToStream = initialChunks;
|
|
843
|
+
let appendToTemp = append;
|
|
844
|
+
if (preserveInitialChunks) {
|
|
845
|
+
writeInitialChunksToStage(tempPath, initialChunks);
|
|
846
|
+
hashing.seed(initialChunks);
|
|
847
|
+
chunksToStream = [];
|
|
848
|
+
appendToTemp = true;
|
|
849
|
+
}
|
|
850
|
+
await pipeline(Readable.from(regularDownloadChunks(chunksToStream, iterator)), hashing, fs.createWriteStream(tempPath, { flags: appendToTemp ? "a" : "wx" }));
|
|
748
851
|
return { hash: hashing.digest(), size: hashing.size };
|
|
749
852
|
}
|
|
853
|
+
async function hashStagedFile(filePath) {
|
|
854
|
+
const hash = crypto.createHash("sha256");
|
|
855
|
+
let size = 0;
|
|
856
|
+
for await (const chunk of fs.createReadStream(filePath)) {
|
|
857
|
+
const bytes = bufferFromChunk(chunk);
|
|
858
|
+
hash.update(bytes);
|
|
859
|
+
size += bytes.length;
|
|
860
|
+
}
|
|
861
|
+
return { hash: hash.digest("hex"), size };
|
|
862
|
+
}
|
|
863
|
+
function writeInitialChunksToStage(filePath, chunks) {
|
|
864
|
+
const descriptor = fs.openSync(filePath, "wx");
|
|
865
|
+
try {
|
|
866
|
+
for (const chunk of chunks) {
|
|
867
|
+
let offset = 0;
|
|
868
|
+
while (offset < chunk.length) {
|
|
869
|
+
offset += fs.writeSync(descriptor, chunk, offset, chunk.length - offset);
|
|
870
|
+
}
|
|
871
|
+
}
|
|
872
|
+
}
|
|
873
|
+
finally {
|
|
874
|
+
fs.closeSync(descriptor);
|
|
875
|
+
}
|
|
876
|
+
}
|
|
877
|
+
function normalizedEtag(value) {
|
|
878
|
+
if (typeof value !== "string")
|
|
879
|
+
return undefined;
|
|
880
|
+
const trimmed = value.trim();
|
|
881
|
+
if (trimmed.length === 0)
|
|
882
|
+
return undefined;
|
|
883
|
+
return trimmed.startsWith('"') && trimmed.endsWith('"')
|
|
884
|
+
? trimmed.slice(1, -1)
|
|
885
|
+
: trimmed;
|
|
886
|
+
}
|
|
887
|
+
function stageMatchesRemote(metadata, head, paths) {
|
|
888
|
+
return (metadata.keyHash === paths.keyHash &&
|
|
889
|
+
head !== null &&
|
|
890
|
+
normalizedEtag(metadata.etag) !== undefined &&
|
|
891
|
+
normalizedEtag(metadata.etag) === normalizedEtag(head.etag) &&
|
|
892
|
+
metadata.expectedSize === head.size &&
|
|
893
|
+
(metadata.versionId === undefined ||
|
|
894
|
+
head.versionId === undefined ||
|
|
895
|
+
metadata.versionId === head.versionId));
|
|
896
|
+
}
|
|
897
|
+
function rangedResponseMatchesStage(response, metadata, offset) {
|
|
898
|
+
const range = response.contentRange?.match(/^bytes (\d+)-(\d+)\/(\d+)$/);
|
|
899
|
+
return (response.statusCode === 206 &&
|
|
900
|
+
range !== null &&
|
|
901
|
+
range?.[1] !== undefined &&
|
|
902
|
+
Number(range[1]) === offset &&
|
|
903
|
+
range[3] !== undefined &&
|
|
904
|
+
Number(range[3]) === metadata.expectedSize &&
|
|
905
|
+
normalizedEtag(response.etag) === normalizedEtag(metadata.etag) &&
|
|
906
|
+
(metadata.versionId === undefined ||
|
|
907
|
+
response.versionId === undefined ||
|
|
908
|
+
metadata.versionId === response.versionId) &&
|
|
909
|
+
(response.contentLength === undefined ||
|
|
910
|
+
response.contentLength === metadata.expectedSize - offset));
|
|
911
|
+
}
|
|
750
912
|
/**
|
|
751
913
|
* Batch pre-mint transport URLs for `keys` under `op` so the subsequent
|
|
752
914
|
* per-file transfer calls (downloadFile/headRemoteFile/…) reuse them instead
|
|
@@ -974,7 +1136,7 @@ export function validateVaultUploadKey(key, scope) {
|
|
|
974
1136
|
function uploadScopeFor(ctx) {
|
|
975
1137
|
return ctx.uid.startsWith("cmp_") ? "company" : "personal";
|
|
976
1138
|
}
|
|
977
|
-
export async function uploadFile(ctx, localPath, key, author, precondition) {
|
|
1139
|
+
export async function uploadFile(ctx, localPath, key, author, precondition, transferOptions) {
|
|
978
1140
|
// Boundary guardrail: never store a non-POSIX key (see toPosixKey), and
|
|
979
1141
|
// never store a key the server-side validator would reject (see
|
|
980
1142
|
// validateVaultUploadKey — the direct-S3 STS transport bypasses server
|
|
@@ -997,6 +1159,9 @@ export async function uploadFile(ctx, localPath, key, author, precondition) {
|
|
|
997
1159
|
body,
|
|
998
1160
|
contentType: getMimeType(key),
|
|
999
1161
|
metadata: {},
|
|
1162
|
+
...(transferOptions?.bypassBandwidthGovernor
|
|
1163
|
+
? { bypassBandwidthGovernor: true }
|
|
1164
|
+
: {}),
|
|
1000
1165
|
});
|
|
1001
1166
|
return { etag: primed.etag };
|
|
1002
1167
|
}
|
|
@@ -1022,6 +1187,9 @@ export async function uploadFile(ctx, localPath, key, author, precondition) {
|
|
|
1022
1187
|
contentType: getMimeType(key),
|
|
1023
1188
|
metadata: Metadata,
|
|
1024
1189
|
...(precondition ?? {}),
|
|
1190
|
+
...(transferOptions?.bypassBandwidthGovernor
|
|
1191
|
+
? { bypassBandwidthGovernor: true }
|
|
1192
|
+
: {}),
|
|
1025
1193
|
});
|
|
1026
1194
|
return { etag: response.etag };
|
|
1027
1195
|
}
|
|
@@ -1037,7 +1205,7 @@ export async function uploadFile(ctx, localPath, key, author, precondition) {
|
|
|
1037
1205
|
* is out of scope for this primitive — the policy decision lives in the
|
|
1038
1206
|
* caller (currently: upload anyway, never silently rewrite).
|
|
1039
1207
|
*/
|
|
1040
|
-
export async function uploadSymlink(ctx, target, key, author, precondition) {
|
|
1208
|
+
export async function uploadSymlink(ctx, target, key, author, precondition, transferOptions) {
|
|
1041
1209
|
// Boundary guardrail: never store a non-POSIX key (see toPosixKey), and
|
|
1042
1210
|
// never store a key the server-side validator would reject (see
|
|
1043
1211
|
// validateVaultUploadKey).
|
|
@@ -1054,6 +1222,9 @@ export async function uploadSymlink(ctx, target, key, author, precondition) {
|
|
|
1054
1222
|
body: symlinkBody,
|
|
1055
1223
|
contentType: "application/octet-stream",
|
|
1056
1224
|
metadata: {},
|
|
1225
|
+
...(transferOptions?.bypassBandwidthGovernor
|
|
1226
|
+
? { bypassBandwidthGovernor: true }
|
|
1227
|
+
: {}),
|
|
1057
1228
|
});
|
|
1058
1229
|
return { etag: primed.etag };
|
|
1059
1230
|
}
|
|
@@ -1080,6 +1251,9 @@ export async function uploadSymlink(ctx, target, key, author, precondition) {
|
|
|
1080
1251
|
contentType: "application/octet-stream",
|
|
1081
1252
|
metadata: Metadata,
|
|
1082
1253
|
...(precondition ?? {}),
|
|
1254
|
+
...(transferOptions?.bypassBandwidthGovernor
|
|
1255
|
+
? { bypassBandwidthGovernor: true }
|
|
1256
|
+
: {}),
|
|
1083
1257
|
});
|
|
1084
1258
|
return { etag: response.etag };
|
|
1085
1259
|
}
|
|
@@ -1134,6 +1308,29 @@ function ensureDownloadParentDir(dir, key) {
|
|
|
1134
1308
|
fs.mkdirSync(dir, { recursive: true });
|
|
1135
1309
|
}
|
|
1136
1310
|
export async function downloadFile(ctx, key, localPath, options = {}) {
|
|
1311
|
+
// Every download of one key into one directory shares a resumable stage
|
|
1312
|
+
// (see resumableStagePaths). A realtime pull may overlap a chore pull of the
|
|
1313
|
+
// same key, so serialize them in-process: otherwise one writer can resume
|
|
1314
|
+
// and append to, delete, or rename the other's partial stage mid-write.
|
|
1315
|
+
const stageLock = resumableStagePaths(path.dirname(localPath), key).stagePath;
|
|
1316
|
+
const previous = downloadStageLocks.get(stageLock) ?? Promise.resolve();
|
|
1317
|
+
let releaseLock = () => { };
|
|
1318
|
+
const held = new Promise((resolve) => { releaseLock = resolve; });
|
|
1319
|
+
const chained = previous.then(() => held);
|
|
1320
|
+
downloadStageLocks.set(stageLock, chained);
|
|
1321
|
+
await previous;
|
|
1322
|
+
try {
|
|
1323
|
+
return await downloadFileUnderStageLock(ctx, key, localPath, options);
|
|
1324
|
+
}
|
|
1325
|
+
finally {
|
|
1326
|
+
releaseLock();
|
|
1327
|
+
if (downloadStageLocks.get(stageLock) === chained)
|
|
1328
|
+
downloadStageLocks.delete(stageLock);
|
|
1329
|
+
}
|
|
1330
|
+
}
|
|
1331
|
+
/** Tail of the in-process download chain for each resumable stage path. */
|
|
1332
|
+
const downloadStageLocks = new Map();
|
|
1333
|
+
async function downloadFileUnderStageLock(ctx, key, localPath, options) {
|
|
1137
1334
|
const io = resolveObjectIO(ctx);
|
|
1138
1335
|
// Outer whole-download retries: mid-stream EPIPE / socket hang up after a
|
|
1139
1336
|
// 200 cannot be fixed by replaying a half-written temp or by buffering the
|
|
@@ -1164,24 +1361,93 @@ export async function downloadFile(ctx, key, localPath, options = {}) {
|
|
|
1164
1361
|
? lastError
|
|
1165
1362
|
: new Error(`download ${key}: failed after ${DOWNLOAD_STREAM_MAX_RETRIES} retries`);
|
|
1166
1363
|
}
|
|
1167
|
-
async function downloadFileAttempt(io, key, localPath, options) {
|
|
1168
|
-
const
|
|
1364
|
+
async function downloadFileAttempt(io, key, localPath, options, allowResume = true) {
|
|
1365
|
+
const dir = path.dirname(localPath);
|
|
1366
|
+
if (!fs.existsSync(dir))
|
|
1367
|
+
ensureDownloadParentDir(dir, key);
|
|
1368
|
+
sweepStaleStagedFilesOnce(dir);
|
|
1369
|
+
const resumePaths = resumableStagePaths(dir, key);
|
|
1370
|
+
let resumeMetadata;
|
|
1371
|
+
let resumeOffset = 0;
|
|
1372
|
+
let isResuming = false;
|
|
1373
|
+
let streamResult;
|
|
1374
|
+
if (allowResume) {
|
|
1375
|
+
const saved = readResumableStageMetadata(resumePaths.metadataPath);
|
|
1376
|
+
let stageStat;
|
|
1377
|
+
try {
|
|
1378
|
+
stageStat = fs.lstatSync(resumePaths.stagePath);
|
|
1379
|
+
}
|
|
1380
|
+
catch {
|
|
1381
|
+
// No stage to resume.
|
|
1382
|
+
}
|
|
1383
|
+
if (saved &&
|
|
1384
|
+
saved.keyHash === resumePaths.keyHash &&
|
|
1385
|
+
stageStat?.isFile() &&
|
|
1386
|
+
!stageStat.isSymbolicLink() &&
|
|
1387
|
+
stageStat.size > 0 &&
|
|
1388
|
+
stageStat.size < saved.expectedSize) {
|
|
1389
|
+
const head = await io.headObject(key);
|
|
1390
|
+
if (stageMatchesRemote(saved, head, resumePaths)) {
|
|
1391
|
+
resumeOffset = stageStat.size;
|
|
1392
|
+
resumeMetadata = saved;
|
|
1393
|
+
try {
|
|
1394
|
+
streamResult = await getObjectStream(io, key, {
|
|
1395
|
+
...(options.bypassBandwidthGovernor
|
|
1396
|
+
? { bypassBandwidthGovernor: true }
|
|
1397
|
+
: {}),
|
|
1398
|
+
rangeStart: resumeOffset,
|
|
1399
|
+
ifMatch: saved.etag,
|
|
1400
|
+
});
|
|
1401
|
+
}
|
|
1402
|
+
catch (error) {
|
|
1403
|
+
if (!isRangePreconditionError(error))
|
|
1404
|
+
throw error;
|
|
1405
|
+
removeResumableStage(resumePaths);
|
|
1406
|
+
resumeMetadata = undefined;
|
|
1407
|
+
resumeOffset = 0;
|
|
1408
|
+
}
|
|
1409
|
+
if (streamResult && rangedResponseMatchesStage(streamResult, saved, resumeOffset)) {
|
|
1410
|
+
isResuming = true;
|
|
1411
|
+
}
|
|
1412
|
+
else if (streamResult) {
|
|
1413
|
+
// Close the rejected response without an error: nothing is listening
|
|
1414
|
+
// on its body yet, so an injected error would surface as an
|
|
1415
|
+
// unhandled stream `error` and could terminate the runner.
|
|
1416
|
+
streamResult.destroy?.();
|
|
1417
|
+
removeResumableStage(resumePaths);
|
|
1418
|
+
streamResult = undefined;
|
|
1419
|
+
resumeMetadata = undefined;
|
|
1420
|
+
resumeOffset = 0;
|
|
1421
|
+
}
|
|
1422
|
+
}
|
|
1423
|
+
else {
|
|
1424
|
+
removeResumableStage(resumePaths);
|
|
1425
|
+
}
|
|
1426
|
+
}
|
|
1427
|
+
else if (saved || stageStat) {
|
|
1428
|
+
removeResumableStage(resumePaths);
|
|
1429
|
+
}
|
|
1430
|
+
}
|
|
1431
|
+
streamResult ??= await getObjectStream(io, key, {
|
|
1432
|
+
...(options.bypassBandwidthGovernor
|
|
1433
|
+
? { bypassBandwidthGovernor: true }
|
|
1434
|
+
: {}),
|
|
1435
|
+
});
|
|
1436
|
+
const { body, metadata } = streamResult;
|
|
1169
1437
|
const iterator = body[Symbol.asyncIterator]();
|
|
1170
1438
|
const initialChunks = [];
|
|
1171
1439
|
let initialLength = 0;
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
if (!fs.existsSync(dir)) {
|
|
1184
|
-
ensureDownloadParentDir(dir, key);
|
|
1440
|
+
if (!isResuming) {
|
|
1441
|
+
while (initialLength < SYMLINK_BODY_PREFIX_BYTES.length) {
|
|
1442
|
+
const next = await iterator.next();
|
|
1443
|
+
if (next.done)
|
|
1444
|
+
break;
|
|
1445
|
+
const chunk = bufferFromChunk(next.value);
|
|
1446
|
+
if (chunk.length === 0)
|
|
1447
|
+
continue;
|
|
1448
|
+
initialChunks.push(chunk);
|
|
1449
|
+
initialLength += chunk.length;
|
|
1450
|
+
}
|
|
1185
1451
|
}
|
|
1186
1452
|
// Symlink path: presence of SYMLINK_TARGET_META_KEY (any non-empty
|
|
1187
1453
|
// value) is the discriminator. The TARGET is now sourced from the
|
|
@@ -1298,14 +1564,39 @@ async function downloadFileAttempt(io, key, localPath, options) {
|
|
|
1298
1564
|
}
|
|
1299
1565
|
return { metadata };
|
|
1300
1566
|
}
|
|
1301
|
-
|
|
1567
|
+
let tempPath;
|
|
1568
|
+
let keepResumableStage = false;
|
|
1569
|
+
if (isResuming && resumeMetadata) {
|
|
1570
|
+
tempPath = resumePaths.stagePath;
|
|
1571
|
+
keepResumableStage = true;
|
|
1572
|
+
}
|
|
1573
|
+
else {
|
|
1574
|
+
tempPath = downloadTempPath(localPath);
|
|
1575
|
+
const responseSize = streamResult.contentLength;
|
|
1576
|
+
if (streamResult.etag &&
|
|
1577
|
+
responseSize !== undefined &&
|
|
1578
|
+
Number.isSafeInteger(responseSize) &&
|
|
1579
|
+
responseSize > initialLength &&
|
|
1580
|
+
!isSymlinkRecord) {
|
|
1581
|
+
resumeMetadata = {
|
|
1582
|
+
version: 1,
|
|
1583
|
+
keyHash: resumePaths.keyHash,
|
|
1584
|
+
etag: streamResult.etag,
|
|
1585
|
+
...(streamResult.versionId ? { versionId: streamResult.versionId } : {}),
|
|
1586
|
+
expectedSize: responseSize,
|
|
1587
|
+
};
|
|
1588
|
+
tempPath = resumePaths.stagePath;
|
|
1589
|
+
writeResumableStageMetadata(resumePaths, resumeMetadata);
|
|
1590
|
+
keepResumableStage = true;
|
|
1591
|
+
}
|
|
1592
|
+
}
|
|
1302
1593
|
// A legacy object has no hq-mode. If it is replacing a regular local file,
|
|
1303
1594
|
// that local mode is the only trustworthy signal we have — retaining it is
|
|
1304
1595
|
// strictly safer than replacing (for example) a 0755 hook with the receiver
|
|
1305
1596
|
// umask's usual 0644. Never inspect a symlink or infer intent from content.
|
|
1306
1597
|
let existingLocalMode;
|
|
1307
1598
|
try {
|
|
1308
|
-
const existing = fs.lstatSync(localPath);
|
|
1599
|
+
const existing = fs.lstatSync(options.modeFallbackPath ?? localPath);
|
|
1309
1600
|
if (existing.isFile())
|
|
1310
1601
|
existingLocalMode = existing.mode & 0o777;
|
|
1311
1602
|
}
|
|
@@ -1316,7 +1607,17 @@ async function downloadFileAttempt(io, key, localPath, options) {
|
|
|
1316
1607
|
let streamed;
|
|
1317
1608
|
const modeWarnings = [];
|
|
1318
1609
|
try {
|
|
1319
|
-
|
|
1610
|
+
if (resumeMetadata && isResuming) {
|
|
1611
|
+
await streamRegularFileToTemp(tempPath, [], iterator, true);
|
|
1612
|
+
streamed = await hashStagedFile(tempPath);
|
|
1613
|
+
}
|
|
1614
|
+
else {
|
|
1615
|
+
streamed = await streamRegularFileToTemp(tempPath, initialChunks, iterator, false, resumeMetadata !== undefined);
|
|
1616
|
+
}
|
|
1617
|
+
if (resumeMetadata &&
|
|
1618
|
+
streamed.size !== resumeMetadata.expectedSize) {
|
|
1619
|
+
throw Object.assign(new Error(`download ${key} size mismatch: expected ${resumeMetadata.expectedSize} bytes, received ${streamed.size}`), { code: "ERR_HQ_TRUNCATED_RESPONSE" });
|
|
1620
|
+
}
|
|
1320
1621
|
// Bug #5 — apply source-side mode after the byte write. See
|
|
1321
1622
|
// FILE_MODE_META_KEY for the metadata contract. The staged path is a
|
|
1322
1623
|
// regular file, then it is atomically renamed over the destination.
|
|
@@ -1404,9 +1705,11 @@ async function downloadFileAttempt(io, key, localPath, options) {
|
|
|
1404
1705
|
options.beforeReplace?.();
|
|
1405
1706
|
replaceStagedPath(tempPath, localPath);
|
|
1406
1707
|
tempReady = false;
|
|
1708
|
+
if (keepResumableStage)
|
|
1709
|
+
removeTempPath(resumePaths.metadataPath);
|
|
1407
1710
|
}
|
|
1408
1711
|
finally {
|
|
1409
|
-
if (tempReady)
|
|
1712
|
+
if (tempReady && !keepResumableStage)
|
|
1410
1713
|
removeTempPath(tempPath);
|
|
1411
1714
|
}
|
|
1412
1715
|
// TODO: stamp hq-btime once Node lands lbirthtime (or birthtimeSync).
|