@indigoai-us/hq-cloud 6.18.0 → 6.18.1-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bandwidth.d.ts +45 -11
- package/dist/bandwidth.d.ts.map +1 -1
- package/dist/bandwidth.js +231 -18
- package/dist/bandwidth.js.map +1 -1
- package/dist/bandwidth.test.js +124 -4
- package/dist/bandwidth.test.js.map +1 -1
- package/dist/bin/sync-runner-company.d.ts +2 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +6 -0
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +26 -1
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +217 -3
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +26 -2
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +14 -5
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +107 -1
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/share.d.ts +18 -6
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +92 -47
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +3 -3
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts +17 -4
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +155 -27
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +315 -1
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/object-io.d.ts +33 -7
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +326 -140
- package/dist/object-io.js.map +1 -1
- package/dist/object-io.test.js +377 -1
- package/dist/object-io.test.js.map +1 -1
- package/dist/s3-download-stream-hash.test.d.ts +2 -0
- package/dist/s3-download-stream-hash.test.d.ts.map +1 -0
- package/dist/s3-download-stream-hash.test.js +70 -0
- package/dist/s3-download-stream-hash.test.js.map +1 -0
- package/dist/s3.d.ts +8 -3
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +304 -27
- package/dist/s3.js.map +1 -1
- package/dist/s3.test.js +163 -0
- package/dist/s3.test.js.map +1 -1
- package/dist/sync/inbound-event-admission.d.ts.map +1 -1
- package/dist/sync/inbound-event-admission.js +3 -0
- package/dist/sync/inbound-event-admission.js.map +1 -1
- package/dist/sync/inbound-event-admission.test.js +6 -0
- package/dist/sync/inbound-event-admission.test.js.map +1 -1
- package/dist/sync-core.d.ts +2 -1
- package/dist/sync-core.d.ts.map +1 -1
- package/dist/sync-core.js +5 -10
- package/dist/sync-core.js.map +1 -1
- package/dist/sync-core.test.js +16 -0
- package/dist/sync-core.test.js.map +1 -1
- package/package.json +1 -1
package/dist/object-io.js
CHANGED
|
@@ -25,6 +25,7 @@
|
|
|
25
25
|
import { S3Client, PutObjectCommand, GetObjectCommand, ListObjectsV2Command, DeleteObjectCommand, HeadObjectCommand, } from "@aws-sdk/client-s3";
|
|
26
26
|
import * as http from "node:http";
|
|
27
27
|
import * as https from "node:https";
|
|
28
|
+
import { Transform } from "node:stream";
|
|
28
29
|
import { isAbortOrTimeoutError, isTransientNetworkError } from "./lib/net-errors.js";
|
|
29
30
|
import { headersForObjectLockPut, isObjectLockChecksumRequiredBody, isObjectLockChecksumRequiredError, ObjectLockChecksumRequiredError, } from "./lib/s3-content-checksum.js";
|
|
30
31
|
export { ObjectLockChecksumRequiredError };
|
|
@@ -36,13 +37,97 @@ import { VaultClientError } from "./vault-client.js";
|
|
|
36
37
|
* socket fails the leg as transient instead of freezing the runner.
|
|
37
38
|
*/
|
|
38
39
|
export const OBJECT_IO_REQUEST_TIMEOUT_MS = 60_000;
|
|
40
|
+
/** Maximum idle interval while consuming one streamed object body. */
|
|
41
|
+
export const OBJECT_IO_BODY_IDLE_TIMEOUT_MS = 15_000;
|
|
39
42
|
let objectIORequestTimeoutMsForTest;
|
|
43
|
+
let objectIOBodyIdleTimeoutMsForTest;
|
|
40
44
|
export function objectIORequestTimeoutMs() {
|
|
41
45
|
return objectIORequestTimeoutMsForTest ?? OBJECT_IO_REQUEST_TIMEOUT_MS;
|
|
42
46
|
}
|
|
43
47
|
export function setObjectIORequestTimeoutMsForTesting(ms) {
|
|
44
48
|
objectIORequestTimeoutMsForTest = ms;
|
|
45
49
|
}
|
|
50
|
+
export function setObjectIOBodyIdleTimeoutMsForTesting(ms) {
|
|
51
|
+
objectIOBodyIdleTimeoutMsForTest = ms;
|
|
52
|
+
}
|
|
53
|
+
function objectIOBodyIdleTimeoutMs() {
|
|
54
|
+
return objectIOBodyIdleTimeoutMsForTest ?? OBJECT_IO_BODY_IDLE_TIMEOUT_MS;
|
|
55
|
+
}
|
|
56
|
+
export class ObjectBodyIdleTimeoutError extends Error {
|
|
57
|
+
constructor(key, timeoutMs) {
|
|
58
|
+
super(`object body made no progress for ${timeoutMs} ms: ${key}`);
|
|
59
|
+
this.name = "ObjectBodyIdleTimeoutError";
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
/** Bound a streamed GET after response headers arrive. */
|
|
63
|
+
export async function* withObjectBodyIdleTimeout(body, key, timeoutMs = objectIOBodyIdleTimeoutMs(), onTimeout) {
|
|
64
|
+
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
|
|
65
|
+
throw new Error("object body idle timeout must be positive");
|
|
66
|
+
}
|
|
67
|
+
const iterator = body[Symbol.asyncIterator]();
|
|
68
|
+
let complete = false;
|
|
69
|
+
let closed = false;
|
|
70
|
+
const close = () => {
|
|
71
|
+
if (closed)
|
|
72
|
+
return;
|
|
73
|
+
closed = true;
|
|
74
|
+
try {
|
|
75
|
+
void Promise.resolve(iterator.return?.()).catch(() => undefined);
|
|
76
|
+
}
|
|
77
|
+
catch {
|
|
78
|
+
// Preserve the timeout/error as the authoritative transfer result.
|
|
79
|
+
}
|
|
80
|
+
};
|
|
81
|
+
try {
|
|
82
|
+
while (true) {
|
|
83
|
+
let timer;
|
|
84
|
+
const item = await Promise.race([
|
|
85
|
+
Promise.resolve(iterator.next()),
|
|
86
|
+
new Promise((_resolve, reject) => {
|
|
87
|
+
timer = setTimeout(() => {
|
|
88
|
+
const error = new ObjectBodyIdleTimeoutError(key, timeoutMs);
|
|
89
|
+
try {
|
|
90
|
+
onTimeout?.(error);
|
|
91
|
+
}
|
|
92
|
+
catch {
|
|
93
|
+
// Preserve the timeout/error as the authoritative transfer result.
|
|
94
|
+
}
|
|
95
|
+
close();
|
|
96
|
+
reject(error);
|
|
97
|
+
}, timeoutMs);
|
|
98
|
+
}),
|
|
99
|
+
]).finally(() => {
|
|
100
|
+
if (timer !== undefined)
|
|
101
|
+
clearTimeout(timer);
|
|
102
|
+
});
|
|
103
|
+
if (item.done) {
|
|
104
|
+
complete = true;
|
|
105
|
+
return;
|
|
106
|
+
}
|
|
107
|
+
yield item.value;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
finally {
|
|
111
|
+
if (!complete)
|
|
112
|
+
close();
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
function cancelObjectBody(body, error) {
|
|
116
|
+
if (!body || typeof body !== "object")
|
|
117
|
+
return;
|
|
118
|
+
const stream = body;
|
|
119
|
+
try {
|
|
120
|
+
if (typeof stream.destroy === "function") {
|
|
121
|
+
stream.destroy(error);
|
|
122
|
+
}
|
|
123
|
+
else if (typeof stream.cancel === "function") {
|
|
124
|
+
void Promise.resolve(stream.cancel(error)).catch(() => undefined);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
catch {
|
|
128
|
+
// The idle timeout remains the authoritative transfer failure.
|
|
129
|
+
}
|
|
130
|
+
}
|
|
46
131
|
function objectIOAbortSignal() {
|
|
47
132
|
return AbortSignal.timeout(objectIORequestTimeoutMs());
|
|
48
133
|
}
|
|
@@ -220,11 +305,19 @@ export class S3SdkObjectIO {
|
|
|
220
305
|
}
|
|
221
306
|
}
|
|
222
307
|
async putObject(input) {
|
|
308
|
+
if (input.bypassBandwidthGovernor)
|
|
309
|
+
return this.putObjectWithGovernor(input);
|
|
310
|
+
return getBandwidthGovernor().withChoreTransfer(() => this.putObjectWithGovernor(input));
|
|
311
|
+
}
|
|
312
|
+
async putObjectWithGovernor(input) {
|
|
313
|
+
const governor = getBandwidthGovernor();
|
|
223
314
|
// Data-plane pacing: acquire tokens for the whole body before the PUT so
|
|
224
315
|
// aggregate upload+download throughput stays under the governor's cap.
|
|
225
|
-
|
|
316
|
+
if (!input.bypassBandwidthGovernor) {
|
|
317
|
+
await governor.acquire(input.body.byteLength);
|
|
318
|
+
}
|
|
226
319
|
try {
|
|
227
|
-
const
|
|
320
|
+
const send = () => this.sendWithCredentialRetry((client, abortSignal) => client.send(new PutObjectCommand({
|
|
228
321
|
Bucket: this.bucket,
|
|
229
322
|
Key: input.key,
|
|
230
323
|
Body: input.body,
|
|
@@ -241,6 +334,9 @@ export class S3SdkObjectIO {
|
|
|
241
334
|
...(input.ifMatch ? { IfMatch: quoteEtag(input.ifMatch) } : {}),
|
|
242
335
|
...(input.ifNoneMatch ? { IfNoneMatch: input.ifNoneMatch } : {}),
|
|
243
336
|
}), { abortSignal }));
|
|
337
|
+
const res = input.bypassBandwidthGovernor
|
|
338
|
+
? await send()
|
|
339
|
+
: await governor.measureRawTransfer(input.body.byteLength, send);
|
|
244
340
|
return { etag: res.ETag || "" };
|
|
245
341
|
}
|
|
246
342
|
catch (err) {
|
|
@@ -250,20 +346,51 @@ export class S3SdkObjectIO {
|
|
|
250
346
|
throw err;
|
|
251
347
|
}
|
|
252
348
|
}
|
|
253
|
-
async getObjectStream(key) {
|
|
254
|
-
const
|
|
255
|
-
|
|
256
|
-
|
|
349
|
+
async getObjectStream(key, options = {}) {
|
|
350
|
+
const governor = getBandwidthGovernor();
|
|
351
|
+
const release = options.bypassBandwidthGovernor
|
|
352
|
+
? undefined
|
|
353
|
+
: await governor.reserveChoreTransfer();
|
|
354
|
+
try {
|
|
355
|
+
const res = await this.sendWithCredentialRetry((client, abortSignal) => client.send(new GetObjectCommand({
|
|
356
|
+
Bucket: this.bucket,
|
|
357
|
+
Key: key,
|
|
358
|
+
...(options.rangeStart !== undefined
|
|
359
|
+
? { Range: `bytes=${options.rangeStart}-` }
|
|
360
|
+
: {}),
|
|
361
|
+
...(options.ifMatch ? { IfMatch: quoteEtag(options.ifMatch) } : {}),
|
|
362
|
+
}), { abortSignal }));
|
|
363
|
+
if (!res.Body) {
|
|
364
|
+
throw new Error(`Empty response for ${key}`);
|
|
365
|
+
}
|
|
366
|
+
const rawBody = res.Body;
|
|
367
|
+
const observedBody = options.bypassBandwidthGovernor
|
|
368
|
+
? rawBody
|
|
369
|
+
: governor.observeRawBody(rawBody);
|
|
370
|
+
const idleBoundBody = withObjectBodyIdleTimeout(observedBody, key, objectIOBodyIdleTimeoutMs(), (error) => cancelObjectBody(res.Body, error));
|
|
371
|
+
return {
|
|
372
|
+
body: options.bypassBandwidthGovernor
|
|
373
|
+
? idleBoundBody
|
|
374
|
+
: governor.governBody(idleBoundBody, release),
|
|
375
|
+
metadata: res.Metadata,
|
|
376
|
+
etag: res.ETag,
|
|
377
|
+
versionId: res.VersionId,
|
|
378
|
+
contentLength: res.ContentLength,
|
|
379
|
+
contentRange: res.ContentRange,
|
|
380
|
+
statusCode: res.$metadata?.httpStatusCode,
|
|
381
|
+
destroy: (error) => {
|
|
382
|
+
cancelObjectBody(res.Body, error ?? new Error(`GET body closed: ${key}`));
|
|
383
|
+
release?.();
|
|
384
|
+
},
|
|
385
|
+
};
|
|
386
|
+
}
|
|
387
|
+
catch (error) {
|
|
388
|
+
release?.();
|
|
389
|
+
throw error;
|
|
257
390
|
}
|
|
258
|
-
return {
|
|
259
|
-
// Data-plane pacing: each chunk acquires governor tokens before it is
|
|
260
|
-
// yielded (backpressure — the SDK stream pauses while we wait).
|
|
261
|
-
body: getBandwidthGovernor().throttleBody(res.Body),
|
|
262
|
-
metadata: res.Metadata,
|
|
263
|
-
};
|
|
264
391
|
}
|
|
265
|
-
async getObject(key) {
|
|
266
|
-
const res = await this.getObjectStream(key);
|
|
392
|
+
async getObject(key, options = {}) {
|
|
393
|
+
const res = await this.getObjectStream(key, options);
|
|
267
394
|
const body = await drainToBuffer(res.body);
|
|
268
395
|
return { body, metadata: res.metadata };
|
|
269
396
|
}
|
|
@@ -300,6 +427,7 @@ export class S3SdkObjectIO {
|
|
|
300
427
|
lastModified: res.LastModified || new Date(),
|
|
301
428
|
etag: res.ETag || "",
|
|
302
429
|
size: res.ContentLength || 0,
|
|
430
|
+
versionId: res.VersionId,
|
|
303
431
|
metadata: res.Metadata,
|
|
304
432
|
storageClass: res.StorageClass ?? "STANDARD",
|
|
305
433
|
};
|
|
@@ -358,39 +486,41 @@ function expectedContentLength(headers) {
|
|
|
358
486
|
function contentLengthError(expected, actual) {
|
|
359
487
|
return Object.assign(new Error(`presigned GET content-length mismatch: expected ${expected} bytes, received ${actual}`), { code: "ERR_HQ_TRUNCATED_RESPONSE" });
|
|
360
488
|
}
|
|
361
|
-
|
|
489
|
+
function checkedNodeBody(message, expected) {
|
|
362
490
|
let received = 0;
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
if (expected !== undefined && received > expected) {
|
|
370
|
-
throw contentLengthError(expected, received);
|
|
371
|
-
}
|
|
372
|
-
yield bytes;
|
|
491
|
+
const body = new Transform({
|
|
492
|
+
transform: (chunk, _encoding, callback) => {
|
|
493
|
+
received += chunk.byteLength;
|
|
494
|
+
if (expected !== undefined && received > expected) {
|
|
495
|
+
callback(contentLengthError(expected, received));
|
|
496
|
+
return;
|
|
373
497
|
}
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
// an "aborted" stream error. Preserve the useful data-integrity cause.
|
|
498
|
+
callback(null, chunk);
|
|
499
|
+
},
|
|
500
|
+
flush: (callback) => {
|
|
378
501
|
if (expected !== undefined && received !== expected) {
|
|
379
|
-
|
|
502
|
+
callback(contentLengthError(expected, received));
|
|
503
|
+
return;
|
|
380
504
|
}
|
|
381
|
-
|
|
382
|
-
}
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
505
|
+
callback();
|
|
506
|
+
},
|
|
507
|
+
});
|
|
508
|
+
message.once("error", (error) => {
|
|
509
|
+
body.destroy(expected !== undefined && received !== expected
|
|
510
|
+
? contentLengthError(expected, received)
|
|
511
|
+
: error);
|
|
512
|
+
});
|
|
513
|
+
message.once("aborted", () => {
|
|
514
|
+
body.destroy(expected !== undefined && received !== expected
|
|
515
|
+
? contentLengthError(expected, received)
|
|
516
|
+
: new Error("presigned GET body aborted"));
|
|
517
|
+
});
|
|
518
|
+
body.once("close", () => {
|
|
519
|
+
if (!message.destroyed)
|
|
392
520
|
message.destroy();
|
|
393
|
-
}
|
|
521
|
+
});
|
|
522
|
+
message.pipe(body);
|
|
523
|
+
return body;
|
|
394
524
|
}
|
|
395
525
|
function nodePresignedGet(url, headers) {
|
|
396
526
|
const target = new URL(url);
|
|
@@ -406,11 +536,15 @@ function nodePresignedGet(url, headers) {
|
|
|
406
536
|
reject(err);
|
|
407
537
|
return;
|
|
408
538
|
}
|
|
539
|
+
const body = checkedNodeBody(message, expected);
|
|
409
540
|
resolve({
|
|
410
541
|
status: message.statusCode ?? 0,
|
|
411
542
|
headers: responseHeaders,
|
|
412
|
-
body
|
|
413
|
-
destroy: () =>
|
|
543
|
+
body,
|
|
544
|
+
destroy: (error) => {
|
|
545
|
+
body.destroy(error);
|
|
546
|
+
message.destroy(error);
|
|
547
|
+
},
|
|
414
548
|
});
|
|
415
549
|
};
|
|
416
550
|
const request = target.protocol === "https:"
|
|
@@ -885,33 +1019,46 @@ export class PresignObjectIO {
|
|
|
885
1019
|
if (isPresignFenceStrictEnabled()) {
|
|
886
1020
|
this.requireSignedPutPreconditions(input.key, input, row.headers);
|
|
887
1021
|
}
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
const res = await fetchWithRetry(row.url, {
|
|
895
|
-
method: "PUT",
|
|
896
|
-
body: input.body,
|
|
897
|
-
headers: headersForObjectLockPut(row.url, row.headers, input.body),
|
|
898
|
-
}, `presigned PUT ${input.key}`);
|
|
899
|
-
if (res.status === 412) {
|
|
900
|
-
// The signed conditional header fenced this write off: the remote
|
|
901
|
-
// moved past our baseline (If-Match) or the key already exists
|
|
902
|
-
// (If-None-Match). Same shape as the SDK's PreconditionFailed so the
|
|
903
|
-
// push path routes both transports through one conflict handler.
|
|
904
|
-
const detail = await safeText(res);
|
|
905
|
-
throw preconditionFailedError(input.key, detail);
|
|
906
|
-
}
|
|
907
|
-
if (!res.ok) {
|
|
908
|
-
const detail = await safeText(res);
|
|
909
|
-
if (isObjectLockChecksumRequiredBody(res.status, detail)) {
|
|
910
|
-
throw new ObjectLockChecksumRequiredError(input.key, detail);
|
|
1022
|
+
const governor = getBandwidthGovernor();
|
|
1023
|
+
const performPut = async () => {
|
|
1024
|
+
// Data-plane pacing: acquire tokens for the whole body before the PUT so
|
|
1025
|
+
// aggregate upload+download throughput stays under the governor's cap.
|
|
1026
|
+
if (!input.bypassBandwidthGovernor) {
|
|
1027
|
+
await governor.acquire(input.body.byteLength);
|
|
911
1028
|
}
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
1029
|
+
const send = async () => {
|
|
1030
|
+
// The server signs Content-Type, SSE-KMS, and every x-amz-meta-* into the
|
|
1031
|
+
// signature and returns them in `headers`; they MUST be replayed verbatim
|
|
1032
|
+
// or SigV4 rejects the PUT.
|
|
1033
|
+
const res = await fetchWithRetry(row.url, {
|
|
1034
|
+
method: "PUT",
|
|
1035
|
+
body: input.body,
|
|
1036
|
+
headers: headersForObjectLockPut(row.url, row.headers, input.body),
|
|
1037
|
+
}, `presigned PUT ${input.key}`);
|
|
1038
|
+
if (res.status === 412) {
|
|
1039
|
+
// The signed conditional header fenced this write off: the remote
|
|
1040
|
+
// moved past our baseline (If-Match) or the key already exists
|
|
1041
|
+
// (If-None-Match). Same shape as the SDK's PreconditionFailed so the
|
|
1042
|
+
// push path routes both transports through one conflict handler.
|
|
1043
|
+
const detail = await safeText(res);
|
|
1044
|
+
throw preconditionFailedError(input.key, detail);
|
|
1045
|
+
}
|
|
1046
|
+
if (!res.ok) {
|
|
1047
|
+
const detail = await safeText(res);
|
|
1048
|
+
if (isObjectLockChecksumRequiredBody(res.status, detail)) {
|
|
1049
|
+
throw new ObjectLockChecksumRequiredError(input.key, detail);
|
|
1050
|
+
}
|
|
1051
|
+
throw new Error(`presigned PUT failed for ${input.key}: ${res.status} ${detail}`);
|
|
1052
|
+
}
|
|
1053
|
+
return { etag: stripQuotes(res.headers.get("etag") ?? undefined) };
|
|
1054
|
+
};
|
|
1055
|
+
return input.bypassBandwidthGovernor
|
|
1056
|
+
? send()
|
|
1057
|
+
: governor.measureRawTransfer(input.body.byteLength, send);
|
|
1058
|
+
};
|
|
1059
|
+
return input.bypassBandwidthGovernor
|
|
1060
|
+
? performPut()
|
|
1061
|
+
: getBandwidthGovernor().withChoreTransfer(performPut);
|
|
915
1062
|
}
|
|
916
1063
|
/**
|
|
917
1064
|
* True-streaming GET with bounded open-time outer retries.
|
|
@@ -927,81 +1074,119 @@ export class PresignObjectIO {
|
|
|
927
1074
|
* consumer that already received prefix bytes). That recovery lives at the
|
|
928
1075
|
* whole-download seam in s3.ts `downloadFile` (fresh stream → fresh temp).
|
|
929
1076
|
*/
|
|
930
|
-
async getObjectStream(key) {
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
row = await this.presignSingle("get", key);
|
|
947
|
-
}
|
|
948
|
-
else {
|
|
949
|
-
row = await this.resolveGetUrlForBody(key);
|
|
950
|
-
}
|
|
951
|
-
forceFreshUrl = false;
|
|
952
|
-
let res;
|
|
953
|
-
try {
|
|
954
|
-
res = await presignedGetWithRetry(row.url, row.headers, `presigned GET ${key}`);
|
|
955
|
-
}
|
|
956
|
-
catch (err) {
|
|
957
|
-
// presignedGetWithRetry already burned same-URL connection retries.
|
|
958
|
-
// One outer pass with a freshly minted URL recovers poisoned sockets
|
|
959
|
-
// that keep failing on the same signed endpoint.
|
|
960
|
-
if (isRetryablePresignedGetError(err) && attempt < FETCH_MAX_RETRIES) {
|
|
961
|
-
lastError = err;
|
|
962
|
-
forceFreshUrl = true;
|
|
963
|
-
continue;
|
|
1077
|
+
async getObjectStream(key, options = {}) {
|
|
1078
|
+
const governor = getBandwidthGovernor();
|
|
1079
|
+
const release = options.bypassBandwidthGovernor
|
|
1080
|
+
? undefined
|
|
1081
|
+
: await governor.reserveChoreTransfer();
|
|
1082
|
+
let leaseHandedOff = false;
|
|
1083
|
+
try {
|
|
1084
|
+
let lastError;
|
|
1085
|
+
// After a proven-expired URL or exhausted same-URL transport failure, the
|
|
1086
|
+
// next attempt must mint a new presign rather than replaying a dead URL.
|
|
1087
|
+
let forceFreshUrl = false;
|
|
1088
|
+
for (let attempt = 0; attempt <= FETCH_MAX_RETRIES; attempt++) {
|
|
1089
|
+
if (attempt > 0) {
|
|
1090
|
+
const backoff = FETCH_BASE_DELAY_MS * 2 ** (attempt - 1);
|
|
1091
|
+
const jitter = Math.floor(Math.random() * FETCH_BASE_DELAY_MS);
|
|
1092
|
+
await sleep(backoff + jitter);
|
|
964
1093
|
}
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
if (res.status === 403) {
|
|
972
|
-
const detail = await safePresignedGetText(res);
|
|
973
|
-
this.invalidateGetCache(key);
|
|
974
|
-
if (isExpiredPresignDetail(detail) && attempt < FETCH_MAX_RETRIES) {
|
|
975
|
-
lastError = new Error(`presigned GET expired for ${key}: 403 ${detail}`);
|
|
976
|
-
forceFreshUrl = true;
|
|
977
|
-
continue;
|
|
1094
|
+
// RateLimited / per-key Forbidden from the mint path are definitive and
|
|
1095
|
+
// propagate to the caller without outer transport retry.
|
|
1096
|
+
let row;
|
|
1097
|
+
if (forceFreshUrl) {
|
|
1098
|
+
this.invalidateGetCache(key);
|
|
1099
|
+
row = await this.presignSingle("get", key);
|
|
978
1100
|
}
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
1101
|
+
else {
|
|
1102
|
+
row = await this.resolveGetUrlForBody(key);
|
|
1103
|
+
}
|
|
1104
|
+
forceFreshUrl = false;
|
|
1105
|
+
let res;
|
|
1106
|
+
try {
|
|
1107
|
+
const requestHeaders = { ...(row.headers ?? {}) };
|
|
1108
|
+
if (options.rangeStart !== undefined) {
|
|
1109
|
+
requestHeaders.Range = `bytes=${options.rangeStart}-`;
|
|
1110
|
+
}
|
|
1111
|
+
if (options.ifMatch) {
|
|
1112
|
+
requestHeaders["If-Match"] = quoteEtag(options.ifMatch);
|
|
1113
|
+
}
|
|
1114
|
+
res = await presignedGetWithRetry(row.url, Object.keys(requestHeaders).length > 0 ? requestHeaders : undefined, `presigned GET ${key}`);
|
|
1115
|
+
}
|
|
1116
|
+
catch (err) {
|
|
1117
|
+
// presignedGetWithRetry already burned same-URL connection retries.
|
|
1118
|
+
// One outer pass with a freshly minted URL recovers poisoned sockets
|
|
1119
|
+
// that keep failing on the same signed endpoint.
|
|
1120
|
+
if (isRetryablePresignedGetError(err) && attempt < FETCH_MAX_RETRIES) {
|
|
1121
|
+
lastError = err;
|
|
1122
|
+
forceFreshUrl = true;
|
|
1123
|
+
continue;
|
|
1124
|
+
}
|
|
1125
|
+
throw err;
|
|
1126
|
+
}
|
|
1127
|
+
if (res.status === 404) {
|
|
1128
|
+
res.destroy();
|
|
1129
|
+
throw notFoundError(key);
|
|
1130
|
+
}
|
|
1131
|
+
if (res.status === 403) {
|
|
1132
|
+
const detail = await safePresignedGetText(res);
|
|
1133
|
+
this.invalidateGetCache(key);
|
|
1134
|
+
if (isExpiredPresignDetail(detail) && attempt < FETCH_MAX_RETRIES) {
|
|
1135
|
+
lastError = new Error(`presigned GET expired for ${key}: 403 ${detail}`);
|
|
1136
|
+
forceFreshUrl = true;
|
|
1137
|
+
continue;
|
|
1138
|
+
}
|
|
1139
|
+
// True ACL/KMS/bucket denial — never conflate with absence, and do
|
|
1140
|
+
// not burn re-presigns on a definitive Forbidden.
|
|
1141
|
+
throw accessDeniedError(key, detail || "presigned GET returned 403");
|
|
1142
|
+
}
|
|
1143
|
+
if (res.status === 412 || res.status === 416) {
|
|
1144
|
+
const detail = await safePresignedGetText(res);
|
|
1145
|
+
throw preconditionFailedError(key, detail);
|
|
1146
|
+
}
|
|
1147
|
+
if (res.status < 200 || res.status >= 300) {
|
|
1148
|
+
// Transient 5xx already retried inside presignedGetWithRetry; a final
|
|
1149
|
+
// non-ok status is reported to the caller without outer re-presign.
|
|
1150
|
+
const detail = await safePresignedGetText(res);
|
|
1151
|
+
throw new Error(`presigned GET failed for ${key}: ${res.status} ${detail}`);
|
|
1152
|
+
}
|
|
1153
|
+
// Live body stream — do not drain. Callers that need a Buffer use
|
|
1154
|
+
// getObject; downloadFile streams to a temp path and retries mid-stream
|
|
1155
|
+
// transport cuts around the whole attempt.
|
|
1156
|
+
const rawBody = res.body ?? (async function* () { })();
|
|
1157
|
+
const observedBody = options.bypassBandwidthGovernor
|
|
1158
|
+
? rawBody
|
|
1159
|
+
: governor.observeRawBody(rawBody);
|
|
1160
|
+
const idleBoundBody = withObjectBodyIdleTimeout(observedBody, key, objectIOBodyIdleTimeoutMs(), (error) => res.destroy(error));
|
|
1161
|
+
const result = {
|
|
1162
|
+
body: options.bypassBandwidthGovernor
|
|
1163
|
+
? idleBoundBody
|
|
1164
|
+
: governor.governBody(idleBoundBody, release),
|
|
1165
|
+
metadata: metaFromHeaders(res.headers),
|
|
1166
|
+
etag: res.headers.get("etag") ?? undefined,
|
|
1167
|
+
versionId: res.headers.get("x-amz-version-id") ?? undefined,
|
|
1168
|
+
contentLength: expectedContentLength(res.headers),
|
|
1169
|
+
contentRange: res.headers.get("content-range") ?? undefined,
|
|
1170
|
+
statusCode: res.status,
|
|
1171
|
+
destroy: (error) => {
|
|
1172
|
+
res.destroy(error);
|
|
1173
|
+
release?.();
|
|
1174
|
+
},
|
|
1175
|
+
};
|
|
1176
|
+
leaseHandedOff = true;
|
|
1177
|
+
return result;
|
|
988
1178
|
}
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
metadata: metaFromHeaders(res.headers),
|
|
997
|
-
};
|
|
1179
|
+
throw lastError instanceof Error
|
|
1180
|
+
? lastError
|
|
1181
|
+
: new Error(`presigned GET ${key}: failed after ${FETCH_MAX_RETRIES} retries`);
|
|
1182
|
+
}
|
|
1183
|
+
finally {
|
|
1184
|
+
if (!leaseHandedOff)
|
|
1185
|
+
release?.();
|
|
998
1186
|
}
|
|
999
|
-
throw lastError instanceof Error
|
|
1000
|
-
? lastError
|
|
1001
|
-
: new Error(`presigned GET ${key}: failed after ${FETCH_MAX_RETRIES} retries`);
|
|
1002
1187
|
}
|
|
1003
|
-
async getObject(key) {
|
|
1004
|
-
const res = await this.getObjectStream(key);
|
|
1188
|
+
async getObject(key, options = {}) {
|
|
1189
|
+
const res = await this.getObjectStream(key, options);
|
|
1005
1190
|
const body = await drainToBuffer(res.body);
|
|
1006
1191
|
return { body, metadata: res.metadata };
|
|
1007
1192
|
}
|
|
@@ -1106,6 +1291,7 @@ export class PresignObjectIO {
|
|
|
1106
1291
|
lastModified: parseLastModified(res.headers.get("last-modified")),
|
|
1107
1292
|
etag: stripQuotes(res.headers.get("etag") ?? undefined),
|
|
1108
1293
|
size: Number(res.headers.get("content-length") ?? "0"),
|
|
1294
|
+
versionId: res.headers.get("x-amz-version-id") ?? undefined,
|
|
1109
1295
|
storageClass: res.headers.get("x-amz-storage-class") ?? "STANDARD",
|
|
1110
1296
|
metadata: metaFromHeaders(res.headers),
|
|
1111
1297
|
};
|