@indigoai-us/hq-cloud 6.18.3 → 6.18.4-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bandwidth.d.ts +56 -10
- package/dist/bandwidth.d.ts.map +1 -1
- package/dist/bandwidth.js +253 -17
- package/dist/bandwidth.js.map +1 -1
- package/dist/bandwidth.test.js +103 -4
- package/dist/bandwidth.test.js.map +1 -1
- package/dist/bin/sync-runner-company.d.ts +2 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +15 -0
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-company.test.js +56 -0
- package/dist/bin/sync-runner-company.test.js.map +1 -1
- package/dist/bin/sync-runner-events.test.js +15 -0
- package/dist/bin/sync-runner-events.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +31 -5
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +228 -10
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +31 -2
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +14 -5
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +120 -39
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/share.d.ts +26 -8
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +227 -62
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +252 -3
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts +21 -4
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +140 -19
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +274 -1
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/lib/machine-id.d.ts +7 -0
- package/dist/lib/machine-id.d.ts.map +1 -1
- package/dist/lib/machine-id.js +54 -7
- package/dist/lib/machine-id.js.map +1 -1
- package/dist/lib/machine-id.test.js +20 -1
- package/dist/lib/machine-id.test.js.map +1 -1
- package/dist/lib/ontology-entity-push.d.ts +51 -0
- package/dist/lib/ontology-entity-push.d.ts.map +1 -0
- package/dist/lib/ontology-entity-push.js +125 -0
- package/dist/lib/ontology-entity-push.js.map +1 -0
- package/dist/lib/ontology-entity-push.test.d.ts +2 -0
- package/dist/lib/ontology-entity-push.test.d.ts.map +1 -0
- package/dist/lib/ontology-entity-push.test.js +149 -0
- package/dist/lib/ontology-entity-push.test.js.map +1 -0
- package/dist/object-io.d.ts +38 -7
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +336 -140
- package/dist/object-io.js.map +1 -1
- package/dist/object-io.test.js +423 -1
- package/dist/object-io.test.js.map +1 -1
- package/dist/s3-download-stream-hash.test.d.ts +2 -0
- package/dist/s3-download-stream-hash.test.d.ts.map +1 -0
- package/dist/s3-download-stream-hash.test.js +70 -0
- package/dist/s3-download-stream-hash.test.js.map +1 -0
- package/dist/s3.d.ts +8 -3
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +330 -27
- package/dist/s3.js.map +1 -1
- package/dist/s3.test.js +304 -0
- package/dist/s3.test.js.map +1 -1
- package/dist/sync/event-sync.d.ts +14 -4
- package/dist/sync/event-sync.d.ts.map +1 -1
- package/dist/sync/event-sync.js +145 -14
- package/dist/sync/event-sync.js.map +1 -1
- package/dist/sync/event-sync.test.js +266 -1
- package/dist/sync/event-sync.test.js.map +1 -1
- package/dist/sync/inbound-event-admission.d.ts.map +1 -1
- package/dist/sync/inbound-event-admission.js +3 -0
- package/dist/sync/inbound-event-admission.js.map +1 -1
- package/dist/sync/inbound-event-admission.test.js +6 -0
- package/dist/sync/inbound-event-admission.test.js.map +1 -1
- package/dist/sync/push-transport.d.ts +15 -0
- package/dist/sync/push-transport.d.ts.map +1 -1
- package/dist/sync/push-transport.js +27 -0
- package/dist/sync/push-transport.js.map +1 -1
- package/dist/sync/push-transport.test.js +14 -1
- package/dist/sync/push-transport.test.js.map +1 -1
- package/dist/sync-core.d.ts +2 -1
- package/dist/sync-core.d.ts.map +1 -1
- package/dist/sync-core.js +5 -10
- package/dist/sync-core.js.map +1 -1
- package/dist/sync-core.test.js +16 -0
- package/dist/sync-core.test.js.map +1 -1
- package/dist/vault-client.d.ts +5 -0
- package/dist/vault-client.d.ts.map +1 -1
- package/dist/vault-client.js +7 -0
- package/dist/vault-client.js.map +1 -1
- package/dist/vault-client.test.js +39 -0
- package/dist/vault-client.test.js.map +1 -1
- package/package.json +1 -1
package/dist/object-io.js
CHANGED
|
@@ -25,6 +25,7 @@
|
|
|
25
25
|
import { S3Client, PutObjectCommand, GetObjectCommand, ListObjectsV2Command, DeleteObjectCommand, HeadObjectCommand, } from "@aws-sdk/client-s3";
|
|
26
26
|
import * as http from "node:http";
|
|
27
27
|
import * as https from "node:https";
|
|
28
|
+
import { Transform } from "node:stream";
|
|
28
29
|
import { isAbortOrTimeoutError, isTransientNetworkError } from "./lib/net-errors.js";
|
|
29
30
|
import { headersForObjectLockPut, isObjectLockChecksumRequiredBody, isObjectLockChecksumRequiredError, ObjectLockChecksumRequiredError, } from "./lib/s3-content-checksum.js";
|
|
30
31
|
export { ObjectLockChecksumRequiredError };
|
|
@@ -36,13 +37,97 @@ import { VaultClientError } from "./vault-client.js";
|
|
|
36
37
|
* socket fails the leg as transient instead of freezing the runner.
|
|
37
38
|
*/
|
|
38
39
|
export const OBJECT_IO_REQUEST_TIMEOUT_MS = 60_000;
|
|
40
|
+
/** Maximum idle interval while consuming one streamed object body. */
|
|
41
|
+
export const OBJECT_IO_BODY_IDLE_TIMEOUT_MS = 15_000;
|
|
39
42
|
let objectIORequestTimeoutMsForTest;
|
|
43
|
+
let objectIOBodyIdleTimeoutMsForTest;
|
|
40
44
|
export function objectIORequestTimeoutMs() {
|
|
41
45
|
return objectIORequestTimeoutMsForTest ?? OBJECT_IO_REQUEST_TIMEOUT_MS;
|
|
42
46
|
}
|
|
43
47
|
export function setObjectIORequestTimeoutMsForTesting(ms) {
|
|
44
48
|
objectIORequestTimeoutMsForTest = ms;
|
|
45
49
|
}
|
|
50
|
+
export function setObjectIOBodyIdleTimeoutMsForTesting(ms) {
|
|
51
|
+
objectIOBodyIdleTimeoutMsForTest = ms;
|
|
52
|
+
}
|
|
53
|
+
function objectIOBodyIdleTimeoutMs() {
|
|
54
|
+
return objectIOBodyIdleTimeoutMsForTest ?? OBJECT_IO_BODY_IDLE_TIMEOUT_MS;
|
|
55
|
+
}
|
|
56
|
+
export class ObjectBodyIdleTimeoutError extends Error {
|
|
57
|
+
constructor(key, timeoutMs) {
|
|
58
|
+
super(`object body made no progress for ${timeoutMs} ms: ${key}`);
|
|
59
|
+
this.name = "ObjectBodyIdleTimeoutError";
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
/** Bound a streamed GET after response headers arrive. */
|
|
63
|
+
export async function* withObjectBodyIdleTimeout(body, key, timeoutMs = objectIOBodyIdleTimeoutMs(), onTimeout) {
|
|
64
|
+
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
|
|
65
|
+
throw new Error("object body idle timeout must be positive");
|
|
66
|
+
}
|
|
67
|
+
const iterator = body[Symbol.asyncIterator]();
|
|
68
|
+
let complete = false;
|
|
69
|
+
let closed = false;
|
|
70
|
+
const close = () => {
|
|
71
|
+
if (closed)
|
|
72
|
+
return;
|
|
73
|
+
closed = true;
|
|
74
|
+
try {
|
|
75
|
+
void Promise.resolve(iterator.return?.()).catch(() => undefined);
|
|
76
|
+
}
|
|
77
|
+
catch {
|
|
78
|
+
// Preserve the timeout/error as the authoritative transfer result.
|
|
79
|
+
}
|
|
80
|
+
};
|
|
81
|
+
try {
|
|
82
|
+
while (true) {
|
|
83
|
+
let timer;
|
|
84
|
+
const item = await Promise.race([
|
|
85
|
+
Promise.resolve(iterator.next()),
|
|
86
|
+
new Promise((_resolve, reject) => {
|
|
87
|
+
timer = setTimeout(() => {
|
|
88
|
+
const error = new ObjectBodyIdleTimeoutError(key, timeoutMs);
|
|
89
|
+
try {
|
|
90
|
+
onTimeout?.(error);
|
|
91
|
+
}
|
|
92
|
+
catch {
|
|
93
|
+
// Preserve the timeout/error as the authoritative transfer result.
|
|
94
|
+
}
|
|
95
|
+
close();
|
|
96
|
+
reject(error);
|
|
97
|
+
}, timeoutMs);
|
|
98
|
+
}),
|
|
99
|
+
]).finally(() => {
|
|
100
|
+
if (timer !== undefined)
|
|
101
|
+
clearTimeout(timer);
|
|
102
|
+
});
|
|
103
|
+
if (item.done) {
|
|
104
|
+
complete = true;
|
|
105
|
+
return;
|
|
106
|
+
}
|
|
107
|
+
yield item.value;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
finally {
|
|
111
|
+
if (!complete)
|
|
112
|
+
close();
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
function cancelObjectBody(body, error) {
|
|
116
|
+
if (!body || typeof body !== "object")
|
|
117
|
+
return;
|
|
118
|
+
const stream = body;
|
|
119
|
+
try {
|
|
120
|
+
if (typeof stream.destroy === "function") {
|
|
121
|
+
stream.destroy(error);
|
|
122
|
+
}
|
|
123
|
+
else if (typeof stream.cancel === "function") {
|
|
124
|
+
void Promise.resolve(stream.cancel(error)).catch(() => undefined);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
catch {
|
|
128
|
+
// The idle timeout remains the authoritative transfer failure.
|
|
129
|
+
}
|
|
130
|
+
}
|
|
46
131
|
function objectIOAbortSignal() {
|
|
47
132
|
return AbortSignal.timeout(objectIORequestTimeoutMs());
|
|
48
133
|
}
|
|
@@ -242,11 +327,19 @@ export class S3SdkObjectIO {
|
|
|
242
327
|
}
|
|
243
328
|
}
|
|
244
329
|
async putObject(input) {
|
|
330
|
+
if (input.bypassBandwidthGovernor)
|
|
331
|
+
return this.putObjectWithGovernor(input);
|
|
332
|
+
return getBandwidthGovernor().withChoreTransfer(() => this.putObjectWithGovernor(input));
|
|
333
|
+
}
|
|
334
|
+
async putObjectWithGovernor(input) {
|
|
335
|
+
const governor = getBandwidthGovernor();
|
|
245
336
|
// Data-plane pacing: acquire tokens for the whole body before the PUT so
|
|
246
337
|
// aggregate upload+download throughput stays under the governor's cap.
|
|
247
|
-
|
|
338
|
+
if (!input.bypassBandwidthGovernor) {
|
|
339
|
+
await governor.acquire(input.body.byteLength);
|
|
340
|
+
}
|
|
248
341
|
try {
|
|
249
|
-
const
|
|
342
|
+
const send = () => this.sendWithCredentialRetry((client, abortSignal) => client.send(new PutObjectCommand({
|
|
250
343
|
Bucket: this.bucket,
|
|
251
344
|
Key: input.key,
|
|
252
345
|
Body: input.body,
|
|
@@ -263,6 +356,9 @@ export class S3SdkObjectIO {
|
|
|
263
356
|
...(input.ifMatch ? { IfMatch: quoteEtag(input.ifMatch) } : {}),
|
|
264
357
|
...(input.ifNoneMatch ? { IfNoneMatch: input.ifNoneMatch } : {}),
|
|
265
358
|
}), { abortSignal }));
|
|
359
|
+
const res = input.bypassBandwidthGovernor
|
|
360
|
+
? await send()
|
|
361
|
+
: await governor.measureRawTransfer(input.body.byteLength, send);
|
|
266
362
|
return { etag: res.ETag || "" };
|
|
267
363
|
}
|
|
268
364
|
catch (err) {
|
|
@@ -272,20 +368,51 @@ export class S3SdkObjectIO {
|
|
|
272
368
|
throw err;
|
|
273
369
|
}
|
|
274
370
|
}
|
|
275
|
-
async getObjectStream(key) {
|
|
276
|
-
const
|
|
277
|
-
|
|
278
|
-
|
|
371
|
+
async getObjectStream(key, options = {}) {
|
|
372
|
+
const governor = getBandwidthGovernor();
|
|
373
|
+
const release = options.bypassBandwidthGovernor
|
|
374
|
+
? undefined
|
|
375
|
+
: await governor.reserveChoreTransfer();
|
|
376
|
+
try {
|
|
377
|
+
const res = await this.sendWithCredentialRetry((client, abortSignal) => client.send(new GetObjectCommand({
|
|
378
|
+
Bucket: this.bucket,
|
|
379
|
+
Key: key,
|
|
380
|
+
...(options.rangeStart !== undefined
|
|
381
|
+
? { Range: `bytes=${options.rangeStart}-` }
|
|
382
|
+
: {}),
|
|
383
|
+
...(options.ifMatch ? { IfMatch: quoteEtag(options.ifMatch) } : {}),
|
|
384
|
+
}), { abortSignal }));
|
|
385
|
+
if (!res.Body) {
|
|
386
|
+
throw new Error(`Empty response for ${key}`);
|
|
387
|
+
}
|
|
388
|
+
const rawBody = res.Body;
|
|
389
|
+
const observedBody = options.bypassBandwidthGovernor
|
|
390
|
+
? rawBody
|
|
391
|
+
: governor.observeRawBody(rawBody);
|
|
392
|
+
const idleBoundBody = withObjectBodyIdleTimeout(observedBody, key, objectIOBodyIdleTimeoutMs(), (error) => cancelObjectBody(res.Body, error));
|
|
393
|
+
return {
|
|
394
|
+
body: options.bypassBandwidthGovernor
|
|
395
|
+
? idleBoundBody
|
|
396
|
+
: governor.governBody(idleBoundBody, release),
|
|
397
|
+
metadata: res.Metadata,
|
|
398
|
+
etag: res.ETag,
|
|
399
|
+
versionId: res.VersionId,
|
|
400
|
+
contentLength: res.ContentLength,
|
|
401
|
+
contentRange: res.ContentRange,
|
|
402
|
+
statusCode: res.$metadata?.httpStatusCode,
|
|
403
|
+
destroy: (error) => {
|
|
404
|
+
cancelObjectBody(res.Body, error ?? new Error(`GET body closed: ${key}`));
|
|
405
|
+
release?.();
|
|
406
|
+
},
|
|
407
|
+
};
|
|
408
|
+
}
|
|
409
|
+
catch (error) {
|
|
410
|
+
release?.();
|
|
411
|
+
throw error;
|
|
279
412
|
}
|
|
280
|
-
return {
|
|
281
|
-
// Data-plane pacing: each chunk acquires governor tokens before it is
|
|
282
|
-
// yielded (backpressure — the SDK stream pauses while we wait).
|
|
283
|
-
body: getBandwidthGovernor().throttleBody(res.Body),
|
|
284
|
-
metadata: res.Metadata,
|
|
285
|
-
};
|
|
286
413
|
}
|
|
287
|
-
async getObject(key) {
|
|
288
|
-
const res = await this.getObjectStream(key);
|
|
414
|
+
async getObject(key, options = {}) {
|
|
415
|
+
const res = await this.getObjectStream(key, options);
|
|
289
416
|
const body = await drainToBuffer(res.body);
|
|
290
417
|
return { body, metadata: res.metadata };
|
|
291
418
|
}
|
|
@@ -322,6 +449,7 @@ export class S3SdkObjectIO {
|
|
|
322
449
|
lastModified: res.LastModified || new Date(),
|
|
323
450
|
etag: res.ETag || "",
|
|
324
451
|
size: res.ContentLength || 0,
|
|
452
|
+
versionId: res.VersionId,
|
|
325
453
|
metadata: res.Metadata,
|
|
326
454
|
storageClass: res.StorageClass ?? "STANDARD",
|
|
327
455
|
};
|
|
@@ -380,40 +508,49 @@ function expectedContentLength(headers) {
|
|
|
380
508
|
function contentLengthError(expected, actual) {
|
|
381
509
|
return Object.assign(new Error(`presigned GET content-length mismatch: expected ${expected} bytes, received ${actual}`), { code: "ERR_HQ_TRUNCATED_RESPONSE" });
|
|
382
510
|
}
|
|
383
|
-
|
|
511
|
+
function checkedNodeBody(message, expected) {
|
|
384
512
|
let received = 0;
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
if (expected !== undefined && received > expected) {
|
|
392
|
-
throw contentLengthError(expected, received);
|
|
393
|
-
}
|
|
394
|
-
yield bytes;
|
|
513
|
+
const body = new Transform({
|
|
514
|
+
transform: (chunk, _encoding, callback) => {
|
|
515
|
+
received += chunk.byteLength;
|
|
516
|
+
if (expected !== undefined && received > expected) {
|
|
517
|
+
callback(contentLengthError(expected, received));
|
|
518
|
+
return;
|
|
395
519
|
}
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
// an "aborted" stream error. Preserve the useful data-integrity cause.
|
|
520
|
+
callback(null, chunk);
|
|
521
|
+
},
|
|
522
|
+
flush: (callback) => {
|
|
400
523
|
if (expected !== undefined && received !== expected) {
|
|
401
|
-
|
|
524
|
+
callback(contentLengthError(expected, received));
|
|
525
|
+
return;
|
|
402
526
|
}
|
|
403
|
-
|
|
404
|
-
}
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
527
|
+
callback();
|
|
528
|
+
},
|
|
529
|
+
});
|
|
530
|
+
message.once("error", (error) => {
|
|
531
|
+
body.destroy(expected !== undefined && received !== expected
|
|
532
|
+
? contentLengthError(expected, received)
|
|
533
|
+
: error);
|
|
534
|
+
});
|
|
535
|
+
message.once("aborted", () => {
|
|
536
|
+
body.destroy(expected !== undefined && received !== expected
|
|
537
|
+
? contentLengthError(expected, received)
|
|
538
|
+
: new Error("presigned GET body aborted"));
|
|
539
|
+
});
|
|
540
|
+
body.once("close", () => {
|
|
541
|
+
if (!message.destroyed)
|
|
414
542
|
message.destroy();
|
|
415
|
-
}
|
|
543
|
+
});
|
|
544
|
+
// The body is piped eagerly, so a socket error or abort can destroy it
|
|
545
|
+
// before the caller starts iterating. Without a listener that destroy would
|
|
546
|
+
// be an uncaught stream error; the error stays on `body.errored` and the
|
|
547
|
+
// reader's iteration still rejects with it, reaching the download retry.
|
|
548
|
+
body.on("error", () => { });
|
|
549
|
+
message.pipe(body);
|
|
550
|
+
return body;
|
|
416
551
|
}
|
|
552
|
+
/** Test hook: the Node presigned GET body wrapper. */
|
|
553
|
+
export const checkedNodeBodyForTesting = checkedNodeBody;
|
|
417
554
|
function nodePresignedGet(url, headers) {
|
|
418
555
|
const target = new URL(url);
|
|
419
556
|
return new Promise((resolve, reject) => {
|
|
@@ -428,11 +565,15 @@ function nodePresignedGet(url, headers) {
|
|
|
428
565
|
reject(err);
|
|
429
566
|
return;
|
|
430
567
|
}
|
|
568
|
+
const body = checkedNodeBody(message, expected);
|
|
431
569
|
resolve({
|
|
432
570
|
status: message.statusCode ?? 0,
|
|
433
571
|
headers: responseHeaders,
|
|
434
|
-
body
|
|
435
|
-
destroy: () =>
|
|
572
|
+
body,
|
|
573
|
+
destroy: (error) => {
|
|
574
|
+
body.destroy(error);
|
|
575
|
+
message.destroy(error);
|
|
576
|
+
},
|
|
436
577
|
});
|
|
437
578
|
};
|
|
438
579
|
const request = target.protocol === "https:"
|
|
@@ -494,6 +635,9 @@ function firstRowOrThrow(results, key, op) {
|
|
|
494
635
|
throw new Error(`presign ${op} returned no row for ${key}`);
|
|
495
636
|
}
|
|
496
637
|
if (row.error || !row.url) {
|
|
638
|
+
if (row.code === "PLAN_LIMIT_REACHED" && row.upgradeUrl) {
|
|
639
|
+
throw new VaultClientError(row.error ?? "Plan limit reached", 402, undefined, { code: row.code, path: key, upgradeUrl: row.upgradeUrl });
|
|
640
|
+
}
|
|
497
641
|
// An expected per-key denial (see {@link isSkipWithLogDenialCode}) must
|
|
498
642
|
// throw the SAME `name: "Forbidden"` shape the GET/HEAD path uses so it
|
|
499
643
|
// routes through the `isAccessDenied` skip in sync.ts / share.ts. Without
|
|
@@ -907,33 +1051,46 @@ export class PresignObjectIO {
|
|
|
907
1051
|
if (isPresignFenceStrictEnabled()) {
|
|
908
1052
|
this.requireSignedPutPreconditions(input.key, input, row.headers);
|
|
909
1053
|
}
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
const res = await fetchWithRetry(row.url, {
|
|
917
|
-
method: "PUT",
|
|
918
|
-
body: input.body,
|
|
919
|
-
headers: headersForObjectLockPut(row.url, row.headers, input.body),
|
|
920
|
-
}, `presigned PUT ${input.key}`);
|
|
921
|
-
if (res.status === 412) {
|
|
922
|
-
// The signed conditional header fenced this write off: the remote
|
|
923
|
-
// moved past our baseline (If-Match) or the key already exists
|
|
924
|
-
// (If-None-Match). Same shape as the SDK's PreconditionFailed so the
|
|
925
|
-
// push path routes both transports through one conflict handler.
|
|
926
|
-
const detail = await safeText(res);
|
|
927
|
-
throw preconditionFailedError(input.key, detail);
|
|
928
|
-
}
|
|
929
|
-
if (!res.ok) {
|
|
930
|
-
const detail = await safeText(res);
|
|
931
|
-
if (isObjectLockChecksumRequiredBody(res.status, detail)) {
|
|
932
|
-
throw new ObjectLockChecksumRequiredError(input.key, detail);
|
|
1054
|
+
const governor = getBandwidthGovernor();
|
|
1055
|
+
const performPut = async () => {
|
|
1056
|
+
// Data-plane pacing: acquire tokens for the whole body before the PUT so
|
|
1057
|
+
// aggregate upload+download throughput stays under the governor's cap.
|
|
1058
|
+
if (!input.bypassBandwidthGovernor) {
|
|
1059
|
+
await governor.acquire(input.body.byteLength);
|
|
933
1060
|
}
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
1061
|
+
const send = async () => {
|
|
1062
|
+
// The server signs Content-Type, SSE-KMS, and every x-amz-meta-* into the
|
|
1063
|
+
// signature and returns them in `headers`; they MUST be replayed verbatim
|
|
1064
|
+
// or SigV4 rejects the PUT.
|
|
1065
|
+
const res = await fetchWithRetry(row.url, {
|
|
1066
|
+
method: "PUT",
|
|
1067
|
+
body: input.body,
|
|
1068
|
+
headers: headersForObjectLockPut(row.url, row.headers, input.body),
|
|
1069
|
+
}, `presigned PUT ${input.key}`);
|
|
1070
|
+
if (res.status === 412) {
|
|
1071
|
+
// The signed conditional header fenced this write off: the remote
|
|
1072
|
+
// moved past our baseline (If-Match) or the key already exists
|
|
1073
|
+
// (If-None-Match). Same shape as the SDK's PreconditionFailed so the
|
|
1074
|
+
// push path routes both transports through one conflict handler.
|
|
1075
|
+
const detail = await safeText(res);
|
|
1076
|
+
throw preconditionFailedError(input.key, detail);
|
|
1077
|
+
}
|
|
1078
|
+
if (!res.ok) {
|
|
1079
|
+
const detail = await safeText(res);
|
|
1080
|
+
if (isObjectLockChecksumRequiredBody(res.status, detail)) {
|
|
1081
|
+
throw new ObjectLockChecksumRequiredError(input.key, detail);
|
|
1082
|
+
}
|
|
1083
|
+
throw new Error(`presigned PUT failed for ${input.key}: ${res.status} ${detail}`);
|
|
1084
|
+
}
|
|
1085
|
+
return { etag: stripQuotes(res.headers.get("etag") ?? undefined) };
|
|
1086
|
+
};
|
|
1087
|
+
return input.bypassBandwidthGovernor
|
|
1088
|
+
? send()
|
|
1089
|
+
: governor.measureRawTransfer(input.body.byteLength, send);
|
|
1090
|
+
};
|
|
1091
|
+
return input.bypassBandwidthGovernor
|
|
1092
|
+
? performPut()
|
|
1093
|
+
: getBandwidthGovernor().withChoreTransfer(performPut);
|
|
937
1094
|
}
|
|
938
1095
|
/**
|
|
939
1096
|
* True-streaming GET with bounded open-time outer retries.
|
|
@@ -949,81 +1106,119 @@ export class PresignObjectIO {
|
|
|
949
1106
|
* consumer that already received prefix bytes). That recovery lives at the
|
|
950
1107
|
* whole-download seam in s3.ts `downloadFile` (fresh stream → fresh temp).
|
|
951
1108
|
*/
|
|
952
|
-
async getObjectStream(key) {
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
row = await this.presignSingle("get", key);
|
|
969
|
-
}
|
|
970
|
-
else {
|
|
971
|
-
row = await this.resolveGetUrlForBody(key);
|
|
972
|
-
}
|
|
973
|
-
forceFreshUrl = false;
|
|
974
|
-
let res;
|
|
975
|
-
try {
|
|
976
|
-
res = await presignedGetWithRetry(row.url, row.headers, `presigned GET ${key}`);
|
|
977
|
-
}
|
|
978
|
-
catch (err) {
|
|
979
|
-
// presignedGetWithRetry already burned same-URL connection retries.
|
|
980
|
-
// One outer pass with a freshly minted URL recovers poisoned sockets
|
|
981
|
-
// that keep failing on the same signed endpoint.
|
|
982
|
-
if (isRetryablePresignedGetError(err) && attempt < FETCH_MAX_RETRIES) {
|
|
983
|
-
lastError = err;
|
|
984
|
-
forceFreshUrl = true;
|
|
985
|
-
continue;
|
|
1109
|
+
async getObjectStream(key, options = {}) {
|
|
1110
|
+
const governor = getBandwidthGovernor();
|
|
1111
|
+
const release = options.bypassBandwidthGovernor
|
|
1112
|
+
? undefined
|
|
1113
|
+
: await governor.reserveChoreTransfer();
|
|
1114
|
+
let leaseHandedOff = false;
|
|
1115
|
+
try {
|
|
1116
|
+
let lastError;
|
|
1117
|
+
// After a proven-expired URL or exhausted same-URL transport failure, the
|
|
1118
|
+
// next attempt must mint a new presign rather than replaying a dead URL.
|
|
1119
|
+
let forceFreshUrl = false;
|
|
1120
|
+
for (let attempt = 0; attempt <= FETCH_MAX_RETRIES; attempt++) {
|
|
1121
|
+
if (attempt > 0) {
|
|
1122
|
+
const backoff = FETCH_BASE_DELAY_MS * 2 ** (attempt - 1);
|
|
1123
|
+
const jitter = Math.floor(Math.random() * FETCH_BASE_DELAY_MS);
|
|
1124
|
+
await sleep(backoff + jitter);
|
|
986
1125
|
}
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
if (res.status === 403) {
|
|
994
|
-
const detail = await safePresignedGetText(res);
|
|
995
|
-
this.invalidateGetCache(key);
|
|
996
|
-
if (isExpiredPresignDetail(detail) && attempt < FETCH_MAX_RETRIES) {
|
|
997
|
-
lastError = new Error(`presigned GET expired for ${key}: 403 ${detail}`);
|
|
998
|
-
forceFreshUrl = true;
|
|
999
|
-
continue;
|
|
1126
|
+
// RateLimited / per-key Forbidden from the mint path are definitive and
|
|
1127
|
+
// propagate to the caller without outer transport retry.
|
|
1128
|
+
let row;
|
|
1129
|
+
if (forceFreshUrl) {
|
|
1130
|
+
this.invalidateGetCache(key);
|
|
1131
|
+
row = await this.presignSingle("get", key);
|
|
1000
1132
|
}
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1133
|
+
else {
|
|
1134
|
+
row = await this.resolveGetUrlForBody(key);
|
|
1135
|
+
}
|
|
1136
|
+
forceFreshUrl = false;
|
|
1137
|
+
let res;
|
|
1138
|
+
try {
|
|
1139
|
+
const requestHeaders = { ...(row.headers ?? {}) };
|
|
1140
|
+
if (options.rangeStart !== undefined) {
|
|
1141
|
+
requestHeaders.Range = `bytes=${options.rangeStart}-`;
|
|
1142
|
+
}
|
|
1143
|
+
if (options.ifMatch) {
|
|
1144
|
+
requestHeaders["If-Match"] = quoteEtag(options.ifMatch);
|
|
1145
|
+
}
|
|
1146
|
+
res = await presignedGetWithRetry(row.url, Object.keys(requestHeaders).length > 0 ? requestHeaders : undefined, `presigned GET ${key}`);
|
|
1147
|
+
}
|
|
1148
|
+
catch (err) {
|
|
1149
|
+
// presignedGetWithRetry already burned same-URL connection retries.
|
|
1150
|
+
// One outer pass with a freshly minted URL recovers poisoned sockets
|
|
1151
|
+
// that keep failing on the same signed endpoint.
|
|
1152
|
+
if (isRetryablePresignedGetError(err) && attempt < FETCH_MAX_RETRIES) {
|
|
1153
|
+
lastError = err;
|
|
1154
|
+
forceFreshUrl = true;
|
|
1155
|
+
continue;
|
|
1156
|
+
}
|
|
1157
|
+
throw err;
|
|
1158
|
+
}
|
|
1159
|
+
if (res.status === 404) {
|
|
1160
|
+
res.destroy();
|
|
1161
|
+
throw notFoundError(key);
|
|
1162
|
+
}
|
|
1163
|
+
if (res.status === 403) {
|
|
1164
|
+
const detail = await safePresignedGetText(res);
|
|
1165
|
+
this.invalidateGetCache(key);
|
|
1166
|
+
if (isExpiredPresignDetail(detail) && attempt < FETCH_MAX_RETRIES) {
|
|
1167
|
+
lastError = new Error(`presigned GET expired for ${key}: 403 ${detail}`);
|
|
1168
|
+
forceFreshUrl = true;
|
|
1169
|
+
continue;
|
|
1170
|
+
}
|
|
1171
|
+
// True ACL/KMS/bucket denial — never conflate with absence, and do
|
|
1172
|
+
// not burn re-presigns on a definitive Forbidden.
|
|
1173
|
+
throw accessDeniedError(key, detail || "presigned GET returned 403");
|
|
1174
|
+
}
|
|
1175
|
+
if (res.status === 412 || res.status === 416) {
|
|
1176
|
+
const detail = await safePresignedGetText(res);
|
|
1177
|
+
throw preconditionFailedError(key, detail);
|
|
1178
|
+
}
|
|
1179
|
+
if (res.status < 200 || res.status >= 300) {
|
|
1180
|
+
// Transient 5xx already retried inside presignedGetWithRetry; a final
|
|
1181
|
+
// non-ok status is reported to the caller without outer re-presign.
|
|
1182
|
+
const detail = await safePresignedGetText(res);
|
|
1183
|
+
throw new Error(`presigned GET failed for ${key}: ${res.status} ${detail}`);
|
|
1184
|
+
}
|
|
1185
|
+
// Live body stream — do not drain. Callers that need a Buffer use
|
|
1186
|
+
// getObject; downloadFile streams to a temp path and retries mid-stream
|
|
1187
|
+
// transport cuts around the whole attempt.
|
|
1188
|
+
const rawBody = res.body ?? (async function* () { })();
|
|
1189
|
+
const observedBody = options.bypassBandwidthGovernor
|
|
1190
|
+
? rawBody
|
|
1191
|
+
: governor.observeRawBody(rawBody);
|
|
1192
|
+
const idleBoundBody = withObjectBodyIdleTimeout(observedBody, key, objectIOBodyIdleTimeoutMs(), (error) => res.destroy(error));
|
|
1193
|
+
const result = {
|
|
1194
|
+
body: options.bypassBandwidthGovernor
|
|
1195
|
+
? idleBoundBody
|
|
1196
|
+
: governor.governBody(idleBoundBody, release),
|
|
1197
|
+
metadata: metaFromHeaders(res.headers),
|
|
1198
|
+
etag: res.headers.get("etag") ?? undefined,
|
|
1199
|
+
versionId: res.headers.get("x-amz-version-id") ?? undefined,
|
|
1200
|
+
contentLength: expectedContentLength(res.headers),
|
|
1201
|
+
contentRange: res.headers.get("content-range") ?? undefined,
|
|
1202
|
+
statusCode: res.status,
|
|
1203
|
+
destroy: (error) => {
|
|
1204
|
+
res.destroy(error);
|
|
1205
|
+
release?.();
|
|
1206
|
+
},
|
|
1207
|
+
};
|
|
1208
|
+
leaseHandedOff = true;
|
|
1209
|
+
return result;
|
|
1010
1210
|
}
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
metadata: metaFromHeaders(res.headers),
|
|
1019
|
-
};
|
|
1211
|
+
throw lastError instanceof Error
|
|
1212
|
+
? lastError
|
|
1213
|
+
: new Error(`presigned GET ${key}: failed after ${FETCH_MAX_RETRIES} retries`);
|
|
1214
|
+
}
|
|
1215
|
+
finally {
|
|
1216
|
+
if (!leaseHandedOff)
|
|
1217
|
+
release?.();
|
|
1020
1218
|
}
|
|
1021
|
-
throw lastError instanceof Error
|
|
1022
|
-
? lastError
|
|
1023
|
-
: new Error(`presigned GET ${key}: failed after ${FETCH_MAX_RETRIES} retries`);
|
|
1024
1219
|
}
|
|
1025
|
-
async getObject(key) {
|
|
1026
|
-
const res = await this.getObjectStream(key);
|
|
1220
|
+
async getObject(key, options = {}) {
|
|
1221
|
+
const res = await this.getObjectStream(key, options);
|
|
1027
1222
|
const body = await drainToBuffer(res.body);
|
|
1028
1223
|
return { body, metadata: res.metadata };
|
|
1029
1224
|
}
|
|
@@ -1133,6 +1328,7 @@ export class PresignObjectIO {
|
|
|
1133
1328
|
lastModified: parseLastModified(res.headers.get("last-modified")),
|
|
1134
1329
|
etag: stripQuotes(res.headers.get("etag") ?? undefined),
|
|
1135
1330
|
size: Number(res.headers.get("content-length") ?? "0"),
|
|
1331
|
+
versionId: res.headers.get("x-amz-version-id") ?? undefined,
|
|
1136
1332
|
storageClass: res.headers.get("x-amz-storage-class") ?? "STANDARD",
|
|
1137
1333
|
metadata: metaFromHeaders(res.headers),
|
|
1138
1334
|
};
|