@bymax-one/nest-storage 1.1.11 → 1.1.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +167 -1
- package/README.md +66 -16
- package/dist/server/index.cjs +686 -207
- package/dist/server/index.d.cts +76 -37
- package/dist/server/index.d.ts +76 -37
- package/dist/server/index.mjs +688 -209
- package/dist/shared/index.cjs +2 -1
- package/dist/shared/index.d.cts +1 -0
- package/dist/shared/index.d.ts +1 -0
- package/dist/shared/index.mjs +2 -1
- package/package.json +1 -1
package/dist/server/index.mjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { HttpStatus, Injectable, Inject, Global, Module,
|
|
1
|
+
import { HttpStatus, Injectable, Inject, Logger, Global, Module, HttpException } from '@nestjs/common';
|
|
2
2
|
import { S3Client, GetBucketLifecycleConfigurationCommand, DeleteBucketLifecycleCommand, PutBucketLifecycleConfigurationCommand, HeadObjectCommand, GetObjectCommand, DeleteObjectCommand, HeadBucketCommand, PutObjectCommand, ListObjectsV2Command, CopyObjectCommand, CreateMultipartUploadCommand, UploadPartCommand, CompleteMultipartUploadCommand, AbortMultipartUploadCommand, DeleteObjectsCommand } from '@aws-sdk/client-s3';
|
|
3
3
|
import { Upload } from '@aws-sdk/lib-storage';
|
|
4
|
-
import { PassThrough, Readable } from 'stream';
|
|
4
|
+
import { PassThrough, Transform, pipeline, Readable } from 'stream';
|
|
5
5
|
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
|
|
6
6
|
import { createHash } from 'crypto';
|
|
7
7
|
|
|
@@ -51,7 +51,8 @@ var STORAGE_ERROR_CODES = {
|
|
|
51
51
|
STORAGE_PRECONDITION_FAILED: "STORAGE_PRECONDITION_FAILED",
|
|
52
52
|
STORAGE_BUCKET_NOT_FOUND: "STORAGE_BUCKET_NOT_FOUND",
|
|
53
53
|
STORAGE_BUCKET_ACCESS_DENIED: "STORAGE_BUCKET_ACCESS_DENIED",
|
|
54
|
-
STORAGE_BUCKET_REGION_MISMATCH: "STORAGE_BUCKET_REGION_MISMATCH"
|
|
54
|
+
STORAGE_BUCKET_REGION_MISMATCH: "STORAGE_BUCKET_REGION_MISMATCH",
|
|
55
|
+
STORAGE_REQUEST_ABORTED: "STORAGE_REQUEST_ABORTED"
|
|
55
56
|
};
|
|
56
57
|
|
|
57
58
|
// src/server/errors/storage-error-messages.ts
|
|
@@ -81,8 +82,10 @@ var STORAGE_ERROR_MESSAGES = {
|
|
|
81
82
|
[STORAGE_ERROR_CODES.STORAGE_PRECONDITION_FAILED]: "The object does not satisfy the supplied precondition",
|
|
82
83
|
[STORAGE_ERROR_CODES.STORAGE_BUCKET_NOT_FOUND]: "The bucket does not exist",
|
|
83
84
|
[STORAGE_ERROR_CODES.STORAGE_BUCKET_ACCESS_DENIED]: "The bucket cannot be accessed with these credentials (it may also not exist)",
|
|
84
|
-
[STORAGE_ERROR_CODES.STORAGE_BUCKET_REGION_MISMATCH]: "The bucket belongs to a different region than the configured one"
|
|
85
|
+
[STORAGE_ERROR_CODES.STORAGE_BUCKET_REGION_MISMATCH]: "The bucket belongs to a different region than the configured one",
|
|
86
|
+
[STORAGE_ERROR_CODES.STORAGE_REQUEST_ABORTED]: "The call was aborted by its AbortSignal"
|
|
85
87
|
};
|
|
88
|
+
var CLIENT_CLOSED_REQUEST = 499;
|
|
86
89
|
var STORAGE_ERROR_STATUS = {
|
|
87
90
|
[STORAGE_ERROR_CODES.STORAGE_NOT_CONFIGURED]: HttpStatus.SERVICE_UNAVAILABLE,
|
|
88
91
|
[STORAGE_ERROR_CODES.STORAGE_KEY_INVALID]: HttpStatus.BAD_REQUEST,
|
|
@@ -101,7 +104,7 @@ var STORAGE_ERROR_STATUS = {
|
|
|
101
104
|
[STORAGE_ERROR_CODES.STORAGE_BUCKET_UNDEFINED]: HttpStatus.BAD_REQUEST,
|
|
102
105
|
[STORAGE_ERROR_CODES.STORAGE_MULTIPART_ABORTED]: HttpStatus.INTERNAL_SERVER_ERROR,
|
|
103
106
|
[STORAGE_ERROR_CODES.STORAGE_INVALID_CONFIG]: HttpStatus.INTERNAL_SERVER_ERROR,
|
|
104
|
-
[STORAGE_ERROR_CODES.STORAGE_TIMEOUT]: HttpStatus.
|
|
107
|
+
[STORAGE_ERROR_CODES.STORAGE_TIMEOUT]: HttpStatus.SERVICE_UNAVAILABLE,
|
|
105
108
|
[STORAGE_ERROR_CODES.STORAGE_LIFECYCLE_RULE_INVALID]: HttpStatus.BAD_REQUEST,
|
|
106
109
|
[STORAGE_ERROR_CODES.STORAGE_LIFECYCLE_UNSUPPORTED]: HttpStatus.NOT_IMPLEMENTED,
|
|
107
110
|
[STORAGE_ERROR_CODES.STORAGE_LIFECYCLE_REJECTED]: HttpStatus.UNPROCESSABLE_ENTITY,
|
|
@@ -109,7 +112,8 @@ var STORAGE_ERROR_STATUS = {
|
|
|
109
112
|
[STORAGE_ERROR_CODES.STORAGE_PRECONDITION_FAILED]: HttpStatus.PRECONDITION_FAILED,
|
|
110
113
|
[STORAGE_ERROR_CODES.STORAGE_BUCKET_NOT_FOUND]: HttpStatus.NOT_FOUND,
|
|
111
114
|
[STORAGE_ERROR_CODES.STORAGE_BUCKET_ACCESS_DENIED]: HttpStatus.FORBIDDEN,
|
|
112
|
-
[STORAGE_ERROR_CODES.STORAGE_BUCKET_REGION_MISMATCH]: HttpStatus.INTERNAL_SERVER_ERROR
|
|
115
|
+
[STORAGE_ERROR_CODES.STORAGE_BUCKET_REGION_MISMATCH]: HttpStatus.INTERNAL_SERVER_ERROR,
|
|
116
|
+
[STORAGE_ERROR_CODES.STORAGE_REQUEST_ABORTED]: CLIENT_CLOSED_REQUEST
|
|
113
117
|
};
|
|
114
118
|
|
|
115
119
|
// src/server/errors/storage-exception.ts
|
|
@@ -130,14 +134,17 @@ var StorageException = class extends HttpException {
|
|
|
130
134
|
}
|
|
131
135
|
};
|
|
132
136
|
|
|
137
|
+
// src/server/constants/timer.constants.ts
|
|
138
|
+
var MAX_TIMER_MS = 2147483647;
|
|
139
|
+
|
|
133
140
|
// src/server/utils/header-value.ts
|
|
134
141
|
var HEADER_VALUE = /^[\t \x21-\x7E]*$/;
|
|
135
142
|
var HEADER_NAME = /^[!#$%&'*+.^_`|~0-9a-z-]+$/;
|
|
136
143
|
var VALUE_REASON = "must be a string of tab, space and printable ASCII characters";
|
|
137
144
|
var KEY_REASON = "every key must be a non-empty header name of letters, digits and the characters !#$%&'*+-.^_`|~";
|
|
138
|
-
function refuse(
|
|
145
|
+
function refuse(field2, reason) {
|
|
139
146
|
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_VALIDATION_FAILED, void 0, {
|
|
140
|
-
field,
|
|
147
|
+
field: field2,
|
|
141
148
|
reason
|
|
142
149
|
});
|
|
143
150
|
}
|
|
@@ -145,9 +152,9 @@ function isHeaderValue(value) {
|
|
|
145
152
|
return typeof value === "string" && HEADER_VALUE.test(value);
|
|
146
153
|
}
|
|
147
154
|
function assertHeaderValues(fields) {
|
|
148
|
-
for (const [
|
|
155
|
+
for (const [field2, value] of Object.entries(fields)) {
|
|
149
156
|
if (value !== void 0 && !isHeaderValue(value)) {
|
|
150
|
-
refuse(
|
|
157
|
+
refuse(field2, VALUE_REASON);
|
|
151
158
|
}
|
|
152
159
|
}
|
|
153
160
|
}
|
|
@@ -178,6 +185,34 @@ function isObject(value) {
|
|
|
178
185
|
function isNonEmptyString(value) {
|
|
179
186
|
return typeof value === "string" && value.length > 0;
|
|
180
187
|
}
|
|
188
|
+
var BEFORE_AUTHORITY = /^(?:[a-z][a-z\d+.-]*:)?[/\\]*/i;
|
|
189
|
+
function asParsed(value) {
|
|
190
|
+
return value.replace(/[\t\n\r]/g, "").replace(/^[\p{Cc} ]+/u, "");
|
|
191
|
+
}
|
|
192
|
+
function parsedCarriesCredentials(value) {
|
|
193
|
+
try {
|
|
194
|
+
const url = new URL(value);
|
|
195
|
+
return url.username !== "" || url.password !== "";
|
|
196
|
+
} catch {
|
|
197
|
+
return false;
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
function authorityCarriesCredentials(text) {
|
|
201
|
+
const rest = text.replace(BEFORE_AUTHORITY, "");
|
|
202
|
+
const end = rest.search(/[/?#]/);
|
|
203
|
+
return (end === -1 ? rest : rest.slice(0, end)).includes("@");
|
|
204
|
+
}
|
|
205
|
+
function urlCarriesCredentials(value) {
|
|
206
|
+
return parsedCarriesCredentials(value) || authorityCarriesCredentials(asParsed(value));
|
|
207
|
+
}
|
|
208
|
+
function assertTimerMs(options, name, min) {
|
|
209
|
+
const value = options[name];
|
|
210
|
+
if (value !== void 0 && (typeof value !== "number" || !Number.isInteger(value) || value < min || value > MAX_TIMER_MS)) {
|
|
211
|
+
throw invalid(
|
|
212
|
+
`${name} must be an integer from ${String(min)} to ${String(MAX_TIMER_MS)}` + (min === 0 ? " (0 disables the guard)" : "")
|
|
213
|
+
);
|
|
214
|
+
}
|
|
215
|
+
}
|
|
181
216
|
function validateOptions(options) {
|
|
182
217
|
if (!isObject(options)) {
|
|
183
218
|
throw invalid("options object is required");
|
|
@@ -185,6 +220,12 @@ function validateOptions(options) {
|
|
|
185
220
|
if (!isNonEmptyString(options.endpoint)) {
|
|
186
221
|
throw invalid("options.endpoint must be a non-empty string");
|
|
187
222
|
}
|
|
223
|
+
for (const name of ["endpoint", "publicBaseUrl", "cdnBaseUrl"]) {
|
|
224
|
+
const value = options[name];
|
|
225
|
+
if (typeof value === "string" && urlCarriesCredentials(value)) {
|
|
226
|
+
throw invalid(`options.${name} must not carry credentials; pass them in options.credentials`);
|
|
227
|
+
}
|
|
228
|
+
}
|
|
188
229
|
if (!isNonEmptyString(options.region)) {
|
|
189
230
|
throw invalid("options.region must be a non-empty string");
|
|
190
231
|
}
|
|
@@ -202,10 +243,13 @@ function validateOptions(options) {
|
|
|
202
243
|
if (isObject(multipart) && typeof multipart.partSizeBytes === "number" && multipart.partSizeBytes < MIN_PART_SIZE_BYTES) {
|
|
203
244
|
throw invalid("multipart.partSizeBytes must be >= 5 MB (S3 hard limit)");
|
|
204
245
|
}
|
|
205
|
-
const
|
|
206
|
-
if (
|
|
207
|
-
throw invalid("
|
|
246
|
+
const maxAttempts = options.maxAttempts;
|
|
247
|
+
if (maxAttempts !== void 0 && (typeof maxAttempts !== "number" || !Number.isInteger(maxAttempts) || maxAttempts < 1)) {
|
|
248
|
+
throw invalid("maxAttempts must be an integer >= 1");
|
|
208
249
|
}
|
|
250
|
+
assertTimerMs(options, "requestTimeoutMs", 1);
|
|
251
|
+
assertTimerMs(options, "uploadIdleTimeoutMs", 0);
|
|
252
|
+
assertTimerMs(options, "bodyIdleTimeoutMs", 0);
|
|
209
253
|
if (options.serverSideEncryption === "aws:kms" && !options.kmsKeyId) {
|
|
210
254
|
throw invalid('kmsKeyId is required when serverSideEncryption === "aws:kms"');
|
|
211
255
|
}
|
|
@@ -287,6 +331,7 @@ function applyDefaults(options) {
|
|
|
287
331
|
maxAttempts: options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS,
|
|
288
332
|
requestTimeoutMs,
|
|
289
333
|
uploadIdleTimeoutMs: options.uploadIdleTimeoutMs ?? requestTimeoutMs,
|
|
334
|
+
bodyIdleTimeoutMs: options.bodyIdleTimeoutMs ?? requestTimeoutMs,
|
|
290
335
|
hasCredentials
|
|
291
336
|
};
|
|
292
337
|
Object.defineProperty(resolved, "credentials", {
|
|
@@ -296,6 +341,18 @@ function applyDefaults(options) {
|
|
|
296
341
|
});
|
|
297
342
|
return resolved;
|
|
298
343
|
}
|
|
344
|
+
|
|
345
|
+
// src/server/utils/describe-endpoint.ts
|
|
346
|
+
function describeEndpoint(endpoint) {
|
|
347
|
+
try {
|
|
348
|
+
const url = new URL(endpoint);
|
|
349
|
+
return `${url.protocol}//${url.host}`;
|
|
350
|
+
} catch {
|
|
351
|
+
return "<unparseable endpoint>";
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
// src/server/providers/s3-client.provider.ts
|
|
299
356
|
var S3ClientProvider = class {
|
|
300
357
|
constructor(options) {
|
|
301
358
|
this.options = options;
|
|
@@ -381,7 +438,7 @@ var S3ClientProvider = class {
|
|
|
381
438
|
responseChecksumValidation: this.options.responseChecksumValidation
|
|
382
439
|
};
|
|
383
440
|
this.logger.log(
|
|
384
|
-
`S3Client initialized: endpoint=${this.options.endpoint} region=${this.options.region}`
|
|
441
|
+
`S3Client initialized: endpoint=${describeEndpoint(this.options.endpoint)} region=${this.options.region}`
|
|
385
442
|
);
|
|
386
443
|
return new S3Client(config);
|
|
387
444
|
}
|
|
@@ -450,6 +507,127 @@ KeyResolverService = __decorateClass([
|
|
|
450
507
|
Injectable(),
|
|
451
508
|
__decorateParam(0, Inject(BYMAX_STORAGE_OPTIONS))
|
|
452
509
|
], KeyResolverService);
|
|
510
|
+
|
|
511
|
+
// src/server/utils/describe-error.ts
|
|
512
|
+
var NON_PRINTABLE_REJECTION = "non-printable rejection value";
|
|
513
|
+
function describeError(value) {
|
|
514
|
+
try {
|
|
515
|
+
if (typeof value === "object" && value !== null) {
|
|
516
|
+
const message = value.message;
|
|
517
|
+
if (typeof message === "string") {
|
|
518
|
+
return message;
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
return String(value);
|
|
522
|
+
} catch {
|
|
523
|
+
return NON_PRINTABLE_REJECTION;
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
// src/server/utils/log-value.ts
|
|
528
|
+
var LEFT_LITERAL_BY_JSON = /[\p{Cc}\p{Cf}\p{Zl}\p{Zp}]/gu;
|
|
529
|
+
function logValue(value) {
|
|
530
|
+
const text = typeof value === "string" ? value : describeError(value);
|
|
531
|
+
return JSON.stringify(text).replace(
|
|
532
|
+
LEFT_LITERAL_BY_JSON,
|
|
533
|
+
(char) => {
|
|
534
|
+
const codePoint = Number(char.codePointAt(0));
|
|
535
|
+
const hex = codePoint.toString(16);
|
|
536
|
+
return codePoint > 65535 ? `\\u{${hex}}` : `\\u${hex.padStart(4, "0")}`;
|
|
537
|
+
}
|
|
538
|
+
);
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
// src/server/utils/provider-failure-log.ts
|
|
542
|
+
var SECRET_IN_MESSAGE = /(X-Amz-(?:Signature|Credential|Security-Token)|AWSAccessKeyId|Signature)=[^&\s"']+/gi;
|
|
543
|
+
var MAX_MESSAGE_LENGTH = 300;
|
|
544
|
+
var logger = new Logger("StorageProvider");
|
|
545
|
+
function field(value) {
|
|
546
|
+
return typeof value === "string" || typeof value === "number" ? logValue(value) : "-";
|
|
547
|
+
}
|
|
548
|
+
function describeProviderFailure(err) {
|
|
549
|
+
const failure = typeof err === "object" && err !== null ? err : {};
|
|
550
|
+
const message = describeError(err).replace(SECRET_IN_MESSAGE, "$1=[redacted]");
|
|
551
|
+
return [
|
|
552
|
+
`name=${field(failure.name)}`,
|
|
553
|
+
`code=${field(failure.Code)}`,
|
|
554
|
+
`status=${field(failure.$metadata?.httpStatusCode)}`,
|
|
555
|
+
`requestId=${field(failure.$metadata?.requestId)}`,
|
|
556
|
+
`message=${logValue(message.slice(0, MAX_MESSAGE_LENGTH))}`
|
|
557
|
+
].join(" ");
|
|
558
|
+
}
|
|
559
|
+
function logProviderFailure(err, context) {
|
|
560
|
+
const op = typeof context?.op === "string" ? logValue(context.op) : "-";
|
|
561
|
+
const key = typeof context?.key === "string" ? logValue(context.key) : "-";
|
|
562
|
+
logger.warn(`Provider call failed: op=${op} key=${key} ${describeProviderFailure(err)}`);
|
|
563
|
+
}
|
|
564
|
+
var UNREACHABLE_CODES = /* @__PURE__ */ new Set([
|
|
565
|
+
"ECONNREFUSED",
|
|
566
|
+
"ECONNRESET",
|
|
567
|
+
"ECONNABORTED",
|
|
568
|
+
"ENOTFOUND",
|
|
569
|
+
"EAI_AGAIN",
|
|
570
|
+
"EHOSTUNREACH",
|
|
571
|
+
"ENETUNREACH",
|
|
572
|
+
"EPIPE",
|
|
573
|
+
"ETIMEDOUT"
|
|
574
|
+
]);
|
|
575
|
+
var UNREACHABLE_IN_MESSAGE = new RegExp(
|
|
576
|
+
`^(?:AWS SDK error wrapper for \\w+: )?(?:(?:connect|read|write|getaddrinfo) (?:${[...UNREACHABLE_CODES].join("|")})\\b|socket hang up)`
|
|
577
|
+
);
|
|
578
|
+
var ADDRESSES_FAILED = "AWS SDK error wrapper for AggregateError";
|
|
579
|
+
var CREDENTIAL_CODES = /* @__PURE__ */ new Set([
|
|
580
|
+
"InvalidAccessKeyId",
|
|
581
|
+
"SignatureDoesNotMatch"
|
|
582
|
+
]);
|
|
583
|
+
function cutConnection(failure, noAnswer) {
|
|
584
|
+
const codes = [failure.code, failure.cause?.code];
|
|
585
|
+
if (Array.isArray(failure.errors)) {
|
|
586
|
+
codes.push(...failure.errors.map((inner) => inner.code));
|
|
587
|
+
}
|
|
588
|
+
if (codes.some((code) => typeof code === "string" && UNREACHABLE_CODES.has(code))) {
|
|
589
|
+
return true;
|
|
590
|
+
}
|
|
591
|
+
const message = typeof failure.message === "string" ? failure.message : "";
|
|
592
|
+
return noAnswer && (UNREACHABLE_IN_MESSAGE.test(message) || message.startsWith(ADDRESSES_FAILED));
|
|
593
|
+
}
|
|
594
|
+
function providerFailureReason(err) {
|
|
595
|
+
const failure = err ?? {};
|
|
596
|
+
const status = failure.$metadata?.httpStatusCode;
|
|
597
|
+
if (failure.name === "TimeoutError" && failure.code === "ETIMEDOUT") {
|
|
598
|
+
return "timeout";
|
|
599
|
+
}
|
|
600
|
+
if (cutConnection(failure, status === void 0)) {
|
|
601
|
+
return "unreachable";
|
|
602
|
+
}
|
|
603
|
+
if (failure.name === "TimeoutError") {
|
|
604
|
+
return "timeout";
|
|
605
|
+
}
|
|
606
|
+
if (typeof status === "number" && status >= 500 && status !== 501) {
|
|
607
|
+
return "provider-error";
|
|
608
|
+
}
|
|
609
|
+
const awsCode = failure.Code ?? failure.name;
|
|
610
|
+
if (status === 403 && typeof awsCode === "string" && CREDENTIAL_CODES.has(awsCode)) {
|
|
611
|
+
return "credentials-refused";
|
|
612
|
+
}
|
|
613
|
+
return "rejected";
|
|
614
|
+
}
|
|
615
|
+
function classifyProviderFailure(err) {
|
|
616
|
+
const reason = providerFailureReason(err);
|
|
617
|
+
if (reason === "credentials-refused") {
|
|
618
|
+
return "credentials-refused";
|
|
619
|
+
}
|
|
620
|
+
return reason === "rejected" ? "other" : "unavailable";
|
|
621
|
+
}
|
|
622
|
+
function providerFailureStatus(err) {
|
|
623
|
+
const failure = classifyProviderFailure(err);
|
|
624
|
+
if (failure === "unavailable") {
|
|
625
|
+
return HttpStatus.SERVICE_UNAVAILABLE;
|
|
626
|
+
}
|
|
627
|
+
return failure === "credentials-refused" ? HttpStatus.INTERNAL_SERVER_ERROR : void 0;
|
|
628
|
+
}
|
|
629
|
+
|
|
630
|
+
// src/server/errors/aws-error-mapper.ts
|
|
453
631
|
function mapAwsError(err, context) {
|
|
454
632
|
const e = err ?? {};
|
|
455
633
|
const httpStatus = e.$metadata?.httpStatusCode;
|
|
@@ -460,7 +638,8 @@ function mapAwsError(err, context) {
|
|
|
460
638
|
awsCode,
|
|
461
639
|
httpStatus,
|
|
462
640
|
requestId,
|
|
463
|
-
|
|
641
|
+
// A fixed word, never what the driver or the provider wrote.
|
|
642
|
+
reason: providerFailureReason(err)
|
|
464
643
|
};
|
|
465
644
|
if (awsCode === "NoSuchBucket") {
|
|
466
645
|
return new StorageException(
|
|
@@ -477,7 +656,11 @@ function mapAwsError(err, context) {
|
|
|
477
656
|
);
|
|
478
657
|
}
|
|
479
658
|
if (httpStatus === 304) {
|
|
480
|
-
return new StorageException(
|
|
659
|
+
return new StorageException(
|
|
660
|
+
STORAGE_ERROR_CODES.STORAGE_NOT_MODIFIED,
|
|
661
|
+
HttpStatus.NOT_MODIFIED,
|
|
662
|
+
details
|
|
663
|
+
);
|
|
481
664
|
}
|
|
482
665
|
if (httpStatus === 412) {
|
|
483
666
|
return new StorageException(
|
|
@@ -487,11 +670,17 @@ function mapAwsError(err, context) {
|
|
|
487
670
|
);
|
|
488
671
|
}
|
|
489
672
|
if (e.name === "TimeoutError") {
|
|
490
|
-
|
|
673
|
+
logProviderFailure(err, context);
|
|
674
|
+
return new StorageException(
|
|
675
|
+
STORAGE_ERROR_CODES.STORAGE_TIMEOUT,
|
|
676
|
+
void 0,
|
|
677
|
+
details
|
|
678
|
+
);
|
|
491
679
|
}
|
|
680
|
+
logProviderFailure(err, context);
|
|
492
681
|
return new StorageException(
|
|
493
682
|
STORAGE_ERROR_CODES.STORAGE_PROVIDER_ERROR,
|
|
494
|
-
|
|
683
|
+
providerFailureStatus(err),
|
|
495
684
|
details
|
|
496
685
|
);
|
|
497
686
|
}
|
|
@@ -520,10 +709,18 @@ function mapLifecycleError(err, context) {
|
|
|
520
709
|
const status = e.$metadata?.httpStatusCode ?? 0;
|
|
521
710
|
const details = mapped.getResponse().error.details;
|
|
522
711
|
if (UNSUPPORTED_NAMES.has(name) || UNSUPPORTED_STATUSES.has(status)) {
|
|
523
|
-
return new StorageException(
|
|
712
|
+
return new StorageException(
|
|
713
|
+
STORAGE_ERROR_CODES.STORAGE_LIFECYCLE_UNSUPPORTED,
|
|
714
|
+
void 0,
|
|
715
|
+
details
|
|
716
|
+
);
|
|
524
717
|
}
|
|
525
718
|
if (REJECTED_NAMES.has(name)) {
|
|
526
|
-
return new StorageException(
|
|
719
|
+
return new StorageException(
|
|
720
|
+
STORAGE_ERROR_CODES.STORAGE_LIFECYCLE_REJECTED,
|
|
721
|
+
void 0,
|
|
722
|
+
details
|
|
723
|
+
);
|
|
527
724
|
}
|
|
528
725
|
return mapped;
|
|
529
726
|
}
|
|
@@ -540,9 +737,9 @@ function ruleInvalid(reason) {
|
|
|
540
737
|
reason
|
|
541
738
|
});
|
|
542
739
|
}
|
|
543
|
-
function assertDays(
|
|
740
|
+
function assertDays(field2, value) {
|
|
544
741
|
if (!Number.isInteger(value) || value < 1) {
|
|
545
|
-
throw ruleInvalid(`${
|
|
742
|
+
throw ruleInvalid(`${field2} must be an integer of at least 1`);
|
|
546
743
|
}
|
|
547
744
|
}
|
|
548
745
|
var BucketLifecycleService = class {
|
|
@@ -860,6 +1057,25 @@ async function readStreamUpTo(source, maxBytes) {
|
|
|
860
1057
|
}
|
|
861
1058
|
return Buffer.concat(chunks);
|
|
862
1059
|
}
|
|
1060
|
+
async function readStream(source) {
|
|
1061
|
+
const chunks = [];
|
|
1062
|
+
for await (const chunk of source) {
|
|
1063
|
+
chunks.push(typeof chunk === "string" ? Buffer.from(chunk) : chunk);
|
|
1064
|
+
}
|
|
1065
|
+
return Buffer.concat(chunks);
|
|
1066
|
+
}
|
|
1067
|
+
function toUploadBody(body) {
|
|
1068
|
+
if (typeof body === "string") {
|
|
1069
|
+
return Buffer.from(body, "utf8");
|
|
1070
|
+
}
|
|
1071
|
+
if (body instanceof ArrayBuffer) {
|
|
1072
|
+
return Buffer.from(body);
|
|
1073
|
+
}
|
|
1074
|
+
if (ArrayBuffer.isView(body) && !(body instanceof Uint8Array)) {
|
|
1075
|
+
return Buffer.from(body.buffer, body.byteOffset, body.byteLength);
|
|
1076
|
+
}
|
|
1077
|
+
return body;
|
|
1078
|
+
}
|
|
863
1079
|
|
|
864
1080
|
// src/server/utils/mime-match.ts
|
|
865
1081
|
var SINGLE_MEDIA_TYPE = /^[ \t]*[!#$%&'*+.^_`|~0-9A-Za-z-]+\/[!#$%&'*+.^_`|~0-9A-Za-z-]+[ \t]*(?:;[\t \x21-\x2B\x2D-\x7E]*)?$/;
|
|
@@ -1042,7 +1258,7 @@ var FileScannerService = class {
|
|
|
1042
1258
|
}
|
|
1043
1259
|
const result = await this.scanner.scan(input);
|
|
1044
1260
|
if (result.status === "infected") {
|
|
1045
|
-
this.logger.warn(`Infected file detected: key=${input.key} engine=${result.engine} threat=${result.threat ?? "unknown"}`);
|
|
1261
|
+
this.logger.warn(`Infected file detected: key=${logValue(input.key)} engine=${logValue(result.engine)} threat=${logValue(result.threat ?? "unknown")}`);
|
|
1046
1262
|
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_SCAN_INFECTED, void 0, {
|
|
1047
1263
|
engine: result.engine,
|
|
1048
1264
|
threat: result.threat,
|
|
@@ -1056,7 +1272,7 @@ var FileScannerService = class {
|
|
|
1056
1272
|
details: result.details
|
|
1057
1273
|
});
|
|
1058
1274
|
}
|
|
1059
|
-
this.logger.warn(`Inconclusive scan result accepted: key=${input.key} engine=${result.engine}`);
|
|
1275
|
+
this.logger.warn(`Inconclusive scan result accepted: key=${logValue(input.key)} engine=${logValue(result.engine)}`);
|
|
1060
1276
|
}
|
|
1061
1277
|
return result;
|
|
1062
1278
|
}
|
|
@@ -1066,22 +1282,6 @@ FileScannerService = __decorateClass([
|
|
|
1066
1282
|
__decorateParam(0, Inject(BYMAX_STORAGE_OPTIONS)),
|
|
1067
1283
|
__decorateParam(1, Inject(BYMAX_STORAGE_FILE_SCANNER))
|
|
1068
1284
|
], FileScannerService);
|
|
1069
|
-
|
|
1070
|
-
// src/server/utils/describe-error.ts
|
|
1071
|
-
var NON_PRINTABLE_REJECTION = "non-printable rejection value";
|
|
1072
|
-
function describeError(value) {
|
|
1073
|
-
try {
|
|
1074
|
-
if (typeof value === "object" && value !== null) {
|
|
1075
|
-
const message = value.message;
|
|
1076
|
-
if (typeof message === "string") {
|
|
1077
|
-
return message;
|
|
1078
|
-
}
|
|
1079
|
-
}
|
|
1080
|
-
return String(value);
|
|
1081
|
-
} catch {
|
|
1082
|
-
return NON_PRINTABLE_REJECTION;
|
|
1083
|
-
}
|
|
1084
|
-
}
|
|
1085
1285
|
var SourceIdleError = class extends Error {
|
|
1086
1286
|
constructor(idleMs) {
|
|
1087
1287
|
super(`The upload source delivered no bytes for ${String(idleMs)} ms`);
|
|
@@ -1107,7 +1307,7 @@ function withinIdleTime(next, idleMs) {
|
|
|
1107
1307
|
);
|
|
1108
1308
|
});
|
|
1109
1309
|
}
|
|
1110
|
-
function withIdleTimeout(source, idleMs) {
|
|
1310
|
+
function withIdleTimeout(source, idleMs, toError) {
|
|
1111
1311
|
const iterator = source[Symbol.asyncIterator]();
|
|
1112
1312
|
const wrapper = new Readable({
|
|
1113
1313
|
objectMode: false,
|
|
@@ -1120,7 +1320,7 @@ function withIdleTimeout(source, idleMs) {
|
|
|
1120
1320
|
wrapper.push(step.done === true ? null : step.value);
|
|
1121
1321
|
},
|
|
1122
1322
|
(error) => {
|
|
1123
|
-
wrapper.destroy(error);
|
|
1323
|
+
wrapper.destroy(toError ? toError(error) : error);
|
|
1124
1324
|
}
|
|
1125
1325
|
);
|
|
1126
1326
|
},
|
|
@@ -1138,25 +1338,71 @@ function guardIdleBody(body, idleMs) {
|
|
|
1138
1338
|
}
|
|
1139
1339
|
return idleMs === 0 ? body : withIdleTimeout(body, idleMs);
|
|
1140
1340
|
}
|
|
1341
|
+
var SizeLimitError = class extends Error {
|
|
1342
|
+
constructor(limitBytes, kind) {
|
|
1343
|
+
super(`The upload body is longer than ${String(limitBytes)} bytes`);
|
|
1344
|
+
this.limitBytes = limitBytes;
|
|
1345
|
+
this.kind = kind;
|
|
1346
|
+
this.name = "SizeLimitError";
|
|
1347
|
+
}
|
|
1348
|
+
limitBytes;
|
|
1349
|
+
kind;
|
|
1350
|
+
};
|
|
1351
|
+
function withSizeLimit(source, limitBytes, kind) {
|
|
1352
|
+
let seen = 0;
|
|
1353
|
+
const guard = new Transform({
|
|
1354
|
+
transform(chunk, _encoding, callback) {
|
|
1355
|
+
seen += Buffer.byteLength(chunk);
|
|
1356
|
+
if (seen > limitBytes) {
|
|
1357
|
+
callback(new SizeLimitError(limitBytes, kind));
|
|
1358
|
+
return;
|
|
1359
|
+
}
|
|
1360
|
+
callback(null, chunk);
|
|
1361
|
+
}
|
|
1362
|
+
});
|
|
1363
|
+
pipeline(source, guard, () => void 0);
|
|
1364
|
+
return guard;
|
|
1365
|
+
}
|
|
1366
|
+
|
|
1367
|
+
// src/server/errors/size-limit-exception.ts
|
|
1368
|
+
function sizeLimitException(limitBytes, kind, context) {
|
|
1369
|
+
const bound = kind === "declared" ? { declaredSize: limitBytes } : { maxSize: limitBytes };
|
|
1370
|
+
return new StorageException(STORAGE_ERROR_CODES.STORAGE_SIZE_EXCEEDED, void 0, {
|
|
1371
|
+
...context,
|
|
1372
|
+
...bound
|
|
1373
|
+
});
|
|
1374
|
+
}
|
|
1375
|
+
function mapSizeLimitError(err, context) {
|
|
1376
|
+
if (!(err instanceof SizeLimitError)) {
|
|
1377
|
+
return void 0;
|
|
1378
|
+
}
|
|
1379
|
+
return sizeLimitException(err.limitBytes, err.kind, context);
|
|
1380
|
+
}
|
|
1141
1381
|
|
|
1142
1382
|
// src/server/errors/multipart-error-mapper.ts
|
|
1143
1383
|
var ABORT_FAILED_MESSAGE = "Multipart upload failed and could not be aborted";
|
|
1144
1384
|
function mapMultipartError(err, context) {
|
|
1145
1385
|
if (err instanceof SourceIdleError) {
|
|
1146
|
-
return new StorageException(STORAGE_ERROR_CODES.STORAGE_TIMEOUT,
|
|
1386
|
+
return new StorageException(STORAGE_ERROR_CODES.STORAGE_TIMEOUT, void 0, {
|
|
1147
1387
|
...context,
|
|
1148
1388
|
phase: "source-idle",
|
|
1149
1389
|
idleMs: err.idleMs
|
|
1150
1390
|
});
|
|
1151
1391
|
}
|
|
1392
|
+
const tooLong = mapSizeLimitError(err, context);
|
|
1393
|
+
if (tooLong) {
|
|
1394
|
+
return tooLong;
|
|
1395
|
+
}
|
|
1152
1396
|
const bucketMissing = mapAwsError(err, context);
|
|
1153
1397
|
if (bucketMissing.code === STORAGE_ERROR_CODES.STORAGE_BUCKET_NOT_FOUND) {
|
|
1154
1398
|
return bucketMissing;
|
|
1155
1399
|
}
|
|
1156
1400
|
return new StorageException(
|
|
1157
1401
|
STORAGE_ERROR_CODES.STORAGE_MULTIPART_ABORTED,
|
|
1158
|
-
|
|
1159
|
-
|
|
1402
|
+
// An outage ends the upload as an outage; any other cause is a failure of the upload itself.
|
|
1403
|
+
providerFailureStatus(err),
|
|
1404
|
+
// A fixed word, never what the driver or the provider wrote.
|
|
1405
|
+
{ ...context, reason: providerFailureReason(err) },
|
|
1160
1406
|
// The code's message says the upload was aborted; it is not, when the abort itself failed.
|
|
1161
1407
|
context.abort === "failed" ? ABORT_FAILED_MESSAGE : void 0
|
|
1162
1408
|
);
|
|
@@ -1165,43 +1411,149 @@ function knownStatus(error) {
|
|
|
1165
1411
|
const name = HttpStatus[error.getStatus()];
|
|
1166
1412
|
return name === void 0 ? void 0 : HttpStatus[name];
|
|
1167
1413
|
}
|
|
1168
|
-
function
|
|
1414
|
+
function cleanupErrorCode(error) {
|
|
1415
|
+
return error instanceof StorageException ? error.code : "UNKNOWN";
|
|
1416
|
+
}
|
|
1417
|
+
function describeCleanupFailure(error) {
|
|
1169
1418
|
if (!(error instanceof StorageException)) {
|
|
1170
1419
|
return describeError(error);
|
|
1171
1420
|
}
|
|
1172
|
-
|
|
1173
|
-
const awsMessage = body.error?.details?.awsMessage;
|
|
1174
|
-
return typeof awsMessage === "string" ? `${error.code}: ${awsMessage}` : error.code;
|
|
1421
|
+
return error.code;
|
|
1175
1422
|
}
|
|
1176
1423
|
function withCleanupFailure(scanError, keys, cleanupError) {
|
|
1177
1424
|
if (!(scanError instanceof StorageException)) {
|
|
1178
1425
|
return scanError;
|
|
1179
1426
|
}
|
|
1180
1427
|
const body = scanError.getResponse();
|
|
1181
|
-
return new StorageException(
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1428
|
+
return new StorageException(
|
|
1429
|
+
scanError.code,
|
|
1430
|
+
knownStatus(scanError),
|
|
1431
|
+
{
|
|
1432
|
+
...body.error?.details ?? {},
|
|
1433
|
+
key: keys.key,
|
|
1434
|
+
storedKey: keys.storedKey,
|
|
1435
|
+
cleanup: "failed",
|
|
1436
|
+
cleanupError: cleanupErrorCode(cleanupError)
|
|
1437
|
+
}
|
|
1438
|
+
);
|
|
1439
|
+
}
|
|
1440
|
+
var CLEANUP_DEADLINE_MS = 1e4;
|
|
1441
|
+
function reasonName(signal) {
|
|
1442
|
+
const reason = signal.reason;
|
|
1443
|
+
if (typeof reason === "object" && reason !== null && "name" in reason && typeof reason.name === "string") {
|
|
1444
|
+
return reason.name;
|
|
1445
|
+
}
|
|
1446
|
+
return "AbortError";
|
|
1447
|
+
}
|
|
1448
|
+
function requestAborted(signal, context) {
|
|
1449
|
+
return new StorageException(STORAGE_ERROR_CODES.STORAGE_REQUEST_ABORTED, void 0, {
|
|
1450
|
+
...context,
|
|
1451
|
+
reason: reasonName(signal)
|
|
1452
|
+
});
|
|
1453
|
+
}
|
|
1454
|
+
function throwIfAborted(signal, context) {
|
|
1455
|
+
if (signal?.aborted) {
|
|
1456
|
+
throw requestAborted(signal, context);
|
|
1457
|
+
}
|
|
1458
|
+
}
|
|
1459
|
+
function sendOptions(signal) {
|
|
1460
|
+
return signal ? { abortSignal: signal } : void 0;
|
|
1461
|
+
}
|
|
1462
|
+
function failureOrAbort(signal, context, mapped) {
|
|
1463
|
+
return signal?.aborted ? requestAborted(signal, context) : mapped;
|
|
1464
|
+
}
|
|
1465
|
+
function bindToSignal(client, signal) {
|
|
1466
|
+
let created = Promise.resolve(void 0);
|
|
1467
|
+
const bound = new Proxy(client, {
|
|
1468
|
+
get(target, property) {
|
|
1469
|
+
if (property !== "send") {
|
|
1470
|
+
const value = Reflect.get(target, property, target);
|
|
1471
|
+
return value;
|
|
1472
|
+
}
|
|
1473
|
+
return (command, options) => {
|
|
1474
|
+
if (command instanceof CreateMultipartUploadCommand) {
|
|
1475
|
+
const sent = target.send(command, options);
|
|
1476
|
+
created = sent.then(
|
|
1477
|
+
(response) => response.UploadId,
|
|
1478
|
+
() => void 0
|
|
1479
|
+
);
|
|
1480
|
+
return sent;
|
|
1481
|
+
}
|
|
1482
|
+
return target.send(command, { ...options, abortSignal: signal });
|
|
1483
|
+
};
|
|
1484
|
+
}
|
|
1485
|
+
});
|
|
1486
|
+
return { client: bound, createdUploadId: () => created };
|
|
1487
|
+
}
|
|
1488
|
+
async function within(promise, ms) {
|
|
1489
|
+
let timer;
|
|
1490
|
+
const late = new Promise((resolve) => {
|
|
1491
|
+
timer = setTimeout(() => {
|
|
1492
|
+
resolve(void 0);
|
|
1493
|
+
}, ms);
|
|
1187
1494
|
});
|
|
1495
|
+
try {
|
|
1496
|
+
return await Promise.race([promise, late]);
|
|
1497
|
+
} finally {
|
|
1498
|
+
clearTimeout(timer);
|
|
1499
|
+
}
|
|
1188
1500
|
}
|
|
1189
|
-
async function
|
|
1501
|
+
async function abortable(work, signal, context) {
|
|
1502
|
+
if (!signal) {
|
|
1503
|
+
return work;
|
|
1504
|
+
}
|
|
1505
|
+
if (signal.aborted) {
|
|
1506
|
+
work.catch(() => void 0);
|
|
1507
|
+
throw requestAborted(signal, context);
|
|
1508
|
+
}
|
|
1509
|
+
const release = new AbortController();
|
|
1510
|
+
const aborted = new Promise((_resolve, reject) => {
|
|
1511
|
+
signal.addEventListener(
|
|
1512
|
+
"abort",
|
|
1513
|
+
() => {
|
|
1514
|
+
reject(requestAborted(signal, context));
|
|
1515
|
+
},
|
|
1516
|
+
// Stryker disable next-line BooleanLiteral: `release.abort()` in the `finally` removes the listener, and a signal aborts once, so `once` is not observable (equivalent).
|
|
1517
|
+
{ once: true, signal: release.signal }
|
|
1518
|
+
);
|
|
1519
|
+
});
|
|
1520
|
+
try {
|
|
1521
|
+
return await Promise.race([work, aborted]);
|
|
1522
|
+
} finally {
|
|
1523
|
+
release.abort();
|
|
1524
|
+
work.catch(() => void 0);
|
|
1525
|
+
}
|
|
1526
|
+
}
|
|
1527
|
+
|
|
1528
|
+
// src/server/utils/multipart-abort.ts
|
|
1529
|
+
async function closeMultipartUpload(client, bucket, key, uploadId, onFailure) {
|
|
1190
1530
|
if (uploadId === void 0) {
|
|
1191
1531
|
return {};
|
|
1192
1532
|
}
|
|
1193
1533
|
try {
|
|
1194
1534
|
await client.send(
|
|
1195
|
-
new AbortMultipartUploadCommand({ Bucket: bucket, Key: key, UploadId: uploadId })
|
|
1535
|
+
new AbortMultipartUploadCommand({ Bucket: bucket, Key: key, UploadId: uploadId }),
|
|
1536
|
+
{ abortSignal: AbortSignal.timeout(CLEANUP_DEADLINE_MS) }
|
|
1196
1537
|
);
|
|
1197
1538
|
return { abort: "completed" };
|
|
1198
1539
|
} catch (err) {
|
|
1199
1540
|
if (err?.name === "NoSuchUpload") {
|
|
1200
1541
|
return { abort: "completed" };
|
|
1201
1542
|
}
|
|
1202
|
-
|
|
1543
|
+
onFailure?.(err);
|
|
1544
|
+
return { abort: "failed" };
|
|
1203
1545
|
}
|
|
1204
1546
|
}
|
|
1547
|
+
function closeWhenCreated(client, bucket, key, created, onFailure) {
|
|
1548
|
+
void created.then((uploadId) => closeMultipartUpload(client, bucket, key, uploadId, onFailure));
|
|
1549
|
+
}
|
|
1550
|
+
function abortFailureLogger(logger2, key) {
|
|
1551
|
+
return (err) => {
|
|
1552
|
+
logger2.warn(
|
|
1553
|
+
`Multipart abort failed, the upload stays open on the provider: ${logValue(key)} ${describeProviderFailure(err)}`
|
|
1554
|
+
);
|
|
1555
|
+
};
|
|
1556
|
+
}
|
|
1205
1557
|
async function deleteObjectsChunk(client, bucket, chunk, stripPrefix) {
|
|
1206
1558
|
const deleted = [];
|
|
1207
1559
|
const failed = [];
|
|
@@ -1222,14 +1574,16 @@ async function deleteObjectsChunk(client, bucket, chunk, stripPrefix) {
|
|
|
1222
1574
|
if (failure.Key !== void 0) {
|
|
1223
1575
|
failed.push({
|
|
1224
1576
|
key: stripPrefix(failure.Key),
|
|
1225
|
-
|
|
1577
|
+
// The provider's code, which a caller can branch on; its sentence is not repeated.
|
|
1578
|
+
error: failure.Code ?? "Unknown"
|
|
1226
1579
|
});
|
|
1227
1580
|
}
|
|
1228
1581
|
}
|
|
1229
1582
|
} catch (err) {
|
|
1230
|
-
|
|
1583
|
+
logProviderFailure(err, { op: "deleteMany" });
|
|
1584
|
+
const reason = providerFailureReason(err);
|
|
1231
1585
|
for (const key of chunk) {
|
|
1232
|
-
failed.push({ key: stripPrefix(key), error:
|
|
1586
|
+
failed.push({ key: stripPrefix(key), error: reason });
|
|
1233
1587
|
}
|
|
1234
1588
|
}
|
|
1235
1589
|
return { deleted, failed };
|
|
@@ -1237,7 +1591,12 @@ async function deleteObjectsChunk(client, bucket, chunk, stripPrefix) {
|
|
|
1237
1591
|
function mapBucketCheckError(err, context) {
|
|
1238
1592
|
const failure = err ?? {};
|
|
1239
1593
|
const status = failure.$metadata?.httpStatusCode;
|
|
1240
|
-
const details = {
|
|
1594
|
+
const details = {
|
|
1595
|
+
...context,
|
|
1596
|
+
httpStatus: status,
|
|
1597
|
+
awsCode: failure.name,
|
|
1598
|
+
reason: providerFailureReason(err)
|
|
1599
|
+
};
|
|
1241
1600
|
if (status === 301 || failure.name === "PermanentRedirect") {
|
|
1242
1601
|
return new StorageException(
|
|
1243
1602
|
STORAGE_ERROR_CODES.STORAGE_BUCKET_REGION_MISMATCH,
|
|
@@ -1320,6 +1679,141 @@ function pickUploadStrategy(body, declaredSize, thresholdBytes) {
|
|
|
1320
1679
|
return "single-shot";
|
|
1321
1680
|
}
|
|
1322
1681
|
|
|
1682
|
+
// src/server/utils/upload-body.ts
|
|
1683
|
+
function withBytesBody(options) {
|
|
1684
|
+
const body = toUploadBody(options.body);
|
|
1685
|
+
return isBufferLike(body) ? { ...options, body, size: body.byteLength } : { ...options, body };
|
|
1686
|
+
}
|
|
1687
|
+
function boundStreamSize(body, declaredSize, options) {
|
|
1688
|
+
const limit = declaredSize ?? options.validation?.maxSizeBytes;
|
|
1689
|
+
const singleShot = pickUploadStrategy(body, declaredSize, options.multipart.thresholdBytes) === "single-shot";
|
|
1690
|
+
if (isBufferLike(body) || singleShot || limit === void 0) {
|
|
1691
|
+
return body;
|
|
1692
|
+
}
|
|
1693
|
+
return withSizeLimit(body, limit, declaredSize === void 0 ? "max" : "declared");
|
|
1694
|
+
}
|
|
1695
|
+
|
|
1696
|
+
// src/server/utils/download-body.ts
|
|
1697
|
+
function guardDownloadBody(body, idleMs, context, signal) {
|
|
1698
|
+
if (idleMs === 0) {
|
|
1699
|
+
return body;
|
|
1700
|
+
}
|
|
1701
|
+
return withIdleTimeout(body, idleMs, (error) => {
|
|
1702
|
+
if (error instanceof SourceIdleError) {
|
|
1703
|
+
return new StorageException(STORAGE_ERROR_CODES.STORAGE_TIMEOUT, void 0, {
|
|
1704
|
+
...context,
|
|
1705
|
+
phase: "body-idle",
|
|
1706
|
+
idleMs: error.idleMs
|
|
1707
|
+
});
|
|
1708
|
+
}
|
|
1709
|
+
return signal?.aborted ? requestAborted(signal, context) : error;
|
|
1710
|
+
});
|
|
1711
|
+
}
|
|
1712
|
+
|
|
1713
|
+
// src/server/utils/public-url.ts
|
|
1714
|
+
function buildPublicUrl(options, finalKey, bucket) {
|
|
1715
|
+
const base = trimTrailingSlashes(options.cdnBaseUrl ?? options.publicBaseUrl);
|
|
1716
|
+
return baseCarriesBucket(base, bucket) ? `${base}/${finalKey}` : `${base}/${bucket}/${finalKey}`;
|
|
1717
|
+
}
|
|
1718
|
+
function baseCarriesBucket(base, bucket) {
|
|
1719
|
+
const schemeless = base.replace(/^[a-z][a-z\d+.-]*:\/\//i, "");
|
|
1720
|
+
const pathStart = schemeless.indexOf("/");
|
|
1721
|
+
const authority = pathStart === -1 ? schemeless : schemeless.slice(0, pathStart);
|
|
1722
|
+
const path = pathStart === -1 ? "" : schemeless.slice(pathStart);
|
|
1723
|
+
const leadingHostLabel = authority.split(".")[0];
|
|
1724
|
+
const pathSegments = path.split("/").filter((segment) => segment.length > 0);
|
|
1725
|
+
return leadingHostLabel === bucket || pathSegments.includes(bucket);
|
|
1726
|
+
}
|
|
1727
|
+
|
|
1728
|
+
// src/server/utils/upload-support.ts
|
|
1729
|
+
function normalizeScanBody(body) {
|
|
1730
|
+
return Buffer.isBuffer(body) ? body : Buffer.from(body);
|
|
1731
|
+
}
|
|
1732
|
+
function forwardProgress(uploader, onProgress) {
|
|
1733
|
+
if (!onProgress) return;
|
|
1734
|
+
uploader.on("httpUploadProgress", (event) => {
|
|
1735
|
+
emitProgress(onProgress, event.loaded, event.total, event.part);
|
|
1736
|
+
});
|
|
1737
|
+
}
|
|
1738
|
+
function normalizeSingleShotBody(body) {
|
|
1739
|
+
if (isBufferLike(body)) {
|
|
1740
|
+
return Buffer.isBuffer(body) ? body : Buffer.from(body);
|
|
1741
|
+
}
|
|
1742
|
+
return body;
|
|
1743
|
+
}
|
|
1744
|
+
function buildUploadResult(keyResolver, options, params) {
|
|
1745
|
+
return {
|
|
1746
|
+
key: keyResolver.stripPrefix(params.finalKey),
|
|
1747
|
+
bucket: params.bucket,
|
|
1748
|
+
etag: params.etag ?? "",
|
|
1749
|
+
contentType: params.contentType,
|
|
1750
|
+
publicUrl: buildPublicUrl(options, params.finalKey, params.bucket),
|
|
1751
|
+
multipart: params.multipart,
|
|
1752
|
+
fromIdempotencyCache: false,
|
|
1753
|
+
...params.size !== void 0 ? { size: params.size } : {},
|
|
1754
|
+
...params.versionId !== void 0 ? { versionId: params.versionId } : {}
|
|
1755
|
+
};
|
|
1756
|
+
}
|
|
1757
|
+
function emitProgress(onProgress, loaded, total, part) {
|
|
1758
|
+
onProgress({
|
|
1759
|
+
loaded: loaded ?? 0,
|
|
1760
|
+
...total !== void 0 ? { total } : {},
|
|
1761
|
+
...part !== void 0 ? { part } : {}
|
|
1762
|
+
});
|
|
1763
|
+
}
|
|
1764
|
+
function putObjectHeaders(options, resolved, finalKey, bucket) {
|
|
1765
|
+
return {
|
|
1766
|
+
Bucket: bucket,
|
|
1767
|
+
Key: finalKey,
|
|
1768
|
+
ContentType: options.contentType,
|
|
1769
|
+
CacheControl: buildCacheControl(options.cacheControl, resolved.defaultCacheControl),
|
|
1770
|
+
ContentDisposition: buildContentDisposition(
|
|
1771
|
+
options.contentDisposition,
|
|
1772
|
+
resolved.defaultContentDisposition
|
|
1773
|
+
),
|
|
1774
|
+
// `public-read` only takes effect on buckets that allow ACLs (see header-utils).
|
|
1775
|
+
ACL: buildACL(options.publicRead, resolved.defaultPublicRead),
|
|
1776
|
+
Metadata: options.metadata,
|
|
1777
|
+
...buildSSE(options.serverSideEncryption, options.kmsKeyId, resolved)
|
|
1778
|
+
};
|
|
1779
|
+
}
|
|
1780
|
+
function validationInput(options, finalKey) {
|
|
1781
|
+
return {
|
|
1782
|
+
key: finalKey,
|
|
1783
|
+
body: options.body,
|
|
1784
|
+
contentType: options.contentType,
|
|
1785
|
+
...options.size !== void 0 ? { size: options.size } : {},
|
|
1786
|
+
...options.metadata !== void 0 ? { metadata: options.metadata } : {},
|
|
1787
|
+
cacheControl: options.cacheControl,
|
|
1788
|
+
contentDisposition: options.contentDisposition,
|
|
1789
|
+
kmsKeyId: options.kmsKeyId
|
|
1790
|
+
};
|
|
1791
|
+
}
|
|
1792
|
+
|
|
1793
|
+
// src/server/utils/download-buffer.ts
|
|
1794
|
+
function assertMaxBytes(maxBytes) {
|
|
1795
|
+
if (maxBytes !== void 0 && (typeof maxBytes !== "number" || !Number.isInteger(maxBytes) || maxBytes < 1)) {
|
|
1796
|
+
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_VALIDATION_FAILED, void 0, {
|
|
1797
|
+
reason: "maxBytes must be an integer >= 1"
|
|
1798
|
+
});
|
|
1799
|
+
}
|
|
1800
|
+
}
|
|
1801
|
+
async function readDownloadBody(stream, metadata, maxBytes) {
|
|
1802
|
+
if (maxBytes === void 0) {
|
|
1803
|
+
return readStream(stream);
|
|
1804
|
+
}
|
|
1805
|
+
const context = { key: metadata.key, bucket: metadata.bucket };
|
|
1806
|
+
if (metadata.size > maxBytes) {
|
|
1807
|
+
destroyStream(stream);
|
|
1808
|
+
throw sizeLimitException(maxBytes, "max", { ...context, contentLength: metadata.size });
|
|
1809
|
+
}
|
|
1810
|
+
const buffer = await readStreamUpTo(stream, maxBytes);
|
|
1811
|
+
if (buffer === void 0) {
|
|
1812
|
+
throw sizeLimitException(maxBytes, "max", context);
|
|
1813
|
+
}
|
|
1814
|
+
return buffer;
|
|
1815
|
+
}
|
|
1816
|
+
|
|
1323
1817
|
// src/server/services/storage.service.ts
|
|
1324
1818
|
var MAX_LIST_KEYS = 1e3;
|
|
1325
1819
|
var DELETE_BATCH_LIMIT = 1e3;
|
|
@@ -1363,17 +1857,19 @@ var StorageService = class {
|
|
|
1363
1857
|
}
|
|
1364
1858
|
}
|
|
1365
1859
|
/** The upload itself; `upload()` destroys a stream body when this rejects. */
|
|
1366
|
-
async uploadOwningBody(
|
|
1860
|
+
async uploadOwningBody(requested) {
|
|
1367
1861
|
this.assertConfigured();
|
|
1368
|
-
const rawBody =
|
|
1862
|
+
const rawBody = requested.body;
|
|
1369
1863
|
if (rawBody === void 0 || rawBody === null) {
|
|
1370
1864
|
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_BODY_MISSING);
|
|
1371
1865
|
}
|
|
1866
|
+
const options = withBytesBody(requested);
|
|
1372
1867
|
if (!options.contentType) {
|
|
1373
1868
|
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_CONTENT_TYPE_REQUIRED);
|
|
1374
1869
|
}
|
|
1375
1870
|
const finalKey = this.keyResolver.normalize(options.key);
|
|
1376
1871
|
const bucket = this.resolveBucket(options.bucket);
|
|
1872
|
+
throwIfAborted(options.abortSignal, { key: finalKey, bucket, op: "upload" });
|
|
1377
1873
|
if (options.idempotencyKey) {
|
|
1378
1874
|
const cacheKey = this.idempotencyCache.computeKey(options.idempotencyKey, finalKey);
|
|
1379
1875
|
const cached = this.idempotencyCache.get(cacheKey);
|
|
@@ -1383,32 +1879,28 @@ var StorageService = class {
|
|
|
1383
1879
|
return { ...cached, fromIdempotencyCache: true };
|
|
1384
1880
|
}
|
|
1385
1881
|
}
|
|
1386
|
-
const
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
const body = await this.bufferForPreUploadScan(validated.body, finalKey);
|
|
1882
|
+
const context = { key: finalKey, bucket, op: "upload" };
|
|
1883
|
+
const signal = options.abortSignal;
|
|
1884
|
+
const input = validationInput(options, finalKey);
|
|
1885
|
+
const validated = await abortable(this.validation.validate(input), signal, context);
|
|
1886
|
+
const scanned = await abortable(
|
|
1887
|
+
this.bufferForPreUploadScan(validated.body, finalKey),
|
|
1888
|
+
signal,
|
|
1889
|
+
context
|
|
1890
|
+
);
|
|
1891
|
+
const body = boundStreamSize(scanned, options.size, this.options);
|
|
1397
1892
|
const validatedOptions = { ...options, body };
|
|
1398
|
-
await this.runPreUploadScan(body, finalKey, bucket, options);
|
|
1893
|
+
await abortable(this.runPreUploadScan(body, finalKey, bucket, options), signal, context);
|
|
1894
|
+
throwIfAborted(signal, context);
|
|
1399
1895
|
const strategy = pickUploadStrategy(
|
|
1400
1896
|
validatedOptions.body,
|
|
1401
1897
|
validatedOptions.size,
|
|
1402
1898
|
this.options.multipart.thresholdBytes
|
|
1403
1899
|
);
|
|
1404
1900
|
const result = strategy === "multipart" ? await this.uploadMultipart(validatedOptions, finalKey, bucket) : await this.uploadSingleShot(validatedOptions, finalKey, bucket);
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
this.idempotencyCache.computeKey(options.idempotencyKey, finalKey),
|
|
1409
|
-
result
|
|
1410
|
-
);
|
|
1411
|
-
}
|
|
1901
|
+
const stored = { ...context, stored: true };
|
|
1902
|
+
await abortable(this.runPostUploadScan(finalKey, bucket, options), signal, stored);
|
|
1903
|
+
this.rememberUpload(options, finalKey, result);
|
|
1412
1904
|
return result;
|
|
1413
1905
|
}
|
|
1414
1906
|
/**
|
|
@@ -1447,20 +1939,20 @@ var StorageService = class {
|
|
|
1447
1939
|
}
|
|
1448
1940
|
await this.scanner.scan({
|
|
1449
1941
|
mode: "pre-upload",
|
|
1450
|
-
body:
|
|
1942
|
+
body: normalizeScanBody(body),
|
|
1451
1943
|
key: finalKey,
|
|
1452
1944
|
bucket,
|
|
1453
1945
|
contentType: options.contentType,
|
|
1454
1946
|
...options.size !== void 0 ? { size: options.size } : {}
|
|
1455
1947
|
});
|
|
1456
1948
|
}
|
|
1457
|
-
/**
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1949
|
+
/** Remembers the result of an upload under its idempotency key, when it has one. */
|
|
1950
|
+
rememberUpload(options, finalKey, result) {
|
|
1951
|
+
if (!options.idempotencyKey) return;
|
|
1952
|
+
this.idempotencyCache.set(
|
|
1953
|
+
this.idempotencyCache.computeKey(options.idempotencyKey, finalKey),
|
|
1954
|
+
result
|
|
1955
|
+
);
|
|
1464
1956
|
}
|
|
1465
1957
|
/**
|
|
1466
1958
|
* Runs a post-upload scan. On infection, deletes the just-uploaded object
|
|
@@ -1481,11 +1973,11 @@ var StorageService = class {
|
|
|
1481
1973
|
} catch (scanErr) {
|
|
1482
1974
|
const key = this.keyResolver.stripPrefix(finalKey);
|
|
1483
1975
|
try {
|
|
1484
|
-
await this.delete(key, { bucket });
|
|
1976
|
+
await this.delete(key, { bucket, abortSignal: AbortSignal.timeout(CLEANUP_DEADLINE_MS) });
|
|
1485
1977
|
} catch (deleteErr) {
|
|
1486
1978
|
this.logger.error(
|
|
1487
|
-
`STORAGE_SCAN_CLEANUP_FAILED: the rejected object is still stored: ${finalKey}`,
|
|
1488
|
-
|
|
1979
|
+
`STORAGE_SCAN_CLEANUP_FAILED: the rejected object is still stored: ${logValue(finalKey)}`,
|
|
1980
|
+
logValue(describeCleanupFailure(deleteErr))
|
|
1489
1981
|
);
|
|
1490
1982
|
throw withCleanupFailure(scanErr, { key, storedKey: finalKey }, deleteErr);
|
|
1491
1983
|
}
|
|
@@ -1504,11 +1996,16 @@ var StorageService = class {
|
|
|
1504
1996
|
this.assertConfigured();
|
|
1505
1997
|
const finalKey = this.keyResolver.normalize(key);
|
|
1506
1998
|
const bucket = this.resolveBucket(options?.bucket);
|
|
1999
|
+
const context = { key: finalKey, bucket, op: "head" };
|
|
2000
|
+
throwIfAborted(options?.abortSignal, context);
|
|
1507
2001
|
try {
|
|
1508
|
-
const response = await this.s3Provider.getClient().send(
|
|
2002
|
+
const response = await this.s3Provider.getClient().send(
|
|
2003
|
+
new HeadObjectCommand({ Bucket: bucket, Key: finalKey }),
|
|
2004
|
+
sendOptions(options?.abortSignal)
|
|
2005
|
+
);
|
|
1509
2006
|
return this.toObjectMetadata(response, finalKey, bucket);
|
|
1510
2007
|
} catch (err) {
|
|
1511
|
-
throw
|
|
2008
|
+
throw failureOrAbort(options?.abortSignal, context, mapAwsError(err, context));
|
|
1512
2009
|
}
|
|
1513
2010
|
}
|
|
1514
2011
|
/**
|
|
@@ -1527,7 +2024,12 @@ var StorageService = class {
|
|
|
1527
2024
|
if (err instanceof StorageException && err.code === STORAGE_ERROR_CODES.STORAGE_OBJECT_NOT_FOUND) {
|
|
1528
2025
|
return false;
|
|
1529
2026
|
}
|
|
1530
|
-
|
|
2027
|
+
if (err instanceof StorageException && err.code === STORAGE_ERROR_CODES.STORAGE_REQUEST_ABORTED) {
|
|
2028
|
+
throw err;
|
|
2029
|
+
}
|
|
2030
|
+
this.logger.warn(
|
|
2031
|
+
`exists() \u2014 treating a non-404 error as "false": ${logValue(describeError(err))}`
|
|
2032
|
+
);
|
|
1531
2033
|
return false;
|
|
1532
2034
|
}
|
|
1533
2035
|
}
|
|
@@ -1542,7 +2044,7 @@ var StorageService = class {
|
|
|
1542
2044
|
getPublicUrl(key, options) {
|
|
1543
2045
|
const finalKey = this.keyResolver.normalize(key);
|
|
1544
2046
|
const bucket = this.resolveBucket(options?.bucket);
|
|
1545
|
-
return this.
|
|
2047
|
+
return buildPublicUrl(this.options, finalKey, bucket);
|
|
1546
2048
|
}
|
|
1547
2049
|
/**
|
|
1548
2050
|
* Streams an object. The returned `stream` is a Node `Readable` consumable via
|
|
@@ -1555,9 +2057,15 @@ var StorageService = class {
|
|
|
1555
2057
|
*/
|
|
1556
2058
|
async download(options) {
|
|
1557
2059
|
this.assertConfigured();
|
|
1558
|
-
assertHeaderValues({
|
|
2060
|
+
assertHeaderValues({
|
|
2061
|
+
range: options.range,
|
|
2062
|
+
ifNoneMatch: options.ifNoneMatch,
|
|
2063
|
+
ifMatch: options.ifMatch
|
|
2064
|
+
});
|
|
1559
2065
|
const finalKey = this.keyResolver.normalize(options.key);
|
|
1560
2066
|
const bucket = this.resolveBucket(options.bucket);
|
|
2067
|
+
const context = { key: finalKey, bucket, op: "download" };
|
|
2068
|
+
throwIfAborted(options.abortSignal, context);
|
|
1561
2069
|
try {
|
|
1562
2070
|
const response = await this.s3Provider.getClient().send(
|
|
1563
2071
|
new GetObjectCommand({
|
|
@@ -1566,7 +2074,8 @@ var StorageService = class {
|
|
|
1566
2074
|
Range: options.range,
|
|
1567
2075
|
IfNoneMatch: options.ifNoneMatch,
|
|
1568
2076
|
IfMatch: options.ifMatch
|
|
1569
|
-
})
|
|
2077
|
+
}),
|
|
2078
|
+
sendOptions(options.abortSignal)
|
|
1570
2079
|
);
|
|
1571
2080
|
if (!response.Body) {
|
|
1572
2081
|
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_OBJECT_NOT_FOUND, void 0, {
|
|
@@ -1574,26 +2083,33 @@ var StorageService = class {
|
|
|
1574
2083
|
bucket
|
|
1575
2084
|
});
|
|
1576
2085
|
}
|
|
1577
|
-
const stream =
|
|
2086
|
+
const stream = guardDownloadBody(
|
|
2087
|
+
response.Body,
|
|
2088
|
+
this.options.bodyIdleTimeoutMs,
|
|
2089
|
+
{ key: finalKey, bucket },
|
|
2090
|
+
options.abortSignal
|
|
2091
|
+
);
|
|
1578
2092
|
return { stream, metadata: this.toObjectMetadata(response, finalKey, bucket) };
|
|
1579
2093
|
} catch (err) {
|
|
1580
2094
|
if (err instanceof StorageException) {
|
|
1581
2095
|
throw err;
|
|
1582
2096
|
}
|
|
1583
|
-
throw
|
|
2097
|
+
throw failureOrAbort(options.abortSignal, context, mapAwsError(err, context));
|
|
1584
2098
|
}
|
|
1585
2099
|
}
|
|
1586
2100
|
/**
|
|
1587
|
-
*
|
|
1588
|
-
*
|
|
2101
|
+
* Reads an object fully into memory through the same idle-guarded stream as {@link download}.
|
|
2102
|
+
* Without `maxBytes` the whole object is held, however large; use {@link download} for large ones.
|
|
1589
2103
|
*
|
|
1590
|
-
* @param options - The download request.
|
|
2104
|
+
* @param options - The download request; `maxBytes` is the most bytes to hold.
|
|
1591
2105
|
* @returns The object buffer and its metadata.
|
|
2106
|
+
* @throws StorageException `STORAGE_SIZE_EXCEEDED` past `maxBytes`, by `Content-Length` or by
|
|
2107
|
+
* bytes; `STORAGE_VALIDATION_FAILED` for a `maxBytes` that is not an integer of at least 1.
|
|
1592
2108
|
*/
|
|
1593
2109
|
async downloadBuffer(options) {
|
|
2110
|
+
assertMaxBytes(options.maxBytes);
|
|
1594
2111
|
const { stream, metadata } = await this.download(options);
|
|
1595
|
-
|
|
1596
|
-
return { buffer: Buffer.from(bytes), metadata };
|
|
2112
|
+
return { buffer: await readDownloadBody(stream, metadata, options.maxBytes), metadata };
|
|
1597
2113
|
}
|
|
1598
2114
|
/**
|
|
1599
2115
|
* Deletes an object. Idempotent: a missing object is a no-op (logged as a
|
|
@@ -1606,14 +2122,19 @@ var StorageService = class {
|
|
|
1606
2122
|
this.assertConfigured();
|
|
1607
2123
|
const finalKey = this.keyResolver.normalize(key);
|
|
1608
2124
|
const bucket = this.resolveBucket(options?.bucket);
|
|
2125
|
+
const context = { key: finalKey, bucket, op: "delete" };
|
|
2126
|
+
throwIfAborted(options?.abortSignal, context);
|
|
1609
2127
|
try {
|
|
1610
|
-
await this.s3Provider.getClient().send(
|
|
2128
|
+
await this.s3Provider.getClient().send(
|
|
2129
|
+
new DeleteObjectCommand({ Bucket: bucket, Key: finalKey }),
|
|
2130
|
+
sendOptions(options?.abortSignal)
|
|
2131
|
+
);
|
|
1611
2132
|
} catch (err) {
|
|
1612
|
-
const mapped =
|
|
2133
|
+
const mapped = failureOrAbort(options?.abortSignal, context, mapAwsError(err, context));
|
|
1613
2134
|
if (mapped.code !== STORAGE_ERROR_CODES.STORAGE_OBJECT_NOT_FOUND) {
|
|
1614
2135
|
throw mapped;
|
|
1615
2136
|
}
|
|
1616
|
-
this.logger.warn(`delete() \u2014 key not found (idempotent no-op): ${finalKey}`);
|
|
2137
|
+
this.logger.warn(`delete() \u2014 key not found (idempotent no-op): ${logValue(finalKey)}`);
|
|
1617
2138
|
}
|
|
1618
2139
|
this.idempotencyCache.invalidate(bucket, this.keyResolver.stripPrefix(finalKey));
|
|
1619
2140
|
}
|
|
@@ -1631,10 +2152,12 @@ var StorageService = class {
|
|
|
1631
2152
|
async checkBucket(options) {
|
|
1632
2153
|
this.assertConfigured();
|
|
1633
2154
|
const bucket = this.resolveBucket(options?.bucket);
|
|
2155
|
+
const context = { bucket, op: "checkBucket" };
|
|
2156
|
+
throwIfAborted(options?.abortSignal, context);
|
|
1634
2157
|
try {
|
|
1635
|
-
await this.s3Provider.getClient().send(new HeadBucketCommand({ Bucket: bucket }));
|
|
2158
|
+
await this.s3Provider.getClient().send(new HeadBucketCommand({ Bucket: bucket }), sendOptions(options?.abortSignal));
|
|
1636
2159
|
} catch (err) {
|
|
1637
|
-
throw
|
|
2160
|
+
throw failureOrAbort(options?.abortSignal, context, mapBucketCheckError(err, context));
|
|
1638
2161
|
}
|
|
1639
2162
|
}
|
|
1640
2163
|
/** Throws when the S3 client was never built (missing credentials). */
|
|
@@ -1651,26 +2174,6 @@ var StorageService = class {
|
|
|
1651
2174
|
}
|
|
1652
2175
|
return bucket;
|
|
1653
2176
|
}
|
|
1654
|
-
/** Builds a public URL, avoiding a duplicated bucket segment in the base. */
|
|
1655
|
-
buildPublicUrl(finalKey, bucket) {
|
|
1656
|
-
const base = trimTrailingSlashes(this.options.cdnBaseUrl ?? this.options.publicBaseUrl);
|
|
1657
|
-
return this.baseCarriesBucket(base, bucket) ? `${base}/${finalKey}` : `${base}/${bucket}/${finalKey}`;
|
|
1658
|
-
}
|
|
1659
|
-
/**
|
|
1660
|
-
* Reports whether the base URL already carries the bucket as a real segment —
|
|
1661
|
-
* the leading host label (virtual-hosted-style `bucket.host`) or a path segment
|
|
1662
|
-
* (path-style `/bucket` or `/bucket/…`) — rather than an incidental substring,
|
|
1663
|
-
* so `bucket="test"` is not matched by a `latest` elsewhere in the base.
|
|
1664
|
-
*/
|
|
1665
|
-
baseCarriesBucket(base, bucket) {
|
|
1666
|
-
const schemeless = base.replace(/^[a-z][a-z\d+.-]*:\/\//i, "");
|
|
1667
|
-
const pathStart = schemeless.indexOf("/");
|
|
1668
|
-
const authority = pathStart === -1 ? schemeless : schemeless.slice(0, pathStart);
|
|
1669
|
-
const path = pathStart === -1 ? "" : schemeless.slice(pathStart);
|
|
1670
|
-
const leadingHostLabel = authority.split(".")[0];
|
|
1671
|
-
const pathSegments = path.split("/").filter((segment) => segment.length > 0);
|
|
1672
|
-
return leadingHostLabel === bucket || pathSegments.includes(bucket);
|
|
1673
|
-
}
|
|
1674
2177
|
/** Maps an S3 metadata response onto the public `ObjectMetadata` shape. */
|
|
1675
2178
|
toObjectMetadata(response, finalKey, bucket) {
|
|
1676
2179
|
return {
|
|
@@ -1691,27 +2194,16 @@ var StorageService = class {
|
|
|
1691
2194
|
async uploadSingleShot(options, finalKey, bucket) {
|
|
1692
2195
|
const total = options.size ?? getBodySize(options.body);
|
|
1693
2196
|
const input = {
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
ContentType: options.contentType,
|
|
1698
|
-
ContentLength: total,
|
|
1699
|
-
CacheControl: buildCacheControl(options.cacheControl, this.options.defaultCacheControl),
|
|
1700
|
-
ContentDisposition: buildContentDisposition(
|
|
1701
|
-
options.contentDisposition,
|
|
1702
|
-
this.options.defaultContentDisposition
|
|
1703
|
-
),
|
|
1704
|
-
// `public-read` only takes effect on buckets that allow ACLs (see header-utils).
|
|
1705
|
-
ACL: buildACL(options.publicRead, this.options.defaultPublicRead),
|
|
1706
|
-
Metadata: options.metadata,
|
|
1707
|
-
...buildSSE(options.serverSideEncryption, options.kmsKeyId, this.options)
|
|
2197
|
+
...putObjectHeaders(options, this.options, finalKey, bucket),
|
|
2198
|
+
Body: normalizeSingleShotBody(options.body),
|
|
2199
|
+
ContentLength: total
|
|
1708
2200
|
};
|
|
1709
2201
|
try {
|
|
1710
|
-
const response = await this.s3Provider.getClient().send(new PutObjectCommand(input));
|
|
2202
|
+
const response = await this.s3Provider.getClient().send(new PutObjectCommand(input), sendOptions(options.abortSignal));
|
|
1711
2203
|
if (options.onProgress) {
|
|
1712
|
-
|
|
2204
|
+
emitProgress(options.onProgress, total, total);
|
|
1713
2205
|
}
|
|
1714
|
-
return this.
|
|
2206
|
+
return buildUploadResult(this.keyResolver, this.options, {
|
|
1715
2207
|
finalKey,
|
|
1716
2208
|
bucket,
|
|
1717
2209
|
etag: response.ETag,
|
|
@@ -1721,7 +2213,8 @@ var StorageService = class {
|
|
|
1721
2213
|
multipart: false
|
|
1722
2214
|
});
|
|
1723
2215
|
} catch (err) {
|
|
1724
|
-
|
|
2216
|
+
const context = { key: finalKey, bucket, op: "upload-single" };
|
|
2217
|
+
throw failureOrAbort(options.abortSignal, context, mapAwsError(err, context));
|
|
1725
2218
|
}
|
|
1726
2219
|
}
|
|
1727
2220
|
/**
|
|
@@ -1732,30 +2225,26 @@ var StorageService = class {
|
|
|
1732
2225
|
*/
|
|
1733
2226
|
async uploadMultipart(options, finalKey, bucket) {
|
|
1734
2227
|
const params = {
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
Body: guardIdleBody(options.body, this.options.uploadIdleTimeoutMs),
|
|
1738
|
-
ContentType: options.contentType,
|
|
1739
|
-
CacheControl: buildCacheControl(options.cacheControl, this.options.defaultCacheControl),
|
|
1740
|
-
ContentDisposition: buildContentDisposition(
|
|
1741
|
-
options.contentDisposition,
|
|
1742
|
-
this.options.defaultContentDisposition
|
|
1743
|
-
),
|
|
1744
|
-
ACL: buildACL(options.publicRead, this.options.defaultPublicRead),
|
|
1745
|
-
Metadata: options.metadata,
|
|
1746
|
-
...buildSSE(options.serverSideEncryption, options.kmsKeyId, this.options)
|
|
2228
|
+
...putObjectHeaders(options, this.options, finalKey, bucket),
|
|
2229
|
+
Body: guardIdleBody(options.body, this.options.uploadIdleTimeoutMs)
|
|
1747
2230
|
};
|
|
2231
|
+
const signal = options.abortSignal;
|
|
2232
|
+
const bound = signal ? bindToSignal(this.s3Provider.getClient(), signal) : void 0;
|
|
1748
2233
|
const uploader = new Upload({
|
|
1749
|
-
client: this.s3Provider.getClient(),
|
|
2234
|
+
client: bound?.client ?? this.s3Provider.getClient(),
|
|
1750
2235
|
params,
|
|
1751
2236
|
queueSize: this.options.multipart.queueSize,
|
|
1752
2237
|
partSize: this.options.multipart.partSizeBytes,
|
|
1753
2238
|
leavePartsOnError: true
|
|
1754
2239
|
});
|
|
1755
|
-
|
|
2240
|
+
forwardProgress(uploader, options.onProgress);
|
|
2241
|
+
const abortUpload = () => {
|
|
2242
|
+
void uploader.abort();
|
|
2243
|
+
};
|
|
2244
|
+
signal?.addEventListener("abort", abortUpload, { once: true });
|
|
1756
2245
|
try {
|
|
1757
2246
|
const response = await uploader.done();
|
|
1758
|
-
return this.
|
|
2247
|
+
return buildUploadResult(this.keyResolver, this.options, {
|
|
1759
2248
|
finalKey,
|
|
1760
2249
|
bucket,
|
|
1761
2250
|
etag: response.ETag,
|
|
@@ -1765,54 +2254,41 @@ var StorageService = class {
|
|
|
1765
2254
|
multipart: true
|
|
1766
2255
|
});
|
|
1767
2256
|
} catch (err) {
|
|
1768
|
-
return this.failMultipart(err, finalKey, bucket,
|
|
2257
|
+
return await this.failMultipart(err, finalKey, bucket, {
|
|
2258
|
+
uploadId: uploader.uploadId,
|
|
2259
|
+
signal,
|
|
2260
|
+
createdUploadId: bound?.createdUploadId
|
|
2261
|
+
});
|
|
2262
|
+
} finally {
|
|
2263
|
+
signal?.removeEventListener("abort", abortUpload);
|
|
1769
2264
|
}
|
|
1770
2265
|
}
|
|
1771
|
-
/**
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
2266
|
+
/**
|
|
2267
|
+
* Aborts the multipart upload that failed, then throws the original failure with the outcome. Whether
|
|
2268
|
+
* the failure is an abort is decided first: a signal that fires during the cleanup does not turn a
|
|
2269
|
+
* real failure into one. An upload id the provider answered late is waited for, within the cleanup
|
|
2270
|
+
* deadline, so that an abort during `CreateMultipartUpload` still aborts the upload it created.
|
|
2271
|
+
*/
|
|
2272
|
+
async failMultipart(err, finalKey, bucket, upload) {
|
|
2273
|
+
const abortedBy = upload.signal?.aborted === true ? upload.signal : void 0;
|
|
2274
|
+
const client = this.s3Provider.getClient();
|
|
2275
|
+
const lookup = abortedBy ? upload.createdUploadId?.() : void 0;
|
|
2276
|
+
let uploadId = upload.uploadId;
|
|
2277
|
+
if (uploadId === void 0 && lookup) {
|
|
2278
|
+
uploadId = await within(lookup, CLEANUP_DEADLINE_MS);
|
|
2279
|
+
if (uploadId === void 0) {
|
|
2280
|
+
closeWhenCreated(client, bucket, finalKey, lookup, abortFailureLogger(this.logger, finalKey));
|
|
2281
|
+
}
|
|
2282
|
+
}
|
|
1780
2283
|
const abort = await closeMultipartUpload(
|
|
1781
|
-
|
|
2284
|
+
client,
|
|
1782
2285
|
bucket,
|
|
1783
2286
|
finalKey,
|
|
1784
|
-
uploadId
|
|
2287
|
+
uploadId,
|
|
2288
|
+
abortFailureLogger(this.logger, finalKey)
|
|
1785
2289
|
);
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
/** Converts a `Uint8Array` body to a `Buffer` for SDK safety; passes others through. */
|
|
1789
|
-
normalizeSingleShotBody(body) {
|
|
1790
|
-
if (isBufferLike(body)) {
|
|
1791
|
-
return Buffer.isBuffer(body) ? body : Buffer.from(body);
|
|
1792
|
-
}
|
|
1793
|
-
return body;
|
|
1794
|
-
}
|
|
1795
|
-
/** Assembles an `UploadResult`, omitting optional fields that are absent. */
|
|
1796
|
-
buildUploadResult(params) {
|
|
1797
|
-
return {
|
|
1798
|
-
key: this.keyResolver.stripPrefix(params.finalKey),
|
|
1799
|
-
bucket: params.bucket,
|
|
1800
|
-
etag: params.etag ?? "",
|
|
1801
|
-
contentType: params.contentType,
|
|
1802
|
-
publicUrl: this.buildPublicUrl(params.finalKey, params.bucket),
|
|
1803
|
-
multipart: params.multipart,
|
|
1804
|
-
fromIdempotencyCache: false,
|
|
1805
|
-
...params.size !== void 0 ? { size: params.size } : {},
|
|
1806
|
-
...params.versionId !== void 0 ? { versionId: params.versionId } : {}
|
|
1807
|
-
};
|
|
1808
|
-
}
|
|
1809
|
-
/** Emits a progress event, omitting `total`/`part` when they are unknown. */
|
|
1810
|
-
emitProgress(onProgress, loaded, total, part) {
|
|
1811
|
-
onProgress({
|
|
1812
|
-
loaded: loaded ?? 0,
|
|
1813
|
-
...total !== void 0 ? { total } : {},
|
|
1814
|
-
...part !== void 0 ? { part } : {}
|
|
1815
|
-
});
|
|
2290
|
+
const context = { key: finalKey, bucket, op: "upload-multipart", ...abort };
|
|
2291
|
+
throw abortedBy ? requestAborted(abortedBy, context) : mapMultipartError(err, context);
|
|
1816
2292
|
}
|
|
1817
2293
|
/**
|
|
1818
2294
|
* Lists objects in the bucket as a single page. Applies the global key prefix
|
|
@@ -1828,6 +2304,8 @@ var StorageService = class {
|
|
|
1828
2304
|
const bucket = this.resolveBucket(options.bucket);
|
|
1829
2305
|
const maxKeys = Math.min(options.maxKeys ?? MAX_LIST_KEYS, MAX_LIST_KEYS);
|
|
1830
2306
|
const prefix = options.prefix ? this.keyResolver.normalize(options.prefix) : this.keyResolver.getPrefix();
|
|
2307
|
+
const context = { bucket, prefix, op: "list" };
|
|
2308
|
+
throwIfAborted(options.abortSignal, context);
|
|
1831
2309
|
try {
|
|
1832
2310
|
const response = await this.s3Provider.getClient().send(
|
|
1833
2311
|
new ListObjectsV2Command({
|
|
@@ -1836,11 +2314,12 @@ var StorageService = class {
|
|
|
1836
2314
|
MaxKeys: maxKeys,
|
|
1837
2315
|
ContinuationToken: options.continuationToken,
|
|
1838
2316
|
Delimiter: options.delimiter
|
|
1839
|
-
})
|
|
2317
|
+
}),
|
|
2318
|
+
sendOptions(options.abortSignal)
|
|
1840
2319
|
);
|
|
1841
2320
|
return toListResult(response, (key) => this.keyResolver.stripPrefix(key));
|
|
1842
2321
|
} catch (err) {
|
|
1843
|
-
throw
|
|
2322
|
+
throw failureOrAbort(options.abortSignal, context, mapAwsError(err, context));
|
|
1844
2323
|
}
|
|
1845
2324
|
}
|
|
1846
2325
|
/**
|