@bymax-one/nest-storage 1.1.12 → 1.1.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -1
- package/README.md +4 -4
- package/dist/server/index.cjs +179 -77
- package/dist/server/index.d.cts +6 -1
- package/dist/server/index.d.ts +6 -1
- package/dist/server/index.mjs +180 -78
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -13,6 +13,29 @@ note on each entry is the compatibility contract — read it, not the number.
|
|
|
13
13
|
|
|
14
14
|
## [Unreleased]
|
|
15
15
|
|
|
16
|
+
## [1.1.13] - 2026-11-16
|
|
17
|
+
|
|
18
|
+
### Security
|
|
19
|
+
|
|
20
|
+
- **A failed call no longer tells its client what the driver or the provider said.** The body of a
|
|
21
|
+
`StorageException` carried `details.awsMessage`, and for a failed multipart abort `abortError`, a
|
|
22
|
+
failed post-upload delete `cleanupError`, and `deleteMany` the text of each failure. For a provider
|
|
23
|
+
that could not be reached that text is the driver's own, such as `connect ECONNREFUSED 10.0.3.7:9000`,
|
|
24
|
+
and a host that returns `getResponse()` unchanged (Nest's default) handed a caller the address and
|
|
25
|
+
port of the storage service. The body now holds a fixed `details.reason` beside `awsCode`, `httpStatus`
|
|
26
|
+
and `requestId`, which stay. What the driver and the provider said is logged for a provider failure, as a warning
|
|
27
|
+
that starts `Provider call failed` (or `Multipart abort failed`), from named fields only: the error's
|
|
28
|
+
name, the provider's code, the status, the request id and the message, with a signature or an access
|
|
29
|
+
key id in it masked and its length bounded. The error object is not attached to the exception.
|
|
30
|
+
`deleteMany`'s `failed[].error` is the provider's error code for a key it refused, or the fixed reason
|
|
31
|
+
when the whole request failed.
|
|
32
|
+
|
|
33
|
+
**Apply to a derived backend:** a host that read `details.awsMessage`, `abortError` or the text in
|
|
34
|
+
`cleanupError` finds that text in the library's warn log instead; the exception no longer carries it,
|
|
35
|
+
so there is nothing to forward to a client by mistake. A host that matched on that text matches
|
|
36
|
+
`details.reason` (`unreachable`, `timeout`, `provider-error`, `credentials-refused` or `rejected`) or
|
|
37
|
+
`awsCode`.
|
|
38
|
+
|
|
16
39
|
## [1.1.12] - 2026-11-14
|
|
17
40
|
|
|
18
41
|
### Added
|
|
@@ -826,4 +849,5 @@ have regressed from. They are kept because the reasoning is worth having.
|
|
|
826
849
|
[1.1.10]: https://github.com/bymaxone/nest-storage/compare/v1.1.9...v1.1.10
|
|
827
850
|
[1.1.11]: https://github.com/bymaxone/nest-storage/compare/v1.1.10...v1.1.11
|
|
828
851
|
[1.1.12]: https://github.com/bymaxone/nest-storage/compare/v1.1.11...v1.1.12
|
|
829
|
-
[
|
|
852
|
+
[1.1.13]: https://github.com/bymaxone/nest-storage/compare/v1.1.12...v1.1.13
|
|
853
|
+
[Unreleased]: https://github.com/bymaxone/nest-storage/compare/v1.1.13...HEAD
|
package/README.md
CHANGED
|
@@ -598,8 +598,8 @@ bodies are not affected, and `post-upload` hands the scanner the key only, no bo
|
|
|
598
598
|
when the scanner rejects it, so the library deletes it and rejects `upload()` with the verdict
|
|
599
599
|
(`STORAGE_SCAN_INFECTED` or `STORAGE_SCAN_INCONCLUSIVE`). If that delete fails the verdict is still the
|
|
600
600
|
error, and `details` carries `cleanup: 'failed'`, the `key` you passed, the `storedKey` (with any
|
|
601
|
-
`keyPrefix`, as a bucket listing shows it) and the delete failure in `cleanupError`; the
|
|
602
|
-
|
|
601
|
+
`keyPrefix`, as a bucket listing shows it) and the code of the delete failure in `cleanupError`; the
|
|
602
|
+
failure, with what the provider said, is logged at error level as `STORAGE_SCAN_CLEANUP_FAILED`. With no `details.cleanup` the object was
|
|
603
603
|
removed; with `cleanup: 'failed'` it is **still stored** and has to be removed by other means.
|
|
604
604
|
|
|
605
605
|
---
|
|
@@ -760,12 +760,12 @@ All errors are thrown as `StorageException extends HttpException`. The response
|
|
|
760
760
|
| `STORAGE_SCAN_INFECTED` | 422 | Scanner returned `'infected'` (`details.threat`); `details.cleanup: 'failed'` means the object is still stored |
|
|
761
761
|
| `STORAGE_SCAN_INCONCLUSIVE` | 422 | Scanner returned `'unknown'` and `rejectOnUnknown: true` |
|
|
762
762
|
| `STORAGE_OBJECT_NOT_FOUND` | 404 | `head()`, `download()`, or `copy()` on a nonexistent key |
|
|
763
|
-
| `STORAGE_PROVIDER_ERROR` | 502 | The provider rejected the request (`AccessControlListNotSupported`, an `AccessDenied` on an object, any other rejection). **503** when it could not be reached (connection refused or reset, DNS failure) or answered with a server error (`SlowDown`, 500, 502, 504; not 501), which is worth retrying; **500** when it refused the credentials (`InvalidAccessKeyId`, `SignatureDoesNotMatch`), a deployment fault that a retry does not fix. `details` carries `awsCode`, `
|
|
763
|
+
| `STORAGE_PROVIDER_ERROR` | 502 | The provider rejected the request (`AccessControlListNotSupported`, an `AccessDenied` on an object, any other rejection). **503** when it could not be reached (connection refused or reset, DNS failure) or answered with a server error (`SlowDown`, 500, 502, 504; not 501), which is worth retrying; **500** when it refused the credentials (`InvalidAccessKeyId`, `SignatureDoesNotMatch`), a deployment fault that a retry does not fix. `details` carries `awsCode`, `httpStatus`, `requestId` and a fixed `reason` (`unreachable`, `provider-error`, `credentials-refused` or `rejected`; never the text of the driver or of the provider) plus the operation context (`op`, `bucket`, and `key` or `prefix`) |
|
|
764
764
|
| `STORAGE_SIGNED_URL_TTL_INVALID` | 400 | Per-request `ttlSeconds` ≤ 0 |
|
|
765
765
|
| `STORAGE_PART_TOO_SMALL` | 400 | Never thrown: a `multipart.partSizeBytes` below 5 MiB fails at init as `STORAGE_INVALID_CONFIG` |
|
|
766
766
|
| `STORAGE_INVALID_PART_COUNT` | 400 | `getMultipartUploadUrls()` with `parts` not an integer from 1 to 10 000 (`details.provided`, `details.reason`) |
|
|
767
767
|
| `STORAGE_BUCKET_UNDEFINED` | 400 | Operation without a bucket and no default in config |
|
|
768
|
-
| `STORAGE_MULTIPART_ABORTED` | 500 | A multipart upload failed (**503** when the cause is an outage, as for `STORAGE_PROVIDER_ERROR`). `details.abort` is `'completed'` when the library aborted it, `'failed'`
|
|
768
|
+
| `STORAGE_MULTIPART_ABORTED` | 500 | A multipart upload failed (**503** when the cause is an outage, as for `STORAGE_PROVIDER_ERROR`). `details.abort` is `'completed'` when the library aborted it, `'failed'` when the abort itself failed (logged as a warning, with what it threw): that upload then stays open on the provider until a lifecycle rule removes it |
|
|
769
769
|
| `STORAGE_INVALID_CONFIG` | 500 | `BymaxStorageModuleOptions` validation failed at initialization |
|
|
770
770
|
| `STORAGE_TIMEOUT` | 503 | An attempt got no response headers within `requestTimeoutMs`, a multipart source stayed silent for `uploadIdleTimeoutMs` (`details.phase: 'source-idle'`), or a download body stayed silent for `bodyIdleTimeoutMs` (`details.phase: 'body-idle'`) |
|
|
771
771
|
| `STORAGE_LIFECYCLE_RULE_INVALID` | 400 | `BucketLifecycleService` input rejected: bad `id`, day count or `prefix`, or an id naming a rule outside `keyPrefix` (`details.reason`) |
|
package/dist/server/index.cjs
CHANGED
|
@@ -144,9 +144,9 @@ var HEADER_VALUE = /^[\t \x21-\x7E]*$/;
|
|
|
144
144
|
var HEADER_NAME = /^[!#$%&'*+.^_`|~0-9a-z-]+$/;
|
|
145
145
|
var VALUE_REASON = "must be a string of tab, space and printable ASCII characters";
|
|
146
146
|
var KEY_REASON = "every key must be a non-empty header name of letters, digits and the characters !#$%&'*+-.^_`|~";
|
|
147
|
-
function refuse(
|
|
147
|
+
function refuse(field2, reason) {
|
|
148
148
|
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_VALIDATION_FAILED, void 0, {
|
|
149
|
-
field,
|
|
149
|
+
field: field2,
|
|
150
150
|
reason
|
|
151
151
|
});
|
|
152
152
|
}
|
|
@@ -154,9 +154,9 @@ function isHeaderValue(value) {
|
|
|
154
154
|
return typeof value === "string" && HEADER_VALUE.test(value);
|
|
155
155
|
}
|
|
156
156
|
function assertHeaderValues(fields) {
|
|
157
|
-
for (const [
|
|
157
|
+
for (const [field2, value] of Object.entries(fields)) {
|
|
158
158
|
if (value !== void 0 && !isHeaderValue(value)) {
|
|
159
|
-
refuse(
|
|
159
|
+
refuse(field2, VALUE_REASON);
|
|
160
160
|
}
|
|
161
161
|
}
|
|
162
162
|
}
|
|
@@ -509,6 +509,60 @@ KeyResolverService = __decorateClass([
|
|
|
509
509
|
common.Injectable(),
|
|
510
510
|
__decorateParam(0, common.Inject(BYMAX_STORAGE_OPTIONS))
|
|
511
511
|
], KeyResolverService);
|
|
512
|
+
|
|
513
|
+
// src/server/utils/describe-error.ts
|
|
514
|
+
var NON_PRINTABLE_REJECTION = "non-printable rejection value";
|
|
515
|
+
function describeError(value) {
|
|
516
|
+
try {
|
|
517
|
+
if (typeof value === "object" && value !== null) {
|
|
518
|
+
const message = value.message;
|
|
519
|
+
if (typeof message === "string") {
|
|
520
|
+
return message;
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
return String(value);
|
|
524
|
+
} catch {
|
|
525
|
+
return NON_PRINTABLE_REJECTION;
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
// src/server/utils/log-value.ts
|
|
530
|
+
var LEFT_LITERAL_BY_JSON = /[\p{Cc}\p{Cf}\p{Zl}\p{Zp}]/gu;
|
|
531
|
+
function logValue(value) {
|
|
532
|
+
const text = typeof value === "string" ? value : describeError(value);
|
|
533
|
+
return JSON.stringify(text).replace(
|
|
534
|
+
LEFT_LITERAL_BY_JSON,
|
|
535
|
+
(char) => {
|
|
536
|
+
const codePoint = Number(char.codePointAt(0));
|
|
537
|
+
const hex = codePoint.toString(16);
|
|
538
|
+
return codePoint > 65535 ? `\\u{${hex}}` : `\\u${hex.padStart(4, "0")}`;
|
|
539
|
+
}
|
|
540
|
+
);
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
// src/server/utils/provider-failure-log.ts
|
|
544
|
+
var SECRET_IN_MESSAGE = /(X-Amz-(?:Signature|Credential|Security-Token)|AWSAccessKeyId|Signature)=[^&\s"']+/gi;
|
|
545
|
+
var MAX_MESSAGE_LENGTH = 300;
|
|
546
|
+
var logger = new common.Logger("StorageProvider");
|
|
547
|
+
function field(value) {
|
|
548
|
+
return typeof value === "string" || typeof value === "number" ? logValue(value) : "-";
|
|
549
|
+
}
|
|
550
|
+
function describeProviderFailure(err) {
|
|
551
|
+
const failure = typeof err === "object" && err !== null ? err : {};
|
|
552
|
+
const message = describeError(err).replace(SECRET_IN_MESSAGE, "$1=[redacted]");
|
|
553
|
+
return [
|
|
554
|
+
`name=${field(failure.name)}`,
|
|
555
|
+
`code=${field(failure.Code)}`,
|
|
556
|
+
`status=${field(failure.$metadata?.httpStatusCode)}`,
|
|
557
|
+
`requestId=${field(failure.$metadata?.requestId)}`,
|
|
558
|
+
`message=${logValue(message.slice(0, MAX_MESSAGE_LENGTH))}`
|
|
559
|
+
].join(" ");
|
|
560
|
+
}
|
|
561
|
+
function logProviderFailure(err, context) {
|
|
562
|
+
const op = typeof context?.op === "string" ? logValue(context.op) : "-";
|
|
563
|
+
const key = typeof context?.key === "string" ? logValue(context.key) : "-";
|
|
564
|
+
logger.warn(`Provider call failed: op=${op} key=${key} ${describeProviderFailure(err)}`);
|
|
565
|
+
}
|
|
512
566
|
var UNREACHABLE_CODES = /* @__PURE__ */ new Set([
|
|
513
567
|
"ECONNREFUSED",
|
|
514
568
|
"ECONNRESET",
|
|
@@ -524,7 +578,10 @@ var UNREACHABLE_IN_MESSAGE = new RegExp(
|
|
|
524
578
|
`^(?:AWS SDK error wrapper for \\w+: )?(?:(?:connect|read|write|getaddrinfo) (?:${[...UNREACHABLE_CODES].join("|")})\\b|socket hang up)`
|
|
525
579
|
);
|
|
526
580
|
var ADDRESSES_FAILED = "AWS SDK error wrapper for AggregateError";
|
|
527
|
-
var CREDENTIAL_CODES = /* @__PURE__ */ new Set([
|
|
581
|
+
var CREDENTIAL_CODES = /* @__PURE__ */ new Set([
|
|
582
|
+
"InvalidAccessKeyId",
|
|
583
|
+
"SignatureDoesNotMatch"
|
|
584
|
+
]);
|
|
528
585
|
function cutConnection(failure, noAnswer) {
|
|
529
586
|
const codes = [failure.code, failure.cause?.code];
|
|
530
587
|
if (Array.isArray(failure.errors)) {
|
|
@@ -536,20 +593,33 @@ function cutConnection(failure, noAnswer) {
|
|
|
536
593
|
const message = typeof failure.message === "string" ? failure.message : "";
|
|
537
594
|
return noAnswer && (UNREACHABLE_IN_MESSAGE.test(message) || message.startsWith(ADDRESSES_FAILED));
|
|
538
595
|
}
|
|
539
|
-
function
|
|
596
|
+
function providerFailureReason(err) {
|
|
540
597
|
const failure = err ?? {};
|
|
541
598
|
const status = failure.$metadata?.httpStatusCode;
|
|
542
|
-
if (failure.name === "TimeoutError"
|
|
543
|
-
return "
|
|
599
|
+
if (failure.name === "TimeoutError" && failure.code === "ETIMEDOUT") {
|
|
600
|
+
return "timeout";
|
|
601
|
+
}
|
|
602
|
+
if (cutConnection(failure, status === void 0)) {
|
|
603
|
+
return "unreachable";
|
|
604
|
+
}
|
|
605
|
+
if (failure.name === "TimeoutError") {
|
|
606
|
+
return "timeout";
|
|
544
607
|
}
|
|
545
608
|
if (typeof status === "number" && status >= 500 && status !== 501) {
|
|
546
|
-
return "
|
|
609
|
+
return "provider-error";
|
|
547
610
|
}
|
|
548
611
|
const awsCode = failure.Code ?? failure.name;
|
|
549
612
|
if (status === 403 && typeof awsCode === "string" && CREDENTIAL_CODES.has(awsCode)) {
|
|
550
613
|
return "credentials-refused";
|
|
551
614
|
}
|
|
552
|
-
return "
|
|
615
|
+
return "rejected";
|
|
616
|
+
}
|
|
617
|
+
function classifyProviderFailure(err) {
|
|
618
|
+
const reason = providerFailureReason(err);
|
|
619
|
+
if (reason === "credentials-refused") {
|
|
620
|
+
return "credentials-refused";
|
|
621
|
+
}
|
|
622
|
+
return reason === "rejected" ? "other" : "unavailable";
|
|
553
623
|
}
|
|
554
624
|
function providerFailureStatus(err) {
|
|
555
625
|
const failure = classifyProviderFailure(err);
|
|
@@ -570,7 +640,8 @@ function mapAwsError(err, context) {
|
|
|
570
640
|
awsCode,
|
|
571
641
|
httpStatus,
|
|
572
642
|
requestId,
|
|
573
|
-
|
|
643
|
+
// A fixed word, never what the driver or the provider wrote.
|
|
644
|
+
reason: providerFailureReason(err)
|
|
574
645
|
};
|
|
575
646
|
if (awsCode === "NoSuchBucket") {
|
|
576
647
|
return new StorageException(
|
|
@@ -587,7 +658,11 @@ function mapAwsError(err, context) {
|
|
|
587
658
|
);
|
|
588
659
|
}
|
|
589
660
|
if (httpStatus === 304) {
|
|
590
|
-
return new StorageException(
|
|
661
|
+
return new StorageException(
|
|
662
|
+
STORAGE_ERROR_CODES.STORAGE_NOT_MODIFIED,
|
|
663
|
+
common.HttpStatus.NOT_MODIFIED,
|
|
664
|
+
details
|
|
665
|
+
);
|
|
591
666
|
}
|
|
592
667
|
if (httpStatus === 412) {
|
|
593
668
|
return new StorageException(
|
|
@@ -597,8 +672,14 @@ function mapAwsError(err, context) {
|
|
|
597
672
|
);
|
|
598
673
|
}
|
|
599
674
|
if (e.name === "TimeoutError") {
|
|
600
|
-
|
|
675
|
+
logProviderFailure(err, context);
|
|
676
|
+
return new StorageException(
|
|
677
|
+
STORAGE_ERROR_CODES.STORAGE_TIMEOUT,
|
|
678
|
+
void 0,
|
|
679
|
+
details
|
|
680
|
+
);
|
|
601
681
|
}
|
|
682
|
+
logProviderFailure(err, context);
|
|
602
683
|
return new StorageException(
|
|
603
684
|
STORAGE_ERROR_CODES.STORAGE_PROVIDER_ERROR,
|
|
604
685
|
providerFailureStatus(err),
|
|
@@ -630,10 +711,18 @@ function mapLifecycleError(err, context) {
|
|
|
630
711
|
const status = e.$metadata?.httpStatusCode ?? 0;
|
|
631
712
|
const details = mapped.getResponse().error.details;
|
|
632
713
|
if (UNSUPPORTED_NAMES.has(name) || UNSUPPORTED_STATUSES.has(status)) {
|
|
633
|
-
return new StorageException(
|
|
714
|
+
return new StorageException(
|
|
715
|
+
STORAGE_ERROR_CODES.STORAGE_LIFECYCLE_UNSUPPORTED,
|
|
716
|
+
void 0,
|
|
717
|
+
details
|
|
718
|
+
);
|
|
634
719
|
}
|
|
635
720
|
if (REJECTED_NAMES.has(name)) {
|
|
636
|
-
return new StorageException(
|
|
721
|
+
return new StorageException(
|
|
722
|
+
STORAGE_ERROR_CODES.STORAGE_LIFECYCLE_REJECTED,
|
|
723
|
+
void 0,
|
|
724
|
+
details
|
|
725
|
+
);
|
|
637
726
|
}
|
|
638
727
|
return mapped;
|
|
639
728
|
}
|
|
@@ -650,9 +739,9 @@ function ruleInvalid(reason) {
|
|
|
650
739
|
reason
|
|
651
740
|
});
|
|
652
741
|
}
|
|
653
|
-
function assertDays(
|
|
742
|
+
function assertDays(field2, value) {
|
|
654
743
|
if (!Number.isInteger(value) || value < 1) {
|
|
655
|
-
throw ruleInvalid(`${
|
|
744
|
+
throw ruleInvalid(`${field2} must be an integer of at least 1`);
|
|
656
745
|
}
|
|
657
746
|
}
|
|
658
747
|
exports.BucketLifecycleService = class BucketLifecycleService {
|
|
@@ -1120,38 +1209,6 @@ ValidationService = __decorateClass([
|
|
|
1120
1209
|
__decorateParam(0, common.Inject(BYMAX_STORAGE_OPTIONS)),
|
|
1121
1210
|
__decorateParam(1, common.Inject(BYMAX_STORAGE_UPLOAD_VALIDATORS))
|
|
1122
1211
|
], ValidationService);
|
|
1123
|
-
|
|
1124
|
-
// src/server/utils/describe-error.ts
|
|
1125
|
-
var NON_PRINTABLE_REJECTION = "non-printable rejection value";
|
|
1126
|
-
function describeError(value) {
|
|
1127
|
-
try {
|
|
1128
|
-
if (typeof value === "object" && value !== null) {
|
|
1129
|
-
const message = value.message;
|
|
1130
|
-
if (typeof message === "string") {
|
|
1131
|
-
return message;
|
|
1132
|
-
}
|
|
1133
|
-
}
|
|
1134
|
-
return String(value);
|
|
1135
|
-
} catch {
|
|
1136
|
-
return NON_PRINTABLE_REJECTION;
|
|
1137
|
-
}
|
|
1138
|
-
}
|
|
1139
|
-
|
|
1140
|
-
// src/server/utils/log-value.ts
|
|
1141
|
-
var LEFT_LITERAL_BY_JSON = /[\p{Cc}\p{Cf}\p{Zl}\p{Zp}]/gu;
|
|
1142
|
-
function logValue(value) {
|
|
1143
|
-
const text = typeof value === "string" ? value : describeError(value);
|
|
1144
|
-
return JSON.stringify(text).replace(
|
|
1145
|
-
LEFT_LITERAL_BY_JSON,
|
|
1146
|
-
(char) => {
|
|
1147
|
-
const codePoint = Number(char.codePointAt(0));
|
|
1148
|
-
const hex = codePoint.toString(16);
|
|
1149
|
-
return codePoint > 65535 ? `\\u{${hex}}` : `\\u${hex.padStart(4, "0")}`;
|
|
1150
|
-
}
|
|
1151
|
-
);
|
|
1152
|
-
}
|
|
1153
|
-
|
|
1154
|
-
// src/server/services/file-scanner.service.ts
|
|
1155
1212
|
var FileScannerService = class {
|
|
1156
1213
|
constructor(options, scanner) {
|
|
1157
1214
|
this.options = options;
|
|
@@ -1346,7 +1403,8 @@ function mapMultipartError(err, context) {
|
|
|
1346
1403
|
STORAGE_ERROR_CODES.STORAGE_MULTIPART_ABORTED,
|
|
1347
1404
|
// An outage ends the upload as an outage; any other cause is a failure of the upload itself.
|
|
1348
1405
|
providerFailureStatus(err),
|
|
1349
|
-
|
|
1406
|
+
// A fixed word, never what the driver or the provider wrote.
|
|
1407
|
+
{ ...context, reason: providerFailureReason(err) },
|
|
1350
1408
|
// The code's message says the upload was aborted; it is not, when the abort itself failed.
|
|
1351
1409
|
context.abort === "failed" ? ABORT_FAILED_MESSAGE : void 0
|
|
1352
1410
|
);
|
|
@@ -1355,26 +1413,31 @@ function knownStatus(error) {
|
|
|
1355
1413
|
const name = common.HttpStatus[error.getStatus()];
|
|
1356
1414
|
return name === void 0 ? void 0 : common.HttpStatus[name];
|
|
1357
1415
|
}
|
|
1358
|
-
function
|
|
1416
|
+
function cleanupErrorCode(error) {
|
|
1417
|
+
return error instanceof StorageException ? error.code : "UNKNOWN";
|
|
1418
|
+
}
|
|
1419
|
+
function describeCleanupFailure(error) {
|
|
1359
1420
|
if (!(error instanceof StorageException)) {
|
|
1360
1421
|
return describeError(error);
|
|
1361
1422
|
}
|
|
1362
|
-
|
|
1363
|
-
const awsMessage = body.error?.details?.awsMessage;
|
|
1364
|
-
return typeof awsMessage === "string" ? `${error.code}: ${awsMessage}` : error.code;
|
|
1423
|
+
return error.code;
|
|
1365
1424
|
}
|
|
1366
1425
|
function withCleanupFailure(scanError, keys, cleanupError) {
|
|
1367
1426
|
if (!(scanError instanceof StorageException)) {
|
|
1368
1427
|
return scanError;
|
|
1369
1428
|
}
|
|
1370
1429
|
const body = scanError.getResponse();
|
|
1371
|
-
return new StorageException(
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1430
|
+
return new StorageException(
|
|
1431
|
+
scanError.code,
|
|
1432
|
+
knownStatus(scanError),
|
|
1433
|
+
{
|
|
1434
|
+
...body.error?.details ?? {},
|
|
1435
|
+
key: keys.key,
|
|
1436
|
+
storedKey: keys.storedKey,
|
|
1437
|
+
cleanup: "failed",
|
|
1438
|
+
cleanupError: cleanupErrorCode(cleanupError)
|
|
1439
|
+
}
|
|
1440
|
+
);
|
|
1378
1441
|
}
|
|
1379
1442
|
var CLEANUP_DEADLINE_MS = 1e4;
|
|
1380
1443
|
function reasonName(signal) {
|
|
@@ -1465,7 +1528,7 @@ async function abortable(work, signal, context) {
|
|
|
1465
1528
|
}
|
|
1466
1529
|
|
|
1467
1530
|
// src/server/utils/multipart-abort.ts
|
|
1468
|
-
async function closeMultipartUpload(client, bucket, key, uploadId) {
|
|
1531
|
+
async function closeMultipartUpload(client, bucket, key, uploadId, onFailure) {
|
|
1469
1532
|
if (uploadId === void 0) {
|
|
1470
1533
|
return {};
|
|
1471
1534
|
}
|
|
@@ -1479,11 +1542,19 @@ async function closeMultipartUpload(client, bucket, key, uploadId) {
|
|
|
1479
1542
|
if (err?.name === "NoSuchUpload") {
|
|
1480
1543
|
return { abort: "completed" };
|
|
1481
1544
|
}
|
|
1482
|
-
|
|
1545
|
+
onFailure?.(err);
|
|
1546
|
+
return { abort: "failed" };
|
|
1483
1547
|
}
|
|
1484
1548
|
}
|
|
1485
|
-
function closeWhenCreated(client, bucket, key, created) {
|
|
1486
|
-
void created.then((uploadId) => closeMultipartUpload(client, bucket, key, uploadId));
|
|
1549
|
+
function closeWhenCreated(client, bucket, key, created, onFailure) {
|
|
1550
|
+
void created.then((uploadId) => closeMultipartUpload(client, bucket, key, uploadId, onFailure));
|
|
1551
|
+
}
|
|
1552
|
+
function abortFailureLogger(logger2, key) {
|
|
1553
|
+
return (err) => {
|
|
1554
|
+
logger2.warn(
|
|
1555
|
+
`Multipart abort failed, the upload stays open on the provider: ${logValue(key)} ${describeProviderFailure(err)}`
|
|
1556
|
+
);
|
|
1557
|
+
};
|
|
1487
1558
|
}
|
|
1488
1559
|
async function deleteObjectsChunk(client, bucket, chunk, stripPrefix) {
|
|
1489
1560
|
const deleted = [];
|
|
@@ -1505,14 +1576,16 @@ async function deleteObjectsChunk(client, bucket, chunk, stripPrefix) {
|
|
|
1505
1576
|
if (failure.Key !== void 0) {
|
|
1506
1577
|
failed.push({
|
|
1507
1578
|
key: stripPrefix(failure.Key),
|
|
1508
|
-
|
|
1579
|
+
// The provider's code, which a caller can branch on; its sentence is not repeated.
|
|
1580
|
+
error: failure.Code ?? "Unknown"
|
|
1509
1581
|
});
|
|
1510
1582
|
}
|
|
1511
1583
|
}
|
|
1512
1584
|
} catch (err) {
|
|
1513
|
-
|
|
1585
|
+
logProviderFailure(err, { op: "deleteMany" });
|
|
1586
|
+
const reason = providerFailureReason(err);
|
|
1514
1587
|
for (const key of chunk) {
|
|
1515
|
-
failed.push({ key: stripPrefix(key), error:
|
|
1588
|
+
failed.push({ key: stripPrefix(key), error: reason });
|
|
1516
1589
|
}
|
|
1517
1590
|
}
|
|
1518
1591
|
return { deleted, failed };
|
|
@@ -1520,7 +1593,12 @@ async function deleteObjectsChunk(client, bucket, chunk, stripPrefix) {
|
|
|
1520
1593
|
function mapBucketCheckError(err, context) {
|
|
1521
1594
|
const failure = err ?? {};
|
|
1522
1595
|
const status = failure.$metadata?.httpStatusCode;
|
|
1523
|
-
const details = {
|
|
1596
|
+
const details = {
|
|
1597
|
+
...context,
|
|
1598
|
+
httpStatus: status,
|
|
1599
|
+
awsCode: failure.name,
|
|
1600
|
+
reason: providerFailureReason(err)
|
|
1601
|
+
};
|
|
1524
1602
|
if (status === 301 || failure.name === "PermanentRedirect") {
|
|
1525
1603
|
return new StorageException(
|
|
1526
1604
|
STORAGE_ERROR_CODES.STORAGE_BUCKET_REGION_MISMATCH,
|
|
@@ -1807,7 +1885,11 @@ exports.StorageService = class StorageService {
|
|
|
1807
1885
|
const signal = options.abortSignal;
|
|
1808
1886
|
const input = validationInput(options, finalKey);
|
|
1809
1887
|
const validated = await abortable(this.validation.validate(input), signal, context);
|
|
1810
|
-
const scanned = await abortable(
|
|
1888
|
+
const scanned = await abortable(
|
|
1889
|
+
this.bufferForPreUploadScan(validated.body, finalKey),
|
|
1890
|
+
signal,
|
|
1891
|
+
context
|
|
1892
|
+
);
|
|
1811
1893
|
const body = boundStreamSize(scanned, options.size, this.options);
|
|
1812
1894
|
const validatedOptions = { ...options, body };
|
|
1813
1895
|
await abortable(this.runPreUploadScan(body, finalKey, bucket, options), signal, context);
|
|
@@ -1897,7 +1979,7 @@ exports.StorageService = class StorageService {
|
|
|
1897
1979
|
} catch (deleteErr) {
|
|
1898
1980
|
this.logger.error(
|
|
1899
1981
|
`STORAGE_SCAN_CLEANUP_FAILED: the rejected object is still stored: ${logValue(finalKey)}`,
|
|
1900
|
-
logValue(
|
|
1982
|
+
logValue(describeCleanupFailure(deleteErr))
|
|
1901
1983
|
);
|
|
1902
1984
|
throw withCleanupFailure(scanErr, { key, storedKey: finalKey }, deleteErr);
|
|
1903
1985
|
}
|
|
@@ -1919,7 +2001,10 @@ exports.StorageService = class StorageService {
|
|
|
1919
2001
|
const context = { key: finalKey, bucket, op: "head" };
|
|
1920
2002
|
throwIfAborted(options?.abortSignal, context);
|
|
1921
2003
|
try {
|
|
1922
|
-
const response = await this.s3Provider.getClient().send(
|
|
2004
|
+
const response = await this.s3Provider.getClient().send(
|
|
2005
|
+
new clientS3.HeadObjectCommand({ Bucket: bucket, Key: finalKey }),
|
|
2006
|
+
sendOptions(options?.abortSignal)
|
|
2007
|
+
);
|
|
1923
2008
|
return this.toObjectMetadata(response, finalKey, bucket);
|
|
1924
2009
|
} catch (err) {
|
|
1925
2010
|
throw failureOrAbort(options?.abortSignal, context, mapAwsError(err, context));
|
|
@@ -1944,7 +2029,9 @@ exports.StorageService = class StorageService {
|
|
|
1944
2029
|
if (err instanceof StorageException && err.code === STORAGE_ERROR_CODES.STORAGE_REQUEST_ABORTED) {
|
|
1945
2030
|
throw err;
|
|
1946
2031
|
}
|
|
1947
|
-
this.logger.warn(
|
|
2032
|
+
this.logger.warn(
|
|
2033
|
+
`exists() \u2014 treating a non-404 error as "false": ${logValue(describeError(err))}`
|
|
2034
|
+
);
|
|
1948
2035
|
return false;
|
|
1949
2036
|
}
|
|
1950
2037
|
}
|
|
@@ -1972,7 +2059,11 @@ exports.StorageService = class StorageService {
|
|
|
1972
2059
|
*/
|
|
1973
2060
|
async download(options) {
|
|
1974
2061
|
this.assertConfigured();
|
|
1975
|
-
assertHeaderValues({
|
|
2062
|
+
assertHeaderValues({
|
|
2063
|
+
range: options.range,
|
|
2064
|
+
ifNoneMatch: options.ifNoneMatch,
|
|
2065
|
+
ifMatch: options.ifMatch
|
|
2066
|
+
});
|
|
1976
2067
|
const finalKey = this.keyResolver.normalize(options.key);
|
|
1977
2068
|
const bucket = this.resolveBucket(options.bucket);
|
|
1978
2069
|
const context = { key: finalKey, bucket, op: "download" };
|
|
@@ -2036,7 +2127,10 @@ exports.StorageService = class StorageService {
|
|
|
2036
2127
|
const context = { key: finalKey, bucket, op: "delete" };
|
|
2037
2128
|
throwIfAborted(options?.abortSignal, context);
|
|
2038
2129
|
try {
|
|
2039
|
-
await this.s3Provider.getClient().send(
|
|
2130
|
+
await this.s3Provider.getClient().send(
|
|
2131
|
+
new clientS3.DeleteObjectCommand({ Bucket: bucket, Key: finalKey }),
|
|
2132
|
+
sendOptions(options?.abortSignal)
|
|
2133
|
+
);
|
|
2040
2134
|
} catch (err) {
|
|
2041
2135
|
const mapped = failureOrAbort(options?.abortSignal, context, mapAwsError(err, context));
|
|
2042
2136
|
if (mapped.code !== STORAGE_ERROR_CODES.STORAGE_OBJECT_NOT_FOUND) {
|
|
@@ -2184,9 +2278,17 @@ exports.StorageService = class StorageService {
|
|
|
2184
2278
|
let uploadId = upload.uploadId;
|
|
2185
2279
|
if (uploadId === void 0 && lookup) {
|
|
2186
2280
|
uploadId = await within(lookup, CLEANUP_DEADLINE_MS);
|
|
2187
|
-
if (uploadId === void 0)
|
|
2281
|
+
if (uploadId === void 0) {
|
|
2282
|
+
closeWhenCreated(client, bucket, finalKey, lookup, abortFailureLogger(this.logger, finalKey));
|
|
2283
|
+
}
|
|
2188
2284
|
}
|
|
2189
|
-
const abort = await closeMultipartUpload(
|
|
2285
|
+
const abort = await closeMultipartUpload(
|
|
2286
|
+
client,
|
|
2287
|
+
bucket,
|
|
2288
|
+
finalKey,
|
|
2289
|
+
uploadId,
|
|
2290
|
+
abortFailureLogger(this.logger, finalKey)
|
|
2291
|
+
);
|
|
2190
2292
|
const context = { key: finalKey, bucket, op: "upload-multipart", ...abort };
|
|
2191
2293
|
throw abortedBy ? requestAborted(abortedBy, context) : mapMultipartError(err, context);
|
|
2192
2294
|
}
|
package/dist/server/index.d.cts
CHANGED
|
@@ -502,9 +502,14 @@ interface CopyOptions {
|
|
|
502
502
|
interface DeleteManyOptions {
|
|
503
503
|
bucket?: string;
|
|
504
504
|
}
|
|
505
|
-
/** A single failed key from a batch delete
|
|
505
|
+
/** A single failed key from a batch delete. */
|
|
506
506
|
interface FailedDeletion {
|
|
507
507
|
key: string;
|
|
508
|
+
/**
|
|
509
|
+
* The provider's error code for this key (for example `AccessDenied`), or, when the whole request
|
|
510
|
+
* failed, a fixed reason: `unreachable`, `timeout`, `provider-error`, `credentials-refused` or
|
|
511
|
+
* `rejected`. Never the text of the driver or of the provider.
|
|
512
|
+
*/
|
|
508
513
|
error: string;
|
|
509
514
|
}
|
|
510
515
|
/** Aggregated outcome of a batch delete: succeeded keys and per-key failures. */
|
package/dist/server/index.d.ts
CHANGED
|
@@ -502,9 +502,14 @@ interface CopyOptions {
|
|
|
502
502
|
interface DeleteManyOptions {
|
|
503
503
|
bucket?: string;
|
|
504
504
|
}
|
|
505
|
-
/** A single failed key from a batch delete
|
|
505
|
+
/** A single failed key from a batch delete. */
|
|
506
506
|
interface FailedDeletion {
|
|
507
507
|
key: string;
|
|
508
|
+
/**
|
|
509
|
+
* The provider's error code for this key (for example `AccessDenied`), or, when the whole request
|
|
510
|
+
* failed, a fixed reason: `unreachable`, `timeout`, `provider-error`, `credentials-refused` or
|
|
511
|
+
* `rejected`. Never the text of the driver or of the provider.
|
|
512
|
+
*/
|
|
508
513
|
error: string;
|
|
509
514
|
}
|
|
510
515
|
/** Aggregated outcome of a batch delete: succeeded keys and per-key failures. */
|
package/dist/server/index.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { HttpStatus, Injectable, Inject, Global, Module,
|
|
1
|
+
import { HttpStatus, Injectable, Inject, Logger, Global, Module, HttpException } from '@nestjs/common';
|
|
2
2
|
import { S3Client, GetBucketLifecycleConfigurationCommand, DeleteBucketLifecycleCommand, PutBucketLifecycleConfigurationCommand, HeadObjectCommand, GetObjectCommand, DeleteObjectCommand, HeadBucketCommand, PutObjectCommand, ListObjectsV2Command, CopyObjectCommand, CreateMultipartUploadCommand, UploadPartCommand, CompleteMultipartUploadCommand, AbortMultipartUploadCommand, DeleteObjectsCommand } from '@aws-sdk/client-s3';
|
|
3
3
|
import { Upload } from '@aws-sdk/lib-storage';
|
|
4
4
|
import { PassThrough, Transform, pipeline, Readable } from 'stream';
|
|
@@ -142,9 +142,9 @@ var HEADER_VALUE = /^[\t \x21-\x7E]*$/;
|
|
|
142
142
|
var HEADER_NAME = /^[!#$%&'*+.^_`|~0-9a-z-]+$/;
|
|
143
143
|
var VALUE_REASON = "must be a string of tab, space and printable ASCII characters";
|
|
144
144
|
var KEY_REASON = "every key must be a non-empty header name of letters, digits and the characters !#$%&'*+-.^_`|~";
|
|
145
|
-
function refuse(
|
|
145
|
+
function refuse(field2, reason) {
|
|
146
146
|
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_VALIDATION_FAILED, void 0, {
|
|
147
|
-
field,
|
|
147
|
+
field: field2,
|
|
148
148
|
reason
|
|
149
149
|
});
|
|
150
150
|
}
|
|
@@ -152,9 +152,9 @@ function isHeaderValue(value) {
|
|
|
152
152
|
return typeof value === "string" && HEADER_VALUE.test(value);
|
|
153
153
|
}
|
|
154
154
|
function assertHeaderValues(fields) {
|
|
155
|
-
for (const [
|
|
155
|
+
for (const [field2, value] of Object.entries(fields)) {
|
|
156
156
|
if (value !== void 0 && !isHeaderValue(value)) {
|
|
157
|
-
refuse(
|
|
157
|
+
refuse(field2, VALUE_REASON);
|
|
158
158
|
}
|
|
159
159
|
}
|
|
160
160
|
}
|
|
@@ -507,6 +507,60 @@ KeyResolverService = __decorateClass([
|
|
|
507
507
|
Injectable(),
|
|
508
508
|
__decorateParam(0, Inject(BYMAX_STORAGE_OPTIONS))
|
|
509
509
|
], KeyResolverService);
|
|
510
|
+
|
|
511
|
+
// src/server/utils/describe-error.ts
|
|
512
|
+
var NON_PRINTABLE_REJECTION = "non-printable rejection value";
|
|
513
|
+
function describeError(value) {
|
|
514
|
+
try {
|
|
515
|
+
if (typeof value === "object" && value !== null) {
|
|
516
|
+
const message = value.message;
|
|
517
|
+
if (typeof message === "string") {
|
|
518
|
+
return message;
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
return String(value);
|
|
522
|
+
} catch {
|
|
523
|
+
return NON_PRINTABLE_REJECTION;
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
// src/server/utils/log-value.ts
|
|
528
|
+
var LEFT_LITERAL_BY_JSON = /[\p{Cc}\p{Cf}\p{Zl}\p{Zp}]/gu;
|
|
529
|
+
function logValue(value) {
|
|
530
|
+
const text = typeof value === "string" ? value : describeError(value);
|
|
531
|
+
return JSON.stringify(text).replace(
|
|
532
|
+
LEFT_LITERAL_BY_JSON,
|
|
533
|
+
(char) => {
|
|
534
|
+
const codePoint = Number(char.codePointAt(0));
|
|
535
|
+
const hex = codePoint.toString(16);
|
|
536
|
+
return codePoint > 65535 ? `\\u{${hex}}` : `\\u${hex.padStart(4, "0")}`;
|
|
537
|
+
}
|
|
538
|
+
);
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
// src/server/utils/provider-failure-log.ts
|
|
542
|
+
var SECRET_IN_MESSAGE = /(X-Amz-(?:Signature|Credential|Security-Token)|AWSAccessKeyId|Signature)=[^&\s"']+/gi;
|
|
543
|
+
var MAX_MESSAGE_LENGTH = 300;
|
|
544
|
+
var logger = new Logger("StorageProvider");
|
|
545
|
+
function field(value) {
|
|
546
|
+
return typeof value === "string" || typeof value === "number" ? logValue(value) : "-";
|
|
547
|
+
}
|
|
548
|
+
function describeProviderFailure(err) {
|
|
549
|
+
const failure = typeof err === "object" && err !== null ? err : {};
|
|
550
|
+
const message = describeError(err).replace(SECRET_IN_MESSAGE, "$1=[redacted]");
|
|
551
|
+
return [
|
|
552
|
+
`name=${field(failure.name)}`,
|
|
553
|
+
`code=${field(failure.Code)}`,
|
|
554
|
+
`status=${field(failure.$metadata?.httpStatusCode)}`,
|
|
555
|
+
`requestId=${field(failure.$metadata?.requestId)}`,
|
|
556
|
+
`message=${logValue(message.slice(0, MAX_MESSAGE_LENGTH))}`
|
|
557
|
+
].join(" ");
|
|
558
|
+
}
|
|
559
|
+
function logProviderFailure(err, context) {
|
|
560
|
+
const op = typeof context?.op === "string" ? logValue(context.op) : "-";
|
|
561
|
+
const key = typeof context?.key === "string" ? logValue(context.key) : "-";
|
|
562
|
+
logger.warn(`Provider call failed: op=${op} key=${key} ${describeProviderFailure(err)}`);
|
|
563
|
+
}
|
|
510
564
|
var UNREACHABLE_CODES = /* @__PURE__ */ new Set([
|
|
511
565
|
"ECONNREFUSED",
|
|
512
566
|
"ECONNRESET",
|
|
@@ -522,7 +576,10 @@ var UNREACHABLE_IN_MESSAGE = new RegExp(
|
|
|
522
576
|
`^(?:AWS SDK error wrapper for \\w+: )?(?:(?:connect|read|write|getaddrinfo) (?:${[...UNREACHABLE_CODES].join("|")})\\b|socket hang up)`
|
|
523
577
|
);
|
|
524
578
|
var ADDRESSES_FAILED = "AWS SDK error wrapper for AggregateError";
|
|
525
|
-
var CREDENTIAL_CODES = /* @__PURE__ */ new Set([
|
|
579
|
+
var CREDENTIAL_CODES = /* @__PURE__ */ new Set([
|
|
580
|
+
"InvalidAccessKeyId",
|
|
581
|
+
"SignatureDoesNotMatch"
|
|
582
|
+
]);
|
|
526
583
|
function cutConnection(failure, noAnswer) {
|
|
527
584
|
const codes = [failure.code, failure.cause?.code];
|
|
528
585
|
if (Array.isArray(failure.errors)) {
|
|
@@ -534,20 +591,33 @@ function cutConnection(failure, noAnswer) {
|
|
|
534
591
|
const message = typeof failure.message === "string" ? failure.message : "";
|
|
535
592
|
return noAnswer && (UNREACHABLE_IN_MESSAGE.test(message) || message.startsWith(ADDRESSES_FAILED));
|
|
536
593
|
}
|
|
537
|
-
function
|
|
594
|
+
function providerFailureReason(err) {
|
|
538
595
|
const failure = err ?? {};
|
|
539
596
|
const status = failure.$metadata?.httpStatusCode;
|
|
540
|
-
if (failure.name === "TimeoutError"
|
|
541
|
-
return "
|
|
597
|
+
if (failure.name === "TimeoutError" && failure.code === "ETIMEDOUT") {
|
|
598
|
+
return "timeout";
|
|
599
|
+
}
|
|
600
|
+
if (cutConnection(failure, status === void 0)) {
|
|
601
|
+
return "unreachable";
|
|
602
|
+
}
|
|
603
|
+
if (failure.name === "TimeoutError") {
|
|
604
|
+
return "timeout";
|
|
542
605
|
}
|
|
543
606
|
if (typeof status === "number" && status >= 500 && status !== 501) {
|
|
544
|
-
return "
|
|
607
|
+
return "provider-error";
|
|
545
608
|
}
|
|
546
609
|
const awsCode = failure.Code ?? failure.name;
|
|
547
610
|
if (status === 403 && typeof awsCode === "string" && CREDENTIAL_CODES.has(awsCode)) {
|
|
548
611
|
return "credentials-refused";
|
|
549
612
|
}
|
|
550
|
-
return "
|
|
613
|
+
return "rejected";
|
|
614
|
+
}
|
|
615
|
+
function classifyProviderFailure(err) {
|
|
616
|
+
const reason = providerFailureReason(err);
|
|
617
|
+
if (reason === "credentials-refused") {
|
|
618
|
+
return "credentials-refused";
|
|
619
|
+
}
|
|
620
|
+
return reason === "rejected" ? "other" : "unavailable";
|
|
551
621
|
}
|
|
552
622
|
function providerFailureStatus(err) {
|
|
553
623
|
const failure = classifyProviderFailure(err);
|
|
@@ -568,7 +638,8 @@ function mapAwsError(err, context) {
|
|
|
568
638
|
awsCode,
|
|
569
639
|
httpStatus,
|
|
570
640
|
requestId,
|
|
571
|
-
|
|
641
|
+
// A fixed word, never what the driver or the provider wrote.
|
|
642
|
+
reason: providerFailureReason(err)
|
|
572
643
|
};
|
|
573
644
|
if (awsCode === "NoSuchBucket") {
|
|
574
645
|
return new StorageException(
|
|
@@ -585,7 +656,11 @@ function mapAwsError(err, context) {
|
|
|
585
656
|
);
|
|
586
657
|
}
|
|
587
658
|
if (httpStatus === 304) {
|
|
588
|
-
return new StorageException(
|
|
659
|
+
return new StorageException(
|
|
660
|
+
STORAGE_ERROR_CODES.STORAGE_NOT_MODIFIED,
|
|
661
|
+
HttpStatus.NOT_MODIFIED,
|
|
662
|
+
details
|
|
663
|
+
);
|
|
589
664
|
}
|
|
590
665
|
if (httpStatus === 412) {
|
|
591
666
|
return new StorageException(
|
|
@@ -595,8 +670,14 @@ function mapAwsError(err, context) {
|
|
|
595
670
|
);
|
|
596
671
|
}
|
|
597
672
|
if (e.name === "TimeoutError") {
|
|
598
|
-
|
|
673
|
+
logProviderFailure(err, context);
|
|
674
|
+
return new StorageException(
|
|
675
|
+
STORAGE_ERROR_CODES.STORAGE_TIMEOUT,
|
|
676
|
+
void 0,
|
|
677
|
+
details
|
|
678
|
+
);
|
|
599
679
|
}
|
|
680
|
+
logProviderFailure(err, context);
|
|
600
681
|
return new StorageException(
|
|
601
682
|
STORAGE_ERROR_CODES.STORAGE_PROVIDER_ERROR,
|
|
602
683
|
providerFailureStatus(err),
|
|
@@ -628,10 +709,18 @@ function mapLifecycleError(err, context) {
|
|
|
628
709
|
const status = e.$metadata?.httpStatusCode ?? 0;
|
|
629
710
|
const details = mapped.getResponse().error.details;
|
|
630
711
|
if (UNSUPPORTED_NAMES.has(name) || UNSUPPORTED_STATUSES.has(status)) {
|
|
631
|
-
return new StorageException(
|
|
712
|
+
return new StorageException(
|
|
713
|
+
STORAGE_ERROR_CODES.STORAGE_LIFECYCLE_UNSUPPORTED,
|
|
714
|
+
void 0,
|
|
715
|
+
details
|
|
716
|
+
);
|
|
632
717
|
}
|
|
633
718
|
if (REJECTED_NAMES.has(name)) {
|
|
634
|
-
return new StorageException(
|
|
719
|
+
return new StorageException(
|
|
720
|
+
STORAGE_ERROR_CODES.STORAGE_LIFECYCLE_REJECTED,
|
|
721
|
+
void 0,
|
|
722
|
+
details
|
|
723
|
+
);
|
|
635
724
|
}
|
|
636
725
|
return mapped;
|
|
637
726
|
}
|
|
@@ -648,9 +737,9 @@ function ruleInvalid(reason) {
|
|
|
648
737
|
reason
|
|
649
738
|
});
|
|
650
739
|
}
|
|
651
|
-
function assertDays(
|
|
740
|
+
function assertDays(field2, value) {
|
|
652
741
|
if (!Number.isInteger(value) || value < 1) {
|
|
653
|
-
throw ruleInvalid(`${
|
|
742
|
+
throw ruleInvalid(`${field2} must be an integer of at least 1`);
|
|
654
743
|
}
|
|
655
744
|
}
|
|
656
745
|
var BucketLifecycleService = class {
|
|
@@ -1118,38 +1207,6 @@ ValidationService = __decorateClass([
|
|
|
1118
1207
|
__decorateParam(0, Inject(BYMAX_STORAGE_OPTIONS)),
|
|
1119
1208
|
__decorateParam(1, Inject(BYMAX_STORAGE_UPLOAD_VALIDATORS))
|
|
1120
1209
|
], ValidationService);
|
|
1121
|
-
|
|
1122
|
-
// src/server/utils/describe-error.ts
|
|
1123
|
-
var NON_PRINTABLE_REJECTION = "non-printable rejection value";
|
|
1124
|
-
function describeError(value) {
|
|
1125
|
-
try {
|
|
1126
|
-
if (typeof value === "object" && value !== null) {
|
|
1127
|
-
const message = value.message;
|
|
1128
|
-
if (typeof message === "string") {
|
|
1129
|
-
return message;
|
|
1130
|
-
}
|
|
1131
|
-
}
|
|
1132
|
-
return String(value);
|
|
1133
|
-
} catch {
|
|
1134
|
-
return NON_PRINTABLE_REJECTION;
|
|
1135
|
-
}
|
|
1136
|
-
}
|
|
1137
|
-
|
|
1138
|
-
// src/server/utils/log-value.ts
|
|
1139
|
-
var LEFT_LITERAL_BY_JSON = /[\p{Cc}\p{Cf}\p{Zl}\p{Zp}]/gu;
|
|
1140
|
-
function logValue(value) {
|
|
1141
|
-
const text = typeof value === "string" ? value : describeError(value);
|
|
1142
|
-
return JSON.stringify(text).replace(
|
|
1143
|
-
LEFT_LITERAL_BY_JSON,
|
|
1144
|
-
(char) => {
|
|
1145
|
-
const codePoint = Number(char.codePointAt(0));
|
|
1146
|
-
const hex = codePoint.toString(16);
|
|
1147
|
-
return codePoint > 65535 ? `\\u{${hex}}` : `\\u${hex.padStart(4, "0")}`;
|
|
1148
|
-
}
|
|
1149
|
-
);
|
|
1150
|
-
}
|
|
1151
|
-
|
|
1152
|
-
// src/server/services/file-scanner.service.ts
|
|
1153
1210
|
var FileScannerService = class {
|
|
1154
1211
|
constructor(options, scanner) {
|
|
1155
1212
|
this.options = options;
|
|
@@ -1344,7 +1401,8 @@ function mapMultipartError(err, context) {
|
|
|
1344
1401
|
STORAGE_ERROR_CODES.STORAGE_MULTIPART_ABORTED,
|
|
1345
1402
|
// An outage ends the upload as an outage; any other cause is a failure of the upload itself.
|
|
1346
1403
|
providerFailureStatus(err),
|
|
1347
|
-
|
|
1404
|
+
// A fixed word, never what the driver or the provider wrote.
|
|
1405
|
+
{ ...context, reason: providerFailureReason(err) },
|
|
1348
1406
|
// The code's message says the upload was aborted; it is not, when the abort itself failed.
|
|
1349
1407
|
context.abort === "failed" ? ABORT_FAILED_MESSAGE : void 0
|
|
1350
1408
|
);
|
|
@@ -1353,26 +1411,31 @@ function knownStatus(error) {
|
|
|
1353
1411
|
const name = HttpStatus[error.getStatus()];
|
|
1354
1412
|
return name === void 0 ? void 0 : HttpStatus[name];
|
|
1355
1413
|
}
|
|
1356
|
-
function
|
|
1414
|
+
function cleanupErrorCode(error) {
|
|
1415
|
+
return error instanceof StorageException ? error.code : "UNKNOWN";
|
|
1416
|
+
}
|
|
1417
|
+
function describeCleanupFailure(error) {
|
|
1357
1418
|
if (!(error instanceof StorageException)) {
|
|
1358
1419
|
return describeError(error);
|
|
1359
1420
|
}
|
|
1360
|
-
|
|
1361
|
-
const awsMessage = body.error?.details?.awsMessage;
|
|
1362
|
-
return typeof awsMessage === "string" ? `${error.code}: ${awsMessage}` : error.code;
|
|
1421
|
+
return error.code;
|
|
1363
1422
|
}
|
|
1364
1423
|
function withCleanupFailure(scanError, keys, cleanupError) {
|
|
1365
1424
|
if (!(scanError instanceof StorageException)) {
|
|
1366
1425
|
return scanError;
|
|
1367
1426
|
}
|
|
1368
1427
|
const body = scanError.getResponse();
|
|
1369
|
-
return new StorageException(
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1428
|
+
return new StorageException(
|
|
1429
|
+
scanError.code,
|
|
1430
|
+
knownStatus(scanError),
|
|
1431
|
+
{
|
|
1432
|
+
...body.error?.details ?? {},
|
|
1433
|
+
key: keys.key,
|
|
1434
|
+
storedKey: keys.storedKey,
|
|
1435
|
+
cleanup: "failed",
|
|
1436
|
+
cleanupError: cleanupErrorCode(cleanupError)
|
|
1437
|
+
}
|
|
1438
|
+
);
|
|
1376
1439
|
}
|
|
1377
1440
|
var CLEANUP_DEADLINE_MS = 1e4;
|
|
1378
1441
|
function reasonName(signal) {
|
|
@@ -1463,7 +1526,7 @@ async function abortable(work, signal, context) {
|
|
|
1463
1526
|
}
|
|
1464
1527
|
|
|
1465
1528
|
// src/server/utils/multipart-abort.ts
|
|
1466
|
-
async function closeMultipartUpload(client, bucket, key, uploadId) {
|
|
1529
|
+
async function closeMultipartUpload(client, bucket, key, uploadId, onFailure) {
|
|
1467
1530
|
if (uploadId === void 0) {
|
|
1468
1531
|
return {};
|
|
1469
1532
|
}
|
|
@@ -1477,11 +1540,19 @@ async function closeMultipartUpload(client, bucket, key, uploadId) {
|
|
|
1477
1540
|
if (err?.name === "NoSuchUpload") {
|
|
1478
1541
|
return { abort: "completed" };
|
|
1479
1542
|
}
|
|
1480
|
-
|
|
1543
|
+
onFailure?.(err);
|
|
1544
|
+
return { abort: "failed" };
|
|
1481
1545
|
}
|
|
1482
1546
|
}
|
|
1483
|
-
function closeWhenCreated(client, bucket, key, created) {
|
|
1484
|
-
void created.then((uploadId) => closeMultipartUpload(client, bucket, key, uploadId));
|
|
1547
|
+
function closeWhenCreated(client, bucket, key, created, onFailure) {
|
|
1548
|
+
void created.then((uploadId) => closeMultipartUpload(client, bucket, key, uploadId, onFailure));
|
|
1549
|
+
}
|
|
1550
|
+
function abortFailureLogger(logger2, key) {
|
|
1551
|
+
return (err) => {
|
|
1552
|
+
logger2.warn(
|
|
1553
|
+
`Multipart abort failed, the upload stays open on the provider: ${logValue(key)} ${describeProviderFailure(err)}`
|
|
1554
|
+
);
|
|
1555
|
+
};
|
|
1485
1556
|
}
|
|
1486
1557
|
async function deleteObjectsChunk(client, bucket, chunk, stripPrefix) {
|
|
1487
1558
|
const deleted = [];
|
|
@@ -1503,14 +1574,16 @@ async function deleteObjectsChunk(client, bucket, chunk, stripPrefix) {
|
|
|
1503
1574
|
if (failure.Key !== void 0) {
|
|
1504
1575
|
failed.push({
|
|
1505
1576
|
key: stripPrefix(failure.Key),
|
|
1506
|
-
|
|
1577
|
+
// The provider's code, which a caller can branch on; its sentence is not repeated.
|
|
1578
|
+
error: failure.Code ?? "Unknown"
|
|
1507
1579
|
});
|
|
1508
1580
|
}
|
|
1509
1581
|
}
|
|
1510
1582
|
} catch (err) {
|
|
1511
|
-
|
|
1583
|
+
logProviderFailure(err, { op: "deleteMany" });
|
|
1584
|
+
const reason = providerFailureReason(err);
|
|
1512
1585
|
for (const key of chunk) {
|
|
1513
|
-
failed.push({ key: stripPrefix(key), error:
|
|
1586
|
+
failed.push({ key: stripPrefix(key), error: reason });
|
|
1514
1587
|
}
|
|
1515
1588
|
}
|
|
1516
1589
|
return { deleted, failed };
|
|
@@ -1518,7 +1591,12 @@ async function deleteObjectsChunk(client, bucket, chunk, stripPrefix) {
|
|
|
1518
1591
|
function mapBucketCheckError(err, context) {
|
|
1519
1592
|
const failure = err ?? {};
|
|
1520
1593
|
const status = failure.$metadata?.httpStatusCode;
|
|
1521
|
-
const details = {
|
|
1594
|
+
const details = {
|
|
1595
|
+
...context,
|
|
1596
|
+
httpStatus: status,
|
|
1597
|
+
awsCode: failure.name,
|
|
1598
|
+
reason: providerFailureReason(err)
|
|
1599
|
+
};
|
|
1522
1600
|
if (status === 301 || failure.name === "PermanentRedirect") {
|
|
1523
1601
|
return new StorageException(
|
|
1524
1602
|
STORAGE_ERROR_CODES.STORAGE_BUCKET_REGION_MISMATCH,
|
|
@@ -1805,7 +1883,11 @@ var StorageService = class {
|
|
|
1805
1883
|
const signal = options.abortSignal;
|
|
1806
1884
|
const input = validationInput(options, finalKey);
|
|
1807
1885
|
const validated = await abortable(this.validation.validate(input), signal, context);
|
|
1808
|
-
const scanned = await abortable(
|
|
1886
|
+
const scanned = await abortable(
|
|
1887
|
+
this.bufferForPreUploadScan(validated.body, finalKey),
|
|
1888
|
+
signal,
|
|
1889
|
+
context
|
|
1890
|
+
);
|
|
1809
1891
|
const body = boundStreamSize(scanned, options.size, this.options);
|
|
1810
1892
|
const validatedOptions = { ...options, body };
|
|
1811
1893
|
await abortable(this.runPreUploadScan(body, finalKey, bucket, options), signal, context);
|
|
@@ -1895,7 +1977,7 @@ var StorageService = class {
|
|
|
1895
1977
|
} catch (deleteErr) {
|
|
1896
1978
|
this.logger.error(
|
|
1897
1979
|
`STORAGE_SCAN_CLEANUP_FAILED: the rejected object is still stored: ${logValue(finalKey)}`,
|
|
1898
|
-
logValue(
|
|
1980
|
+
logValue(describeCleanupFailure(deleteErr))
|
|
1899
1981
|
);
|
|
1900
1982
|
throw withCleanupFailure(scanErr, { key, storedKey: finalKey }, deleteErr);
|
|
1901
1983
|
}
|
|
@@ -1917,7 +1999,10 @@ var StorageService = class {
|
|
|
1917
1999
|
const context = { key: finalKey, bucket, op: "head" };
|
|
1918
2000
|
throwIfAborted(options?.abortSignal, context);
|
|
1919
2001
|
try {
|
|
1920
|
-
const response = await this.s3Provider.getClient().send(
|
|
2002
|
+
const response = await this.s3Provider.getClient().send(
|
|
2003
|
+
new HeadObjectCommand({ Bucket: bucket, Key: finalKey }),
|
|
2004
|
+
sendOptions(options?.abortSignal)
|
|
2005
|
+
);
|
|
1921
2006
|
return this.toObjectMetadata(response, finalKey, bucket);
|
|
1922
2007
|
} catch (err) {
|
|
1923
2008
|
throw failureOrAbort(options?.abortSignal, context, mapAwsError(err, context));
|
|
@@ -1942,7 +2027,9 @@ var StorageService = class {
|
|
|
1942
2027
|
if (err instanceof StorageException && err.code === STORAGE_ERROR_CODES.STORAGE_REQUEST_ABORTED) {
|
|
1943
2028
|
throw err;
|
|
1944
2029
|
}
|
|
1945
|
-
this.logger.warn(
|
|
2030
|
+
this.logger.warn(
|
|
2031
|
+
`exists() \u2014 treating a non-404 error as "false": ${logValue(describeError(err))}`
|
|
2032
|
+
);
|
|
1946
2033
|
return false;
|
|
1947
2034
|
}
|
|
1948
2035
|
}
|
|
@@ -1970,7 +2057,11 @@ var StorageService = class {
|
|
|
1970
2057
|
*/
|
|
1971
2058
|
async download(options) {
|
|
1972
2059
|
this.assertConfigured();
|
|
1973
|
-
assertHeaderValues({
|
|
2060
|
+
assertHeaderValues({
|
|
2061
|
+
range: options.range,
|
|
2062
|
+
ifNoneMatch: options.ifNoneMatch,
|
|
2063
|
+
ifMatch: options.ifMatch
|
|
2064
|
+
});
|
|
1974
2065
|
const finalKey = this.keyResolver.normalize(options.key);
|
|
1975
2066
|
const bucket = this.resolveBucket(options.bucket);
|
|
1976
2067
|
const context = { key: finalKey, bucket, op: "download" };
|
|
@@ -2034,7 +2125,10 @@ var StorageService = class {
|
|
|
2034
2125
|
const context = { key: finalKey, bucket, op: "delete" };
|
|
2035
2126
|
throwIfAborted(options?.abortSignal, context);
|
|
2036
2127
|
try {
|
|
2037
|
-
await this.s3Provider.getClient().send(
|
|
2128
|
+
await this.s3Provider.getClient().send(
|
|
2129
|
+
new DeleteObjectCommand({ Bucket: bucket, Key: finalKey }),
|
|
2130
|
+
sendOptions(options?.abortSignal)
|
|
2131
|
+
);
|
|
2038
2132
|
} catch (err) {
|
|
2039
2133
|
const mapped = failureOrAbort(options?.abortSignal, context, mapAwsError(err, context));
|
|
2040
2134
|
if (mapped.code !== STORAGE_ERROR_CODES.STORAGE_OBJECT_NOT_FOUND) {
|
|
@@ -2182,9 +2276,17 @@ var StorageService = class {
|
|
|
2182
2276
|
let uploadId = upload.uploadId;
|
|
2183
2277
|
if (uploadId === void 0 && lookup) {
|
|
2184
2278
|
uploadId = await within(lookup, CLEANUP_DEADLINE_MS);
|
|
2185
|
-
if (uploadId === void 0)
|
|
2279
|
+
if (uploadId === void 0) {
|
|
2280
|
+
closeWhenCreated(client, bucket, finalKey, lookup, abortFailureLogger(this.logger, finalKey));
|
|
2281
|
+
}
|
|
2186
2282
|
}
|
|
2187
|
-
const abort = await closeMultipartUpload(
|
|
2283
|
+
const abort = await closeMultipartUpload(
|
|
2284
|
+
client,
|
|
2285
|
+
bucket,
|
|
2286
|
+
finalKey,
|
|
2287
|
+
uploadId,
|
|
2288
|
+
abortFailureLogger(this.logger, finalKey)
|
|
2289
|
+
);
|
|
2188
2290
|
const context = { key: finalKey, bucket, op: "upload-multipart", ...abort };
|
|
2189
2291
|
throw abortedBy ? requestAborted(abortedBy, context) : mapMultipartError(err, context);
|
|
2190
2292
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bymax-one/nest-storage",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.13",
|
|
4
4
|
"description": "Provider-agnostic S3-compatible object storage for NestJS. Works with AWS S3, DigitalOcean Spaces, Cloudflare R2, Backblaze B2, MinIO, Wasabi.",
|
|
5
5
|
"author": "Bymax One <support@bymax.one>",
|
|
6
6
|
"license": "MIT",
|