@bymax-one/nest-storage 1.1.12 → 1.1.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +40 -1
- package/README.md +4 -4
- package/dist/server/index.cjs +190 -81
- package/dist/server/index.d.cts +6 -1
- package/dist/server/index.d.ts +6 -1
- package/dist/server/index.mjs +191 -82
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -13,6 +13,43 @@ note on each entry is the compatibility contract — read it, not the number.
|
|
|
13
13
|
|
|
14
14
|
## [Unreleased]
|
|
15
15
|
|
|
16
|
+
## [1.1.14] - 2026-11-16
|
|
17
|
+
|
|
18
|
+
### Changed
|
|
19
|
+
|
|
20
|
+
- **The provider-failure log masks more, and says less when nothing failed.** The warning that carries what
|
|
21
|
+
the driver or the provider said now also masks the credential scope and signature of an `Authorization`
|
|
22
|
+
header (`Credential=...`) and a bare access key id (`AKIA...`, `ASIA...`), still before it cuts the
|
|
23
|
+
message. A request the caller cancelled is no longer logged, and a multipart upload that fails opens its
|
|
24
|
+
line with `Multipart upload failed` (it was `Provider call failed`, which a failure of the caller's own
|
|
25
|
+
source stream is not). Nothing a client receives changes.
|
|
26
|
+
|
|
27
|
+
**Apply to a derived backend:** nothing, unless a log search matched the old line start for a multipart
|
|
28
|
+
failure or the cancelled-request warning.
|
|
29
|
+
|
|
30
|
+
## [1.1.13] - 2026-11-16
|
|
31
|
+
|
|
32
|
+
### Security
|
|
33
|
+
|
|
34
|
+
- **A failed call no longer tells its client what the driver or the provider said.** The body of a
|
|
35
|
+
`StorageException` carried `details.awsMessage`, and for a failed multipart abort `abortError`, a
|
|
36
|
+
failed post-upload delete `cleanupError`, and `deleteMany` the text of each failure. For a provider
|
|
37
|
+
that could not be reached that text is the driver's own, such as `connect ECONNREFUSED 10.0.3.7:9000`,
|
|
38
|
+
and a host that returns `getResponse()` unchanged (Nest's default) handed a caller the address and
|
|
39
|
+
port of the storage service. The body now holds a fixed `details.reason` beside `awsCode`, `httpStatus`
|
|
40
|
+
and `requestId`, which stay. What the driver and the provider said is logged for a provider failure, as a warning
|
|
41
|
+
that starts `Provider call failed` (or `Multipart abort failed`), from named fields only: the error's
|
|
42
|
+
name, the provider's code, the status, the request id and the message, with a signature or an access
|
|
43
|
+
key id in it masked and its length bounded. The error object is not attached to the exception.
|
|
44
|
+
`deleteMany`'s `failed[].error` is the provider's error code for a key it refused, or the fixed reason
|
|
45
|
+
when the whole request failed.
|
|
46
|
+
|
|
47
|
+
**Apply to a derived backend:** a host that read `details.awsMessage`, `abortError` or the text in
|
|
48
|
+
`cleanupError` finds that text in the library's warn log instead; the exception no longer carries it,
|
|
49
|
+
so there is nothing to forward to a client by mistake. A host that matched on that text matches
|
|
50
|
+
`details.reason` (`unreachable`, `timeout`, `provider-error`, `credentials-refused` or `rejected`) or
|
|
51
|
+
`awsCode`.
|
|
52
|
+
|
|
16
53
|
## [1.1.12] - 2026-11-14
|
|
17
54
|
|
|
18
55
|
### Added
|
|
@@ -826,4 +863,6 @@ have regressed from. They are kept because the reasoning is worth having.
|
|
|
826
863
|
[1.1.10]: https://github.com/bymaxone/nest-storage/compare/v1.1.9...v1.1.10
|
|
827
864
|
[1.1.11]: https://github.com/bymaxone/nest-storage/compare/v1.1.10...v1.1.11
|
|
828
865
|
[1.1.12]: https://github.com/bymaxone/nest-storage/compare/v1.1.11...v1.1.12
|
|
829
|
-
[
|
|
866
|
+
[1.1.13]: https://github.com/bymaxone/nest-storage/compare/v1.1.12...v1.1.13
|
|
867
|
+
[1.1.14]: https://github.com/bymaxone/nest-storage/compare/v1.1.13...v1.1.14
|
|
868
|
+
[Unreleased]: https://github.com/bymaxone/nest-storage/compare/v1.1.14...HEAD
|
package/README.md
CHANGED
|
@@ -598,8 +598,8 @@ bodies are not affected, and `post-upload` hands the scanner the key only, no bo
|
|
|
598
598
|
when the scanner rejects it, so the library deletes it and rejects `upload()` with the verdict
|
|
599
599
|
(`STORAGE_SCAN_INFECTED` or `STORAGE_SCAN_INCONCLUSIVE`). If that delete fails the verdict is still the
|
|
600
600
|
error, and `details` carries `cleanup: 'failed'`, the `key` you passed, the `storedKey` (with any
|
|
601
|
-
`keyPrefix`, as a bucket listing shows it) and the delete failure in `cleanupError`; the
|
|
602
|
-
|
|
601
|
+
`keyPrefix`, as a bucket listing shows it) and the code of the delete failure in `cleanupError`; the
|
|
602
|
+
failure, with what the provider said, is logged at error level as `STORAGE_SCAN_CLEANUP_FAILED`. With no `details.cleanup` the object was
|
|
603
603
|
removed; with `cleanup: 'failed'` it is **still stored** and has to be removed by other means.
|
|
604
604
|
|
|
605
605
|
---
|
|
@@ -760,12 +760,12 @@ All errors are thrown as `StorageException extends HttpException`. The response
|
|
|
760
760
|
| `STORAGE_SCAN_INFECTED` | 422 | Scanner returned `'infected'` (`details.threat`); `details.cleanup: 'failed'` means the object is still stored |
|
|
761
761
|
| `STORAGE_SCAN_INCONCLUSIVE` | 422 | Scanner returned `'unknown'` and `rejectOnUnknown: true` |
|
|
762
762
|
| `STORAGE_OBJECT_NOT_FOUND` | 404 | `head()`, `download()`, or `copy()` on a nonexistent key |
|
|
763
|
-
| `STORAGE_PROVIDER_ERROR` | 502 | The provider rejected the request (`AccessControlListNotSupported`, an `AccessDenied` on an object, any other rejection). **503** when it could not be reached (connection refused or reset, DNS failure) or answered with a server error (`SlowDown`, 500, 502, 504; not 501), which is worth retrying; **500** when it refused the credentials (`InvalidAccessKeyId`, `SignatureDoesNotMatch`), a deployment fault that a retry does not fix. `details` carries `awsCode`, `
|
|
763
|
+
| `STORAGE_PROVIDER_ERROR` | 502 | The provider rejected the request (`AccessControlListNotSupported`, an `AccessDenied` on an object, any other rejection). **503** when it could not be reached (connection refused or reset, DNS failure) or answered with a server error (`SlowDown`, 500, 502, 504; not 501), which is worth retrying; **500** when it refused the credentials (`InvalidAccessKeyId`, `SignatureDoesNotMatch`), a deployment fault that a retry does not fix (a `HEAD` answer carries no body and so no code: a refused `head()` stays **502** with reason `rejected`, and the credentials case is not told apart there). `details` carries `awsCode`, `httpStatus`, `requestId` and a fixed `reason` (`unreachable`, `provider-error`, `credentials-refused` or `rejected`; never the text of the driver or of the provider) plus the operation context (`op`, `bucket`, and `key` or `prefix`) |
|
|
764
764
|
| `STORAGE_SIGNED_URL_TTL_INVALID` | 400 | Per-request `ttlSeconds` ≤ 0 |
|
|
765
765
|
| `STORAGE_PART_TOO_SMALL` | 400 | Never thrown: a `multipart.partSizeBytes` below 5 MiB fails at init as `STORAGE_INVALID_CONFIG` |
|
|
766
766
|
| `STORAGE_INVALID_PART_COUNT` | 400 | `getMultipartUploadUrls()` with `parts` not an integer from 1 to 10 000 (`details.provided`, `details.reason`) |
|
|
767
767
|
| `STORAGE_BUCKET_UNDEFINED` | 400 | Operation without a bucket and no default in config |
|
|
768
|
-
| `STORAGE_MULTIPART_ABORTED` | 500 | A multipart upload failed (**503** when the cause is an outage, as for `STORAGE_PROVIDER_ERROR`). `details.abort` is `'completed'` when the library aborted it, `'failed'`
|
|
768
|
+
| `STORAGE_MULTIPART_ABORTED` | 500 | A multipart upload failed (**503** when the cause is an outage, as for `STORAGE_PROVIDER_ERROR`). `details.abort` is `'completed'` when the library aborted it, `'failed'` when the abort itself failed (logged as a warning, with what it threw): that upload then stays open on the provider until a lifecycle rule removes it |
|
|
769
769
|
| `STORAGE_INVALID_CONFIG` | 500 | `BymaxStorageModuleOptions` validation failed at initialization |
|
|
770
770
|
| `STORAGE_TIMEOUT` | 503 | An attempt got no response headers within `requestTimeoutMs`, a multipart source stayed silent for `uploadIdleTimeoutMs` (`details.phase: 'source-idle'`), or a download body stayed silent for `bodyIdleTimeoutMs` (`details.phase: 'body-idle'`) |
|
|
771
771
|
| `STORAGE_LIFECYCLE_RULE_INVALID` | 400 | `BucketLifecycleService` input rejected: bad `id`, day count or `prefix`, or an id naming a rule outside `keyPrefix` (`details.reason`) |
|
package/dist/server/index.cjs
CHANGED
|
@@ -144,9 +144,9 @@ var HEADER_VALUE = /^[\t \x21-\x7E]*$/;
|
|
|
144
144
|
var HEADER_NAME = /^[!#$%&'*+.^_`|~0-9a-z-]+$/;
|
|
145
145
|
var VALUE_REASON = "must be a string of tab, space and printable ASCII characters";
|
|
146
146
|
var KEY_REASON = "every key must be a non-empty header name of letters, digits and the characters !#$%&'*+-.^_`|~";
|
|
147
|
-
function refuse(
|
|
147
|
+
function refuse(field2, reason) {
|
|
148
148
|
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_VALIDATION_FAILED, void 0, {
|
|
149
|
-
field,
|
|
149
|
+
field: field2,
|
|
150
150
|
reason
|
|
151
151
|
});
|
|
152
152
|
}
|
|
@@ -154,9 +154,9 @@ function isHeaderValue(value) {
|
|
|
154
154
|
return typeof value === "string" && HEADER_VALUE.test(value);
|
|
155
155
|
}
|
|
156
156
|
function assertHeaderValues(fields) {
|
|
157
|
-
for (const [
|
|
157
|
+
for (const [field2, value] of Object.entries(fields)) {
|
|
158
158
|
if (value !== void 0 && !isHeaderValue(value)) {
|
|
159
|
-
refuse(
|
|
159
|
+
refuse(field2, VALUE_REASON);
|
|
160
160
|
}
|
|
161
161
|
}
|
|
162
162
|
}
|
|
@@ -509,6 +509,64 @@ KeyResolverService = __decorateClass([
|
|
|
509
509
|
common.Injectable(),
|
|
510
510
|
__decorateParam(0, common.Inject(BYMAX_STORAGE_OPTIONS))
|
|
511
511
|
], KeyResolverService);
|
|
512
|
+
|
|
513
|
+
// src/server/utils/describe-error.ts
|
|
514
|
+
var NON_PRINTABLE_REJECTION = "non-printable rejection value";
|
|
515
|
+
function describeError(value) {
|
|
516
|
+
try {
|
|
517
|
+
if (typeof value === "object" && value !== null) {
|
|
518
|
+
const message = value.message;
|
|
519
|
+
if (typeof message === "string") {
|
|
520
|
+
return message;
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
return String(value);
|
|
524
|
+
} catch {
|
|
525
|
+
return NON_PRINTABLE_REJECTION;
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
// src/server/utils/log-value.ts
|
|
530
|
+
var LEFT_LITERAL_BY_JSON = /[\p{Cc}\p{Cf}\p{Zl}\p{Zp}]/gu;
|
|
531
|
+
function logValue(value) {
|
|
532
|
+
const text = typeof value === "string" ? value : describeError(value);
|
|
533
|
+
return JSON.stringify(text).replace(
|
|
534
|
+
LEFT_LITERAL_BY_JSON,
|
|
535
|
+
(char) => {
|
|
536
|
+
const codePoint = Number(char.codePointAt(0));
|
|
537
|
+
const hex = codePoint.toString(16);
|
|
538
|
+
return codePoint > 65535 ? `\\u{${hex}}` : `\\u${hex.padStart(4, "0")}`;
|
|
539
|
+
}
|
|
540
|
+
);
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
// src/server/utils/provider-failure-log.ts
|
|
544
|
+
var SECRET_IN_MESSAGE = /(X-Amz-(?:Signature|Credential|Security-Token)|AWSAccessKeyId|Credential|Signature)=[^&\s"']+/gi;
|
|
545
|
+
var BARE_ACCESS_KEY_ID = /\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/g;
|
|
546
|
+
var MAX_MESSAGE_LENGTH = 300;
|
|
547
|
+
var logger = new common.Logger("StorageProvider");
|
|
548
|
+
function field(value) {
|
|
549
|
+
return typeof value === "string" || typeof value === "number" ? logValue(value) : "-";
|
|
550
|
+
}
|
|
551
|
+
function describeProviderFailure(err) {
|
|
552
|
+
const failure = typeof err === "object" && err !== null ? err : {};
|
|
553
|
+
const message = describeError(err).replace(SECRET_IN_MESSAGE, "$1=[redacted]").replace(BARE_ACCESS_KEY_ID, "[redacted-key-id]");
|
|
554
|
+
return [
|
|
555
|
+
`name=${field(failure.name)}`,
|
|
556
|
+
`code=${field(failure.Code)}`,
|
|
557
|
+
`status=${field(failure.$metadata?.httpStatusCode)}`,
|
|
558
|
+
`requestId=${field(failure.$metadata?.requestId)}`,
|
|
559
|
+
`message=${logValue(message.slice(0, MAX_MESSAGE_LENGTH))}`
|
|
560
|
+
].join(" ");
|
|
561
|
+
}
|
|
562
|
+
function logProviderFailure(err, context, label = "Provider call failed") {
|
|
563
|
+
if (err?.name === "AbortError") {
|
|
564
|
+
return;
|
|
565
|
+
}
|
|
566
|
+
const op = typeof context?.op === "string" ? logValue(context.op) : "-";
|
|
567
|
+
const key = typeof context?.key === "string" ? logValue(context.key) : "-";
|
|
568
|
+
logger.warn(`${label}: op=${op} key=${key} ${describeProviderFailure(err)}`);
|
|
569
|
+
}
|
|
512
570
|
var UNREACHABLE_CODES = /* @__PURE__ */ new Set([
|
|
513
571
|
"ECONNREFUSED",
|
|
514
572
|
"ECONNRESET",
|
|
@@ -524,7 +582,10 @@ var UNREACHABLE_IN_MESSAGE = new RegExp(
|
|
|
524
582
|
`^(?:AWS SDK error wrapper for \\w+: )?(?:(?:connect|read|write|getaddrinfo) (?:${[...UNREACHABLE_CODES].join("|")})\\b|socket hang up)`
|
|
525
583
|
);
|
|
526
584
|
var ADDRESSES_FAILED = "AWS SDK error wrapper for AggregateError";
|
|
527
|
-
var CREDENTIAL_CODES = /* @__PURE__ */ new Set([
|
|
585
|
+
var CREDENTIAL_CODES = /* @__PURE__ */ new Set([
|
|
586
|
+
"InvalidAccessKeyId",
|
|
587
|
+
"SignatureDoesNotMatch"
|
|
588
|
+
]);
|
|
528
589
|
function cutConnection(failure, noAnswer) {
|
|
529
590
|
const codes = [failure.code, failure.cause?.code];
|
|
530
591
|
if (Array.isArray(failure.errors)) {
|
|
@@ -536,20 +597,33 @@ function cutConnection(failure, noAnswer) {
|
|
|
536
597
|
const message = typeof failure.message === "string" ? failure.message : "";
|
|
537
598
|
return noAnswer && (UNREACHABLE_IN_MESSAGE.test(message) || message.startsWith(ADDRESSES_FAILED));
|
|
538
599
|
}
|
|
539
|
-
function
|
|
600
|
+
function providerFailureReason(err) {
|
|
540
601
|
const failure = err ?? {};
|
|
541
602
|
const status = failure.$metadata?.httpStatusCode;
|
|
542
|
-
if (failure.name === "TimeoutError"
|
|
543
|
-
return "
|
|
603
|
+
if (failure.name === "TimeoutError" && failure.code === "ETIMEDOUT") {
|
|
604
|
+
return "timeout";
|
|
605
|
+
}
|
|
606
|
+
if (cutConnection(failure, status === void 0)) {
|
|
607
|
+
return "unreachable";
|
|
608
|
+
}
|
|
609
|
+
if (failure.name === "TimeoutError") {
|
|
610
|
+
return "timeout";
|
|
544
611
|
}
|
|
545
612
|
if (typeof status === "number" && status >= 500 && status !== 501) {
|
|
546
|
-
return "
|
|
613
|
+
return "provider-error";
|
|
547
614
|
}
|
|
548
615
|
const awsCode = failure.Code ?? failure.name;
|
|
549
616
|
if (status === 403 && typeof awsCode === "string" && CREDENTIAL_CODES.has(awsCode)) {
|
|
550
617
|
return "credentials-refused";
|
|
551
618
|
}
|
|
552
|
-
return "
|
|
619
|
+
return "rejected";
|
|
620
|
+
}
|
|
621
|
+
function classifyProviderFailure(err) {
|
|
622
|
+
const reason = providerFailureReason(err);
|
|
623
|
+
if (reason === "credentials-refused") {
|
|
624
|
+
return "credentials-refused";
|
|
625
|
+
}
|
|
626
|
+
return reason === "rejected" ? "other" : "unavailable";
|
|
553
627
|
}
|
|
554
628
|
function providerFailureStatus(err) {
|
|
555
629
|
const failure = classifyProviderFailure(err);
|
|
@@ -560,19 +634,27 @@ function providerFailureStatus(err) {
|
|
|
560
634
|
}
|
|
561
635
|
|
|
562
636
|
// src/server/errors/aws-error-mapper.ts
|
|
637
|
+
function awsCodeOf(err) {
|
|
638
|
+
const e = err ?? {};
|
|
639
|
+
return e.Code ?? e.name;
|
|
640
|
+
}
|
|
641
|
+
function isNoSuchBucket(err) {
|
|
642
|
+
return awsCodeOf(err) === "NoSuchBucket";
|
|
643
|
+
}
|
|
563
644
|
function mapAwsError(err, context) {
|
|
564
645
|
const e = err ?? {};
|
|
565
646
|
const httpStatus = e.$metadata?.httpStatusCode;
|
|
566
|
-
const awsCode =
|
|
647
|
+
const awsCode = awsCodeOf(err);
|
|
567
648
|
const requestId = e.$metadata?.requestId;
|
|
568
649
|
const details = {
|
|
569
650
|
...context ?? {},
|
|
570
651
|
awsCode,
|
|
571
652
|
httpStatus,
|
|
572
653
|
requestId,
|
|
573
|
-
|
|
654
|
+
// A fixed word, never what the driver or the provider wrote.
|
|
655
|
+
reason: providerFailureReason(err)
|
|
574
656
|
};
|
|
575
|
-
if (
|
|
657
|
+
if (isNoSuchBucket(err)) {
|
|
576
658
|
return new StorageException(
|
|
577
659
|
STORAGE_ERROR_CODES.STORAGE_BUCKET_NOT_FOUND,
|
|
578
660
|
common.HttpStatus.NOT_FOUND,
|
|
@@ -587,7 +669,11 @@ function mapAwsError(err, context) {
|
|
|
587
669
|
);
|
|
588
670
|
}
|
|
589
671
|
if (httpStatus === 304) {
|
|
590
|
-
return new StorageException(
|
|
672
|
+
return new StorageException(
|
|
673
|
+
STORAGE_ERROR_CODES.STORAGE_NOT_MODIFIED,
|
|
674
|
+
common.HttpStatus.NOT_MODIFIED,
|
|
675
|
+
details
|
|
676
|
+
);
|
|
591
677
|
}
|
|
592
678
|
if (httpStatus === 412) {
|
|
593
679
|
return new StorageException(
|
|
@@ -597,8 +683,10 @@ function mapAwsError(err, context) {
|
|
|
597
683
|
);
|
|
598
684
|
}
|
|
599
685
|
if (e.name === "TimeoutError") {
|
|
686
|
+
logProviderFailure(err, context);
|
|
600
687
|
return new StorageException(STORAGE_ERROR_CODES.STORAGE_TIMEOUT, void 0, details);
|
|
601
688
|
}
|
|
689
|
+
logProviderFailure(err, context);
|
|
602
690
|
return new StorageException(
|
|
603
691
|
STORAGE_ERROR_CODES.STORAGE_PROVIDER_ERROR,
|
|
604
692
|
providerFailureStatus(err),
|
|
@@ -630,10 +718,18 @@ function mapLifecycleError(err, context) {
|
|
|
630
718
|
const status = e.$metadata?.httpStatusCode ?? 0;
|
|
631
719
|
const details = mapped.getResponse().error.details;
|
|
632
720
|
if (UNSUPPORTED_NAMES.has(name) || UNSUPPORTED_STATUSES.has(status)) {
|
|
633
|
-
return new StorageException(
|
|
721
|
+
return new StorageException(
|
|
722
|
+
STORAGE_ERROR_CODES.STORAGE_LIFECYCLE_UNSUPPORTED,
|
|
723
|
+
void 0,
|
|
724
|
+
details
|
|
725
|
+
);
|
|
634
726
|
}
|
|
635
727
|
if (REJECTED_NAMES.has(name)) {
|
|
636
|
-
return new StorageException(
|
|
728
|
+
return new StorageException(
|
|
729
|
+
STORAGE_ERROR_CODES.STORAGE_LIFECYCLE_REJECTED,
|
|
730
|
+
void 0,
|
|
731
|
+
details
|
|
732
|
+
);
|
|
637
733
|
}
|
|
638
734
|
return mapped;
|
|
639
735
|
}
|
|
@@ -650,9 +746,9 @@ function ruleInvalid(reason) {
|
|
|
650
746
|
reason
|
|
651
747
|
});
|
|
652
748
|
}
|
|
653
|
-
function assertDays(
|
|
749
|
+
function assertDays(field2, value) {
|
|
654
750
|
if (!Number.isInteger(value) || value < 1) {
|
|
655
|
-
throw ruleInvalid(`${
|
|
751
|
+
throw ruleInvalid(`${field2} must be an integer of at least 1`);
|
|
656
752
|
}
|
|
657
753
|
}
|
|
658
754
|
exports.BucketLifecycleService = class BucketLifecycleService {
|
|
@@ -1120,38 +1216,6 @@ ValidationService = __decorateClass([
|
|
|
1120
1216
|
__decorateParam(0, common.Inject(BYMAX_STORAGE_OPTIONS)),
|
|
1121
1217
|
__decorateParam(1, common.Inject(BYMAX_STORAGE_UPLOAD_VALIDATORS))
|
|
1122
1218
|
], ValidationService);
|
|
1123
|
-
|
|
1124
|
-
// src/server/utils/describe-error.ts
|
|
1125
|
-
var NON_PRINTABLE_REJECTION = "non-printable rejection value";
|
|
1126
|
-
function describeError(value) {
|
|
1127
|
-
try {
|
|
1128
|
-
if (typeof value === "object" && value !== null) {
|
|
1129
|
-
const message = value.message;
|
|
1130
|
-
if (typeof message === "string") {
|
|
1131
|
-
return message;
|
|
1132
|
-
}
|
|
1133
|
-
}
|
|
1134
|
-
return String(value);
|
|
1135
|
-
} catch {
|
|
1136
|
-
return NON_PRINTABLE_REJECTION;
|
|
1137
|
-
}
|
|
1138
|
-
}
|
|
1139
|
-
|
|
1140
|
-
// src/server/utils/log-value.ts
|
|
1141
|
-
var LEFT_LITERAL_BY_JSON = /[\p{Cc}\p{Cf}\p{Zl}\p{Zp}]/gu;
|
|
1142
|
-
function logValue(value) {
|
|
1143
|
-
const text = typeof value === "string" ? value : describeError(value);
|
|
1144
|
-
return JSON.stringify(text).replace(
|
|
1145
|
-
LEFT_LITERAL_BY_JSON,
|
|
1146
|
-
(char) => {
|
|
1147
|
-
const codePoint = Number(char.codePointAt(0));
|
|
1148
|
-
const hex = codePoint.toString(16);
|
|
1149
|
-
return codePoint > 65535 ? `\\u{${hex}}` : `\\u${hex.padStart(4, "0")}`;
|
|
1150
|
-
}
|
|
1151
|
-
);
|
|
1152
|
-
}
|
|
1153
|
-
|
|
1154
|
-
// src/server/services/file-scanner.service.ts
|
|
1155
1219
|
var FileScannerService = class {
|
|
1156
1220
|
constructor(options, scanner) {
|
|
1157
1221
|
this.options = options;
|
|
@@ -1338,15 +1402,16 @@ function mapMultipartError(err, context) {
|
|
|
1338
1402
|
if (tooLong) {
|
|
1339
1403
|
return tooLong;
|
|
1340
1404
|
}
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
return bucketMissing;
|
|
1405
|
+
if (isNoSuchBucket(err)) {
|
|
1406
|
+
return mapAwsError(err, context);
|
|
1344
1407
|
}
|
|
1408
|
+
logProviderFailure(err, context, "Multipart upload failed");
|
|
1345
1409
|
return new StorageException(
|
|
1346
1410
|
STORAGE_ERROR_CODES.STORAGE_MULTIPART_ABORTED,
|
|
1347
1411
|
// An outage ends the upload as an outage; any other cause is a failure of the upload itself.
|
|
1348
1412
|
providerFailureStatus(err),
|
|
1349
|
-
|
|
1413
|
+
// A fixed word, never what the driver or the provider wrote.
|
|
1414
|
+
{ ...context, reason: providerFailureReason(err) },
|
|
1350
1415
|
// The code's message says the upload was aborted; it is not, when the abort itself failed.
|
|
1351
1416
|
context.abort === "failed" ? ABORT_FAILED_MESSAGE : void 0
|
|
1352
1417
|
);
|
|
@@ -1355,26 +1420,31 @@ function knownStatus(error) {
|
|
|
1355
1420
|
const name = common.HttpStatus[error.getStatus()];
|
|
1356
1421
|
return name === void 0 ? void 0 : common.HttpStatus[name];
|
|
1357
1422
|
}
|
|
1358
|
-
function
|
|
1423
|
+
function cleanupErrorCode(error) {
|
|
1424
|
+
return error instanceof StorageException ? error.code : "UNKNOWN";
|
|
1425
|
+
}
|
|
1426
|
+
function describeCleanupFailure(error) {
|
|
1359
1427
|
if (!(error instanceof StorageException)) {
|
|
1360
1428
|
return describeError(error);
|
|
1361
1429
|
}
|
|
1362
|
-
|
|
1363
|
-
const awsMessage = body.error?.details?.awsMessage;
|
|
1364
|
-
return typeof awsMessage === "string" ? `${error.code}: ${awsMessage}` : error.code;
|
|
1430
|
+
return error.code;
|
|
1365
1431
|
}
|
|
1366
1432
|
function withCleanupFailure(scanError, keys, cleanupError) {
|
|
1367
1433
|
if (!(scanError instanceof StorageException)) {
|
|
1368
1434
|
return scanError;
|
|
1369
1435
|
}
|
|
1370
1436
|
const body = scanError.getResponse();
|
|
1371
|
-
return new StorageException(
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1437
|
+
return new StorageException(
|
|
1438
|
+
scanError.code,
|
|
1439
|
+
knownStatus(scanError),
|
|
1440
|
+
{
|
|
1441
|
+
...body.error?.details ?? {},
|
|
1442
|
+
key: keys.key,
|
|
1443
|
+
storedKey: keys.storedKey,
|
|
1444
|
+
cleanup: "failed",
|
|
1445
|
+
cleanupError: cleanupErrorCode(cleanupError)
|
|
1446
|
+
}
|
|
1447
|
+
);
|
|
1378
1448
|
}
|
|
1379
1449
|
var CLEANUP_DEADLINE_MS = 1e4;
|
|
1380
1450
|
function reasonName(signal) {
|
|
@@ -1465,7 +1535,7 @@ async function abortable(work, signal, context) {
|
|
|
1465
1535
|
}
|
|
1466
1536
|
|
|
1467
1537
|
// src/server/utils/multipart-abort.ts
|
|
1468
|
-
async function closeMultipartUpload(client, bucket, key, uploadId) {
|
|
1538
|
+
async function closeMultipartUpload(client, bucket, key, uploadId, onFailure) {
|
|
1469
1539
|
if (uploadId === void 0) {
|
|
1470
1540
|
return {};
|
|
1471
1541
|
}
|
|
@@ -1479,11 +1549,19 @@ async function closeMultipartUpload(client, bucket, key, uploadId) {
|
|
|
1479
1549
|
if (err?.name === "NoSuchUpload") {
|
|
1480
1550
|
return { abort: "completed" };
|
|
1481
1551
|
}
|
|
1482
|
-
|
|
1552
|
+
onFailure?.(err);
|
|
1553
|
+
return { abort: "failed" };
|
|
1483
1554
|
}
|
|
1484
1555
|
}
|
|
1485
|
-
function closeWhenCreated(client, bucket, key, created) {
|
|
1486
|
-
void created.then((uploadId) => closeMultipartUpload(client, bucket, key, uploadId));
|
|
1556
|
+
function closeWhenCreated(client, bucket, key, created, onFailure) {
|
|
1557
|
+
void created.then((uploadId) => closeMultipartUpload(client, bucket, key, uploadId, onFailure));
|
|
1558
|
+
}
|
|
1559
|
+
function abortFailureLogger(logger2, key) {
|
|
1560
|
+
return (err) => {
|
|
1561
|
+
logger2.warn(
|
|
1562
|
+
`Multipart abort failed, the upload stays open on the provider: ${logValue(key)} ${describeProviderFailure(err)}`
|
|
1563
|
+
);
|
|
1564
|
+
};
|
|
1487
1565
|
}
|
|
1488
1566
|
async function deleteObjectsChunk(client, bucket, chunk, stripPrefix) {
|
|
1489
1567
|
const deleted = [];
|
|
@@ -1505,14 +1583,16 @@ async function deleteObjectsChunk(client, bucket, chunk, stripPrefix) {
|
|
|
1505
1583
|
if (failure.Key !== void 0) {
|
|
1506
1584
|
failed.push({
|
|
1507
1585
|
key: stripPrefix(failure.Key),
|
|
1508
|
-
|
|
1586
|
+
// The provider's code, which a caller can branch on; its sentence is not repeated.
|
|
1587
|
+
error: failure.Code ?? "Unknown"
|
|
1509
1588
|
});
|
|
1510
1589
|
}
|
|
1511
1590
|
}
|
|
1512
1591
|
} catch (err) {
|
|
1513
|
-
|
|
1592
|
+
logProviderFailure(err, { op: "deleteMany" });
|
|
1593
|
+
const reason = providerFailureReason(err);
|
|
1514
1594
|
for (const key of chunk) {
|
|
1515
|
-
failed.push({ key: stripPrefix(key), error:
|
|
1595
|
+
failed.push({ key: stripPrefix(key), error: reason });
|
|
1516
1596
|
}
|
|
1517
1597
|
}
|
|
1518
1598
|
return { deleted, failed };
|
|
@@ -1520,7 +1600,12 @@ async function deleteObjectsChunk(client, bucket, chunk, stripPrefix) {
|
|
|
1520
1600
|
function mapBucketCheckError(err, context) {
|
|
1521
1601
|
const failure = err ?? {};
|
|
1522
1602
|
const status = failure.$metadata?.httpStatusCode;
|
|
1523
|
-
const details = {
|
|
1603
|
+
const details = {
|
|
1604
|
+
...context,
|
|
1605
|
+
httpStatus: status,
|
|
1606
|
+
awsCode: failure.name,
|
|
1607
|
+
reason: providerFailureReason(err)
|
|
1608
|
+
};
|
|
1524
1609
|
if (status === 301 || failure.name === "PermanentRedirect") {
|
|
1525
1610
|
return new StorageException(
|
|
1526
1611
|
STORAGE_ERROR_CODES.STORAGE_BUCKET_REGION_MISMATCH,
|
|
@@ -1807,7 +1892,11 @@ exports.StorageService = class StorageService {
|
|
|
1807
1892
|
const signal = options.abortSignal;
|
|
1808
1893
|
const input = validationInput(options, finalKey);
|
|
1809
1894
|
const validated = await abortable(this.validation.validate(input), signal, context);
|
|
1810
|
-
const scanned = await abortable(
|
|
1895
|
+
const scanned = await abortable(
|
|
1896
|
+
this.bufferForPreUploadScan(validated.body, finalKey),
|
|
1897
|
+
signal,
|
|
1898
|
+
context
|
|
1899
|
+
);
|
|
1811
1900
|
const body = boundStreamSize(scanned, options.size, this.options);
|
|
1812
1901
|
const validatedOptions = { ...options, body };
|
|
1813
1902
|
await abortable(this.runPreUploadScan(body, finalKey, bucket, options), signal, context);
|
|
@@ -1897,7 +1986,7 @@ exports.StorageService = class StorageService {
|
|
|
1897
1986
|
} catch (deleteErr) {
|
|
1898
1987
|
this.logger.error(
|
|
1899
1988
|
`STORAGE_SCAN_CLEANUP_FAILED: the rejected object is still stored: ${logValue(finalKey)}`,
|
|
1900
|
-
logValue(
|
|
1989
|
+
logValue(describeCleanupFailure(deleteErr))
|
|
1901
1990
|
);
|
|
1902
1991
|
throw withCleanupFailure(scanErr, { key, storedKey: finalKey }, deleteErr);
|
|
1903
1992
|
}
|
|
@@ -1919,7 +2008,10 @@ exports.StorageService = class StorageService {
|
|
|
1919
2008
|
const context = { key: finalKey, bucket, op: "head" };
|
|
1920
2009
|
throwIfAborted(options?.abortSignal, context);
|
|
1921
2010
|
try {
|
|
1922
|
-
const response = await this.s3Provider.getClient().send(
|
|
2011
|
+
const response = await this.s3Provider.getClient().send(
|
|
2012
|
+
new clientS3.HeadObjectCommand({ Bucket: bucket, Key: finalKey }),
|
|
2013
|
+
sendOptions(options?.abortSignal)
|
|
2014
|
+
);
|
|
1923
2015
|
return this.toObjectMetadata(response, finalKey, bucket);
|
|
1924
2016
|
} catch (err) {
|
|
1925
2017
|
throw failureOrAbort(options?.abortSignal, context, mapAwsError(err, context));
|
|
@@ -1944,7 +2036,9 @@ exports.StorageService = class StorageService {
|
|
|
1944
2036
|
if (err instanceof StorageException && err.code === STORAGE_ERROR_CODES.STORAGE_REQUEST_ABORTED) {
|
|
1945
2037
|
throw err;
|
|
1946
2038
|
}
|
|
1947
|
-
this.logger.warn(
|
|
2039
|
+
this.logger.warn(
|
|
2040
|
+
`exists() \u2014 treating a non-404 error as "false": ${logValue(describeError(err))}`
|
|
2041
|
+
);
|
|
1948
2042
|
return false;
|
|
1949
2043
|
}
|
|
1950
2044
|
}
|
|
@@ -1972,7 +2066,11 @@ exports.StorageService = class StorageService {
|
|
|
1972
2066
|
*/
|
|
1973
2067
|
async download(options) {
|
|
1974
2068
|
this.assertConfigured();
|
|
1975
|
-
assertHeaderValues({
|
|
2069
|
+
assertHeaderValues({
|
|
2070
|
+
range: options.range,
|
|
2071
|
+
ifNoneMatch: options.ifNoneMatch,
|
|
2072
|
+
ifMatch: options.ifMatch
|
|
2073
|
+
});
|
|
1976
2074
|
const finalKey = this.keyResolver.normalize(options.key);
|
|
1977
2075
|
const bucket = this.resolveBucket(options.bucket);
|
|
1978
2076
|
const context = { key: finalKey, bucket, op: "download" };
|
|
@@ -2036,7 +2134,10 @@ exports.StorageService = class StorageService {
|
|
|
2036
2134
|
const context = { key: finalKey, bucket, op: "delete" };
|
|
2037
2135
|
throwIfAborted(options?.abortSignal, context);
|
|
2038
2136
|
try {
|
|
2039
|
-
await this.s3Provider.getClient().send(
|
|
2137
|
+
await this.s3Provider.getClient().send(
|
|
2138
|
+
new clientS3.DeleteObjectCommand({ Bucket: bucket, Key: finalKey }),
|
|
2139
|
+
sendOptions(options?.abortSignal)
|
|
2140
|
+
);
|
|
2040
2141
|
} catch (err) {
|
|
2041
2142
|
const mapped = failureOrAbort(options?.abortSignal, context, mapAwsError(err, context));
|
|
2042
2143
|
if (mapped.code !== STORAGE_ERROR_CODES.STORAGE_OBJECT_NOT_FOUND) {
|
|
@@ -2184,9 +2285,17 @@ exports.StorageService = class StorageService {
|
|
|
2184
2285
|
let uploadId = upload.uploadId;
|
|
2185
2286
|
if (uploadId === void 0 && lookup) {
|
|
2186
2287
|
uploadId = await within(lookup, CLEANUP_DEADLINE_MS);
|
|
2187
|
-
if (uploadId === void 0)
|
|
2288
|
+
if (uploadId === void 0) {
|
|
2289
|
+
closeWhenCreated(client, bucket, finalKey, lookup, abortFailureLogger(this.logger, finalKey));
|
|
2290
|
+
}
|
|
2188
2291
|
}
|
|
2189
|
-
const abort = await closeMultipartUpload(
|
|
2292
|
+
const abort = await closeMultipartUpload(
|
|
2293
|
+
client,
|
|
2294
|
+
bucket,
|
|
2295
|
+
finalKey,
|
|
2296
|
+
uploadId,
|
|
2297
|
+
abortFailureLogger(this.logger, finalKey)
|
|
2298
|
+
);
|
|
2190
2299
|
const context = { key: finalKey, bucket, op: "upload-multipart", ...abort };
|
|
2191
2300
|
throw abortedBy ? requestAborted(abortedBy, context) : mapMultipartError(err, context);
|
|
2192
2301
|
}
|
package/dist/server/index.d.cts
CHANGED
|
@@ -502,9 +502,14 @@ interface CopyOptions {
|
|
|
502
502
|
interface DeleteManyOptions {
|
|
503
503
|
bucket?: string;
|
|
504
504
|
}
|
|
505
|
-
/** A single failed key from a batch delete
|
|
505
|
+
/** A single failed key from a batch delete. */
|
|
506
506
|
interface FailedDeletion {
|
|
507
507
|
key: string;
|
|
508
|
+
/**
|
|
509
|
+
* The provider's error code for this key (for example `AccessDenied`), or, when the whole request
|
|
510
|
+
* failed, a fixed reason: `unreachable`, `timeout`, `provider-error`, `credentials-refused` or
|
|
511
|
+
* `rejected`. Never the text of the driver or of the provider.
|
|
512
|
+
*/
|
|
508
513
|
error: string;
|
|
509
514
|
}
|
|
510
515
|
/** Aggregated outcome of a batch delete: succeeded keys and per-key failures. */
|
package/dist/server/index.d.ts
CHANGED
|
@@ -502,9 +502,14 @@ interface CopyOptions {
|
|
|
502
502
|
interface DeleteManyOptions {
|
|
503
503
|
bucket?: string;
|
|
504
504
|
}
|
|
505
|
-
/** A single failed key from a batch delete
|
|
505
|
+
/** A single failed key from a batch delete. */
|
|
506
506
|
interface FailedDeletion {
|
|
507
507
|
key: string;
|
|
508
|
+
/**
|
|
509
|
+
* The provider's error code for this key (for example `AccessDenied`), or, when the whole request
|
|
510
|
+
* failed, a fixed reason: `unreachable`, `timeout`, `provider-error`, `credentials-refused` or
|
|
511
|
+
* `rejected`. Never the text of the driver or of the provider.
|
|
512
|
+
*/
|
|
508
513
|
error: string;
|
|
509
514
|
}
|
|
510
515
|
/** Aggregated outcome of a batch delete: succeeded keys and per-key failures. */
|
package/dist/server/index.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { HttpStatus, Injectable, Inject, Global, Module,
|
|
1
|
+
import { HttpStatus, Injectable, Inject, Logger, Global, Module, HttpException } from '@nestjs/common';
|
|
2
2
|
import { S3Client, GetBucketLifecycleConfigurationCommand, DeleteBucketLifecycleCommand, PutBucketLifecycleConfigurationCommand, HeadObjectCommand, GetObjectCommand, DeleteObjectCommand, HeadBucketCommand, PutObjectCommand, ListObjectsV2Command, CopyObjectCommand, CreateMultipartUploadCommand, UploadPartCommand, CompleteMultipartUploadCommand, AbortMultipartUploadCommand, DeleteObjectsCommand } from '@aws-sdk/client-s3';
|
|
3
3
|
import { Upload } from '@aws-sdk/lib-storage';
|
|
4
4
|
import { PassThrough, Transform, pipeline, Readable } from 'stream';
|
|
@@ -142,9 +142,9 @@ var HEADER_VALUE = /^[\t \x21-\x7E]*$/;
|
|
|
142
142
|
var HEADER_NAME = /^[!#$%&'*+.^_`|~0-9a-z-]+$/;
|
|
143
143
|
var VALUE_REASON = "must be a string of tab, space and printable ASCII characters";
|
|
144
144
|
var KEY_REASON = "every key must be a non-empty header name of letters, digits and the characters !#$%&'*+-.^_`|~";
|
|
145
|
-
function refuse(
|
|
145
|
+
function refuse(field2, reason) {
|
|
146
146
|
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_VALIDATION_FAILED, void 0, {
|
|
147
|
-
field,
|
|
147
|
+
field: field2,
|
|
148
148
|
reason
|
|
149
149
|
});
|
|
150
150
|
}
|
|
@@ -152,9 +152,9 @@ function isHeaderValue(value) {
|
|
|
152
152
|
return typeof value === "string" && HEADER_VALUE.test(value);
|
|
153
153
|
}
|
|
154
154
|
function assertHeaderValues(fields) {
|
|
155
|
-
for (const [
|
|
155
|
+
for (const [field2, value] of Object.entries(fields)) {
|
|
156
156
|
if (value !== void 0 && !isHeaderValue(value)) {
|
|
157
|
-
refuse(
|
|
157
|
+
refuse(field2, VALUE_REASON);
|
|
158
158
|
}
|
|
159
159
|
}
|
|
160
160
|
}
|
|
@@ -507,6 +507,64 @@ KeyResolverService = __decorateClass([
|
|
|
507
507
|
Injectable(),
|
|
508
508
|
__decorateParam(0, Inject(BYMAX_STORAGE_OPTIONS))
|
|
509
509
|
], KeyResolverService);
|
|
510
|
+
|
|
511
|
+
// src/server/utils/describe-error.ts
|
|
512
|
+
var NON_PRINTABLE_REJECTION = "non-printable rejection value";
|
|
513
|
+
function describeError(value) {
|
|
514
|
+
try {
|
|
515
|
+
if (typeof value === "object" && value !== null) {
|
|
516
|
+
const message = value.message;
|
|
517
|
+
if (typeof message === "string") {
|
|
518
|
+
return message;
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
return String(value);
|
|
522
|
+
} catch {
|
|
523
|
+
return NON_PRINTABLE_REJECTION;
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
// src/server/utils/log-value.ts
|
|
528
|
+
var LEFT_LITERAL_BY_JSON = /[\p{Cc}\p{Cf}\p{Zl}\p{Zp}]/gu;
|
|
529
|
+
function logValue(value) {
|
|
530
|
+
const text = typeof value === "string" ? value : describeError(value);
|
|
531
|
+
return JSON.stringify(text).replace(
|
|
532
|
+
LEFT_LITERAL_BY_JSON,
|
|
533
|
+
(char) => {
|
|
534
|
+
const codePoint = Number(char.codePointAt(0));
|
|
535
|
+
const hex = codePoint.toString(16);
|
|
536
|
+
return codePoint > 65535 ? `\\u{${hex}}` : `\\u${hex.padStart(4, "0")}`;
|
|
537
|
+
}
|
|
538
|
+
);
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
// src/server/utils/provider-failure-log.ts
|
|
542
|
+
var SECRET_IN_MESSAGE = /(X-Amz-(?:Signature|Credential|Security-Token)|AWSAccessKeyId|Credential|Signature)=[^&\s"']+/gi;
|
|
543
|
+
var BARE_ACCESS_KEY_ID = /\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/g;
|
|
544
|
+
var MAX_MESSAGE_LENGTH = 300;
|
|
545
|
+
var logger = new Logger("StorageProvider");
|
|
546
|
+
function field(value) {
|
|
547
|
+
return typeof value === "string" || typeof value === "number" ? logValue(value) : "-";
|
|
548
|
+
}
|
|
549
|
+
function describeProviderFailure(err) {
|
|
550
|
+
const failure = typeof err === "object" && err !== null ? err : {};
|
|
551
|
+
const message = describeError(err).replace(SECRET_IN_MESSAGE, "$1=[redacted]").replace(BARE_ACCESS_KEY_ID, "[redacted-key-id]");
|
|
552
|
+
return [
|
|
553
|
+
`name=${field(failure.name)}`,
|
|
554
|
+
`code=${field(failure.Code)}`,
|
|
555
|
+
`status=${field(failure.$metadata?.httpStatusCode)}`,
|
|
556
|
+
`requestId=${field(failure.$metadata?.requestId)}`,
|
|
557
|
+
`message=${logValue(message.slice(0, MAX_MESSAGE_LENGTH))}`
|
|
558
|
+
].join(" ");
|
|
559
|
+
}
|
|
560
|
+
function logProviderFailure(err, context, label = "Provider call failed") {
|
|
561
|
+
if (err?.name === "AbortError") {
|
|
562
|
+
return;
|
|
563
|
+
}
|
|
564
|
+
const op = typeof context?.op === "string" ? logValue(context.op) : "-";
|
|
565
|
+
const key = typeof context?.key === "string" ? logValue(context.key) : "-";
|
|
566
|
+
logger.warn(`${label}: op=${op} key=${key} ${describeProviderFailure(err)}`);
|
|
567
|
+
}
|
|
510
568
|
var UNREACHABLE_CODES = /* @__PURE__ */ new Set([
|
|
511
569
|
"ECONNREFUSED",
|
|
512
570
|
"ECONNRESET",
|
|
@@ -522,7 +580,10 @@ var UNREACHABLE_IN_MESSAGE = new RegExp(
|
|
|
522
580
|
`^(?:AWS SDK error wrapper for \\w+: )?(?:(?:connect|read|write|getaddrinfo) (?:${[...UNREACHABLE_CODES].join("|")})\\b|socket hang up)`
|
|
523
581
|
);
|
|
524
582
|
var ADDRESSES_FAILED = "AWS SDK error wrapper for AggregateError";
|
|
525
|
-
var CREDENTIAL_CODES = /* @__PURE__ */ new Set([
|
|
583
|
+
var CREDENTIAL_CODES = /* @__PURE__ */ new Set([
|
|
584
|
+
"InvalidAccessKeyId",
|
|
585
|
+
"SignatureDoesNotMatch"
|
|
586
|
+
]);
|
|
526
587
|
function cutConnection(failure, noAnswer) {
|
|
527
588
|
const codes = [failure.code, failure.cause?.code];
|
|
528
589
|
if (Array.isArray(failure.errors)) {
|
|
@@ -534,20 +595,33 @@ function cutConnection(failure, noAnswer) {
|
|
|
534
595
|
const message = typeof failure.message === "string" ? failure.message : "";
|
|
535
596
|
return noAnswer && (UNREACHABLE_IN_MESSAGE.test(message) || message.startsWith(ADDRESSES_FAILED));
|
|
536
597
|
}
|
|
537
|
-
function
|
|
598
|
+
function providerFailureReason(err) {
|
|
538
599
|
const failure = err ?? {};
|
|
539
600
|
const status = failure.$metadata?.httpStatusCode;
|
|
540
|
-
if (failure.name === "TimeoutError"
|
|
541
|
-
return "
|
|
601
|
+
if (failure.name === "TimeoutError" && failure.code === "ETIMEDOUT") {
|
|
602
|
+
return "timeout";
|
|
603
|
+
}
|
|
604
|
+
if (cutConnection(failure, status === void 0)) {
|
|
605
|
+
return "unreachable";
|
|
606
|
+
}
|
|
607
|
+
if (failure.name === "TimeoutError") {
|
|
608
|
+
return "timeout";
|
|
542
609
|
}
|
|
543
610
|
if (typeof status === "number" && status >= 500 && status !== 501) {
|
|
544
|
-
return "
|
|
611
|
+
return "provider-error";
|
|
545
612
|
}
|
|
546
613
|
const awsCode = failure.Code ?? failure.name;
|
|
547
614
|
if (status === 403 && typeof awsCode === "string" && CREDENTIAL_CODES.has(awsCode)) {
|
|
548
615
|
return "credentials-refused";
|
|
549
616
|
}
|
|
550
|
-
return "
|
|
617
|
+
return "rejected";
|
|
618
|
+
}
|
|
619
|
+
function classifyProviderFailure(err) {
|
|
620
|
+
const reason = providerFailureReason(err);
|
|
621
|
+
if (reason === "credentials-refused") {
|
|
622
|
+
return "credentials-refused";
|
|
623
|
+
}
|
|
624
|
+
return reason === "rejected" ? "other" : "unavailable";
|
|
551
625
|
}
|
|
552
626
|
function providerFailureStatus(err) {
|
|
553
627
|
const failure = classifyProviderFailure(err);
|
|
@@ -558,19 +632,27 @@ function providerFailureStatus(err) {
|
|
|
558
632
|
}
|
|
559
633
|
|
|
560
634
|
// src/server/errors/aws-error-mapper.ts
|
|
635
|
+
function awsCodeOf(err) {
|
|
636
|
+
const e = err ?? {};
|
|
637
|
+
return e.Code ?? e.name;
|
|
638
|
+
}
|
|
639
|
+
function isNoSuchBucket(err) {
|
|
640
|
+
return awsCodeOf(err) === "NoSuchBucket";
|
|
641
|
+
}
|
|
561
642
|
function mapAwsError(err, context) {
|
|
562
643
|
const e = err ?? {};
|
|
563
644
|
const httpStatus = e.$metadata?.httpStatusCode;
|
|
564
|
-
const awsCode =
|
|
645
|
+
const awsCode = awsCodeOf(err);
|
|
565
646
|
const requestId = e.$metadata?.requestId;
|
|
566
647
|
const details = {
|
|
567
648
|
...context ?? {},
|
|
568
649
|
awsCode,
|
|
569
650
|
httpStatus,
|
|
570
651
|
requestId,
|
|
571
|
-
|
|
652
|
+
// A fixed word, never what the driver or the provider wrote.
|
|
653
|
+
reason: providerFailureReason(err)
|
|
572
654
|
};
|
|
573
|
-
if (
|
|
655
|
+
if (isNoSuchBucket(err)) {
|
|
574
656
|
return new StorageException(
|
|
575
657
|
STORAGE_ERROR_CODES.STORAGE_BUCKET_NOT_FOUND,
|
|
576
658
|
HttpStatus.NOT_FOUND,
|
|
@@ -585,7 +667,11 @@ function mapAwsError(err, context) {
|
|
|
585
667
|
);
|
|
586
668
|
}
|
|
587
669
|
if (httpStatus === 304) {
|
|
588
|
-
return new StorageException(
|
|
670
|
+
return new StorageException(
|
|
671
|
+
STORAGE_ERROR_CODES.STORAGE_NOT_MODIFIED,
|
|
672
|
+
HttpStatus.NOT_MODIFIED,
|
|
673
|
+
details
|
|
674
|
+
);
|
|
589
675
|
}
|
|
590
676
|
if (httpStatus === 412) {
|
|
591
677
|
return new StorageException(
|
|
@@ -595,8 +681,10 @@ function mapAwsError(err, context) {
|
|
|
595
681
|
);
|
|
596
682
|
}
|
|
597
683
|
if (e.name === "TimeoutError") {
|
|
684
|
+
logProviderFailure(err, context);
|
|
598
685
|
return new StorageException(STORAGE_ERROR_CODES.STORAGE_TIMEOUT, void 0, details);
|
|
599
686
|
}
|
|
687
|
+
logProviderFailure(err, context);
|
|
600
688
|
return new StorageException(
|
|
601
689
|
STORAGE_ERROR_CODES.STORAGE_PROVIDER_ERROR,
|
|
602
690
|
providerFailureStatus(err),
|
|
@@ -628,10 +716,18 @@ function mapLifecycleError(err, context) {
|
|
|
628
716
|
const status = e.$metadata?.httpStatusCode ?? 0;
|
|
629
717
|
const details = mapped.getResponse().error.details;
|
|
630
718
|
if (UNSUPPORTED_NAMES.has(name) || UNSUPPORTED_STATUSES.has(status)) {
|
|
631
|
-
return new StorageException(
|
|
719
|
+
return new StorageException(
|
|
720
|
+
STORAGE_ERROR_CODES.STORAGE_LIFECYCLE_UNSUPPORTED,
|
|
721
|
+
void 0,
|
|
722
|
+
details
|
|
723
|
+
);
|
|
632
724
|
}
|
|
633
725
|
if (REJECTED_NAMES.has(name)) {
|
|
634
|
-
return new StorageException(
|
|
726
|
+
return new StorageException(
|
|
727
|
+
STORAGE_ERROR_CODES.STORAGE_LIFECYCLE_REJECTED,
|
|
728
|
+
void 0,
|
|
729
|
+
details
|
|
730
|
+
);
|
|
635
731
|
}
|
|
636
732
|
return mapped;
|
|
637
733
|
}
|
|
@@ -648,9 +744,9 @@ function ruleInvalid(reason) {
|
|
|
648
744
|
reason
|
|
649
745
|
});
|
|
650
746
|
}
|
|
651
|
-
function assertDays(
|
|
747
|
+
function assertDays(field2, value) {
|
|
652
748
|
if (!Number.isInteger(value) || value < 1) {
|
|
653
|
-
throw ruleInvalid(`${
|
|
749
|
+
throw ruleInvalid(`${field2} must be an integer of at least 1`);
|
|
654
750
|
}
|
|
655
751
|
}
|
|
656
752
|
var BucketLifecycleService = class {
|
|
@@ -1118,38 +1214,6 @@ ValidationService = __decorateClass([
|
|
|
1118
1214
|
__decorateParam(0, Inject(BYMAX_STORAGE_OPTIONS)),
|
|
1119
1215
|
__decorateParam(1, Inject(BYMAX_STORAGE_UPLOAD_VALIDATORS))
|
|
1120
1216
|
], ValidationService);
|
|
1121
|
-
|
|
1122
|
-
// src/server/utils/describe-error.ts
|
|
1123
|
-
var NON_PRINTABLE_REJECTION = "non-printable rejection value";
|
|
1124
|
-
function describeError(value) {
|
|
1125
|
-
try {
|
|
1126
|
-
if (typeof value === "object" && value !== null) {
|
|
1127
|
-
const message = value.message;
|
|
1128
|
-
if (typeof message === "string") {
|
|
1129
|
-
return message;
|
|
1130
|
-
}
|
|
1131
|
-
}
|
|
1132
|
-
return String(value);
|
|
1133
|
-
} catch {
|
|
1134
|
-
return NON_PRINTABLE_REJECTION;
|
|
1135
|
-
}
|
|
1136
|
-
}
|
|
1137
|
-
|
|
1138
|
-
// src/server/utils/log-value.ts
|
|
1139
|
-
var LEFT_LITERAL_BY_JSON = /[\p{Cc}\p{Cf}\p{Zl}\p{Zp}]/gu;
|
|
1140
|
-
function logValue(value) {
|
|
1141
|
-
const text = typeof value === "string" ? value : describeError(value);
|
|
1142
|
-
return JSON.stringify(text).replace(
|
|
1143
|
-
LEFT_LITERAL_BY_JSON,
|
|
1144
|
-
(char) => {
|
|
1145
|
-
const codePoint = Number(char.codePointAt(0));
|
|
1146
|
-
const hex = codePoint.toString(16);
|
|
1147
|
-
return codePoint > 65535 ? `\\u{${hex}}` : `\\u${hex.padStart(4, "0")}`;
|
|
1148
|
-
}
|
|
1149
|
-
);
|
|
1150
|
-
}
|
|
1151
|
-
|
|
1152
|
-
// src/server/services/file-scanner.service.ts
|
|
1153
1217
|
var FileScannerService = class {
|
|
1154
1218
|
constructor(options, scanner) {
|
|
1155
1219
|
this.options = options;
|
|
@@ -1336,15 +1400,16 @@ function mapMultipartError(err, context) {
|
|
|
1336
1400
|
if (tooLong) {
|
|
1337
1401
|
return tooLong;
|
|
1338
1402
|
}
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
return bucketMissing;
|
|
1403
|
+
if (isNoSuchBucket(err)) {
|
|
1404
|
+
return mapAwsError(err, context);
|
|
1342
1405
|
}
|
|
1406
|
+
logProviderFailure(err, context, "Multipart upload failed");
|
|
1343
1407
|
return new StorageException(
|
|
1344
1408
|
STORAGE_ERROR_CODES.STORAGE_MULTIPART_ABORTED,
|
|
1345
1409
|
// An outage ends the upload as an outage; any other cause is a failure of the upload itself.
|
|
1346
1410
|
providerFailureStatus(err),
|
|
1347
|
-
|
|
1411
|
+
// A fixed word, never what the driver or the provider wrote.
|
|
1412
|
+
{ ...context, reason: providerFailureReason(err) },
|
|
1348
1413
|
// The code's message says the upload was aborted; it is not, when the abort itself failed.
|
|
1349
1414
|
context.abort === "failed" ? ABORT_FAILED_MESSAGE : void 0
|
|
1350
1415
|
);
|
|
@@ -1353,26 +1418,31 @@ function knownStatus(error) {
|
|
|
1353
1418
|
const name = HttpStatus[error.getStatus()];
|
|
1354
1419
|
return name === void 0 ? void 0 : HttpStatus[name];
|
|
1355
1420
|
}
|
|
1356
|
-
function
|
|
1421
|
+
function cleanupErrorCode(error) {
|
|
1422
|
+
return error instanceof StorageException ? error.code : "UNKNOWN";
|
|
1423
|
+
}
|
|
1424
|
+
function describeCleanupFailure(error) {
|
|
1357
1425
|
if (!(error instanceof StorageException)) {
|
|
1358
1426
|
return describeError(error);
|
|
1359
1427
|
}
|
|
1360
|
-
|
|
1361
|
-
const awsMessage = body.error?.details?.awsMessage;
|
|
1362
|
-
return typeof awsMessage === "string" ? `${error.code}: ${awsMessage}` : error.code;
|
|
1428
|
+
return error.code;
|
|
1363
1429
|
}
|
|
1364
1430
|
function withCleanupFailure(scanError, keys, cleanupError) {
|
|
1365
1431
|
if (!(scanError instanceof StorageException)) {
|
|
1366
1432
|
return scanError;
|
|
1367
1433
|
}
|
|
1368
1434
|
const body = scanError.getResponse();
|
|
1369
|
-
return new StorageException(
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1435
|
+
return new StorageException(
|
|
1436
|
+
scanError.code,
|
|
1437
|
+
knownStatus(scanError),
|
|
1438
|
+
{
|
|
1439
|
+
...body.error?.details ?? {},
|
|
1440
|
+
key: keys.key,
|
|
1441
|
+
storedKey: keys.storedKey,
|
|
1442
|
+
cleanup: "failed",
|
|
1443
|
+
cleanupError: cleanupErrorCode(cleanupError)
|
|
1444
|
+
}
|
|
1445
|
+
);
|
|
1376
1446
|
}
|
|
1377
1447
|
var CLEANUP_DEADLINE_MS = 1e4;
|
|
1378
1448
|
function reasonName(signal) {
|
|
@@ -1463,7 +1533,7 @@ async function abortable(work, signal, context) {
|
|
|
1463
1533
|
}
|
|
1464
1534
|
|
|
1465
1535
|
// src/server/utils/multipart-abort.ts
|
|
1466
|
-
async function closeMultipartUpload(client, bucket, key, uploadId) {
|
|
1536
|
+
async function closeMultipartUpload(client, bucket, key, uploadId, onFailure) {
|
|
1467
1537
|
if (uploadId === void 0) {
|
|
1468
1538
|
return {};
|
|
1469
1539
|
}
|
|
@@ -1477,11 +1547,19 @@ async function closeMultipartUpload(client, bucket, key, uploadId) {
|
|
|
1477
1547
|
if (err?.name === "NoSuchUpload") {
|
|
1478
1548
|
return { abort: "completed" };
|
|
1479
1549
|
}
|
|
1480
|
-
|
|
1550
|
+
onFailure?.(err);
|
|
1551
|
+
return { abort: "failed" };
|
|
1481
1552
|
}
|
|
1482
1553
|
}
|
|
1483
|
-
function closeWhenCreated(client, bucket, key, created) {
|
|
1484
|
-
void created.then((uploadId) => closeMultipartUpload(client, bucket, key, uploadId));
|
|
1554
|
+
function closeWhenCreated(client, bucket, key, created, onFailure) {
|
|
1555
|
+
void created.then((uploadId) => closeMultipartUpload(client, bucket, key, uploadId, onFailure));
|
|
1556
|
+
}
|
|
1557
|
+
function abortFailureLogger(logger2, key) {
|
|
1558
|
+
return (err) => {
|
|
1559
|
+
logger2.warn(
|
|
1560
|
+
`Multipart abort failed, the upload stays open on the provider: ${logValue(key)} ${describeProviderFailure(err)}`
|
|
1561
|
+
);
|
|
1562
|
+
};
|
|
1485
1563
|
}
|
|
1486
1564
|
async function deleteObjectsChunk(client, bucket, chunk, stripPrefix) {
|
|
1487
1565
|
const deleted = [];
|
|
@@ -1503,14 +1581,16 @@ async function deleteObjectsChunk(client, bucket, chunk, stripPrefix) {
|
|
|
1503
1581
|
if (failure.Key !== void 0) {
|
|
1504
1582
|
failed.push({
|
|
1505
1583
|
key: stripPrefix(failure.Key),
|
|
1506
|
-
|
|
1584
|
+
// The provider's code, which a caller can branch on; its sentence is not repeated.
|
|
1585
|
+
error: failure.Code ?? "Unknown"
|
|
1507
1586
|
});
|
|
1508
1587
|
}
|
|
1509
1588
|
}
|
|
1510
1589
|
} catch (err) {
|
|
1511
|
-
|
|
1590
|
+
logProviderFailure(err, { op: "deleteMany" });
|
|
1591
|
+
const reason = providerFailureReason(err);
|
|
1512
1592
|
for (const key of chunk) {
|
|
1513
|
-
failed.push({ key: stripPrefix(key), error:
|
|
1593
|
+
failed.push({ key: stripPrefix(key), error: reason });
|
|
1514
1594
|
}
|
|
1515
1595
|
}
|
|
1516
1596
|
return { deleted, failed };
|
|
@@ -1518,7 +1598,12 @@ async function deleteObjectsChunk(client, bucket, chunk, stripPrefix) {
|
|
|
1518
1598
|
function mapBucketCheckError(err, context) {
|
|
1519
1599
|
const failure = err ?? {};
|
|
1520
1600
|
const status = failure.$metadata?.httpStatusCode;
|
|
1521
|
-
const details = {
|
|
1601
|
+
const details = {
|
|
1602
|
+
...context,
|
|
1603
|
+
httpStatus: status,
|
|
1604
|
+
awsCode: failure.name,
|
|
1605
|
+
reason: providerFailureReason(err)
|
|
1606
|
+
};
|
|
1522
1607
|
if (status === 301 || failure.name === "PermanentRedirect") {
|
|
1523
1608
|
return new StorageException(
|
|
1524
1609
|
STORAGE_ERROR_CODES.STORAGE_BUCKET_REGION_MISMATCH,
|
|
@@ -1805,7 +1890,11 @@ var StorageService = class {
|
|
|
1805
1890
|
const signal = options.abortSignal;
|
|
1806
1891
|
const input = validationInput(options, finalKey);
|
|
1807
1892
|
const validated = await abortable(this.validation.validate(input), signal, context);
|
|
1808
|
-
const scanned = await abortable(
|
|
1893
|
+
const scanned = await abortable(
|
|
1894
|
+
this.bufferForPreUploadScan(validated.body, finalKey),
|
|
1895
|
+
signal,
|
|
1896
|
+
context
|
|
1897
|
+
);
|
|
1809
1898
|
const body = boundStreamSize(scanned, options.size, this.options);
|
|
1810
1899
|
const validatedOptions = { ...options, body };
|
|
1811
1900
|
await abortable(this.runPreUploadScan(body, finalKey, bucket, options), signal, context);
|
|
@@ -1895,7 +1984,7 @@ var StorageService = class {
|
|
|
1895
1984
|
} catch (deleteErr) {
|
|
1896
1985
|
this.logger.error(
|
|
1897
1986
|
`STORAGE_SCAN_CLEANUP_FAILED: the rejected object is still stored: ${logValue(finalKey)}`,
|
|
1898
|
-
logValue(
|
|
1987
|
+
logValue(describeCleanupFailure(deleteErr))
|
|
1899
1988
|
);
|
|
1900
1989
|
throw withCleanupFailure(scanErr, { key, storedKey: finalKey }, deleteErr);
|
|
1901
1990
|
}
|
|
@@ -1917,7 +2006,10 @@ var StorageService = class {
|
|
|
1917
2006
|
const context = { key: finalKey, bucket, op: "head" };
|
|
1918
2007
|
throwIfAborted(options?.abortSignal, context);
|
|
1919
2008
|
try {
|
|
1920
|
-
const response = await this.s3Provider.getClient().send(
|
|
2009
|
+
const response = await this.s3Provider.getClient().send(
|
|
2010
|
+
new HeadObjectCommand({ Bucket: bucket, Key: finalKey }),
|
|
2011
|
+
sendOptions(options?.abortSignal)
|
|
2012
|
+
);
|
|
1921
2013
|
return this.toObjectMetadata(response, finalKey, bucket);
|
|
1922
2014
|
} catch (err) {
|
|
1923
2015
|
throw failureOrAbort(options?.abortSignal, context, mapAwsError(err, context));
|
|
@@ -1942,7 +2034,9 @@ var StorageService = class {
|
|
|
1942
2034
|
if (err instanceof StorageException && err.code === STORAGE_ERROR_CODES.STORAGE_REQUEST_ABORTED) {
|
|
1943
2035
|
throw err;
|
|
1944
2036
|
}
|
|
1945
|
-
this.logger.warn(
|
|
2037
|
+
this.logger.warn(
|
|
2038
|
+
`exists() \u2014 treating a non-404 error as "false": ${logValue(describeError(err))}`
|
|
2039
|
+
);
|
|
1946
2040
|
return false;
|
|
1947
2041
|
}
|
|
1948
2042
|
}
|
|
@@ -1970,7 +2064,11 @@ var StorageService = class {
|
|
|
1970
2064
|
*/
|
|
1971
2065
|
async download(options) {
|
|
1972
2066
|
this.assertConfigured();
|
|
1973
|
-
assertHeaderValues({
|
|
2067
|
+
assertHeaderValues({
|
|
2068
|
+
range: options.range,
|
|
2069
|
+
ifNoneMatch: options.ifNoneMatch,
|
|
2070
|
+
ifMatch: options.ifMatch
|
|
2071
|
+
});
|
|
1974
2072
|
const finalKey = this.keyResolver.normalize(options.key);
|
|
1975
2073
|
const bucket = this.resolveBucket(options.bucket);
|
|
1976
2074
|
const context = { key: finalKey, bucket, op: "download" };
|
|
@@ -2034,7 +2132,10 @@ var StorageService = class {
|
|
|
2034
2132
|
const context = { key: finalKey, bucket, op: "delete" };
|
|
2035
2133
|
throwIfAborted(options?.abortSignal, context);
|
|
2036
2134
|
try {
|
|
2037
|
-
await this.s3Provider.getClient().send(
|
|
2135
|
+
await this.s3Provider.getClient().send(
|
|
2136
|
+
new DeleteObjectCommand({ Bucket: bucket, Key: finalKey }),
|
|
2137
|
+
sendOptions(options?.abortSignal)
|
|
2138
|
+
);
|
|
2038
2139
|
} catch (err) {
|
|
2039
2140
|
const mapped = failureOrAbort(options?.abortSignal, context, mapAwsError(err, context));
|
|
2040
2141
|
if (mapped.code !== STORAGE_ERROR_CODES.STORAGE_OBJECT_NOT_FOUND) {
|
|
@@ -2182,9 +2283,17 @@ var StorageService = class {
|
|
|
2182
2283
|
let uploadId = upload.uploadId;
|
|
2183
2284
|
if (uploadId === void 0 && lookup) {
|
|
2184
2285
|
uploadId = await within(lookup, CLEANUP_DEADLINE_MS);
|
|
2185
|
-
if (uploadId === void 0)
|
|
2286
|
+
if (uploadId === void 0) {
|
|
2287
|
+
closeWhenCreated(client, bucket, finalKey, lookup, abortFailureLogger(this.logger, finalKey));
|
|
2288
|
+
}
|
|
2186
2289
|
}
|
|
2187
|
-
const abort = await closeMultipartUpload(
|
|
2290
|
+
const abort = await closeMultipartUpload(
|
|
2291
|
+
client,
|
|
2292
|
+
bucket,
|
|
2293
|
+
finalKey,
|
|
2294
|
+
uploadId,
|
|
2295
|
+
abortFailureLogger(this.logger, finalKey)
|
|
2296
|
+
);
|
|
2188
2297
|
const context = { key: finalKey, bucket, op: "upload-multipart", ...abort };
|
|
2189
2298
|
throw abortedBy ? requestAborted(abortedBy, context) : mapMultipartError(err, context);
|
|
2190
2299
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bymax-one/nest-storage",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.14",
|
|
4
4
|
"description": "Provider-agnostic S3-compatible object storage for NestJS. Works with AWS S3, DigitalOcean Spaces, Cloudflare R2, Backblaze B2, MinIO, Wasabi.",
|
|
5
5
|
"author": "Bymax One <support@bymax.one>",
|
|
6
6
|
"license": "MIT",
|