@bymax-one/nest-storage 1.1.18 → 1.1.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +76 -1
- package/README.md +160 -11
- package/dist/observability/index.cjs +151 -0
- package/dist/observability/index.d.cts +72 -0
- package/dist/observability/index.d.ts +72 -0
- package/dist/observability/index.mjs +145 -0
- package/dist/server/index.cjs +751 -421
- package/dist/server/index.d.cts +137 -11
- package/dist/server/index.d.ts +137 -11
- package/dist/server/index.mjs +751 -424
- package/package.json +32 -5
package/dist/server/index.mjs
CHANGED
|
@@ -1,19 +1,28 @@
|
|
|
1
|
-
import { HttpStatus, Injectable, Inject, Logger, Global, Module, HttpException } from '@nestjs/common';
|
|
2
|
-
import { S3Client, GetBucketLifecycleConfigurationCommand, DeleteBucketLifecycleCommand, PutBucketLifecycleConfigurationCommand, HeadObjectCommand, GetObjectCommand, DeleteObjectCommand, HeadBucketCommand,
|
|
1
|
+
import { HttpStatus, Injectable, Inject, Logger, Optional, Global, Module, HttpException } from '@nestjs/common';
|
|
2
|
+
import { S3Client, GetBucketLifecycleConfigurationCommand, DeleteBucketLifecycleCommand, PutBucketLifecycleConfigurationCommand, HeadObjectCommand, GetObjectCommand, DeleteObjectCommand, HeadBucketCommand, ListObjectsV2Command, CopyObjectCommand, PutObjectCommand, CreateMultipartUploadCommand, UploadPartCommand, CompleteMultipartUploadCommand, AbortMultipartUploadCommand, DeleteObjectsCommand } from '@aws-sdk/client-s3';
|
|
3
3
|
import { Upload } from '@aws-sdk/lib-storage';
|
|
4
4
|
import { PassThrough, Transform, pipeline, Readable } from 'stream';
|
|
5
5
|
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
|
|
6
6
|
import { createHash } from 'crypto';
|
|
7
7
|
|
|
8
|
+
var __defProp = Object.defineProperty;
|
|
8
9
|
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
10
|
+
var __typeError = (msg) => {
|
|
11
|
+
throw TypeError(msg);
|
|
12
|
+
};
|
|
9
13
|
var __decorateClass = (decorators, target, key, kind) => {
|
|
10
14
|
var result = kind > 1 ? void 0 : kind ? __getOwnPropDesc(target, key) : target;
|
|
11
15
|
for (var i = decorators.length - 1, decorator; i >= 0; i--)
|
|
12
16
|
if (decorator = decorators[i])
|
|
13
|
-
result = (decorator(result)) || result;
|
|
17
|
+
result = (kind ? decorator(target, key, result) : decorator(result)) || result;
|
|
18
|
+
if (kind && result) __defProp(target, key, result);
|
|
14
19
|
return result;
|
|
15
20
|
};
|
|
16
21
|
var __decorateParam = (index, decorator) => (target, key) => decorator(target, key, index);
|
|
22
|
+
var __accessCheck = (obj, member, msg) => member.has(obj) || __typeError("Cannot " + msg);
|
|
23
|
+
var __privateGet = (obj, member, getter) => (__accessCheck(obj, member, "read from private field"), getter ? getter.call(obj) : member.get(obj));
|
|
24
|
+
var __privateAdd = (obj, member, value) => member.has(obj) ? __typeError("Cannot add the same private member more than once") : member instanceof WeakSet ? member.add(obj) : member.set(obj, value);
|
|
25
|
+
var __privateSet = (obj, member, value, setter) => (__accessCheck(obj, member, "write to private field"), member.set(obj, value), value);
|
|
17
26
|
|
|
18
27
|
// src/server/bymax-storage.constants.ts
|
|
19
28
|
var BYMAX_STORAGE_OPTIONS = /* @__PURE__ */ Symbol("BYMAX_STORAGE_OPTIONS");
|
|
@@ -142,9 +151,9 @@ var HEADER_VALUE = /^[\t \x21-\x7E]*$/;
|
|
|
142
151
|
var HEADER_NAME = /^[!#$%&'*+.^_`|~0-9a-z-]+$/;
|
|
143
152
|
var VALUE_REASON = "must be a string of tab, space and printable ASCII characters";
|
|
144
153
|
var KEY_REASON = "every key must be a non-empty header name of letters, digits and the characters !#$%&'*+-.^_`|~";
|
|
145
|
-
function refuse(
|
|
154
|
+
function refuse(field, reason) {
|
|
146
155
|
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_VALIDATION_FAILED, void 0, {
|
|
147
|
-
field
|
|
156
|
+
field,
|
|
148
157
|
reason
|
|
149
158
|
});
|
|
150
159
|
}
|
|
@@ -152,9 +161,9 @@ function isHeaderValue(value) {
|
|
|
152
161
|
return typeof value === "string" && HEADER_VALUE.test(value);
|
|
153
162
|
}
|
|
154
163
|
function assertHeaderValues(fields) {
|
|
155
|
-
for (const [
|
|
164
|
+
for (const [field, value] of Object.entries(fields)) {
|
|
156
165
|
if (value !== void 0 && !isHeaderValue(value)) {
|
|
157
|
-
refuse(
|
|
166
|
+
refuse(field, VALUE_REASON);
|
|
158
167
|
}
|
|
159
168
|
}
|
|
160
169
|
}
|
|
@@ -251,7 +260,7 @@ function assertValidationKinds(validation) {
|
|
|
251
260
|
if (mimeWhitelist === null) {
|
|
252
261
|
throw invalid("validation.mimeWhitelist must not be null; omit it to allow every type");
|
|
253
262
|
}
|
|
254
|
-
if (mimeWhitelist !== void 0 &&
|
|
263
|
+
if (mimeWhitelist !== void 0 && !isStringArray(mimeWhitelist)) {
|
|
255
264
|
throw invalid("validation.mimeWhitelist must be an array of strings");
|
|
256
265
|
}
|
|
257
266
|
if (customValidators === null) {
|
|
@@ -260,6 +269,31 @@ function assertValidationKinds(validation) {
|
|
|
260
269
|
if (customValidators !== void 0 && !Array.isArray(customValidators)) {
|
|
261
270
|
throw invalid("validation.customValidators must be an array");
|
|
262
271
|
}
|
|
272
|
+
if (customValidators !== void 0 && !isValidatorList(customValidators)) {
|
|
273
|
+
throw invalid("validation.customValidators must hold objects with a validate function");
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
function isStringArray(value) {
|
|
277
|
+
try {
|
|
278
|
+
return Array.isArray(value) && Array.from(value).every((entry) => typeof entry === "string");
|
|
279
|
+
} catch {
|
|
280
|
+
return false;
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
function isValidatorList(entries) {
|
|
284
|
+
try {
|
|
285
|
+
return Array.from(entries).every((entry) => isObject(entry) && typeof entry.validate === "function");
|
|
286
|
+
} catch {
|
|
287
|
+
return false;
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
function assertGroupKinds(options) {
|
|
291
|
+
for (const group of ["signedUrls", "multipart", "validation"]) {
|
|
292
|
+
const value = options[group];
|
|
293
|
+
if (value !== void 0 && value !== null && (!isObject(value) || Array.isArray(value))) {
|
|
294
|
+
throw invalid(`options.${group} must be an object`);
|
|
295
|
+
}
|
|
296
|
+
}
|
|
263
297
|
}
|
|
264
298
|
function assertNumericSections(options) {
|
|
265
299
|
const MAX = Number.MAX_SAFE_INTEGER;
|
|
@@ -277,6 +311,7 @@ function assertNumericSections(options) {
|
|
|
277
311
|
assertBoundedInteger(options.multipart, "multipart", "queueSize", { min: 1, max: MAX_QUEUE_SIZE });
|
|
278
312
|
}
|
|
279
313
|
function assertLimitsAndEncryption(options) {
|
|
314
|
+
assertGroupKinds(options);
|
|
280
315
|
assertNumericSections(options);
|
|
281
316
|
assertValidationKinds(options.validation);
|
|
282
317
|
assertScanner(options.scanner);
|
|
@@ -326,6 +361,9 @@ function validateOptions(options) {
|
|
|
326
361
|
if (!isObject(options.credentials)) {
|
|
327
362
|
throw invalid("options.credentials is required");
|
|
328
363
|
}
|
|
364
|
+
if (Array.isArray(options.credentials)) {
|
|
365
|
+
throw invalid("options.credentials must be an object");
|
|
366
|
+
}
|
|
329
367
|
assertLimitsAndEncryption(options);
|
|
330
368
|
}
|
|
331
369
|
|
|
@@ -472,7 +510,7 @@ function pick(source, keys) {
|
|
|
472
510
|
return copy;
|
|
473
511
|
}
|
|
474
512
|
function pickGroup(value, keys) {
|
|
475
|
-
return isObject2(value) ? pick(value, keys) : value;
|
|
513
|
+
return isObject2(value) && !Array.isArray(value) ? pick(value, keys) : value;
|
|
476
514
|
}
|
|
477
515
|
function copyArray(value) {
|
|
478
516
|
if (!Array.isArray(value)) return value;
|
|
@@ -504,16 +542,36 @@ function snapshotOptions(options) {
|
|
|
504
542
|
return copy;
|
|
505
543
|
}
|
|
506
544
|
|
|
507
|
-
// src/server/utils/
|
|
508
|
-
function
|
|
545
|
+
// src/server/utils/absorb.ts
|
|
546
|
+
function absorb(call) {
|
|
509
547
|
try {
|
|
510
|
-
|
|
511
|
-
return `${url.protocol}//${url.host}`;
|
|
548
|
+
void Promise.resolve(call()).then(void 0, () => void 0);
|
|
512
549
|
} catch {
|
|
513
|
-
return "<unparseable endpoint>";
|
|
514
550
|
}
|
|
515
551
|
}
|
|
516
552
|
|
|
553
|
+
// src/server/utils/log-event.ts
|
|
554
|
+
function emit(sink, level, entry) {
|
|
555
|
+
absorb(() => sink[level](entry));
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
// src/server/utils/log-events.ts
|
|
559
|
+
var STORAGE_LOG_EVENTS = {
|
|
560
|
+
ttlClamped: "STORAGE_TTL_CLAMPED",
|
|
561
|
+
credentialsMissing: "STORAGE_CREDENTIALS_MISSING",
|
|
562
|
+
clientReady: "STORAGE_CLIENT_READY",
|
|
563
|
+
providerCallFailed: "STORAGE_PROVIDER_CALL_FAILED",
|
|
564
|
+
multipartAbortFailed: "STORAGE_MULTIPART_ABORT_FAILED",
|
|
565
|
+
scanInfected: "STORAGE_SCAN_INFECTED",
|
|
566
|
+
scanInconclusiveAccepted: "STORAGE_SCAN_INCONCLUSIVE_ACCEPTED",
|
|
567
|
+
scanInconclusiveRejected: "STORAGE_SCAN_INCONCLUSIVE_REJECTED",
|
|
568
|
+
scanFailed: "STORAGE_SCAN_FAILED",
|
|
569
|
+
scanTimeout: "STORAGE_SCAN_TIMEOUT",
|
|
570
|
+
scanCleanupFailed: "STORAGE_SCAN_CLEANUP_FAILED",
|
|
571
|
+
existsFailed: "STORAGE_EXISTS_FAILED",
|
|
572
|
+
deleteNotFound: "STORAGE_DELETE_NOT_FOUND"
|
|
573
|
+
};
|
|
574
|
+
|
|
517
575
|
// src/server/providers/s3-client.provider.ts
|
|
518
576
|
var S3ClientProvider = class {
|
|
519
577
|
constructor(options) {
|
|
@@ -524,9 +582,10 @@ var S3ClientProvider = class {
|
|
|
524
582
|
client;
|
|
525
583
|
onModuleInit() {
|
|
526
584
|
if (!this.options.hasCredentials) {
|
|
527
|
-
this.logger
|
|
528
|
-
|
|
529
|
-
|
|
585
|
+
emit(this.logger, "warn", {
|
|
586
|
+
event: STORAGE_LOG_EVENTS.credentialsMissing,
|
|
587
|
+
message: "Storage credentials are missing \u2014 module is registered but operations will throw STORAGE_NOT_CONFIGURED."
|
|
588
|
+
});
|
|
530
589
|
return;
|
|
531
590
|
}
|
|
532
591
|
this.resolveClient();
|
|
@@ -599,9 +658,10 @@ var S3ClientProvider = class {
|
|
|
599
658
|
requestChecksumCalculation: this.options.requestChecksumCalculation,
|
|
600
659
|
responseChecksumValidation: this.options.responseChecksumValidation
|
|
601
660
|
};
|
|
602
|
-
this.logger
|
|
603
|
-
|
|
604
|
-
|
|
661
|
+
emit(this.logger, "log", {
|
|
662
|
+
event: STORAGE_LOG_EVENTS.clientReady,
|
|
663
|
+
message: "S3Client initialized"
|
|
664
|
+
});
|
|
605
665
|
return new S3Client(config);
|
|
606
666
|
}
|
|
607
667
|
};
|
|
@@ -670,11 +730,256 @@ KeyResolverService = __decorateClass([
|
|
|
670
730
|
__decorateParam(0, Inject(BYMAX_STORAGE_OPTIONS))
|
|
671
731
|
], KeyResolverService);
|
|
672
732
|
|
|
733
|
+
// src/server/observability/storage-operations.ts
|
|
734
|
+
var STORAGE_OPERATIONS = [
|
|
735
|
+
"upload",
|
|
736
|
+
"download",
|
|
737
|
+
"head",
|
|
738
|
+
"delete",
|
|
739
|
+
"delete_many",
|
|
740
|
+
"list",
|
|
741
|
+
"copy",
|
|
742
|
+
"get_download_url",
|
|
743
|
+
"get_upload_url",
|
|
744
|
+
"get_multipart_upload_urls",
|
|
745
|
+
"check_bucket",
|
|
746
|
+
"lifecycle_get",
|
|
747
|
+
"lifecycle_upsert",
|
|
748
|
+
"lifecycle_remove"
|
|
749
|
+
];
|
|
750
|
+
var STORAGE_OUTCOMES = ["success", "not_found", "aborted", "error"];
|
|
751
|
+
var SCAN_VERDICTS = ["clean", "infected", "unknown", "timeout", "error"];
|
|
752
|
+
function outcomeOf(error) {
|
|
753
|
+
if (!(error instanceof StorageException)) return "error";
|
|
754
|
+
if (error.code === STORAGE_ERROR_CODES.STORAGE_OBJECT_NOT_FOUND) return "not_found";
|
|
755
|
+
if (error.code === STORAGE_ERROR_CODES.STORAGE_REQUEST_ABORTED || error.code === STORAGE_ERROR_CODES.STORAGE_MULTIPART_ABORTED) {
|
|
756
|
+
return "aborted";
|
|
757
|
+
}
|
|
758
|
+
return "error";
|
|
759
|
+
}
|
|
760
|
+
function scanVerdictOfError(error) {
|
|
761
|
+
if (!(error instanceof StorageException)) return "error";
|
|
762
|
+
if (error.code === STORAGE_ERROR_CODES.STORAGE_SCAN_INFECTED) return "infected";
|
|
763
|
+
if (error.code === STORAGE_ERROR_CODES.STORAGE_SCAN_INCONCLUSIVE) return "unknown";
|
|
764
|
+
if (error.code === STORAGE_ERROR_CODES.STORAGE_TIMEOUT) return "timeout";
|
|
765
|
+
return "error";
|
|
766
|
+
}
|
|
767
|
+
|
|
768
|
+
// src/server/observability/storage-metrics-tally.ts
|
|
769
|
+
function zeroed(keys) {
|
|
770
|
+
return Object.fromEntries(keys.map((key) => [key, 0]));
|
|
771
|
+
}
|
|
772
|
+
var _operations, _scans, _listeners, _bytesUploaded;
|
|
773
|
+
var StorageMetricsTally = class {
|
|
774
|
+
constructor() {
|
|
775
|
+
__privateAdd(this, _operations, Object.fromEntries(
|
|
776
|
+
STORAGE_OPERATIONS.map((operation) => [operation, zeroed(STORAGE_OUTCOMES)])
|
|
777
|
+
));
|
|
778
|
+
__privateAdd(this, _scans, zeroed(SCAN_VERDICTS));
|
|
779
|
+
__privateAdd(this, _listeners, []);
|
|
780
|
+
__privateAdd(this, _bytesUploaded, 0);
|
|
781
|
+
}
|
|
782
|
+
/** An operation ended as `outcome` after `seconds`. */
|
|
783
|
+
operation(operation, outcome, seconds) {
|
|
784
|
+
__privateGet(this, _operations)[operation][outcome] += 1;
|
|
785
|
+
for (const listener of __privateGet(this, _listeners)) absorb(() => listener(operation, seconds));
|
|
786
|
+
}
|
|
787
|
+
/** `bytes` of an object reached the provider; anything that is not a positive finite number is ignored. */
|
|
788
|
+
bytesUploaded(bytes) {
|
|
789
|
+
if (typeof bytes === "number" && Number.isFinite(bytes) && bytes > 0)
|
|
790
|
+
__privateSet(this, _bytesUploaded, __privateGet(this, _bytesUploaded) + bytes);
|
|
791
|
+
}
|
|
792
|
+
/** A scan ended as `verdict`. */
|
|
793
|
+
scan(verdict) {
|
|
794
|
+
__privateGet(this, _scans)[verdict] += 1;
|
|
795
|
+
}
|
|
796
|
+
/** Calls `listener` with the duration of every operation that ends from now on. */
|
|
797
|
+
onDuration(listener) {
|
|
798
|
+
__privateGet(this, _listeners).push(listener);
|
|
799
|
+
}
|
|
800
|
+
/** The counts so far, as copies. */
|
|
801
|
+
snapshot() {
|
|
802
|
+
return {
|
|
803
|
+
operations: Object.fromEntries(
|
|
804
|
+
STORAGE_OPERATIONS.map((operation) => [operation, { ...__privateGet(this, _operations)[operation] }])
|
|
805
|
+
),
|
|
806
|
+
bytesUploaded: __privateGet(this, _bytesUploaded),
|
|
807
|
+
scans: { ...__privateGet(this, _scans) }
|
|
808
|
+
};
|
|
809
|
+
}
|
|
810
|
+
};
|
|
811
|
+
_operations = new WeakMap();
|
|
812
|
+
_scans = new WeakMap();
|
|
813
|
+
_listeners = new WeakMap();
|
|
814
|
+
_bytesUploaded = new WeakMap();
|
|
815
|
+
StorageMetricsTally = __decorateClass([
|
|
816
|
+
Injectable()
|
|
817
|
+
], StorageMetricsTally);
|
|
818
|
+
var STATUS_ERROR = 2;
|
|
819
|
+
var TRACER_NAME = "@bymax-one/nest-storage";
|
|
820
|
+
var LIBRARY_CODE = /^STORAGE_[A-Z_]+$/;
|
|
821
|
+
function codeOf(error) {
|
|
822
|
+
if (!(error instanceof StorageException)) return "STORAGE_ERROR";
|
|
823
|
+
return LIBRARY_CODE.test(error.code) ? error.code : "STORAGE_ERROR";
|
|
824
|
+
}
|
|
825
|
+
var _api;
|
|
826
|
+
var StorageTracing = class {
|
|
827
|
+
constructor() {
|
|
828
|
+
__privateAdd(this, _api);
|
|
829
|
+
}
|
|
830
|
+
/** Loads the API once, at start-up; a package that is not there leaves the seam inert. */
|
|
831
|
+
async onModuleInit() {
|
|
832
|
+
try {
|
|
833
|
+
const loaded = await import('@opentelemetry/api');
|
|
834
|
+
__privateSet(this, _api, loaded);
|
|
835
|
+
} catch {
|
|
836
|
+
}
|
|
837
|
+
}
|
|
838
|
+
/** Runs the operation and reports it on the span, where nothing the span does can change what comes back. */
|
|
839
|
+
async traced(span, run, describe) {
|
|
840
|
+
let result;
|
|
841
|
+
try {
|
|
842
|
+
result = await run();
|
|
843
|
+
} catch (error) {
|
|
844
|
+
absorb(() => span.setAttributes({ "bymax.storage.outcome": outcomeOf(error) }));
|
|
845
|
+
absorb(() => span.setStatus({ code: STATUS_ERROR, message: codeOf(error) }));
|
|
846
|
+
absorb(() => span.end());
|
|
847
|
+
throw error;
|
|
848
|
+
}
|
|
849
|
+
absorb(() => span.setAttributes({ "bymax.storage.outcome": "success", ...describe(result) }));
|
|
850
|
+
absorb(() => span.end());
|
|
851
|
+
return result;
|
|
852
|
+
}
|
|
853
|
+
/**
|
|
854
|
+
* Runs `run` inside a span named `name`.
|
|
855
|
+
*
|
|
856
|
+
* The tracer belongs to the host's SDK, so it is never trusted with the operation: whatever it returns or
|
|
857
|
+
* throws is ignored, the operation runs exactly once, through the span when the tracer started it and bare
|
|
858
|
+
* otherwise, and its own result or error is what comes back.
|
|
859
|
+
*
|
|
860
|
+
* @param name - One of the library's span names.
|
|
861
|
+
* @param attributes - Closed-set attributes of the span.
|
|
862
|
+
* @param run - The operation; its result or its error passes through unchanged.
|
|
863
|
+
* @param describe - What the span states of a successful result; a throw of its own is the operation's own.
|
|
864
|
+
*/
|
|
865
|
+
span(name, attributes, run, describe = () => ({})) {
|
|
866
|
+
const api = __privateGet(this, _api);
|
|
867
|
+
if (!api) return run();
|
|
868
|
+
const latch = { claimed: false };
|
|
869
|
+
absorb(
|
|
870
|
+
() => api.trace.getTracer(TRACER_NAME).startActiveSpan(name, { attributes }, (span) => {
|
|
871
|
+
if (!latch.claimed) {
|
|
872
|
+
latch.claimed = true;
|
|
873
|
+
latch.started = this.traced(span, run, describe);
|
|
874
|
+
}
|
|
875
|
+
return latch.started;
|
|
876
|
+
})
|
|
877
|
+
);
|
|
878
|
+
if (latch.started) return latch.started;
|
|
879
|
+
latch.claimed = true;
|
|
880
|
+
return run();
|
|
881
|
+
}
|
|
882
|
+
};
|
|
883
|
+
_api = new WeakMap();
|
|
884
|
+
StorageTracing = __decorateClass([
|
|
885
|
+
Injectable()
|
|
886
|
+
], StorageTracing);
|
|
887
|
+
|
|
888
|
+
// src/server/observability/storage-observer.ts
|
|
889
|
+
function secondsSince(startedAt) {
|
|
890
|
+
return Number(process.hrtime.bigint() - startedAt) / 1e9;
|
|
891
|
+
}
|
|
892
|
+
var StorageObserver = class {
|
|
893
|
+
constructor(tally, tracing) {
|
|
894
|
+
this.tally = tally;
|
|
895
|
+
this.tracing = tracing;
|
|
896
|
+
}
|
|
897
|
+
tally;
|
|
898
|
+
tracing;
|
|
899
|
+
/**
|
|
900
|
+
* Runs `run`, counting how it ended and how long it took.
|
|
901
|
+
*
|
|
902
|
+
* @param operation - The operation, one of the closed set.
|
|
903
|
+
* @param run - The operation; its result or its error passes through unchanged.
|
|
904
|
+
* @param options - A span to start, and what to state of the result.
|
|
905
|
+
*/
|
|
906
|
+
observe(operation, run, options = {}) {
|
|
907
|
+
const counted = async () => {
|
|
908
|
+
const startedAt = process.hrtime.bigint();
|
|
909
|
+
let result;
|
|
910
|
+
try {
|
|
911
|
+
result = await run();
|
|
912
|
+
} catch (error) {
|
|
913
|
+
absorb(() => {
|
|
914
|
+
this.tally.operation(operation, outcomeOf(error), secondsSince(startedAt));
|
|
915
|
+
});
|
|
916
|
+
throw error;
|
|
917
|
+
}
|
|
918
|
+
absorb(() => {
|
|
919
|
+
this.tally.operation(operation, "success", secondsSince(startedAt));
|
|
920
|
+
this.tally.bytesUploaded(options.bytes?.(result));
|
|
921
|
+
});
|
|
922
|
+
return result;
|
|
923
|
+
};
|
|
924
|
+
return options.span === void 0 ? counted() : this.tracing.span(options.span, options.attributes ?? {}, counted, options.describe);
|
|
925
|
+
}
|
|
926
|
+
/**
|
|
927
|
+
* Runs a scan, counting its verdict and wrapping it in the `storage.scan` span.
|
|
928
|
+
*
|
|
929
|
+
* @param mode - When the scan runs, `pre-upload` or `post-upload`.
|
|
930
|
+
* @param run - The scan; its result or its error passes through unchanged.
|
|
931
|
+
*/
|
|
932
|
+
scan(mode, run) {
|
|
933
|
+
const counted = async () => {
|
|
934
|
+
try {
|
|
935
|
+
const result = await run();
|
|
936
|
+
absorb(() => {
|
|
937
|
+
this.tally.scan(result.status === "unknown" ? "unknown" : "clean");
|
|
938
|
+
});
|
|
939
|
+
return result;
|
|
940
|
+
} catch (error) {
|
|
941
|
+
absorb(() => {
|
|
942
|
+
this.tally.scan(scanVerdictOfError(error));
|
|
943
|
+
});
|
|
944
|
+
throw error;
|
|
945
|
+
}
|
|
946
|
+
};
|
|
947
|
+
return this.tracing.span("storage.scan", { "bymax.storage.scan_mode": mode }, counted);
|
|
948
|
+
}
|
|
949
|
+
};
|
|
950
|
+
StorageObserver = __decorateClass([
|
|
951
|
+
Injectable(),
|
|
952
|
+
__decorateParam(0, Inject(StorageMetricsTally)),
|
|
953
|
+
__decorateParam(1, Inject(StorageTracing))
|
|
954
|
+
], StorageObserver);
|
|
955
|
+
function Observed(operation, options = {}) {
|
|
956
|
+
return (_target, _key, descriptor) => {
|
|
957
|
+
const original = descriptor.value;
|
|
958
|
+
descriptor.value = function(...args) {
|
|
959
|
+
const run = () => original.apply(this, args);
|
|
960
|
+
return this.observer === void 0 ? run() : this.observer.observe(operation, run, options);
|
|
961
|
+
};
|
|
962
|
+
};
|
|
963
|
+
}
|
|
964
|
+
|
|
965
|
+
// src/server/utils/call-details.ts
|
|
966
|
+
var MAX_CALLER_TEXT_LENGTH = 1024;
|
|
967
|
+
function callerFields(given) {
|
|
968
|
+
const fields = {};
|
|
969
|
+
for (const [field, value] of Object.entries(given)) {
|
|
970
|
+
if (typeof value === "string" && value.length <= MAX_CALLER_TEXT_LENGTH) fields[field] = value;
|
|
971
|
+
}
|
|
972
|
+
return fields;
|
|
973
|
+
}
|
|
974
|
+
function callDetails(op, given, extra = {}) {
|
|
975
|
+
return { ...callerFields(given), op, ...extra };
|
|
976
|
+
}
|
|
977
|
+
|
|
673
978
|
// src/server/utils/call-input.ts
|
|
674
|
-
function wrongKind(
|
|
979
|
+
function wrongKind(field, needs) {
|
|
675
980
|
return new StorageException(STORAGE_ERROR_CODES.STORAGE_VALIDATION_FAILED, void 0, {
|
|
676
|
-
field
|
|
677
|
-
reason: `${
|
|
981
|
+
field,
|
|
982
|
+
reason: `${field} must be ${needs}`
|
|
678
983
|
});
|
|
679
984
|
}
|
|
680
985
|
function resolveBucket(perCall, moduleDefault) {
|
|
@@ -702,64 +1007,6 @@ function hasSignalSurface(signal) {
|
|
|
702
1007
|
return false;
|
|
703
1008
|
}
|
|
704
1009
|
}
|
|
705
|
-
|
|
706
|
-
// src/server/utils/describe-error.ts
|
|
707
|
-
var NON_PRINTABLE_REJECTION = "non-printable rejection value";
|
|
708
|
-
function describeError(value) {
|
|
709
|
-
try {
|
|
710
|
-
if (typeof value === "object" && value !== null) {
|
|
711
|
-
const message = value.message;
|
|
712
|
-
if (typeof message === "string") {
|
|
713
|
-
return message;
|
|
714
|
-
}
|
|
715
|
-
}
|
|
716
|
-
return String(value);
|
|
717
|
-
} catch {
|
|
718
|
-
return NON_PRINTABLE_REJECTION;
|
|
719
|
-
}
|
|
720
|
-
}
|
|
721
|
-
|
|
722
|
-
// src/server/utils/log-value.ts
|
|
723
|
-
var LEFT_LITERAL_BY_JSON = /[\p{Cc}\p{Cf}\p{Zl}\p{Zp}]/gu;
|
|
724
|
-
function logValue(value) {
|
|
725
|
-
const text = typeof value === "string" ? value : describeError(value);
|
|
726
|
-
return JSON.stringify(text).replace(
|
|
727
|
-
LEFT_LITERAL_BY_JSON,
|
|
728
|
-
(char) => {
|
|
729
|
-
const codePoint = Number(char.codePointAt(0));
|
|
730
|
-
const hex = codePoint.toString(16);
|
|
731
|
-
return codePoint > 65535 ? `\\u{${hex}}` : `\\u${hex.padStart(4, "0")}`;
|
|
732
|
-
}
|
|
733
|
-
);
|
|
734
|
-
}
|
|
735
|
-
|
|
736
|
-
// src/server/utils/provider-failure-log.ts
|
|
737
|
-
var SECRET_IN_MESSAGE = /(X-Amz-(?:Signature|Credential|Security-Token)|AWSAccessKeyId|Credential|Signature)=[^&\s"']+/gi;
|
|
738
|
-
var BARE_ACCESS_KEY_ID = /\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/g;
|
|
739
|
-
var MAX_MESSAGE_LENGTH = 300;
|
|
740
|
-
var logger = new Logger("StorageProvider");
|
|
741
|
-
function field(value) {
|
|
742
|
-
return typeof value === "string" || typeof value === "number" ? logValue(value) : "-";
|
|
743
|
-
}
|
|
744
|
-
function describeProviderFailure(err) {
|
|
745
|
-
const failure = typeof err === "object" && err !== null ? err : {};
|
|
746
|
-
const message = describeError(err).replace(SECRET_IN_MESSAGE, "$1=[redacted]").replace(BARE_ACCESS_KEY_ID, "[redacted-key-id]");
|
|
747
|
-
return [
|
|
748
|
-
`name=${field(failure.name)}`,
|
|
749
|
-
`code=${field(failure.Code)}`,
|
|
750
|
-
`status=${field(failure.$metadata?.httpStatusCode)}`,
|
|
751
|
-
`requestId=${field(failure.$metadata?.requestId)}`,
|
|
752
|
-
`message=${logValue(message.slice(0, MAX_MESSAGE_LENGTH))}`
|
|
753
|
-
].join(" ");
|
|
754
|
-
}
|
|
755
|
-
function logProviderFailure(err, context, label = "Provider call failed") {
|
|
756
|
-
if (err?.name === "AbortError") {
|
|
757
|
-
return;
|
|
758
|
-
}
|
|
759
|
-
const op = typeof context?.op === "string" ? logValue(context.op) : "-";
|
|
760
|
-
const key = typeof context?.key === "string" ? logValue(context.key) : "-";
|
|
761
|
-
logger.warn(`${label}: op=${op} key=${key} ${describeProviderFailure(err)}`);
|
|
762
|
-
}
|
|
763
1010
|
var UNREACHABLE_CODES = /* @__PURE__ */ new Set([
|
|
764
1011
|
"ECONNREFUSED",
|
|
765
1012
|
"ECONNRESET",
|
|
@@ -826,6 +1073,106 @@ function providerFailureStatus(err) {
|
|
|
826
1073
|
return failure === "credentials-refused" ? HttpStatus.INTERNAL_SERVER_ERROR : void 0;
|
|
827
1074
|
}
|
|
828
1075
|
|
|
1076
|
+
// src/server/utils/cut-text.ts
|
|
1077
|
+
function cutText(text, max) {
|
|
1078
|
+
if (text.length <= max) return text;
|
|
1079
|
+
const lastKept = text.charCodeAt(max - 1);
|
|
1080
|
+
const splitsPair = lastKept >= 55296 && lastKept <= 56319;
|
|
1081
|
+
return text.slice(0, splitsPair ? max - 1 : max);
|
|
1082
|
+
}
|
|
1083
|
+
|
|
1084
|
+
// src/server/utils/describe-error.ts
|
|
1085
|
+
var NON_PRINTABLE_REJECTION = "non-printable rejection value";
|
|
1086
|
+
function describeError(value) {
|
|
1087
|
+
try {
|
|
1088
|
+
if (typeof value === "object" && value !== null) {
|
|
1089
|
+
const message = value.message;
|
|
1090
|
+
if (typeof message === "string") {
|
|
1091
|
+
return message;
|
|
1092
|
+
}
|
|
1093
|
+
}
|
|
1094
|
+
return String(value);
|
|
1095
|
+
} catch {
|
|
1096
|
+
return NON_PRINTABLE_REJECTION;
|
|
1097
|
+
}
|
|
1098
|
+
}
|
|
1099
|
+
|
|
1100
|
+
// src/server/utils/log-value.ts
|
|
1101
|
+
var LEFT_LITERAL_BY_JSON = /[\p{Cc}\p{Cf}\p{Zl}\p{Zp}]/gu;
|
|
1102
|
+
var SHORT_ESCAPES = {
|
|
1103
|
+
"\b": "\\b",
|
|
1104
|
+
"\f": "\\f",
|
|
1105
|
+
"\n": "\\n",
|
|
1106
|
+
"\r": "\\r",
|
|
1107
|
+
" ": "\\t"
|
|
1108
|
+
};
|
|
1109
|
+
function escapeChar(char) {
|
|
1110
|
+
const short = SHORT_ESCAPES[char];
|
|
1111
|
+
if (short !== void 0) return short;
|
|
1112
|
+
const codePoint = Number(char.codePointAt(0));
|
|
1113
|
+
const hex = codePoint.toString(16);
|
|
1114
|
+
return codePoint > 65535 ? `\\u{${hex}}` : `\\u${hex.padStart(4, "0")}`;
|
|
1115
|
+
}
|
|
1116
|
+
function escapeForLog(text) {
|
|
1117
|
+
return text.replace(LEFT_LITERAL_BY_JSON, escapeChar);
|
|
1118
|
+
}
|
|
1119
|
+
|
|
1120
|
+
// src/server/utils/provider-failure-log.ts
|
|
1121
|
+
var SECRET_IN_MESSAGE = /(X-Amz-(?:Signature|Credential|Security-Token)|AWSAccessKeyId|Credential|Signature)=[^&\s"']+/gi;
|
|
1122
|
+
var BARE_ACCESS_KEY_ID = /\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/g;
|
|
1123
|
+
var MAX_MESSAGE_LENGTH = 300;
|
|
1124
|
+
var logger = new Logger("StorageProvider");
|
|
1125
|
+
var MAX_FIELD_LENGTH = 128;
|
|
1126
|
+
function readOrUndefined(read) {
|
|
1127
|
+
try {
|
|
1128
|
+
return read();
|
|
1129
|
+
} catch {
|
|
1130
|
+
return void 0;
|
|
1131
|
+
}
|
|
1132
|
+
}
|
|
1133
|
+
function textField(value) {
|
|
1134
|
+
return typeof value === "string" ? escapeForLog(cutText(value, MAX_FIELD_LENGTH)) : void 0;
|
|
1135
|
+
}
|
|
1136
|
+
function projectProviderFailure(err) {
|
|
1137
|
+
const failure = typeof err === "object" && err !== null ? err : {};
|
|
1138
|
+
const message = describeError(err).replace(SECRET_IN_MESSAGE, "$1=[redacted]").replace(BARE_ACCESS_KEY_ID, "[redacted-key-id]");
|
|
1139
|
+
const status = readOrUndefined(() => failure.$metadata?.httpStatusCode);
|
|
1140
|
+
const name = textField(readOrUndefined(() => failure.name));
|
|
1141
|
+
const awsCode = textField(readOrUndefined(() => failure.Code));
|
|
1142
|
+
const requestId = textField(readOrUndefined(() => failure.$metadata?.requestId));
|
|
1143
|
+
return {
|
|
1144
|
+
...name !== void 0 ? { name } : {},
|
|
1145
|
+
...awsCode !== void 0 ? { awsCode } : {},
|
|
1146
|
+
...typeof status === "number" ? { httpStatus: status } : {},
|
|
1147
|
+
...requestId !== void 0 ? { requestId } : {},
|
|
1148
|
+
message: escapeForLog(cutText(message, MAX_MESSAGE_LENGTH))
|
|
1149
|
+
};
|
|
1150
|
+
}
|
|
1151
|
+
function failureError(err) {
|
|
1152
|
+
const view = projectProviderFailure(err);
|
|
1153
|
+
const error = new Error(view.message);
|
|
1154
|
+
if (view.name !== void 0) error.name = view.name;
|
|
1155
|
+
if (view.awsCode !== void 0)
|
|
1156
|
+
Object.defineProperty(error, "awsCode", { value: view.awsCode, enumerable: true });
|
|
1157
|
+
return error;
|
|
1158
|
+
}
|
|
1159
|
+
function logProviderFailure(err, context, label = "Provider call failed") {
|
|
1160
|
+
if (readOrUndefined(() => err?.name) === "AbortError") {
|
|
1161
|
+
return;
|
|
1162
|
+
}
|
|
1163
|
+
const view = projectProviderFailure(err);
|
|
1164
|
+
emit(logger, "warn", {
|
|
1165
|
+
event: STORAGE_LOG_EVENTS.providerCallFailed,
|
|
1166
|
+
message: label,
|
|
1167
|
+
...typeof context?.op === "string" ? { operation: escapeForLog(context.op) } : {},
|
|
1168
|
+
...view.awsCode !== void 0 ? { awsCode: view.awsCode } : {},
|
|
1169
|
+
...view.httpStatus !== void 0 ? { httpStatus: view.httpStatus } : {},
|
|
1170
|
+
...view.requestId !== void 0 ? { requestId: view.requestId } : {},
|
|
1171
|
+
reason: readOrUndefined(() => providerFailureReason(err)) ?? "rejected",
|
|
1172
|
+
err: failureError(err)
|
|
1173
|
+
});
|
|
1174
|
+
}
|
|
1175
|
+
|
|
829
1176
|
// src/server/errors/aws-error-mapper.ts
|
|
830
1177
|
function awsCodeOf(err) {
|
|
831
1178
|
const e = err ?? {};
|
|
@@ -945,54 +1292,32 @@ function ruleInvalid(reason) {
|
|
|
945
1292
|
reason
|
|
946
1293
|
});
|
|
947
1294
|
}
|
|
948
|
-
function assertDays(
|
|
1295
|
+
function assertDays(field, value) {
|
|
949
1296
|
if (!Number.isInteger(value) || value < 1) {
|
|
950
|
-
throw ruleInvalid(`${
|
|
1297
|
+
throw ruleInvalid(`${field} must be an integer of at least 1`);
|
|
951
1298
|
}
|
|
952
1299
|
}
|
|
953
1300
|
var BucketLifecycleService = class {
|
|
954
|
-
constructor(options, s3Provider, keyResolver) {
|
|
1301
|
+
constructor(options, s3Provider, keyResolver, observer) {
|
|
955
1302
|
this.options = options;
|
|
956
1303
|
this.s3Provider = s3Provider;
|
|
957
1304
|
this.keyResolver = keyResolver;
|
|
1305
|
+
this.observer = observer;
|
|
958
1306
|
}
|
|
959
1307
|
options;
|
|
960
1308
|
s3Provider;
|
|
961
1309
|
keyResolver;
|
|
962
|
-
|
|
963
|
-
* Returns the bucket's lifecycle rules. A bucket with no lifecycle configuration
|
|
964
|
-
* yields an empty array. Key prefixes inside the module's `keyPrefix` come back
|
|
965
|
-
* relative to it, as `list()` returns keys, so a prefix read here can be passed
|
|
966
|
-
* straight to `upsert()` without being prefixed twice. A rule that is not confined
|
|
967
|
-
* to `keyPrefix` is returned exactly as stored, flagged `isOutsideKeyPrefix: true`;
|
|
968
|
-
* everything else about a rule is returned as the provider stores it.
|
|
969
|
-
*
|
|
970
|
-
* @param options - Optional per-call bucket override.
|
|
971
|
-
* @returns Every rule in the bucket.
|
|
972
|
-
* @throws StorageException for missing configuration, an unsupported provider, or a provider error.
|
|
973
|
-
*/
|
|
1310
|
+
observer;
|
|
974
1311
|
async get(options) {
|
|
975
1312
|
this.assertConfigured();
|
|
976
1313
|
const bucket = this.resolveBucket(options?.bucket);
|
|
977
|
-
return (await this.read(bucket)).rules.map((rule) => this.toView(rule));
|
|
1314
|
+
return (await this.read(bucket, options?.bucket)).rules.map((rule) => this.toView(rule));
|
|
978
1315
|
}
|
|
979
|
-
/**
|
|
980
|
-
* Adds the expiration or multipart-abort rule, or replaces the rule with the same id, and writes
|
|
981
|
-
* the full set back. Rules with other ids are sent exactly as they were read.
|
|
982
|
-
*
|
|
983
|
-
* @param rule - The rule id, key prefix and either `expirationDays` or `abortIncompleteMultipartDays`.
|
|
984
|
-
* @param options - Optional per-call bucket override.
|
|
985
|
-
* @throws StorageException `STORAGE_LIFECYCLE_RULE_INVALID` for a bad id, prefix or day count, or an id
|
|
986
|
-
* that names a rule outside `keyPrefix`;
|
|
987
|
-
* `STORAGE_LIFECYCLE_UNSUPPORTED` / `STORAGE_LIFECYCLE_REJECTED` from the provider.
|
|
988
|
-
* @example
|
|
989
|
-
* await lifecycle.upsert({ id: 'expire-tmp', prefix: 'expiring/', expirationDays: 7 })
|
|
990
|
-
*/
|
|
991
1316
|
async upsert(rule, options) {
|
|
992
1317
|
this.assertConfigured();
|
|
993
1318
|
const owned = this.toRule(rule);
|
|
994
1319
|
const bucket = this.resolveBucket(options?.bucket);
|
|
995
|
-
const snapshot = await this.read(bucket);
|
|
1320
|
+
const snapshot = await this.read(bucket, options?.bucket);
|
|
996
1321
|
const index = snapshot.rules.findIndex((existing) => existing.ID === owned.ID);
|
|
997
1322
|
this.assertOwnable(snapshot.rules[index]);
|
|
998
1323
|
const rules = [...snapshot.rules];
|
|
@@ -1001,30 +1326,20 @@ var BucketLifecycleService = class {
|
|
|
1001
1326
|
} else {
|
|
1002
1327
|
rules[index] = owned;
|
|
1003
1328
|
}
|
|
1004
|
-
await this.write(bucket, rules, snapshot, "upsert");
|
|
1329
|
+
await this.write(bucket, options?.bucket, rules, snapshot, "upsert");
|
|
1005
1330
|
}
|
|
1006
|
-
/**
|
|
1007
|
-
* Removes the rule with this id. When it was the last rule the configuration is
|
|
1008
|
-
* deleted, because S3 rejects a write with no rules. An unknown id changes nothing.
|
|
1009
|
-
*
|
|
1010
|
-
* @param id - The rule id.
|
|
1011
|
-
* @param options - Optional per-call bucket override.
|
|
1012
|
-
* @returns `true` when a rule was removed, `false` when no rule had that id.
|
|
1013
|
-
* @throws StorageException `STORAGE_LIFECYCLE_RULE_INVALID` for a bad id or an id that names a rule
|
|
1014
|
-
* outside `keyPrefix`; a provider failure otherwise.
|
|
1015
|
-
*/
|
|
1016
1331
|
async remove(id, options) {
|
|
1017
1332
|
this.assertConfigured();
|
|
1018
1333
|
this.assertRuleId(id);
|
|
1019
1334
|
const bucket = this.resolveBucket(options?.bucket);
|
|
1020
|
-
const snapshot = await this.read(bucket);
|
|
1335
|
+
const snapshot = await this.read(bucket, options?.bucket);
|
|
1021
1336
|
const target = snapshot.rules.find((existing) => existing.ID === id);
|
|
1022
1337
|
if (target === void 0) {
|
|
1023
1338
|
return false;
|
|
1024
1339
|
}
|
|
1025
1340
|
this.assertOwnable(target);
|
|
1026
1341
|
const rules = snapshot.rules.filter((existing) => existing.ID !== id);
|
|
1027
|
-
await this.write(bucket, rules, snapshot, "remove");
|
|
1342
|
+
await this.write(bucket, options?.bucket, rules, snapshot, "remove");
|
|
1028
1343
|
return true;
|
|
1029
1344
|
}
|
|
1030
1345
|
/**
|
|
@@ -1078,7 +1393,7 @@ var BucketLifecycleService = class {
|
|
|
1078
1393
|
};
|
|
1079
1394
|
}
|
|
1080
1395
|
/** Reads the configuration; "none yet" is an empty snapshot. */
|
|
1081
|
-
async read(bucket) {
|
|
1396
|
+
async read(bucket, perCall) {
|
|
1082
1397
|
try {
|
|
1083
1398
|
const response = await this.s3Provider.getClient().send(new GetBucketLifecycleConfigurationCommand({ Bucket: bucket }));
|
|
1084
1399
|
return {
|
|
@@ -1089,7 +1404,7 @@ var BucketLifecycleService = class {
|
|
|
1089
1404
|
if (isNoLifecycleConfiguration(err)) {
|
|
1090
1405
|
return { rules: [], transitionDefaultMinimumObjectSize: void 0 };
|
|
1091
1406
|
}
|
|
1092
|
-
throw mapLifecycleError(err,
|
|
1407
|
+
throw mapLifecycleError(err, callDetails("lifecycle.get", { bucket: perCall }));
|
|
1093
1408
|
}
|
|
1094
1409
|
}
|
|
1095
1410
|
/**
|
|
@@ -1097,7 +1412,7 @@ var BucketLifecycleService = class {
|
|
|
1097
1412
|
* requires at least one rule on a put. The default-minimum-object-size setting
|
|
1098
1413
|
* read earlier is sent back so a write does not silently change transition behaviour.
|
|
1099
1414
|
*/
|
|
1100
|
-
async write(bucket, rules, snapshot, op) {
|
|
1415
|
+
async write(bucket, perCall, rules, snapshot, op) {
|
|
1101
1416
|
const client = this.s3Provider.getClient();
|
|
1102
1417
|
try {
|
|
1103
1418
|
if (rules.length === 0) {
|
|
@@ -1112,7 +1427,7 @@ var BucketLifecycleService = class {
|
|
|
1112
1427
|
})
|
|
1113
1428
|
);
|
|
1114
1429
|
} catch (err) {
|
|
1115
|
-
throw mapLifecycleError(err,
|
|
1430
|
+
throw mapLifecycleError(err, callDetails(`lifecycle.${op}`, { bucket: perCall }));
|
|
1116
1431
|
}
|
|
1117
1432
|
}
|
|
1118
1433
|
/** Validates the input and builds the rule, an expiration or a multipart-abort rule. */
|
|
@@ -1162,11 +1477,22 @@ var BucketLifecycleService = class {
|
|
|
1162
1477
|
return resolveBucket(perCall, this.options.bucket);
|
|
1163
1478
|
}
|
|
1164
1479
|
};
|
|
1480
|
+
__decorateClass([
|
|
1481
|
+
Observed("lifecycle_get")
|
|
1482
|
+
], BucketLifecycleService.prototype, "get", 1);
|
|
1483
|
+
__decorateClass([
|
|
1484
|
+
Observed("lifecycle_upsert")
|
|
1485
|
+
], BucketLifecycleService.prototype, "upsert", 1);
|
|
1486
|
+
__decorateClass([
|
|
1487
|
+
Observed("lifecycle_remove")
|
|
1488
|
+
], BucketLifecycleService.prototype, "remove", 1);
|
|
1165
1489
|
BucketLifecycleService = __decorateClass([
|
|
1166
1490
|
Injectable(),
|
|
1167
1491
|
__decorateParam(0, Inject(BYMAX_STORAGE_OPTIONS)),
|
|
1168
1492
|
__decorateParam(1, Inject(S3ClientProvider)),
|
|
1169
|
-
__decorateParam(2, Inject(KeyResolverService))
|
|
1493
|
+
__decorateParam(2, Inject(KeyResolverService)),
|
|
1494
|
+
__decorateParam(3, Optional()),
|
|
1495
|
+
__decorateParam(3, Inject(StorageObserver))
|
|
1170
1496
|
], BucketLifecycleService);
|
|
1171
1497
|
function isBufferLike(body) {
|
|
1172
1498
|
return body instanceof Uint8Array;
|
|
@@ -1320,18 +1646,17 @@ function assertMimeAllowed(whitelist, contentType) {
|
|
|
1320
1646
|
return;
|
|
1321
1647
|
}
|
|
1322
1648
|
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_MIME_NOT_ALLOWED, void 0, {
|
|
1323
|
-
...isHeaderValue(contentType) ? { contentType } : {}
|
|
1324
|
-
whitelist
|
|
1649
|
+
...isHeaderValue(contentType) ? { contentType } : {}
|
|
1325
1650
|
});
|
|
1326
1651
|
}
|
|
1327
1652
|
|
|
1328
1653
|
// src/server/utils/numeric-input.ts
|
|
1329
|
-
function assertIntegerInput(value,
|
|
1654
|
+
function assertIntegerInput(value, field, min) {
|
|
1330
1655
|
if (value === void 0) return;
|
|
1331
1656
|
if (!Number.isInteger(value) || value < min) {
|
|
1332
1657
|
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_VALIDATION_FAILED, void 0, {
|
|
1333
|
-
field
|
|
1334
|
-
reason: `${
|
|
1658
|
+
field,
|
|
1659
|
+
reason: `${field} must be an integer of at least ${String(min)}`
|
|
1335
1660
|
});
|
|
1336
1661
|
}
|
|
1337
1662
|
}
|
|
@@ -1431,46 +1756,41 @@ ValidationService = __decorateClass([
|
|
|
1431
1756
|
__decorateParam(0, Inject(BYMAX_STORAGE_OPTIONS)),
|
|
1432
1757
|
__decorateParam(1, Inject(BYMAX_STORAGE_UPLOAD_VALIDATORS))
|
|
1433
1758
|
], ValidationService);
|
|
1434
|
-
|
|
1435
|
-
// src/server/utils/cut-text.ts
|
|
1436
|
-
function cutText(text, max) {
|
|
1437
|
-
if (text.length <= max) return text;
|
|
1438
|
-
const lastKept = text.charCodeAt(max - 1);
|
|
1439
|
-
const splitsPair = lastKept >= 55296 && lastKept <= 56319;
|
|
1440
|
-
return text.slice(0, splitsPair ? max - 1 : max);
|
|
1441
|
-
}
|
|
1442
|
-
|
|
1443
|
-
// src/server/services/file-scanner.service.ts
|
|
1444
1759
|
var UNSERIALISABLE = "[unserialisable]";
|
|
1445
1760
|
var MAX_SCANNER_LABEL_LENGTH = 200;
|
|
1446
1761
|
var MAX_SCANNER_DETAILS_LENGTH = 300;
|
|
1447
1762
|
var MAX_STATUS_LOG_LENGTH = 40;
|
|
1448
1763
|
function statusForLog(status) {
|
|
1449
|
-
return typeof status === "string" ?
|
|
1764
|
+
return typeof status === "string" ? escapeForLog(cutText(status, MAX_STATUS_LOG_LENGTH)) : `<${typeof status}>`;
|
|
1450
1765
|
}
|
|
1451
1766
|
function scannerLabel(value) {
|
|
1452
1767
|
return typeof value === "string" ? cutText(value, MAX_SCANNER_LABEL_LENGTH) : void 0;
|
|
1453
1768
|
}
|
|
1769
|
+
function labelForLog(label) {
|
|
1770
|
+
return label === void 0 ? void 0 : escapeForLog(label);
|
|
1771
|
+
}
|
|
1454
1772
|
function toJson(value) {
|
|
1455
1773
|
return JSON.stringify(value);
|
|
1456
1774
|
}
|
|
1457
1775
|
function scannerDetailsForLog(details) {
|
|
1458
|
-
if (details === void 0) return
|
|
1776
|
+
if (details === void 0) return {};
|
|
1459
1777
|
let text;
|
|
1460
1778
|
try {
|
|
1461
1779
|
text = toJson(details) ?? UNSERIALISABLE;
|
|
1462
1780
|
} catch {
|
|
1463
1781
|
text = UNSERIALISABLE;
|
|
1464
1782
|
}
|
|
1465
|
-
return
|
|
1783
|
+
return { details: escapeForLog(cutText(text, MAX_SCANNER_DETAILS_LENGTH)) };
|
|
1466
1784
|
}
|
|
1467
1785
|
var FileScannerService = class {
|
|
1468
|
-
constructor(options, scanner) {
|
|
1786
|
+
constructor(options, scanner, observer) {
|
|
1469
1787
|
this.options = options;
|
|
1470
1788
|
this.scanner = scanner;
|
|
1789
|
+
this.observer = observer;
|
|
1471
1790
|
}
|
|
1472
1791
|
options;
|
|
1473
1792
|
scanner;
|
|
1793
|
+
observer;
|
|
1474
1794
|
logger = new Logger(FileScannerService.name);
|
|
1475
1795
|
/**
|
|
1476
1796
|
* Returns `true` when both a scanner implementation and scanner options are
|
|
@@ -1504,19 +1824,27 @@ var FileScannerService = class {
|
|
|
1504
1824
|
* says, and a scan that outlasts `scanner.timeoutMs` throws `STORAGE_TIMEOUT`.
|
|
1505
1825
|
*
|
|
1506
1826
|
* @param input - The scan input.
|
|
1827
|
+
* @param caller - What the deadline error repeats of the call (see `callerFields`): the caller's own key and
|
|
1828
|
+
* bucket, never the prefixed key or the configured bucket that `input` holds for the scanner.
|
|
1507
1829
|
* @returns The scan result when `status` is `'clean'` or `'unknown'` (non-rejecting).
|
|
1508
1830
|
* @throws Error when called without a configured scanner.
|
|
1509
1831
|
* @throws StorageException `STORAGE_SCAN_INFECTED` on an infected result.
|
|
1510
1832
|
* @throws StorageException `STORAGE_SCAN_INCONCLUSIVE` on an unknown result with `rejectOnUnknown`, or a result outside the contract.
|
|
1511
1833
|
* @throws StorageException `STORAGE_TIMEOUT` when the scan outlasts `scanner.timeoutMs`.
|
|
1512
1834
|
*/
|
|
1513
|
-
async scan(input) {
|
|
1835
|
+
async scan(input, caller = {}) {
|
|
1514
1836
|
if (!this.scanner) {
|
|
1515
1837
|
throw new Error(
|
|
1516
1838
|
"FileScannerService.scan() called without a configured scanner \u2014 guard with isEnabled() first."
|
|
1517
1839
|
);
|
|
1518
1840
|
}
|
|
1519
|
-
const
|
|
1841
|
+
const scanner = this.scanner;
|
|
1842
|
+
const run = () => this.evaluate(scanner, input, caller);
|
|
1843
|
+
return this.observer === void 0 ? run() : this.observer.scan(input.mode, run);
|
|
1844
|
+
}
|
|
1845
|
+
/** Runs the scanner and applies the library's policy to what it answered. */
|
|
1846
|
+
async evaluate(scanner, input, caller) {
|
|
1847
|
+
const result = await this.scanLogged(scanner, input, caller);
|
|
1520
1848
|
const status = result?.status;
|
|
1521
1849
|
const engine = scannerLabel(result?.engine);
|
|
1522
1850
|
const threat = scannerLabel(result?.threat);
|
|
@@ -1525,18 +1853,41 @@ var FileScannerService = class {
|
|
|
1525
1853
|
return result;
|
|
1526
1854
|
}
|
|
1527
1855
|
if (status === "infected") {
|
|
1528
|
-
this.logger
|
|
1856
|
+
emit(this.logger, "warn", {
|
|
1857
|
+
event: STORAGE_LOG_EVENTS.scanInfected,
|
|
1858
|
+
message: "Infected file detected",
|
|
1859
|
+
engine: labelForLog(engine),
|
|
1860
|
+
threat: escapeForLog(threat ?? "unknown"),
|
|
1861
|
+
...details
|
|
1862
|
+
});
|
|
1529
1863
|
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_SCAN_INFECTED, void 0, {
|
|
1530
1864
|
engine,
|
|
1531
1865
|
threat
|
|
1532
1866
|
});
|
|
1533
1867
|
}
|
|
1534
1868
|
if (status === "unknown" && !this.options.scanner?.rejectOnUnknown) {
|
|
1535
|
-
this.logger
|
|
1869
|
+
emit(this.logger, "warn", {
|
|
1870
|
+
event: STORAGE_LOG_EVENTS.scanInconclusiveAccepted,
|
|
1871
|
+
message: "Inconclusive scan result accepted",
|
|
1872
|
+
engine: labelForLog(engine)
|
|
1873
|
+
});
|
|
1536
1874
|
return result;
|
|
1537
1875
|
}
|
|
1876
|
+
return this.rejectInconclusive(status, engine, details);
|
|
1877
|
+
}
|
|
1878
|
+
/**
|
|
1879
|
+
* `unknown` with `rejectOnUnknown`, or a result outside the contract (no result, a status that is not
|
|
1880
|
+
* exactly one of the three): the object is not known to be clean, so it is not stored.
|
|
1881
|
+
*/
|
|
1882
|
+
rejectInconclusive(status, engine, details) {
|
|
1538
1883
|
const unrecognised = status !== "unknown";
|
|
1539
|
-
this.logger
|
|
1884
|
+
emit(this.logger, "warn", {
|
|
1885
|
+
event: STORAGE_LOG_EVENTS.scanInconclusiveRejected,
|
|
1886
|
+
message: "Inconclusive scan result rejected",
|
|
1887
|
+
engine: labelForLog(engine),
|
|
1888
|
+
...unrecognised ? { status: statusForLog(status) } : {},
|
|
1889
|
+
...details
|
|
1890
|
+
});
|
|
1540
1891
|
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_SCAN_INCONCLUSIVE, void 0, {
|
|
1541
1892
|
engine,
|
|
1542
1893
|
...unrecognised ? { reason: "unrecognised scanner status" } : {}
|
|
@@ -1546,14 +1897,16 @@ var FileScannerService = class {
|
|
|
1546
1897
|
* Runs the scan and logs a scanner that throws: the caller is answered 500 for that, and without the
|
|
1547
1898
|
* line there would be no trace of why. A typed failure of ours (the deadline) says it itself.
|
|
1548
1899
|
*/
|
|
1549
|
-
async scanLogged(scanner, input) {
|
|
1900
|
+
async scanLogged(scanner, input, caller) {
|
|
1550
1901
|
try {
|
|
1551
|
-
return await this.scanWithinDeadline(scanner, input);
|
|
1902
|
+
return await this.scanWithinDeadline(scanner, input, caller);
|
|
1552
1903
|
} catch (err) {
|
|
1553
1904
|
if (!(err instanceof StorageException)) {
|
|
1554
|
-
this.logger
|
|
1555
|
-
|
|
1556
|
-
|
|
1905
|
+
emit(this.logger, "warn", {
|
|
1906
|
+
event: STORAGE_LOG_EVENTS.scanFailed,
|
|
1907
|
+
message: "Scanner failed",
|
|
1908
|
+
err: failureError(err)
|
|
1909
|
+
});
|
|
1557
1910
|
}
|
|
1558
1911
|
throw err;
|
|
1559
1912
|
}
|
|
@@ -1563,17 +1916,20 @@ var FileScannerService = class {
|
|
|
1563
1916
|
* the upload for good. The scanner is handed a signal that aborts then; one that ignores it keeps
|
|
1564
1917
|
* running.
|
|
1565
1918
|
*/
|
|
1566
|
-
async scanWithinDeadline(scanner, input) {
|
|
1919
|
+
async scanWithinDeadline(scanner, input, caller) {
|
|
1567
1920
|
const timeoutMs = this.options.scanner?.timeoutMs ?? DEFAULT_SCANNER_TIMEOUT_MS;
|
|
1568
1921
|
const controller = new AbortController();
|
|
1569
1922
|
let timer;
|
|
1570
1923
|
const deadline = new Promise((_resolve, reject) => {
|
|
1571
1924
|
timer = setTimeout(() => {
|
|
1572
|
-
this.logger
|
|
1925
|
+
emit(this.logger, "warn", {
|
|
1926
|
+
event: STORAGE_LOG_EVENTS.scanTimeout,
|
|
1927
|
+
message: "Scan timed out",
|
|
1928
|
+
timeoutMs
|
|
1929
|
+
});
|
|
1573
1930
|
reject(
|
|
1574
1931
|
new StorageException(STORAGE_ERROR_CODES.STORAGE_TIMEOUT, void 0, {
|
|
1575
|
-
|
|
1576
|
-
bucket: input.bucket,
|
|
1932
|
+
...caller,
|
|
1577
1933
|
phase: "scan",
|
|
1578
1934
|
timeoutMs
|
|
1579
1935
|
})
|
|
@@ -1591,7 +1947,9 @@ var FileScannerService = class {
|
|
|
1591
1947
|
FileScannerService = __decorateClass([
|
|
1592
1948
|
Injectable(),
|
|
1593
1949
|
__decorateParam(0, Inject(BYMAX_STORAGE_OPTIONS)),
|
|
1594
|
-
__decorateParam(1, Inject(BYMAX_STORAGE_FILE_SCANNER))
|
|
1950
|
+
__decorateParam(1, Inject(BYMAX_STORAGE_FILE_SCANNER)),
|
|
1951
|
+
__decorateParam(2, Optional()),
|
|
1952
|
+
__decorateParam(2, Inject(StorageObserver))
|
|
1595
1953
|
], FileScannerService);
|
|
1596
1954
|
var SourceIdleError = class extends Error {
|
|
1597
1955
|
constructor(idleMs) {
|
|
@@ -1805,7 +2163,7 @@ function cleanupErrorCode(error) {
|
|
|
1805
2163
|
}
|
|
1806
2164
|
function describeCleanupFailure(error) {
|
|
1807
2165
|
if (!(error instanceof StorageException)) {
|
|
1808
|
-
return
|
|
2166
|
+
return projectProviderFailure(error).message;
|
|
1809
2167
|
}
|
|
1810
2168
|
return error.code;
|
|
1811
2169
|
}
|
|
@@ -1820,7 +2178,6 @@ function withCleanupFailure(scanError, keys, cleanupError) {
|
|
|
1820
2178
|
{
|
|
1821
2179
|
...body.error?.details ?? {},
|
|
1822
2180
|
key: keys.key,
|
|
1823
|
-
storedKey: keys.storedKey,
|
|
1824
2181
|
cleanup: "failed",
|
|
1825
2182
|
cleanupError: cleanupErrorCode(cleanupError)
|
|
1826
2183
|
}
|
|
@@ -1953,11 +2310,13 @@ async function closeMultipartUpload(client, bucket, key, uploadId, onFailure) {
|
|
|
1953
2310
|
function closeWhenCreated(client, bucket, key, created, onFailure) {
|
|
1954
2311
|
void created.then((uploadId) => closeMultipartUpload(client, bucket, key, uploadId, onFailure));
|
|
1955
2312
|
}
|
|
1956
|
-
function abortFailureLogger(logger2
|
|
2313
|
+
function abortFailureLogger(logger2) {
|
|
1957
2314
|
return (err) => {
|
|
1958
|
-
logger2
|
|
1959
|
-
|
|
1960
|
-
|
|
2315
|
+
emit(logger2, "warn", {
|
|
2316
|
+
event: STORAGE_LOG_EVENTS.multipartAbortFailed,
|
|
2317
|
+
message: "Multipart abort failed, the upload stays open on the provider",
|
|
2318
|
+
err: failureError(err)
|
|
2319
|
+
});
|
|
1961
2320
|
};
|
|
1962
2321
|
}
|
|
1963
2322
|
async function deleteObjectsChunk(client, bucket, chunk, stripPrefix) {
|
|
@@ -2073,6 +2432,23 @@ function buildACL(perCall, defaultValue) {
|
|
|
2073
2432
|
return perCall ?? defaultValue ? "public-read" : void 0;
|
|
2074
2433
|
}
|
|
2075
2434
|
|
|
2435
|
+
// src/server/utils/object-metadata.ts
|
|
2436
|
+
function toObjectMetadata(response, key, bucket) {
|
|
2437
|
+
return {
|
|
2438
|
+
key,
|
|
2439
|
+
bucket,
|
|
2440
|
+
size: response.ContentLength ?? 0,
|
|
2441
|
+
contentType: response.ContentType ?? "application/octet-stream",
|
|
2442
|
+
etag: response.ETag ?? "",
|
|
2443
|
+
lastModified: response.LastModified ?? /* @__PURE__ */ new Date(0),
|
|
2444
|
+
metadata: response.Metadata ?? {},
|
|
2445
|
+
...response.CacheControl !== void 0 ? { cacheControl: response.CacheControl } : {},
|
|
2446
|
+
...response.ContentDisposition !== void 0 ? { contentDisposition: response.ContentDisposition } : {},
|
|
2447
|
+
...response.StorageClass !== void 0 ? { storageClass: response.StorageClass } : {},
|
|
2448
|
+
...response.VersionId !== void 0 ? { versionId: response.VersionId } : {}
|
|
2449
|
+
};
|
|
2450
|
+
}
|
|
2451
|
+
|
|
2076
2452
|
// src/server/utils/upload-strategy.ts
|
|
2077
2453
|
function pickUploadStrategy(body, declaredSize, thresholdBytes) {
|
|
2078
2454
|
const size = declaredSize ?? getBodySize(body);
|
|
@@ -2101,18 +2477,18 @@ function boundStreamSize(body, declaredSize, options) {
|
|
|
2101
2477
|
}
|
|
2102
2478
|
return withSizeLimit(body, limit, declaredSize === void 0 ? "max" : "declared");
|
|
2103
2479
|
}
|
|
2104
|
-
async function bufferStreamForScan(body,
|
|
2480
|
+
async function bufferStreamForScan(body, context, maxSizeBytes) {
|
|
2105
2481
|
if (maxSizeBytes === void 0) {
|
|
2106
2482
|
destroyStream(body);
|
|
2107
2483
|
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_VALIDATION_FAILED, void 0, {
|
|
2108
|
-
|
|
2484
|
+
...context,
|
|
2109
2485
|
reason: "Pre-upload scanning of a stream needs validation.maxSizeBytes to bound the buffer; set it or use scanner mode post-upload"
|
|
2110
2486
|
});
|
|
2111
2487
|
}
|
|
2112
2488
|
const buffered = await readStreamUpTo(body, maxSizeBytes);
|
|
2113
2489
|
if (buffered === void 0) {
|
|
2114
2490
|
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_VALIDATION_FAILED, void 0, {
|
|
2115
|
-
|
|
2491
|
+
...context,
|
|
2116
2492
|
maxSizeBytes,
|
|
2117
2493
|
reason: "The stream is longer than validation.maxSizeBytes, so it cannot be scanned before upload"
|
|
2118
2494
|
});
|
|
@@ -2170,6 +2546,13 @@ function baseCarriesBucket(base, bucket) {
|
|
|
2170
2546
|
function normalizeScanBody(body) {
|
|
2171
2547
|
return body;
|
|
2172
2548
|
}
|
|
2549
|
+
function trackLoaded(uploader) {
|
|
2550
|
+
let loaded = 0;
|
|
2551
|
+
uploader.on("httpUploadProgress", (event) => {
|
|
2552
|
+
loaded = event.loaded ?? loaded;
|
|
2553
|
+
});
|
|
2554
|
+
return () => loaded;
|
|
2555
|
+
}
|
|
2173
2556
|
function forwardProgress(uploader, onProgress) {
|
|
2174
2557
|
if (!onProgress) return;
|
|
2175
2558
|
uploader.on("httpUploadProgress", (event) => {
|
|
@@ -2239,11 +2622,10 @@ function assertMaxBytes(maxBytes) {
|
|
|
2239
2622
|
});
|
|
2240
2623
|
}
|
|
2241
2624
|
}
|
|
2242
|
-
async function readBody(stream, metadata, maxBytes) {
|
|
2625
|
+
async function readBody(stream, metadata, maxBytes, context) {
|
|
2243
2626
|
if (maxBytes === void 0) {
|
|
2244
2627
|
return readStream(stream);
|
|
2245
2628
|
}
|
|
2246
|
-
const context = { key: metadata.key, bucket: metadata.bucket };
|
|
2247
2629
|
if (metadata.size > maxBytes) {
|
|
2248
2630
|
destroyStream(stream);
|
|
2249
2631
|
throw sizeLimitException(maxBytes, "max", { ...context, contentLength: metadata.size });
|
|
@@ -2254,15 +2636,50 @@ async function readBody(stream, metadata, maxBytes) {
|
|
|
2254
2636
|
}
|
|
2255
2637
|
return buffer;
|
|
2256
2638
|
}
|
|
2257
|
-
async function readDownloadBody(stream, metadata, maxBytes, signal) {
|
|
2639
|
+
async function readDownloadBody(stream, metadata, maxBytes, signal, caller) {
|
|
2640
|
+
const context = { ...caller, op: "downloadBuffer" };
|
|
2258
2641
|
try {
|
|
2259
|
-
return await readBody(stream, metadata, maxBytes);
|
|
2642
|
+
return await readBody(stream, metadata, maxBytes, context);
|
|
2260
2643
|
} catch (err) {
|
|
2261
|
-
throw mapBodyReadFailure(err, signal,
|
|
2262
|
-
|
|
2263
|
-
|
|
2264
|
-
|
|
2644
|
+
throw mapBodyReadFailure(err, signal, context);
|
|
2645
|
+
}
|
|
2646
|
+
}
|
|
2647
|
+
|
|
2648
|
+
// src/server/utils/uploaded-bytes.ts
|
|
2649
|
+
var SENT = /* @__PURE__ */ new WeakMap();
|
|
2650
|
+
function rememberUploadedBytes(result, bytes) {
|
|
2651
|
+
SENT.set(result, bytes);
|
|
2652
|
+
return result;
|
|
2653
|
+
}
|
|
2654
|
+
function uploadedBytesOf(result) {
|
|
2655
|
+
return result.fromIdempotencyCache ? void 0 : SENT.get(result) ?? result.size;
|
|
2656
|
+
}
|
|
2657
|
+
async function uploadSingleShot(context, options, finalKey, bucket, caller) {
|
|
2658
|
+
const total = options.size ?? getBodySize(options.body);
|
|
2659
|
+
const input = {
|
|
2660
|
+
...putObjectHeaders(options, context.options, finalKey, bucket),
|
|
2661
|
+
Body: normalizeSingleShotBody(options.body),
|
|
2662
|
+
ContentLength: total
|
|
2663
|
+
};
|
|
2664
|
+
const watch = abortOnBodyFailure(input.Body, options.abortSignal);
|
|
2665
|
+
try {
|
|
2666
|
+
const response = await context.s3Provider.getClient().send(new PutObjectCommand(input), sendOptions(watch.signal));
|
|
2667
|
+
if (options.onProgress) {
|
|
2668
|
+
emitProgress(options.onProgress, total, total);
|
|
2669
|
+
}
|
|
2670
|
+
return buildUploadResult(context.keyResolver, context.options, {
|
|
2671
|
+
finalKey,
|
|
2672
|
+
bucket,
|
|
2673
|
+
etag: response.ETag,
|
|
2674
|
+
versionId: response.VersionId,
|
|
2675
|
+
size: total,
|
|
2676
|
+
contentType: options.contentType,
|
|
2677
|
+
multipart: false
|
|
2265
2678
|
});
|
|
2679
|
+
} catch (err) {
|
|
2680
|
+
const context2 = { ...caller, op: "upload-single" };
|
|
2681
|
+
const mapped = mapSizeLimitError(watch.failure(), context2) ?? mapAwsError(err, context2);
|
|
2682
|
+
throw failureOrAbort(options.abortSignal, context2, mapped);
|
|
2266
2683
|
}
|
|
2267
2684
|
}
|
|
2268
2685
|
|
|
@@ -2270,13 +2687,14 @@ async function readDownloadBody(stream, metadata, maxBytes, signal) {
|
|
|
2270
2687
|
var MAX_LIST_KEYS = 1e3;
|
|
2271
2688
|
var DELETE_BATCH_LIMIT = 1e3;
|
|
2272
2689
|
var StorageService = class {
|
|
2273
|
-
constructor(options, s3Provider, keyResolver, idempotencyCache, validation, scanner) {
|
|
2690
|
+
constructor(options, s3Provider, keyResolver, idempotencyCache, validation, scanner, observer) {
|
|
2274
2691
|
this.options = options;
|
|
2275
2692
|
this.s3Provider = s3Provider;
|
|
2276
2693
|
this.keyResolver = keyResolver;
|
|
2277
2694
|
this.idempotencyCache = idempotencyCache;
|
|
2278
2695
|
this.validation = validation;
|
|
2279
2696
|
this.scanner = scanner;
|
|
2697
|
+
this.observer = observer;
|
|
2280
2698
|
}
|
|
2281
2699
|
options;
|
|
2282
2700
|
s3Provider;
|
|
@@ -2284,22 +2702,8 @@ var StorageService = class {
|
|
|
2284
2702
|
idempotencyCache;
|
|
2285
2703
|
validation;
|
|
2286
2704
|
scanner;
|
|
2705
|
+
observer;
|
|
2287
2706
|
logger = new Logger(StorageService.name);
|
|
2288
|
-
/**
|
|
2289
|
-
* Uploads an object, choosing single-shot or multipart automatically. When an
|
|
2290
|
-
* `idempotencyKey` is supplied and a matching result is cached within the TTL
|
|
2291
|
-
* window, the cached result is returned without re-uploading.
|
|
2292
|
-
*
|
|
2293
|
-
* The library takes ownership of a `Readable` body, as `stream.pipeline` does: when the
|
|
2294
|
-
* upload is rejected for any reason (configuration, validation, scan, provider failure)
|
|
2295
|
-
* the stream is destroyed (without an error: the SDK forwards a stream's error to wrappers
|
|
2296
|
-
* that nothing listens to), which releases the file handle or socket behind it; a cache hit
|
|
2297
|
-
* destroys it unread. A stream consumed by a successful upload ends on its own.
|
|
2298
|
-
*
|
|
2299
|
-
* @param options - The upload request.
|
|
2300
|
-
* @returns The upload result.
|
|
2301
|
-
* @throws StorageException for missing configuration, body, or content type.
|
|
2302
|
-
*/
|
|
2303
2707
|
async upload(options) {
|
|
2304
2708
|
try {
|
|
2305
2709
|
return await this.uploadOwningBody(options);
|
|
@@ -2322,7 +2726,7 @@ var StorageService = class {
|
|
|
2322
2726
|
}
|
|
2323
2727
|
const finalKey = this.keyResolver.normalize(options.key);
|
|
2324
2728
|
const bucket = this.resolveBucket(options.bucket);
|
|
2325
|
-
const context = { key:
|
|
2729
|
+
const context = callDetails("upload", { key: options.key, bucket: options.bucket });
|
|
2326
2730
|
throwIfAborted(options.abortSignal, context);
|
|
2327
2731
|
if (options.idempotencyKey) {
|
|
2328
2732
|
const cacheKey = this.idempotencyCache.computeKey(options.idempotencyKey, finalKey);
|
|
@@ -2340,7 +2744,7 @@ var StorageService = class {
|
|
|
2340
2744
|
validatedOptions.size,
|
|
2341
2745
|
this.options.multipart.thresholdBytes
|
|
2342
2746
|
);
|
|
2343
|
-
const result = strategy === "multipart" ? await this.uploadMultipart(validatedOptions, finalKey, bucket) : await this.
|
|
2747
|
+
const result = strategy === "multipart" ? await this.uploadMultipart(validatedOptions, finalKey, bucket, context) : await uploadSingleShot(this.uploadContext, validatedOptions, finalKey, bucket, context);
|
|
2344
2748
|
const stored = { ...context, stored: true };
|
|
2345
2749
|
await abortable(this.runPostUploadScan(finalKey, bucket, options), options.abortSignal, stored);
|
|
2346
2750
|
this.rememberUpload(options, finalKey, result);
|
|
@@ -2356,7 +2760,7 @@ var StorageService = class {
|
|
|
2356
2760
|
const input = validationInput(options, finalKey);
|
|
2357
2761
|
const validated = await abortable(this.validation.validate(input), signal, context);
|
|
2358
2762
|
const scanned = await abortable(
|
|
2359
|
-
this.bufferForPreUploadScan(validated.body,
|
|
2763
|
+
this.bufferForPreUploadScan(validated.body, context),
|
|
2360
2764
|
signal,
|
|
2361
2765
|
context
|
|
2362
2766
|
);
|
|
@@ -2373,28 +2777,31 @@ var StorageService = class {
|
|
|
2373
2777
|
* `validation.maxSizeBytes`, and a stream with no bound is refused rather than
|
|
2374
2778
|
* buffered without limit. Other bodies and modes pass through unchanged.
|
|
2375
2779
|
*/
|
|
2376
|
-
async bufferForPreUploadScan(body,
|
|
2780
|
+
async bufferForPreUploadScan(body, context) {
|
|
2377
2781
|
if (!this.scanner.isEnabled() || this.scanner.getMode() !== "pre-upload") {
|
|
2378
2782
|
return body;
|
|
2379
2783
|
}
|
|
2380
2784
|
if (isBufferLike(body)) {
|
|
2381
2785
|
return Buffer.from(body);
|
|
2382
2786
|
}
|
|
2383
|
-
return bufferStreamForScan(body,
|
|
2787
|
+
return bufferStreamForScan(body, context, this.options.validation?.maxSizeBytes);
|
|
2384
2788
|
}
|
|
2385
2789
|
/** Runs a pre-upload scan when configured; throws on infected. */
|
|
2386
2790
|
async runPreUploadScan(body, finalKey, bucket, options) {
|
|
2387
2791
|
if (!this.scanner.isEnabled() || this.scanner.getMode() !== "pre-upload") {
|
|
2388
2792
|
return;
|
|
2389
2793
|
}
|
|
2390
|
-
await this.scanner.scan(
|
|
2391
|
-
|
|
2392
|
-
|
|
2393
|
-
|
|
2394
|
-
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
|
|
2794
|
+
await this.scanner.scan(
|
|
2795
|
+
{
|
|
2796
|
+
mode: "pre-upload",
|
|
2797
|
+
body: normalizeScanBody(body),
|
|
2798
|
+
key: finalKey,
|
|
2799
|
+
bucket,
|
|
2800
|
+
contentType: options.contentType,
|
|
2801
|
+
...options.size !== void 0 ? { size: options.size } : {}
|
|
2802
|
+
},
|
|
2803
|
+
callerFields({ key: options.key, bucket: options.bucket })
|
|
2804
|
+
);
|
|
2398
2805
|
}
|
|
2399
2806
|
/** Remembers the result of an upload under its idempotency key, when it has one. */
|
|
2400
2807
|
rememberUpload(options, finalKey, result) {
|
|
@@ -2413,47 +2820,44 @@ var StorageService = class {
|
|
|
2413
2820
|
return;
|
|
2414
2821
|
}
|
|
2415
2822
|
try {
|
|
2416
|
-
await this.scanner.scan(
|
|
2417
|
-
|
|
2418
|
-
|
|
2419
|
-
|
|
2420
|
-
|
|
2421
|
-
|
|
2422
|
-
|
|
2823
|
+
await this.scanner.scan(
|
|
2824
|
+
{
|
|
2825
|
+
mode: "post-upload",
|
|
2826
|
+
key: finalKey,
|
|
2827
|
+
bucket,
|
|
2828
|
+
contentType: options.contentType,
|
|
2829
|
+
...options.size !== void 0 ? { size: options.size } : {}
|
|
2830
|
+
},
|
|
2831
|
+
callerFields({ key: options.key, bucket: options.bucket })
|
|
2832
|
+
);
|
|
2423
2833
|
} catch (scanErr) {
|
|
2424
2834
|
const key = this.keyResolver.stripPrefix(finalKey);
|
|
2425
2835
|
try {
|
|
2426
2836
|
await this.delete(key, { bucket, abortSignal: AbortSignal.timeout(CLEANUP_DEADLINE_MS) });
|
|
2427
2837
|
} catch (deleteErr) {
|
|
2428
|
-
this.logger
|
|
2429
|
-
|
|
2430
|
-
|
|
2431
|
-
|
|
2432
|
-
|
|
2838
|
+
emit(this.logger, "error", {
|
|
2839
|
+
event: STORAGE_LOG_EVENTS.scanCleanupFailed,
|
|
2840
|
+
message: "The rejected object is still stored",
|
|
2841
|
+
storedKey: escapeForLog(finalKey),
|
|
2842
|
+
cleanupFailure: describeCleanupFailure(deleteErr)
|
|
2843
|
+
});
|
|
2844
|
+
throw withCleanupFailure(scanErr, { key }, deleteErr);
|
|
2433
2845
|
}
|
|
2434
2846
|
throw scanErr;
|
|
2435
2847
|
}
|
|
2436
2848
|
}
|
|
2437
|
-
/**
|
|
2438
|
-
* Returns metadata for an object without downloading its body.
|
|
2439
|
-
*
|
|
2440
|
-
* @param key - The raw object key (normalized internally).
|
|
2441
|
-
* @param options - Optional per-call bucket override.
|
|
2442
|
-
* @returns The object metadata.
|
|
2443
|
-
* @throws StorageException `STORAGE_OBJECT_NOT_FOUND` when the key is absent.
|
|
2444
|
-
*/
|
|
2445
2849
|
async head(key, options) {
|
|
2446
2850
|
this.assertConfigured();
|
|
2447
2851
|
const finalKey = this.keyResolver.normalize(key);
|
|
2448
2852
|
const bucket = this.resolveBucket(options?.bucket);
|
|
2449
|
-
const context = { key
|
|
2853
|
+
const context = callDetails("head", { key, bucket: options?.bucket });
|
|
2450
2854
|
throwIfAborted(options?.abortSignal, context);
|
|
2451
2855
|
try {
|
|
2452
2856
|
const response = await this.s3Provider.getClient().send(
|
|
2453
2857
|
new HeadObjectCommand({ Bucket: bucket, Key: finalKey }),
|
|
2454
2858
|
sendOptions(options?.abortSignal)
|
|
2455
2859
|
);
|
|
2456
|
-
return
|
|
2860
|
+
return toObjectMetadata(response, this.keyResolver.stripPrefix(finalKey), bucket);
|
|
2457
2861
|
} catch (err) {
|
|
2458
2862
|
throw failureOrAbort(options?.abortSignal, context, mapAwsError(err, context));
|
|
2459
2863
|
}
|
|
@@ -2477,9 +2881,11 @@ var StorageService = class {
|
|
|
2477
2881
|
if (err instanceof StorageException && err.code === STORAGE_ERROR_CODES.STORAGE_REQUEST_ABORTED) {
|
|
2478
2882
|
throw err;
|
|
2479
2883
|
}
|
|
2480
|
-
this.logger
|
|
2481
|
-
|
|
2482
|
-
|
|
2884
|
+
emit(this.logger, "warn", {
|
|
2885
|
+
event: STORAGE_LOG_EVENTS.existsFailed,
|
|
2886
|
+
message: 'exists() \u2014 treating a non-404 error as "false"',
|
|
2887
|
+
err: failureError(err)
|
|
2888
|
+
});
|
|
2483
2889
|
return false;
|
|
2484
2890
|
}
|
|
2485
2891
|
}
|
|
@@ -2496,15 +2902,6 @@ var StorageService = class {
|
|
|
2496
2902
|
const bucket = this.resolveBucket(options?.bucket);
|
|
2497
2903
|
return buildPublicUrl(this.options, finalKey, bucket);
|
|
2498
2904
|
}
|
|
2499
|
-
/**
|
|
2500
|
-
* Streams an object. The returned `stream` is a Node `Readable` consumable via
|
|
2501
|
-
* `for await` or `.pipe()`. `range` / `ifNoneMatch` / `ifMatch` are forwarded as
|
|
2502
|
-
* conditional/partial GET headers.
|
|
2503
|
-
*
|
|
2504
|
-
* @param options - The download request.
|
|
2505
|
-
* @returns The object stream and its metadata.
|
|
2506
|
-
* @throws StorageException `STORAGE_OBJECT_NOT_FOUND` when the body is absent.
|
|
2507
|
-
*/
|
|
2508
2905
|
async download(options) {
|
|
2509
2906
|
this.assertConfigured();
|
|
2510
2907
|
assertHeaderValues({
|
|
@@ -2514,7 +2911,7 @@ var StorageService = class {
|
|
|
2514
2911
|
});
|
|
2515
2912
|
const finalKey = this.keyResolver.normalize(options.key);
|
|
2516
2913
|
const bucket = this.resolveBucket(options.bucket);
|
|
2517
|
-
const context = { key:
|
|
2914
|
+
const context = callDetails("download", { key: options.key, bucket: options.bucket });
|
|
2518
2915
|
throwIfAborted(options.abortSignal, context);
|
|
2519
2916
|
try {
|
|
2520
2917
|
const response = await this.s3Provider.getClient().send(
|
|
@@ -2528,10 +2925,7 @@ var StorageService = class {
|
|
|
2528
2925
|
sendOptions(options.abortSignal)
|
|
2529
2926
|
);
|
|
2530
2927
|
if (!response.Body) {
|
|
2531
|
-
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_OBJECT_NOT_FOUND, void 0,
|
|
2532
|
-
key: finalKey,
|
|
2533
|
-
bucket
|
|
2534
|
-
});
|
|
2928
|
+
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_OBJECT_NOT_FOUND, void 0, context);
|
|
2535
2929
|
}
|
|
2536
2930
|
const stream = guardDownloadBody(
|
|
2537
2931
|
response.Body,
|
|
@@ -2539,7 +2933,10 @@ var StorageService = class {
|
|
|
2539
2933
|
context,
|
|
2540
2934
|
options.abortSignal
|
|
2541
2935
|
);
|
|
2542
|
-
return {
|
|
2936
|
+
return {
|
|
2937
|
+
stream,
|
|
2938
|
+
metadata: toObjectMetadata(response, this.keyResolver.stripPrefix(finalKey), bucket)
|
|
2939
|
+
};
|
|
2543
2940
|
} catch (err) {
|
|
2544
2941
|
if (err instanceof StorageException) {
|
|
2545
2942
|
throw err;
|
|
@@ -2559,21 +2956,20 @@ var StorageService = class {
|
|
|
2559
2956
|
async downloadBuffer(options) {
|
|
2560
2957
|
assertMaxBytes(options.maxBytes);
|
|
2561
2958
|
const { stream, metadata } = await this.download(options);
|
|
2562
|
-
const buffer = await readDownloadBody(
|
|
2959
|
+
const buffer = await readDownloadBody(
|
|
2960
|
+
stream,
|
|
2961
|
+
metadata,
|
|
2962
|
+
options.maxBytes,
|
|
2963
|
+
options.abortSignal,
|
|
2964
|
+
callerFields({ key: options.key, bucket: options.bucket })
|
|
2965
|
+
);
|
|
2563
2966
|
return { buffer, metadata };
|
|
2564
2967
|
}
|
|
2565
|
-
/**
|
|
2566
|
-
* Deletes an object. Idempotent: a missing object is a no-op (logged as a
|
|
2567
|
-
* warning), not an error. Any other failure is propagated.
|
|
2568
|
-
*
|
|
2569
|
-
* @param key - The raw object key.
|
|
2570
|
-
* @param options - Optional per-call bucket override.
|
|
2571
|
-
*/
|
|
2572
2968
|
async delete(key, options) {
|
|
2573
2969
|
this.assertConfigured();
|
|
2574
2970
|
const finalKey = this.keyResolver.normalize(key);
|
|
2575
2971
|
const bucket = this.resolveBucket(options?.bucket);
|
|
2576
|
-
const context = { key
|
|
2972
|
+
const context = callDetails("delete", { key, bucket: options?.bucket });
|
|
2577
2973
|
throwIfAborted(options?.abortSignal, context);
|
|
2578
2974
|
try {
|
|
2579
2975
|
await this.s3Provider.getClient().send(
|
|
@@ -2585,25 +2981,17 @@ var StorageService = class {
|
|
|
2585
2981
|
if (mapped.code !== STORAGE_ERROR_CODES.STORAGE_OBJECT_NOT_FOUND) {
|
|
2586
2982
|
throw mapped;
|
|
2587
2983
|
}
|
|
2588
|
-
this.logger
|
|
2984
|
+
emit(this.logger, "warn", {
|
|
2985
|
+
event: STORAGE_LOG_EVENTS.deleteNotFound,
|
|
2986
|
+
message: "delete() \u2014 key not found (idempotent no-op)"
|
|
2987
|
+
});
|
|
2589
2988
|
}
|
|
2590
2989
|
this.idempotencyCache.invalidate(bucket, this.keyResolver.stripPrefix(finalKey));
|
|
2591
2990
|
}
|
|
2592
|
-
/**
|
|
2593
|
-
* Probes the bucket with one `HeadBucket`, for a readiness check: it resolves when the bucket
|
|
2594
|
-
* is reachable with these credentials and otherwise rejects with a typed code. Nothing in the
|
|
2595
|
-
* module calls it on its own, so booting costs no request. Needs `s3:ListBucket`.
|
|
2596
|
-
*
|
|
2597
|
-
* @param options - Optional per-call bucket override.
|
|
2598
|
-
* @throws StorageException `STORAGE_BUCKET_NOT_FOUND` (404), `STORAGE_BUCKET_ACCESS_DENIED` (403,
|
|
2599
|
-
* which a provider also answers for a bucket the caller cannot see), `STORAGE_BUCKET_REGION_MISMATCH`
|
|
2600
|
-
* (301), `STORAGE_TIMEOUT`, or `STORAGE_PROVIDER_ERROR`; `STORAGE_NOT_CONFIGURED` and
|
|
2601
|
-
* `STORAGE_BUCKET_UNDEFINED` before any request.
|
|
2602
|
-
*/
|
|
2603
2991
|
async checkBucket(options) {
|
|
2604
2992
|
this.assertConfigured();
|
|
2605
2993
|
const bucket = this.resolveBucket(options?.bucket);
|
|
2606
|
-
const context = { bucket
|
|
2994
|
+
const context = callDetails("checkBucket", { bucket: options?.bucket });
|
|
2607
2995
|
throwIfAborted(options?.abortSignal, context);
|
|
2608
2996
|
try {
|
|
2609
2997
|
await this.s3Provider.getClient().send(new HeadBucketCommand({ Bucket: bucket }), sendOptions(options?.abortSignal));
|
|
@@ -2611,6 +2999,10 @@ var StorageService = class {
|
|
|
2611
2999
|
throw failureOrAbort(options?.abortSignal, context, mapBucketCheckError(err, context));
|
|
2612
3000
|
}
|
|
2613
3001
|
}
|
|
3002
|
+
/** What the single-shot upload needs of this service. */
|
|
3003
|
+
get uploadContext() {
|
|
3004
|
+
return { s3Provider: this.s3Provider, keyResolver: this.keyResolver, options: this.options };
|
|
3005
|
+
}
|
|
2614
3006
|
/** Throws when the S3 client was never built (missing credentials). */
|
|
2615
3007
|
assertConfigured() {
|
|
2616
3008
|
if (!this.s3Provider.isConfigured()) {
|
|
@@ -2621,58 +3013,13 @@ var StorageService = class {
|
|
|
2621
3013
|
resolveBucket(perCall) {
|
|
2622
3014
|
return resolveBucket(perCall, this.options.bucket);
|
|
2623
3015
|
}
|
|
2624
|
-
/** Maps an S3 metadata response onto the public `ObjectMetadata` shape. */
|
|
2625
|
-
toObjectMetadata(response, finalKey, bucket) {
|
|
2626
|
-
return {
|
|
2627
|
-
key: this.keyResolver.stripPrefix(finalKey),
|
|
2628
|
-
bucket,
|
|
2629
|
-
size: response.ContentLength ?? 0,
|
|
2630
|
-
contentType: response.ContentType ?? "application/octet-stream",
|
|
2631
|
-
etag: response.ETag ?? "",
|
|
2632
|
-
lastModified: response.LastModified ?? /* @__PURE__ */ new Date(0),
|
|
2633
|
-
metadata: response.Metadata ?? {},
|
|
2634
|
-
...response.CacheControl !== void 0 ? { cacheControl: response.CacheControl } : {},
|
|
2635
|
-
...response.ContentDisposition !== void 0 ? { contentDisposition: response.ContentDisposition } : {},
|
|
2636
|
-
...response.StorageClass !== void 0 ? { storageClass: response.StorageClass } : {},
|
|
2637
|
-
...response.VersionId !== void 0 ? { versionId: response.VersionId } : {}
|
|
2638
|
-
};
|
|
2639
|
-
}
|
|
2640
|
-
/** Single `PutObject` upload for bodies below the multipart threshold. */
|
|
2641
|
-
async uploadSingleShot(options, finalKey, bucket) {
|
|
2642
|
-
const total = options.size ?? getBodySize(options.body);
|
|
2643
|
-
const input = {
|
|
2644
|
-
...putObjectHeaders(options, this.options, finalKey, bucket),
|
|
2645
|
-
Body: normalizeSingleShotBody(options.body),
|
|
2646
|
-
ContentLength: total
|
|
2647
|
-
};
|
|
2648
|
-
const watch = abortOnBodyFailure(input.Body, options.abortSignal);
|
|
2649
|
-
try {
|
|
2650
|
-
const response = await this.s3Provider.getClient().send(new PutObjectCommand(input), sendOptions(watch.signal));
|
|
2651
|
-
if (options.onProgress) {
|
|
2652
|
-
emitProgress(options.onProgress, total, total);
|
|
2653
|
-
}
|
|
2654
|
-
return buildUploadResult(this.keyResolver, this.options, {
|
|
2655
|
-
finalKey,
|
|
2656
|
-
bucket,
|
|
2657
|
-
etag: response.ETag,
|
|
2658
|
-
versionId: response.VersionId,
|
|
2659
|
-
size: total,
|
|
2660
|
-
contentType: options.contentType,
|
|
2661
|
-
multipart: false
|
|
2662
|
-
});
|
|
2663
|
-
} catch (err) {
|
|
2664
|
-
const context = { key: finalKey, bucket, op: "upload-single" };
|
|
2665
|
-
const mapped = mapSizeLimitError(watch.failure(), context) ?? mapAwsError(err, context);
|
|
2666
|
-
throw failureOrAbort(options.abortSignal, context, mapped);
|
|
2667
|
-
}
|
|
2668
|
-
}
|
|
2669
3016
|
/**
|
|
2670
3017
|
* Multipart upload via `@aws-sdk/lib-storage`. The SDK is told to leave the parts
|
|
2671
3018
|
* (`leavePartsOnError: true`) because its own abort runs inside its failure path: a failing
|
|
2672
3019
|
* abort replaced the failure that caused it, and a failed `CompleteMultipartUpload` was never
|
|
2673
3020
|
* aborted. This method aborts the upload itself and reports the outcome in the error details.
|
|
2674
3021
|
*/
|
|
2675
|
-
async uploadMultipart(options, finalKey, bucket) {
|
|
3022
|
+
async uploadMultipart(options, finalKey, bucket, caller) {
|
|
2676
3023
|
const params = {
|
|
2677
3024
|
...putObjectHeaders(options, this.options, finalKey, bucket),
|
|
2678
3025
|
Body: guardIdleBody(options.body, this.options.uploadIdleTimeoutMs)
|
|
@@ -2686,6 +3033,7 @@ var StorageService = class {
|
|
|
2686
3033
|
partSize: this.options.multipart.partSizeBytes,
|
|
2687
3034
|
leavePartsOnError: true
|
|
2688
3035
|
});
|
|
3036
|
+
const sent = trackLoaded(uploader);
|
|
2689
3037
|
forwardProgress(uploader, options.onProgress);
|
|
2690
3038
|
const abortUpload = () => {
|
|
2691
3039
|
void uploader.abort();
|
|
@@ -2693,17 +3041,20 @@ var StorageService = class {
|
|
|
2693
3041
|
signal?.addEventListener("abort", abortUpload, { once: true });
|
|
2694
3042
|
try {
|
|
2695
3043
|
const response = await uploader.done();
|
|
2696
|
-
return
|
|
2697
|
-
|
|
2698
|
-
|
|
2699
|
-
|
|
2700
|
-
|
|
2701
|
-
|
|
2702
|
-
|
|
2703
|
-
|
|
2704
|
-
|
|
3044
|
+
return rememberUploadedBytes(
|
|
3045
|
+
buildUploadResult(this.keyResolver, this.options, {
|
|
3046
|
+
finalKey,
|
|
3047
|
+
bucket,
|
|
3048
|
+
etag: response.ETag,
|
|
3049
|
+
versionId: response.VersionId,
|
|
3050
|
+
size: options.size,
|
|
3051
|
+
contentType: options.contentType,
|
|
3052
|
+
multipart: true
|
|
3053
|
+
}),
|
|
3054
|
+
sent()
|
|
3055
|
+
);
|
|
2705
3056
|
} catch (err) {
|
|
2706
|
-
return await this.failMultipart(err, finalKey, bucket, {
|
|
3057
|
+
return await this.failMultipart(err, finalKey, bucket, caller, {
|
|
2707
3058
|
uploadId: uploader.uploadId,
|
|
2708
3059
|
signal,
|
|
2709
3060
|
createdUploadId: bound?.createdUploadId
|
|
@@ -2718,7 +3069,7 @@ var StorageService = class {
|
|
|
2718
3069
|
* real failure into one. An upload id the provider answered late is waited for, within the cleanup
|
|
2719
3070
|
* deadline, so that an abort during `CreateMultipartUpload` still aborts the upload it created.
|
|
2720
3071
|
*/
|
|
2721
|
-
async failMultipart(err, finalKey, bucket, upload) {
|
|
3072
|
+
async failMultipart(err, finalKey, bucket, caller, upload) {
|
|
2722
3073
|
const abortedBy = upload.signal?.aborted === true ? upload.signal : void 0;
|
|
2723
3074
|
const client = this.s3Provider.getClient();
|
|
2724
3075
|
const lookup = abortedBy ? upload.createdUploadId?.() : void 0;
|
|
@@ -2726,7 +3077,7 @@ var StorageService = class {
|
|
|
2726
3077
|
if (uploadId === void 0 && lookup) {
|
|
2727
3078
|
uploadId = await within(lookup, CLEANUP_DEADLINE_MS);
|
|
2728
3079
|
if (uploadId === void 0) {
|
|
2729
|
-
closeWhenCreated(client, bucket, finalKey, lookup, abortFailureLogger(this.logger
|
|
3080
|
+
closeWhenCreated(client, bucket, finalKey, lookup, abortFailureLogger(this.logger));
|
|
2730
3081
|
}
|
|
2731
3082
|
}
|
|
2732
3083
|
const abort = await closeMultipartUpload(
|
|
@@ -2734,27 +3085,18 @@ var StorageService = class {
|
|
|
2734
3085
|
bucket,
|
|
2735
3086
|
finalKey,
|
|
2736
3087
|
uploadId,
|
|
2737
|
-
abortFailureLogger(this.logger
|
|
3088
|
+
abortFailureLogger(this.logger)
|
|
2738
3089
|
);
|
|
2739
|
-
const context = {
|
|
3090
|
+
const context = { ...caller, op: "upload-multipart", ...abort };
|
|
2740
3091
|
throw abortedBy ? requestAborted(abortedBy, context) : mapMultipartError(err, context);
|
|
2741
3092
|
}
|
|
2742
|
-
/**
|
|
2743
|
-
* Lists objects in the bucket as a single page. Applies the global key prefix
|
|
2744
|
-
* to the caller's `prefix`, caps `maxKeys` at the S3 hard limit of 1000, threads
|
|
2745
|
-
* the continuation token, and strips the global prefix from every returned key.
|
|
2746
|
-
*
|
|
2747
|
-
* @param options - The listing request.
|
|
2748
|
-
* @returns One page of objects plus `commonPrefixes`, truncation, and the next token.
|
|
2749
|
-
* @throws StorageException mapped from any AWS failure.
|
|
2750
|
-
*/
|
|
2751
3093
|
async list(options) {
|
|
2752
3094
|
this.assertConfigured();
|
|
2753
3095
|
const bucket = this.resolveBucket(options.bucket);
|
|
2754
3096
|
assertIntegerInput(options.maxKeys, "maxKeys", 1);
|
|
2755
3097
|
const maxKeys = Math.min(options.maxKeys ?? MAX_LIST_KEYS, MAX_LIST_KEYS);
|
|
2756
3098
|
const prefix = options.prefix ? this.keyResolver.normalize(options.prefix) : this.keyResolver.getPrefix();
|
|
2757
|
-
const context = { bucket, prefix
|
|
3099
|
+
const context = callDetails("list", { bucket: options.bucket, prefix: options.prefix });
|
|
2758
3100
|
throwIfAborted(options.abortSignal, context);
|
|
2759
3101
|
try {
|
|
2760
3102
|
const response = await this.s3Provider.getClient().send(
|
|
@@ -2772,14 +3114,6 @@ var StorageService = class {
|
|
|
2772
3114
|
throw failureOrAbort(options.abortSignal, context, mapAwsError(err, context));
|
|
2773
3115
|
}
|
|
2774
3116
|
}
|
|
2775
|
-
/**
|
|
2776
|
-
* Server-side copy of an object (no bytes flow through the app). Supports both
|
|
2777
|
-
* same-bucket and cross-bucket copies and preserves the source metadata.
|
|
2778
|
-
*
|
|
2779
|
-
* @param options - Source/destination keys and buckets plus copy overrides.
|
|
2780
|
-
* @returns The new object's ETag.
|
|
2781
|
-
* @throws StorageException mapped from any AWS failure.
|
|
2782
|
-
*/
|
|
2783
3117
|
async copy(options) {
|
|
2784
3118
|
this.assertConfigured();
|
|
2785
3119
|
assertHeaderValues({ cacheControl: options.cacheControl });
|
|
@@ -2803,19 +3137,12 @@ var StorageService = class {
|
|
|
2803
3137
|
);
|
|
2804
3138
|
return { etag: response.CopyObjectResult?.ETag ?? "" };
|
|
2805
3139
|
} catch (err) {
|
|
2806
|
-
throw mapAwsError(
|
|
3140
|
+
throw mapAwsError(
|
|
3141
|
+
err,
|
|
3142
|
+
callDetails("copy", { sourceKey: options.sourceKey, destKey: options.destinationKey })
|
|
3143
|
+
);
|
|
2807
3144
|
}
|
|
2808
3145
|
}
|
|
2809
|
-
/**
|
|
2810
|
-
* Deletes many objects, chunking the input at the S3 hard limit of 1000 keys
|
|
2811
|
-
* per request. Reports both successes and per-key failures; a whole-chunk send
|
|
2812
|
-
* failure marks every key in that chunk as failed. Returned keys are stripped of
|
|
2813
|
-
* the global prefix.
|
|
2814
|
-
*
|
|
2815
|
-
* @param keys - The raw object keys to delete.
|
|
2816
|
-
* @param options - Optional per-call bucket override.
|
|
2817
|
-
* @returns The deleted keys and the per-key failures.
|
|
2818
|
-
*/
|
|
2819
3146
|
async deleteMany(keys, options) {
|
|
2820
3147
|
this.assertConfigured();
|
|
2821
3148
|
if (keys.length === 0) {
|
|
@@ -2842,6 +3169,34 @@ var StorageService = class {
|
|
|
2842
3169
|
return { deleted, failed };
|
|
2843
3170
|
}
|
|
2844
3171
|
};
|
|
3172
|
+
__decorateClass([
|
|
3173
|
+
Observed("upload", {
|
|
3174
|
+
span: "storage.upload",
|
|
3175
|
+
describe: (result) => ({ "bymax.storage.multipart": String(result.multipart) }),
|
|
3176
|
+
bytes: uploadedBytesOf
|
|
3177
|
+
})
|
|
3178
|
+
], StorageService.prototype, "upload", 1);
|
|
3179
|
+
__decorateClass([
|
|
3180
|
+
Observed("head")
|
|
3181
|
+
], StorageService.prototype, "head", 1);
|
|
3182
|
+
__decorateClass([
|
|
3183
|
+
Observed("download")
|
|
3184
|
+
], StorageService.prototype, "download", 1);
|
|
3185
|
+
__decorateClass([
|
|
3186
|
+
Observed("delete")
|
|
3187
|
+
], StorageService.prototype, "delete", 1);
|
|
3188
|
+
__decorateClass([
|
|
3189
|
+
Observed("check_bucket")
|
|
3190
|
+
], StorageService.prototype, "checkBucket", 1);
|
|
3191
|
+
__decorateClass([
|
|
3192
|
+
Observed("list")
|
|
3193
|
+
], StorageService.prototype, "list", 1);
|
|
3194
|
+
__decorateClass([
|
|
3195
|
+
Observed("copy")
|
|
3196
|
+
], StorageService.prototype, "copy", 1);
|
|
3197
|
+
__decorateClass([
|
|
3198
|
+
Observed("delete_many")
|
|
3199
|
+
], StorageService.prototype, "deleteMany", 1);
|
|
2845
3200
|
StorageService = __decorateClass([
|
|
2846
3201
|
Injectable(),
|
|
2847
3202
|
__decorateParam(0, Inject(BYMAX_STORAGE_OPTIONS)),
|
|
@@ -2849,7 +3204,9 @@ StorageService = __decorateClass([
|
|
|
2849
3204
|
__decorateParam(2, Inject(KeyResolverService)),
|
|
2850
3205
|
__decorateParam(3, Inject(BYMAX_STORAGE_IDEMPOTENCY_CACHE)),
|
|
2851
3206
|
__decorateParam(4, Inject(ValidationService)),
|
|
2852
|
-
__decorateParam(5, Inject(FileScannerService))
|
|
3207
|
+
__decorateParam(5, Inject(FileScannerService)),
|
|
3208
|
+
__decorateParam(6, Optional()),
|
|
3209
|
+
__decorateParam(6, Inject(StorageObserver))
|
|
2853
3210
|
], StorageService);
|
|
2854
3211
|
|
|
2855
3212
|
// src/server/utils/ttl-clamp.ts
|
|
@@ -2869,25 +3226,16 @@ function clampTtl(ttlSeconds, defaultTtl, maxTtl) {
|
|
|
2869
3226
|
// src/server/services/signed-url.service.ts
|
|
2870
3227
|
var MAX_MULTIPART_PARTS = 1e4;
|
|
2871
3228
|
var SignedUrlService = class {
|
|
2872
|
-
constructor(options, s3Provider, keyResolver) {
|
|
3229
|
+
constructor(options, s3Provider, keyResolver, observer) {
|
|
2873
3230
|
this.options = options;
|
|
2874
3231
|
this.s3Provider = s3Provider;
|
|
2875
3232
|
this.keyResolver = keyResolver;
|
|
3233
|
+
this.observer = observer;
|
|
2876
3234
|
}
|
|
2877
3235
|
options;
|
|
2878
3236
|
s3Provider;
|
|
2879
3237
|
keyResolver;
|
|
2880
|
-
|
|
2881
|
-
* Issues a presigned GET URL for downloading an object.
|
|
2882
|
-
*
|
|
2883
|
-
* SECURITY: The returned URL is a temporary credential. NEVER log it, store it
|
|
2884
|
-
* in a database, or cache it across users. Issue a fresh URL per request.
|
|
2885
|
-
*
|
|
2886
|
-
* @param options - The download URL options.
|
|
2887
|
-
* @returns The presigned URL and its expiry metadata.
|
|
2888
|
-
* @throws StorageException `STORAGE_NOT_CONFIGURED` when the client is absent.
|
|
2889
|
-
* @throws StorageException `STORAGE_SIGNED_URL_TTL_INVALID` for non-positive TTL.
|
|
2890
|
-
*/
|
|
3238
|
+
observer;
|
|
2891
3239
|
async getDownloadUrl(options) {
|
|
2892
3240
|
this.assertConfigured();
|
|
2893
3241
|
assertHeaderValues({
|
|
@@ -2912,36 +3260,9 @@ var SignedUrlService = class {
|
|
|
2912
3260
|
return { url, expiresAt: new Date(Date.now() + ttl * 1e3), method: "GET", requiredHeaders: {} };
|
|
2913
3261
|
} catch (err) {
|
|
2914
3262
|
if (err instanceof StorageException) throw err;
|
|
2915
|
-
throw mapAwsError(err, { key:
|
|
3263
|
+
throw mapAwsError(err, callDetails("getDownloadUrl", { key: options.key, bucket: options.bucket }));
|
|
2916
3264
|
}
|
|
2917
3265
|
}
|
|
2918
|
-
/**
|
|
2919
|
-
* Issues a presigned PUT URL for uploading an object. The client MUST send
|
|
2920
|
-
* every header listed in `requiredHeaders` verbatim — each one is part of the
|
|
2921
|
-
* URL signature, so an omitted header yields `SignatureDoesNotMatch`. The map
|
|
2922
|
-
* always carries `Content-Type` and additionally lists `x-amz-acl` when an ACL
|
|
2923
|
-
* is signed and an `x-amz-meta-<key>` entry per metadata pair. The `Content-Type`
|
|
2924
|
-
* is signed (`X-Amz-SignedHeaders` lists `content-type`), so a PUT that sends any
|
|
2925
|
-
* other type is refused by the provider and nothing is stored.
|
|
2926
|
-
*
|
|
2927
|
-
* SECURITY: The returned URL is a temporary credential — NEVER log it. A signed
|
|
2928
|
-
* PUT bypasses the server-side MIME/size validation pipeline. Size limits are
|
|
2929
|
-
* NOT enforced at presign time: a SigV4 PUT signature can only pin an exact
|
|
2930
|
-
* `Content-Length`, never a maximum, so binding one would reject valid smaller
|
|
2931
|
-
* uploads. Enforce size after the fact via a post-upload HEAD/size check plus
|
|
2932
|
-
* the post-upload scanner path for defence-in-depth.
|
|
2933
|
-
*
|
|
2934
|
-
* ACL caveat: `publicRead` emits `ACL: public-read`, which returns HTTP 400
|
|
2935
|
-
* `AccessControlListNotSupported` on modern AWS S3 buckets (Object Ownership
|
|
2936
|
-
* = "Bucket owner enforced") and is a no-op on Cloudflare R2. Prefer a bucket
|
|
2937
|
-
* policy, CDN, or signed GET URL for public delivery.
|
|
2938
|
-
*
|
|
2939
|
-
* @param options - The upload URL options.
|
|
2940
|
-
* @returns The presigned URL, its expiry metadata, and required headers.
|
|
2941
|
-
* @throws StorageException `STORAGE_NOT_CONFIGURED` when the client is absent.
|
|
2942
|
-
* @throws StorageException `STORAGE_SIGNED_URL_TTL_INVALID` for non-positive TTL.
|
|
2943
|
-
* @throws StorageException `STORAGE_MIME_NOT_ALLOWED` when `validation.mimeWhitelist` is set and excludes `contentType`.
|
|
2944
|
-
*/
|
|
2945
3266
|
async getUploadUrl(options) {
|
|
2946
3267
|
this.assertConfigured();
|
|
2947
3268
|
assertMimeAllowed(this.options.validation?.mimeWhitelist, options.contentType);
|
|
@@ -2975,7 +3296,7 @@ var SignedUrlService = class {
|
|
|
2975
3296
|
};
|
|
2976
3297
|
} catch (err) {
|
|
2977
3298
|
if (err instanceof StorageException) throw err;
|
|
2978
|
-
throw mapAwsError(err, { key:
|
|
3299
|
+
throw mapAwsError(err, callDetails("getUploadUrl", { key: options.key, bucket: options.bucket }));
|
|
2979
3300
|
}
|
|
2980
3301
|
}
|
|
2981
3302
|
/**
|
|
@@ -2996,21 +3317,6 @@ var SignedUrlService = class {
|
|
|
2996
3317
|
}
|
|
2997
3318
|
return headers;
|
|
2998
3319
|
}
|
|
2999
|
-
/**
|
|
3000
|
-
* Presigns a multipart upload: creates the upload session on S3, then returns
|
|
3001
|
-
* N presigned `UploadPart` URLs (1-indexed) and a presigned `CompleteMultipart`
|
|
3002
|
-
* URL. The client uploads parts directly to S3 and then calls the complete URL.
|
|
3003
|
-
*
|
|
3004
|
-
* SECURITY: All returned URLs are temporary credentials — NEVER log them.
|
|
3005
|
-
*
|
|
3006
|
-
* @param options - The multipart URL options.
|
|
3007
|
-
* @returns `uploadId`, `partUrls`, and `completeUrl`.
|
|
3008
|
-
* @throws StorageException `STORAGE_NOT_CONFIGURED` when the client is absent.
|
|
3009
|
-
* @throws StorageException `STORAGE_INVALID_PART_COUNT` when `parts` is not an integer from 1 to 10 000.
|
|
3010
|
-
* @throws StorageException `STORAGE_SIGNED_URL_TTL_INVALID` for non-positive TTL.
|
|
3011
|
-
* @throws StorageException `STORAGE_PROVIDER_ERROR` when no `UploadId` is returned.
|
|
3012
|
-
* @throws StorageException `STORAGE_MIME_NOT_ALLOWED` when `validation.mimeWhitelist` is set and excludes `contentType`.
|
|
3013
|
-
*/
|
|
3014
3320
|
async getMultipartUploadUrls(options) {
|
|
3015
3321
|
this.assertConfigured();
|
|
3016
3322
|
assertMimeAllowed(this.options.validation?.mimeWhitelist, options.contentType);
|
|
@@ -3040,14 +3346,16 @@ var SignedUrlService = class {
|
|
|
3040
3346
|
if (!uploadId) {
|
|
3041
3347
|
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_PROVIDER_ERROR, void 0, {
|
|
3042
3348
|
reason: "CreateMultipartUpload returned no UploadId",
|
|
3043
|
-
key:
|
|
3044
|
-
bucket
|
|
3349
|
+
...callerFields({ key: options.key, bucket: options.bucket })
|
|
3045
3350
|
});
|
|
3046
3351
|
}
|
|
3047
3352
|
return await this.presignMultipartParts({ finalKey, bucket, uploadId, parts: options.parts, ttl });
|
|
3048
3353
|
} catch (err) {
|
|
3049
3354
|
if (err instanceof StorageException) throw err;
|
|
3050
|
-
throw mapAwsError(
|
|
3355
|
+
throw mapAwsError(
|
|
3356
|
+
err,
|
|
3357
|
+
callDetails("getMultipartUploadUrls", { key: options.key, bucket: options.bucket })
|
|
3358
|
+
);
|
|
3051
3359
|
}
|
|
3052
3360
|
}
|
|
3053
3361
|
/** Presigns all part URLs and the complete URL in parallel. */
|
|
@@ -3078,11 +3386,22 @@ var SignedUrlService = class {
|
|
|
3078
3386
|
return resolveBucket(perCall, this.options.bucket);
|
|
3079
3387
|
}
|
|
3080
3388
|
};
|
|
3389
|
+
__decorateClass([
|
|
3390
|
+
Observed("get_download_url", { span: "storage.download_url" })
|
|
3391
|
+
], SignedUrlService.prototype, "getDownloadUrl", 1);
|
|
3392
|
+
__decorateClass([
|
|
3393
|
+
Observed("get_upload_url")
|
|
3394
|
+
], SignedUrlService.prototype, "getUploadUrl", 1);
|
|
3395
|
+
__decorateClass([
|
|
3396
|
+
Observed("get_multipart_upload_urls")
|
|
3397
|
+
], SignedUrlService.prototype, "getMultipartUploadUrls", 1);
|
|
3081
3398
|
SignedUrlService = __decorateClass([
|
|
3082
3399
|
Injectable(),
|
|
3083
3400
|
__decorateParam(0, Inject(BYMAX_STORAGE_OPTIONS)),
|
|
3084
3401
|
__decorateParam(1, Inject(S3ClientProvider)),
|
|
3085
|
-
__decorateParam(2, Inject(KeyResolverService))
|
|
3402
|
+
__decorateParam(2, Inject(KeyResolverService)),
|
|
3403
|
+
__decorateParam(3, Optional()),
|
|
3404
|
+
__decorateParam(3, Inject(StorageObserver))
|
|
3086
3405
|
], SignedUrlService);
|
|
3087
3406
|
var IdempotencyCache = class {
|
|
3088
3407
|
/**
|
|
@@ -3192,9 +3511,12 @@ function resolveAsyncOptions(raw) {
|
|
|
3192
3511
|
const resolved = applyDefaults(options);
|
|
3193
3512
|
const configured = options.signedUrls?.maxTtlSeconds;
|
|
3194
3513
|
if (configured !== void 0 && configured > resolved.signedUrls.maxTtlSeconds) {
|
|
3195
|
-
new Logger(BymaxStorageModule.name)
|
|
3196
|
-
|
|
3197
|
-
|
|
3514
|
+
emit(new Logger(BymaxStorageModule.name), "warn", {
|
|
3515
|
+
event: STORAGE_LOG_EVENTS.ttlClamped,
|
|
3516
|
+
message: "signedUrls.maxTtlSeconds exceeds the SigV4 presigned-URL limit; the limit is used instead",
|
|
3517
|
+
configuredSeconds: configured,
|
|
3518
|
+
appliedSeconds: resolved.signedUrls.maxTtlSeconds
|
|
3519
|
+
});
|
|
3198
3520
|
}
|
|
3199
3521
|
return resolved;
|
|
3200
3522
|
}
|
|
@@ -3292,6 +3614,10 @@ var BymaxStorageModule = class {
|
|
|
3292
3614
|
DEFAULT_IDEMPOTENCY_CACHE_TTL_MS
|
|
3293
3615
|
)
|
|
3294
3616
|
},
|
|
3617
|
+
// The counts and the span seam the services report to; the optional `observability` subpath reads the tally.
|
|
3618
|
+
StorageMetricsTally,
|
|
3619
|
+
StorageTracing,
|
|
3620
|
+
StorageObserver,
|
|
3295
3621
|
StorageService,
|
|
3296
3622
|
SignedUrlService,
|
|
3297
3623
|
BucketLifecycleService,
|
|
@@ -3318,6 +3644,7 @@ var BymaxStorageModule = class {
|
|
|
3318
3644
|
StorageService,
|
|
3319
3645
|
SignedUrlService,
|
|
3320
3646
|
BucketLifecycleService,
|
|
3647
|
+
StorageMetricsTally,
|
|
3321
3648
|
BYMAX_STORAGE_OPTIONS,
|
|
3322
3649
|
BYMAX_STORAGE_S3_CLIENT,
|
|
3323
3650
|
BYMAX_STORAGE_UPLOAD_VALIDATORS,
|
|
@@ -3480,4 +3807,4 @@ var providerRecipes = {
|
|
|
3480
3807
|
}
|
|
3481
3808
|
};
|
|
3482
3809
|
|
|
3483
|
-
export { BYMAX_STORAGE_FILE_SCANNER, BYMAX_STORAGE_OPTIONS, BYMAX_STORAGE_S3_CLIENT, BYMAX_STORAGE_UPLOAD_VALIDATORS, BucketLifecycleService, BymaxStorageModule, NoOpFileScanner, NoOpUploadValidator, STORAGE_ERROR_CODES, SignedUrlService, StorageException, StorageService, providerRecipes };
|
|
3810
|
+
export { BYMAX_STORAGE_FILE_SCANNER, BYMAX_STORAGE_OPTIONS, BYMAX_STORAGE_S3_CLIENT, BYMAX_STORAGE_UPLOAD_VALIDATORS, BucketLifecycleService, BymaxStorageModule, NoOpFileScanner, NoOpUploadValidator, SCAN_VERDICTS, STORAGE_ERROR_CODES, STORAGE_OPERATIONS, STORAGE_OUTCOMES, SignedUrlService, StorageException, StorageMetricsTally, StorageService, providerRecipes };
|