@bymax-one/nest-storage 1.1.18 → 1.1.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +76 -1
- package/README.md +160 -11
- package/dist/observability/index.cjs +151 -0
- package/dist/observability/index.d.cts +72 -0
- package/dist/observability/index.d.ts +72 -0
- package/dist/observability/index.mjs +145 -0
- package/dist/server/index.cjs +751 -421
- package/dist/server/index.d.cts +137 -11
- package/dist/server/index.d.ts +137 -11
- package/dist/server/index.mjs +751 -424
- package/package.json +32 -5
package/dist/server/index.cjs
CHANGED
|
@@ -7,15 +7,24 @@ var stream = require('stream');
|
|
|
7
7
|
var s3RequestPresigner = require('@aws-sdk/s3-request-presigner');
|
|
8
8
|
var crypto = require('crypto');
|
|
9
9
|
|
|
10
|
+
var __defProp = Object.defineProperty;
|
|
10
11
|
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
12
|
+
var __typeError = (msg) => {
|
|
13
|
+
throw TypeError(msg);
|
|
14
|
+
};
|
|
11
15
|
var __decorateClass = (decorators, target, key, kind) => {
|
|
12
16
|
var result = kind > 1 ? void 0 : kind ? __getOwnPropDesc(target, key) : target;
|
|
13
17
|
for (var i = decorators.length - 1, decorator; i >= 0; i--)
|
|
14
18
|
if (decorator = decorators[i])
|
|
15
|
-
result = (decorator(result)) || result;
|
|
19
|
+
result = (kind ? decorator(target, key, result) : decorator(result)) || result;
|
|
20
|
+
if (kind && result) __defProp(target, key, result);
|
|
16
21
|
return result;
|
|
17
22
|
};
|
|
18
23
|
var __decorateParam = (index, decorator) => (target, key) => decorator(target, key, index);
|
|
24
|
+
var __accessCheck = (obj, member, msg) => member.has(obj) || __typeError("Cannot " + msg);
|
|
25
|
+
var __privateGet = (obj, member, getter) => (__accessCheck(obj, member, "read from private field"), getter ? getter.call(obj) : member.get(obj));
|
|
26
|
+
var __privateAdd = (obj, member, value) => member.has(obj) ? __typeError("Cannot add the same private member more than once") : member instanceof WeakSet ? member.add(obj) : member.set(obj, value);
|
|
27
|
+
var __privateSet = (obj, member, value, setter) => (__accessCheck(obj, member, "write to private field"), member.set(obj, value), value);
|
|
19
28
|
|
|
20
29
|
// src/server/bymax-storage.constants.ts
|
|
21
30
|
var BYMAX_STORAGE_OPTIONS = /* @__PURE__ */ Symbol("BYMAX_STORAGE_OPTIONS");
|
|
@@ -144,9 +153,9 @@ var HEADER_VALUE = /^[\t \x21-\x7E]*$/;
|
|
|
144
153
|
var HEADER_NAME = /^[!#$%&'*+.^_`|~0-9a-z-]+$/;
|
|
145
154
|
var VALUE_REASON = "must be a string of tab, space and printable ASCII characters";
|
|
146
155
|
var KEY_REASON = "every key must be a non-empty header name of letters, digits and the characters !#$%&'*+-.^_`|~";
|
|
147
|
-
function refuse(
|
|
156
|
+
function refuse(field, reason) {
|
|
148
157
|
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_VALIDATION_FAILED, void 0, {
|
|
149
|
-
field
|
|
158
|
+
field,
|
|
150
159
|
reason
|
|
151
160
|
});
|
|
152
161
|
}
|
|
@@ -154,9 +163,9 @@ function isHeaderValue(value) {
|
|
|
154
163
|
return typeof value === "string" && HEADER_VALUE.test(value);
|
|
155
164
|
}
|
|
156
165
|
function assertHeaderValues(fields) {
|
|
157
|
-
for (const [
|
|
166
|
+
for (const [field, value] of Object.entries(fields)) {
|
|
158
167
|
if (value !== void 0 && !isHeaderValue(value)) {
|
|
159
|
-
refuse(
|
|
168
|
+
refuse(field, VALUE_REASON);
|
|
160
169
|
}
|
|
161
170
|
}
|
|
162
171
|
}
|
|
@@ -253,7 +262,7 @@ function assertValidationKinds(validation) {
|
|
|
253
262
|
if (mimeWhitelist === null) {
|
|
254
263
|
throw invalid("validation.mimeWhitelist must not be null; omit it to allow every type");
|
|
255
264
|
}
|
|
256
|
-
if (mimeWhitelist !== void 0 &&
|
|
265
|
+
if (mimeWhitelist !== void 0 && !isStringArray(mimeWhitelist)) {
|
|
257
266
|
throw invalid("validation.mimeWhitelist must be an array of strings");
|
|
258
267
|
}
|
|
259
268
|
if (customValidators === null) {
|
|
@@ -262,6 +271,31 @@ function assertValidationKinds(validation) {
|
|
|
262
271
|
if (customValidators !== void 0 && !Array.isArray(customValidators)) {
|
|
263
272
|
throw invalid("validation.customValidators must be an array");
|
|
264
273
|
}
|
|
274
|
+
if (customValidators !== void 0 && !isValidatorList(customValidators)) {
|
|
275
|
+
throw invalid("validation.customValidators must hold objects with a validate function");
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
function isStringArray(value) {
|
|
279
|
+
try {
|
|
280
|
+
return Array.isArray(value) && Array.from(value).every((entry) => typeof entry === "string");
|
|
281
|
+
} catch {
|
|
282
|
+
return false;
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
function isValidatorList(entries) {
|
|
286
|
+
try {
|
|
287
|
+
return Array.from(entries).every((entry) => isObject(entry) && typeof entry.validate === "function");
|
|
288
|
+
} catch {
|
|
289
|
+
return false;
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
function assertGroupKinds(options) {
|
|
293
|
+
for (const group of ["signedUrls", "multipart", "validation"]) {
|
|
294
|
+
const value = options[group];
|
|
295
|
+
if (value !== void 0 && value !== null && (!isObject(value) || Array.isArray(value))) {
|
|
296
|
+
throw invalid(`options.${group} must be an object`);
|
|
297
|
+
}
|
|
298
|
+
}
|
|
265
299
|
}
|
|
266
300
|
function assertNumericSections(options) {
|
|
267
301
|
const MAX = Number.MAX_SAFE_INTEGER;
|
|
@@ -279,6 +313,7 @@ function assertNumericSections(options) {
|
|
|
279
313
|
assertBoundedInteger(options.multipart, "multipart", "queueSize", { min: 1, max: MAX_QUEUE_SIZE });
|
|
280
314
|
}
|
|
281
315
|
function assertLimitsAndEncryption(options) {
|
|
316
|
+
assertGroupKinds(options);
|
|
282
317
|
assertNumericSections(options);
|
|
283
318
|
assertValidationKinds(options.validation);
|
|
284
319
|
assertScanner(options.scanner);
|
|
@@ -328,6 +363,9 @@ function validateOptions(options) {
|
|
|
328
363
|
if (!isObject(options.credentials)) {
|
|
329
364
|
throw invalid("options.credentials is required");
|
|
330
365
|
}
|
|
366
|
+
if (Array.isArray(options.credentials)) {
|
|
367
|
+
throw invalid("options.credentials must be an object");
|
|
368
|
+
}
|
|
331
369
|
assertLimitsAndEncryption(options);
|
|
332
370
|
}
|
|
333
371
|
|
|
@@ -474,7 +512,7 @@ function pick(source, keys) {
|
|
|
474
512
|
return copy;
|
|
475
513
|
}
|
|
476
514
|
function pickGroup(value, keys) {
|
|
477
|
-
return isObject2(value) ? pick(value, keys) : value;
|
|
515
|
+
return isObject2(value) && !Array.isArray(value) ? pick(value, keys) : value;
|
|
478
516
|
}
|
|
479
517
|
function copyArray(value) {
|
|
480
518
|
if (!Array.isArray(value)) return value;
|
|
@@ -506,16 +544,36 @@ function snapshotOptions(options) {
|
|
|
506
544
|
return copy;
|
|
507
545
|
}
|
|
508
546
|
|
|
509
|
-
// src/server/utils/
|
|
510
|
-
function
|
|
547
|
+
// src/server/utils/absorb.ts
|
|
548
|
+
function absorb(call) {
|
|
511
549
|
try {
|
|
512
|
-
|
|
513
|
-
return `${url.protocol}//${url.host}`;
|
|
550
|
+
void Promise.resolve(call()).then(void 0, () => void 0);
|
|
514
551
|
} catch {
|
|
515
|
-
return "<unparseable endpoint>";
|
|
516
552
|
}
|
|
517
553
|
}
|
|
518
554
|
|
|
555
|
+
// src/server/utils/log-event.ts
|
|
556
|
+
function emit(sink, level, entry) {
|
|
557
|
+
absorb(() => sink[level](entry));
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
// src/server/utils/log-events.ts
|
|
561
|
+
var STORAGE_LOG_EVENTS = {
|
|
562
|
+
ttlClamped: "STORAGE_TTL_CLAMPED",
|
|
563
|
+
credentialsMissing: "STORAGE_CREDENTIALS_MISSING",
|
|
564
|
+
clientReady: "STORAGE_CLIENT_READY",
|
|
565
|
+
providerCallFailed: "STORAGE_PROVIDER_CALL_FAILED",
|
|
566
|
+
multipartAbortFailed: "STORAGE_MULTIPART_ABORT_FAILED",
|
|
567
|
+
scanInfected: "STORAGE_SCAN_INFECTED",
|
|
568
|
+
scanInconclusiveAccepted: "STORAGE_SCAN_INCONCLUSIVE_ACCEPTED",
|
|
569
|
+
scanInconclusiveRejected: "STORAGE_SCAN_INCONCLUSIVE_REJECTED",
|
|
570
|
+
scanFailed: "STORAGE_SCAN_FAILED",
|
|
571
|
+
scanTimeout: "STORAGE_SCAN_TIMEOUT",
|
|
572
|
+
scanCleanupFailed: "STORAGE_SCAN_CLEANUP_FAILED",
|
|
573
|
+
existsFailed: "STORAGE_EXISTS_FAILED",
|
|
574
|
+
deleteNotFound: "STORAGE_DELETE_NOT_FOUND"
|
|
575
|
+
};
|
|
576
|
+
|
|
519
577
|
// src/server/providers/s3-client.provider.ts
|
|
520
578
|
var S3ClientProvider = class {
|
|
521
579
|
constructor(options) {
|
|
@@ -526,9 +584,10 @@ var S3ClientProvider = class {
|
|
|
526
584
|
client;
|
|
527
585
|
onModuleInit() {
|
|
528
586
|
if (!this.options.hasCredentials) {
|
|
529
|
-
this.logger
|
|
530
|
-
|
|
531
|
-
|
|
587
|
+
emit(this.logger, "warn", {
|
|
588
|
+
event: STORAGE_LOG_EVENTS.credentialsMissing,
|
|
589
|
+
message: "Storage credentials are missing \u2014 module is registered but operations will throw STORAGE_NOT_CONFIGURED."
|
|
590
|
+
});
|
|
532
591
|
return;
|
|
533
592
|
}
|
|
534
593
|
this.resolveClient();
|
|
@@ -601,9 +660,10 @@ var S3ClientProvider = class {
|
|
|
601
660
|
requestChecksumCalculation: this.options.requestChecksumCalculation,
|
|
602
661
|
responseChecksumValidation: this.options.responseChecksumValidation
|
|
603
662
|
};
|
|
604
|
-
this.logger
|
|
605
|
-
|
|
606
|
-
|
|
663
|
+
emit(this.logger, "log", {
|
|
664
|
+
event: STORAGE_LOG_EVENTS.clientReady,
|
|
665
|
+
message: "S3Client initialized"
|
|
666
|
+
});
|
|
607
667
|
return new clientS3.S3Client(config);
|
|
608
668
|
}
|
|
609
669
|
};
|
|
@@ -672,11 +732,256 @@ KeyResolverService = __decorateClass([
|
|
|
672
732
|
__decorateParam(0, common.Inject(BYMAX_STORAGE_OPTIONS))
|
|
673
733
|
], KeyResolverService);
|
|
674
734
|
|
|
735
|
+
// src/server/observability/storage-operations.ts
|
|
736
|
+
var STORAGE_OPERATIONS = [
|
|
737
|
+
"upload",
|
|
738
|
+
"download",
|
|
739
|
+
"head",
|
|
740
|
+
"delete",
|
|
741
|
+
"delete_many",
|
|
742
|
+
"list",
|
|
743
|
+
"copy",
|
|
744
|
+
"get_download_url",
|
|
745
|
+
"get_upload_url",
|
|
746
|
+
"get_multipart_upload_urls",
|
|
747
|
+
"check_bucket",
|
|
748
|
+
"lifecycle_get",
|
|
749
|
+
"lifecycle_upsert",
|
|
750
|
+
"lifecycle_remove"
|
|
751
|
+
];
|
|
752
|
+
var STORAGE_OUTCOMES = ["success", "not_found", "aborted", "error"];
|
|
753
|
+
var SCAN_VERDICTS = ["clean", "infected", "unknown", "timeout", "error"];
|
|
754
|
+
function outcomeOf(error) {
|
|
755
|
+
if (!(error instanceof StorageException)) return "error";
|
|
756
|
+
if (error.code === STORAGE_ERROR_CODES.STORAGE_OBJECT_NOT_FOUND) return "not_found";
|
|
757
|
+
if (error.code === STORAGE_ERROR_CODES.STORAGE_REQUEST_ABORTED || error.code === STORAGE_ERROR_CODES.STORAGE_MULTIPART_ABORTED) {
|
|
758
|
+
return "aborted";
|
|
759
|
+
}
|
|
760
|
+
return "error";
|
|
761
|
+
}
|
|
762
|
+
function scanVerdictOfError(error) {
|
|
763
|
+
if (!(error instanceof StorageException)) return "error";
|
|
764
|
+
if (error.code === STORAGE_ERROR_CODES.STORAGE_SCAN_INFECTED) return "infected";
|
|
765
|
+
if (error.code === STORAGE_ERROR_CODES.STORAGE_SCAN_INCONCLUSIVE) return "unknown";
|
|
766
|
+
if (error.code === STORAGE_ERROR_CODES.STORAGE_TIMEOUT) return "timeout";
|
|
767
|
+
return "error";
|
|
768
|
+
}
|
|
769
|
+
|
|
770
|
+
// src/server/observability/storage-metrics-tally.ts
|
|
771
|
+
function zeroed(keys) {
|
|
772
|
+
return Object.fromEntries(keys.map((key) => [key, 0]));
|
|
773
|
+
}
|
|
774
|
+
var _operations, _scans, _listeners, _bytesUploaded;
|
|
775
|
+
exports.StorageMetricsTally = class StorageMetricsTally {
|
|
776
|
+
constructor() {
|
|
777
|
+
__privateAdd(this, _operations, Object.fromEntries(
|
|
778
|
+
STORAGE_OPERATIONS.map((operation) => [operation, zeroed(STORAGE_OUTCOMES)])
|
|
779
|
+
));
|
|
780
|
+
__privateAdd(this, _scans, zeroed(SCAN_VERDICTS));
|
|
781
|
+
__privateAdd(this, _listeners, []);
|
|
782
|
+
__privateAdd(this, _bytesUploaded, 0);
|
|
783
|
+
}
|
|
784
|
+
/** An operation ended as `outcome` after `seconds`. */
|
|
785
|
+
operation(operation, outcome, seconds) {
|
|
786
|
+
__privateGet(this, _operations)[operation][outcome] += 1;
|
|
787
|
+
for (const listener of __privateGet(this, _listeners)) absorb(() => listener(operation, seconds));
|
|
788
|
+
}
|
|
789
|
+
/** `bytes` of an object reached the provider; anything that is not a positive finite number is ignored. */
|
|
790
|
+
bytesUploaded(bytes) {
|
|
791
|
+
if (typeof bytes === "number" && Number.isFinite(bytes) && bytes > 0)
|
|
792
|
+
__privateSet(this, _bytesUploaded, __privateGet(this, _bytesUploaded) + bytes);
|
|
793
|
+
}
|
|
794
|
+
/** A scan ended as `verdict`. */
|
|
795
|
+
scan(verdict) {
|
|
796
|
+
__privateGet(this, _scans)[verdict] += 1;
|
|
797
|
+
}
|
|
798
|
+
/** Calls `listener` with the duration of every operation that ends from now on. */
|
|
799
|
+
onDuration(listener) {
|
|
800
|
+
__privateGet(this, _listeners).push(listener);
|
|
801
|
+
}
|
|
802
|
+
/** The counts so far, as copies. */
|
|
803
|
+
snapshot() {
|
|
804
|
+
return {
|
|
805
|
+
operations: Object.fromEntries(
|
|
806
|
+
STORAGE_OPERATIONS.map((operation) => [operation, { ...__privateGet(this, _operations)[operation] }])
|
|
807
|
+
),
|
|
808
|
+
bytesUploaded: __privateGet(this, _bytesUploaded),
|
|
809
|
+
scans: { ...__privateGet(this, _scans) }
|
|
810
|
+
};
|
|
811
|
+
}
|
|
812
|
+
};
|
|
813
|
+
_operations = new WeakMap();
|
|
814
|
+
_scans = new WeakMap();
|
|
815
|
+
_listeners = new WeakMap();
|
|
816
|
+
_bytesUploaded = new WeakMap();
|
|
817
|
+
exports.StorageMetricsTally = __decorateClass([
|
|
818
|
+
common.Injectable()
|
|
819
|
+
], exports.StorageMetricsTally);
|
|
820
|
+
var STATUS_ERROR = 2;
|
|
821
|
+
var TRACER_NAME = "@bymax-one/nest-storage";
|
|
822
|
+
var LIBRARY_CODE = /^STORAGE_[A-Z_]+$/;
|
|
823
|
+
function codeOf(error) {
|
|
824
|
+
if (!(error instanceof StorageException)) return "STORAGE_ERROR";
|
|
825
|
+
return LIBRARY_CODE.test(error.code) ? error.code : "STORAGE_ERROR";
|
|
826
|
+
}
|
|
827
|
+
var _api;
|
|
828
|
+
var StorageTracing = class {
|
|
829
|
+
constructor() {
|
|
830
|
+
__privateAdd(this, _api);
|
|
831
|
+
}
|
|
832
|
+
/** Loads the API once, at start-up; a package that is not there leaves the seam inert. */
|
|
833
|
+
async onModuleInit() {
|
|
834
|
+
try {
|
|
835
|
+
const loaded = await import('@opentelemetry/api');
|
|
836
|
+
__privateSet(this, _api, loaded);
|
|
837
|
+
} catch {
|
|
838
|
+
}
|
|
839
|
+
}
|
|
840
|
+
/** Runs the operation and reports it on the span, where nothing the span does can change what comes back. */
|
|
841
|
+
async traced(span, run, describe) {
|
|
842
|
+
let result;
|
|
843
|
+
try {
|
|
844
|
+
result = await run();
|
|
845
|
+
} catch (error) {
|
|
846
|
+
absorb(() => span.setAttributes({ "bymax.storage.outcome": outcomeOf(error) }));
|
|
847
|
+
absorb(() => span.setStatus({ code: STATUS_ERROR, message: codeOf(error) }));
|
|
848
|
+
absorb(() => span.end());
|
|
849
|
+
throw error;
|
|
850
|
+
}
|
|
851
|
+
absorb(() => span.setAttributes({ "bymax.storage.outcome": "success", ...describe(result) }));
|
|
852
|
+
absorb(() => span.end());
|
|
853
|
+
return result;
|
|
854
|
+
}
|
|
855
|
+
/**
|
|
856
|
+
* Runs `run` inside a span named `name`.
|
|
857
|
+
*
|
|
858
|
+
* The tracer belongs to the host's SDK, so it is never trusted with the operation: whatever it returns or
|
|
859
|
+
* throws is ignored, the operation runs exactly once, through the span when the tracer started it and bare
|
|
860
|
+
* otherwise, and its own result or error is what comes back.
|
|
861
|
+
*
|
|
862
|
+
* @param name - One of the library's span names.
|
|
863
|
+
* @param attributes - Closed-set attributes of the span.
|
|
864
|
+
* @param run - The operation; its result or its error passes through unchanged.
|
|
865
|
+
* @param describe - What the span states of a successful result; a throw of its own is the operation's own.
|
|
866
|
+
*/
|
|
867
|
+
span(name, attributes, run, describe = () => ({})) {
|
|
868
|
+
const api = __privateGet(this, _api);
|
|
869
|
+
if (!api) return run();
|
|
870
|
+
const latch = { claimed: false };
|
|
871
|
+
absorb(
|
|
872
|
+
() => api.trace.getTracer(TRACER_NAME).startActiveSpan(name, { attributes }, (span) => {
|
|
873
|
+
if (!latch.claimed) {
|
|
874
|
+
latch.claimed = true;
|
|
875
|
+
latch.started = this.traced(span, run, describe);
|
|
876
|
+
}
|
|
877
|
+
return latch.started;
|
|
878
|
+
})
|
|
879
|
+
);
|
|
880
|
+
if (latch.started) return latch.started;
|
|
881
|
+
latch.claimed = true;
|
|
882
|
+
return run();
|
|
883
|
+
}
|
|
884
|
+
};
|
|
885
|
+
_api = new WeakMap();
|
|
886
|
+
StorageTracing = __decorateClass([
|
|
887
|
+
common.Injectable()
|
|
888
|
+
], StorageTracing);
|
|
889
|
+
|
|
890
|
+
// src/server/observability/storage-observer.ts
|
|
891
|
+
function secondsSince(startedAt) {
|
|
892
|
+
return Number(process.hrtime.bigint() - startedAt) / 1e9;
|
|
893
|
+
}
|
|
894
|
+
var StorageObserver = class {
|
|
895
|
+
constructor(tally, tracing) {
|
|
896
|
+
this.tally = tally;
|
|
897
|
+
this.tracing = tracing;
|
|
898
|
+
}
|
|
899
|
+
tally;
|
|
900
|
+
tracing;
|
|
901
|
+
/**
|
|
902
|
+
* Runs `run`, counting how it ended and how long it took.
|
|
903
|
+
*
|
|
904
|
+
* @param operation - The operation, one of the closed set.
|
|
905
|
+
* @param run - The operation; its result or its error passes through unchanged.
|
|
906
|
+
* @param options - A span to start, and what to state of the result.
|
|
907
|
+
*/
|
|
908
|
+
observe(operation, run, options = {}) {
|
|
909
|
+
const counted = async () => {
|
|
910
|
+
const startedAt = process.hrtime.bigint();
|
|
911
|
+
let result;
|
|
912
|
+
try {
|
|
913
|
+
result = await run();
|
|
914
|
+
} catch (error) {
|
|
915
|
+
absorb(() => {
|
|
916
|
+
this.tally.operation(operation, outcomeOf(error), secondsSince(startedAt));
|
|
917
|
+
});
|
|
918
|
+
throw error;
|
|
919
|
+
}
|
|
920
|
+
absorb(() => {
|
|
921
|
+
this.tally.operation(operation, "success", secondsSince(startedAt));
|
|
922
|
+
this.tally.bytesUploaded(options.bytes?.(result));
|
|
923
|
+
});
|
|
924
|
+
return result;
|
|
925
|
+
};
|
|
926
|
+
return options.span === void 0 ? counted() : this.tracing.span(options.span, options.attributes ?? {}, counted, options.describe);
|
|
927
|
+
}
|
|
928
|
+
/**
|
|
929
|
+
* Runs a scan, counting its verdict and wrapping it in the `storage.scan` span.
|
|
930
|
+
*
|
|
931
|
+
* @param mode - When the scan runs, `pre-upload` or `post-upload`.
|
|
932
|
+
* @param run - The scan; its result or its error passes through unchanged.
|
|
933
|
+
*/
|
|
934
|
+
scan(mode, run) {
|
|
935
|
+
const counted = async () => {
|
|
936
|
+
try {
|
|
937
|
+
const result = await run();
|
|
938
|
+
absorb(() => {
|
|
939
|
+
this.tally.scan(result.status === "unknown" ? "unknown" : "clean");
|
|
940
|
+
});
|
|
941
|
+
return result;
|
|
942
|
+
} catch (error) {
|
|
943
|
+
absorb(() => {
|
|
944
|
+
this.tally.scan(scanVerdictOfError(error));
|
|
945
|
+
});
|
|
946
|
+
throw error;
|
|
947
|
+
}
|
|
948
|
+
};
|
|
949
|
+
return this.tracing.span("storage.scan", { "bymax.storage.scan_mode": mode }, counted);
|
|
950
|
+
}
|
|
951
|
+
};
|
|
952
|
+
StorageObserver = __decorateClass([
|
|
953
|
+
common.Injectable(),
|
|
954
|
+
__decorateParam(0, common.Inject(exports.StorageMetricsTally)),
|
|
955
|
+
__decorateParam(1, common.Inject(StorageTracing))
|
|
956
|
+
], StorageObserver);
|
|
957
|
+
function Observed(operation, options = {}) {
|
|
958
|
+
return (_target, _key, descriptor) => {
|
|
959
|
+
const original = descriptor.value;
|
|
960
|
+
descriptor.value = function(...args) {
|
|
961
|
+
const run = () => original.apply(this, args);
|
|
962
|
+
return this.observer === void 0 ? run() : this.observer.observe(operation, run, options);
|
|
963
|
+
};
|
|
964
|
+
};
|
|
965
|
+
}
|
|
966
|
+
|
|
967
|
+
// src/server/utils/call-details.ts
|
|
968
|
+
var MAX_CALLER_TEXT_LENGTH = 1024;
|
|
969
|
+
function callerFields(given) {
|
|
970
|
+
const fields = {};
|
|
971
|
+
for (const [field, value] of Object.entries(given)) {
|
|
972
|
+
if (typeof value === "string" && value.length <= MAX_CALLER_TEXT_LENGTH) fields[field] = value;
|
|
973
|
+
}
|
|
974
|
+
return fields;
|
|
975
|
+
}
|
|
976
|
+
function callDetails(op, given, extra = {}) {
|
|
977
|
+
return { ...callerFields(given), op, ...extra };
|
|
978
|
+
}
|
|
979
|
+
|
|
675
980
|
// src/server/utils/call-input.ts
|
|
676
|
-
function wrongKind(
|
|
981
|
+
function wrongKind(field, needs) {
|
|
677
982
|
return new StorageException(STORAGE_ERROR_CODES.STORAGE_VALIDATION_FAILED, void 0, {
|
|
678
|
-
field
|
|
679
|
-
reason: `${
|
|
983
|
+
field,
|
|
984
|
+
reason: `${field} must be ${needs}`
|
|
680
985
|
});
|
|
681
986
|
}
|
|
682
987
|
function resolveBucket(perCall, moduleDefault) {
|
|
@@ -704,64 +1009,6 @@ function hasSignalSurface(signal) {
|
|
|
704
1009
|
return false;
|
|
705
1010
|
}
|
|
706
1011
|
}
|
|
707
|
-
|
|
708
|
-
// src/server/utils/describe-error.ts
|
|
709
|
-
var NON_PRINTABLE_REJECTION = "non-printable rejection value";
|
|
710
|
-
function describeError(value) {
|
|
711
|
-
try {
|
|
712
|
-
if (typeof value === "object" && value !== null) {
|
|
713
|
-
const message = value.message;
|
|
714
|
-
if (typeof message === "string") {
|
|
715
|
-
return message;
|
|
716
|
-
}
|
|
717
|
-
}
|
|
718
|
-
return String(value);
|
|
719
|
-
} catch {
|
|
720
|
-
return NON_PRINTABLE_REJECTION;
|
|
721
|
-
}
|
|
722
|
-
}
|
|
723
|
-
|
|
724
|
-
// src/server/utils/log-value.ts
|
|
725
|
-
var LEFT_LITERAL_BY_JSON = /[\p{Cc}\p{Cf}\p{Zl}\p{Zp}]/gu;
|
|
726
|
-
function logValue(value) {
|
|
727
|
-
const text = typeof value === "string" ? value : describeError(value);
|
|
728
|
-
return JSON.stringify(text).replace(
|
|
729
|
-
LEFT_LITERAL_BY_JSON,
|
|
730
|
-
(char) => {
|
|
731
|
-
const codePoint = Number(char.codePointAt(0));
|
|
732
|
-
const hex = codePoint.toString(16);
|
|
733
|
-
return codePoint > 65535 ? `\\u{${hex}}` : `\\u${hex.padStart(4, "0")}`;
|
|
734
|
-
}
|
|
735
|
-
);
|
|
736
|
-
}
|
|
737
|
-
|
|
738
|
-
// src/server/utils/provider-failure-log.ts
|
|
739
|
-
var SECRET_IN_MESSAGE = /(X-Amz-(?:Signature|Credential|Security-Token)|AWSAccessKeyId|Credential|Signature)=[^&\s"']+/gi;
|
|
740
|
-
var BARE_ACCESS_KEY_ID = /\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/g;
|
|
741
|
-
var MAX_MESSAGE_LENGTH = 300;
|
|
742
|
-
var logger = new common.Logger("StorageProvider");
|
|
743
|
-
function field(value) {
|
|
744
|
-
return typeof value === "string" || typeof value === "number" ? logValue(value) : "-";
|
|
745
|
-
}
|
|
746
|
-
function describeProviderFailure(err) {
|
|
747
|
-
const failure = typeof err === "object" && err !== null ? err : {};
|
|
748
|
-
const message = describeError(err).replace(SECRET_IN_MESSAGE, "$1=[redacted]").replace(BARE_ACCESS_KEY_ID, "[redacted-key-id]");
|
|
749
|
-
return [
|
|
750
|
-
`name=${field(failure.name)}`,
|
|
751
|
-
`code=${field(failure.Code)}`,
|
|
752
|
-
`status=${field(failure.$metadata?.httpStatusCode)}`,
|
|
753
|
-
`requestId=${field(failure.$metadata?.requestId)}`,
|
|
754
|
-
`message=${logValue(message.slice(0, MAX_MESSAGE_LENGTH))}`
|
|
755
|
-
].join(" ");
|
|
756
|
-
}
|
|
757
|
-
function logProviderFailure(err, context, label = "Provider call failed") {
|
|
758
|
-
if (err?.name === "AbortError") {
|
|
759
|
-
return;
|
|
760
|
-
}
|
|
761
|
-
const op = typeof context?.op === "string" ? logValue(context.op) : "-";
|
|
762
|
-
const key = typeof context?.key === "string" ? logValue(context.key) : "-";
|
|
763
|
-
logger.warn(`${label}: op=${op} key=${key} ${describeProviderFailure(err)}`);
|
|
764
|
-
}
|
|
765
1012
|
var UNREACHABLE_CODES = /* @__PURE__ */ new Set([
|
|
766
1013
|
"ECONNREFUSED",
|
|
767
1014
|
"ECONNRESET",
|
|
@@ -828,6 +1075,106 @@ function providerFailureStatus(err) {
|
|
|
828
1075
|
return failure === "credentials-refused" ? common.HttpStatus.INTERNAL_SERVER_ERROR : void 0;
|
|
829
1076
|
}
|
|
830
1077
|
|
|
1078
|
+
// src/server/utils/cut-text.ts
|
|
1079
|
+
function cutText(text, max) {
|
|
1080
|
+
if (text.length <= max) return text;
|
|
1081
|
+
const lastKept = text.charCodeAt(max - 1);
|
|
1082
|
+
const splitsPair = lastKept >= 55296 && lastKept <= 56319;
|
|
1083
|
+
return text.slice(0, splitsPair ? max - 1 : max);
|
|
1084
|
+
}
|
|
1085
|
+
|
|
1086
|
+
// src/server/utils/describe-error.ts
|
|
1087
|
+
var NON_PRINTABLE_REJECTION = "non-printable rejection value";
|
|
1088
|
+
function describeError(value) {
|
|
1089
|
+
try {
|
|
1090
|
+
if (typeof value === "object" && value !== null) {
|
|
1091
|
+
const message = value.message;
|
|
1092
|
+
if (typeof message === "string") {
|
|
1093
|
+
return message;
|
|
1094
|
+
}
|
|
1095
|
+
}
|
|
1096
|
+
return String(value);
|
|
1097
|
+
} catch {
|
|
1098
|
+
return NON_PRINTABLE_REJECTION;
|
|
1099
|
+
}
|
|
1100
|
+
}
|
|
1101
|
+
|
|
1102
|
+
// src/server/utils/log-value.ts
|
|
1103
|
+
var LEFT_LITERAL_BY_JSON = /[\p{Cc}\p{Cf}\p{Zl}\p{Zp}]/gu;
|
|
1104
|
+
var SHORT_ESCAPES = {
|
|
1105
|
+
"\b": "\\b",
|
|
1106
|
+
"\f": "\\f",
|
|
1107
|
+
"\n": "\\n",
|
|
1108
|
+
"\r": "\\r",
|
|
1109
|
+
" ": "\\t"
|
|
1110
|
+
};
|
|
1111
|
+
function escapeChar(char) {
|
|
1112
|
+
const short = SHORT_ESCAPES[char];
|
|
1113
|
+
if (short !== void 0) return short;
|
|
1114
|
+
const codePoint = Number(char.codePointAt(0));
|
|
1115
|
+
const hex = codePoint.toString(16);
|
|
1116
|
+
return codePoint > 65535 ? `\\u{${hex}}` : `\\u${hex.padStart(4, "0")}`;
|
|
1117
|
+
}
|
|
1118
|
+
function escapeForLog(text) {
|
|
1119
|
+
return text.replace(LEFT_LITERAL_BY_JSON, escapeChar);
|
|
1120
|
+
}
|
|
1121
|
+
|
|
1122
|
+
// src/server/utils/provider-failure-log.ts
|
|
1123
|
+
var SECRET_IN_MESSAGE = /(X-Amz-(?:Signature|Credential|Security-Token)|AWSAccessKeyId|Credential|Signature)=[^&\s"']+/gi;
|
|
1124
|
+
var BARE_ACCESS_KEY_ID = /\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/g;
|
|
1125
|
+
var MAX_MESSAGE_LENGTH = 300;
|
|
1126
|
+
var logger = new common.Logger("StorageProvider");
|
|
1127
|
+
var MAX_FIELD_LENGTH = 128;
|
|
1128
|
+
function readOrUndefined(read) {
|
|
1129
|
+
try {
|
|
1130
|
+
return read();
|
|
1131
|
+
} catch {
|
|
1132
|
+
return void 0;
|
|
1133
|
+
}
|
|
1134
|
+
}
|
|
1135
|
+
function textField(value) {
|
|
1136
|
+
return typeof value === "string" ? escapeForLog(cutText(value, MAX_FIELD_LENGTH)) : void 0;
|
|
1137
|
+
}
|
|
1138
|
+
function projectProviderFailure(err) {
|
|
1139
|
+
const failure = typeof err === "object" && err !== null ? err : {};
|
|
1140
|
+
const message = describeError(err).replace(SECRET_IN_MESSAGE, "$1=[redacted]").replace(BARE_ACCESS_KEY_ID, "[redacted-key-id]");
|
|
1141
|
+
const status = readOrUndefined(() => failure.$metadata?.httpStatusCode);
|
|
1142
|
+
const name = textField(readOrUndefined(() => failure.name));
|
|
1143
|
+
const awsCode = textField(readOrUndefined(() => failure.Code));
|
|
1144
|
+
const requestId = textField(readOrUndefined(() => failure.$metadata?.requestId));
|
|
1145
|
+
return {
|
|
1146
|
+
...name !== void 0 ? { name } : {},
|
|
1147
|
+
...awsCode !== void 0 ? { awsCode } : {},
|
|
1148
|
+
...typeof status === "number" ? { httpStatus: status } : {},
|
|
1149
|
+
...requestId !== void 0 ? { requestId } : {},
|
|
1150
|
+
message: escapeForLog(cutText(message, MAX_MESSAGE_LENGTH))
|
|
1151
|
+
};
|
|
1152
|
+
}
|
|
1153
|
+
function failureError(err) {
|
|
1154
|
+
const view = projectProviderFailure(err);
|
|
1155
|
+
const error = new Error(view.message);
|
|
1156
|
+
if (view.name !== void 0) error.name = view.name;
|
|
1157
|
+
if (view.awsCode !== void 0)
|
|
1158
|
+
Object.defineProperty(error, "awsCode", { value: view.awsCode, enumerable: true });
|
|
1159
|
+
return error;
|
|
1160
|
+
}
|
|
1161
|
+
function logProviderFailure(err, context, label = "Provider call failed") {
|
|
1162
|
+
if (readOrUndefined(() => err?.name) === "AbortError") {
|
|
1163
|
+
return;
|
|
1164
|
+
}
|
|
1165
|
+
const view = projectProviderFailure(err);
|
|
1166
|
+
emit(logger, "warn", {
|
|
1167
|
+
event: STORAGE_LOG_EVENTS.providerCallFailed,
|
|
1168
|
+
message: label,
|
|
1169
|
+
...typeof context?.op === "string" ? { operation: escapeForLog(context.op) } : {},
|
|
1170
|
+
...view.awsCode !== void 0 ? { awsCode: view.awsCode } : {},
|
|
1171
|
+
...view.httpStatus !== void 0 ? { httpStatus: view.httpStatus } : {},
|
|
1172
|
+
...view.requestId !== void 0 ? { requestId: view.requestId } : {},
|
|
1173
|
+
reason: readOrUndefined(() => providerFailureReason(err)) ?? "rejected",
|
|
1174
|
+
err: failureError(err)
|
|
1175
|
+
});
|
|
1176
|
+
}
|
|
1177
|
+
|
|
831
1178
|
// src/server/errors/aws-error-mapper.ts
|
|
832
1179
|
function awsCodeOf(err) {
|
|
833
1180
|
const e = err ?? {};
|
|
@@ -947,54 +1294,32 @@ function ruleInvalid(reason) {
|
|
|
947
1294
|
reason
|
|
948
1295
|
});
|
|
949
1296
|
}
|
|
950
|
-
function assertDays(
|
|
1297
|
+
function assertDays(field, value) {
|
|
951
1298
|
if (!Number.isInteger(value) || value < 1) {
|
|
952
|
-
throw ruleInvalid(`${
|
|
1299
|
+
throw ruleInvalid(`${field} must be an integer of at least 1`);
|
|
953
1300
|
}
|
|
954
1301
|
}
|
|
955
1302
|
exports.BucketLifecycleService = class BucketLifecycleService {
|
|
956
|
-
constructor(options, s3Provider, keyResolver) {
|
|
1303
|
+
constructor(options, s3Provider, keyResolver, observer) {
|
|
957
1304
|
this.options = options;
|
|
958
1305
|
this.s3Provider = s3Provider;
|
|
959
1306
|
this.keyResolver = keyResolver;
|
|
1307
|
+
this.observer = observer;
|
|
960
1308
|
}
|
|
961
1309
|
options;
|
|
962
1310
|
s3Provider;
|
|
963
1311
|
keyResolver;
|
|
964
|
-
|
|
965
|
-
* Returns the bucket's lifecycle rules. A bucket with no lifecycle configuration
|
|
966
|
-
* yields an empty array. Key prefixes inside the module's `keyPrefix` come back
|
|
967
|
-
* relative to it, as `list()` returns keys, so a prefix read here can be passed
|
|
968
|
-
* straight to `upsert()` without being prefixed twice. A rule that is not confined
|
|
969
|
-
* to `keyPrefix` is returned exactly as stored, flagged `isOutsideKeyPrefix: true`;
|
|
970
|
-
* everything else about a rule is returned as the provider stores it.
|
|
971
|
-
*
|
|
972
|
-
* @param options - Optional per-call bucket override.
|
|
973
|
-
* @returns Every rule in the bucket.
|
|
974
|
-
* @throws StorageException for missing configuration, an unsupported provider, or a provider error.
|
|
975
|
-
*/
|
|
1312
|
+
observer;
|
|
976
1313
|
async get(options) {
|
|
977
1314
|
this.assertConfigured();
|
|
978
1315
|
const bucket = this.resolveBucket(options?.bucket);
|
|
979
|
-
return (await this.read(bucket)).rules.map((rule) => this.toView(rule));
|
|
1316
|
+
return (await this.read(bucket, options?.bucket)).rules.map((rule) => this.toView(rule));
|
|
980
1317
|
}
|
|
981
|
-
/**
|
|
982
|
-
* Adds the expiration or multipart-abort rule, or replaces the rule with the same id, and writes
|
|
983
|
-
* the full set back. Rules with other ids are sent exactly as they were read.
|
|
984
|
-
*
|
|
985
|
-
* @param rule - The rule id, key prefix and either `expirationDays` or `abortIncompleteMultipartDays`.
|
|
986
|
-
* @param options - Optional per-call bucket override.
|
|
987
|
-
* @throws StorageException `STORAGE_LIFECYCLE_RULE_INVALID` for a bad id, prefix or day count, or an id
|
|
988
|
-
* that names a rule outside `keyPrefix`;
|
|
989
|
-
* `STORAGE_LIFECYCLE_UNSUPPORTED` / `STORAGE_LIFECYCLE_REJECTED` from the provider.
|
|
990
|
-
* @example
|
|
991
|
-
* await lifecycle.upsert({ id: 'expire-tmp', prefix: 'expiring/', expirationDays: 7 })
|
|
992
|
-
*/
|
|
993
1318
|
async upsert(rule, options) {
|
|
994
1319
|
this.assertConfigured();
|
|
995
1320
|
const owned = this.toRule(rule);
|
|
996
1321
|
const bucket = this.resolveBucket(options?.bucket);
|
|
997
|
-
const snapshot = await this.read(bucket);
|
|
1322
|
+
const snapshot = await this.read(bucket, options?.bucket);
|
|
998
1323
|
const index = snapshot.rules.findIndex((existing) => existing.ID === owned.ID);
|
|
999
1324
|
this.assertOwnable(snapshot.rules[index]);
|
|
1000
1325
|
const rules = [...snapshot.rules];
|
|
@@ -1003,30 +1328,20 @@ exports.BucketLifecycleService = class BucketLifecycleService {
|
|
|
1003
1328
|
} else {
|
|
1004
1329
|
rules[index] = owned;
|
|
1005
1330
|
}
|
|
1006
|
-
await this.write(bucket, rules, snapshot, "upsert");
|
|
1331
|
+
await this.write(bucket, options?.bucket, rules, snapshot, "upsert");
|
|
1007
1332
|
}
|
|
1008
|
-
/**
|
|
1009
|
-
* Removes the rule with this id. When it was the last rule the configuration is
|
|
1010
|
-
* deleted, because S3 rejects a write with no rules. An unknown id changes nothing.
|
|
1011
|
-
*
|
|
1012
|
-
* @param id - The rule id.
|
|
1013
|
-
* @param options - Optional per-call bucket override.
|
|
1014
|
-
* @returns `true` when a rule was removed, `false` when no rule had that id.
|
|
1015
|
-
* @throws StorageException `STORAGE_LIFECYCLE_RULE_INVALID` for a bad id or an id that names a rule
|
|
1016
|
-
* outside `keyPrefix`; a provider failure otherwise.
|
|
1017
|
-
*/
|
|
1018
1333
|
async remove(id, options) {
|
|
1019
1334
|
this.assertConfigured();
|
|
1020
1335
|
this.assertRuleId(id);
|
|
1021
1336
|
const bucket = this.resolveBucket(options?.bucket);
|
|
1022
|
-
const snapshot = await this.read(bucket);
|
|
1337
|
+
const snapshot = await this.read(bucket, options?.bucket);
|
|
1023
1338
|
const target = snapshot.rules.find((existing) => existing.ID === id);
|
|
1024
1339
|
if (target === void 0) {
|
|
1025
1340
|
return false;
|
|
1026
1341
|
}
|
|
1027
1342
|
this.assertOwnable(target);
|
|
1028
1343
|
const rules = snapshot.rules.filter((existing) => existing.ID !== id);
|
|
1029
|
-
await this.write(bucket, rules, snapshot, "remove");
|
|
1344
|
+
await this.write(bucket, options?.bucket, rules, snapshot, "remove");
|
|
1030
1345
|
return true;
|
|
1031
1346
|
}
|
|
1032
1347
|
/**
|
|
@@ -1080,7 +1395,7 @@ exports.BucketLifecycleService = class BucketLifecycleService {
|
|
|
1080
1395
|
};
|
|
1081
1396
|
}
|
|
1082
1397
|
/** Reads the configuration; "none yet" is an empty snapshot. */
|
|
1083
|
-
async read(bucket) {
|
|
1398
|
+
async read(bucket, perCall) {
|
|
1084
1399
|
try {
|
|
1085
1400
|
const response = await this.s3Provider.getClient().send(new clientS3.GetBucketLifecycleConfigurationCommand({ Bucket: bucket }));
|
|
1086
1401
|
return {
|
|
@@ -1091,7 +1406,7 @@ exports.BucketLifecycleService = class BucketLifecycleService {
|
|
|
1091
1406
|
if (isNoLifecycleConfiguration(err)) {
|
|
1092
1407
|
return { rules: [], transitionDefaultMinimumObjectSize: void 0 };
|
|
1093
1408
|
}
|
|
1094
|
-
throw mapLifecycleError(err,
|
|
1409
|
+
throw mapLifecycleError(err, callDetails("lifecycle.get", { bucket: perCall }));
|
|
1095
1410
|
}
|
|
1096
1411
|
}
|
|
1097
1412
|
/**
|
|
@@ -1099,7 +1414,7 @@ exports.BucketLifecycleService = class BucketLifecycleService {
|
|
|
1099
1414
|
* requires at least one rule on a put. The default-minimum-object-size setting
|
|
1100
1415
|
* read earlier is sent back so a write does not silently change transition behaviour.
|
|
1101
1416
|
*/
|
|
1102
|
-
async write(bucket, rules, snapshot, op) {
|
|
1417
|
+
async write(bucket, perCall, rules, snapshot, op) {
|
|
1103
1418
|
const client = this.s3Provider.getClient();
|
|
1104
1419
|
try {
|
|
1105
1420
|
if (rules.length === 0) {
|
|
@@ -1114,7 +1429,7 @@ exports.BucketLifecycleService = class BucketLifecycleService {
|
|
|
1114
1429
|
})
|
|
1115
1430
|
);
|
|
1116
1431
|
} catch (err) {
|
|
1117
|
-
throw mapLifecycleError(err,
|
|
1432
|
+
throw mapLifecycleError(err, callDetails(`lifecycle.${op}`, { bucket: perCall }));
|
|
1118
1433
|
}
|
|
1119
1434
|
}
|
|
1120
1435
|
/** Validates the input and builds the rule, an expiration or a multipart-abort rule. */
|
|
@@ -1164,11 +1479,22 @@ exports.BucketLifecycleService = class BucketLifecycleService {
|
|
|
1164
1479
|
return resolveBucket(perCall, this.options.bucket);
|
|
1165
1480
|
}
|
|
1166
1481
|
};
|
|
1482
|
+
__decorateClass([
|
|
1483
|
+
Observed("lifecycle_get")
|
|
1484
|
+
], exports.BucketLifecycleService.prototype, "get", 1);
|
|
1485
|
+
__decorateClass([
|
|
1486
|
+
Observed("lifecycle_upsert")
|
|
1487
|
+
], exports.BucketLifecycleService.prototype, "upsert", 1);
|
|
1488
|
+
__decorateClass([
|
|
1489
|
+
Observed("lifecycle_remove")
|
|
1490
|
+
], exports.BucketLifecycleService.prototype, "remove", 1);
|
|
1167
1491
|
exports.BucketLifecycleService = __decorateClass([
|
|
1168
1492
|
common.Injectable(),
|
|
1169
1493
|
__decorateParam(0, common.Inject(BYMAX_STORAGE_OPTIONS)),
|
|
1170
1494
|
__decorateParam(1, common.Inject(S3ClientProvider)),
|
|
1171
|
-
__decorateParam(2, common.Inject(KeyResolverService))
|
|
1495
|
+
__decorateParam(2, common.Inject(KeyResolverService)),
|
|
1496
|
+
__decorateParam(3, common.Optional()),
|
|
1497
|
+
__decorateParam(3, common.Inject(StorageObserver))
|
|
1172
1498
|
], exports.BucketLifecycleService);
|
|
1173
1499
|
function isBufferLike(body) {
|
|
1174
1500
|
return body instanceof Uint8Array;
|
|
@@ -1322,18 +1648,17 @@ function assertMimeAllowed(whitelist, contentType) {
|
|
|
1322
1648
|
return;
|
|
1323
1649
|
}
|
|
1324
1650
|
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_MIME_NOT_ALLOWED, void 0, {
|
|
1325
|
-
...isHeaderValue(contentType) ? { contentType } : {}
|
|
1326
|
-
whitelist
|
|
1651
|
+
...isHeaderValue(contentType) ? { contentType } : {}
|
|
1327
1652
|
});
|
|
1328
1653
|
}
|
|
1329
1654
|
|
|
1330
1655
|
// src/server/utils/numeric-input.ts
|
|
1331
|
-
function assertIntegerInput(value,
|
|
1656
|
+
function assertIntegerInput(value, field, min) {
|
|
1332
1657
|
if (value === void 0) return;
|
|
1333
1658
|
if (!Number.isInteger(value) || value < min) {
|
|
1334
1659
|
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_VALIDATION_FAILED, void 0, {
|
|
1335
|
-
field
|
|
1336
|
-
reason: `${
|
|
1660
|
+
field,
|
|
1661
|
+
reason: `${field} must be an integer of at least ${String(min)}`
|
|
1337
1662
|
});
|
|
1338
1663
|
}
|
|
1339
1664
|
}
|
|
@@ -1433,46 +1758,41 @@ ValidationService = __decorateClass([
|
|
|
1433
1758
|
__decorateParam(0, common.Inject(BYMAX_STORAGE_OPTIONS)),
|
|
1434
1759
|
__decorateParam(1, common.Inject(BYMAX_STORAGE_UPLOAD_VALIDATORS))
|
|
1435
1760
|
], ValidationService);
|
|
1436
|
-
|
|
1437
|
-
// src/server/utils/cut-text.ts
|
|
1438
|
-
function cutText(text, max) {
|
|
1439
|
-
if (text.length <= max) return text;
|
|
1440
|
-
const lastKept = text.charCodeAt(max - 1);
|
|
1441
|
-
const splitsPair = lastKept >= 55296 && lastKept <= 56319;
|
|
1442
|
-
return text.slice(0, splitsPair ? max - 1 : max);
|
|
1443
|
-
}
|
|
1444
|
-
|
|
1445
|
-
// src/server/services/file-scanner.service.ts
|
|
1446
1761
|
var UNSERIALISABLE = "[unserialisable]";
|
|
1447
1762
|
var MAX_SCANNER_LABEL_LENGTH = 200;
|
|
1448
1763
|
var MAX_SCANNER_DETAILS_LENGTH = 300;
|
|
1449
1764
|
var MAX_STATUS_LOG_LENGTH = 40;
|
|
1450
1765
|
function statusForLog(status) {
|
|
1451
|
-
return typeof status === "string" ?
|
|
1766
|
+
return typeof status === "string" ? escapeForLog(cutText(status, MAX_STATUS_LOG_LENGTH)) : `<${typeof status}>`;
|
|
1452
1767
|
}
|
|
1453
1768
|
function scannerLabel(value) {
|
|
1454
1769
|
return typeof value === "string" ? cutText(value, MAX_SCANNER_LABEL_LENGTH) : void 0;
|
|
1455
1770
|
}
|
|
1771
|
+
function labelForLog(label) {
|
|
1772
|
+
return label === void 0 ? void 0 : escapeForLog(label);
|
|
1773
|
+
}
|
|
1456
1774
|
function toJson(value) {
|
|
1457
1775
|
return JSON.stringify(value);
|
|
1458
1776
|
}
|
|
1459
1777
|
function scannerDetailsForLog(details) {
|
|
1460
|
-
if (details === void 0) return
|
|
1778
|
+
if (details === void 0) return {};
|
|
1461
1779
|
let text;
|
|
1462
1780
|
try {
|
|
1463
1781
|
text = toJson(details) ?? UNSERIALISABLE;
|
|
1464
1782
|
} catch {
|
|
1465
1783
|
text = UNSERIALISABLE;
|
|
1466
1784
|
}
|
|
1467
|
-
return
|
|
1785
|
+
return { details: escapeForLog(cutText(text, MAX_SCANNER_DETAILS_LENGTH)) };
|
|
1468
1786
|
}
|
|
1469
1787
|
var FileScannerService = class {
|
|
1470
|
-
constructor(options, scanner) {
|
|
1788
|
+
constructor(options, scanner, observer) {
|
|
1471
1789
|
this.options = options;
|
|
1472
1790
|
this.scanner = scanner;
|
|
1791
|
+
this.observer = observer;
|
|
1473
1792
|
}
|
|
1474
1793
|
options;
|
|
1475
1794
|
scanner;
|
|
1795
|
+
observer;
|
|
1476
1796
|
logger = new common.Logger(FileScannerService.name);
|
|
1477
1797
|
/**
|
|
1478
1798
|
* Returns `true` when both a scanner implementation and scanner options are
|
|
@@ -1506,19 +1826,27 @@ var FileScannerService = class {
|
|
|
1506
1826
|
* says, and a scan that outlasts `scanner.timeoutMs` throws `STORAGE_TIMEOUT`.
|
|
1507
1827
|
*
|
|
1508
1828
|
* @param input - The scan input.
|
|
1829
|
+
* @param caller - What the deadline error repeats of the call (see `callerFields`): the caller's own key and
|
|
1830
|
+
* bucket, never the prefixed key or the configured bucket that `input` holds for the scanner.
|
|
1509
1831
|
* @returns The scan result when `status` is `'clean'` or `'unknown'` (non-rejecting).
|
|
1510
1832
|
* @throws Error when called without a configured scanner.
|
|
1511
1833
|
* @throws StorageException `STORAGE_SCAN_INFECTED` on an infected result.
|
|
1512
1834
|
* @throws StorageException `STORAGE_SCAN_INCONCLUSIVE` on an unknown result with `rejectOnUnknown`, or a result outside the contract.
|
|
1513
1835
|
* @throws StorageException `STORAGE_TIMEOUT` when the scan outlasts `scanner.timeoutMs`.
|
|
1514
1836
|
*/
|
|
1515
|
-
async scan(input) {
|
|
1837
|
+
async scan(input, caller = {}) {
|
|
1516
1838
|
if (!this.scanner) {
|
|
1517
1839
|
throw new Error(
|
|
1518
1840
|
"FileScannerService.scan() called without a configured scanner \u2014 guard with isEnabled() first."
|
|
1519
1841
|
);
|
|
1520
1842
|
}
|
|
1521
|
-
const
|
|
1843
|
+
const scanner = this.scanner;
|
|
1844
|
+
const run = () => this.evaluate(scanner, input, caller);
|
|
1845
|
+
return this.observer === void 0 ? run() : this.observer.scan(input.mode, run);
|
|
1846
|
+
}
|
|
1847
|
+
/** Runs the scanner and applies the library's policy to what it answered. */
|
|
1848
|
+
async evaluate(scanner, input, caller) {
|
|
1849
|
+
const result = await this.scanLogged(scanner, input, caller);
|
|
1522
1850
|
const status = result?.status;
|
|
1523
1851
|
const engine = scannerLabel(result?.engine);
|
|
1524
1852
|
const threat = scannerLabel(result?.threat);
|
|
@@ -1527,18 +1855,41 @@ var FileScannerService = class {
|
|
|
1527
1855
|
return result;
|
|
1528
1856
|
}
|
|
1529
1857
|
if (status === "infected") {
|
|
1530
|
-
this.logger
|
|
1858
|
+
emit(this.logger, "warn", {
|
|
1859
|
+
event: STORAGE_LOG_EVENTS.scanInfected,
|
|
1860
|
+
message: "Infected file detected",
|
|
1861
|
+
engine: labelForLog(engine),
|
|
1862
|
+
threat: escapeForLog(threat ?? "unknown"),
|
|
1863
|
+
...details
|
|
1864
|
+
});
|
|
1531
1865
|
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_SCAN_INFECTED, void 0, {
|
|
1532
1866
|
engine,
|
|
1533
1867
|
threat
|
|
1534
1868
|
});
|
|
1535
1869
|
}
|
|
1536
1870
|
if (status === "unknown" && !this.options.scanner?.rejectOnUnknown) {
|
|
1537
|
-
this.logger
|
|
1871
|
+
emit(this.logger, "warn", {
|
|
1872
|
+
event: STORAGE_LOG_EVENTS.scanInconclusiveAccepted,
|
|
1873
|
+
message: "Inconclusive scan result accepted",
|
|
1874
|
+
engine: labelForLog(engine)
|
|
1875
|
+
});
|
|
1538
1876
|
return result;
|
|
1539
1877
|
}
|
|
1878
|
+
return this.rejectInconclusive(status, engine, details);
|
|
1879
|
+
}
|
|
1880
|
+
/**
|
|
1881
|
+
* `unknown` with `rejectOnUnknown`, or a result outside the contract (no result, a status that is not
|
|
1882
|
+
* exactly one of the three): the object is not known to be clean, so it is not stored.
|
|
1883
|
+
*/
|
|
1884
|
+
rejectInconclusive(status, engine, details) {
|
|
1540
1885
|
const unrecognised = status !== "unknown";
|
|
1541
|
-
this.logger
|
|
1886
|
+
emit(this.logger, "warn", {
|
|
1887
|
+
event: STORAGE_LOG_EVENTS.scanInconclusiveRejected,
|
|
1888
|
+
message: "Inconclusive scan result rejected",
|
|
1889
|
+
engine: labelForLog(engine),
|
|
1890
|
+
...unrecognised ? { status: statusForLog(status) } : {},
|
|
1891
|
+
...details
|
|
1892
|
+
});
|
|
1542
1893
|
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_SCAN_INCONCLUSIVE, void 0, {
|
|
1543
1894
|
engine,
|
|
1544
1895
|
...unrecognised ? { reason: "unrecognised scanner status" } : {}
|
|
@@ -1548,14 +1899,16 @@ var FileScannerService = class {
|
|
|
1548
1899
|
* Runs the scan and logs a scanner that throws: the caller is answered 500 for that, and without the
|
|
1549
1900
|
* line there would be no trace of why. A typed failure of ours (the deadline) says it itself.
|
|
1550
1901
|
*/
|
|
1551
|
-
async scanLogged(scanner, input) {
|
|
1902
|
+
async scanLogged(scanner, input, caller) {
|
|
1552
1903
|
try {
|
|
1553
|
-
return await this.scanWithinDeadline(scanner, input);
|
|
1904
|
+
return await this.scanWithinDeadline(scanner, input, caller);
|
|
1554
1905
|
} catch (err) {
|
|
1555
1906
|
if (!(err instanceof StorageException)) {
|
|
1556
|
-
this.logger
|
|
1557
|
-
|
|
1558
|
-
|
|
1907
|
+
emit(this.logger, "warn", {
|
|
1908
|
+
event: STORAGE_LOG_EVENTS.scanFailed,
|
|
1909
|
+
message: "Scanner failed",
|
|
1910
|
+
err: failureError(err)
|
|
1911
|
+
});
|
|
1559
1912
|
}
|
|
1560
1913
|
throw err;
|
|
1561
1914
|
}
|
|
@@ -1565,17 +1918,20 @@ var FileScannerService = class {
|
|
|
1565
1918
|
* the upload for good. The scanner is handed a signal that aborts then; one that ignores it keeps
|
|
1566
1919
|
* running.
|
|
1567
1920
|
*/
|
|
1568
|
-
async scanWithinDeadline(scanner, input) {
|
|
1921
|
+
async scanWithinDeadline(scanner, input, caller) {
|
|
1569
1922
|
const timeoutMs = this.options.scanner?.timeoutMs ?? DEFAULT_SCANNER_TIMEOUT_MS;
|
|
1570
1923
|
const controller = new AbortController();
|
|
1571
1924
|
let timer;
|
|
1572
1925
|
const deadline = new Promise((_resolve, reject) => {
|
|
1573
1926
|
timer = setTimeout(() => {
|
|
1574
|
-
this.logger
|
|
1927
|
+
emit(this.logger, "warn", {
|
|
1928
|
+
event: STORAGE_LOG_EVENTS.scanTimeout,
|
|
1929
|
+
message: "Scan timed out",
|
|
1930
|
+
timeoutMs
|
|
1931
|
+
});
|
|
1575
1932
|
reject(
|
|
1576
1933
|
new StorageException(STORAGE_ERROR_CODES.STORAGE_TIMEOUT, void 0, {
|
|
1577
|
-
|
|
1578
|
-
bucket: input.bucket,
|
|
1934
|
+
...caller,
|
|
1579
1935
|
phase: "scan",
|
|
1580
1936
|
timeoutMs
|
|
1581
1937
|
})
|
|
@@ -1593,7 +1949,9 @@ var FileScannerService = class {
|
|
|
1593
1949
|
FileScannerService = __decorateClass([
|
|
1594
1950
|
common.Injectable(),
|
|
1595
1951
|
__decorateParam(0, common.Inject(BYMAX_STORAGE_OPTIONS)),
|
|
1596
|
-
__decorateParam(1, common.Inject(BYMAX_STORAGE_FILE_SCANNER))
|
|
1952
|
+
__decorateParam(1, common.Inject(BYMAX_STORAGE_FILE_SCANNER)),
|
|
1953
|
+
__decorateParam(2, common.Optional()),
|
|
1954
|
+
__decorateParam(2, common.Inject(StorageObserver))
|
|
1597
1955
|
], FileScannerService);
|
|
1598
1956
|
var SourceIdleError = class extends Error {
|
|
1599
1957
|
constructor(idleMs) {
|
|
@@ -1807,7 +2165,7 @@ function cleanupErrorCode(error) {
|
|
|
1807
2165
|
}
|
|
1808
2166
|
function describeCleanupFailure(error) {
|
|
1809
2167
|
if (!(error instanceof StorageException)) {
|
|
1810
|
-
return
|
|
2168
|
+
return projectProviderFailure(error).message;
|
|
1811
2169
|
}
|
|
1812
2170
|
return error.code;
|
|
1813
2171
|
}
|
|
@@ -1822,7 +2180,6 @@ function withCleanupFailure(scanError, keys, cleanupError) {
|
|
|
1822
2180
|
{
|
|
1823
2181
|
...body.error?.details ?? {},
|
|
1824
2182
|
key: keys.key,
|
|
1825
|
-
storedKey: keys.storedKey,
|
|
1826
2183
|
cleanup: "failed",
|
|
1827
2184
|
cleanupError: cleanupErrorCode(cleanupError)
|
|
1828
2185
|
}
|
|
@@ -1955,11 +2312,13 @@ async function closeMultipartUpload(client, bucket, key, uploadId, onFailure) {
|
|
|
1955
2312
|
function closeWhenCreated(client, bucket, key, created, onFailure) {
|
|
1956
2313
|
void created.then((uploadId) => closeMultipartUpload(client, bucket, key, uploadId, onFailure));
|
|
1957
2314
|
}
|
|
1958
|
-
function abortFailureLogger(logger2
|
|
2315
|
+
function abortFailureLogger(logger2) {
|
|
1959
2316
|
return (err) => {
|
|
1960
|
-
logger2
|
|
1961
|
-
|
|
1962
|
-
|
|
2317
|
+
emit(logger2, "warn", {
|
|
2318
|
+
event: STORAGE_LOG_EVENTS.multipartAbortFailed,
|
|
2319
|
+
message: "Multipart abort failed, the upload stays open on the provider",
|
|
2320
|
+
err: failureError(err)
|
|
2321
|
+
});
|
|
1963
2322
|
};
|
|
1964
2323
|
}
|
|
1965
2324
|
async function deleteObjectsChunk(client, bucket, chunk, stripPrefix) {
|
|
@@ -2075,6 +2434,23 @@ function buildACL(perCall, defaultValue) {
|
|
|
2075
2434
|
return perCall ?? defaultValue ? "public-read" : void 0;
|
|
2076
2435
|
}
|
|
2077
2436
|
|
|
2437
|
+
// src/server/utils/object-metadata.ts
|
|
2438
|
+
function toObjectMetadata(response, key, bucket) {
|
|
2439
|
+
return {
|
|
2440
|
+
key,
|
|
2441
|
+
bucket,
|
|
2442
|
+
size: response.ContentLength ?? 0,
|
|
2443
|
+
contentType: response.ContentType ?? "application/octet-stream",
|
|
2444
|
+
etag: response.ETag ?? "",
|
|
2445
|
+
lastModified: response.LastModified ?? /* @__PURE__ */ new Date(0),
|
|
2446
|
+
metadata: response.Metadata ?? {},
|
|
2447
|
+
...response.CacheControl !== void 0 ? { cacheControl: response.CacheControl } : {},
|
|
2448
|
+
...response.ContentDisposition !== void 0 ? { contentDisposition: response.ContentDisposition } : {},
|
|
2449
|
+
...response.StorageClass !== void 0 ? { storageClass: response.StorageClass } : {},
|
|
2450
|
+
...response.VersionId !== void 0 ? { versionId: response.VersionId } : {}
|
|
2451
|
+
};
|
|
2452
|
+
}
|
|
2453
|
+
|
|
2078
2454
|
// src/server/utils/upload-strategy.ts
|
|
2079
2455
|
function pickUploadStrategy(body, declaredSize, thresholdBytes) {
|
|
2080
2456
|
const size = declaredSize ?? getBodySize(body);
|
|
@@ -2103,18 +2479,18 @@ function boundStreamSize(body, declaredSize, options) {
|
|
|
2103
2479
|
}
|
|
2104
2480
|
return withSizeLimit(body, limit, declaredSize === void 0 ? "max" : "declared");
|
|
2105
2481
|
}
|
|
2106
|
-
async function bufferStreamForScan(body,
|
|
2482
|
+
async function bufferStreamForScan(body, context, maxSizeBytes) {
|
|
2107
2483
|
if (maxSizeBytes === void 0) {
|
|
2108
2484
|
destroyStream(body);
|
|
2109
2485
|
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_VALIDATION_FAILED, void 0, {
|
|
2110
|
-
|
|
2486
|
+
...context,
|
|
2111
2487
|
reason: "Pre-upload scanning of a stream needs validation.maxSizeBytes to bound the buffer; set it or use scanner mode post-upload"
|
|
2112
2488
|
});
|
|
2113
2489
|
}
|
|
2114
2490
|
const buffered = await readStreamUpTo(body, maxSizeBytes);
|
|
2115
2491
|
if (buffered === void 0) {
|
|
2116
2492
|
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_VALIDATION_FAILED, void 0, {
|
|
2117
|
-
|
|
2493
|
+
...context,
|
|
2118
2494
|
maxSizeBytes,
|
|
2119
2495
|
reason: "The stream is longer than validation.maxSizeBytes, so it cannot be scanned before upload"
|
|
2120
2496
|
});
|
|
@@ -2172,6 +2548,13 @@ function baseCarriesBucket(base, bucket) {
|
|
|
2172
2548
|
function normalizeScanBody(body) {
|
|
2173
2549
|
return body;
|
|
2174
2550
|
}
|
|
2551
|
+
function trackLoaded(uploader) {
|
|
2552
|
+
let loaded = 0;
|
|
2553
|
+
uploader.on("httpUploadProgress", (event) => {
|
|
2554
|
+
loaded = event.loaded ?? loaded;
|
|
2555
|
+
});
|
|
2556
|
+
return () => loaded;
|
|
2557
|
+
}
|
|
2175
2558
|
function forwardProgress(uploader, onProgress) {
|
|
2176
2559
|
if (!onProgress) return;
|
|
2177
2560
|
uploader.on("httpUploadProgress", (event) => {
|
|
@@ -2241,11 +2624,10 @@ function assertMaxBytes(maxBytes) {
|
|
|
2241
2624
|
});
|
|
2242
2625
|
}
|
|
2243
2626
|
}
|
|
2244
|
-
async function readBody(stream, metadata, maxBytes) {
|
|
2627
|
+
async function readBody(stream, metadata, maxBytes, context) {
|
|
2245
2628
|
if (maxBytes === void 0) {
|
|
2246
2629
|
return readStream(stream);
|
|
2247
2630
|
}
|
|
2248
|
-
const context = { key: metadata.key, bucket: metadata.bucket };
|
|
2249
2631
|
if (metadata.size > maxBytes) {
|
|
2250
2632
|
destroyStream(stream);
|
|
2251
2633
|
throw sizeLimitException(maxBytes, "max", { ...context, contentLength: metadata.size });
|
|
@@ -2256,15 +2638,50 @@ async function readBody(stream, metadata, maxBytes) {
|
|
|
2256
2638
|
}
|
|
2257
2639
|
return buffer;
|
|
2258
2640
|
}
|
|
2259
|
-
async function readDownloadBody(stream, metadata, maxBytes, signal) {
|
|
2641
|
+
async function readDownloadBody(stream, metadata, maxBytes, signal, caller) {
|
|
2642
|
+
const context = { ...caller, op: "downloadBuffer" };
|
|
2260
2643
|
try {
|
|
2261
|
-
return await readBody(stream, metadata, maxBytes);
|
|
2644
|
+
return await readBody(stream, metadata, maxBytes, context);
|
|
2262
2645
|
} catch (err) {
|
|
2263
|
-
throw mapBodyReadFailure(err, signal,
|
|
2264
|
-
|
|
2265
|
-
|
|
2266
|
-
|
|
2646
|
+
throw mapBodyReadFailure(err, signal, context);
|
|
2647
|
+
}
|
|
2648
|
+
}
|
|
2649
|
+
|
|
2650
|
+
// src/server/utils/uploaded-bytes.ts
|
|
2651
|
+
var SENT = /* @__PURE__ */ new WeakMap();
|
|
2652
|
+
function rememberUploadedBytes(result, bytes) {
|
|
2653
|
+
SENT.set(result, bytes);
|
|
2654
|
+
return result;
|
|
2655
|
+
}
|
|
2656
|
+
function uploadedBytesOf(result) {
|
|
2657
|
+
return result.fromIdempotencyCache ? void 0 : SENT.get(result) ?? result.size;
|
|
2658
|
+
}
|
|
2659
|
+
async function uploadSingleShot(context, options, finalKey, bucket, caller) {
|
|
2660
|
+
const total = options.size ?? getBodySize(options.body);
|
|
2661
|
+
const input = {
|
|
2662
|
+
...putObjectHeaders(options, context.options, finalKey, bucket),
|
|
2663
|
+
Body: normalizeSingleShotBody(options.body),
|
|
2664
|
+
ContentLength: total
|
|
2665
|
+
};
|
|
2666
|
+
const watch = abortOnBodyFailure(input.Body, options.abortSignal);
|
|
2667
|
+
try {
|
|
2668
|
+
const response = await context.s3Provider.getClient().send(new clientS3.PutObjectCommand(input), sendOptions(watch.signal));
|
|
2669
|
+
if (options.onProgress) {
|
|
2670
|
+
emitProgress(options.onProgress, total, total);
|
|
2671
|
+
}
|
|
2672
|
+
return buildUploadResult(context.keyResolver, context.options, {
|
|
2673
|
+
finalKey,
|
|
2674
|
+
bucket,
|
|
2675
|
+
etag: response.ETag,
|
|
2676
|
+
versionId: response.VersionId,
|
|
2677
|
+
size: total,
|
|
2678
|
+
contentType: options.contentType,
|
|
2679
|
+
multipart: false
|
|
2267
2680
|
});
|
|
2681
|
+
} catch (err) {
|
|
2682
|
+
const context2 = { ...caller, op: "upload-single" };
|
|
2683
|
+
const mapped = mapSizeLimitError(watch.failure(), context2) ?? mapAwsError(err, context2);
|
|
2684
|
+
throw failureOrAbort(options.abortSignal, context2, mapped);
|
|
2268
2685
|
}
|
|
2269
2686
|
}
|
|
2270
2687
|
|
|
@@ -2272,13 +2689,14 @@ async function readDownloadBody(stream, metadata, maxBytes, signal) {
|
|
|
2272
2689
|
var MAX_LIST_KEYS = 1e3;
|
|
2273
2690
|
var DELETE_BATCH_LIMIT = 1e3;
|
|
2274
2691
|
exports.StorageService = class StorageService {
|
|
2275
|
-
constructor(options, s3Provider, keyResolver, idempotencyCache, validation, scanner) {
|
|
2692
|
+
constructor(options, s3Provider, keyResolver, idempotencyCache, validation, scanner, observer) {
|
|
2276
2693
|
this.options = options;
|
|
2277
2694
|
this.s3Provider = s3Provider;
|
|
2278
2695
|
this.keyResolver = keyResolver;
|
|
2279
2696
|
this.idempotencyCache = idempotencyCache;
|
|
2280
2697
|
this.validation = validation;
|
|
2281
2698
|
this.scanner = scanner;
|
|
2699
|
+
this.observer = observer;
|
|
2282
2700
|
}
|
|
2283
2701
|
options;
|
|
2284
2702
|
s3Provider;
|
|
@@ -2286,22 +2704,8 @@ exports.StorageService = class StorageService {
|
|
|
2286
2704
|
idempotencyCache;
|
|
2287
2705
|
validation;
|
|
2288
2706
|
scanner;
|
|
2707
|
+
observer;
|
|
2289
2708
|
logger = new common.Logger(exports.StorageService.name);
|
|
2290
|
-
/**
|
|
2291
|
-
* Uploads an object, choosing single-shot or multipart automatically. When an
|
|
2292
|
-
* `idempotencyKey` is supplied and a matching result is cached within the TTL
|
|
2293
|
-
* window, the cached result is returned without re-uploading.
|
|
2294
|
-
*
|
|
2295
|
-
* The library takes ownership of a `Readable` body, as `stream.pipeline` does: when the
|
|
2296
|
-
* upload is rejected for any reason (configuration, validation, scan, provider failure)
|
|
2297
|
-
* the stream is destroyed (without an error: the SDK forwards a stream's error to wrappers
|
|
2298
|
-
* that nothing listens to), which releases the file handle or socket behind it; a cache hit
|
|
2299
|
-
* destroys it unread. A stream consumed by a successful upload ends on its own.
|
|
2300
|
-
*
|
|
2301
|
-
* @param options - The upload request.
|
|
2302
|
-
* @returns The upload result.
|
|
2303
|
-
* @throws StorageException for missing configuration, body, or content type.
|
|
2304
|
-
*/
|
|
2305
2709
|
async upload(options) {
|
|
2306
2710
|
try {
|
|
2307
2711
|
return await this.uploadOwningBody(options);
|
|
@@ -2324,7 +2728,7 @@ exports.StorageService = class StorageService {
|
|
|
2324
2728
|
}
|
|
2325
2729
|
const finalKey = this.keyResolver.normalize(options.key);
|
|
2326
2730
|
const bucket = this.resolveBucket(options.bucket);
|
|
2327
|
-
const context = { key:
|
|
2731
|
+
const context = callDetails("upload", { key: options.key, bucket: options.bucket });
|
|
2328
2732
|
throwIfAborted(options.abortSignal, context);
|
|
2329
2733
|
if (options.idempotencyKey) {
|
|
2330
2734
|
const cacheKey = this.idempotencyCache.computeKey(options.idempotencyKey, finalKey);
|
|
@@ -2342,7 +2746,7 @@ exports.StorageService = class StorageService {
|
|
|
2342
2746
|
validatedOptions.size,
|
|
2343
2747
|
this.options.multipart.thresholdBytes
|
|
2344
2748
|
);
|
|
2345
|
-
const result = strategy === "multipart" ? await this.uploadMultipart(validatedOptions, finalKey, bucket) : await this.
|
|
2749
|
+
const result = strategy === "multipart" ? await this.uploadMultipart(validatedOptions, finalKey, bucket, context) : await uploadSingleShot(this.uploadContext, validatedOptions, finalKey, bucket, context);
|
|
2346
2750
|
const stored = { ...context, stored: true };
|
|
2347
2751
|
await abortable(this.runPostUploadScan(finalKey, bucket, options), options.abortSignal, stored);
|
|
2348
2752
|
this.rememberUpload(options, finalKey, result);
|
|
@@ -2358,7 +2762,7 @@ exports.StorageService = class StorageService {
|
|
|
2358
2762
|
const input = validationInput(options, finalKey);
|
|
2359
2763
|
const validated = await abortable(this.validation.validate(input), signal, context);
|
|
2360
2764
|
const scanned = await abortable(
|
|
2361
|
-
this.bufferForPreUploadScan(validated.body,
|
|
2765
|
+
this.bufferForPreUploadScan(validated.body, context),
|
|
2362
2766
|
signal,
|
|
2363
2767
|
context
|
|
2364
2768
|
);
|
|
@@ -2375,28 +2779,31 @@ exports.StorageService = class StorageService {
|
|
|
2375
2779
|
* `validation.maxSizeBytes`, and a stream with no bound is refused rather than
|
|
2376
2780
|
* buffered without limit. Other bodies and modes pass through unchanged.
|
|
2377
2781
|
*/
|
|
2378
|
-
async bufferForPreUploadScan(body,
|
|
2782
|
+
async bufferForPreUploadScan(body, context) {
|
|
2379
2783
|
if (!this.scanner.isEnabled() || this.scanner.getMode() !== "pre-upload") {
|
|
2380
2784
|
return body;
|
|
2381
2785
|
}
|
|
2382
2786
|
if (isBufferLike(body)) {
|
|
2383
2787
|
return Buffer.from(body);
|
|
2384
2788
|
}
|
|
2385
|
-
return bufferStreamForScan(body,
|
|
2789
|
+
return bufferStreamForScan(body, context, this.options.validation?.maxSizeBytes);
|
|
2386
2790
|
}
|
|
2387
2791
|
/** Runs a pre-upload scan when configured; throws on infected. */
|
|
2388
2792
|
async runPreUploadScan(body, finalKey, bucket, options) {
|
|
2389
2793
|
if (!this.scanner.isEnabled() || this.scanner.getMode() !== "pre-upload") {
|
|
2390
2794
|
return;
|
|
2391
2795
|
}
|
|
2392
|
-
await this.scanner.scan(
|
|
2393
|
-
|
|
2394
|
-
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
|
|
2398
|
-
|
|
2399
|
-
|
|
2796
|
+
await this.scanner.scan(
|
|
2797
|
+
{
|
|
2798
|
+
mode: "pre-upload",
|
|
2799
|
+
body: normalizeScanBody(body),
|
|
2800
|
+
key: finalKey,
|
|
2801
|
+
bucket,
|
|
2802
|
+
contentType: options.contentType,
|
|
2803
|
+
...options.size !== void 0 ? { size: options.size } : {}
|
|
2804
|
+
},
|
|
2805
|
+
callerFields({ key: options.key, bucket: options.bucket })
|
|
2806
|
+
);
|
|
2400
2807
|
}
|
|
2401
2808
|
/** Remembers the result of an upload under its idempotency key, when it has one. */
|
|
2402
2809
|
rememberUpload(options, finalKey, result) {
|
|
@@ -2415,47 +2822,44 @@ exports.StorageService = class StorageService {
|
|
|
2415
2822
|
return;
|
|
2416
2823
|
}
|
|
2417
2824
|
try {
|
|
2418
|
-
await this.scanner.scan(
|
|
2419
|
-
|
|
2420
|
-
|
|
2421
|
-
|
|
2422
|
-
|
|
2423
|
-
|
|
2424
|
-
|
|
2825
|
+
await this.scanner.scan(
|
|
2826
|
+
{
|
|
2827
|
+
mode: "post-upload",
|
|
2828
|
+
key: finalKey,
|
|
2829
|
+
bucket,
|
|
2830
|
+
contentType: options.contentType,
|
|
2831
|
+
...options.size !== void 0 ? { size: options.size } : {}
|
|
2832
|
+
},
|
|
2833
|
+
callerFields({ key: options.key, bucket: options.bucket })
|
|
2834
|
+
);
|
|
2425
2835
|
} catch (scanErr) {
|
|
2426
2836
|
const key = this.keyResolver.stripPrefix(finalKey);
|
|
2427
2837
|
try {
|
|
2428
2838
|
await this.delete(key, { bucket, abortSignal: AbortSignal.timeout(CLEANUP_DEADLINE_MS) });
|
|
2429
2839
|
} catch (deleteErr) {
|
|
2430
|
-
this.logger
|
|
2431
|
-
|
|
2432
|
-
|
|
2433
|
-
|
|
2434
|
-
|
|
2840
|
+
emit(this.logger, "error", {
|
|
2841
|
+
event: STORAGE_LOG_EVENTS.scanCleanupFailed,
|
|
2842
|
+
message: "The rejected object is still stored",
|
|
2843
|
+
storedKey: escapeForLog(finalKey),
|
|
2844
|
+
cleanupFailure: describeCleanupFailure(deleteErr)
|
|
2845
|
+
});
|
|
2846
|
+
throw withCleanupFailure(scanErr, { key }, deleteErr);
|
|
2435
2847
|
}
|
|
2436
2848
|
throw scanErr;
|
|
2437
2849
|
}
|
|
2438
2850
|
}
|
|
2439
|
-
/**
|
|
2440
|
-
* Returns metadata for an object without downloading its body.
|
|
2441
|
-
*
|
|
2442
|
-
* @param key - The raw object key (normalized internally).
|
|
2443
|
-
* @param options - Optional per-call bucket override.
|
|
2444
|
-
* @returns The object metadata.
|
|
2445
|
-
* @throws StorageException `STORAGE_OBJECT_NOT_FOUND` when the key is absent.
|
|
2446
|
-
*/
|
|
2447
2851
|
async head(key, options) {
|
|
2448
2852
|
this.assertConfigured();
|
|
2449
2853
|
const finalKey = this.keyResolver.normalize(key);
|
|
2450
2854
|
const bucket = this.resolveBucket(options?.bucket);
|
|
2451
|
-
const context = { key
|
|
2855
|
+
const context = callDetails("head", { key, bucket: options?.bucket });
|
|
2452
2856
|
throwIfAborted(options?.abortSignal, context);
|
|
2453
2857
|
try {
|
|
2454
2858
|
const response = await this.s3Provider.getClient().send(
|
|
2455
2859
|
new clientS3.HeadObjectCommand({ Bucket: bucket, Key: finalKey }),
|
|
2456
2860
|
sendOptions(options?.abortSignal)
|
|
2457
2861
|
);
|
|
2458
|
-
return
|
|
2862
|
+
return toObjectMetadata(response, this.keyResolver.stripPrefix(finalKey), bucket);
|
|
2459
2863
|
} catch (err) {
|
|
2460
2864
|
throw failureOrAbort(options?.abortSignal, context, mapAwsError(err, context));
|
|
2461
2865
|
}
|
|
@@ -2479,9 +2883,11 @@ exports.StorageService = class StorageService {
|
|
|
2479
2883
|
if (err instanceof StorageException && err.code === STORAGE_ERROR_CODES.STORAGE_REQUEST_ABORTED) {
|
|
2480
2884
|
throw err;
|
|
2481
2885
|
}
|
|
2482
|
-
this.logger
|
|
2483
|
-
|
|
2484
|
-
|
|
2886
|
+
emit(this.logger, "warn", {
|
|
2887
|
+
event: STORAGE_LOG_EVENTS.existsFailed,
|
|
2888
|
+
message: 'exists() \u2014 treating a non-404 error as "false"',
|
|
2889
|
+
err: failureError(err)
|
|
2890
|
+
});
|
|
2485
2891
|
return false;
|
|
2486
2892
|
}
|
|
2487
2893
|
}
|
|
@@ -2498,15 +2904,6 @@ exports.StorageService = class StorageService {
|
|
|
2498
2904
|
const bucket = this.resolveBucket(options?.bucket);
|
|
2499
2905
|
return buildPublicUrl(this.options, finalKey, bucket);
|
|
2500
2906
|
}
|
|
2501
|
-
/**
|
|
2502
|
-
* Streams an object. The returned `stream` is a Node `Readable` consumable via
|
|
2503
|
-
* `for await` or `.pipe()`. `range` / `ifNoneMatch` / `ifMatch` are forwarded as
|
|
2504
|
-
* conditional/partial GET headers.
|
|
2505
|
-
*
|
|
2506
|
-
* @param options - The download request.
|
|
2507
|
-
* @returns The object stream and its metadata.
|
|
2508
|
-
* @throws StorageException `STORAGE_OBJECT_NOT_FOUND` when the body is absent.
|
|
2509
|
-
*/
|
|
2510
2907
|
async download(options) {
|
|
2511
2908
|
this.assertConfigured();
|
|
2512
2909
|
assertHeaderValues({
|
|
@@ -2516,7 +2913,7 @@ exports.StorageService = class StorageService {
|
|
|
2516
2913
|
});
|
|
2517
2914
|
const finalKey = this.keyResolver.normalize(options.key);
|
|
2518
2915
|
const bucket = this.resolveBucket(options.bucket);
|
|
2519
|
-
const context = { key:
|
|
2916
|
+
const context = callDetails("download", { key: options.key, bucket: options.bucket });
|
|
2520
2917
|
throwIfAborted(options.abortSignal, context);
|
|
2521
2918
|
try {
|
|
2522
2919
|
const response = await this.s3Provider.getClient().send(
|
|
@@ -2530,10 +2927,7 @@ exports.StorageService = class StorageService {
|
|
|
2530
2927
|
sendOptions(options.abortSignal)
|
|
2531
2928
|
);
|
|
2532
2929
|
if (!response.Body) {
|
|
2533
|
-
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_OBJECT_NOT_FOUND, void 0,
|
|
2534
|
-
key: finalKey,
|
|
2535
|
-
bucket
|
|
2536
|
-
});
|
|
2930
|
+
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_OBJECT_NOT_FOUND, void 0, context);
|
|
2537
2931
|
}
|
|
2538
2932
|
const stream = guardDownloadBody(
|
|
2539
2933
|
response.Body,
|
|
@@ -2541,7 +2935,10 @@ exports.StorageService = class StorageService {
|
|
|
2541
2935
|
context,
|
|
2542
2936
|
options.abortSignal
|
|
2543
2937
|
);
|
|
2544
|
-
return {
|
|
2938
|
+
return {
|
|
2939
|
+
stream,
|
|
2940
|
+
metadata: toObjectMetadata(response, this.keyResolver.stripPrefix(finalKey), bucket)
|
|
2941
|
+
};
|
|
2545
2942
|
} catch (err) {
|
|
2546
2943
|
if (err instanceof StorageException) {
|
|
2547
2944
|
throw err;
|
|
@@ -2561,21 +2958,20 @@ exports.StorageService = class StorageService {
|
|
|
2561
2958
|
async downloadBuffer(options) {
|
|
2562
2959
|
assertMaxBytes(options.maxBytes);
|
|
2563
2960
|
const { stream, metadata } = await this.download(options);
|
|
2564
|
-
const buffer = await readDownloadBody(
|
|
2961
|
+
const buffer = await readDownloadBody(
|
|
2962
|
+
stream,
|
|
2963
|
+
metadata,
|
|
2964
|
+
options.maxBytes,
|
|
2965
|
+
options.abortSignal,
|
|
2966
|
+
callerFields({ key: options.key, bucket: options.bucket })
|
|
2967
|
+
);
|
|
2565
2968
|
return { buffer, metadata };
|
|
2566
2969
|
}
|
|
2567
|
-
/**
|
|
2568
|
-
* Deletes an object. Idempotent: a missing object is a no-op (logged as a
|
|
2569
|
-
* warning), not an error. Any other failure is propagated.
|
|
2570
|
-
*
|
|
2571
|
-
* @param key - The raw object key.
|
|
2572
|
-
* @param options - Optional per-call bucket override.
|
|
2573
|
-
*/
|
|
2574
2970
|
async delete(key, options) {
|
|
2575
2971
|
this.assertConfigured();
|
|
2576
2972
|
const finalKey = this.keyResolver.normalize(key);
|
|
2577
2973
|
const bucket = this.resolveBucket(options?.bucket);
|
|
2578
|
-
const context = { key
|
|
2974
|
+
const context = callDetails("delete", { key, bucket: options?.bucket });
|
|
2579
2975
|
throwIfAborted(options?.abortSignal, context);
|
|
2580
2976
|
try {
|
|
2581
2977
|
await this.s3Provider.getClient().send(
|
|
@@ -2587,25 +2983,17 @@ exports.StorageService = class StorageService {
|
|
|
2587
2983
|
if (mapped.code !== STORAGE_ERROR_CODES.STORAGE_OBJECT_NOT_FOUND) {
|
|
2588
2984
|
throw mapped;
|
|
2589
2985
|
}
|
|
2590
|
-
this.logger
|
|
2986
|
+
emit(this.logger, "warn", {
|
|
2987
|
+
event: STORAGE_LOG_EVENTS.deleteNotFound,
|
|
2988
|
+
message: "delete() \u2014 key not found (idempotent no-op)"
|
|
2989
|
+
});
|
|
2591
2990
|
}
|
|
2592
2991
|
this.idempotencyCache.invalidate(bucket, this.keyResolver.stripPrefix(finalKey));
|
|
2593
2992
|
}
|
|
2594
|
-
/**
|
|
2595
|
-
* Probes the bucket with one `HeadBucket`, for a readiness check: it resolves when the bucket
|
|
2596
|
-
* is reachable with these credentials and otherwise rejects with a typed code. Nothing in the
|
|
2597
|
-
* module calls it on its own, so booting costs no request. Needs `s3:ListBucket`.
|
|
2598
|
-
*
|
|
2599
|
-
* @param options - Optional per-call bucket override.
|
|
2600
|
-
* @throws StorageException `STORAGE_BUCKET_NOT_FOUND` (404), `STORAGE_BUCKET_ACCESS_DENIED` (403,
|
|
2601
|
-
* which a provider also answers for a bucket the caller cannot see), `STORAGE_BUCKET_REGION_MISMATCH`
|
|
2602
|
-
* (301), `STORAGE_TIMEOUT`, or `STORAGE_PROVIDER_ERROR`; `STORAGE_NOT_CONFIGURED` and
|
|
2603
|
-
* `STORAGE_BUCKET_UNDEFINED` before any request.
|
|
2604
|
-
*/
|
|
2605
2993
|
async checkBucket(options) {
|
|
2606
2994
|
this.assertConfigured();
|
|
2607
2995
|
const bucket = this.resolveBucket(options?.bucket);
|
|
2608
|
-
const context = { bucket
|
|
2996
|
+
const context = callDetails("checkBucket", { bucket: options?.bucket });
|
|
2609
2997
|
throwIfAborted(options?.abortSignal, context);
|
|
2610
2998
|
try {
|
|
2611
2999
|
await this.s3Provider.getClient().send(new clientS3.HeadBucketCommand({ Bucket: bucket }), sendOptions(options?.abortSignal));
|
|
@@ -2613,6 +3001,10 @@ exports.StorageService = class StorageService {
|
|
|
2613
3001
|
throw failureOrAbort(options?.abortSignal, context, mapBucketCheckError(err, context));
|
|
2614
3002
|
}
|
|
2615
3003
|
}
|
|
3004
|
+
/** What the single-shot upload needs of this service. */
|
|
3005
|
+
get uploadContext() {
|
|
3006
|
+
return { s3Provider: this.s3Provider, keyResolver: this.keyResolver, options: this.options };
|
|
3007
|
+
}
|
|
2616
3008
|
/** Throws when the S3 client was never built (missing credentials). */
|
|
2617
3009
|
assertConfigured() {
|
|
2618
3010
|
if (!this.s3Provider.isConfigured()) {
|
|
@@ -2623,58 +3015,13 @@ exports.StorageService = class StorageService {
|
|
|
2623
3015
|
resolveBucket(perCall) {
|
|
2624
3016
|
return resolveBucket(perCall, this.options.bucket);
|
|
2625
3017
|
}
|
|
2626
|
-
/** Maps an S3 metadata response onto the public `ObjectMetadata` shape. */
|
|
2627
|
-
toObjectMetadata(response, finalKey, bucket) {
|
|
2628
|
-
return {
|
|
2629
|
-
key: this.keyResolver.stripPrefix(finalKey),
|
|
2630
|
-
bucket,
|
|
2631
|
-
size: response.ContentLength ?? 0,
|
|
2632
|
-
contentType: response.ContentType ?? "application/octet-stream",
|
|
2633
|
-
etag: response.ETag ?? "",
|
|
2634
|
-
lastModified: response.LastModified ?? /* @__PURE__ */ new Date(0),
|
|
2635
|
-
metadata: response.Metadata ?? {},
|
|
2636
|
-
...response.CacheControl !== void 0 ? { cacheControl: response.CacheControl } : {},
|
|
2637
|
-
...response.ContentDisposition !== void 0 ? { contentDisposition: response.ContentDisposition } : {},
|
|
2638
|
-
...response.StorageClass !== void 0 ? { storageClass: response.StorageClass } : {},
|
|
2639
|
-
...response.VersionId !== void 0 ? { versionId: response.VersionId } : {}
|
|
2640
|
-
};
|
|
2641
|
-
}
|
|
2642
|
-
/** Single `PutObject` upload for bodies below the multipart threshold. */
|
|
2643
|
-
async uploadSingleShot(options, finalKey, bucket) {
|
|
2644
|
-
const total = options.size ?? getBodySize(options.body);
|
|
2645
|
-
const input = {
|
|
2646
|
-
...putObjectHeaders(options, this.options, finalKey, bucket),
|
|
2647
|
-
Body: normalizeSingleShotBody(options.body),
|
|
2648
|
-
ContentLength: total
|
|
2649
|
-
};
|
|
2650
|
-
const watch = abortOnBodyFailure(input.Body, options.abortSignal);
|
|
2651
|
-
try {
|
|
2652
|
-
const response = await this.s3Provider.getClient().send(new clientS3.PutObjectCommand(input), sendOptions(watch.signal));
|
|
2653
|
-
if (options.onProgress) {
|
|
2654
|
-
emitProgress(options.onProgress, total, total);
|
|
2655
|
-
}
|
|
2656
|
-
return buildUploadResult(this.keyResolver, this.options, {
|
|
2657
|
-
finalKey,
|
|
2658
|
-
bucket,
|
|
2659
|
-
etag: response.ETag,
|
|
2660
|
-
versionId: response.VersionId,
|
|
2661
|
-
size: total,
|
|
2662
|
-
contentType: options.contentType,
|
|
2663
|
-
multipart: false
|
|
2664
|
-
});
|
|
2665
|
-
} catch (err) {
|
|
2666
|
-
const context = { key: finalKey, bucket, op: "upload-single" };
|
|
2667
|
-
const mapped = mapSizeLimitError(watch.failure(), context) ?? mapAwsError(err, context);
|
|
2668
|
-
throw failureOrAbort(options.abortSignal, context, mapped);
|
|
2669
|
-
}
|
|
2670
|
-
}
|
|
2671
3018
|
/**
|
|
2672
3019
|
* Multipart upload via `@aws-sdk/lib-storage`. The SDK is told to leave the parts
|
|
2673
3020
|
* (`leavePartsOnError: true`) because its own abort runs inside its failure path: a failing
|
|
2674
3021
|
* abort replaced the failure that caused it, and a failed `CompleteMultipartUpload` was never
|
|
2675
3022
|
* aborted. This method aborts the upload itself and reports the outcome in the error details.
|
|
2676
3023
|
*/
|
|
2677
|
-
async uploadMultipart(options, finalKey, bucket) {
|
|
3024
|
+
async uploadMultipart(options, finalKey, bucket, caller) {
|
|
2678
3025
|
const params = {
|
|
2679
3026
|
...putObjectHeaders(options, this.options, finalKey, bucket),
|
|
2680
3027
|
Body: guardIdleBody(options.body, this.options.uploadIdleTimeoutMs)
|
|
@@ -2688,6 +3035,7 @@ exports.StorageService = class StorageService {
|
|
|
2688
3035
|
partSize: this.options.multipart.partSizeBytes,
|
|
2689
3036
|
leavePartsOnError: true
|
|
2690
3037
|
});
|
|
3038
|
+
const sent = trackLoaded(uploader);
|
|
2691
3039
|
forwardProgress(uploader, options.onProgress);
|
|
2692
3040
|
const abortUpload = () => {
|
|
2693
3041
|
void uploader.abort();
|
|
@@ -2695,17 +3043,20 @@ exports.StorageService = class StorageService {
|
|
|
2695
3043
|
signal?.addEventListener("abort", abortUpload, { once: true });
|
|
2696
3044
|
try {
|
|
2697
3045
|
const response = await uploader.done();
|
|
2698
|
-
return
|
|
2699
|
-
|
|
2700
|
-
|
|
2701
|
-
|
|
2702
|
-
|
|
2703
|
-
|
|
2704
|
-
|
|
2705
|
-
|
|
2706
|
-
|
|
3046
|
+
return rememberUploadedBytes(
|
|
3047
|
+
buildUploadResult(this.keyResolver, this.options, {
|
|
3048
|
+
finalKey,
|
|
3049
|
+
bucket,
|
|
3050
|
+
etag: response.ETag,
|
|
3051
|
+
versionId: response.VersionId,
|
|
3052
|
+
size: options.size,
|
|
3053
|
+
contentType: options.contentType,
|
|
3054
|
+
multipart: true
|
|
3055
|
+
}),
|
|
3056
|
+
sent()
|
|
3057
|
+
);
|
|
2707
3058
|
} catch (err) {
|
|
2708
|
-
return await this.failMultipart(err, finalKey, bucket, {
|
|
3059
|
+
return await this.failMultipart(err, finalKey, bucket, caller, {
|
|
2709
3060
|
uploadId: uploader.uploadId,
|
|
2710
3061
|
signal,
|
|
2711
3062
|
createdUploadId: bound?.createdUploadId
|
|
@@ -2720,7 +3071,7 @@ exports.StorageService = class StorageService {
|
|
|
2720
3071
|
* real failure into one. An upload id the provider answered late is waited for, within the cleanup
|
|
2721
3072
|
* deadline, so that an abort during `CreateMultipartUpload` still aborts the upload it created.
|
|
2722
3073
|
*/
|
|
2723
|
-
async failMultipart(err, finalKey, bucket, upload) {
|
|
3074
|
+
async failMultipart(err, finalKey, bucket, caller, upload) {
|
|
2724
3075
|
const abortedBy = upload.signal?.aborted === true ? upload.signal : void 0;
|
|
2725
3076
|
const client = this.s3Provider.getClient();
|
|
2726
3077
|
const lookup = abortedBy ? upload.createdUploadId?.() : void 0;
|
|
@@ -2728,7 +3079,7 @@ exports.StorageService = class StorageService {
|
|
|
2728
3079
|
if (uploadId === void 0 && lookup) {
|
|
2729
3080
|
uploadId = await within(lookup, CLEANUP_DEADLINE_MS);
|
|
2730
3081
|
if (uploadId === void 0) {
|
|
2731
|
-
closeWhenCreated(client, bucket, finalKey, lookup, abortFailureLogger(this.logger
|
|
3082
|
+
closeWhenCreated(client, bucket, finalKey, lookup, abortFailureLogger(this.logger));
|
|
2732
3083
|
}
|
|
2733
3084
|
}
|
|
2734
3085
|
const abort = await closeMultipartUpload(
|
|
@@ -2736,27 +3087,18 @@ exports.StorageService = class StorageService {
|
|
|
2736
3087
|
bucket,
|
|
2737
3088
|
finalKey,
|
|
2738
3089
|
uploadId,
|
|
2739
|
-
abortFailureLogger(this.logger
|
|
3090
|
+
abortFailureLogger(this.logger)
|
|
2740
3091
|
);
|
|
2741
|
-
const context = {
|
|
3092
|
+
const context = { ...caller, op: "upload-multipart", ...abort };
|
|
2742
3093
|
throw abortedBy ? requestAborted(abortedBy, context) : mapMultipartError(err, context);
|
|
2743
3094
|
}
|
|
2744
|
-
/**
|
|
2745
|
-
* Lists objects in the bucket as a single page. Applies the global key prefix
|
|
2746
|
-
* to the caller's `prefix`, caps `maxKeys` at the S3 hard limit of 1000, threads
|
|
2747
|
-
* the continuation token, and strips the global prefix from every returned key.
|
|
2748
|
-
*
|
|
2749
|
-
* @param options - The listing request.
|
|
2750
|
-
* @returns One page of objects plus `commonPrefixes`, truncation, and the next token.
|
|
2751
|
-
* @throws StorageException mapped from any AWS failure.
|
|
2752
|
-
*/
|
|
2753
3095
|
async list(options) {
|
|
2754
3096
|
this.assertConfigured();
|
|
2755
3097
|
const bucket = this.resolveBucket(options.bucket);
|
|
2756
3098
|
assertIntegerInput(options.maxKeys, "maxKeys", 1);
|
|
2757
3099
|
const maxKeys = Math.min(options.maxKeys ?? MAX_LIST_KEYS, MAX_LIST_KEYS);
|
|
2758
3100
|
const prefix = options.prefix ? this.keyResolver.normalize(options.prefix) : this.keyResolver.getPrefix();
|
|
2759
|
-
const context = { bucket, prefix
|
|
3101
|
+
const context = callDetails("list", { bucket: options.bucket, prefix: options.prefix });
|
|
2760
3102
|
throwIfAborted(options.abortSignal, context);
|
|
2761
3103
|
try {
|
|
2762
3104
|
const response = await this.s3Provider.getClient().send(
|
|
@@ -2774,14 +3116,6 @@ exports.StorageService = class StorageService {
|
|
|
2774
3116
|
throw failureOrAbort(options.abortSignal, context, mapAwsError(err, context));
|
|
2775
3117
|
}
|
|
2776
3118
|
}
|
|
2777
|
-
/**
|
|
2778
|
-
* Server-side copy of an object (no bytes flow through the app). Supports both
|
|
2779
|
-
* same-bucket and cross-bucket copies and preserves the source metadata.
|
|
2780
|
-
*
|
|
2781
|
-
* @param options - Source/destination keys and buckets plus copy overrides.
|
|
2782
|
-
* @returns The new object's ETag.
|
|
2783
|
-
* @throws StorageException mapped from any AWS failure.
|
|
2784
|
-
*/
|
|
2785
3119
|
async copy(options) {
|
|
2786
3120
|
this.assertConfigured();
|
|
2787
3121
|
assertHeaderValues({ cacheControl: options.cacheControl });
|
|
@@ -2805,19 +3139,12 @@ exports.StorageService = class StorageService {
|
|
|
2805
3139
|
);
|
|
2806
3140
|
return { etag: response.CopyObjectResult?.ETag ?? "" };
|
|
2807
3141
|
} catch (err) {
|
|
2808
|
-
throw mapAwsError(
|
|
3142
|
+
throw mapAwsError(
|
|
3143
|
+
err,
|
|
3144
|
+
callDetails("copy", { sourceKey: options.sourceKey, destKey: options.destinationKey })
|
|
3145
|
+
);
|
|
2809
3146
|
}
|
|
2810
3147
|
}
|
|
2811
|
-
/**
|
|
2812
|
-
* Deletes many objects, chunking the input at the S3 hard limit of 1000 keys
|
|
2813
|
-
* per request. Reports both successes and per-key failures; a whole-chunk send
|
|
2814
|
-
* failure marks every key in that chunk as failed. Returned keys are stripped of
|
|
2815
|
-
* the global prefix.
|
|
2816
|
-
*
|
|
2817
|
-
* @param keys - The raw object keys to delete.
|
|
2818
|
-
* @param options - Optional per-call bucket override.
|
|
2819
|
-
* @returns The deleted keys and the per-key failures.
|
|
2820
|
-
*/
|
|
2821
3148
|
async deleteMany(keys, options) {
|
|
2822
3149
|
this.assertConfigured();
|
|
2823
3150
|
if (keys.length === 0) {
|
|
@@ -2844,6 +3171,34 @@ exports.StorageService = class StorageService {
|
|
|
2844
3171
|
return { deleted, failed };
|
|
2845
3172
|
}
|
|
2846
3173
|
};
|
|
3174
|
+
__decorateClass([
|
|
3175
|
+
Observed("upload", {
|
|
3176
|
+
span: "storage.upload",
|
|
3177
|
+
describe: (result) => ({ "bymax.storage.multipart": String(result.multipart) }),
|
|
3178
|
+
bytes: uploadedBytesOf
|
|
3179
|
+
})
|
|
3180
|
+
], exports.StorageService.prototype, "upload", 1);
|
|
3181
|
+
__decorateClass([
|
|
3182
|
+
Observed("head")
|
|
3183
|
+
], exports.StorageService.prototype, "head", 1);
|
|
3184
|
+
__decorateClass([
|
|
3185
|
+
Observed("download")
|
|
3186
|
+
], exports.StorageService.prototype, "download", 1);
|
|
3187
|
+
__decorateClass([
|
|
3188
|
+
Observed("delete")
|
|
3189
|
+
], exports.StorageService.prototype, "delete", 1);
|
|
3190
|
+
__decorateClass([
|
|
3191
|
+
Observed("check_bucket")
|
|
3192
|
+
], exports.StorageService.prototype, "checkBucket", 1);
|
|
3193
|
+
__decorateClass([
|
|
3194
|
+
Observed("list")
|
|
3195
|
+
], exports.StorageService.prototype, "list", 1);
|
|
3196
|
+
__decorateClass([
|
|
3197
|
+
Observed("copy")
|
|
3198
|
+
], exports.StorageService.prototype, "copy", 1);
|
|
3199
|
+
__decorateClass([
|
|
3200
|
+
Observed("delete_many")
|
|
3201
|
+
], exports.StorageService.prototype, "deleteMany", 1);
|
|
2847
3202
|
exports.StorageService = __decorateClass([
|
|
2848
3203
|
common.Injectable(),
|
|
2849
3204
|
__decorateParam(0, common.Inject(BYMAX_STORAGE_OPTIONS)),
|
|
@@ -2851,7 +3206,9 @@ exports.StorageService = __decorateClass([
|
|
|
2851
3206
|
__decorateParam(2, common.Inject(KeyResolverService)),
|
|
2852
3207
|
__decorateParam(3, common.Inject(BYMAX_STORAGE_IDEMPOTENCY_CACHE)),
|
|
2853
3208
|
__decorateParam(4, common.Inject(ValidationService)),
|
|
2854
|
-
__decorateParam(5, common.Inject(FileScannerService))
|
|
3209
|
+
__decorateParam(5, common.Inject(FileScannerService)),
|
|
3210
|
+
__decorateParam(6, common.Optional()),
|
|
3211
|
+
__decorateParam(6, common.Inject(StorageObserver))
|
|
2855
3212
|
], exports.StorageService);
|
|
2856
3213
|
|
|
2857
3214
|
// src/server/utils/ttl-clamp.ts
|
|
@@ -2871,25 +3228,16 @@ function clampTtl(ttlSeconds, defaultTtl, maxTtl) {
|
|
|
2871
3228
|
// src/server/services/signed-url.service.ts
|
|
2872
3229
|
var MAX_MULTIPART_PARTS = 1e4;
|
|
2873
3230
|
exports.SignedUrlService = class SignedUrlService {
|
|
2874
|
-
constructor(options, s3Provider, keyResolver) {
|
|
3231
|
+
constructor(options, s3Provider, keyResolver, observer) {
|
|
2875
3232
|
this.options = options;
|
|
2876
3233
|
this.s3Provider = s3Provider;
|
|
2877
3234
|
this.keyResolver = keyResolver;
|
|
3235
|
+
this.observer = observer;
|
|
2878
3236
|
}
|
|
2879
3237
|
options;
|
|
2880
3238
|
s3Provider;
|
|
2881
3239
|
keyResolver;
|
|
2882
|
-
|
|
2883
|
-
* Issues a presigned GET URL for downloading an object.
|
|
2884
|
-
*
|
|
2885
|
-
* SECURITY: The returned URL is a temporary credential. NEVER log it, store it
|
|
2886
|
-
* in a database, or cache it across users. Issue a fresh URL per request.
|
|
2887
|
-
*
|
|
2888
|
-
* @param options - The download URL options.
|
|
2889
|
-
* @returns The presigned URL and its expiry metadata.
|
|
2890
|
-
* @throws StorageException `STORAGE_NOT_CONFIGURED` when the client is absent.
|
|
2891
|
-
* @throws StorageException `STORAGE_SIGNED_URL_TTL_INVALID` for non-positive TTL.
|
|
2892
|
-
*/
|
|
3240
|
+
observer;
|
|
2893
3241
|
async getDownloadUrl(options) {
|
|
2894
3242
|
this.assertConfigured();
|
|
2895
3243
|
assertHeaderValues({
|
|
@@ -2914,36 +3262,9 @@ exports.SignedUrlService = class SignedUrlService {
|
|
|
2914
3262
|
return { url, expiresAt: new Date(Date.now() + ttl * 1e3), method: "GET", requiredHeaders: {} };
|
|
2915
3263
|
} catch (err) {
|
|
2916
3264
|
if (err instanceof StorageException) throw err;
|
|
2917
|
-
throw mapAwsError(err, { key:
|
|
3265
|
+
throw mapAwsError(err, callDetails("getDownloadUrl", { key: options.key, bucket: options.bucket }));
|
|
2918
3266
|
}
|
|
2919
3267
|
}
|
|
2920
|
-
/**
|
|
2921
|
-
* Issues a presigned PUT URL for uploading an object. The client MUST send
|
|
2922
|
-
* every header listed in `requiredHeaders` verbatim — each one is part of the
|
|
2923
|
-
* URL signature, so an omitted header yields `SignatureDoesNotMatch`. The map
|
|
2924
|
-
* always carries `Content-Type` and additionally lists `x-amz-acl` when an ACL
|
|
2925
|
-
* is signed and an `x-amz-meta-<key>` entry per metadata pair. The `Content-Type`
|
|
2926
|
-
* is signed (`X-Amz-SignedHeaders` lists `content-type`), so a PUT that sends any
|
|
2927
|
-
* other type is refused by the provider and nothing is stored.
|
|
2928
|
-
*
|
|
2929
|
-
* SECURITY: The returned URL is a temporary credential — NEVER log it. A signed
|
|
2930
|
-
* PUT bypasses the server-side MIME/size validation pipeline. Size limits are
|
|
2931
|
-
* NOT enforced at presign time: a SigV4 PUT signature can only pin an exact
|
|
2932
|
-
* `Content-Length`, never a maximum, so binding one would reject valid smaller
|
|
2933
|
-
* uploads. Enforce size after the fact via a post-upload HEAD/size check plus
|
|
2934
|
-
* the post-upload scanner path for defence-in-depth.
|
|
2935
|
-
*
|
|
2936
|
-
* ACL caveat: `publicRead` emits `ACL: public-read`, which returns HTTP 400
|
|
2937
|
-
* `AccessControlListNotSupported` on modern AWS S3 buckets (Object Ownership
|
|
2938
|
-
* = "Bucket owner enforced") and is a no-op on Cloudflare R2. Prefer a bucket
|
|
2939
|
-
* policy, CDN, or signed GET URL for public delivery.
|
|
2940
|
-
*
|
|
2941
|
-
* @param options - The upload URL options.
|
|
2942
|
-
* @returns The presigned URL, its expiry metadata, and required headers.
|
|
2943
|
-
* @throws StorageException `STORAGE_NOT_CONFIGURED` when the client is absent.
|
|
2944
|
-
* @throws StorageException `STORAGE_SIGNED_URL_TTL_INVALID` for non-positive TTL.
|
|
2945
|
-
* @throws StorageException `STORAGE_MIME_NOT_ALLOWED` when `validation.mimeWhitelist` is set and excludes `contentType`.
|
|
2946
|
-
*/
|
|
2947
3268
|
async getUploadUrl(options) {
|
|
2948
3269
|
this.assertConfigured();
|
|
2949
3270
|
assertMimeAllowed(this.options.validation?.mimeWhitelist, options.contentType);
|
|
@@ -2977,7 +3298,7 @@ exports.SignedUrlService = class SignedUrlService {
|
|
|
2977
3298
|
};
|
|
2978
3299
|
} catch (err) {
|
|
2979
3300
|
if (err instanceof StorageException) throw err;
|
|
2980
|
-
throw mapAwsError(err, { key:
|
|
3301
|
+
throw mapAwsError(err, callDetails("getUploadUrl", { key: options.key, bucket: options.bucket }));
|
|
2981
3302
|
}
|
|
2982
3303
|
}
|
|
2983
3304
|
/**
|
|
@@ -2998,21 +3319,6 @@ exports.SignedUrlService = class SignedUrlService {
|
|
|
2998
3319
|
}
|
|
2999
3320
|
return headers;
|
|
3000
3321
|
}
|
|
3001
|
-
/**
|
|
3002
|
-
* Presigns a multipart upload: creates the upload session on S3, then returns
|
|
3003
|
-
* N presigned `UploadPart` URLs (1-indexed) and a presigned `CompleteMultipart`
|
|
3004
|
-
* URL. The client uploads parts directly to S3 and then calls the complete URL.
|
|
3005
|
-
*
|
|
3006
|
-
* SECURITY: All returned URLs are temporary credentials — NEVER log them.
|
|
3007
|
-
*
|
|
3008
|
-
* @param options - The multipart URL options.
|
|
3009
|
-
* @returns `uploadId`, `partUrls`, and `completeUrl`.
|
|
3010
|
-
* @throws StorageException `STORAGE_NOT_CONFIGURED` when the client is absent.
|
|
3011
|
-
* @throws StorageException `STORAGE_INVALID_PART_COUNT` when `parts` is not an integer from 1 to 10 000.
|
|
3012
|
-
* @throws StorageException `STORAGE_SIGNED_URL_TTL_INVALID` for non-positive TTL.
|
|
3013
|
-
* @throws StorageException `STORAGE_PROVIDER_ERROR` when no `UploadId` is returned.
|
|
3014
|
-
* @throws StorageException `STORAGE_MIME_NOT_ALLOWED` when `validation.mimeWhitelist` is set and excludes `contentType`.
|
|
3015
|
-
*/
|
|
3016
3322
|
async getMultipartUploadUrls(options) {
|
|
3017
3323
|
this.assertConfigured();
|
|
3018
3324
|
assertMimeAllowed(this.options.validation?.mimeWhitelist, options.contentType);
|
|
@@ -3042,14 +3348,16 @@ exports.SignedUrlService = class SignedUrlService {
|
|
|
3042
3348
|
if (!uploadId) {
|
|
3043
3349
|
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_PROVIDER_ERROR, void 0, {
|
|
3044
3350
|
reason: "CreateMultipartUpload returned no UploadId",
|
|
3045
|
-
key:
|
|
3046
|
-
bucket
|
|
3351
|
+
...callerFields({ key: options.key, bucket: options.bucket })
|
|
3047
3352
|
});
|
|
3048
3353
|
}
|
|
3049
3354
|
return await this.presignMultipartParts({ finalKey, bucket, uploadId, parts: options.parts, ttl });
|
|
3050
3355
|
} catch (err) {
|
|
3051
3356
|
if (err instanceof StorageException) throw err;
|
|
3052
|
-
throw mapAwsError(
|
|
3357
|
+
throw mapAwsError(
|
|
3358
|
+
err,
|
|
3359
|
+
callDetails("getMultipartUploadUrls", { key: options.key, bucket: options.bucket })
|
|
3360
|
+
);
|
|
3053
3361
|
}
|
|
3054
3362
|
}
|
|
3055
3363
|
/** Presigns all part URLs and the complete URL in parallel. */
|
|
@@ -3080,11 +3388,22 @@ exports.SignedUrlService = class SignedUrlService {
|
|
|
3080
3388
|
return resolveBucket(perCall, this.options.bucket);
|
|
3081
3389
|
}
|
|
3082
3390
|
};
|
|
3391
|
+
__decorateClass([
|
|
3392
|
+
Observed("get_download_url", { span: "storage.download_url" })
|
|
3393
|
+
], exports.SignedUrlService.prototype, "getDownloadUrl", 1);
|
|
3394
|
+
__decorateClass([
|
|
3395
|
+
Observed("get_upload_url")
|
|
3396
|
+
], exports.SignedUrlService.prototype, "getUploadUrl", 1);
|
|
3397
|
+
__decorateClass([
|
|
3398
|
+
Observed("get_multipart_upload_urls")
|
|
3399
|
+
], exports.SignedUrlService.prototype, "getMultipartUploadUrls", 1);
|
|
3083
3400
|
exports.SignedUrlService = __decorateClass([
|
|
3084
3401
|
common.Injectable(),
|
|
3085
3402
|
__decorateParam(0, common.Inject(BYMAX_STORAGE_OPTIONS)),
|
|
3086
3403
|
__decorateParam(1, common.Inject(S3ClientProvider)),
|
|
3087
|
-
__decorateParam(2, common.Inject(KeyResolverService))
|
|
3404
|
+
__decorateParam(2, common.Inject(KeyResolverService)),
|
|
3405
|
+
__decorateParam(3, common.Optional()),
|
|
3406
|
+
__decorateParam(3, common.Inject(StorageObserver))
|
|
3088
3407
|
], exports.SignedUrlService);
|
|
3089
3408
|
var IdempotencyCache = class {
|
|
3090
3409
|
/**
|
|
@@ -3194,9 +3513,12 @@ function resolveAsyncOptions(raw) {
|
|
|
3194
3513
|
const resolved = applyDefaults(options);
|
|
3195
3514
|
const configured = options.signedUrls?.maxTtlSeconds;
|
|
3196
3515
|
if (configured !== void 0 && configured > resolved.signedUrls.maxTtlSeconds) {
|
|
3197
|
-
new common.Logger(exports.BymaxStorageModule.name)
|
|
3198
|
-
|
|
3199
|
-
|
|
3516
|
+
emit(new common.Logger(exports.BymaxStorageModule.name), "warn", {
|
|
3517
|
+
event: STORAGE_LOG_EVENTS.ttlClamped,
|
|
3518
|
+
message: "signedUrls.maxTtlSeconds exceeds the SigV4 presigned-URL limit; the limit is used instead",
|
|
3519
|
+
configuredSeconds: configured,
|
|
3520
|
+
appliedSeconds: resolved.signedUrls.maxTtlSeconds
|
|
3521
|
+
});
|
|
3200
3522
|
}
|
|
3201
3523
|
return resolved;
|
|
3202
3524
|
}
|
|
@@ -3294,6 +3616,10 @@ exports.BymaxStorageModule = class BymaxStorageModule {
|
|
|
3294
3616
|
DEFAULT_IDEMPOTENCY_CACHE_TTL_MS
|
|
3295
3617
|
)
|
|
3296
3618
|
},
|
|
3619
|
+
// The counts and the span seam the services report to; the optional `observability` subpath reads the tally.
|
|
3620
|
+
exports.StorageMetricsTally,
|
|
3621
|
+
StorageTracing,
|
|
3622
|
+
StorageObserver,
|
|
3297
3623
|
exports.StorageService,
|
|
3298
3624
|
exports.SignedUrlService,
|
|
3299
3625
|
exports.BucketLifecycleService,
|
|
@@ -3320,6 +3646,7 @@ exports.BymaxStorageModule = class BymaxStorageModule {
|
|
|
3320
3646
|
exports.StorageService,
|
|
3321
3647
|
exports.SignedUrlService,
|
|
3322
3648
|
exports.BucketLifecycleService,
|
|
3649
|
+
exports.StorageMetricsTally,
|
|
3323
3650
|
BYMAX_STORAGE_OPTIONS,
|
|
3324
3651
|
BYMAX_STORAGE_S3_CLIENT,
|
|
3325
3652
|
BYMAX_STORAGE_UPLOAD_VALIDATORS,
|
|
@@ -3488,6 +3815,9 @@ exports.BYMAX_STORAGE_S3_CLIENT = BYMAX_STORAGE_S3_CLIENT;
|
|
|
3488
3815
|
exports.BYMAX_STORAGE_UPLOAD_VALIDATORS = BYMAX_STORAGE_UPLOAD_VALIDATORS;
|
|
3489
3816
|
exports.NoOpFileScanner = NoOpFileScanner;
|
|
3490
3817
|
exports.NoOpUploadValidator = NoOpUploadValidator;
|
|
3818
|
+
exports.SCAN_VERDICTS = SCAN_VERDICTS;
|
|
3491
3819
|
exports.STORAGE_ERROR_CODES = STORAGE_ERROR_CODES;
|
|
3820
|
+
exports.STORAGE_OPERATIONS = STORAGE_OPERATIONS;
|
|
3821
|
+
exports.STORAGE_OUTCOMES = STORAGE_OUTCOMES;
|
|
3492
3822
|
exports.StorageException = StorageException;
|
|
3493
3823
|
exports.providerRecipes = providerRecipes;
|