@bymax-one/nest-storage 1.1.17 → 1.1.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +169 -1
- package/README.md +172 -21
- package/dist/observability/index.cjs +151 -0
- package/dist/observability/index.d.cts +72 -0
- package/dist/observability/index.d.ts +72 -0
- package/dist/observability/index.mjs +145 -0
- package/dist/server/index.cjs +895 -420
- package/dist/server/index.d.cts +137 -11
- package/dist/server/index.d.ts +137 -11
- package/dist/server/index.mjs +895 -423
- package/package.json +32 -5
package/dist/server/index.cjs
CHANGED
|
@@ -7,15 +7,24 @@ var stream = require('stream');
|
|
|
7
7
|
var s3RequestPresigner = require('@aws-sdk/s3-request-presigner');
|
|
8
8
|
var crypto = require('crypto');
|
|
9
9
|
|
|
10
|
+
var __defProp = Object.defineProperty;
|
|
10
11
|
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
12
|
+
var __typeError = (msg) => {
|
|
13
|
+
throw TypeError(msg);
|
|
14
|
+
};
|
|
11
15
|
var __decorateClass = (decorators, target, key, kind) => {
|
|
12
16
|
var result = kind > 1 ? void 0 : kind ? __getOwnPropDesc(target, key) : target;
|
|
13
17
|
for (var i = decorators.length - 1, decorator; i >= 0; i--)
|
|
14
18
|
if (decorator = decorators[i])
|
|
15
|
-
result = (decorator(result)) || result;
|
|
19
|
+
result = (kind ? decorator(target, key, result) : decorator(result)) || result;
|
|
20
|
+
if (kind && result) __defProp(target, key, result);
|
|
16
21
|
return result;
|
|
17
22
|
};
|
|
18
23
|
var __decorateParam = (index, decorator) => (target, key) => decorator(target, key, index);
|
|
24
|
+
var __accessCheck = (obj, member, msg) => member.has(obj) || __typeError("Cannot " + msg);
|
|
25
|
+
var __privateGet = (obj, member, getter) => (__accessCheck(obj, member, "read from private field"), getter ? getter.call(obj) : member.get(obj));
|
|
26
|
+
var __privateAdd = (obj, member, value) => member.has(obj) ? __typeError("Cannot add the same private member more than once") : member instanceof WeakSet ? member.add(obj) : member.set(obj, value);
|
|
27
|
+
var __privateSet = (obj, member, value, setter) => (__accessCheck(obj, member, "write to private field"), member.set(obj, value), value);
|
|
19
28
|
|
|
20
29
|
// src/server/bymax-storage.constants.ts
|
|
21
30
|
var BYMAX_STORAGE_OPTIONS = /* @__PURE__ */ Symbol("BYMAX_STORAGE_OPTIONS");
|
|
@@ -144,9 +153,9 @@ var HEADER_VALUE = /^[\t \x21-\x7E]*$/;
|
|
|
144
153
|
var HEADER_NAME = /^[!#$%&'*+.^_`|~0-9a-z-]+$/;
|
|
145
154
|
var VALUE_REASON = "must be a string of tab, space and printable ASCII characters";
|
|
146
155
|
var KEY_REASON = "every key must be a non-empty header name of letters, digits and the characters !#$%&'*+-.^_`|~";
|
|
147
|
-
function refuse(
|
|
156
|
+
function refuse(field, reason) {
|
|
148
157
|
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_VALIDATION_FAILED, void 0, {
|
|
149
|
-
field
|
|
158
|
+
field,
|
|
150
159
|
reason
|
|
151
160
|
});
|
|
152
161
|
}
|
|
@@ -154,9 +163,9 @@ function isHeaderValue(value) {
|
|
|
154
163
|
return typeof value === "string" && HEADER_VALUE.test(value);
|
|
155
164
|
}
|
|
156
165
|
function assertHeaderValues(fields) {
|
|
157
|
-
for (const [
|
|
166
|
+
for (const [field, value] of Object.entries(fields)) {
|
|
158
167
|
if (value !== void 0 && !isHeaderValue(value)) {
|
|
159
|
-
refuse(
|
|
168
|
+
refuse(field, VALUE_REASON);
|
|
160
169
|
}
|
|
161
170
|
}
|
|
162
171
|
}
|
|
@@ -178,6 +187,7 @@ function assertMetadata(metadata) {
|
|
|
178
187
|
var MIN_PART_SIZE_BYTES = 5 * 1024 * 1024;
|
|
179
188
|
var MAX_PART_SIZE_BYTES = 5 * 1024 * 1024 * 1024;
|
|
180
189
|
var MAX_QUEUE_SIZE = 1e3;
|
|
190
|
+
var REGION_SHAPE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,62}$/;
|
|
181
191
|
function invalid(reason) {
|
|
182
192
|
return new StorageException(STORAGE_ERROR_CODES.STORAGE_INVALID_CONFIG, common.HttpStatus.INTERNAL_SERVER_ERROR, {
|
|
183
193
|
reason
|
|
@@ -228,7 +238,8 @@ function assertBoundedInteger(section, group, key, range) {
|
|
|
228
238
|
}
|
|
229
239
|
}
|
|
230
240
|
function assertScanner(scanner) {
|
|
231
|
-
if (scanner === void 0
|
|
241
|
+
if (scanner === void 0) return;
|
|
242
|
+
if (scanner === null) throw invalid("scanner must not be null; omit it for no scanner");
|
|
232
243
|
if (!isObject(scanner)) throw invalid("scanner must be an object");
|
|
233
244
|
const impl = scanner.impl;
|
|
234
245
|
if (!isObject(impl) || typeof impl.scan !== "function") {
|
|
@@ -242,6 +253,50 @@ function assertScanner(scanner) {
|
|
|
242
253
|
throw invalid("scanner.rejectOnUnknown must be a boolean");
|
|
243
254
|
}
|
|
244
255
|
}
|
|
256
|
+
function assertValidationKinds(validation) {
|
|
257
|
+
if (!isObject(validation)) return;
|
|
258
|
+
const { maxSizeBytes, mimeWhitelist, customValidators } = validation;
|
|
259
|
+
if (maxSizeBytes === null) {
|
|
260
|
+
throw invalid("validation.maxSizeBytes must not be null; omit it for no size limit");
|
|
261
|
+
}
|
|
262
|
+
if (mimeWhitelist === null) {
|
|
263
|
+
throw invalid("validation.mimeWhitelist must not be null; omit it to allow every type");
|
|
264
|
+
}
|
|
265
|
+
if (mimeWhitelist !== void 0 && !isStringArray(mimeWhitelist)) {
|
|
266
|
+
throw invalid("validation.mimeWhitelist must be an array of strings");
|
|
267
|
+
}
|
|
268
|
+
if (customValidators === null) {
|
|
269
|
+
throw invalid("validation.customValidators must not be null; omit it to run no custom validator");
|
|
270
|
+
}
|
|
271
|
+
if (customValidators !== void 0 && !Array.isArray(customValidators)) {
|
|
272
|
+
throw invalid("validation.customValidators must be an array");
|
|
273
|
+
}
|
|
274
|
+
if (customValidators !== void 0 && !isValidatorList(customValidators)) {
|
|
275
|
+
throw invalid("validation.customValidators must hold objects with a validate function");
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
function isStringArray(value) {
|
|
279
|
+
try {
|
|
280
|
+
return Array.isArray(value) && Array.from(value).every((entry) => typeof entry === "string");
|
|
281
|
+
} catch {
|
|
282
|
+
return false;
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
function isValidatorList(entries) {
|
|
286
|
+
try {
|
|
287
|
+
return Array.from(entries).every((entry) => isObject(entry) && typeof entry.validate === "function");
|
|
288
|
+
} catch {
|
|
289
|
+
return false;
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
function assertGroupKinds(options) {
|
|
293
|
+
for (const group of ["signedUrls", "multipart", "validation"]) {
|
|
294
|
+
const value = options[group];
|
|
295
|
+
if (value !== void 0 && value !== null && (!isObject(value) || Array.isArray(value))) {
|
|
296
|
+
throw invalid(`options.${group} must be an object`);
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
}
|
|
245
300
|
function assertNumericSections(options) {
|
|
246
301
|
const MAX = Number.MAX_SAFE_INTEGER;
|
|
247
302
|
assertBoundedInteger(options.validation, "validation", "maxSizeBytes", { min: 1, max: MAX });
|
|
@@ -258,7 +313,9 @@ function assertNumericSections(options) {
|
|
|
258
313
|
assertBoundedInteger(options.multipart, "multipart", "queueSize", { min: 1, max: MAX_QUEUE_SIZE });
|
|
259
314
|
}
|
|
260
315
|
function assertLimitsAndEncryption(options) {
|
|
316
|
+
assertGroupKinds(options);
|
|
261
317
|
assertNumericSections(options);
|
|
318
|
+
assertValidationKinds(options.validation);
|
|
262
319
|
assertScanner(options.scanner);
|
|
263
320
|
const maxAttempts = options.maxAttempts;
|
|
264
321
|
if (maxAttempts !== void 0 && (typeof maxAttempts !== "number" || !Number.isInteger(maxAttempts) || maxAttempts < 1 || maxAttempts > Number.MAX_SAFE_INTEGER)) {
|
|
@@ -292,12 +349,23 @@ function validateOptions(options) {
|
|
|
292
349
|
if (!isNonEmptyString(options.region)) {
|
|
293
350
|
throw invalid("options.region must be a non-empty string");
|
|
294
351
|
}
|
|
352
|
+
if (!REGION_SHAPE.test(options.region)) {
|
|
353
|
+
throw invalid(
|
|
354
|
+
"options.region must be 1 to 63 letters, digits, '.', '_' or '-', starting with a letter or digit"
|
|
355
|
+
);
|
|
356
|
+
}
|
|
295
357
|
if (!isNonEmptyString(options.bucket)) {
|
|
296
358
|
throw invalid("options.bucket must be a non-empty string");
|
|
297
359
|
}
|
|
360
|
+
if (options.keyPrefix !== void 0 && options.keyPrefix !== null && typeof options.keyPrefix !== "string") {
|
|
361
|
+
throw invalid("options.keyPrefix must be a string");
|
|
362
|
+
}
|
|
298
363
|
if (!isObject(options.credentials)) {
|
|
299
364
|
throw invalid("options.credentials is required");
|
|
300
365
|
}
|
|
366
|
+
if (Array.isArray(options.credentials)) {
|
|
367
|
+
throw invalid("options.credentials must be an object");
|
|
368
|
+
}
|
|
301
369
|
assertLimitsAndEncryption(options);
|
|
302
370
|
}
|
|
303
371
|
|
|
@@ -341,6 +409,41 @@ var DEFAULT_MULTIPART = {
|
|
|
341
409
|
var DEFAULT_IDEMPOTENCY_CACHE_MAX_ENTRIES = 1e3;
|
|
342
410
|
var DEFAULT_IDEMPOTENCY_CACHE_TTL_MS = 864e5;
|
|
343
411
|
|
|
412
|
+
// src/server/config/option-keys.ts
|
|
413
|
+
function keysOf() {
|
|
414
|
+
return (keys) => keys;
|
|
415
|
+
}
|
|
416
|
+
var OPTION_KEYS = keysOf()([
|
|
417
|
+
"endpoint",
|
|
418
|
+
"region",
|
|
419
|
+
"bucket",
|
|
420
|
+
"credentials",
|
|
421
|
+
"forcePathStyle",
|
|
422
|
+
"publicBaseUrl",
|
|
423
|
+
"cdnBaseUrl",
|
|
424
|
+
"defaultPublicRead",
|
|
425
|
+
"keyPrefix",
|
|
426
|
+
"defaultCacheControl",
|
|
427
|
+
"defaultContentDisposition",
|
|
428
|
+
"signedUrls",
|
|
429
|
+
"multipart",
|
|
430
|
+
"validation",
|
|
431
|
+
"scanner",
|
|
432
|
+
"serverSideEncryption",
|
|
433
|
+
"kmsKeyId",
|
|
434
|
+
"requestChecksumCalculation",
|
|
435
|
+
"responseChecksumValidation",
|
|
436
|
+
"maxAttempts",
|
|
437
|
+
"requestTimeoutMs",
|
|
438
|
+
"uploadIdleTimeoutMs",
|
|
439
|
+
"bodyIdleTimeoutMs"
|
|
440
|
+
]);
|
|
441
|
+
var CREDENTIAL_KEYS = keysOf()(["accessKeyId", "secretAccessKey", "sessionToken"]);
|
|
442
|
+
var SIGNED_URL_KEYS = keysOf()(["defaultGetTtlSeconds", "defaultPutTtlSeconds", "maxTtlSeconds"]);
|
|
443
|
+
var MULTIPART_KEYS = keysOf()(["thresholdBytes", "partSizeBytes", "queueSize"]);
|
|
444
|
+
var VALIDATION_KEYS = keysOf()(["mimeWhitelist", "maxSizeBytes", "customValidators"]);
|
|
445
|
+
var SCANNER_KEYS = keysOf()(["impl", "mode", "rejectOnUnknown", "timeoutMs"]);
|
|
446
|
+
|
|
344
447
|
// src/server/config/apply-defaults.ts
|
|
345
448
|
function definedOnly(group, keys) {
|
|
346
449
|
const picked = {};
|
|
@@ -350,10 +453,6 @@ function definedOnly(group, keys) {
|
|
|
350
453
|
}
|
|
351
454
|
return picked;
|
|
352
455
|
}
|
|
353
|
-
var SIGNED_URL_KEYS = ["defaultGetTtlSeconds", "defaultPutTtlSeconds", "maxTtlSeconds"];
|
|
354
|
-
var MULTIPART_KEYS = ["thresholdBytes", "partSizeBytes", "queueSize"];
|
|
355
|
-
var VALIDATION_KEYS = ["mimeWhitelist", "maxSizeBytes", "customValidators"];
|
|
356
|
-
var SCANNER_KEYS = ["impl", "mode", "rejectOnUnknown", "timeoutMs"];
|
|
357
456
|
function resolveSignedUrls(configured) {
|
|
358
457
|
const merged = { ...DEFAULT_SIGNED_URLS, ...definedOnly(configured, SIGNED_URL_KEYS) };
|
|
359
458
|
return { ...merged, maxTtlSeconds: Math.min(merged.maxTtlSeconds, MAX_SIGNED_URL_TTL_SECONDS) };
|
|
@@ -384,7 +483,7 @@ function applyDefaults(options) {
|
|
|
384
483
|
signedUrls: resolveSignedUrls(options.signedUrls),
|
|
385
484
|
multipart: { ...DEFAULT_MULTIPART, ...definedOnly(options.multipart, MULTIPART_KEYS) },
|
|
386
485
|
...options.validation !== void 0 ? { validation: definedOnly(options.validation, VALIDATION_KEYS) } : {},
|
|
387
|
-
//
|
|
486
|
+
// A group is an object or it is not configured. `forRoot` refuses a `null` scanner before this runs; if one reaches here it adds no scanner.
|
|
388
487
|
...options.scanner ? { scanner: definedOnly(options.scanner, SCANNER_KEYS) } : {},
|
|
389
488
|
...options.serverSideEncryption !== void 0 ? { serverSideEncryption: options.serverSideEncryption } : {},
|
|
390
489
|
...options.kmsKeyId !== void 0 ? { kmsKeyId: options.kmsKeyId } : {},
|
|
@@ -400,16 +499,81 @@ function applyDefaults(options) {
|
|
|
400
499
|
return resolved;
|
|
401
500
|
}
|
|
402
501
|
|
|
403
|
-
// src/server/
|
|
404
|
-
function
|
|
502
|
+
// src/server/config/snapshot-options.ts
|
|
503
|
+
function isObject2(value) {
|
|
504
|
+
return typeof value === "object" && value !== null;
|
|
505
|
+
}
|
|
506
|
+
function pick(source, keys) {
|
|
507
|
+
const copy = {};
|
|
508
|
+
for (const key of keys) {
|
|
509
|
+
const value = source[key];
|
|
510
|
+
if (value !== void 0) copy[key] = value;
|
|
511
|
+
}
|
|
512
|
+
return copy;
|
|
513
|
+
}
|
|
514
|
+
function pickGroup(value, keys) {
|
|
515
|
+
return isObject2(value) && !Array.isArray(value) ? pick(value, keys) : value;
|
|
516
|
+
}
|
|
517
|
+
function copyArray(value) {
|
|
518
|
+
if (!Array.isArray(value)) return value;
|
|
519
|
+
try {
|
|
520
|
+
return Array.from(value);
|
|
521
|
+
} catch {
|
|
522
|
+
return value;
|
|
523
|
+
}
|
|
524
|
+
}
|
|
525
|
+
var GROUPS = {
|
|
526
|
+
credentials: CREDENTIAL_KEYS,
|
|
527
|
+
signedUrls: SIGNED_URL_KEYS,
|
|
528
|
+
multipart: MULTIPART_KEYS,
|
|
529
|
+
validation: VALIDATION_KEYS,
|
|
530
|
+
scanner: SCANNER_KEYS
|
|
531
|
+
};
|
|
532
|
+
function snapshotOptions(options) {
|
|
533
|
+
if (!isObject2(options)) return options;
|
|
534
|
+
const copy = pick(options, OPTION_KEYS);
|
|
535
|
+
for (const [group, keys] of Object.entries(GROUPS)) {
|
|
536
|
+
if (group in copy) copy[group] = pickGroup(copy[group], keys);
|
|
537
|
+
}
|
|
538
|
+
const validation = copy.validation;
|
|
539
|
+
if (isObject2(validation)) {
|
|
540
|
+
for (const key of ["mimeWhitelist", "customValidators"]) {
|
|
541
|
+
if (key in validation) validation[key] = copyArray(validation[key]);
|
|
542
|
+
}
|
|
543
|
+
}
|
|
544
|
+
return copy;
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
// src/server/utils/absorb.ts
|
|
548
|
+
function absorb(call) {
|
|
405
549
|
try {
|
|
406
|
-
|
|
407
|
-
return `${url.protocol}//${url.host}`;
|
|
550
|
+
void Promise.resolve(call()).then(void 0, () => void 0);
|
|
408
551
|
} catch {
|
|
409
|
-
return "<unparseable endpoint>";
|
|
410
552
|
}
|
|
411
553
|
}
|
|
412
554
|
|
|
555
|
+
// src/server/utils/log-event.ts
|
|
556
|
+
function emit(sink, level, entry) {
|
|
557
|
+
absorb(() => sink[level](entry));
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
// src/server/utils/log-events.ts
|
|
561
|
+
var STORAGE_LOG_EVENTS = {
|
|
562
|
+
ttlClamped: "STORAGE_TTL_CLAMPED",
|
|
563
|
+
credentialsMissing: "STORAGE_CREDENTIALS_MISSING",
|
|
564
|
+
clientReady: "STORAGE_CLIENT_READY",
|
|
565
|
+
providerCallFailed: "STORAGE_PROVIDER_CALL_FAILED",
|
|
566
|
+
multipartAbortFailed: "STORAGE_MULTIPART_ABORT_FAILED",
|
|
567
|
+
scanInfected: "STORAGE_SCAN_INFECTED",
|
|
568
|
+
scanInconclusiveAccepted: "STORAGE_SCAN_INCONCLUSIVE_ACCEPTED",
|
|
569
|
+
scanInconclusiveRejected: "STORAGE_SCAN_INCONCLUSIVE_REJECTED",
|
|
570
|
+
scanFailed: "STORAGE_SCAN_FAILED",
|
|
571
|
+
scanTimeout: "STORAGE_SCAN_TIMEOUT",
|
|
572
|
+
scanCleanupFailed: "STORAGE_SCAN_CLEANUP_FAILED",
|
|
573
|
+
existsFailed: "STORAGE_EXISTS_FAILED",
|
|
574
|
+
deleteNotFound: "STORAGE_DELETE_NOT_FOUND"
|
|
575
|
+
};
|
|
576
|
+
|
|
413
577
|
// src/server/providers/s3-client.provider.ts
|
|
414
578
|
var S3ClientProvider = class {
|
|
415
579
|
constructor(options) {
|
|
@@ -420,9 +584,10 @@ var S3ClientProvider = class {
|
|
|
420
584
|
client;
|
|
421
585
|
onModuleInit() {
|
|
422
586
|
if (!this.options.hasCredentials) {
|
|
423
|
-
this.logger
|
|
424
|
-
|
|
425
|
-
|
|
587
|
+
emit(this.logger, "warn", {
|
|
588
|
+
event: STORAGE_LOG_EVENTS.credentialsMissing,
|
|
589
|
+
message: "Storage credentials are missing \u2014 module is registered but operations will throw STORAGE_NOT_CONFIGURED."
|
|
590
|
+
});
|
|
426
591
|
return;
|
|
427
592
|
}
|
|
428
593
|
this.resolveClient();
|
|
@@ -495,9 +660,10 @@ var S3ClientProvider = class {
|
|
|
495
660
|
requestChecksumCalculation: this.options.requestChecksumCalculation,
|
|
496
661
|
responseChecksumValidation: this.options.responseChecksumValidation
|
|
497
662
|
};
|
|
498
|
-
this.logger
|
|
499
|
-
|
|
500
|
-
|
|
663
|
+
emit(this.logger, "log", {
|
|
664
|
+
event: STORAGE_LOG_EVENTS.clientReady,
|
|
665
|
+
message: "S3Client initialized"
|
|
666
|
+
});
|
|
501
667
|
return new clientS3.S3Client(config);
|
|
502
668
|
}
|
|
503
669
|
};
|
|
@@ -566,62 +732,282 @@ KeyResolverService = __decorateClass([
|
|
|
566
732
|
__decorateParam(0, common.Inject(BYMAX_STORAGE_OPTIONS))
|
|
567
733
|
], KeyResolverService);
|
|
568
734
|
|
|
569
|
-
// src/server/
|
|
570
|
-
var
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
735
|
+
// src/server/observability/storage-operations.ts
|
|
736
|
+
var STORAGE_OPERATIONS = [
|
|
737
|
+
"upload",
|
|
738
|
+
"download",
|
|
739
|
+
"head",
|
|
740
|
+
"delete",
|
|
741
|
+
"delete_many",
|
|
742
|
+
"list",
|
|
743
|
+
"copy",
|
|
744
|
+
"get_download_url",
|
|
745
|
+
"get_upload_url",
|
|
746
|
+
"get_multipart_upload_urls",
|
|
747
|
+
"check_bucket",
|
|
748
|
+
"lifecycle_get",
|
|
749
|
+
"lifecycle_upsert",
|
|
750
|
+
"lifecycle_remove"
|
|
751
|
+
];
|
|
752
|
+
var STORAGE_OUTCOMES = ["success", "not_found", "aborted", "error"];
|
|
753
|
+
var SCAN_VERDICTS = ["clean", "infected", "unknown", "timeout", "error"];
|
|
754
|
+
function outcomeOf(error) {
|
|
755
|
+
if (!(error instanceof StorageException)) return "error";
|
|
756
|
+
if (error.code === STORAGE_ERROR_CODES.STORAGE_OBJECT_NOT_FOUND) return "not_found";
|
|
757
|
+
if (error.code === STORAGE_ERROR_CODES.STORAGE_REQUEST_ABORTED || error.code === STORAGE_ERROR_CODES.STORAGE_MULTIPART_ABORTED) {
|
|
758
|
+
return "aborted";
|
|
759
|
+
}
|
|
760
|
+
return "error";
|
|
761
|
+
}
|
|
762
|
+
function scanVerdictOfError(error) {
|
|
763
|
+
if (!(error instanceof StorageException)) return "error";
|
|
764
|
+
if (error.code === STORAGE_ERROR_CODES.STORAGE_SCAN_INFECTED) return "infected";
|
|
765
|
+
if (error.code === STORAGE_ERROR_CODES.STORAGE_SCAN_INCONCLUSIVE) return "unknown";
|
|
766
|
+
if (error.code === STORAGE_ERROR_CODES.STORAGE_TIMEOUT) return "timeout";
|
|
767
|
+
return "error";
|
|
768
|
+
}
|
|
769
|
+
|
|
770
|
+
// src/server/observability/storage-metrics-tally.ts
|
|
771
|
+
function zeroed(keys) {
|
|
772
|
+
return Object.fromEntries(keys.map((key) => [key, 0]));
|
|
773
|
+
}
|
|
774
|
+
var _operations, _scans, _listeners, _bytesUploaded;
|
|
775
|
+
exports.StorageMetricsTally = class StorageMetricsTally {
|
|
776
|
+
constructor() {
|
|
777
|
+
__privateAdd(this, _operations, Object.fromEntries(
|
|
778
|
+
STORAGE_OPERATIONS.map((operation) => [operation, zeroed(STORAGE_OUTCOMES)])
|
|
779
|
+
));
|
|
780
|
+
__privateAdd(this, _scans, zeroed(SCAN_VERDICTS));
|
|
781
|
+
__privateAdd(this, _listeners, []);
|
|
782
|
+
__privateAdd(this, _bytesUploaded, 0);
|
|
783
|
+
}
|
|
784
|
+
/** An operation ended as `outcome` after `seconds`. */
|
|
785
|
+
operation(operation, outcome, seconds) {
|
|
786
|
+
__privateGet(this, _operations)[operation][outcome] += 1;
|
|
787
|
+
for (const listener of __privateGet(this, _listeners)) absorb(() => listener(operation, seconds));
|
|
788
|
+
}
|
|
789
|
+
/** `bytes` of an object reached the provider; anything that is not a positive finite number is ignored. */
|
|
790
|
+
bytesUploaded(bytes) {
|
|
791
|
+
if (typeof bytes === "number" && Number.isFinite(bytes) && bytes > 0)
|
|
792
|
+
__privateSet(this, _bytesUploaded, __privateGet(this, _bytesUploaded) + bytes);
|
|
793
|
+
}
|
|
794
|
+
/** A scan ended as `verdict`. */
|
|
795
|
+
scan(verdict) {
|
|
796
|
+
__privateGet(this, _scans)[verdict] += 1;
|
|
797
|
+
}
|
|
798
|
+
/** Calls `listener` with the duration of every operation that ends from now on. */
|
|
799
|
+
onDuration(listener) {
|
|
800
|
+
__privateGet(this, _listeners).push(listener);
|
|
801
|
+
}
|
|
802
|
+
/** The counts so far, as copies. */
|
|
803
|
+
snapshot() {
|
|
804
|
+
return {
|
|
805
|
+
operations: Object.fromEntries(
|
|
806
|
+
STORAGE_OPERATIONS.map((operation) => [operation, { ...__privateGet(this, _operations)[operation] }])
|
|
807
|
+
),
|
|
808
|
+
bytesUploaded: __privateGet(this, _bytesUploaded),
|
|
809
|
+
scans: { ...__privateGet(this, _scans) }
|
|
810
|
+
};
|
|
811
|
+
}
|
|
812
|
+
};
|
|
813
|
+
_operations = new WeakMap();
|
|
814
|
+
_scans = new WeakMap();
|
|
815
|
+
_listeners = new WeakMap();
|
|
816
|
+
_bytesUploaded = new WeakMap();
|
|
817
|
+
exports.StorageMetricsTally = __decorateClass([
|
|
818
|
+
common.Injectable()
|
|
819
|
+
], exports.StorageMetricsTally);
|
|
820
|
+
var STATUS_ERROR = 2;
|
|
821
|
+
var TRACER_NAME = "@bymax-one/nest-storage";
|
|
822
|
+
var LIBRARY_CODE = /^STORAGE_[A-Z_]+$/;
|
|
823
|
+
function codeOf(error) {
|
|
824
|
+
if (!(error instanceof StorageException)) return "STORAGE_ERROR";
|
|
825
|
+
return LIBRARY_CODE.test(error.code) ? error.code : "STORAGE_ERROR";
|
|
826
|
+
}
|
|
827
|
+
var _api;
|
|
828
|
+
var StorageTracing = class {
|
|
829
|
+
constructor() {
|
|
830
|
+
__privateAdd(this, _api);
|
|
831
|
+
}
|
|
832
|
+
/** Loads the API once, at start-up; a package that is not there leaves the seam inert. */
|
|
833
|
+
async onModuleInit() {
|
|
834
|
+
try {
|
|
835
|
+
const loaded = await import('@opentelemetry/api');
|
|
836
|
+
__privateSet(this, _api, loaded);
|
|
837
|
+
} catch {
|
|
578
838
|
}
|
|
579
|
-
return String(value);
|
|
580
|
-
} catch {
|
|
581
|
-
return NON_PRINTABLE_REJECTION;
|
|
582
839
|
}
|
|
840
|
+
/** Runs the operation and reports it on the span, where nothing the span does can change what comes back. */
|
|
841
|
+
async traced(span, run, describe) {
|
|
842
|
+
let result;
|
|
843
|
+
try {
|
|
844
|
+
result = await run();
|
|
845
|
+
} catch (error) {
|
|
846
|
+
absorb(() => span.setAttributes({ "bymax.storage.outcome": outcomeOf(error) }));
|
|
847
|
+
absorb(() => span.setStatus({ code: STATUS_ERROR, message: codeOf(error) }));
|
|
848
|
+
absorb(() => span.end());
|
|
849
|
+
throw error;
|
|
850
|
+
}
|
|
851
|
+
absorb(() => span.setAttributes({ "bymax.storage.outcome": "success", ...describe(result) }));
|
|
852
|
+
absorb(() => span.end());
|
|
853
|
+
return result;
|
|
854
|
+
}
|
|
855
|
+
/**
|
|
856
|
+
* Runs `run` inside a span named `name`.
|
|
857
|
+
*
|
|
858
|
+
* The tracer belongs to the host's SDK, so it is never trusted with the operation: whatever it returns or
|
|
859
|
+
* throws is ignored, the operation runs exactly once, through the span when the tracer started it and bare
|
|
860
|
+
* otherwise, and its own result or error is what comes back.
|
|
861
|
+
*
|
|
862
|
+
* @param name - One of the library's span names.
|
|
863
|
+
* @param attributes - Closed-set attributes of the span.
|
|
864
|
+
* @param run - The operation; its result or its error passes through unchanged.
|
|
865
|
+
* @param describe - What the span states of a successful result; a throw of its own is the operation's own.
|
|
866
|
+
*/
|
|
867
|
+
span(name, attributes, run, describe = () => ({})) {
|
|
868
|
+
const api = __privateGet(this, _api);
|
|
869
|
+
if (!api) return run();
|
|
870
|
+
const latch = { claimed: false };
|
|
871
|
+
absorb(
|
|
872
|
+
() => api.trace.getTracer(TRACER_NAME).startActiveSpan(name, { attributes }, (span) => {
|
|
873
|
+
if (!latch.claimed) {
|
|
874
|
+
latch.claimed = true;
|
|
875
|
+
latch.started = this.traced(span, run, describe);
|
|
876
|
+
}
|
|
877
|
+
return latch.started;
|
|
878
|
+
})
|
|
879
|
+
);
|
|
880
|
+
if (latch.started) return latch.started;
|
|
881
|
+
latch.claimed = true;
|
|
882
|
+
return run();
|
|
883
|
+
}
|
|
884
|
+
};
|
|
885
|
+
_api = new WeakMap();
|
|
886
|
+
StorageTracing = __decorateClass([
|
|
887
|
+
common.Injectable()
|
|
888
|
+
], StorageTracing);
|
|
889
|
+
|
|
890
|
+
// src/server/observability/storage-observer.ts
|
|
891
|
+
function secondsSince(startedAt) {
|
|
892
|
+
return Number(process.hrtime.bigint() - startedAt) / 1e9;
|
|
893
|
+
}
|
|
894
|
+
var StorageObserver = class {
|
|
895
|
+
constructor(tally, tracing) {
|
|
896
|
+
this.tally = tally;
|
|
897
|
+
this.tracing = tracing;
|
|
898
|
+
}
|
|
899
|
+
tally;
|
|
900
|
+
tracing;
|
|
901
|
+
/**
|
|
902
|
+
* Runs `run`, counting how it ended and how long it took.
|
|
903
|
+
*
|
|
904
|
+
* @param operation - The operation, one of the closed set.
|
|
905
|
+
* @param run - The operation; its result or its error passes through unchanged.
|
|
906
|
+
* @param options - A span to start, and what to state of the result.
|
|
907
|
+
*/
|
|
908
|
+
observe(operation, run, options = {}) {
|
|
909
|
+
const counted = async () => {
|
|
910
|
+
const startedAt = process.hrtime.bigint();
|
|
911
|
+
let result;
|
|
912
|
+
try {
|
|
913
|
+
result = await run();
|
|
914
|
+
} catch (error) {
|
|
915
|
+
absorb(() => {
|
|
916
|
+
this.tally.operation(operation, outcomeOf(error), secondsSince(startedAt));
|
|
917
|
+
});
|
|
918
|
+
throw error;
|
|
919
|
+
}
|
|
920
|
+
absorb(() => {
|
|
921
|
+
this.tally.operation(operation, "success", secondsSince(startedAt));
|
|
922
|
+
this.tally.bytesUploaded(options.bytes?.(result));
|
|
923
|
+
});
|
|
924
|
+
return result;
|
|
925
|
+
};
|
|
926
|
+
return options.span === void 0 ? counted() : this.tracing.span(options.span, options.attributes ?? {}, counted, options.describe);
|
|
927
|
+
}
|
|
928
|
+
/**
|
|
929
|
+
* Runs a scan, counting its verdict and wrapping it in the `storage.scan` span.
|
|
930
|
+
*
|
|
931
|
+
* @param mode - When the scan runs, `pre-upload` or `post-upload`.
|
|
932
|
+
* @param run - The scan; its result or its error passes through unchanged.
|
|
933
|
+
*/
|
|
934
|
+
scan(mode, run) {
|
|
935
|
+
const counted = async () => {
|
|
936
|
+
try {
|
|
937
|
+
const result = await run();
|
|
938
|
+
absorb(() => {
|
|
939
|
+
this.tally.scan(result.status === "unknown" ? "unknown" : "clean");
|
|
940
|
+
});
|
|
941
|
+
return result;
|
|
942
|
+
} catch (error) {
|
|
943
|
+
absorb(() => {
|
|
944
|
+
this.tally.scan(scanVerdictOfError(error));
|
|
945
|
+
});
|
|
946
|
+
throw error;
|
|
947
|
+
}
|
|
948
|
+
};
|
|
949
|
+
return this.tracing.span("storage.scan", { "bymax.storage.scan_mode": mode }, counted);
|
|
950
|
+
}
|
|
951
|
+
};
|
|
952
|
+
StorageObserver = __decorateClass([
|
|
953
|
+
common.Injectable(),
|
|
954
|
+
__decorateParam(0, common.Inject(exports.StorageMetricsTally)),
|
|
955
|
+
__decorateParam(1, common.Inject(StorageTracing))
|
|
956
|
+
], StorageObserver);
|
|
957
|
+
function Observed(operation, options = {}) {
|
|
958
|
+
return (_target, _key, descriptor) => {
|
|
959
|
+
const original = descriptor.value;
|
|
960
|
+
descriptor.value = function(...args) {
|
|
961
|
+
const run = () => original.apply(this, args);
|
|
962
|
+
return this.observer === void 0 ? run() : this.observer.observe(operation, run, options);
|
|
963
|
+
};
|
|
964
|
+
};
|
|
583
965
|
}
|
|
584
966
|
|
|
585
|
-
// src/server/utils/
|
|
586
|
-
var
|
|
587
|
-
function
|
|
588
|
-
const
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
);
|
|
967
|
+
// src/server/utils/call-details.ts
|
|
968
|
+
var MAX_CALLER_TEXT_LENGTH = 1024;
|
|
969
|
+
function callerFields(given) {
|
|
970
|
+
const fields = {};
|
|
971
|
+
for (const [field, value] of Object.entries(given)) {
|
|
972
|
+
if (typeof value === "string" && value.length <= MAX_CALLER_TEXT_LENGTH) fields[field] = value;
|
|
973
|
+
}
|
|
974
|
+
return fields;
|
|
975
|
+
}
|
|
976
|
+
function callDetails(op, given, extra = {}) {
|
|
977
|
+
return { ...callerFields(given), op, ...extra };
|
|
597
978
|
}
|
|
598
979
|
|
|
599
|
-
// src/server/utils/
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
return typeof value === "string" || typeof value === "number" ? logValue(value) : "-";
|
|
980
|
+
// src/server/utils/call-input.ts
|
|
981
|
+
function wrongKind(field, needs) {
|
|
982
|
+
return new StorageException(STORAGE_ERROR_CODES.STORAGE_VALIDATION_FAILED, void 0, {
|
|
983
|
+
field,
|
|
984
|
+
reason: `${field} must be ${needs}`
|
|
985
|
+
});
|
|
606
986
|
}
|
|
607
|
-
function
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
].join(" ");
|
|
987
|
+
function resolveBucket(perCall, moduleDefault) {
|
|
988
|
+
if (perCall !== void 0 && perCall !== null && typeof perCall !== "string") {
|
|
989
|
+
throw wrongKind("bucket", "a string");
|
|
990
|
+
}
|
|
991
|
+
const bucket = perCall ?? moduleDefault;
|
|
992
|
+
if (!bucket) {
|
|
993
|
+
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_BUCKET_UNDEFINED);
|
|
994
|
+
}
|
|
995
|
+
return bucket;
|
|
617
996
|
}
|
|
618
|
-
function
|
|
619
|
-
if (
|
|
620
|
-
|
|
997
|
+
function assertAbortSignal(signal) {
|
|
998
|
+
if (signal === void 0 || signal === null) return;
|
|
999
|
+
if (!hasSignalSurface(signal)) {
|
|
1000
|
+
throw wrongKind("abortSignal", "an AbortSignal");
|
|
1001
|
+
}
|
|
1002
|
+
}
|
|
1003
|
+
function hasSignalSurface(signal) {
|
|
1004
|
+
if (typeof signal !== "object") return false;
|
|
1005
|
+
const candidate = signal;
|
|
1006
|
+
try {
|
|
1007
|
+
return typeof candidate.aborted === "boolean" && typeof candidate.addEventListener === "function";
|
|
1008
|
+
} catch {
|
|
1009
|
+
return false;
|
|
621
1010
|
}
|
|
622
|
-
const op = typeof context?.op === "string" ? logValue(context.op) : "-";
|
|
623
|
-
const key = typeof context?.key === "string" ? logValue(context.key) : "-";
|
|
624
|
-
logger.warn(`${label}: op=${op} key=${key} ${describeProviderFailure(err)}`);
|
|
625
1011
|
}
|
|
626
1012
|
var UNREACHABLE_CODES = /* @__PURE__ */ new Set([
|
|
627
1013
|
"ECONNREFUSED",
|
|
@@ -689,6 +1075,106 @@ function providerFailureStatus(err) {
|
|
|
689
1075
|
return failure === "credentials-refused" ? common.HttpStatus.INTERNAL_SERVER_ERROR : void 0;
|
|
690
1076
|
}
|
|
691
1077
|
|
|
1078
|
+
// src/server/utils/cut-text.ts
|
|
1079
|
+
function cutText(text, max) {
|
|
1080
|
+
if (text.length <= max) return text;
|
|
1081
|
+
const lastKept = text.charCodeAt(max - 1);
|
|
1082
|
+
const splitsPair = lastKept >= 55296 && lastKept <= 56319;
|
|
1083
|
+
return text.slice(0, splitsPair ? max - 1 : max);
|
|
1084
|
+
}
|
|
1085
|
+
|
|
1086
|
+
// src/server/utils/describe-error.ts
|
|
1087
|
+
var NON_PRINTABLE_REJECTION = "non-printable rejection value";
|
|
1088
|
+
function describeError(value) {
|
|
1089
|
+
try {
|
|
1090
|
+
if (typeof value === "object" && value !== null) {
|
|
1091
|
+
const message = value.message;
|
|
1092
|
+
if (typeof message === "string") {
|
|
1093
|
+
return message;
|
|
1094
|
+
}
|
|
1095
|
+
}
|
|
1096
|
+
return String(value);
|
|
1097
|
+
} catch {
|
|
1098
|
+
return NON_PRINTABLE_REJECTION;
|
|
1099
|
+
}
|
|
1100
|
+
}
|
|
1101
|
+
|
|
1102
|
+
// src/server/utils/log-value.ts
|
|
1103
|
+
var LEFT_LITERAL_BY_JSON = /[\p{Cc}\p{Cf}\p{Zl}\p{Zp}]/gu;
|
|
1104
|
+
var SHORT_ESCAPES = {
|
|
1105
|
+
"\b": "\\b",
|
|
1106
|
+
"\f": "\\f",
|
|
1107
|
+
"\n": "\\n",
|
|
1108
|
+
"\r": "\\r",
|
|
1109
|
+
" ": "\\t"
|
|
1110
|
+
};
|
|
1111
|
+
function escapeChar(char) {
|
|
1112
|
+
const short = SHORT_ESCAPES[char];
|
|
1113
|
+
if (short !== void 0) return short;
|
|
1114
|
+
const codePoint = Number(char.codePointAt(0));
|
|
1115
|
+
const hex = codePoint.toString(16);
|
|
1116
|
+
return codePoint > 65535 ? `\\u{${hex}}` : `\\u${hex.padStart(4, "0")}`;
|
|
1117
|
+
}
|
|
1118
|
+
function escapeForLog(text) {
|
|
1119
|
+
return text.replace(LEFT_LITERAL_BY_JSON, escapeChar);
|
|
1120
|
+
}
|
|
1121
|
+
|
|
1122
|
+
// src/server/utils/provider-failure-log.ts
|
|
1123
|
+
var SECRET_IN_MESSAGE = /(X-Amz-(?:Signature|Credential|Security-Token)|AWSAccessKeyId|Credential|Signature)=[^&\s"']+/gi;
|
|
1124
|
+
var BARE_ACCESS_KEY_ID = /\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/g;
|
|
1125
|
+
var MAX_MESSAGE_LENGTH = 300;
|
|
1126
|
+
var logger = new common.Logger("StorageProvider");
|
|
1127
|
+
var MAX_FIELD_LENGTH = 128;
|
|
1128
|
+
function readOrUndefined(read) {
|
|
1129
|
+
try {
|
|
1130
|
+
return read();
|
|
1131
|
+
} catch {
|
|
1132
|
+
return void 0;
|
|
1133
|
+
}
|
|
1134
|
+
}
|
|
1135
|
+
function textField(value) {
|
|
1136
|
+
return typeof value === "string" ? escapeForLog(cutText(value, MAX_FIELD_LENGTH)) : void 0;
|
|
1137
|
+
}
|
|
1138
|
+
function projectProviderFailure(err) {
|
|
1139
|
+
const failure = typeof err === "object" && err !== null ? err : {};
|
|
1140
|
+
const message = describeError(err).replace(SECRET_IN_MESSAGE, "$1=[redacted]").replace(BARE_ACCESS_KEY_ID, "[redacted-key-id]");
|
|
1141
|
+
const status = readOrUndefined(() => failure.$metadata?.httpStatusCode);
|
|
1142
|
+
const name = textField(readOrUndefined(() => failure.name));
|
|
1143
|
+
const awsCode = textField(readOrUndefined(() => failure.Code));
|
|
1144
|
+
const requestId = textField(readOrUndefined(() => failure.$metadata?.requestId));
|
|
1145
|
+
return {
|
|
1146
|
+
...name !== void 0 ? { name } : {},
|
|
1147
|
+
...awsCode !== void 0 ? { awsCode } : {},
|
|
1148
|
+
...typeof status === "number" ? { httpStatus: status } : {},
|
|
1149
|
+
...requestId !== void 0 ? { requestId } : {},
|
|
1150
|
+
message: escapeForLog(cutText(message, MAX_MESSAGE_LENGTH))
|
|
1151
|
+
};
|
|
1152
|
+
}
|
|
1153
|
+
function failureError(err) {
|
|
1154
|
+
const view = projectProviderFailure(err);
|
|
1155
|
+
const error = new Error(view.message);
|
|
1156
|
+
if (view.name !== void 0) error.name = view.name;
|
|
1157
|
+
if (view.awsCode !== void 0)
|
|
1158
|
+
Object.defineProperty(error, "awsCode", { value: view.awsCode, enumerable: true });
|
|
1159
|
+
return error;
|
|
1160
|
+
}
|
|
1161
|
+
function logProviderFailure(err, context, label = "Provider call failed") {
|
|
1162
|
+
if (readOrUndefined(() => err?.name) === "AbortError") {
|
|
1163
|
+
return;
|
|
1164
|
+
}
|
|
1165
|
+
const view = projectProviderFailure(err);
|
|
1166
|
+
emit(logger, "warn", {
|
|
1167
|
+
event: STORAGE_LOG_EVENTS.providerCallFailed,
|
|
1168
|
+
message: label,
|
|
1169
|
+
...typeof context?.op === "string" ? { operation: escapeForLog(context.op) } : {},
|
|
1170
|
+
...view.awsCode !== void 0 ? { awsCode: view.awsCode } : {},
|
|
1171
|
+
...view.httpStatus !== void 0 ? { httpStatus: view.httpStatus } : {},
|
|
1172
|
+
...view.requestId !== void 0 ? { requestId: view.requestId } : {},
|
|
1173
|
+
reason: readOrUndefined(() => providerFailureReason(err)) ?? "rejected",
|
|
1174
|
+
err: failureError(err)
|
|
1175
|
+
});
|
|
1176
|
+
}
|
|
1177
|
+
|
|
692
1178
|
// src/server/errors/aws-error-mapper.ts
|
|
693
1179
|
function awsCodeOf(err) {
|
|
694
1180
|
const e = err ?? {};
|
|
@@ -808,54 +1294,32 @@ function ruleInvalid(reason) {
|
|
|
808
1294
|
reason
|
|
809
1295
|
});
|
|
810
1296
|
}
|
|
811
|
-
function assertDays(
|
|
1297
|
+
function assertDays(field, value) {
|
|
812
1298
|
if (!Number.isInteger(value) || value < 1) {
|
|
813
|
-
throw ruleInvalid(`${
|
|
1299
|
+
throw ruleInvalid(`${field} must be an integer of at least 1`);
|
|
814
1300
|
}
|
|
815
1301
|
}
|
|
816
1302
|
exports.BucketLifecycleService = class BucketLifecycleService {
|
|
817
|
-
constructor(options, s3Provider, keyResolver) {
|
|
1303
|
+
constructor(options, s3Provider, keyResolver, observer) {
|
|
818
1304
|
this.options = options;
|
|
819
1305
|
this.s3Provider = s3Provider;
|
|
820
1306
|
this.keyResolver = keyResolver;
|
|
1307
|
+
this.observer = observer;
|
|
821
1308
|
}
|
|
822
1309
|
options;
|
|
823
1310
|
s3Provider;
|
|
824
1311
|
keyResolver;
|
|
825
|
-
|
|
826
|
-
* Returns the bucket's lifecycle rules. A bucket with no lifecycle configuration
|
|
827
|
-
* yields an empty array. Key prefixes inside the module's `keyPrefix` come back
|
|
828
|
-
* relative to it, as `list()` returns keys, so a prefix read here can be passed
|
|
829
|
-
* straight to `upsert()` without being prefixed twice. A rule that is not confined
|
|
830
|
-
* to `keyPrefix` is returned exactly as stored, flagged `isOutsideKeyPrefix: true`;
|
|
831
|
-
* everything else about a rule is returned as the provider stores it.
|
|
832
|
-
*
|
|
833
|
-
* @param options - Optional per-call bucket override.
|
|
834
|
-
* @returns Every rule in the bucket.
|
|
835
|
-
* @throws StorageException for missing configuration, an unsupported provider, or a provider error.
|
|
836
|
-
*/
|
|
1312
|
+
observer;
|
|
837
1313
|
async get(options) {
|
|
838
1314
|
this.assertConfigured();
|
|
839
1315
|
const bucket = this.resolveBucket(options?.bucket);
|
|
840
|
-
return (await this.read(bucket)).rules.map((rule) => this.toView(rule));
|
|
1316
|
+
return (await this.read(bucket, options?.bucket)).rules.map((rule) => this.toView(rule));
|
|
841
1317
|
}
|
|
842
|
-
/**
|
|
843
|
-
* Adds the expiration or multipart-abort rule, or replaces the rule with the same id, and writes
|
|
844
|
-
* the full set back. Rules with other ids are sent exactly as they were read.
|
|
845
|
-
*
|
|
846
|
-
* @param rule - The rule id, key prefix and either `expirationDays` or `abortIncompleteMultipartDays`.
|
|
847
|
-
* @param options - Optional per-call bucket override.
|
|
848
|
-
* @throws StorageException `STORAGE_LIFECYCLE_RULE_INVALID` for a bad id, prefix or day count, or an id
|
|
849
|
-
* that names a rule outside `keyPrefix`;
|
|
850
|
-
* `STORAGE_LIFECYCLE_UNSUPPORTED` / `STORAGE_LIFECYCLE_REJECTED` from the provider.
|
|
851
|
-
* @example
|
|
852
|
-
* await lifecycle.upsert({ id: 'expire-tmp', prefix: 'expiring/', expirationDays: 7 })
|
|
853
|
-
*/
|
|
854
1318
|
async upsert(rule, options) {
|
|
855
1319
|
this.assertConfigured();
|
|
856
1320
|
const owned = this.toRule(rule);
|
|
857
1321
|
const bucket = this.resolveBucket(options?.bucket);
|
|
858
|
-
const snapshot = await this.read(bucket);
|
|
1322
|
+
const snapshot = await this.read(bucket, options?.bucket);
|
|
859
1323
|
const index = snapshot.rules.findIndex((existing) => existing.ID === owned.ID);
|
|
860
1324
|
this.assertOwnable(snapshot.rules[index]);
|
|
861
1325
|
const rules = [...snapshot.rules];
|
|
@@ -864,30 +1328,20 @@ exports.BucketLifecycleService = class BucketLifecycleService {
|
|
|
864
1328
|
} else {
|
|
865
1329
|
rules[index] = owned;
|
|
866
1330
|
}
|
|
867
|
-
await this.write(bucket, rules, snapshot, "upsert");
|
|
1331
|
+
await this.write(bucket, options?.bucket, rules, snapshot, "upsert");
|
|
868
1332
|
}
|
|
869
|
-
/**
|
|
870
|
-
* Removes the rule with this id. When it was the last rule the configuration is
|
|
871
|
-
* deleted, because S3 rejects a write with no rules. An unknown id changes nothing.
|
|
872
|
-
*
|
|
873
|
-
* @param id - The rule id.
|
|
874
|
-
* @param options - Optional per-call bucket override.
|
|
875
|
-
* @returns `true` when a rule was removed, `false` when no rule had that id.
|
|
876
|
-
* @throws StorageException `STORAGE_LIFECYCLE_RULE_INVALID` for a bad id or an id that names a rule
|
|
877
|
-
* outside `keyPrefix`; a provider failure otherwise.
|
|
878
|
-
*/
|
|
879
1333
|
async remove(id, options) {
|
|
880
1334
|
this.assertConfigured();
|
|
881
1335
|
this.assertRuleId(id);
|
|
882
1336
|
const bucket = this.resolveBucket(options?.bucket);
|
|
883
|
-
const snapshot = await this.read(bucket);
|
|
1337
|
+
const snapshot = await this.read(bucket, options?.bucket);
|
|
884
1338
|
const target = snapshot.rules.find((existing) => existing.ID === id);
|
|
885
1339
|
if (target === void 0) {
|
|
886
1340
|
return false;
|
|
887
1341
|
}
|
|
888
1342
|
this.assertOwnable(target);
|
|
889
1343
|
const rules = snapshot.rules.filter((existing) => existing.ID !== id);
|
|
890
|
-
await this.write(bucket, rules, snapshot, "remove");
|
|
1344
|
+
await this.write(bucket, options?.bucket, rules, snapshot, "remove");
|
|
891
1345
|
return true;
|
|
892
1346
|
}
|
|
893
1347
|
/**
|
|
@@ -941,7 +1395,7 @@ exports.BucketLifecycleService = class BucketLifecycleService {
|
|
|
941
1395
|
};
|
|
942
1396
|
}
|
|
943
1397
|
/** Reads the configuration; "none yet" is an empty snapshot. */
|
|
944
|
-
async read(bucket) {
|
|
1398
|
+
async read(bucket, perCall) {
|
|
945
1399
|
try {
|
|
946
1400
|
const response = await this.s3Provider.getClient().send(new clientS3.GetBucketLifecycleConfigurationCommand({ Bucket: bucket }));
|
|
947
1401
|
return {
|
|
@@ -952,7 +1406,7 @@ exports.BucketLifecycleService = class BucketLifecycleService {
|
|
|
952
1406
|
if (isNoLifecycleConfiguration(err)) {
|
|
953
1407
|
return { rules: [], transitionDefaultMinimumObjectSize: void 0 };
|
|
954
1408
|
}
|
|
955
|
-
throw mapLifecycleError(err,
|
|
1409
|
+
throw mapLifecycleError(err, callDetails("lifecycle.get", { bucket: perCall }));
|
|
956
1410
|
}
|
|
957
1411
|
}
|
|
958
1412
|
/**
|
|
@@ -960,7 +1414,7 @@ exports.BucketLifecycleService = class BucketLifecycleService {
|
|
|
960
1414
|
* requires at least one rule on a put. The default-minimum-object-size setting
|
|
961
1415
|
* read earlier is sent back so a write does not silently change transition behaviour.
|
|
962
1416
|
*/
|
|
963
|
-
async write(bucket, rules, snapshot, op) {
|
|
1417
|
+
async write(bucket, perCall, rules, snapshot, op) {
|
|
964
1418
|
const client = this.s3Provider.getClient();
|
|
965
1419
|
try {
|
|
966
1420
|
if (rules.length === 0) {
|
|
@@ -975,7 +1429,7 @@ exports.BucketLifecycleService = class BucketLifecycleService {
|
|
|
975
1429
|
})
|
|
976
1430
|
);
|
|
977
1431
|
} catch (err) {
|
|
978
|
-
throw mapLifecycleError(err,
|
|
1432
|
+
throw mapLifecycleError(err, callDetails(`lifecycle.${op}`, { bucket: perCall }));
|
|
979
1433
|
}
|
|
980
1434
|
}
|
|
981
1435
|
/** Validates the input and builds the rule, an expiration or a multipart-abort rule. */
|
|
@@ -1022,18 +1476,25 @@ exports.BucketLifecycleService = class BucketLifecycleService {
|
|
|
1022
1476
|
}
|
|
1023
1477
|
}
|
|
1024
1478
|
resolveBucket(perCall) {
|
|
1025
|
-
|
|
1026
|
-
if (!bucket) {
|
|
1027
|
-
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_BUCKET_UNDEFINED);
|
|
1028
|
-
}
|
|
1029
|
-
return bucket;
|
|
1479
|
+
return resolveBucket(perCall, this.options.bucket);
|
|
1030
1480
|
}
|
|
1031
1481
|
};
|
|
1482
|
+
__decorateClass([
|
|
1483
|
+
Observed("lifecycle_get")
|
|
1484
|
+
], exports.BucketLifecycleService.prototype, "get", 1);
|
|
1485
|
+
__decorateClass([
|
|
1486
|
+
Observed("lifecycle_upsert")
|
|
1487
|
+
], exports.BucketLifecycleService.prototype, "upsert", 1);
|
|
1488
|
+
__decorateClass([
|
|
1489
|
+
Observed("lifecycle_remove")
|
|
1490
|
+
], exports.BucketLifecycleService.prototype, "remove", 1);
|
|
1032
1491
|
exports.BucketLifecycleService = __decorateClass([
|
|
1033
1492
|
common.Injectable(),
|
|
1034
1493
|
__decorateParam(0, common.Inject(BYMAX_STORAGE_OPTIONS)),
|
|
1035
1494
|
__decorateParam(1, common.Inject(S3ClientProvider)),
|
|
1036
|
-
__decorateParam(2, common.Inject(KeyResolverService))
|
|
1495
|
+
__decorateParam(2, common.Inject(KeyResolverService)),
|
|
1496
|
+
__decorateParam(3, common.Optional()),
|
|
1497
|
+
__decorateParam(3, common.Inject(StorageObserver))
|
|
1037
1498
|
], exports.BucketLifecycleService);
|
|
1038
1499
|
function isBufferLike(body) {
|
|
1039
1500
|
return body instanceof Uint8Array;
|
|
@@ -1146,7 +1607,7 @@ function toUploadBody(body) {
|
|
|
1146
1607
|
return Buffer.from(body.buffer, body.byteOffset, body.byteLength);
|
|
1147
1608
|
}
|
|
1148
1609
|
const stream = body;
|
|
1149
|
-
const readable = typeof stream?.pipe === "function" && typeof stream.on === "function" && typeof stream.once === "function";
|
|
1610
|
+
const readable = typeof stream?.pipe === "function" && typeof stream.on === "function" && typeof stream.once === "function" && typeof stream.pause === "function" && typeof stream.resume === "function";
|
|
1150
1611
|
if (!(body instanceof Uint8Array) && !readable) {
|
|
1151
1612
|
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_VALIDATION_FAILED, void 0, {
|
|
1152
1613
|
field: "body",
|
|
@@ -1187,18 +1648,17 @@ function assertMimeAllowed(whitelist, contentType) {
|
|
|
1187
1648
|
return;
|
|
1188
1649
|
}
|
|
1189
1650
|
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_MIME_NOT_ALLOWED, void 0, {
|
|
1190
|
-
...isHeaderValue(contentType) ? { contentType } : {}
|
|
1191
|
-
whitelist
|
|
1651
|
+
...isHeaderValue(contentType) ? { contentType } : {}
|
|
1192
1652
|
});
|
|
1193
1653
|
}
|
|
1194
1654
|
|
|
1195
1655
|
// src/server/utils/numeric-input.ts
|
|
1196
|
-
function assertIntegerInput(value,
|
|
1656
|
+
function assertIntegerInput(value, field, min) {
|
|
1197
1657
|
if (value === void 0) return;
|
|
1198
1658
|
if (!Number.isInteger(value) || value < min) {
|
|
1199
1659
|
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_VALIDATION_FAILED, void 0, {
|
|
1200
|
-
field
|
|
1201
|
-
reason: `${
|
|
1660
|
+
field,
|
|
1661
|
+
reason: `${field} must be an integer of at least ${String(min)}`
|
|
1202
1662
|
});
|
|
1203
1663
|
}
|
|
1204
1664
|
}
|
|
@@ -1303,31 +1763,36 @@ var MAX_SCANNER_LABEL_LENGTH = 200;
|
|
|
1303
1763
|
var MAX_SCANNER_DETAILS_LENGTH = 300;
|
|
1304
1764
|
var MAX_STATUS_LOG_LENGTH = 40;
|
|
1305
1765
|
function statusForLog(status) {
|
|
1306
|
-
return typeof status === "string" ?
|
|
1766
|
+
return typeof status === "string" ? escapeForLog(cutText(status, MAX_STATUS_LOG_LENGTH)) : `<${typeof status}>`;
|
|
1307
1767
|
}
|
|
1308
1768
|
function scannerLabel(value) {
|
|
1309
|
-
return typeof value === "string" ? value
|
|
1769
|
+
return typeof value === "string" ? cutText(value, MAX_SCANNER_LABEL_LENGTH) : void 0;
|
|
1770
|
+
}
|
|
1771
|
+
function labelForLog(label) {
|
|
1772
|
+
return label === void 0 ? void 0 : escapeForLog(label);
|
|
1310
1773
|
}
|
|
1311
1774
|
function toJson(value) {
|
|
1312
1775
|
return JSON.stringify(value);
|
|
1313
1776
|
}
|
|
1314
1777
|
function scannerDetailsForLog(details) {
|
|
1315
|
-
if (details === void 0) return
|
|
1778
|
+
if (details === void 0) return {};
|
|
1316
1779
|
let text;
|
|
1317
1780
|
try {
|
|
1318
1781
|
text = toJson(details) ?? UNSERIALISABLE;
|
|
1319
1782
|
} catch {
|
|
1320
1783
|
text = UNSERIALISABLE;
|
|
1321
1784
|
}
|
|
1322
|
-
return
|
|
1785
|
+
return { details: escapeForLog(cutText(text, MAX_SCANNER_DETAILS_LENGTH)) };
|
|
1323
1786
|
}
|
|
1324
1787
|
var FileScannerService = class {
|
|
1325
|
-
constructor(options, scanner) {
|
|
1788
|
+
constructor(options, scanner, observer) {
|
|
1326
1789
|
this.options = options;
|
|
1327
1790
|
this.scanner = scanner;
|
|
1791
|
+
this.observer = observer;
|
|
1328
1792
|
}
|
|
1329
1793
|
options;
|
|
1330
1794
|
scanner;
|
|
1795
|
+
observer;
|
|
1331
1796
|
logger = new common.Logger(FileScannerService.name);
|
|
1332
1797
|
/**
|
|
1333
1798
|
* Returns `true` when both a scanner implementation and scanner options are
|
|
@@ -1361,19 +1826,27 @@ var FileScannerService = class {
|
|
|
1361
1826
|
* says, and a scan that outlasts `scanner.timeoutMs` throws `STORAGE_TIMEOUT`.
|
|
1362
1827
|
*
|
|
1363
1828
|
* @param input - The scan input.
|
|
1829
|
+
* @param caller - What the deadline error repeats of the call (see `callerFields`): the caller's own key and
|
|
1830
|
+
* bucket, never the prefixed key or the configured bucket that `input` holds for the scanner.
|
|
1364
1831
|
* @returns The scan result when `status` is `'clean'` or `'unknown'` (non-rejecting).
|
|
1365
1832
|
* @throws Error when called without a configured scanner.
|
|
1366
1833
|
* @throws StorageException `STORAGE_SCAN_INFECTED` on an infected result.
|
|
1367
1834
|
* @throws StorageException `STORAGE_SCAN_INCONCLUSIVE` on an unknown result with `rejectOnUnknown`, or a result outside the contract.
|
|
1368
1835
|
* @throws StorageException `STORAGE_TIMEOUT` when the scan outlasts `scanner.timeoutMs`.
|
|
1369
1836
|
*/
|
|
1370
|
-
async scan(input) {
|
|
1837
|
+
async scan(input, caller = {}) {
|
|
1371
1838
|
if (!this.scanner) {
|
|
1372
1839
|
throw new Error(
|
|
1373
1840
|
"FileScannerService.scan() called without a configured scanner \u2014 guard with isEnabled() first."
|
|
1374
1841
|
);
|
|
1375
1842
|
}
|
|
1376
|
-
const
|
|
1843
|
+
const scanner = this.scanner;
|
|
1844
|
+
const run = () => this.evaluate(scanner, input, caller);
|
|
1845
|
+
return this.observer === void 0 ? run() : this.observer.scan(input.mode, run);
|
|
1846
|
+
}
|
|
1847
|
+
/** Runs the scanner and applies the library's policy to what it answered. */
|
|
1848
|
+
async evaluate(scanner, input, caller) {
|
|
1849
|
+
const result = await this.scanLogged(scanner, input, caller);
|
|
1377
1850
|
const status = result?.status;
|
|
1378
1851
|
const engine = scannerLabel(result?.engine);
|
|
1379
1852
|
const threat = scannerLabel(result?.threat);
|
|
@@ -1382,18 +1855,41 @@ var FileScannerService = class {
|
|
|
1382
1855
|
return result;
|
|
1383
1856
|
}
|
|
1384
1857
|
if (status === "infected") {
|
|
1385
|
-
this.logger
|
|
1858
|
+
emit(this.logger, "warn", {
|
|
1859
|
+
event: STORAGE_LOG_EVENTS.scanInfected,
|
|
1860
|
+
message: "Infected file detected",
|
|
1861
|
+
engine: labelForLog(engine),
|
|
1862
|
+
threat: escapeForLog(threat ?? "unknown"),
|
|
1863
|
+
...details
|
|
1864
|
+
});
|
|
1386
1865
|
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_SCAN_INFECTED, void 0, {
|
|
1387
1866
|
engine,
|
|
1388
1867
|
threat
|
|
1389
1868
|
});
|
|
1390
1869
|
}
|
|
1391
1870
|
if (status === "unknown" && !this.options.scanner?.rejectOnUnknown) {
|
|
1392
|
-
this.logger
|
|
1871
|
+
emit(this.logger, "warn", {
|
|
1872
|
+
event: STORAGE_LOG_EVENTS.scanInconclusiveAccepted,
|
|
1873
|
+
message: "Inconclusive scan result accepted",
|
|
1874
|
+
engine: labelForLog(engine)
|
|
1875
|
+
});
|
|
1393
1876
|
return result;
|
|
1394
1877
|
}
|
|
1878
|
+
return this.rejectInconclusive(status, engine, details);
|
|
1879
|
+
}
|
|
1880
|
+
/**
|
|
1881
|
+
* `unknown` with `rejectOnUnknown`, or a result outside the contract (no result, a status that is not
|
|
1882
|
+
* exactly one of the three): the object is not known to be clean, so it is not stored.
|
|
1883
|
+
*/
|
|
1884
|
+
rejectInconclusive(status, engine, details) {
|
|
1395
1885
|
const unrecognised = status !== "unknown";
|
|
1396
|
-
this.logger
|
|
1886
|
+
emit(this.logger, "warn", {
|
|
1887
|
+
event: STORAGE_LOG_EVENTS.scanInconclusiveRejected,
|
|
1888
|
+
message: "Inconclusive scan result rejected",
|
|
1889
|
+
engine: labelForLog(engine),
|
|
1890
|
+
...unrecognised ? { status: statusForLog(status) } : {},
|
|
1891
|
+
...details
|
|
1892
|
+
});
|
|
1397
1893
|
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_SCAN_INCONCLUSIVE, void 0, {
|
|
1398
1894
|
engine,
|
|
1399
1895
|
...unrecognised ? { reason: "unrecognised scanner status" } : {}
|
|
@@ -1403,14 +1899,16 @@ var FileScannerService = class {
|
|
|
1403
1899
|
* Runs the scan and logs a scanner that throws: the caller is answered 500 for that, and without the
|
|
1404
1900
|
* line there would be no trace of why. A typed failure of ours (the deadline) says it itself.
|
|
1405
1901
|
*/
|
|
1406
|
-
async scanLogged(scanner, input) {
|
|
1902
|
+
async scanLogged(scanner, input, caller) {
|
|
1407
1903
|
try {
|
|
1408
|
-
return await this.scanWithinDeadline(scanner, input);
|
|
1904
|
+
return await this.scanWithinDeadline(scanner, input, caller);
|
|
1409
1905
|
} catch (err) {
|
|
1410
1906
|
if (!(err instanceof StorageException)) {
|
|
1411
|
-
this.logger
|
|
1412
|
-
|
|
1413
|
-
|
|
1907
|
+
emit(this.logger, "warn", {
|
|
1908
|
+
event: STORAGE_LOG_EVENTS.scanFailed,
|
|
1909
|
+
message: "Scanner failed",
|
|
1910
|
+
err: failureError(err)
|
|
1911
|
+
});
|
|
1414
1912
|
}
|
|
1415
1913
|
throw err;
|
|
1416
1914
|
}
|
|
@@ -1420,16 +1918,20 @@ var FileScannerService = class {
|
|
|
1420
1918
|
* the upload for good. The scanner is handed a signal that aborts then; one that ignores it keeps
|
|
1421
1919
|
* running.
|
|
1422
1920
|
*/
|
|
1423
|
-
async scanWithinDeadline(scanner, input) {
|
|
1921
|
+
async scanWithinDeadline(scanner, input, caller) {
|
|
1424
1922
|
const timeoutMs = this.options.scanner?.timeoutMs ?? DEFAULT_SCANNER_TIMEOUT_MS;
|
|
1425
1923
|
const controller = new AbortController();
|
|
1426
1924
|
let timer;
|
|
1427
1925
|
const deadline = new Promise((_resolve, reject) => {
|
|
1428
1926
|
timer = setTimeout(() => {
|
|
1927
|
+
emit(this.logger, "warn", {
|
|
1928
|
+
event: STORAGE_LOG_EVENTS.scanTimeout,
|
|
1929
|
+
message: "Scan timed out",
|
|
1930
|
+
timeoutMs
|
|
1931
|
+
});
|
|
1429
1932
|
reject(
|
|
1430
1933
|
new StorageException(STORAGE_ERROR_CODES.STORAGE_TIMEOUT, void 0, {
|
|
1431
|
-
|
|
1432
|
-
bucket: input.bucket,
|
|
1934
|
+
...caller,
|
|
1433
1935
|
phase: "scan",
|
|
1434
1936
|
timeoutMs
|
|
1435
1937
|
})
|
|
@@ -1447,7 +1949,9 @@ var FileScannerService = class {
|
|
|
1447
1949
|
FileScannerService = __decorateClass([
|
|
1448
1950
|
common.Injectable(),
|
|
1449
1951
|
__decorateParam(0, common.Inject(BYMAX_STORAGE_OPTIONS)),
|
|
1450
|
-
__decorateParam(1, common.Inject(BYMAX_STORAGE_FILE_SCANNER))
|
|
1952
|
+
__decorateParam(1, common.Inject(BYMAX_STORAGE_FILE_SCANNER)),
|
|
1953
|
+
__decorateParam(2, common.Optional()),
|
|
1954
|
+
__decorateParam(2, common.Inject(StorageObserver))
|
|
1451
1955
|
], FileScannerService);
|
|
1452
1956
|
var SourceIdleError = class extends Error {
|
|
1453
1957
|
constructor(idleMs) {
|
|
@@ -1459,7 +1963,11 @@ var SourceIdleError = class extends Error {
|
|
|
1459
1963
|
};
|
|
1460
1964
|
function isError(value) {
|
|
1461
1965
|
const candidate = value;
|
|
1462
|
-
|
|
1966
|
+
try {
|
|
1967
|
+
return typeof candidate?.name === "string" && typeof candidate.message === "string";
|
|
1968
|
+
} catch {
|
|
1969
|
+
return false;
|
|
1970
|
+
}
|
|
1463
1971
|
}
|
|
1464
1972
|
function withinIdleTime(next, idleMs) {
|
|
1465
1973
|
return new Promise((resolve, reject) => {
|
|
@@ -1657,7 +2165,7 @@ function cleanupErrorCode(error) {
|
|
|
1657
2165
|
}
|
|
1658
2166
|
function describeCleanupFailure(error) {
|
|
1659
2167
|
if (!(error instanceof StorageException)) {
|
|
1660
|
-
return
|
|
2168
|
+
return projectProviderFailure(error).message;
|
|
1661
2169
|
}
|
|
1662
2170
|
return error.code;
|
|
1663
2171
|
}
|
|
@@ -1672,7 +2180,6 @@ function withCleanupFailure(scanError, keys, cleanupError) {
|
|
|
1672
2180
|
{
|
|
1673
2181
|
...body.error?.details ?? {},
|
|
1674
2182
|
key: keys.key,
|
|
1675
|
-
storedKey: keys.storedKey,
|
|
1676
2183
|
cleanup: "failed",
|
|
1677
2184
|
cleanupError: cleanupErrorCode(cleanupError)
|
|
1678
2185
|
}
|
|
@@ -1693,6 +2200,7 @@ function requestAborted(signal, context) {
|
|
|
1693
2200
|
});
|
|
1694
2201
|
}
|
|
1695
2202
|
function throwIfAborted(signal, context) {
|
|
2203
|
+
assertAbortSignal(signal);
|
|
1696
2204
|
if (signal?.aborted) {
|
|
1697
2205
|
throw requestAborted(signal, context);
|
|
1698
2206
|
}
|
|
@@ -1804,11 +2312,13 @@ async function closeMultipartUpload(client, bucket, key, uploadId, onFailure) {
|
|
|
1804
2312
|
function closeWhenCreated(client, bucket, key, created, onFailure) {
|
|
1805
2313
|
void created.then((uploadId) => closeMultipartUpload(client, bucket, key, uploadId, onFailure));
|
|
1806
2314
|
}
|
|
1807
|
-
function abortFailureLogger(logger2
|
|
2315
|
+
function abortFailureLogger(logger2) {
|
|
1808
2316
|
return (err) => {
|
|
1809
|
-
logger2
|
|
1810
|
-
|
|
1811
|
-
|
|
2317
|
+
emit(logger2, "warn", {
|
|
2318
|
+
event: STORAGE_LOG_EVENTS.multipartAbortFailed,
|
|
2319
|
+
message: "Multipart abort failed, the upload stays open on the provider",
|
|
2320
|
+
err: failureError(err)
|
|
2321
|
+
});
|
|
1812
2322
|
};
|
|
1813
2323
|
}
|
|
1814
2324
|
async function deleteObjectsChunk(client, bucket, chunk, stripPrefix) {
|
|
@@ -1924,6 +2434,23 @@ function buildACL(perCall, defaultValue) {
|
|
|
1924
2434
|
return perCall ?? defaultValue ? "public-read" : void 0;
|
|
1925
2435
|
}
|
|
1926
2436
|
|
|
2437
|
+
// src/server/utils/object-metadata.ts
|
|
2438
|
+
function toObjectMetadata(response, key, bucket) {
|
|
2439
|
+
return {
|
|
2440
|
+
key,
|
|
2441
|
+
bucket,
|
|
2442
|
+
size: response.ContentLength ?? 0,
|
|
2443
|
+
contentType: response.ContentType ?? "application/octet-stream",
|
|
2444
|
+
etag: response.ETag ?? "",
|
|
2445
|
+
lastModified: response.LastModified ?? /* @__PURE__ */ new Date(0),
|
|
2446
|
+
metadata: response.Metadata ?? {},
|
|
2447
|
+
...response.CacheControl !== void 0 ? { cacheControl: response.CacheControl } : {},
|
|
2448
|
+
...response.ContentDisposition !== void 0 ? { contentDisposition: response.ContentDisposition } : {},
|
|
2449
|
+
...response.StorageClass !== void 0 ? { storageClass: response.StorageClass } : {},
|
|
2450
|
+
...response.VersionId !== void 0 ? { versionId: response.VersionId } : {}
|
|
2451
|
+
};
|
|
2452
|
+
}
|
|
2453
|
+
|
|
1927
2454
|
// src/server/utils/upload-strategy.ts
|
|
1928
2455
|
function pickUploadStrategy(body, declaredSize, thresholdBytes) {
|
|
1929
2456
|
const size = declaredSize ?? getBodySize(body);
|
|
@@ -1952,18 +2479,18 @@ function boundStreamSize(body, declaredSize, options) {
|
|
|
1952
2479
|
}
|
|
1953
2480
|
return withSizeLimit(body, limit, declaredSize === void 0 ? "max" : "declared");
|
|
1954
2481
|
}
|
|
1955
|
-
async function bufferStreamForScan(body,
|
|
2482
|
+
async function bufferStreamForScan(body, context, maxSizeBytes) {
|
|
1956
2483
|
if (maxSizeBytes === void 0) {
|
|
1957
2484
|
destroyStream(body);
|
|
1958
2485
|
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_VALIDATION_FAILED, void 0, {
|
|
1959
|
-
|
|
2486
|
+
...context,
|
|
1960
2487
|
reason: "Pre-upload scanning of a stream needs validation.maxSizeBytes to bound the buffer; set it or use scanner mode post-upload"
|
|
1961
2488
|
});
|
|
1962
2489
|
}
|
|
1963
2490
|
const buffered = await readStreamUpTo(body, maxSizeBytes);
|
|
1964
2491
|
if (buffered === void 0) {
|
|
1965
2492
|
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_VALIDATION_FAILED, void 0, {
|
|
1966
|
-
|
|
2493
|
+
...context,
|
|
1967
2494
|
maxSizeBytes,
|
|
1968
2495
|
reason: "The stream is longer than validation.maxSizeBytes, so it cannot be scanned before upload"
|
|
1969
2496
|
});
|
|
@@ -2021,6 +2548,13 @@ function baseCarriesBucket(base, bucket) {
|
|
|
2021
2548
|
function normalizeScanBody(body) {
|
|
2022
2549
|
return body;
|
|
2023
2550
|
}
|
|
2551
|
+
function trackLoaded(uploader) {
|
|
2552
|
+
let loaded = 0;
|
|
2553
|
+
uploader.on("httpUploadProgress", (event) => {
|
|
2554
|
+
loaded = event.loaded ?? loaded;
|
|
2555
|
+
});
|
|
2556
|
+
return () => loaded;
|
|
2557
|
+
}
|
|
2024
2558
|
function forwardProgress(uploader, onProgress) {
|
|
2025
2559
|
if (!onProgress) return;
|
|
2026
2560
|
uploader.on("httpUploadProgress", (event) => {
|
|
@@ -2090,11 +2624,10 @@ function assertMaxBytes(maxBytes) {
|
|
|
2090
2624
|
});
|
|
2091
2625
|
}
|
|
2092
2626
|
}
|
|
2093
|
-
async function readBody(stream, metadata, maxBytes) {
|
|
2627
|
+
async function readBody(stream, metadata, maxBytes, context) {
|
|
2094
2628
|
if (maxBytes === void 0) {
|
|
2095
2629
|
return readStream(stream);
|
|
2096
2630
|
}
|
|
2097
|
-
const context = { key: metadata.key, bucket: metadata.bucket };
|
|
2098
2631
|
if (metadata.size > maxBytes) {
|
|
2099
2632
|
destroyStream(stream);
|
|
2100
2633
|
throw sizeLimitException(maxBytes, "max", { ...context, contentLength: metadata.size });
|
|
@@ -2105,15 +2638,50 @@ async function readBody(stream, metadata, maxBytes) {
|
|
|
2105
2638
|
}
|
|
2106
2639
|
return buffer;
|
|
2107
2640
|
}
|
|
2108
|
-
async function readDownloadBody(stream, metadata, maxBytes, signal) {
|
|
2641
|
+
async function readDownloadBody(stream, metadata, maxBytes, signal, caller) {
|
|
2642
|
+
const context = { ...caller, op: "downloadBuffer" };
|
|
2109
2643
|
try {
|
|
2110
|
-
return await readBody(stream, metadata, maxBytes);
|
|
2644
|
+
return await readBody(stream, metadata, maxBytes, context);
|
|
2111
2645
|
} catch (err) {
|
|
2112
|
-
throw mapBodyReadFailure(err, signal,
|
|
2113
|
-
|
|
2114
|
-
|
|
2115
|
-
|
|
2646
|
+
throw mapBodyReadFailure(err, signal, context);
|
|
2647
|
+
}
|
|
2648
|
+
}
|
|
2649
|
+
|
|
2650
|
+
// src/server/utils/uploaded-bytes.ts
|
|
2651
|
+
var SENT = /* @__PURE__ */ new WeakMap();
|
|
2652
|
+
function rememberUploadedBytes(result, bytes) {
|
|
2653
|
+
SENT.set(result, bytes);
|
|
2654
|
+
return result;
|
|
2655
|
+
}
|
|
2656
|
+
function uploadedBytesOf(result) {
|
|
2657
|
+
return result.fromIdempotencyCache ? void 0 : SENT.get(result) ?? result.size;
|
|
2658
|
+
}
|
|
2659
|
+
async function uploadSingleShot(context, options, finalKey, bucket, caller) {
|
|
2660
|
+
const total = options.size ?? getBodySize(options.body);
|
|
2661
|
+
const input = {
|
|
2662
|
+
...putObjectHeaders(options, context.options, finalKey, bucket),
|
|
2663
|
+
Body: normalizeSingleShotBody(options.body),
|
|
2664
|
+
ContentLength: total
|
|
2665
|
+
};
|
|
2666
|
+
const watch = abortOnBodyFailure(input.Body, options.abortSignal);
|
|
2667
|
+
try {
|
|
2668
|
+
const response = await context.s3Provider.getClient().send(new clientS3.PutObjectCommand(input), sendOptions(watch.signal));
|
|
2669
|
+
if (options.onProgress) {
|
|
2670
|
+
emitProgress(options.onProgress, total, total);
|
|
2671
|
+
}
|
|
2672
|
+
return buildUploadResult(context.keyResolver, context.options, {
|
|
2673
|
+
finalKey,
|
|
2674
|
+
bucket,
|
|
2675
|
+
etag: response.ETag,
|
|
2676
|
+
versionId: response.VersionId,
|
|
2677
|
+
size: total,
|
|
2678
|
+
contentType: options.contentType,
|
|
2679
|
+
multipart: false
|
|
2116
2680
|
});
|
|
2681
|
+
} catch (err) {
|
|
2682
|
+
const context2 = { ...caller, op: "upload-single" };
|
|
2683
|
+
const mapped = mapSizeLimitError(watch.failure(), context2) ?? mapAwsError(err, context2);
|
|
2684
|
+
throw failureOrAbort(options.abortSignal, context2, mapped);
|
|
2117
2685
|
}
|
|
2118
2686
|
}
|
|
2119
2687
|
|
|
@@ -2121,13 +2689,14 @@ async function readDownloadBody(stream, metadata, maxBytes, signal) {
|
|
|
2121
2689
|
var MAX_LIST_KEYS = 1e3;
|
|
2122
2690
|
var DELETE_BATCH_LIMIT = 1e3;
|
|
2123
2691
|
exports.StorageService = class StorageService {
|
|
2124
|
-
constructor(options, s3Provider, keyResolver, idempotencyCache, validation, scanner) {
|
|
2692
|
+
constructor(options, s3Provider, keyResolver, idempotencyCache, validation, scanner, observer) {
|
|
2125
2693
|
this.options = options;
|
|
2126
2694
|
this.s3Provider = s3Provider;
|
|
2127
2695
|
this.keyResolver = keyResolver;
|
|
2128
2696
|
this.idempotencyCache = idempotencyCache;
|
|
2129
2697
|
this.validation = validation;
|
|
2130
2698
|
this.scanner = scanner;
|
|
2699
|
+
this.observer = observer;
|
|
2131
2700
|
}
|
|
2132
2701
|
options;
|
|
2133
2702
|
s3Provider;
|
|
@@ -2135,22 +2704,8 @@ exports.StorageService = class StorageService {
|
|
|
2135
2704
|
idempotencyCache;
|
|
2136
2705
|
validation;
|
|
2137
2706
|
scanner;
|
|
2707
|
+
observer;
|
|
2138
2708
|
logger = new common.Logger(exports.StorageService.name);
|
|
2139
|
-
/**
|
|
2140
|
-
* Uploads an object, choosing single-shot or multipart automatically. When an
|
|
2141
|
-
* `idempotencyKey` is supplied and a matching result is cached within the TTL
|
|
2142
|
-
* window, the cached result is returned without re-uploading.
|
|
2143
|
-
*
|
|
2144
|
-
* The library takes ownership of a `Readable` body, as `stream.pipeline` does: when the
|
|
2145
|
-
* upload is rejected for any reason (configuration, validation, scan, provider failure)
|
|
2146
|
-
* the stream is destroyed (without an error: the SDK forwards a stream's error to wrappers
|
|
2147
|
-
* that nothing listens to), which releases the file handle or socket behind it; a cache hit
|
|
2148
|
-
* destroys it unread. A stream consumed by a successful upload ends on its own.
|
|
2149
|
-
*
|
|
2150
|
-
* @param options - The upload request.
|
|
2151
|
-
* @returns The upload result.
|
|
2152
|
-
* @throws StorageException for missing configuration, body, or content type.
|
|
2153
|
-
*/
|
|
2154
2709
|
async upload(options) {
|
|
2155
2710
|
try {
|
|
2156
2711
|
return await this.uploadOwningBody(options);
|
|
@@ -2173,7 +2728,7 @@ exports.StorageService = class StorageService {
|
|
|
2173
2728
|
}
|
|
2174
2729
|
const finalKey = this.keyResolver.normalize(options.key);
|
|
2175
2730
|
const bucket = this.resolveBucket(options.bucket);
|
|
2176
|
-
const context = { key:
|
|
2731
|
+
const context = callDetails("upload", { key: options.key, bucket: options.bucket });
|
|
2177
2732
|
throwIfAborted(options.abortSignal, context);
|
|
2178
2733
|
if (options.idempotencyKey) {
|
|
2179
2734
|
const cacheKey = this.idempotencyCache.computeKey(options.idempotencyKey, finalKey);
|
|
@@ -2191,7 +2746,7 @@ exports.StorageService = class StorageService {
|
|
|
2191
2746
|
validatedOptions.size,
|
|
2192
2747
|
this.options.multipart.thresholdBytes
|
|
2193
2748
|
);
|
|
2194
|
-
const result = strategy === "multipart" ? await this.uploadMultipart(validatedOptions, finalKey, bucket) : await this.
|
|
2749
|
+
const result = strategy === "multipart" ? await this.uploadMultipart(validatedOptions, finalKey, bucket, context) : await uploadSingleShot(this.uploadContext, validatedOptions, finalKey, bucket, context);
|
|
2195
2750
|
const stored = { ...context, stored: true };
|
|
2196
2751
|
await abortable(this.runPostUploadScan(finalKey, bucket, options), options.abortSignal, stored);
|
|
2197
2752
|
this.rememberUpload(options, finalKey, result);
|
|
@@ -2207,7 +2762,7 @@ exports.StorageService = class StorageService {
|
|
|
2207
2762
|
const input = validationInput(options, finalKey);
|
|
2208
2763
|
const validated = await abortable(this.validation.validate(input), signal, context);
|
|
2209
2764
|
const scanned = await abortable(
|
|
2210
|
-
this.bufferForPreUploadScan(validated.body,
|
|
2765
|
+
this.bufferForPreUploadScan(validated.body, context),
|
|
2211
2766
|
signal,
|
|
2212
2767
|
context
|
|
2213
2768
|
);
|
|
@@ -2224,28 +2779,31 @@ exports.StorageService = class StorageService {
|
|
|
2224
2779
|
* `validation.maxSizeBytes`, and a stream with no bound is refused rather than
|
|
2225
2780
|
* buffered without limit. Other bodies and modes pass through unchanged.
|
|
2226
2781
|
*/
|
|
2227
|
-
async bufferForPreUploadScan(body,
|
|
2782
|
+
async bufferForPreUploadScan(body, context) {
|
|
2228
2783
|
if (!this.scanner.isEnabled() || this.scanner.getMode() !== "pre-upload") {
|
|
2229
2784
|
return body;
|
|
2230
2785
|
}
|
|
2231
2786
|
if (isBufferLike(body)) {
|
|
2232
2787
|
return Buffer.from(body);
|
|
2233
2788
|
}
|
|
2234
|
-
return bufferStreamForScan(body,
|
|
2789
|
+
return bufferStreamForScan(body, context, this.options.validation?.maxSizeBytes);
|
|
2235
2790
|
}
|
|
2236
2791
|
/** Runs a pre-upload scan when configured; throws on infected. */
|
|
2237
2792
|
async runPreUploadScan(body, finalKey, bucket, options) {
|
|
2238
2793
|
if (!this.scanner.isEnabled() || this.scanner.getMode() !== "pre-upload") {
|
|
2239
2794
|
return;
|
|
2240
2795
|
}
|
|
2241
|
-
await this.scanner.scan(
|
|
2242
|
-
|
|
2243
|
-
|
|
2244
|
-
|
|
2245
|
-
|
|
2246
|
-
|
|
2247
|
-
|
|
2248
|
-
|
|
2796
|
+
await this.scanner.scan(
|
|
2797
|
+
{
|
|
2798
|
+
mode: "pre-upload",
|
|
2799
|
+
body: normalizeScanBody(body),
|
|
2800
|
+
key: finalKey,
|
|
2801
|
+
bucket,
|
|
2802
|
+
contentType: options.contentType,
|
|
2803
|
+
...options.size !== void 0 ? { size: options.size } : {}
|
|
2804
|
+
},
|
|
2805
|
+
callerFields({ key: options.key, bucket: options.bucket })
|
|
2806
|
+
);
|
|
2249
2807
|
}
|
|
2250
2808
|
/** Remembers the result of an upload under its idempotency key, when it has one. */
|
|
2251
2809
|
rememberUpload(options, finalKey, result) {
|
|
@@ -2264,47 +2822,44 @@ exports.StorageService = class StorageService {
|
|
|
2264
2822
|
return;
|
|
2265
2823
|
}
|
|
2266
2824
|
try {
|
|
2267
|
-
await this.scanner.scan(
|
|
2268
|
-
|
|
2269
|
-
|
|
2270
|
-
|
|
2271
|
-
|
|
2272
|
-
|
|
2273
|
-
|
|
2825
|
+
await this.scanner.scan(
|
|
2826
|
+
{
|
|
2827
|
+
mode: "post-upload",
|
|
2828
|
+
key: finalKey,
|
|
2829
|
+
bucket,
|
|
2830
|
+
contentType: options.contentType,
|
|
2831
|
+
...options.size !== void 0 ? { size: options.size } : {}
|
|
2832
|
+
},
|
|
2833
|
+
callerFields({ key: options.key, bucket: options.bucket })
|
|
2834
|
+
);
|
|
2274
2835
|
} catch (scanErr) {
|
|
2275
2836
|
const key = this.keyResolver.stripPrefix(finalKey);
|
|
2276
2837
|
try {
|
|
2277
2838
|
await this.delete(key, { bucket, abortSignal: AbortSignal.timeout(CLEANUP_DEADLINE_MS) });
|
|
2278
2839
|
} catch (deleteErr) {
|
|
2279
|
-
this.logger
|
|
2280
|
-
|
|
2281
|
-
|
|
2282
|
-
|
|
2283
|
-
|
|
2840
|
+
emit(this.logger, "error", {
|
|
2841
|
+
event: STORAGE_LOG_EVENTS.scanCleanupFailed,
|
|
2842
|
+
message: "The rejected object is still stored",
|
|
2843
|
+
storedKey: escapeForLog(finalKey),
|
|
2844
|
+
cleanupFailure: describeCleanupFailure(deleteErr)
|
|
2845
|
+
});
|
|
2846
|
+
throw withCleanupFailure(scanErr, { key }, deleteErr);
|
|
2284
2847
|
}
|
|
2285
2848
|
throw scanErr;
|
|
2286
2849
|
}
|
|
2287
2850
|
}
|
|
2288
|
-
/**
|
|
2289
|
-
* Returns metadata for an object without downloading its body.
|
|
2290
|
-
*
|
|
2291
|
-
* @param key - The raw object key (normalized internally).
|
|
2292
|
-
* @param options - Optional per-call bucket override.
|
|
2293
|
-
* @returns The object metadata.
|
|
2294
|
-
* @throws StorageException `STORAGE_OBJECT_NOT_FOUND` when the key is absent.
|
|
2295
|
-
*/
|
|
2296
2851
|
async head(key, options) {
|
|
2297
2852
|
this.assertConfigured();
|
|
2298
2853
|
const finalKey = this.keyResolver.normalize(key);
|
|
2299
2854
|
const bucket = this.resolveBucket(options?.bucket);
|
|
2300
|
-
const context = { key
|
|
2855
|
+
const context = callDetails("head", { key, bucket: options?.bucket });
|
|
2301
2856
|
throwIfAborted(options?.abortSignal, context);
|
|
2302
2857
|
try {
|
|
2303
2858
|
const response = await this.s3Provider.getClient().send(
|
|
2304
2859
|
new clientS3.HeadObjectCommand({ Bucket: bucket, Key: finalKey }),
|
|
2305
2860
|
sendOptions(options?.abortSignal)
|
|
2306
2861
|
);
|
|
2307
|
-
return
|
|
2862
|
+
return toObjectMetadata(response, this.keyResolver.stripPrefix(finalKey), bucket);
|
|
2308
2863
|
} catch (err) {
|
|
2309
2864
|
throw failureOrAbort(options?.abortSignal, context, mapAwsError(err, context));
|
|
2310
2865
|
}
|
|
@@ -2328,9 +2883,11 @@ exports.StorageService = class StorageService {
|
|
|
2328
2883
|
if (err instanceof StorageException && err.code === STORAGE_ERROR_CODES.STORAGE_REQUEST_ABORTED) {
|
|
2329
2884
|
throw err;
|
|
2330
2885
|
}
|
|
2331
|
-
this.logger
|
|
2332
|
-
|
|
2333
|
-
|
|
2886
|
+
emit(this.logger, "warn", {
|
|
2887
|
+
event: STORAGE_LOG_EVENTS.existsFailed,
|
|
2888
|
+
message: 'exists() \u2014 treating a non-404 error as "false"',
|
|
2889
|
+
err: failureError(err)
|
|
2890
|
+
});
|
|
2334
2891
|
return false;
|
|
2335
2892
|
}
|
|
2336
2893
|
}
|
|
@@ -2347,15 +2904,6 @@ exports.StorageService = class StorageService {
|
|
|
2347
2904
|
const bucket = this.resolveBucket(options?.bucket);
|
|
2348
2905
|
return buildPublicUrl(this.options, finalKey, bucket);
|
|
2349
2906
|
}
|
|
2350
|
-
/**
|
|
2351
|
-
* Streams an object. The returned `stream` is a Node `Readable` consumable via
|
|
2352
|
-
* `for await` or `.pipe()`. `range` / `ifNoneMatch` / `ifMatch` are forwarded as
|
|
2353
|
-
* conditional/partial GET headers.
|
|
2354
|
-
*
|
|
2355
|
-
* @param options - The download request.
|
|
2356
|
-
* @returns The object stream and its metadata.
|
|
2357
|
-
* @throws StorageException `STORAGE_OBJECT_NOT_FOUND` when the body is absent.
|
|
2358
|
-
*/
|
|
2359
2907
|
async download(options) {
|
|
2360
2908
|
this.assertConfigured();
|
|
2361
2909
|
assertHeaderValues({
|
|
@@ -2365,7 +2913,7 @@ exports.StorageService = class StorageService {
|
|
|
2365
2913
|
});
|
|
2366
2914
|
const finalKey = this.keyResolver.normalize(options.key);
|
|
2367
2915
|
const bucket = this.resolveBucket(options.bucket);
|
|
2368
|
-
const context = { key:
|
|
2916
|
+
const context = callDetails("download", { key: options.key, bucket: options.bucket });
|
|
2369
2917
|
throwIfAborted(options.abortSignal, context);
|
|
2370
2918
|
try {
|
|
2371
2919
|
const response = await this.s3Provider.getClient().send(
|
|
@@ -2379,10 +2927,7 @@ exports.StorageService = class StorageService {
|
|
|
2379
2927
|
sendOptions(options.abortSignal)
|
|
2380
2928
|
);
|
|
2381
2929
|
if (!response.Body) {
|
|
2382
|
-
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_OBJECT_NOT_FOUND, void 0,
|
|
2383
|
-
key: finalKey,
|
|
2384
|
-
bucket
|
|
2385
|
-
});
|
|
2930
|
+
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_OBJECT_NOT_FOUND, void 0, context);
|
|
2386
2931
|
}
|
|
2387
2932
|
const stream = guardDownloadBody(
|
|
2388
2933
|
response.Body,
|
|
@@ -2390,7 +2935,10 @@ exports.StorageService = class StorageService {
|
|
|
2390
2935
|
context,
|
|
2391
2936
|
options.abortSignal
|
|
2392
2937
|
);
|
|
2393
|
-
return {
|
|
2938
|
+
return {
|
|
2939
|
+
stream,
|
|
2940
|
+
metadata: toObjectMetadata(response, this.keyResolver.stripPrefix(finalKey), bucket)
|
|
2941
|
+
};
|
|
2394
2942
|
} catch (err) {
|
|
2395
2943
|
if (err instanceof StorageException) {
|
|
2396
2944
|
throw err;
|
|
@@ -2410,21 +2958,20 @@ exports.StorageService = class StorageService {
|
|
|
2410
2958
|
async downloadBuffer(options) {
|
|
2411
2959
|
assertMaxBytes(options.maxBytes);
|
|
2412
2960
|
const { stream, metadata } = await this.download(options);
|
|
2413
|
-
const buffer = await readDownloadBody(
|
|
2961
|
+
const buffer = await readDownloadBody(
|
|
2962
|
+
stream,
|
|
2963
|
+
metadata,
|
|
2964
|
+
options.maxBytes,
|
|
2965
|
+
options.abortSignal,
|
|
2966
|
+
callerFields({ key: options.key, bucket: options.bucket })
|
|
2967
|
+
);
|
|
2414
2968
|
return { buffer, metadata };
|
|
2415
2969
|
}
|
|
2416
|
-
/**
|
|
2417
|
-
* Deletes an object. Idempotent: a missing object is a no-op (logged as a
|
|
2418
|
-
* warning), not an error. Any other failure is propagated.
|
|
2419
|
-
*
|
|
2420
|
-
* @param key - The raw object key.
|
|
2421
|
-
* @param options - Optional per-call bucket override.
|
|
2422
|
-
*/
|
|
2423
2970
|
async delete(key, options) {
|
|
2424
2971
|
this.assertConfigured();
|
|
2425
2972
|
const finalKey = this.keyResolver.normalize(key);
|
|
2426
2973
|
const bucket = this.resolveBucket(options?.bucket);
|
|
2427
|
-
const context = { key
|
|
2974
|
+
const context = callDetails("delete", { key, bucket: options?.bucket });
|
|
2428
2975
|
throwIfAborted(options?.abortSignal, context);
|
|
2429
2976
|
try {
|
|
2430
2977
|
await this.s3Provider.getClient().send(
|
|
@@ -2436,25 +2983,17 @@ exports.StorageService = class StorageService {
|
|
|
2436
2983
|
if (mapped.code !== STORAGE_ERROR_CODES.STORAGE_OBJECT_NOT_FOUND) {
|
|
2437
2984
|
throw mapped;
|
|
2438
2985
|
}
|
|
2439
|
-
this.logger
|
|
2986
|
+
emit(this.logger, "warn", {
|
|
2987
|
+
event: STORAGE_LOG_EVENTS.deleteNotFound,
|
|
2988
|
+
message: "delete() \u2014 key not found (idempotent no-op)"
|
|
2989
|
+
});
|
|
2440
2990
|
}
|
|
2441
2991
|
this.idempotencyCache.invalidate(bucket, this.keyResolver.stripPrefix(finalKey));
|
|
2442
2992
|
}
|
|
2443
|
-
/**
|
|
2444
|
-
* Probes the bucket with one `HeadBucket`, for a readiness check: it resolves when the bucket
|
|
2445
|
-
* is reachable with these credentials and otherwise rejects with a typed code. Nothing in the
|
|
2446
|
-
* module calls it on its own, so booting costs no request. Needs `s3:ListBucket`.
|
|
2447
|
-
*
|
|
2448
|
-
* @param options - Optional per-call bucket override.
|
|
2449
|
-
* @throws StorageException `STORAGE_BUCKET_NOT_FOUND` (404), `STORAGE_BUCKET_ACCESS_DENIED` (403,
|
|
2450
|
-
* which a provider also answers for a bucket the caller cannot see), `STORAGE_BUCKET_REGION_MISMATCH`
|
|
2451
|
-
* (301), `STORAGE_TIMEOUT`, or `STORAGE_PROVIDER_ERROR`; `STORAGE_NOT_CONFIGURED` and
|
|
2452
|
-
* `STORAGE_BUCKET_UNDEFINED` before any request.
|
|
2453
|
-
*/
|
|
2454
2993
|
async checkBucket(options) {
|
|
2455
2994
|
this.assertConfigured();
|
|
2456
2995
|
const bucket = this.resolveBucket(options?.bucket);
|
|
2457
|
-
const context = { bucket
|
|
2996
|
+
const context = callDetails("checkBucket", { bucket: options?.bucket });
|
|
2458
2997
|
throwIfAborted(options?.abortSignal, context);
|
|
2459
2998
|
try {
|
|
2460
2999
|
await this.s3Provider.getClient().send(new clientS3.HeadBucketCommand({ Bucket: bucket }), sendOptions(options?.abortSignal));
|
|
@@ -2462,6 +3001,10 @@ exports.StorageService = class StorageService {
|
|
|
2462
3001
|
throw failureOrAbort(options?.abortSignal, context, mapBucketCheckError(err, context));
|
|
2463
3002
|
}
|
|
2464
3003
|
}
|
|
3004
|
+
/** What the single-shot upload needs of this service. */
|
|
3005
|
+
get uploadContext() {
|
|
3006
|
+
return { s3Provider: this.s3Provider, keyResolver: this.keyResolver, options: this.options };
|
|
3007
|
+
}
|
|
2465
3008
|
/** Throws when the S3 client was never built (missing credentials). */
|
|
2466
3009
|
assertConfigured() {
|
|
2467
3010
|
if (!this.s3Provider.isConfigured()) {
|
|
@@ -2470,56 +3013,7 @@ exports.StorageService = class StorageService {
|
|
|
2470
3013
|
}
|
|
2471
3014
|
/** Resolves the target bucket from a per-call override or the module default. */
|
|
2472
3015
|
resolveBucket(perCall) {
|
|
2473
|
-
|
|
2474
|
-
if (!bucket) {
|
|
2475
|
-
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_BUCKET_UNDEFINED);
|
|
2476
|
-
}
|
|
2477
|
-
return bucket;
|
|
2478
|
-
}
|
|
2479
|
-
/** Maps an S3 metadata response onto the public `ObjectMetadata` shape. */
|
|
2480
|
-
toObjectMetadata(response, finalKey, bucket) {
|
|
2481
|
-
return {
|
|
2482
|
-
key: this.keyResolver.stripPrefix(finalKey),
|
|
2483
|
-
bucket,
|
|
2484
|
-
size: response.ContentLength ?? 0,
|
|
2485
|
-
contentType: response.ContentType ?? "application/octet-stream",
|
|
2486
|
-
etag: response.ETag ?? "",
|
|
2487
|
-
lastModified: response.LastModified ?? /* @__PURE__ */ new Date(0),
|
|
2488
|
-
metadata: response.Metadata ?? {},
|
|
2489
|
-
...response.CacheControl !== void 0 ? { cacheControl: response.CacheControl } : {},
|
|
2490
|
-
...response.ContentDisposition !== void 0 ? { contentDisposition: response.ContentDisposition } : {},
|
|
2491
|
-
...response.StorageClass !== void 0 ? { storageClass: response.StorageClass } : {},
|
|
2492
|
-
...response.VersionId !== void 0 ? { versionId: response.VersionId } : {}
|
|
2493
|
-
};
|
|
2494
|
-
}
|
|
2495
|
-
/** Single `PutObject` upload for bodies below the multipart threshold. */
|
|
2496
|
-
async uploadSingleShot(options, finalKey, bucket) {
|
|
2497
|
-
const total = options.size ?? getBodySize(options.body);
|
|
2498
|
-
const input = {
|
|
2499
|
-
...putObjectHeaders(options, this.options, finalKey, bucket),
|
|
2500
|
-
Body: normalizeSingleShotBody(options.body),
|
|
2501
|
-
ContentLength: total
|
|
2502
|
-
};
|
|
2503
|
-
const watch = abortOnBodyFailure(input.Body, options.abortSignal);
|
|
2504
|
-
try {
|
|
2505
|
-
const response = await this.s3Provider.getClient().send(new clientS3.PutObjectCommand(input), sendOptions(watch.signal));
|
|
2506
|
-
if (options.onProgress) {
|
|
2507
|
-
emitProgress(options.onProgress, total, total);
|
|
2508
|
-
}
|
|
2509
|
-
return buildUploadResult(this.keyResolver, this.options, {
|
|
2510
|
-
finalKey,
|
|
2511
|
-
bucket,
|
|
2512
|
-
etag: response.ETag,
|
|
2513
|
-
versionId: response.VersionId,
|
|
2514
|
-
size: total,
|
|
2515
|
-
contentType: options.contentType,
|
|
2516
|
-
multipart: false
|
|
2517
|
-
});
|
|
2518
|
-
} catch (err) {
|
|
2519
|
-
const context = { key: finalKey, bucket, op: "upload-single" };
|
|
2520
|
-
const mapped = mapSizeLimitError(watch.failure(), context) ?? mapAwsError(err, context);
|
|
2521
|
-
throw failureOrAbort(options.abortSignal, context, mapped);
|
|
2522
|
-
}
|
|
3016
|
+
return resolveBucket(perCall, this.options.bucket);
|
|
2523
3017
|
}
|
|
2524
3018
|
/**
|
|
2525
3019
|
* Multipart upload via `@aws-sdk/lib-storage`. The SDK is told to leave the parts
|
|
@@ -2527,7 +3021,7 @@ exports.StorageService = class StorageService {
|
|
|
2527
3021
|
* abort replaced the failure that caused it, and a failed `CompleteMultipartUpload` was never
|
|
2528
3022
|
* aborted. This method aborts the upload itself and reports the outcome in the error details.
|
|
2529
3023
|
*/
|
|
2530
|
-
async uploadMultipart(options, finalKey, bucket) {
|
|
3024
|
+
async uploadMultipart(options, finalKey, bucket, caller) {
|
|
2531
3025
|
const params = {
|
|
2532
3026
|
...putObjectHeaders(options, this.options, finalKey, bucket),
|
|
2533
3027
|
Body: guardIdleBody(options.body, this.options.uploadIdleTimeoutMs)
|
|
@@ -2541,6 +3035,7 @@ exports.StorageService = class StorageService {
|
|
|
2541
3035
|
partSize: this.options.multipart.partSizeBytes,
|
|
2542
3036
|
leavePartsOnError: true
|
|
2543
3037
|
});
|
|
3038
|
+
const sent = trackLoaded(uploader);
|
|
2544
3039
|
forwardProgress(uploader, options.onProgress);
|
|
2545
3040
|
const abortUpload = () => {
|
|
2546
3041
|
void uploader.abort();
|
|
@@ -2548,17 +3043,20 @@ exports.StorageService = class StorageService {
|
|
|
2548
3043
|
signal?.addEventListener("abort", abortUpload, { once: true });
|
|
2549
3044
|
try {
|
|
2550
3045
|
const response = await uploader.done();
|
|
2551
|
-
return
|
|
2552
|
-
|
|
2553
|
-
|
|
2554
|
-
|
|
2555
|
-
|
|
2556
|
-
|
|
2557
|
-
|
|
2558
|
-
|
|
2559
|
-
|
|
3046
|
+
return rememberUploadedBytes(
|
|
3047
|
+
buildUploadResult(this.keyResolver, this.options, {
|
|
3048
|
+
finalKey,
|
|
3049
|
+
bucket,
|
|
3050
|
+
etag: response.ETag,
|
|
3051
|
+
versionId: response.VersionId,
|
|
3052
|
+
size: options.size,
|
|
3053
|
+
contentType: options.contentType,
|
|
3054
|
+
multipart: true
|
|
3055
|
+
}),
|
|
3056
|
+
sent()
|
|
3057
|
+
);
|
|
2560
3058
|
} catch (err) {
|
|
2561
|
-
return await this.failMultipart(err, finalKey, bucket, {
|
|
3059
|
+
return await this.failMultipart(err, finalKey, bucket, caller, {
|
|
2562
3060
|
uploadId: uploader.uploadId,
|
|
2563
3061
|
signal,
|
|
2564
3062
|
createdUploadId: bound?.createdUploadId
|
|
@@ -2573,7 +3071,7 @@ exports.StorageService = class StorageService {
|
|
|
2573
3071
|
* real failure into one. An upload id the provider answered late is waited for, within the cleanup
|
|
2574
3072
|
* deadline, so that an abort during `CreateMultipartUpload` still aborts the upload it created.
|
|
2575
3073
|
*/
|
|
2576
|
-
async failMultipart(err, finalKey, bucket, upload) {
|
|
3074
|
+
async failMultipart(err, finalKey, bucket, caller, upload) {
|
|
2577
3075
|
const abortedBy = upload.signal?.aborted === true ? upload.signal : void 0;
|
|
2578
3076
|
const client = this.s3Provider.getClient();
|
|
2579
3077
|
const lookup = abortedBy ? upload.createdUploadId?.() : void 0;
|
|
@@ -2581,7 +3079,7 @@ exports.StorageService = class StorageService {
|
|
|
2581
3079
|
if (uploadId === void 0 && lookup) {
|
|
2582
3080
|
uploadId = await within(lookup, CLEANUP_DEADLINE_MS);
|
|
2583
3081
|
if (uploadId === void 0) {
|
|
2584
|
-
closeWhenCreated(client, bucket, finalKey, lookup, abortFailureLogger(this.logger
|
|
3082
|
+
closeWhenCreated(client, bucket, finalKey, lookup, abortFailureLogger(this.logger));
|
|
2585
3083
|
}
|
|
2586
3084
|
}
|
|
2587
3085
|
const abort = await closeMultipartUpload(
|
|
@@ -2589,27 +3087,18 @@ exports.StorageService = class StorageService {
|
|
|
2589
3087
|
bucket,
|
|
2590
3088
|
finalKey,
|
|
2591
3089
|
uploadId,
|
|
2592
|
-
abortFailureLogger(this.logger
|
|
3090
|
+
abortFailureLogger(this.logger)
|
|
2593
3091
|
);
|
|
2594
|
-
const context = {
|
|
3092
|
+
const context = { ...caller, op: "upload-multipart", ...abort };
|
|
2595
3093
|
throw abortedBy ? requestAborted(abortedBy, context) : mapMultipartError(err, context);
|
|
2596
3094
|
}
|
|
2597
|
-
/**
|
|
2598
|
-
* Lists objects in the bucket as a single page. Applies the global key prefix
|
|
2599
|
-
* to the caller's `prefix`, caps `maxKeys` at the S3 hard limit of 1000, threads
|
|
2600
|
-
* the continuation token, and strips the global prefix from every returned key.
|
|
2601
|
-
*
|
|
2602
|
-
* @param options - The listing request.
|
|
2603
|
-
* @returns One page of objects plus `commonPrefixes`, truncation, and the next token.
|
|
2604
|
-
* @throws StorageException mapped from any AWS failure.
|
|
2605
|
-
*/
|
|
2606
3095
|
async list(options) {
|
|
2607
3096
|
this.assertConfigured();
|
|
2608
3097
|
const bucket = this.resolveBucket(options.bucket);
|
|
2609
3098
|
assertIntegerInput(options.maxKeys, "maxKeys", 1);
|
|
2610
3099
|
const maxKeys = Math.min(options.maxKeys ?? MAX_LIST_KEYS, MAX_LIST_KEYS);
|
|
2611
3100
|
const prefix = options.prefix ? this.keyResolver.normalize(options.prefix) : this.keyResolver.getPrefix();
|
|
2612
|
-
const context = { bucket, prefix
|
|
3101
|
+
const context = callDetails("list", { bucket: options.bucket, prefix: options.prefix });
|
|
2613
3102
|
throwIfAborted(options.abortSignal, context);
|
|
2614
3103
|
try {
|
|
2615
3104
|
const response = await this.s3Provider.getClient().send(
|
|
@@ -2627,14 +3116,6 @@ exports.StorageService = class StorageService {
|
|
|
2627
3116
|
throw failureOrAbort(options.abortSignal, context, mapAwsError(err, context));
|
|
2628
3117
|
}
|
|
2629
3118
|
}
|
|
2630
|
-
/**
|
|
2631
|
-
* Server-side copy of an object (no bytes flow through the app). Supports both
|
|
2632
|
-
* same-bucket and cross-bucket copies and preserves the source metadata.
|
|
2633
|
-
*
|
|
2634
|
-
* @param options - Source/destination keys and buckets plus copy overrides.
|
|
2635
|
-
* @returns The new object's ETag.
|
|
2636
|
-
* @throws StorageException mapped from any AWS failure.
|
|
2637
|
-
*/
|
|
2638
3119
|
async copy(options) {
|
|
2639
3120
|
this.assertConfigured();
|
|
2640
3121
|
assertHeaderValues({ cacheControl: options.cacheControl });
|
|
@@ -2658,19 +3139,12 @@ exports.StorageService = class StorageService {
|
|
|
2658
3139
|
);
|
|
2659
3140
|
return { etag: response.CopyObjectResult?.ETag ?? "" };
|
|
2660
3141
|
} catch (err) {
|
|
2661
|
-
throw mapAwsError(
|
|
3142
|
+
throw mapAwsError(
|
|
3143
|
+
err,
|
|
3144
|
+
callDetails("copy", { sourceKey: options.sourceKey, destKey: options.destinationKey })
|
|
3145
|
+
);
|
|
2662
3146
|
}
|
|
2663
3147
|
}
|
|
2664
|
-
/**
|
|
2665
|
-
* Deletes many objects, chunking the input at the S3 hard limit of 1000 keys
|
|
2666
|
-
* per request. Reports both successes and per-key failures; a whole-chunk send
|
|
2667
|
-
* failure marks every key in that chunk as failed. Returned keys are stripped of
|
|
2668
|
-
* the global prefix.
|
|
2669
|
-
*
|
|
2670
|
-
* @param keys - The raw object keys to delete.
|
|
2671
|
-
* @param options - Optional per-call bucket override.
|
|
2672
|
-
* @returns The deleted keys and the per-key failures.
|
|
2673
|
-
*/
|
|
2674
3148
|
async deleteMany(keys, options) {
|
|
2675
3149
|
this.assertConfigured();
|
|
2676
3150
|
if (keys.length === 0) {
|
|
@@ -2697,6 +3171,34 @@ exports.StorageService = class StorageService {
|
|
|
2697
3171
|
return { deleted, failed };
|
|
2698
3172
|
}
|
|
2699
3173
|
};
|
|
3174
|
+
__decorateClass([
|
|
3175
|
+
Observed("upload", {
|
|
3176
|
+
span: "storage.upload",
|
|
3177
|
+
describe: (result) => ({ "bymax.storage.multipart": String(result.multipart) }),
|
|
3178
|
+
bytes: uploadedBytesOf
|
|
3179
|
+
})
|
|
3180
|
+
], exports.StorageService.prototype, "upload", 1);
|
|
3181
|
+
__decorateClass([
|
|
3182
|
+
Observed("head")
|
|
3183
|
+
], exports.StorageService.prototype, "head", 1);
|
|
3184
|
+
__decorateClass([
|
|
3185
|
+
Observed("download")
|
|
3186
|
+
], exports.StorageService.prototype, "download", 1);
|
|
3187
|
+
__decorateClass([
|
|
3188
|
+
Observed("delete")
|
|
3189
|
+
], exports.StorageService.prototype, "delete", 1);
|
|
3190
|
+
__decorateClass([
|
|
3191
|
+
Observed("check_bucket")
|
|
3192
|
+
], exports.StorageService.prototype, "checkBucket", 1);
|
|
3193
|
+
__decorateClass([
|
|
3194
|
+
Observed("list")
|
|
3195
|
+
], exports.StorageService.prototype, "list", 1);
|
|
3196
|
+
__decorateClass([
|
|
3197
|
+
Observed("copy")
|
|
3198
|
+
], exports.StorageService.prototype, "copy", 1);
|
|
3199
|
+
__decorateClass([
|
|
3200
|
+
Observed("delete_many")
|
|
3201
|
+
], exports.StorageService.prototype, "deleteMany", 1);
|
|
2700
3202
|
exports.StorageService = __decorateClass([
|
|
2701
3203
|
common.Injectable(),
|
|
2702
3204
|
__decorateParam(0, common.Inject(BYMAX_STORAGE_OPTIONS)),
|
|
@@ -2704,7 +3206,9 @@ exports.StorageService = __decorateClass([
|
|
|
2704
3206
|
__decorateParam(2, common.Inject(KeyResolverService)),
|
|
2705
3207
|
__decorateParam(3, common.Inject(BYMAX_STORAGE_IDEMPOTENCY_CACHE)),
|
|
2706
3208
|
__decorateParam(4, common.Inject(ValidationService)),
|
|
2707
|
-
__decorateParam(5, common.Inject(FileScannerService))
|
|
3209
|
+
__decorateParam(5, common.Inject(FileScannerService)),
|
|
3210
|
+
__decorateParam(6, common.Optional()),
|
|
3211
|
+
__decorateParam(6, common.Inject(StorageObserver))
|
|
2708
3212
|
], exports.StorageService);
|
|
2709
3213
|
|
|
2710
3214
|
// src/server/utils/ttl-clamp.ts
|
|
@@ -2724,25 +3228,16 @@ function clampTtl(ttlSeconds, defaultTtl, maxTtl) {
|
|
|
2724
3228
|
// src/server/services/signed-url.service.ts
|
|
2725
3229
|
var MAX_MULTIPART_PARTS = 1e4;
|
|
2726
3230
|
exports.SignedUrlService = class SignedUrlService {
|
|
2727
|
-
constructor(options, s3Provider, keyResolver) {
|
|
3231
|
+
constructor(options, s3Provider, keyResolver, observer) {
|
|
2728
3232
|
this.options = options;
|
|
2729
3233
|
this.s3Provider = s3Provider;
|
|
2730
3234
|
this.keyResolver = keyResolver;
|
|
3235
|
+
this.observer = observer;
|
|
2731
3236
|
}
|
|
2732
3237
|
options;
|
|
2733
3238
|
s3Provider;
|
|
2734
3239
|
keyResolver;
|
|
2735
|
-
|
|
2736
|
-
* Issues a presigned GET URL for downloading an object.
|
|
2737
|
-
*
|
|
2738
|
-
* SECURITY: The returned URL is a temporary credential. NEVER log it, store it
|
|
2739
|
-
* in a database, or cache it across users. Issue a fresh URL per request.
|
|
2740
|
-
*
|
|
2741
|
-
* @param options - The download URL options.
|
|
2742
|
-
* @returns The presigned URL and its expiry metadata.
|
|
2743
|
-
* @throws StorageException `STORAGE_NOT_CONFIGURED` when the client is absent.
|
|
2744
|
-
* @throws StorageException `STORAGE_SIGNED_URL_TTL_INVALID` for non-positive TTL.
|
|
2745
|
-
*/
|
|
3240
|
+
observer;
|
|
2746
3241
|
async getDownloadUrl(options) {
|
|
2747
3242
|
this.assertConfigured();
|
|
2748
3243
|
assertHeaderValues({
|
|
@@ -2767,36 +3262,9 @@ exports.SignedUrlService = class SignedUrlService {
|
|
|
2767
3262
|
return { url, expiresAt: new Date(Date.now() + ttl * 1e3), method: "GET", requiredHeaders: {} };
|
|
2768
3263
|
} catch (err) {
|
|
2769
3264
|
if (err instanceof StorageException) throw err;
|
|
2770
|
-
throw mapAwsError(err, { key:
|
|
3265
|
+
throw mapAwsError(err, callDetails("getDownloadUrl", { key: options.key, bucket: options.bucket }));
|
|
2771
3266
|
}
|
|
2772
3267
|
}
|
|
2773
|
-
/**
|
|
2774
|
-
* Issues a presigned PUT URL for uploading an object. The client MUST send
|
|
2775
|
-
* every header listed in `requiredHeaders` verbatim — each one is part of the
|
|
2776
|
-
* URL signature, so an omitted header yields `SignatureDoesNotMatch`. The map
|
|
2777
|
-
* always carries `Content-Type` and additionally lists `x-amz-acl` when an ACL
|
|
2778
|
-
* is signed and an `x-amz-meta-<key>` entry per metadata pair. The `Content-Type`
|
|
2779
|
-
* is signed (`X-Amz-SignedHeaders` lists `content-type`), so a PUT that sends any
|
|
2780
|
-
* other type is refused by the provider and nothing is stored.
|
|
2781
|
-
*
|
|
2782
|
-
* SECURITY: The returned URL is a temporary credential — NEVER log it. A signed
|
|
2783
|
-
* PUT bypasses the server-side MIME/size validation pipeline. Size limits are
|
|
2784
|
-
* NOT enforced at presign time: a SigV4 PUT signature can only pin an exact
|
|
2785
|
-
* `Content-Length`, never a maximum, so binding one would reject valid smaller
|
|
2786
|
-
* uploads. Enforce size after the fact via a post-upload HEAD/size check plus
|
|
2787
|
-
* the post-upload scanner path for defence-in-depth.
|
|
2788
|
-
*
|
|
2789
|
-
* ACL caveat: `publicRead` emits `ACL: public-read`, which returns HTTP 400
|
|
2790
|
-
* `AccessControlListNotSupported` on modern AWS S3 buckets (Object Ownership
|
|
2791
|
-
* = "Bucket owner enforced") and is a no-op on Cloudflare R2. Prefer a bucket
|
|
2792
|
-
* policy, CDN, or signed GET URL for public delivery.
|
|
2793
|
-
*
|
|
2794
|
-
* @param options - The upload URL options.
|
|
2795
|
-
* @returns The presigned URL, its expiry metadata, and required headers.
|
|
2796
|
-
* @throws StorageException `STORAGE_NOT_CONFIGURED` when the client is absent.
|
|
2797
|
-
* @throws StorageException `STORAGE_SIGNED_URL_TTL_INVALID` for non-positive TTL.
|
|
2798
|
-
* @throws StorageException `STORAGE_MIME_NOT_ALLOWED` when `validation.mimeWhitelist` is set and excludes `contentType`.
|
|
2799
|
-
*/
|
|
2800
3268
|
async getUploadUrl(options) {
|
|
2801
3269
|
this.assertConfigured();
|
|
2802
3270
|
assertMimeAllowed(this.options.validation?.mimeWhitelist, options.contentType);
|
|
@@ -2830,7 +3298,7 @@ exports.SignedUrlService = class SignedUrlService {
|
|
|
2830
3298
|
};
|
|
2831
3299
|
} catch (err) {
|
|
2832
3300
|
if (err instanceof StorageException) throw err;
|
|
2833
|
-
throw mapAwsError(err, { key:
|
|
3301
|
+
throw mapAwsError(err, callDetails("getUploadUrl", { key: options.key, bucket: options.bucket }));
|
|
2834
3302
|
}
|
|
2835
3303
|
}
|
|
2836
3304
|
/**
|
|
@@ -2851,21 +3319,6 @@ exports.SignedUrlService = class SignedUrlService {
|
|
|
2851
3319
|
}
|
|
2852
3320
|
return headers;
|
|
2853
3321
|
}
|
|
2854
|
-
/**
|
|
2855
|
-
* Presigns a multipart upload: creates the upload session on S3, then returns
|
|
2856
|
-
* N presigned `UploadPart` URLs (1-indexed) and a presigned `CompleteMultipart`
|
|
2857
|
-
* URL. The client uploads parts directly to S3 and then calls the complete URL.
|
|
2858
|
-
*
|
|
2859
|
-
* SECURITY: All returned URLs are temporary credentials — NEVER log them.
|
|
2860
|
-
*
|
|
2861
|
-
* @param options - The multipart URL options.
|
|
2862
|
-
* @returns `uploadId`, `partUrls`, and `completeUrl`.
|
|
2863
|
-
* @throws StorageException `STORAGE_NOT_CONFIGURED` when the client is absent.
|
|
2864
|
-
* @throws StorageException `STORAGE_INVALID_PART_COUNT` when `parts` is not an integer from 1 to 10 000.
|
|
2865
|
-
* @throws StorageException `STORAGE_SIGNED_URL_TTL_INVALID` for non-positive TTL.
|
|
2866
|
-
* @throws StorageException `STORAGE_PROVIDER_ERROR` when no `UploadId` is returned.
|
|
2867
|
-
* @throws StorageException `STORAGE_MIME_NOT_ALLOWED` when `validation.mimeWhitelist` is set and excludes `contentType`.
|
|
2868
|
-
*/
|
|
2869
3322
|
async getMultipartUploadUrls(options) {
|
|
2870
3323
|
this.assertConfigured();
|
|
2871
3324
|
assertMimeAllowed(this.options.validation?.mimeWhitelist, options.contentType);
|
|
@@ -2873,7 +3326,8 @@ exports.SignedUrlService = class SignedUrlService {
|
|
|
2873
3326
|
if (!Number.isInteger(options.parts) || options.parts < 1 || options.parts > MAX_MULTIPART_PARTS) {
|
|
2874
3327
|
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_INVALID_PART_COUNT, void 0, {
|
|
2875
3328
|
reason: `parts must be an integer from 1 to ${String(MAX_MULTIPART_PARTS)}`,
|
|
2876
|
-
|
|
3329
|
+
// Only a number is repeated: any other value is the caller's to keep.
|
|
3330
|
+
...Number.isFinite(options.parts) ? { provided: options.parts } : {}
|
|
2877
3331
|
});
|
|
2878
3332
|
}
|
|
2879
3333
|
const finalKey = this.keyResolver.normalize(options.key);
|
|
@@ -2894,14 +3348,16 @@ exports.SignedUrlService = class SignedUrlService {
|
|
|
2894
3348
|
if (!uploadId) {
|
|
2895
3349
|
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_PROVIDER_ERROR, void 0, {
|
|
2896
3350
|
reason: "CreateMultipartUpload returned no UploadId",
|
|
2897
|
-
key:
|
|
2898
|
-
bucket
|
|
3351
|
+
...callerFields({ key: options.key, bucket: options.bucket })
|
|
2899
3352
|
});
|
|
2900
3353
|
}
|
|
2901
3354
|
return await this.presignMultipartParts({ finalKey, bucket, uploadId, parts: options.parts, ttl });
|
|
2902
3355
|
} catch (err) {
|
|
2903
3356
|
if (err instanceof StorageException) throw err;
|
|
2904
|
-
throw mapAwsError(
|
|
3357
|
+
throw mapAwsError(
|
|
3358
|
+
err,
|
|
3359
|
+
callDetails("getMultipartUploadUrls", { key: options.key, bucket: options.bucket })
|
|
3360
|
+
);
|
|
2905
3361
|
}
|
|
2906
3362
|
}
|
|
2907
3363
|
/** Presigns all part URLs and the complete URL in parallel. */
|
|
@@ -2929,18 +3385,25 @@ exports.SignedUrlService = class SignedUrlService {
|
|
|
2929
3385
|
}
|
|
2930
3386
|
/** Resolves the target bucket from a per-call override or the module default. */
|
|
2931
3387
|
resolveBucket(perCall) {
|
|
2932
|
-
|
|
2933
|
-
if (!bucket) {
|
|
2934
|
-
throw new StorageException(STORAGE_ERROR_CODES.STORAGE_BUCKET_UNDEFINED);
|
|
2935
|
-
}
|
|
2936
|
-
return bucket;
|
|
3388
|
+
return resolveBucket(perCall, this.options.bucket);
|
|
2937
3389
|
}
|
|
2938
3390
|
};
|
|
3391
|
+
__decorateClass([
|
|
3392
|
+
Observed("get_download_url", { span: "storage.download_url" })
|
|
3393
|
+
], exports.SignedUrlService.prototype, "getDownloadUrl", 1);
|
|
3394
|
+
__decorateClass([
|
|
3395
|
+
Observed("get_upload_url")
|
|
3396
|
+
], exports.SignedUrlService.prototype, "getUploadUrl", 1);
|
|
3397
|
+
__decorateClass([
|
|
3398
|
+
Observed("get_multipart_upload_urls")
|
|
3399
|
+
], exports.SignedUrlService.prototype, "getMultipartUploadUrls", 1);
|
|
2939
3400
|
exports.SignedUrlService = __decorateClass([
|
|
2940
3401
|
common.Injectable(),
|
|
2941
3402
|
__decorateParam(0, common.Inject(BYMAX_STORAGE_OPTIONS)),
|
|
2942
3403
|
__decorateParam(1, common.Inject(S3ClientProvider)),
|
|
2943
|
-
__decorateParam(2, common.Inject(KeyResolverService))
|
|
3404
|
+
__decorateParam(2, common.Inject(KeyResolverService)),
|
|
3405
|
+
__decorateParam(3, common.Optional()),
|
|
3406
|
+
__decorateParam(3, common.Inject(StorageObserver))
|
|
2944
3407
|
], exports.SignedUrlService);
|
|
2945
3408
|
var IdempotencyCache = class {
|
|
2946
3409
|
/**
|
|
@@ -3044,14 +3507,18 @@ var IdempotencyCache = class {
|
|
|
3044
3507
|
};
|
|
3045
3508
|
|
|
3046
3509
|
// src/server/bymax-storage.module.ts
|
|
3047
|
-
function resolveAsyncOptions(
|
|
3510
|
+
function resolveAsyncOptions(raw) {
|
|
3511
|
+
const options = snapshotOptions(raw);
|
|
3048
3512
|
validateOptions(options);
|
|
3049
3513
|
const resolved = applyDefaults(options);
|
|
3050
3514
|
const configured = options.signedUrls?.maxTtlSeconds;
|
|
3051
3515
|
if (configured !== void 0 && configured > resolved.signedUrls.maxTtlSeconds) {
|
|
3052
|
-
new common.Logger(exports.BymaxStorageModule.name)
|
|
3053
|
-
|
|
3054
|
-
|
|
3516
|
+
emit(new common.Logger(exports.BymaxStorageModule.name), "warn", {
|
|
3517
|
+
event: STORAGE_LOG_EVENTS.ttlClamped,
|
|
3518
|
+
message: "signedUrls.maxTtlSeconds exceeds the SigV4 presigned-URL limit; the limit is used instead",
|
|
3519
|
+
configuredSeconds: configured,
|
|
3520
|
+
appliedSeconds: resolved.signedUrls.maxTtlSeconds
|
|
3521
|
+
});
|
|
3055
3522
|
}
|
|
3056
3523
|
return resolved;
|
|
3057
3524
|
}
|
|
@@ -3149,6 +3616,10 @@ exports.BymaxStorageModule = class BymaxStorageModule {
|
|
|
3149
3616
|
DEFAULT_IDEMPOTENCY_CACHE_TTL_MS
|
|
3150
3617
|
)
|
|
3151
3618
|
},
|
|
3619
|
+
// The counts and the span seam the services report to; the optional `observability` subpath reads the tally.
|
|
3620
|
+
exports.StorageMetricsTally,
|
|
3621
|
+
StorageTracing,
|
|
3622
|
+
StorageObserver,
|
|
3152
3623
|
exports.StorageService,
|
|
3153
3624
|
exports.SignedUrlService,
|
|
3154
3625
|
exports.BucketLifecycleService,
|
|
@@ -3175,6 +3646,7 @@ exports.BymaxStorageModule = class BymaxStorageModule {
|
|
|
3175
3646
|
exports.StorageService,
|
|
3176
3647
|
exports.SignedUrlService,
|
|
3177
3648
|
exports.BucketLifecycleService,
|
|
3649
|
+
exports.StorageMetricsTally,
|
|
3178
3650
|
BYMAX_STORAGE_OPTIONS,
|
|
3179
3651
|
BYMAX_STORAGE_S3_CLIENT,
|
|
3180
3652
|
BYMAX_STORAGE_UPLOAD_VALIDATORS,
|
|
@@ -3343,6 +3815,9 @@ exports.BYMAX_STORAGE_S3_CLIENT = BYMAX_STORAGE_S3_CLIENT;
|
|
|
3343
3815
|
exports.BYMAX_STORAGE_UPLOAD_VALIDATORS = BYMAX_STORAGE_UPLOAD_VALIDATORS;
|
|
3344
3816
|
exports.NoOpFileScanner = NoOpFileScanner;
|
|
3345
3817
|
exports.NoOpUploadValidator = NoOpUploadValidator;
|
|
3818
|
+
exports.SCAN_VERDICTS = SCAN_VERDICTS;
|
|
3346
3819
|
exports.STORAGE_ERROR_CODES = STORAGE_ERROR_CODES;
|
|
3820
|
+
exports.STORAGE_OPERATIONS = STORAGE_OPERATIONS;
|
|
3821
|
+
exports.STORAGE_OUTCOMES = STORAGE_OUTCOMES;
|
|
3347
3822
|
exports.StorageException = StorageException;
|
|
3348
3823
|
exports.providerRecipes = providerRecipes;
|