@cloudbitmaps/s3 0.20.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -2
- package/dist/aws-errors.d.ts +4 -8
- package/dist/bounded-body.d.ts +2 -1
- package/dist/client-probe.d.ts +0 -5
- package/dist/defined.d.ts +8 -0
- package/dist/index.js +110 -150
- package/dist/index.js.map +4 -4
- package/dist/read-timeout.d.ts +0 -4
- package/dist/registry.d.ts +2 -1
- package/dist/s3-errors.d.ts +11 -0
- package/dist/storage.d.ts +10 -8
- package/package.json +2 -2
- package/dist/keys.d.ts +0 -17
package/README.md
CHANGED
|
@@ -57,8 +57,9 @@ It builds its own client from your usual AWS credentials. Any other key is refus
|
|
|
57
57
|
MinIO, 3.640.0 silently overwrites an existing object, which loses a published generation. `If-Match` on
|
|
58
58
|
`PutObject`, which the registry's compare-and-swap is built on, is modelled from 3.700.0: 3.699.0's serializer
|
|
59
59
|
omits it, so a fenced row write goes out unconditionally and can land over a concurrent writer's. This package's
|
|
60
|
-
own range never resolves below 3.700.0. The registry also checks, before its first request, that the client it is given sends `If-Match`
|
|
61
|
-
`
|
|
60
|
+
own range never resolves below 3.700.0. The registry also checks, before its first request, that the client it is given sends `If-None-Match` and
|
|
61
|
+
`If-Match` on `PutObject`, and refuses a registry write it would send without its precondition; a client that does
|
|
62
|
+
not send `If-Match` on `DeleteObject` turns `conditionalDelete` off.
|
|
62
63
|
- **Grant `s3:GetObject`, `s3:PutObject`, `s3:DeleteObject`, `s3:AbortMultipartUpload` and `s3:ListBucket`.** Without
|
|
63
64
|
`s3:ListBucket`, S3 answers a missing key with `403` instead of `404`.
|
|
64
65
|
- **Add a lifecycle rule that aborts incomplete multipart uploads**, and never one that expires current objects or the
|
package/dist/aws-errors.d.ts
CHANGED
|
@@ -1,10 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* SDK-free helpers for classifying AWS-style errors (used by the S3 drivers).
|
|
3
|
-
*
|
|
4
|
-
* These only read structural shapes an AWS SDK v3 error carries — `name`, `$metadata.httpStatusCode`, a
|
|
5
|
-
* lower-level `code`/`errno`, and the SDK's own `$retryable` marker — so the (subtle, easy-to-get-wrong)
|
|
6
|
-
* transient-vs-fatal decision is unit-testable without a live backend or even the SDK installed. They import no SDK.
|
|
7
|
-
*/
|
|
8
1
|
export declare function httpStatus(err: unknown): number | undefined;
|
|
9
2
|
export declare function errorName(err: unknown): string | undefined;
|
|
10
3
|
/** A lower-level transport code (e.g. `ECONNRESET`) — the Node networking layer sets `code`. */
|
|
@@ -13,5 +6,8 @@ export declare function errorCode(err: unknown): string | undefined;
|
|
|
13
6
|
export declare function isSdkRetryable(err: unknown): boolean;
|
|
14
7
|
/** Any 5xx is a server-side fault that's safe to retry (the request didn't deterministically fail). */
|
|
15
8
|
export declare function isServerSide(err: unknown): boolean;
|
|
16
|
-
/**
|
|
9
|
+
/**
|
|
10
|
+
* A dropped/timed-out connection — transient by nature; a retry on a fresh connection usually succeeds. The SDK's
|
|
11
|
+
* transport error names, the codes Node gives a failed connection ({@link isTransportFault}), and the messages above.
|
|
12
|
+
*/
|
|
17
13
|
export declare function isNetworkOrTimeout(err: unknown): boolean;
|
package/dist/bounded-body.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/** The `GetObject` response body, narrowed to the parts used. */
|
|
2
|
-
|
|
2
|
+
type ResponseBody = {
|
|
3
3
|
transformToByteArray?: () => Promise<Uint8Array>;
|
|
4
4
|
[Symbol.asyncIterator]?: () => AsyncIterator<unknown>;
|
|
5
5
|
destroy?: () => void;
|
|
@@ -12,3 +12,4 @@ export declare function destroyBody(body: unknown): void;
|
|
|
12
12
|
* propagates as it is.
|
|
13
13
|
*/
|
|
14
14
|
export declare function readBounded(body: ResponseBody, maxBytes: number, oversize: () => Error, advertised?: number): Promise<Uint8Array>;
|
|
15
|
+
export {};
|
package/dist/client-probe.d.ts
CHANGED
|
@@ -24,11 +24,6 @@ export declare function probeClient(client: S3Client, bucket: string): Promise<C
|
|
|
24
24
|
* asked, since an S3-compatible store (MinIO, for one) can accept the header and ignore it.
|
|
25
25
|
*/
|
|
26
26
|
export declare function appliesDeleteIfMatch(explicit: boolean | undefined, facts: ClientFacts | undefined): boolean;
|
|
27
|
-
/**
|
|
28
|
-
* A `conditionalDelete` option, checked: `undefined` leaves the decision to the client ({@link appliesDeleteIfMatch}),
|
|
29
|
-
* and anything but a boolean is refused.
|
|
30
|
-
*/
|
|
31
|
-
export declare function checkConditionalDelete(value: unknown): boolean | undefined;
|
|
32
27
|
/**
|
|
33
28
|
* Whether `hostname` is an AWS S3 host, by its structure: under an AWS domain, with a label that names S3 (`s3`,
|
|
34
29
|
* `s3-fips`, an access point's, a multi-region access point's `s3-global`, Object Lambda's, an Outpost's, a dash-joined
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `values` without the entries that are `undefined`, typed as optional properties, so a constructor can pass on the
|
|
3
|
+
* options it was given: `exactOptionalPropertyTypes` refuses `{ x: undefined }` for an optional `x`, and a sub-driver
|
|
4
|
+
* must see an option the caller left out as absent.
|
|
5
|
+
*/
|
|
6
|
+
export declare function defined<T extends Record<string, unknown>>(values: T): {
|
|
7
|
+
[K in keyof T]?: Exclude<T[K], undefined>;
|
|
8
|
+
};
|
package/dist/index.js
CHANGED
|
@@ -1,17 +1,35 @@
|
|
|
1
1
|
// src/backend.ts
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
ValidationError as ValidationError5,
|
|
4
|
+
brandAsBackend,
|
|
5
|
+
refuseUnknownOptions,
|
|
6
|
+
requirePositiveInt as requirePositiveInt2
|
|
7
|
+
} from "@cloudbitmaps/core/driver-kit";
|
|
3
8
|
import { S3Client } from "@aws-sdk/client-s3";
|
|
4
9
|
|
|
10
|
+
// src/defined.ts
|
|
11
|
+
function defined(values) {
|
|
12
|
+
return Object.fromEntries(Object.entries(values).filter(([, v]) => v !== void 0));
|
|
13
|
+
}
|
|
14
|
+
|
|
5
15
|
// src/storage.ts
|
|
6
16
|
import {
|
|
7
17
|
IntegrityError,
|
|
8
18
|
NotFoundError as NotFoundError2,
|
|
9
19
|
TransientError as TransientError2,
|
|
10
|
-
ValidationError as
|
|
20
|
+
ValidationError as ValidationError3,
|
|
11
21
|
WriteConflictError,
|
|
22
|
+
concatBytes,
|
|
12
23
|
isNotFoundError,
|
|
13
24
|
isValidationError,
|
|
14
|
-
isWriteConflictError
|
|
25
|
+
isWriteConflictError,
|
|
26
|
+
normalizeObjectPrefix,
|
|
27
|
+
parseGenerationFromKey,
|
|
28
|
+
requireBool,
|
|
29
|
+
requirePositiveInt,
|
|
30
|
+
segmentObjectPrefix,
|
|
31
|
+
storageObjectKey,
|
|
32
|
+
toTransient
|
|
15
33
|
} from "@cloudbitmaps/core/driver-kit";
|
|
16
34
|
import { createHash, randomBytes } from "node:crypto";
|
|
17
35
|
import {
|
|
@@ -26,44 +44,17 @@ import {
|
|
|
26
44
|
UploadPartCommand
|
|
27
45
|
} from "@aws-sdk/client-s3";
|
|
28
46
|
|
|
29
|
-
// src/
|
|
30
|
-
import {
|
|
31
|
-
ValidationError,
|
|
32
|
-
encodeNameForKey,
|
|
33
|
-
namespaceKeyPart,
|
|
34
|
-
prefixPart,
|
|
35
|
-
validateSegmentRef
|
|
36
|
-
} from "@cloudbitmaps/core/driver-kit";
|
|
37
|
-
import { normalizeObjectPrefix } from "@cloudbitmaps/core/driver-kit";
|
|
38
|
-
var SUFFIX = ".crbm";
|
|
39
|
-
function segmentObjectPrefix(prefix, ref) {
|
|
40
|
-
validateSegmentRef(ref);
|
|
41
|
-
return `${prefixPart(prefix)}${namespaceKeyPart(ref.namespace)}/segments/${encodeNameForKey(ref.segment)}.`;
|
|
42
|
-
}
|
|
43
|
-
function storageObjectKey(prefix, key) {
|
|
44
|
-
if (!Number.isInteger(key.generation) || key.generation < 0) {
|
|
45
|
-
throw new ValidationError(`generation must be a non-negative integer; got ${key.generation}`);
|
|
46
|
-
}
|
|
47
|
-
return `${segmentObjectPrefix(prefix, key)}${key.generation}${SUFFIX}`;
|
|
48
|
-
}
|
|
49
|
-
function parseGenerationFromKey(segmentPrefix, objectKey) {
|
|
50
|
-
if (!objectKey.startsWith(segmentPrefix) || !objectKey.endsWith(SUFFIX)) return null;
|
|
51
|
-
const middle = objectKey.slice(segmentPrefix.length, objectKey.length - SUFFIX.length);
|
|
52
|
-
if (!/^(0|[1-9]\d*)$/.test(middle)) return null;
|
|
53
|
-
const generation = Number(middle);
|
|
54
|
-
return Number.isSafeInteger(generation) ? generation : null;
|
|
55
|
-
}
|
|
47
|
+
// src/s3-errors.ts
|
|
48
|
+
import { ValidationError } from "@cloudbitmaps/core/driver-kit";
|
|
56
49
|
|
|
57
50
|
// src/aws-errors.ts
|
|
51
|
+
import { isTransportFault } from "@cloudbitmaps/core/driver-kit";
|
|
58
52
|
function httpStatus(err) {
|
|
59
53
|
return err?.$metadata?.httpStatusCode;
|
|
60
54
|
}
|
|
61
55
|
function errorName(err) {
|
|
62
56
|
return err?.name;
|
|
63
57
|
}
|
|
64
|
-
function errorCode(err) {
|
|
65
|
-
return err?.code;
|
|
66
|
-
}
|
|
67
58
|
function isSdkRetryable(err) {
|
|
68
59
|
return err?.$retryable != null;
|
|
69
60
|
}
|
|
@@ -78,19 +69,10 @@ var NETWORK_NAMES = /* @__PURE__ */ new Set([
|
|
|
78
69
|
"NetworkingError",
|
|
79
70
|
"AbortError"
|
|
80
71
|
]);
|
|
81
|
-
var NETWORK_CODES = /* @__PURE__ */ new Set([
|
|
82
|
-
"ETIMEDOUT",
|
|
83
|
-
"ECONNRESET",
|
|
84
|
-
"ECONNREFUSED",
|
|
85
|
-
"EPIPE",
|
|
86
|
-
"ENOTFOUND",
|
|
87
|
-
"EAI_AGAIN",
|
|
88
|
-
"ECONNABORTED"
|
|
89
|
-
]);
|
|
90
72
|
var NETWORK_MESSAGE = /socket hang up|network (error|failure)|(connection|request|socket|operation|read|write)\s+tim(e|ed)\s?out|connection (reset|refused|aborted|closed)/i;
|
|
91
73
|
function isNetworkOrTimeout(err) {
|
|
92
74
|
if (NETWORK_NAMES.has(errorName(err) ?? "")) return true;
|
|
93
|
-
if (
|
|
75
|
+
if (isTransportFault(err)) return true;
|
|
94
76
|
return NETWORK_MESSAGE.test(err?.message ?? "");
|
|
95
77
|
}
|
|
96
78
|
|
|
@@ -110,6 +92,15 @@ function isNoSuchUpload(err) {
|
|
|
110
92
|
function isMissingBucket(err) {
|
|
111
93
|
return errorName(err) === "NoSuchBucket";
|
|
112
94
|
}
|
|
95
|
+
function asMissingBucket(err, bucket) {
|
|
96
|
+
if (!isMissingBucket(err)) return err;
|
|
97
|
+
return Object.assign(
|
|
98
|
+
new ValidationError(
|
|
99
|
+
`the S3 bucket does not exist: ${bucket}; check the bucket name, and the region and endpoint of the client`
|
|
100
|
+
),
|
|
101
|
+
{ cause: err }
|
|
102
|
+
);
|
|
103
|
+
}
|
|
113
104
|
function isNotFound(err) {
|
|
114
105
|
if (isMissingBucket(err)) return false;
|
|
115
106
|
const name = errorName(err);
|
|
@@ -126,6 +117,9 @@ function isTransient(err) {
|
|
|
126
117
|
if (isConditionalConflict(err) || isNotFound(err) || isInvalidRange(err)) return false;
|
|
127
118
|
return errorName(err) === "SlowDown" || isServerSide(err) || isNetworkOrTimeout(err) || isSdkRetryable(err) || isClockSkewCorrected(err);
|
|
128
119
|
}
|
|
120
|
+
function transientFaultMessage(err) {
|
|
121
|
+
return `transient S3 fault: ${errorName(err) ?? "unknown"}`;
|
|
122
|
+
}
|
|
129
123
|
function spanFromContentRange(contentRange) {
|
|
130
124
|
if (contentRange === void 0) return void 0;
|
|
131
125
|
const match = /^\s*bytes\s+(\d+)-(\d+)\//.exec(contentRange);
|
|
@@ -314,7 +308,6 @@ function scrubCredentials(err) {
|
|
|
314
308
|
}
|
|
315
309
|
|
|
316
310
|
// src/client-probe.ts
|
|
317
|
-
import { ValidationError as ValidationError3 } from "@cloudbitmaps/core/driver-kit";
|
|
318
311
|
import { DeleteObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3";
|
|
319
312
|
var PROBE_KEY = "cloudbitmaps-probe";
|
|
320
313
|
var PLACEHOLDER_CREDENTIALS = { accessKeyId: "cloudbitmaps-probe", secretAccessKey: "unused" };
|
|
@@ -403,13 +396,6 @@ function appliesDeleteIfMatch(explicit, facts) {
|
|
|
403
396
|
if (explicit === true) return facts?.sendsDeleteIfMatch !== false;
|
|
404
397
|
return facts !== void 0 && facts.sendsDeleteIfMatch && isAwsS3Host(facts.host);
|
|
405
398
|
}
|
|
406
|
-
function checkConditionalDelete(value) {
|
|
407
|
-
if (value === void 0) return void 0;
|
|
408
|
-
if (typeof value !== "boolean") {
|
|
409
|
-
throw new ValidationError3(`conditionalDelete must be a boolean; got ${String(value)}`);
|
|
410
|
-
}
|
|
411
|
-
return value;
|
|
412
|
-
}
|
|
413
399
|
var AWS_DOMAINS = [".amazonaws.com", ".amazonaws.com.cn"];
|
|
414
400
|
var S3_LABELS = /* @__PURE__ */ new Set([
|
|
415
401
|
"s3",
|
|
@@ -454,6 +440,7 @@ var THROTTLE_BASE_DELAY_MS = 500;
|
|
|
454
440
|
var REAL_TIME = {
|
|
455
441
|
sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms))
|
|
456
442
|
};
|
|
443
|
+
var REAL_RNG = { next: () => Math.random() };
|
|
457
444
|
var S3StorageDriver = class {
|
|
458
445
|
client;
|
|
459
446
|
bucket;
|
|
@@ -462,32 +449,28 @@ var S3StorageDriver = class {
|
|
|
462
449
|
partBytes;
|
|
463
450
|
readTimeoutMs;
|
|
464
451
|
clock;
|
|
452
|
+
rng;
|
|
465
453
|
sockets;
|
|
466
454
|
explicitConditionalDelete;
|
|
467
455
|
probed;
|
|
468
456
|
facts;
|
|
469
457
|
constructor(options) {
|
|
470
458
|
this.client = options.client;
|
|
471
|
-
this.explicitConditionalDelete =
|
|
459
|
+
this.explicitConditionalDelete = requireBool("conditionalDelete", options.conditionalDelete);
|
|
472
460
|
this.sockets = options.sockets;
|
|
473
461
|
this.bucket = options.bucket;
|
|
474
462
|
this.prefix = normalizeObjectPrefix(options.prefix);
|
|
475
463
|
this.clock = options.clock ?? REAL_TIME;
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
]) {
|
|
480
|
-
if (value !== void 0 && (!Number.isSafeInteger(value) || value < 1)) {
|
|
481
|
-
throw new ValidationError4(`${name} must be a positive safe integer; got ${value}`);
|
|
482
|
-
}
|
|
483
|
-
}
|
|
464
|
+
this.rng = options.rng ?? REAL_RNG;
|
|
465
|
+
requirePositiveInt("partBytes", options.partBytes);
|
|
466
|
+
requirePositiveInt("maxObjectBytes", options.maxObjectBytes);
|
|
484
467
|
if (options.maxObjectBytes !== void 0 && options.maxObjectBytes > S3_MAX_OBJECT_BYTES) {
|
|
485
|
-
throw new
|
|
468
|
+
throw new ValidationError3(
|
|
486
469
|
`maxObjectBytes must be at most S3's 5 TiB object limit (${S3_MAX_OBJECT_BYTES}); got ${options.maxObjectBytes}`
|
|
487
470
|
);
|
|
488
471
|
}
|
|
489
472
|
if (options.partBytes !== void 0 && options.partBytes > S3_MAX_PART_BYTES) {
|
|
490
|
-
throw new
|
|
473
|
+
throw new ValidationError3(
|
|
491
474
|
`partBytes must be at most S3's 5 GiB part limit (${S3_MAX_PART_BYTES}); got ${options.partBytes}`
|
|
492
475
|
);
|
|
493
476
|
}
|
|
@@ -520,7 +503,8 @@ var S3StorageDriver = class {
|
|
|
520
503
|
this.partBytes,
|
|
521
504
|
this.maxObjectBytes,
|
|
522
505
|
this.readTimeoutMs,
|
|
523
|
-
this.clock
|
|
506
|
+
this.clock,
|
|
507
|
+
this.rng
|
|
524
508
|
);
|
|
525
509
|
try {
|
|
526
510
|
await write(sink);
|
|
@@ -540,7 +524,7 @@ var S3StorageDriver = class {
|
|
|
540
524
|
}
|
|
541
525
|
async getRange(key, offset, length) {
|
|
542
526
|
if (!Number.isInteger(offset) || !Number.isInteger(length) || offset < 0 || length < 0) {
|
|
543
|
-
throw new
|
|
527
|
+
throw new ValidationError3(`invalid range offset=${offset} length=${length}`);
|
|
544
528
|
}
|
|
545
529
|
const objectKey = storageObjectKey(this.prefix, key);
|
|
546
530
|
if (length === 0) return new Uint8Array(0);
|
|
@@ -560,7 +544,7 @@ var S3StorageDriver = class {
|
|
|
560
544
|
res.ContentLength
|
|
561
545
|
);
|
|
562
546
|
if (bytes.length !== length) {
|
|
563
|
-
throw new
|
|
547
|
+
throw new ValidationError3(
|
|
564
548
|
`range [${offset}, ${offset + length}) out of bounds (got ${bytes.length}B)`
|
|
565
549
|
);
|
|
566
550
|
}
|
|
@@ -578,6 +562,9 @@ var S3StorageDriver = class {
|
|
|
578
562
|
}
|
|
579
563
|
async getTail(key, maxBytes) {
|
|
580
564
|
const objectKey = storageObjectKey(this.prefix, key);
|
|
565
|
+
if (!Number.isSafeInteger(maxBytes)) {
|
|
566
|
+
throw new ValidationError3(`invalid tail length ${String(maxBytes)}`);
|
|
567
|
+
}
|
|
581
568
|
if (maxBytes <= 0) {
|
|
582
569
|
const head = await this.head(key, objectKey);
|
|
583
570
|
return withVersion({ bytes: new Uint8Array(0), size: head.size ?? 0 }, head.version);
|
|
@@ -628,7 +615,7 @@ var S3StorageDriver = class {
|
|
|
628
615
|
return withVersion({ bytes, size }, version);
|
|
629
616
|
}
|
|
630
617
|
badRead(key, what, why) {
|
|
631
|
-
return new
|
|
618
|
+
return new ValidationError3(
|
|
632
619
|
`S3 ${what} read of ${key.segment}.${key.generation} refused: ${why}`
|
|
633
620
|
);
|
|
634
621
|
}
|
|
@@ -737,41 +724,26 @@ var S3StorageDriver = class {
|
|
|
737
724
|
return new NotFoundError2(`no such generation: ${key.segment}.${key.generation}`);
|
|
738
725
|
}
|
|
739
726
|
if (isInvalidRange(err)) {
|
|
740
|
-
return new
|
|
727
|
+
return new ValidationError3(`range out of bounds for ${key.segment}.${key.generation}`);
|
|
741
728
|
}
|
|
742
729
|
return this.mapError(err);
|
|
743
730
|
}
|
|
744
731
|
/**
|
|
745
|
-
* Reclassify a transient S3 fault (throttle/5xx/dropped connection) as a retryable {@link TransientError},
|
|
746
|
-
*
|
|
747
|
-
* everything else propagates unchanged. The final fallback at every
|
|
748
|
-
* read retry only ever see typed errors.
|
|
732
|
+
* Reclassify a transient S3 fault (throttle/5xx/dropped connection) as a retryable {@link TransientError}, so the
|
|
733
|
+
* store's read retry can ride it out and a write's caller can tell it from a deterministic failure, and a missing
|
|
734
|
+
* bucket as a `ValidationError` that names it; everything else propagates unchanged. The final fallback at every
|
|
735
|
+
* `client.send` site, so callers and the read retry only ever see typed errors.
|
|
749
736
|
*/
|
|
750
737
|
mapError(err) {
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
return new TransientError2(
|
|
754
|
-
`transient S3 fault: ${err?.name ?? "unknown"}`,
|
|
755
|
-
{ cause: err }
|
|
756
|
-
);
|
|
757
|
-
}
|
|
758
|
-
return err;
|
|
738
|
+
const scrubbed = asMissingBucket(scrubCredentials(err), this.bucket);
|
|
739
|
+
return toTransient(scrubbed, isTransient, transientFaultMessage);
|
|
759
740
|
}
|
|
760
741
|
};
|
|
761
742
|
function withVersion(tail, version) {
|
|
762
743
|
return version === void 0 ? tail : { ...tail, version };
|
|
763
744
|
}
|
|
764
|
-
function concatBytes(parts, total) {
|
|
765
|
-
const out = new Uint8Array(total);
|
|
766
|
-
let offset = 0;
|
|
767
|
-
for (const p of parts) {
|
|
768
|
-
out.set(p, offset);
|
|
769
|
-
offset += p.length;
|
|
770
|
-
}
|
|
771
|
-
return out;
|
|
772
|
-
}
|
|
773
745
|
var S3MultipartSink = class {
|
|
774
|
-
constructor(client, bucket, objectKey, partBytes, maxObjectBytes, readTimeoutMs, clock) {
|
|
746
|
+
constructor(client, bucket, objectKey, partBytes, maxObjectBytes, readTimeoutMs, clock, rng) {
|
|
775
747
|
this.client = client;
|
|
776
748
|
this.bucket = bucket;
|
|
777
749
|
this.objectKey = objectKey;
|
|
@@ -779,6 +751,7 @@ var S3MultipartSink = class {
|
|
|
779
751
|
this.maxObjectBytes = maxObjectBytes;
|
|
780
752
|
this.readTimeoutMs = readTimeoutMs;
|
|
781
753
|
this.clock = clock;
|
|
754
|
+
this.rng = rng;
|
|
782
755
|
}
|
|
783
756
|
client;
|
|
784
757
|
bucket;
|
|
@@ -787,6 +760,7 @@ var S3MultipartSink = class {
|
|
|
787
760
|
maxObjectBytes;
|
|
788
761
|
readTimeoutMs;
|
|
789
762
|
clock;
|
|
763
|
+
rng;
|
|
790
764
|
hash = createHash("sha256");
|
|
791
765
|
pending = [];
|
|
792
766
|
pendingLen = 0;
|
|
@@ -800,7 +774,7 @@ var S3MultipartSink = class {
|
|
|
800
774
|
if (bytes.length === 0) return;
|
|
801
775
|
this.total += bytes.length;
|
|
802
776
|
if (this.total > this.maxObjectBytes) {
|
|
803
|
-
throw new
|
|
777
|
+
throw new ValidationError3(`object exceeds maxObjectBytes ${this.maxObjectBytes}`);
|
|
804
778
|
}
|
|
805
779
|
this.hash.update(bytes);
|
|
806
780
|
this.pending.push(new Uint8Array(bytes));
|
|
@@ -823,12 +797,12 @@ var S3MultipartSink = class {
|
|
|
823
797
|
}
|
|
824
798
|
this.uploadId = res2.UploadId;
|
|
825
799
|
}
|
|
826
|
-
const body = concatBytes(this.pending
|
|
800
|
+
const body = concatBytes(this.pending);
|
|
827
801
|
this.pending.length = 0;
|
|
828
802
|
this.pendingLen = 0;
|
|
829
803
|
this.partNumber += 1;
|
|
830
804
|
if (this.partNumber > S3_MAX_PARTS) {
|
|
831
|
-
throw new
|
|
805
|
+
throw new ValidationError3(`multipart upload exceeded the S3 ${S3_MAX_PARTS}-part limit`);
|
|
832
806
|
}
|
|
833
807
|
const res = await this.client.send(
|
|
834
808
|
new UploadPartCommand({
|
|
@@ -845,7 +819,7 @@ var S3MultipartSink = class {
|
|
|
845
819
|
async finish() {
|
|
846
820
|
const sha256 = this.hash.digest("hex");
|
|
847
821
|
if (this.uploadId === void 0) {
|
|
848
|
-
const body = concatBytes(this.pending
|
|
822
|
+
const body = concatBytes(this.pending);
|
|
849
823
|
await this.commit(
|
|
850
824
|
"PutObject",
|
|
851
825
|
() => sendOnce(
|
|
@@ -922,7 +896,7 @@ var S3MultipartSink = class {
|
|
|
922
896
|
{ cause: err }
|
|
923
897
|
);
|
|
924
898
|
}
|
|
925
|
-
await this.clock.sleep(Math.floor(
|
|
899
|
+
await this.clock.sleep(Math.floor(this.rng.next() * THROTTLE_BASE_DELAY_MS * 2 ** resent));
|
|
926
900
|
}
|
|
927
901
|
}
|
|
928
902
|
}
|
|
@@ -966,10 +940,11 @@ import {
|
|
|
966
940
|
isIntegrityError,
|
|
967
941
|
MAX_ROW_BYTES,
|
|
968
942
|
ObjectStoreRegistry,
|
|
969
|
-
|
|
970
|
-
ValidationError as ValidationError5,
|
|
943
|
+
ValidationError as ValidationError4,
|
|
971
944
|
WriteConflictError as WriteConflictError2,
|
|
972
|
-
normalizeObjectPrefix as normalizeObjectPrefix2
|
|
945
|
+
normalizeObjectPrefix as normalizeObjectPrefix2,
|
|
946
|
+
requireBool as requireBool2,
|
|
947
|
+
toTransient as toTransient2
|
|
973
948
|
} from "@cloudbitmaps/core/driver-kit";
|
|
974
949
|
import {
|
|
975
950
|
DeleteObjectCommand as DeleteObjectCommand3,
|
|
@@ -1011,7 +986,7 @@ var S3RegistryStore = class {
|
|
|
1011
986
|
const header = expect === "absent" ? "If-None-Match" : "If-Match";
|
|
1012
987
|
const sent = expect === "absent" ? this.facts?.sendsPutIfNoneMatch : this.facts?.sendsPutIfMatch;
|
|
1013
988
|
if (sent === false) {
|
|
1014
|
-
throw new
|
|
989
|
+
throw new ValidationError4(
|
|
1015
990
|
`the S3 client does not send ${header} on a PutObject: its @aws-sdk/client-s3 predates the member, so a registry write would overwrite whatever is there and lose a concurrent writer's change. Use a newer @aws-sdk/client-s3; the registry sent nothing`
|
|
1016
991
|
);
|
|
1017
992
|
}
|
|
@@ -1027,7 +1002,7 @@ var S3RegistryStore = class {
|
|
|
1027
1002
|
);
|
|
1028
1003
|
} catch (err) {
|
|
1029
1004
|
if (isNotFound(err)) return null;
|
|
1030
|
-
throw mapError(err);
|
|
1005
|
+
throw mapError(err, this.bucket);
|
|
1031
1006
|
}
|
|
1032
1007
|
try {
|
|
1033
1008
|
if ((res.ContentLength ?? 0) > MAX_ROW_BYTES) {
|
|
@@ -1046,7 +1021,7 @@ var S3RegistryStore = class {
|
|
|
1046
1021
|
return { bytes, version: res.ETag ?? "" };
|
|
1047
1022
|
} catch (err) {
|
|
1048
1023
|
destroyBody(res.Body);
|
|
1049
|
-
throw isIntegrityError(err) ? err : mapError(err);
|
|
1024
|
+
throw isIntegrityError(err) ? err : mapError(err, this.bucket);
|
|
1050
1025
|
}
|
|
1051
1026
|
});
|
|
1052
1027
|
}
|
|
@@ -1069,7 +1044,7 @@ var S3RegistryStore = class {
|
|
|
1069
1044
|
if (isConditionalConflict(err)) {
|
|
1070
1045
|
throw new WriteConflictError2(`registry OCC conflict for ${key}`);
|
|
1071
1046
|
}
|
|
1072
|
-
throw mapError(err);
|
|
1047
|
+
throw mapError(err, this.bucket);
|
|
1073
1048
|
}
|
|
1074
1049
|
}
|
|
1075
1050
|
async delete(key, expect) {
|
|
@@ -1082,7 +1057,7 @@ var S3RegistryStore = class {
|
|
|
1082
1057
|
if (isConditionalConflict(err) || isNotFound(err)) {
|
|
1083
1058
|
throw new WriteConflictError2(`registry OCC conflict deleting ${key}`);
|
|
1084
1059
|
}
|
|
1085
|
-
throw mapError(err);
|
|
1060
|
+
throw mapError(err, this.bucket);
|
|
1086
1061
|
}
|
|
1087
1062
|
}
|
|
1088
1063
|
async *listKeys(prefix) {
|
|
@@ -1099,7 +1074,7 @@ var S3RegistryStore = class {
|
|
|
1099
1074
|
})
|
|
1100
1075
|
);
|
|
1101
1076
|
} catch (err) {
|
|
1102
|
-
throw mapError(err);
|
|
1077
|
+
throw mapError(err, this.bucket);
|
|
1103
1078
|
}
|
|
1104
1079
|
for (const obj of res.Contents ?? []) {
|
|
1105
1080
|
if (obj.Key !== void 0) yield obj.Key;
|
|
@@ -1114,15 +1089,12 @@ var S3RegistryStore = class {
|
|
|
1114
1089
|
} while (token !== void 0);
|
|
1115
1090
|
}
|
|
1116
1091
|
};
|
|
1117
|
-
function mapError(err) {
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
);
|
|
1124
|
-
}
|
|
1125
|
-
return err;
|
|
1092
|
+
function mapError(err, bucket) {
|
|
1093
|
+
return toTransient2(
|
|
1094
|
+
asMissingBucket(scrubCredentials(err), bucket),
|
|
1095
|
+
isTransient,
|
|
1096
|
+
transientFaultMessage
|
|
1097
|
+
);
|
|
1126
1098
|
}
|
|
1127
1099
|
var S3RegistryDriver = class extends ObjectStoreRegistry {
|
|
1128
1100
|
constructor(options) {
|
|
@@ -1131,7 +1103,7 @@ var S3RegistryDriver = class extends ObjectStoreRegistry {
|
|
|
1131
1103
|
options.client,
|
|
1132
1104
|
options.bucket,
|
|
1133
1105
|
resolveReadTimeoutMs(options.readTimeoutMs),
|
|
1134
|
-
|
|
1106
|
+
requireBool2("conditionalDelete", options.conditionalDelete)
|
|
1135
1107
|
),
|
|
1136
1108
|
normalizeObjectPrefix2(options.prefix),
|
|
1137
1109
|
options.now ?? (() => Date.now())
|
|
@@ -1269,20 +1241,6 @@ function limitSockets(client, maxSockets) {
|
|
|
1269
1241
|
return handle(request, options);
|
|
1270
1242
|
};
|
|
1271
1243
|
}
|
|
1272
|
-
function refuseUnknown(name, options, keys, hint) {
|
|
1273
|
-
if (options === null || typeof options !== "object") {
|
|
1274
|
-
throw new ValidationError6(
|
|
1275
|
-
`${name} needs an options object \u2014 got ${options === null ? "null" : typeof options}`
|
|
1276
|
-
);
|
|
1277
|
-
}
|
|
1278
|
-
const unknown = Object.keys(options).filter((k) => !keys.includes(k));
|
|
1279
|
-
if (unknown.length > 0) {
|
|
1280
|
-
const list = (ks) => ks.map((k) => `\`${k}\``).join(", ");
|
|
1281
|
-
throw new ValidationError6(
|
|
1282
|
-
`${name} does not take ${list(unknown)}. It takes ${list(keys)}; ${hint}.`
|
|
1283
|
-
);
|
|
1284
|
-
}
|
|
1285
|
-
}
|
|
1286
1244
|
var S3Storage = class {
|
|
1287
1245
|
storage;
|
|
1288
1246
|
registry;
|
|
@@ -1290,56 +1248,58 @@ var S3Storage = class {
|
|
|
1290
1248
|
client;
|
|
1291
1249
|
sockets;
|
|
1292
1250
|
constructor(options) {
|
|
1293
|
-
|
|
1251
|
+
refuseUnknownOptions(
|
|
1252
|
+
"S3Storage",
|
|
1253
|
+
options,
|
|
1254
|
+
S3_STORAGE_OPTION_KEYS,
|
|
1255
|
+
"an S3 client goes in `client`"
|
|
1256
|
+
);
|
|
1294
1257
|
if (typeof options.bucket !== "string" || options.bucket.length === 0) {
|
|
1295
|
-
throw new
|
|
1258
|
+
throw new ValidationError5("S3Storage needs a `bucket`: the name of one, a non-empty string");
|
|
1296
1259
|
}
|
|
1297
1260
|
if (options.now !== void 0 && typeof options.now !== "function") {
|
|
1298
|
-
throw new
|
|
1261
|
+
throw new ValidationError5(
|
|
1299
1262
|
"S3Storage's `now` must be a function that returns epoch milliseconds"
|
|
1300
1263
|
);
|
|
1301
1264
|
}
|
|
1302
1265
|
if (options.client !== void 0 && options.client !== null) {
|
|
1303
1266
|
const ignored = CLIENT_SETTINGS.filter((k) => options[k] !== void 0);
|
|
1304
1267
|
if (ignored.length > 0) {
|
|
1305
|
-
throw new
|
|
1268
|
+
throw new ValidationError5(
|
|
1306
1269
|
`S3Storage takes \`client\` OR ${CLIENT_SETTINGS.map((k) => `\`${k}\``).join(" / ")}, not both \u2014 got \`client\` with ${ignored.map((k) => `\`${k}\``).join(", ")}; the \`client\` already carries them, so configure them on the client, or drop \`client\``
|
|
1307
1270
|
);
|
|
1308
1271
|
}
|
|
1309
1272
|
this.client = options.client;
|
|
1310
1273
|
} else {
|
|
1311
|
-
const
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
...options.pathStyle === void 0 ? {} : { forcePathStyle: options.pathStyle },
|
|
1321
|
-
...options.credentials === void 0 ? {} : { credentials: options.credentials }
|
|
1322
|
-
});
|
|
1274
|
+
const maxSockets = requirePositiveInt2("maxSockets", options.maxSockets) ?? DEFAULT_MAX_SOCKETS;
|
|
1275
|
+
this.client = new S3Client(
|
|
1276
|
+
defined({
|
|
1277
|
+
credentials: options.credentials,
|
|
1278
|
+
region: options.region,
|
|
1279
|
+
endpoint: options.endpoint,
|
|
1280
|
+
forcePathStyle: options.pathStyle
|
|
1281
|
+
})
|
|
1282
|
+
);
|
|
1323
1283
|
limitSockets(this.client, maxSockets);
|
|
1324
1284
|
}
|
|
1325
1285
|
this.sockets = new SocketAdvisory(this.client, options.bucket);
|
|
1326
1286
|
const shared = {
|
|
1327
1287
|
client: this.client,
|
|
1328
1288
|
bucket: options.bucket,
|
|
1329
|
-
...
|
|
1330
|
-
...options.readTimeoutMs === void 0 ? {} : { readTimeoutMs: options.readTimeoutMs }
|
|
1289
|
+
...defined({ prefix: options.prefix, readTimeoutMs: options.readTimeoutMs })
|
|
1331
1290
|
};
|
|
1332
1291
|
this.storage = new S3StorageDriver({
|
|
1333
1292
|
...shared,
|
|
1334
1293
|
sockets: this.sockets,
|
|
1335
|
-
...
|
|
1336
|
-
|
|
1337
|
-
|
|
1294
|
+
...defined({
|
|
1295
|
+
maxObjectBytes: options.maxObjectBytes,
|
|
1296
|
+
partBytes: options.partBytes,
|
|
1297
|
+
conditionalDelete: options.conditionalDelete
|
|
1298
|
+
})
|
|
1338
1299
|
});
|
|
1339
1300
|
this.registry = new S3RegistryDriver({
|
|
1340
1301
|
...shared,
|
|
1341
|
-
...
|
|
1342
|
-
...options.conditionalDelete === void 0 ? {} : { conditionalDelete: options.conditionalDelete }
|
|
1302
|
+
...defined({ now: options.now, conditionalDelete: options.conditionalDelete })
|
|
1343
1303
|
});
|
|
1344
1304
|
brandAsBackend(this);
|
|
1345
1305
|
}
|