@cloudbitmaps/s3 0.19.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/backend.d.ts +7 -5
- package/dist/client-probe.d.ts +12 -0
- package/dist/index.js +221 -167
- package/dist/index.js.map +4 -4
- package/dist/registry.d.ts +1 -2
- package/dist/storage.d.ts +24 -8
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -44,7 +44,7 @@ It builds its own client from your usual AWS credentials. Any other key is refus
|
|
|
44
44
|
| `maxSockets` | most sockets the client you do not pass opens at once, for `https` and plain `http` alike; a positive integer, default 128 (the SDK's own is 50; if a future SDK changes its handler, the built client keeps the SDK's own limit); refused beside `client` |
|
|
45
45
|
| `partBytes`, `maxObjectBytes` | multipart sizing: part size (default 8 MiB; the upload buffers one part at a time) and the largest object (about 80 GiB by default, up to S3's 5 TiB). A cap above 5 TiB, or a part above S3's 5 GiB part limit, is a `ValidationError` |
|
|
46
46
|
| `readTimeoutMs` | how long one read (a `GetObject` or `HeadObject`, its body included) may take before it throws `TransientError` and the store retries it; `0`, the default, sets no timeout. Writes are not timed |
|
|
47
|
-
| `conditionalDelete` | whether the registry removes a deleted row
|
|
47
|
+
| `conditionalDelete` | whether the store applies `If-Match` on a `DeleteObject`: the registry then removes a deleted row rather than leaving a tombstone, and the storage half deletes an erasure's holder only while it is the object the erasure read, by its ETag. On by default when the host the client resolves is an AWS S3 host, however its endpoint was set (`endpoint`, `AWS_ENDPOINT_URL_S3`, `AWS_ENDPOINT_URL`, the shared config file); off for any other host: set it for an S3-compatible store only once you know it applies the header (MinIO ignores it) |
|
|
48
48
|
|
|
49
49
|
## Before production
|
|
50
50
|
|
package/dist/backend.d.ts
CHANGED
|
@@ -89,11 +89,13 @@ export interface S3StorageOptions {
|
|
|
89
89
|
/** Injected clock for the registry's `createdAt`/`updatedAt`; defaults to `Date.now`. */
|
|
90
90
|
readonly now?: () => number;
|
|
91
91
|
/**
|
|
92
|
-
* Whether the
|
|
93
|
-
*
|
|
94
|
-
*
|
|
95
|
-
*
|
|
96
|
-
*
|
|
92
|
+
* Whether the store applies `If-Match` on a `DeleteObject`, which both halves rely on when it is on: the registry
|
|
93
|
+
* removes a deleted row for good rather than leaving a tombstone a full listing reads forever, and the storage half
|
|
94
|
+
* removes an erasure's holder only while it is the object the erasure searched, by its ETag. Defaults to `true` when
|
|
95
|
+
* the host the client resolves is an AWS S3 host, whichever way its endpoint was set (`endpoint`,
|
|
96
|
+
* `AWS_ENDPOINT_URL_S3`, `AWS_ENDPOINT_URL`, the shared config file), and to `false` for any other host: set it for an
|
|
97
|
+
* S3-compatible store only once you know the store applies `If-Match` on a delete. MinIO, for one, ignores it. It is
|
|
98
|
+
* never `true` for an SDK that does not send the header.
|
|
97
99
|
*/
|
|
98
100
|
readonly conditionalDelete?: boolean;
|
|
99
101
|
}
|
package/dist/client-probe.d.ts
CHANGED
|
@@ -17,6 +17,18 @@ export declare const PROBE_KEY = "cloudbitmaps-probe";
|
|
|
17
17
|
* found out. Never throws, never sends a request, and never runs anything the caller added to `client`.
|
|
18
18
|
*/
|
|
19
19
|
export declare function probeClient(client: S3Client, bucket: string): Promise<ClientFacts | undefined>;
|
|
20
|
+
/**
|
|
21
|
+
* Whether a `DeleteObject` may rely on `If-Match`, for the registry's row delete and the storage half's conditional
|
|
22
|
+
* delete alike: `explicit` when the caller set it, but never for an SDK seen not to send the header; otherwise only
|
|
23
|
+
* when the client resolves to an AWS S3 host, which applies the precondition, and `false` until the client has been
|
|
24
|
+
* asked, since an S3-compatible store (MinIO, for one) can accept the header and ignore it.
|
|
25
|
+
*/
|
|
26
|
+
export declare function appliesDeleteIfMatch(explicit: boolean | undefined, facts: ClientFacts | undefined): boolean;
|
|
27
|
+
/**
|
|
28
|
+
* A `conditionalDelete` option, checked: `undefined` leaves the decision to the client ({@link appliesDeleteIfMatch}),
|
|
29
|
+
* and anything but a boolean is refused.
|
|
30
|
+
*/
|
|
31
|
+
export declare function checkConditionalDelete(value: unknown): boolean | undefined;
|
|
20
32
|
/**
|
|
21
33
|
* Whether `hostname` is an AWS S3 host, by its structure: under an AWS domain, with a label that names S3 (`s3`,
|
|
22
34
|
* `s3-fips`, an access point's, a multi-region access point's `s3-global`, Object Lambda's, an Outpost's, a dash-joined
|
package/dist/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// src/backend.ts
|
|
2
|
-
import { ValidationError as
|
|
2
|
+
import { ValidationError as ValidationError6, brandAsBackend } from "@cloudbitmaps/core/driver-kit";
|
|
3
3
|
import { S3Client } from "@aws-sdk/client-s3";
|
|
4
4
|
|
|
5
5
|
// src/storage.ts
|
|
@@ -7,7 +7,7 @@ import {
|
|
|
7
7
|
IntegrityError,
|
|
8
8
|
NotFoundError as NotFoundError2,
|
|
9
9
|
TransientError as TransientError2,
|
|
10
|
-
ValidationError as
|
|
10
|
+
ValidationError as ValidationError4,
|
|
11
11
|
WriteConflictError,
|
|
12
12
|
isNotFoundError,
|
|
13
13
|
isValidationError,
|
|
@@ -18,11 +18,11 @@ import {
|
|
|
18
18
|
AbortMultipartUploadCommand,
|
|
19
19
|
CompleteMultipartUploadCommand,
|
|
20
20
|
CreateMultipartUploadCommand,
|
|
21
|
-
DeleteObjectCommand,
|
|
21
|
+
DeleteObjectCommand as DeleteObjectCommand2,
|
|
22
22
|
GetObjectCommand,
|
|
23
23
|
HeadObjectCommand,
|
|
24
24
|
ListObjectsV2Command,
|
|
25
|
-
PutObjectCommand,
|
|
25
|
+
PutObjectCommand as PutObjectCommand2,
|
|
26
26
|
UploadPartCommand
|
|
27
27
|
} from "@aws-sdk/client-s3";
|
|
28
28
|
|
|
@@ -313,6 +313,136 @@ function scrubCredentials(err) {
|
|
|
313
313
|
return safe;
|
|
314
314
|
}
|
|
315
315
|
|
|
316
|
+
// src/client-probe.ts
|
|
317
|
+
import { ValidationError as ValidationError3 } from "@cloudbitmaps/core/driver-kit";
|
|
318
|
+
import { DeleteObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3";
|
|
319
|
+
var PROBE_KEY = "cloudbitmaps-probe";
|
|
320
|
+
var PLACEHOLDER_CREDENTIALS = { accessKeyId: "cloudbitmaps-probe", secretAccessKey: "unused" };
|
|
321
|
+
var NEVER_SENDS = {
|
|
322
|
+
handle: () => Promise.reject(new Error("the client probe was asked to send a request")),
|
|
323
|
+
updateHttpClientConfig: () => {
|
|
324
|
+
},
|
|
325
|
+
httpHandlerConfigs: () => ({}),
|
|
326
|
+
destroy: () => {
|
|
327
|
+
}
|
|
328
|
+
};
|
|
329
|
+
var SILENT_LOGGER = {
|
|
330
|
+
debug: () => {
|
|
331
|
+
},
|
|
332
|
+
info: () => {
|
|
333
|
+
},
|
|
334
|
+
warn: () => {
|
|
335
|
+
},
|
|
336
|
+
error: () => {
|
|
337
|
+
}
|
|
338
|
+
};
|
|
339
|
+
function cloneForProbe(client) {
|
|
340
|
+
const parts = client;
|
|
341
|
+
if (typeof parts.send !== "function" || typeof parts.config !== "object" || parts.config === null) {
|
|
342
|
+
return void 0;
|
|
343
|
+
}
|
|
344
|
+
if (typeof parts.constructor !== "function" || parts.constructor === Object) return void 0;
|
|
345
|
+
const Class = parts.constructor;
|
|
346
|
+
const clone = new Class({
|
|
347
|
+
...parts.config,
|
|
348
|
+
credentials: PLACEHOLDER_CREDENTIALS,
|
|
349
|
+
requestHandler: NEVER_SENDS,
|
|
350
|
+
logger: SILENT_LOGGER
|
|
351
|
+
});
|
|
352
|
+
return typeof clone.send === "function" && clone.middlewareStack != null ? clone : void 0;
|
|
353
|
+
}
|
|
354
|
+
async function serialise(client, command) {
|
|
355
|
+
let seen;
|
|
356
|
+
command.middlewareStack.addRelativeTo(
|
|
357
|
+
() => (args) => {
|
|
358
|
+
seen = args.request;
|
|
359
|
+
return Promise.resolve({ response: {}, output: { $metadata: {} } });
|
|
360
|
+
},
|
|
361
|
+
{ relation: "after", toMiddleware: "serializerMiddleware", name: "cloudbitmapsProbe" }
|
|
362
|
+
);
|
|
363
|
+
const handler = command.resolveMiddleware(client.middlewareStack, client.config, {});
|
|
364
|
+
await handler(command);
|
|
365
|
+
if (seen === void 0) throw new Error("the probe saw no request");
|
|
366
|
+
return seen;
|
|
367
|
+
}
|
|
368
|
+
var hasHeader = (request, name) => request.headers?.[name] !== void 0;
|
|
369
|
+
async function probeClient(client, bucket) {
|
|
370
|
+
let probe;
|
|
371
|
+
try {
|
|
372
|
+
probe = cloneForProbe(client);
|
|
373
|
+
if (probe === void 0) return void 0;
|
|
374
|
+
const del = await serialise(
|
|
375
|
+
probe,
|
|
376
|
+
new DeleteObjectCommand({ Bucket: bucket, Key: PROBE_KEY, IfMatch: '"probe"' })
|
|
377
|
+
);
|
|
378
|
+
const put = await serialise(
|
|
379
|
+
probe,
|
|
380
|
+
new PutObjectCommand({
|
|
381
|
+
Bucket: bucket,
|
|
382
|
+
Key: PROBE_KEY,
|
|
383
|
+
Body: new Uint8Array(0),
|
|
384
|
+
IfMatch: '"probe"',
|
|
385
|
+
IfNoneMatch: "*"
|
|
386
|
+
})
|
|
387
|
+
);
|
|
388
|
+
if (typeof del.hostname !== "string" || del.hostname === "") return void 0;
|
|
389
|
+
return {
|
|
390
|
+
host: del.hostname,
|
|
391
|
+
sendsDeleteIfMatch: hasHeader(del, "if-match"),
|
|
392
|
+
sendsPutIfMatch: hasHeader(put, "if-match"),
|
|
393
|
+
sendsPutIfNoneMatch: hasHeader(put, "if-none-match")
|
|
394
|
+
};
|
|
395
|
+
} catch {
|
|
396
|
+
return void 0;
|
|
397
|
+
} finally {
|
|
398
|
+
probe?.destroy();
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
function appliesDeleteIfMatch(explicit, facts) {
|
|
402
|
+
if (explicit === false) return false;
|
|
403
|
+
if (explicit === true) return facts?.sendsDeleteIfMatch !== false;
|
|
404
|
+
return facts !== void 0 && facts.sendsDeleteIfMatch && isAwsS3Host(facts.host);
|
|
405
|
+
}
|
|
406
|
+
function checkConditionalDelete(value) {
|
|
407
|
+
if (value === void 0) return void 0;
|
|
408
|
+
if (typeof value !== "boolean") {
|
|
409
|
+
throw new ValidationError3(`conditionalDelete must be a boolean; got ${String(value)}`);
|
|
410
|
+
}
|
|
411
|
+
return value;
|
|
412
|
+
}
|
|
413
|
+
var AWS_DOMAINS = [".amazonaws.com", ".amazonaws.com.cn"];
|
|
414
|
+
var S3_LABELS = /* @__PURE__ */ new Set([
|
|
415
|
+
"s3",
|
|
416
|
+
"s3-fips",
|
|
417
|
+
"s3-accesspoint",
|
|
418
|
+
"s3-accesspoint-fips",
|
|
419
|
+
"s3-global",
|
|
420
|
+
"s3-object-lambda",
|
|
421
|
+
"s3-outposts",
|
|
422
|
+
"s3-external-1"
|
|
423
|
+
]);
|
|
424
|
+
var S3_DASH_REGION_LABEL = /^s3-(fips-)?[a-z]{2}(-gov)?-[a-z]+-\d$/;
|
|
425
|
+
var S3_EXPRESS_LABEL = /^s3express-[a-z0-9-]+$/;
|
|
426
|
+
var REGION_LABEL = /^[a-z]{2}(-[a-z]+)+-\d+$/;
|
|
427
|
+
var isS3Label = (label) => S3_LABELS.has(label) || S3_DASH_REGION_LABEL.test(label) || S3_EXPRESS_LABEL.test(label);
|
|
428
|
+
function isAwsS3Host(hostname) {
|
|
429
|
+
const host = hostname.toLowerCase().replace(/\.$/, "");
|
|
430
|
+
const domain = AWS_DOMAINS.find((d) => host.endsWith(d));
|
|
431
|
+
if (domain === void 0) return false;
|
|
432
|
+
const labels = host.slice(0, -domain.length).split(".");
|
|
433
|
+
let at = -1;
|
|
434
|
+
labels.forEach((label, i) => {
|
|
435
|
+
if (isS3Label(label)) at = i;
|
|
436
|
+
});
|
|
437
|
+
if (at === -1) return false;
|
|
438
|
+
const rest = labels.slice(at + 1);
|
|
439
|
+
let k = 0;
|
|
440
|
+
if (rest[k] === "dualstack") k++;
|
|
441
|
+
if (k < rest.length && REGION_LABEL.test(rest[k])) k++;
|
|
442
|
+
if (rest[k] === "vpce") k++;
|
|
443
|
+
return k === rest.length;
|
|
444
|
+
}
|
|
445
|
+
|
|
316
446
|
// src/storage.ts
|
|
317
447
|
var S3_PART_BYTES = 8 * 1024 * 1024;
|
|
318
448
|
var S3_MAX_PARTS = 1e4;
|
|
@@ -333,8 +463,12 @@ var S3StorageDriver = class {
|
|
|
333
463
|
readTimeoutMs;
|
|
334
464
|
clock;
|
|
335
465
|
sockets;
|
|
466
|
+
explicitConditionalDelete;
|
|
467
|
+
probed;
|
|
468
|
+
facts;
|
|
336
469
|
constructor(options) {
|
|
337
470
|
this.client = options.client;
|
|
471
|
+
this.explicitConditionalDelete = checkConditionalDelete(options.conditionalDelete);
|
|
338
472
|
this.sockets = options.sockets;
|
|
339
473
|
this.bucket = options.bucket;
|
|
340
474
|
this.prefix = normalizeObjectPrefix(options.prefix);
|
|
@@ -344,16 +478,16 @@ var S3StorageDriver = class {
|
|
|
344
478
|
["maxObjectBytes", options.maxObjectBytes]
|
|
345
479
|
]) {
|
|
346
480
|
if (value !== void 0 && (!Number.isSafeInteger(value) || value < 1)) {
|
|
347
|
-
throw new
|
|
481
|
+
throw new ValidationError4(`${name} must be a positive safe integer; got ${value}`);
|
|
348
482
|
}
|
|
349
483
|
}
|
|
350
484
|
if (options.maxObjectBytes !== void 0 && options.maxObjectBytes > S3_MAX_OBJECT_BYTES) {
|
|
351
|
-
throw new
|
|
485
|
+
throw new ValidationError4(
|
|
352
486
|
`maxObjectBytes must be at most S3's 5 TiB object limit (${S3_MAX_OBJECT_BYTES}); got ${options.maxObjectBytes}`
|
|
353
487
|
);
|
|
354
488
|
}
|
|
355
489
|
if (options.partBytes !== void 0 && options.partBytes > S3_MAX_PART_BYTES) {
|
|
356
|
-
throw new
|
|
490
|
+
throw new ValidationError4(
|
|
357
491
|
`partBytes must be at most S3's 5 GiB part limit (${S3_MAX_PART_BYTES}); got ${options.partBytes}`
|
|
358
492
|
);
|
|
359
493
|
}
|
|
@@ -363,7 +497,19 @@ var S3StorageDriver = class {
|
|
|
363
497
|
this.readTimeoutMs = resolveReadTimeoutMs(options.readTimeoutMs);
|
|
364
498
|
}
|
|
365
499
|
capabilities() {
|
|
366
|
-
return {
|
|
500
|
+
return {
|
|
501
|
+
rangeRead: true,
|
|
502
|
+
maxObjectBytes: this.maxObjectBytes,
|
|
503
|
+
conditionalPut: true,
|
|
504
|
+
conditionalDelete: appliesDeleteIfMatch(this.explicitConditionalDelete, this.facts)
|
|
505
|
+
};
|
|
506
|
+
}
|
|
507
|
+
/** Ask the client once where its requests go and whether it sends `If-Match` on a delete; it sends nothing. */
|
|
508
|
+
settle() {
|
|
509
|
+
this.probed ??= probeClient(this.client, this.bucket).then((facts) => {
|
|
510
|
+
this.facts = facts;
|
|
511
|
+
});
|
|
512
|
+
return this.probed;
|
|
367
513
|
}
|
|
368
514
|
async putImmutable(key, write) {
|
|
369
515
|
const objectKey = storageObjectKey(this.prefix, key);
|
|
@@ -394,7 +540,7 @@ var S3StorageDriver = class {
|
|
|
394
540
|
}
|
|
395
541
|
async getRange(key, offset, length) {
|
|
396
542
|
if (!Number.isInteger(offset) || !Number.isInteger(length) || offset < 0 || length < 0) {
|
|
397
|
-
throw new
|
|
543
|
+
throw new ValidationError4(`invalid range offset=${offset} length=${length}`);
|
|
398
544
|
}
|
|
399
545
|
const objectKey = storageObjectKey(this.prefix, key);
|
|
400
546
|
if (length === 0) return new Uint8Array(0);
|
|
@@ -414,7 +560,7 @@ var S3StorageDriver = class {
|
|
|
414
560
|
res.ContentLength
|
|
415
561
|
);
|
|
416
562
|
if (bytes.length !== length) {
|
|
417
|
-
throw new
|
|
563
|
+
throw new ValidationError4(
|
|
418
564
|
`range [${offset}, ${offset + length}) out of bounds (got ${bytes.length}B)`
|
|
419
565
|
);
|
|
420
566
|
}
|
|
@@ -433,7 +579,8 @@ var S3StorageDriver = class {
|
|
|
433
579
|
async getTail(key, maxBytes) {
|
|
434
580
|
const objectKey = storageObjectKey(this.prefix, key);
|
|
435
581
|
if (maxBytes <= 0) {
|
|
436
|
-
|
|
582
|
+
const head = await this.head(key, objectKey);
|
|
583
|
+
return withVersion({ bytes: new Uint8Array(0), size: head.size ?? 0 }, head.version);
|
|
437
584
|
}
|
|
438
585
|
const read = await this.read("GetObject", key, async (options) => {
|
|
439
586
|
try {
|
|
@@ -451,17 +598,18 @@ var S3StorageDriver = class {
|
|
|
451
598
|
() => this.badRead(key, "tail", `the response is longer than the ${maxBytes}B requested`),
|
|
452
599
|
res.ContentLength
|
|
453
600
|
);
|
|
454
|
-
return { bytes: bytes2, contentRange: res.ContentRange };
|
|
601
|
+
return { bytes: bytes2, contentRange: res.ContentRange, version: res.ETag };
|
|
455
602
|
} catch (err) {
|
|
456
603
|
if (isInvalidRange(err)) return { refused: err };
|
|
457
604
|
throw err;
|
|
458
605
|
}
|
|
459
606
|
});
|
|
460
607
|
if ("refused" in read) {
|
|
461
|
-
|
|
462
|
-
|
|
608
|
+
const head = await this.head(key, objectKey);
|
|
609
|
+
if (head.size !== 0) throw this.mapReadError(read.refused, key);
|
|
610
|
+
return withVersion({ bytes: new Uint8Array(0), size: 0 }, head.version);
|
|
463
611
|
}
|
|
464
|
-
const { bytes, contentRange } = read;
|
|
612
|
+
const { bytes, contentRange, version } = read;
|
|
465
613
|
let size = totalFromContentRange(contentRange);
|
|
466
614
|
if (contentRange !== void 0 && !(size === 0 && bytes.length === 0)) {
|
|
467
615
|
const span = spanFromContentRange(contentRange);
|
|
@@ -475,23 +623,26 @@ var S3StorageDriver = class {
|
|
|
475
623
|
}
|
|
476
624
|
}
|
|
477
625
|
if (size === void 0) {
|
|
478
|
-
size = bytes.length === maxBytes ? await this.
|
|
626
|
+
size = bytes.length === maxBytes ? (await this.head(key, objectKey)).size ?? bytes.length : bytes.length;
|
|
479
627
|
}
|
|
480
|
-
return { bytes, size };
|
|
628
|
+
return withVersion({ bytes, size }, version);
|
|
481
629
|
}
|
|
482
630
|
badRead(key, what, why) {
|
|
483
|
-
return new
|
|
631
|
+
return new ValidationError4(
|
|
484
632
|
`S3 ${what} read of ${key.segment}.${key.generation} refused: ${why}`
|
|
485
633
|
);
|
|
486
634
|
}
|
|
487
|
-
/**
|
|
488
|
-
|
|
635
|
+
/**
|
|
636
|
+
* The object's size and ETag from a `HeadObject`, each `undefined` when the response does not carry it. A missing
|
|
637
|
+
* object throws {@link NotFoundError}.
|
|
638
|
+
*/
|
|
639
|
+
head(key, objectKey) {
|
|
489
640
|
return this.read("HeadObject", key, async (options) => {
|
|
490
641
|
const head = await this.client.send(
|
|
491
642
|
new HeadObjectCommand({ Bucket: this.bucket, Key: objectKey }),
|
|
492
643
|
options
|
|
493
644
|
);
|
|
494
|
-
return head.ContentLength;
|
|
645
|
+
return { size: head.ContentLength, version: head.ETag };
|
|
495
646
|
});
|
|
496
647
|
}
|
|
497
648
|
/**
|
|
@@ -509,15 +660,43 @@ var S3StorageDriver = class {
|
|
|
509
660
|
}
|
|
510
661
|
});
|
|
511
662
|
}
|
|
512
|
-
async delete(key) {
|
|
663
|
+
async delete(key, options) {
|
|
664
|
+
const objectKey = storageObjectKey(this.prefix, key);
|
|
665
|
+
const ifVersion = options?.ifVersion;
|
|
666
|
+
if (ifVersion !== void 0 && this.explicitConditionalDelete !== false) await this.settle();
|
|
667
|
+
const conditional = ifVersion !== void 0 && appliesDeleteIfMatch(this.explicitConditionalDelete, this.facts);
|
|
513
668
|
try {
|
|
514
669
|
await this.client.send(
|
|
515
|
-
new
|
|
670
|
+
new DeleteObjectCommand2({
|
|
671
|
+
Bucket: this.bucket,
|
|
672
|
+
Key: objectKey,
|
|
673
|
+
...conditional ? { IfMatch: ifVersion } : {}
|
|
674
|
+
})
|
|
516
675
|
);
|
|
517
|
-
} catch (
|
|
676
|
+
} catch (raw) {
|
|
677
|
+
const err = scrubCredentials(raw);
|
|
678
|
+
if (conditional) {
|
|
679
|
+
if (isNotFound(err)) return;
|
|
680
|
+
if (isConditionalConflict(err)) {
|
|
681
|
+
if (!await this.exists(key, objectKey)) return;
|
|
682
|
+
throw new WriteConflictError(
|
|
683
|
+
`generation ${key.segment}.${key.generation} is another object than the version given; not deleted`
|
|
684
|
+
);
|
|
685
|
+
}
|
|
686
|
+
}
|
|
518
687
|
throw this.mapError(err);
|
|
519
688
|
}
|
|
520
689
|
}
|
|
690
|
+
/** Whether an object is under the key, from one `HeadObject`. A read that fails in any other way throws. */
|
|
691
|
+
async exists(key, objectKey) {
|
|
692
|
+
try {
|
|
693
|
+
await this.head(key, objectKey);
|
|
694
|
+
return true;
|
|
695
|
+
} catch (err) {
|
|
696
|
+
if (isNotFoundError(err)) return false;
|
|
697
|
+
throw err;
|
|
698
|
+
}
|
|
699
|
+
}
|
|
521
700
|
async *list(ref) {
|
|
522
701
|
const prefix = segmentObjectPrefix(this.prefix, ref);
|
|
523
702
|
let token;
|
|
@@ -558,7 +737,7 @@ var S3StorageDriver = class {
|
|
|
558
737
|
return new NotFoundError2(`no such generation: ${key.segment}.${key.generation}`);
|
|
559
738
|
}
|
|
560
739
|
if (isInvalidRange(err)) {
|
|
561
|
-
return new
|
|
740
|
+
return new ValidationError4(`range out of bounds for ${key.segment}.${key.generation}`);
|
|
562
741
|
}
|
|
563
742
|
return this.mapError(err);
|
|
564
743
|
}
|
|
@@ -579,6 +758,9 @@ var S3StorageDriver = class {
|
|
|
579
758
|
return err;
|
|
580
759
|
}
|
|
581
760
|
};
|
|
761
|
+
function withVersion(tail, version) {
|
|
762
|
+
return version === void 0 ? tail : { ...tail, version };
|
|
763
|
+
}
|
|
582
764
|
function concatBytes(parts, total) {
|
|
583
765
|
const out = new Uint8Array(total);
|
|
584
766
|
let offset = 0;
|
|
@@ -618,7 +800,7 @@ var S3MultipartSink = class {
|
|
|
618
800
|
if (bytes.length === 0) return;
|
|
619
801
|
this.total += bytes.length;
|
|
620
802
|
if (this.total > this.maxObjectBytes) {
|
|
621
|
-
throw new
|
|
803
|
+
throw new ValidationError4(`object exceeds maxObjectBytes ${this.maxObjectBytes}`);
|
|
622
804
|
}
|
|
623
805
|
this.hash.update(bytes);
|
|
624
806
|
this.pending.push(new Uint8Array(bytes));
|
|
@@ -646,7 +828,7 @@ var S3MultipartSink = class {
|
|
|
646
828
|
this.pendingLen = 0;
|
|
647
829
|
this.partNumber += 1;
|
|
648
830
|
if (this.partNumber > S3_MAX_PARTS) {
|
|
649
|
-
throw new
|
|
831
|
+
throw new ValidationError4(`multipart upload exceeded the S3 ${S3_MAX_PARTS}-part limit`);
|
|
650
832
|
}
|
|
651
833
|
const res = await this.client.send(
|
|
652
834
|
new UploadPartCommand({
|
|
@@ -668,7 +850,7 @@ var S3MultipartSink = class {
|
|
|
668
850
|
"PutObject",
|
|
669
851
|
() => sendOnce(
|
|
670
852
|
this.client,
|
|
671
|
-
new
|
|
853
|
+
new PutObjectCommand2({
|
|
672
854
|
Bucket: this.bucket,
|
|
673
855
|
Key: this.objectKey,
|
|
674
856
|
Body: body,
|
|
@@ -785,7 +967,7 @@ import {
|
|
|
785
967
|
MAX_ROW_BYTES,
|
|
786
968
|
ObjectStoreRegistry,
|
|
787
969
|
TransientError as TransientError3,
|
|
788
|
-
ValidationError as
|
|
970
|
+
ValidationError as ValidationError5,
|
|
789
971
|
WriteConflictError as WriteConflictError2,
|
|
790
972
|
normalizeObjectPrefix as normalizeObjectPrefix2
|
|
791
973
|
} from "@cloudbitmaps/core/driver-kit";
|
|
@@ -795,125 +977,6 @@ import {
|
|
|
795
977
|
ListObjectsV2Command as ListObjectsV2Command2,
|
|
796
978
|
PutObjectCommand as PutObjectCommand3
|
|
797
979
|
} from "@aws-sdk/client-s3";
|
|
798
|
-
|
|
799
|
-
// src/client-probe.ts
|
|
800
|
-
import { DeleteObjectCommand as DeleteObjectCommand2, PutObjectCommand as PutObjectCommand2 } from "@aws-sdk/client-s3";
|
|
801
|
-
var PROBE_KEY = "cloudbitmaps-probe";
|
|
802
|
-
var PLACEHOLDER_CREDENTIALS = { accessKeyId: "cloudbitmaps-probe", secretAccessKey: "unused" };
|
|
803
|
-
var NEVER_SENDS = {
|
|
804
|
-
handle: () => Promise.reject(new Error("the client probe was asked to send a request")),
|
|
805
|
-
updateHttpClientConfig: () => {
|
|
806
|
-
},
|
|
807
|
-
httpHandlerConfigs: () => ({}),
|
|
808
|
-
destroy: () => {
|
|
809
|
-
}
|
|
810
|
-
};
|
|
811
|
-
var SILENT_LOGGER = {
|
|
812
|
-
debug: () => {
|
|
813
|
-
},
|
|
814
|
-
info: () => {
|
|
815
|
-
},
|
|
816
|
-
warn: () => {
|
|
817
|
-
},
|
|
818
|
-
error: () => {
|
|
819
|
-
}
|
|
820
|
-
};
|
|
821
|
-
function cloneForProbe(client) {
|
|
822
|
-
const parts = client;
|
|
823
|
-
if (typeof parts.send !== "function" || typeof parts.config !== "object" || parts.config === null) {
|
|
824
|
-
return void 0;
|
|
825
|
-
}
|
|
826
|
-
if (typeof parts.constructor !== "function" || parts.constructor === Object) return void 0;
|
|
827
|
-
const Class = parts.constructor;
|
|
828
|
-
const clone = new Class({
|
|
829
|
-
...parts.config,
|
|
830
|
-
credentials: PLACEHOLDER_CREDENTIALS,
|
|
831
|
-
requestHandler: NEVER_SENDS,
|
|
832
|
-
logger: SILENT_LOGGER
|
|
833
|
-
});
|
|
834
|
-
return typeof clone.send === "function" && clone.middlewareStack != null ? clone : void 0;
|
|
835
|
-
}
|
|
836
|
-
async function serialise(client, command) {
|
|
837
|
-
let seen;
|
|
838
|
-
command.middlewareStack.addRelativeTo(
|
|
839
|
-
() => (args) => {
|
|
840
|
-
seen = args.request;
|
|
841
|
-
return Promise.resolve({ response: {}, output: { $metadata: {} } });
|
|
842
|
-
},
|
|
843
|
-
{ relation: "after", toMiddleware: "serializerMiddleware", name: "cloudbitmapsProbe" }
|
|
844
|
-
);
|
|
845
|
-
const handler = command.resolveMiddleware(client.middlewareStack, client.config, {});
|
|
846
|
-
await handler(command);
|
|
847
|
-
if (seen === void 0) throw new Error("the probe saw no request");
|
|
848
|
-
return seen;
|
|
849
|
-
}
|
|
850
|
-
var hasHeader = (request, name) => request.headers?.[name] !== void 0;
|
|
851
|
-
async function probeClient(client, bucket) {
|
|
852
|
-
let probe;
|
|
853
|
-
try {
|
|
854
|
-
probe = cloneForProbe(client);
|
|
855
|
-
if (probe === void 0) return void 0;
|
|
856
|
-
const del = await serialise(
|
|
857
|
-
probe,
|
|
858
|
-
new DeleteObjectCommand2({ Bucket: bucket, Key: PROBE_KEY, IfMatch: '"probe"' })
|
|
859
|
-
);
|
|
860
|
-
const put = await serialise(
|
|
861
|
-
probe,
|
|
862
|
-
new PutObjectCommand2({
|
|
863
|
-
Bucket: bucket,
|
|
864
|
-
Key: PROBE_KEY,
|
|
865
|
-
Body: new Uint8Array(0),
|
|
866
|
-
IfMatch: '"probe"',
|
|
867
|
-
IfNoneMatch: "*"
|
|
868
|
-
})
|
|
869
|
-
);
|
|
870
|
-
if (typeof del.hostname !== "string" || del.hostname === "") return void 0;
|
|
871
|
-
return {
|
|
872
|
-
host: del.hostname,
|
|
873
|
-
sendsDeleteIfMatch: hasHeader(del, "if-match"),
|
|
874
|
-
sendsPutIfMatch: hasHeader(put, "if-match"),
|
|
875
|
-
sendsPutIfNoneMatch: hasHeader(put, "if-none-match")
|
|
876
|
-
};
|
|
877
|
-
} catch {
|
|
878
|
-
return void 0;
|
|
879
|
-
} finally {
|
|
880
|
-
probe?.destroy();
|
|
881
|
-
}
|
|
882
|
-
}
|
|
883
|
-
var AWS_DOMAINS = [".amazonaws.com", ".amazonaws.com.cn"];
|
|
884
|
-
var S3_LABELS = /* @__PURE__ */ new Set([
|
|
885
|
-
"s3",
|
|
886
|
-
"s3-fips",
|
|
887
|
-
"s3-accesspoint",
|
|
888
|
-
"s3-accesspoint-fips",
|
|
889
|
-
"s3-global",
|
|
890
|
-
"s3-object-lambda",
|
|
891
|
-
"s3-outposts",
|
|
892
|
-
"s3-external-1"
|
|
893
|
-
]);
|
|
894
|
-
var S3_DASH_REGION_LABEL = /^s3-(fips-)?[a-z]{2}(-gov)?-[a-z]+-\d$/;
|
|
895
|
-
var S3_EXPRESS_LABEL = /^s3express-[a-z0-9-]+$/;
|
|
896
|
-
var REGION_LABEL = /^[a-z]{2}(-[a-z]+)+-\d+$/;
|
|
897
|
-
var isS3Label = (label) => S3_LABELS.has(label) || S3_DASH_REGION_LABEL.test(label) || S3_EXPRESS_LABEL.test(label);
|
|
898
|
-
function isAwsS3Host(hostname) {
|
|
899
|
-
const host = hostname.toLowerCase().replace(/\.$/, "");
|
|
900
|
-
const domain = AWS_DOMAINS.find((d) => host.endsWith(d));
|
|
901
|
-
if (domain === void 0) return false;
|
|
902
|
-
const labels = host.slice(0, -domain.length).split(".");
|
|
903
|
-
let at = -1;
|
|
904
|
-
labels.forEach((label, i) => {
|
|
905
|
-
if (isS3Label(label)) at = i;
|
|
906
|
-
});
|
|
907
|
-
if (at === -1) return false;
|
|
908
|
-
const rest = labels.slice(at + 1);
|
|
909
|
-
let k = 0;
|
|
910
|
-
if (rest[k] === "dualstack") k++;
|
|
911
|
-
if (k < rest.length && REGION_LABEL.test(rest[k])) k++;
|
|
912
|
-
if (rest[k] === "vpce") k++;
|
|
913
|
-
return k === rest.length;
|
|
914
|
-
}
|
|
915
|
-
|
|
916
|
-
// src/registry.ts
|
|
917
980
|
var S3RegistryStore = class {
|
|
918
981
|
constructor(client, bucket, readTimeoutMs, explicitConditionalDelete) {
|
|
919
982
|
this.client = client;
|
|
@@ -934,10 +997,7 @@ var S3RegistryStore = class {
|
|
|
934
997
|
* vouched for the store.
|
|
935
998
|
*/
|
|
936
999
|
get conditionalDelete() {
|
|
937
|
-
|
|
938
|
-
const facts = this.facts;
|
|
939
|
-
if (this.explicitConditionalDelete === true) return facts?.sendsDeleteIfMatch !== false;
|
|
940
|
-
return facts !== void 0 && facts.sendsDeleteIfMatch && isAwsS3Host(facts.host);
|
|
1000
|
+
return appliesDeleteIfMatch(this.explicitConditionalDelete, this.facts);
|
|
941
1001
|
}
|
|
942
1002
|
/** Ask the client once what it does with the registry's requests; every caller waits for the same answer. */
|
|
943
1003
|
settle() {
|
|
@@ -951,7 +1011,7 @@ var S3RegistryStore = class {
|
|
|
951
1011
|
const header = expect === "absent" ? "If-None-Match" : "If-Match";
|
|
952
1012
|
const sent = expect === "absent" ? this.facts?.sendsPutIfNoneMatch : this.facts?.sendsPutIfMatch;
|
|
953
1013
|
if (sent === false) {
|
|
954
|
-
throw new
|
|
1014
|
+
throw new ValidationError5(
|
|
955
1015
|
`the S3 client does not send ${header} on a PutObject: its @aws-sdk/client-s3 predates the member, so a registry write would overwrite whatever is there and lose a concurrent writer's change. Use a newer @aws-sdk/client-s3; the registry sent nothing`
|
|
956
1016
|
);
|
|
957
1017
|
}
|
|
@@ -1064,13 +1124,6 @@ function mapError(err) {
|
|
|
1064
1124
|
}
|
|
1065
1125
|
return err;
|
|
1066
1126
|
}
|
|
1067
|
-
function checkConditionalDelete(value) {
|
|
1068
|
-
if (value === void 0) return void 0;
|
|
1069
|
-
if (typeof value !== "boolean") {
|
|
1070
|
-
throw new ValidationError4(`conditionalDelete must be a boolean; got ${String(value)}`);
|
|
1071
|
-
}
|
|
1072
|
-
return value;
|
|
1073
|
-
}
|
|
1074
1127
|
var S3RegistryDriver = class extends ObjectStoreRegistry {
|
|
1075
1128
|
constructor(options) {
|
|
1076
1129
|
super(
|
|
@@ -1218,14 +1271,14 @@ function limitSockets(client, maxSockets) {
|
|
|
1218
1271
|
}
|
|
1219
1272
|
function refuseUnknown(name, options, keys, hint) {
|
|
1220
1273
|
if (options === null || typeof options !== "object") {
|
|
1221
|
-
throw new
|
|
1274
|
+
throw new ValidationError6(
|
|
1222
1275
|
`${name} needs an options object \u2014 got ${options === null ? "null" : typeof options}`
|
|
1223
1276
|
);
|
|
1224
1277
|
}
|
|
1225
1278
|
const unknown = Object.keys(options).filter((k) => !keys.includes(k));
|
|
1226
1279
|
if (unknown.length > 0) {
|
|
1227
1280
|
const list = (ks) => ks.map((k) => `\`${k}\``).join(", ");
|
|
1228
|
-
throw new
|
|
1281
|
+
throw new ValidationError6(
|
|
1229
1282
|
`${name} does not take ${list(unknown)}. It takes ${list(keys)}; ${hint}.`
|
|
1230
1283
|
);
|
|
1231
1284
|
}
|
|
@@ -1239,17 +1292,17 @@ var S3Storage = class {
|
|
|
1239
1292
|
constructor(options) {
|
|
1240
1293
|
refuseUnknown("S3Storage", options, S3_STORAGE_OPTION_KEYS, "an S3 client goes in `client`");
|
|
1241
1294
|
if (typeof options.bucket !== "string" || options.bucket.length === 0) {
|
|
1242
|
-
throw new
|
|
1295
|
+
throw new ValidationError6("S3Storage needs a `bucket`: the name of one, a non-empty string");
|
|
1243
1296
|
}
|
|
1244
1297
|
if (options.now !== void 0 && typeof options.now !== "function") {
|
|
1245
|
-
throw new
|
|
1298
|
+
throw new ValidationError6(
|
|
1246
1299
|
"S3Storage's `now` must be a function that returns epoch milliseconds"
|
|
1247
1300
|
);
|
|
1248
1301
|
}
|
|
1249
1302
|
if (options.client !== void 0 && options.client !== null) {
|
|
1250
1303
|
const ignored = CLIENT_SETTINGS.filter((k) => options[k] !== void 0);
|
|
1251
1304
|
if (ignored.length > 0) {
|
|
1252
|
-
throw new
|
|
1305
|
+
throw new ValidationError6(
|
|
1253
1306
|
`S3Storage takes \`client\` OR ${CLIENT_SETTINGS.map((k) => `\`${k}\``).join(" / ")}, not both \u2014 got \`client\` with ${ignored.map((k) => `\`${k}\``).join(", ")}; the \`client\` already carries them, so configure them on the client, or drop \`client\``
|
|
1254
1307
|
);
|
|
1255
1308
|
}
|
|
@@ -1257,7 +1310,7 @@ var S3Storage = class {
|
|
|
1257
1310
|
} else {
|
|
1258
1311
|
const { maxSockets = DEFAULT_MAX_SOCKETS } = options;
|
|
1259
1312
|
if (!Number.isSafeInteger(maxSockets) || maxSockets < 1) {
|
|
1260
|
-
throw new
|
|
1313
|
+
throw new ValidationError6(
|
|
1261
1314
|
`maxSockets must be a positive safe integer; got ${describe(options.maxSockets)}`
|
|
1262
1315
|
);
|
|
1263
1316
|
}
|
|
@@ -1280,7 +1333,8 @@ var S3Storage = class {
|
|
|
1280
1333
|
...shared,
|
|
1281
1334
|
sockets: this.sockets,
|
|
1282
1335
|
...options.maxObjectBytes === void 0 ? {} : { maxObjectBytes: options.maxObjectBytes },
|
|
1283
|
-
...options.partBytes === void 0 ? {} : { partBytes: options.partBytes }
|
|
1336
|
+
...options.partBytes === void 0 ? {} : { partBytes: options.partBytes },
|
|
1337
|
+
...options.conditionalDelete === void 0 ? {} : { conditionalDelete: options.conditionalDelete }
|
|
1284
1338
|
});
|
|
1285
1339
|
this.registry = new S3RegistryDriver({
|
|
1286
1340
|
...shared,
|