@stowage/adapter-s3 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -16
- package/dist/index.d.ts +10 -3
- package/dist/index.js +197 -573
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -11,8 +11,9 @@ npm install @stowage/adapter-s3
|
|
|
11
11
|
|
|
12
12
|
## Example
|
|
13
13
|
|
|
14
|
-
A server signs a URL for one upload of the type and length the browser reported.
|
|
15
|
-
|
|
14
|
+
A server signs a URL for one upload of the type and length the browser reported. `presignPut`
|
|
15
|
+
resolves with the URL and the `headers` the upload sends beside the body, here `content-type`. The
|
|
16
|
+
browser then calls it with a plain `fetch`, `PUT` and those `headers`, against AWS S3 or R2 alike.
|
|
16
17
|
|
|
17
18
|
```ts
|
|
18
19
|
import { fromEnv, s3Storage } from "@stowage/adapter-s3";
|
|
@@ -34,40 +35,43 @@ const r2 = s3Storage({
|
|
|
34
35
|
});
|
|
35
36
|
|
|
36
37
|
for (const storage of [aws, r2]) {
|
|
37
|
-
const url = await storage.presignPut("avatars/alice.png", {
|
|
38
|
+
const { url, headers } = await storage.presignPut("avatars/alice.png", {
|
|
38
39
|
expiresIn: 300,
|
|
39
40
|
contentType: "image/png",
|
|
40
41
|
contentLength: 48_213,
|
|
41
42
|
});
|
|
42
43
|
|
|
43
|
-
console.log(url);
|
|
44
|
+
console.log(url, headers);
|
|
44
45
|
}
|
|
45
46
|
```
|
|
46
47
|
|
|
47
48
|
The bucket has to allow `UNSIGNED-PAYLOAD` and carry a CORS rule for the uploading origin; stowage
|
|
48
49
|
configures neither
|
|
49
|
-
([flow 2](https://github.com/stowage-js/stowage/blob/@stowage/adapter-s3@0.
|
|
50
|
+
([flow 2](https://github.com/stowage-js/stowage/blob/@stowage/adapter-s3@0.2.0/docs/spec.md#flow-2-browser-upload-through-a-presigned-put)).
|
|
50
51
|
|
|
51
52
|
## Runtimes
|
|
52
53
|
|
|
53
|
-
Node 24 and later, Bun, Deno and `workerd` at the compatibility date `2026-09-01
|
|
54
|
+
Node 24 and later, Bun, Deno and `workerd` at the compatibility date `2026-09-01` without Node APIs. `fromEnv` reads
|
|
54
55
|
the environment on `workerd` under `nodejs_compat` and on Deno under `--allow-env`. CI last ran green on Bun 1.4.2 and Deno 2.9.6.
|
|
55
56
|
|
|
56
|
-
The bundle measures 13.
|
|
57
|
+
The bundle measures 13.9 kB minified and gzipped, `@stowage/core` included.
|
|
57
58
|
|
|
58
59
|
## Limits
|
|
59
60
|
|
|
60
61
|
- `keyBytesPreserved` is not declared
|
|
61
|
-
([spec 4.9](https://github.com/stowage-js/stowage/blob/@stowage/adapter-s3@0.
|
|
62
|
+
([spec 4.9](https://github.com/stowage-js/stowage/blob/@stowage/adapter-s3@0.2.0/docs/spec.md#49-capabilities)).
|
|
62
63
|
R2 normalizes a key to NFC, so two Unicode-equivalent keys name one object there and two on AWS
|
|
63
64
|
S3.
|
|
65
|
+
- `delete` sends at most one `DeleteObjects` per 1000 keys, plus at most one `DELETE` per key
|
|
66
|
+
holding `U+FFFE` or `U+FFFF`, which XML carries neither raw nor as a reference
|
|
67
|
+
([spec 7.1](https://github.com/stowage-js/stowage/blob/@stowage/adapter-s3@0.2.0/docs/spec.md#71-construction)).
|
|
64
68
|
- Where AWS S3 and R2 answer differently, the adapter is written to the stricter side
|
|
65
|
-
([spec 7.2](https://github.com/stowage-js/stowage/blob/@stowage/adapter-s3@0.
|
|
69
|
+
([spec 7.2](https://github.com/stowage-js/stowage/blob/@stowage/adapter-s3@0.2.0/docs/spec.md#72-promised-providers)):
|
|
66
70
|
|
|
67
71
|
| Point | What holds |
|
|
68
72
|
| ---------------------------------- | ------------------------------------------------------------------------------------------------------------------------ |
|
|
69
73
|
| Listing order | None. A page holds at most 1000 keys |
|
|
70
|
-
| `userMetadata` | 2 KB of encoded header bytes;
|
|
74
|
+
| `userMetadata` | 2 KB of encoded header bytes; keys handed back in lower case |
|
|
71
75
|
| Single `PUT` | Up to 5 GB for a `Uint8Array` or string; a stream that fills more than one part goes as a multipart upload |
|
|
72
76
|
| Object size ceiling | The provider's, answered with `EntityTooLarge` |
|
|
73
77
|
| `Content-Type` | Always sent by `put`, `application/octet-stream` where none was given |
|
|
@@ -80,7 +84,7 @@ The bundle measures 13.4 kB minified and gzipped, `@stowage/core` included.
|
|
|
80
84
|
## Notes
|
|
81
85
|
|
|
82
86
|
No package takes a connection URL
|
|
83
|
-
([spec 7.3](https://github.com/stowage-js/stowage/blob/@stowage/adapter-s3@0.
|
|
87
|
+
([spec 7.3](https://github.com/stowage-js/stowage/blob/@stowage/adapter-s3@0.2.0/docs/spec.md#73-credentials)).
|
|
84
88
|
A caller holding one splits it into the four options. `credentials: fromEnv` keeps the secret out of
|
|
85
89
|
the URL; one that stays in it has to be percent-encoded, because a `/` in the secret makes
|
|
86
90
|
`new URL` throw.
|
|
@@ -109,7 +113,7 @@ export function s3StorageFromUrl(connection: string): S3Storage {
|
|
|
109
113
|
```
|
|
110
114
|
|
|
111
115
|
`put` takes no `Blob`. A caller holding one passes its stream
|
|
112
|
-
([spec 4.2](https://github.com/stowage-js/stowage/blob/@stowage/adapter-s3@0.
|
|
116
|
+
([spec 4.2](https://github.com/stowage-js/stowage/blob/@stowage/adapter-s3@0.2.0/docs/spec.md#42-bodies)):
|
|
113
117
|
|
|
114
118
|
```ts
|
|
115
119
|
import { fromEnv, s3Storage } from "@stowage/adapter-s3";
|
|
@@ -121,7 +125,7 @@ await storage.put("reports/2026/q3.csv", blob.stream(), { contentType: blob.type
|
|
|
121
125
|
```
|
|
122
126
|
|
|
123
127
|
stowage reports no progress, no upload id and no resume
|
|
124
|
-
([spec 7.6](https://github.com/stowage-js/stowage/blob/@stowage/adapter-s3@0.
|
|
128
|
+
([spec 7.6](https://github.com/stowage-js/stowage/blob/@stowage/adapter-s3@0.2.0/docs/spec.md#76-uploads)).
|
|
125
129
|
A caller who wants progress counts the bytes on their way into `put`:
|
|
126
130
|
|
|
127
131
|
```ts
|
|
@@ -152,10 +156,10 @@ await storage.put("videos/intro.mp4", response.body.pipeThrough(countBytes(conso
|
|
|
152
156
|
|
|
153
157
|
## Specification
|
|
154
158
|
|
|
155
|
-
[`docs/spec.md` at `@stowage/adapter-s3@0.
|
|
159
|
+
[`docs/spec.md` at `@stowage/adapter-s3@0.2.0`](https://github.com/stowage-js/stowage/blob/@stowage/adapter-s3@0.2.0/docs/spec.md#7-stowageadapter-s3)
|
|
156
160
|
is the contract: a caller may rely on what it states and on nothing else this package happens to
|
|
157
|
-
export. The [terms it uses](https://github.com/stowage-js/stowage/blob/@stowage/adapter-s3@0.
|
|
158
|
-
and the [decisions behind it](https://github.com/stowage-js/stowage/tree/@stowage/adapter-s3@0.
|
|
161
|
+
export. The [terms it uses](https://github.com/stowage-js/stowage/blob/@stowage/adapter-s3@0.2.0/CONTEXT.md)
|
|
162
|
+
and the [decisions behind it](https://github.com/stowage-js/stowage/tree/@stowage/adapter-s3@0.2.0/docs/adr)
|
|
159
163
|
are at the same tag.
|
|
160
164
|
|
|
161
165
|
## License
|
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Resolvable, ResolverOptions, Storage } from "@stowage/core";
|
|
1
|
+
import { DeleteReport, PresignedPut, Resolvable, ResolverOptions, Storage } from "@stowage/core";
|
|
2
2
|
//#region src/credentials.d.ts
|
|
3
3
|
/**
|
|
4
4
|
* Checked before signing: `accessKeyId` and `secretAccessKey` are non-empty strings, and no
|
|
@@ -76,6 +76,12 @@ interface S3PresignPutOptions {
|
|
|
76
76
|
//#region src/index.d.ts
|
|
77
77
|
export interface S3Storage extends Storage {
|
|
78
78
|
readonly provider: "s3";
|
|
79
|
+
/**
|
|
80
|
+
* Sends at most one `DeleteObjects` per 1000 keys, plus at most one `DELETE` per key
|
|
81
|
+
* holding `U+FFFE` or `U+FFFF`, which XML carries neither raw nor as a reference. Those
|
|
82
|
+
* go after the batches, one after another.
|
|
83
|
+
*/
|
|
84
|
+
delete(...keys: readonly string[]): Promise<DeleteReport>;
|
|
79
85
|
/**
|
|
80
86
|
* A URL a client holding no credential calls with a plain `GET` for this one key until
|
|
81
87
|
* it expires. Whoever holds it may read the object: it is a bearer token. Sends no
|
|
@@ -87,9 +93,10 @@ export interface S3Storage extends Storage {
|
|
|
87
93
|
* key until it expires. `contentType` and `contentLength` bind exactly: the provider
|
|
88
94
|
* refuses a body of another type or another length, so a body of unknown length cannot
|
|
89
95
|
* be uploaded through it. It signs `UNSIGNED-PAYLOAD` and no checksum, so the upload
|
|
90
|
-
* carries no integrity check.
|
|
96
|
+
* carries no integrity check. Resolves with the URL and the `content-type` the `PUT`
|
|
97
|
+
* sends beside the body. Sends no request.
|
|
91
98
|
*/
|
|
92
|
-
presignPut(key: string, options: S3PresignPutOptions): Promise<
|
|
99
|
+
presignPut(key: string, options: S3PresignPutOptions): Promise<PresignedPut>;
|
|
93
100
|
}
|
|
94
101
|
export declare function s3Storage(options: S3AdapterOptions): S3Storage;
|
|
95
102
|
//#endregion
|
package/dist/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { StorageError, errorCodeForStatus, invalidKeyReason, isStorageError, isTransientStatus, withRetry } from "@stowage/core";
|
|
1
|
+
import { StorageError, XmlSyntaxError, checkUserMetadata, decodeUserMetadataValue, encodeUserMetadataValue, errorCodeForStatus, invalidKeyReason, isStorageError, isTransientStatus, parseXml, rangeBoundsRefusal, rangeCoversWhole, rangeHeader, rangeStartRefusal, readEnvironment, uploadStream, wholeSizeOf, withRetry } from "@stowage/core";
|
|
2
2
|
//#region src/storage-error.ts
|
|
3
3
|
function s3Error(bucket, fields) {
|
|
4
4
|
return new StorageError({
|
|
@@ -267,7 +267,7 @@ const permanentRedirect = 301;
|
|
|
267
267
|
const badRequest = 400;
|
|
268
268
|
/** The longest key AWS S3 and R2 hold, in UTF-8 bytes, above which they answer `KeyTooLongError`. */
|
|
269
269
|
const longestHeldKey = 1024;
|
|
270
|
-
const utf8$
|
|
270
|
+
const utf8$4 = new TextEncoder();
|
|
271
271
|
/**
|
|
272
272
|
* What the provider's answer means, decided by its own code where the table recognizes
|
|
273
273
|
* one and by the status where it does not. The message is the provider's word for word
|
|
@@ -280,7 +280,7 @@ function readProviderFailure(answer) {
|
|
|
280
280
|
code: "InvalidOption",
|
|
281
281
|
message: `The option \`region\` is not the bucket's${answer.bucketRegion === void 0 ? "" : `, which is \`${answer.bucketRegion}\``}: ${said}`
|
|
282
282
|
};
|
|
283
|
-
if (answer.method === "HEAD" && answer.status === badRequest && answer.key !== void 0 && utf8$
|
|
283
|
+
if (answer.method === "HEAD" && answer.status === badRequest && answer.key !== void 0 && utf8$4.encode(answer.key).byteLength > longestHeldKey) return {
|
|
284
284
|
code: "InvalidKey",
|
|
285
285
|
message: `The key is longer than the ${longestHeldKey} bytes the provider holds: ${said}`
|
|
286
286
|
};
|
|
@@ -319,196 +319,6 @@ function readEmbeddedFailure(providerCode, providerMessage) {
|
|
|
319
319
|
};
|
|
320
320
|
}
|
|
321
321
|
//#endregion
|
|
322
|
-
//#region src/xml.ts
|
|
323
|
-
/** A document outside the subset ADR 0003 reads, reported rather than read around. */
|
|
324
|
-
var XmlSyntaxError = class extends Error {
|
|
325
|
-
name = "XmlSyntaxError";
|
|
326
|
-
};
|
|
327
|
-
/**
|
|
328
|
-
* The root element of an XML document of the subset S3 answers a listing with: elements,
|
|
329
|
-
* attributes, text and comments under one optional declaration.
|
|
330
|
-
*/
|
|
331
|
-
function parseXml(document) {
|
|
332
|
-
const scanner = new Scanner(document);
|
|
333
|
-
scanner.skipDeclaration();
|
|
334
|
-
scanner.skipMisc();
|
|
335
|
-
const root = scanner.readElement();
|
|
336
|
-
scanner.skipMisc();
|
|
337
|
-
scanner.requireEnd();
|
|
338
|
-
return root;
|
|
339
|
-
}
|
|
340
|
-
const namePattern = /[:A-Z_a-z\u00C0-\uFFFF][:A-Z_a-z\u00C0-\uFFFF.\d-]*/uy;
|
|
341
|
-
const whitespacePattern = /[ \t\r\n]*/y;
|
|
342
|
-
const entityPattern = /&(?:#x([\da-fA-F]+)|#(\d+)|([A-Za-z]\w*));/uy;
|
|
343
|
-
/** `Char` of XML 1.0, section 2.2: what a character reference may name. */
|
|
344
|
-
function isXmlCharacter(codePoint) {
|
|
345
|
-
return codePoint === 9 || codePoint === 10 || codePoint === 13 || codePoint >= 32 && codePoint <= 55295 || codePoint >= 57344 && codePoint <= 65533 || codePoint >= 65536 && codePoint <= 1114111;
|
|
346
|
-
}
|
|
347
|
-
/** The five entities XML defines without a DTD, which is every one a document here has. */
|
|
348
|
-
const predefinedEntities = /* @__PURE__ */ new Map([
|
|
349
|
-
["amp", "&"],
|
|
350
|
-
["lt", "<"],
|
|
351
|
-
["gt", ">"],
|
|
352
|
-
["quot", "\""],
|
|
353
|
-
["apos", "'"]
|
|
354
|
-
]);
|
|
355
|
-
const xmlEscapes = {
|
|
356
|
-
"&": "&",
|
|
357
|
-
"<": "<",
|
|
358
|
-
">": ">",
|
|
359
|
-
"\"": """,
|
|
360
|
-
"'": "'"
|
|
361
|
-
};
|
|
362
|
-
/** Text as it stands inside an element of a request document stowage writes. */
|
|
363
|
-
function escapeXml(text) {
|
|
364
|
-
return text.replaceAll(/[&<>"']/gu, (character) => xmlEscapes[character] ?? character);
|
|
365
|
-
}
|
|
366
|
-
var Scanner = class {
|
|
367
|
-
#document;
|
|
368
|
-
#position = 0;
|
|
369
|
-
constructor(document) {
|
|
370
|
-
this.#document = document;
|
|
371
|
-
}
|
|
372
|
-
skipDeclaration() {
|
|
373
|
-
if (!this.#startsWith("<?xml")) return;
|
|
374
|
-
this.#position = this.#positionPast("?>", "The XML declaration is never closed");
|
|
375
|
-
}
|
|
376
|
-
/** Whitespace and comments, which are all that may stand beside the root element. */
|
|
377
|
-
skipMisc() {
|
|
378
|
-
for (;;) {
|
|
379
|
-
this.#skipWhitespace();
|
|
380
|
-
if (this.#startsWith("<!DOCTYPE")) throw this.#error("A DTD is refused");
|
|
381
|
-
if (!this.#startsWith("<!--")) return;
|
|
382
|
-
this.#skipComment();
|
|
383
|
-
}
|
|
384
|
-
}
|
|
385
|
-
requireEnd() {
|
|
386
|
-
if (this.#position < this.#document.length) throw this.#error("Something follows the root element");
|
|
387
|
-
}
|
|
388
|
-
readElement() {
|
|
389
|
-
this.#expect("<");
|
|
390
|
-
const name = this.#readName();
|
|
391
|
-
if (this.#readStartTagEnd() === "empty") return {
|
|
392
|
-
name,
|
|
393
|
-
children: [],
|
|
394
|
-
text: ""
|
|
395
|
-
};
|
|
396
|
-
const children = [];
|
|
397
|
-
let text = "";
|
|
398
|
-
for (;;) {
|
|
399
|
-
if (this.#position >= this.#document.length) throw this.#error(`The element <${name}> is never closed`);
|
|
400
|
-
if (this.#startsWith("</")) {
|
|
401
|
-
this.#position += 2;
|
|
402
|
-
this.#closeElement(name);
|
|
403
|
-
return {
|
|
404
|
-
name,
|
|
405
|
-
children,
|
|
406
|
-
text
|
|
407
|
-
};
|
|
408
|
-
}
|
|
409
|
-
if (this.#startsWith("<!--")) this.#skipComment();
|
|
410
|
-
else if (this.#startsWith("<![CDATA[")) throw this.#error("A CDATA section is refused");
|
|
411
|
-
else if (this.#startsWith("<")) children.push(this.readElement());
|
|
412
|
-
else text += this.#readText();
|
|
413
|
-
}
|
|
414
|
-
}
|
|
415
|
-
#closeElement(name) {
|
|
416
|
-
const closing = this.#readName();
|
|
417
|
-
if (closing !== name) throw this.#error(`The element <${name}> is closed by </${closing}>`);
|
|
418
|
-
this.#skipWhitespace();
|
|
419
|
-
this.#expect(">");
|
|
420
|
-
}
|
|
421
|
-
#readStartTagEnd() {
|
|
422
|
-
for (;;) {
|
|
423
|
-
const before = this.#position;
|
|
424
|
-
this.#skipWhitespace();
|
|
425
|
-
if (this.#startsWith("/>")) {
|
|
426
|
-
this.#position += 2;
|
|
427
|
-
return "empty";
|
|
428
|
-
}
|
|
429
|
-
if (this.#startsWith(">")) {
|
|
430
|
-
this.#position += 1;
|
|
431
|
-
return "open";
|
|
432
|
-
}
|
|
433
|
-
if (this.#position === before) throw this.#error("An attribute follows no whitespace");
|
|
434
|
-
this.#readName();
|
|
435
|
-
this.#skipWhitespace();
|
|
436
|
-
this.#expect("=");
|
|
437
|
-
this.#skipWhitespace();
|
|
438
|
-
this.#skipQuoted();
|
|
439
|
-
}
|
|
440
|
-
}
|
|
441
|
-
#skipQuoted() {
|
|
442
|
-
const quote = this.#document[this.#position];
|
|
443
|
-
if (quote !== "\"" && quote !== "'") throw this.#error("An attribute value is not quoted");
|
|
444
|
-
this.#position = this.#positionPast(quote, "An attribute value is never closed", 1);
|
|
445
|
-
}
|
|
446
|
-
/** Text up to the next markup, every `&` in it the start of an entity it decodes. */
|
|
447
|
-
#readText() {
|
|
448
|
-
const markup = this.#document.indexOf("<", this.#position);
|
|
449
|
-
const end = markup === -1 ? this.#document.length : markup;
|
|
450
|
-
let text = "";
|
|
451
|
-
while (this.#position < end) {
|
|
452
|
-
const ampersand = this.#document.indexOf("&", this.#position);
|
|
453
|
-
if (ampersand === -1 || ampersand >= end) {
|
|
454
|
-
text += this.#document.slice(this.#position, end);
|
|
455
|
-
this.#position = end;
|
|
456
|
-
break;
|
|
457
|
-
}
|
|
458
|
-
text += this.#document.slice(this.#position, ampersand);
|
|
459
|
-
this.#position = ampersand;
|
|
460
|
-
text += this.#readEntity();
|
|
461
|
-
}
|
|
462
|
-
return text;
|
|
463
|
-
}
|
|
464
|
-
#readEntity() {
|
|
465
|
-
entityPattern.lastIndex = this.#position;
|
|
466
|
-
const found = entityPattern.exec(this.#document);
|
|
467
|
-
if (found === null) throw this.#error("An `&` starts no entity");
|
|
468
|
-
const [entity, hex, decimal, name] = found;
|
|
469
|
-
if (name !== void 0) {
|
|
470
|
-
const character = predefinedEntities.get(name);
|
|
471
|
-
if (character === void 0) throw this.#error(`The entity ${entity} is not defined`);
|
|
472
|
-
this.#position = entityPattern.lastIndex;
|
|
473
|
-
return character;
|
|
474
|
-
}
|
|
475
|
-
const codePoint = hex === void 0 ? Number(decimal) : Number.parseInt(hex, 16);
|
|
476
|
-
if (!isXmlCharacter(codePoint)) throw this.#error(`The reference ${entity} names no character XML carries`);
|
|
477
|
-
this.#position = entityPattern.lastIndex;
|
|
478
|
-
return String.fromCodePoint(codePoint);
|
|
479
|
-
}
|
|
480
|
-
#skipComment() {
|
|
481
|
-
this.#position = this.#positionPast("-->", "A comment is never closed", 4);
|
|
482
|
-
}
|
|
483
|
-
#readName() {
|
|
484
|
-
namePattern.lastIndex = this.#position;
|
|
485
|
-
const found = namePattern.exec(this.#document);
|
|
486
|
-
if (found === null) throw this.#error("A name is missing where one belongs");
|
|
487
|
-
this.#position = namePattern.lastIndex;
|
|
488
|
-
return found[0];
|
|
489
|
-
}
|
|
490
|
-
#skipWhitespace() {
|
|
491
|
-
whitespacePattern.lastIndex = this.#position;
|
|
492
|
-
whitespacePattern.exec(this.#document);
|
|
493
|
-
this.#position = whitespacePattern.lastIndex;
|
|
494
|
-
}
|
|
495
|
-
#expect(literal) {
|
|
496
|
-
if (!this.#startsWith(literal)) throw this.#error(`\`${literal}\` is missing`);
|
|
497
|
-
this.#position += literal.length;
|
|
498
|
-
}
|
|
499
|
-
#positionPast(terminator, unterminated, offset = 0) {
|
|
500
|
-
const end = this.#document.indexOf(terminator, this.#position + offset);
|
|
501
|
-
if (end === -1) throw this.#error(unterminated);
|
|
502
|
-
return end + terminator.length;
|
|
503
|
-
}
|
|
504
|
-
#startsWith(literal) {
|
|
505
|
-
return this.#document.startsWith(literal, this.#position);
|
|
506
|
-
}
|
|
507
|
-
#error(message) {
|
|
508
|
-
return new XmlSyntaxError(`${message} (at character ${this.#position})`);
|
|
509
|
-
}
|
|
510
|
-
};
|
|
511
|
-
//#endregion
|
|
512
322
|
//#region src/answer-document.ts
|
|
513
323
|
/**
|
|
514
324
|
* The root of the document a `200` carries, read through the parser of spec 7.4. S3 may
|
|
@@ -629,36 +439,24 @@ function compareFields(one, other) {
|
|
|
629
439
|
//#region src/range.ts
|
|
630
440
|
/** Refuses bounds the spec does not allow, before any request goes out (spec 4.3). */
|
|
631
441
|
function requireRange(bucket, range) {
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
return `bytes=${range.start}-${range.end ?? ""}`;
|
|
639
|
-
}
|
|
640
|
-
const contentRange = /^bytes (\d+)-(\d+)\/(\d+)$/u;
|
|
641
|
-
/**
|
|
642
|
-
* The size of the whole object out of a `206`'s `Content-Range`, which is what spec 4.4
|
|
643
|
-
* has a ranged `get` report rather than the length of the range.
|
|
644
|
-
*/
|
|
645
|
-
function wholeSizeOf(response) {
|
|
646
|
-
const found = contentRange.exec(response.headers.get("content-range")?.trim() ?? "");
|
|
647
|
-
const size = Number(found?.[3]);
|
|
648
|
-
return Number.isSafeInteger(size) ? size : void 0;
|
|
442
|
+
const refusal = rangeBoundsRefusal(range);
|
|
443
|
+
if (refusal !== void 0) throw s3Error(bucket, {
|
|
444
|
+
...refusal,
|
|
445
|
+
operation: "get",
|
|
446
|
+
attempts: 0
|
|
447
|
+
});
|
|
649
448
|
}
|
|
650
449
|
/**
|
|
651
450
|
* A provider that answers a ranged `GET` with `200` sent the whole object instead, which
|
|
652
|
-
* RFC 9110 allows. Where the range covers the object,
|
|
653
|
-
*
|
|
654
|
-
*
|
|
655
|
-
* the caller did not ask for.
|
|
451
|
+
* RFC 9110 allows. Where the range covers the object, that is the body asked for. For an
|
|
452
|
+
* object the range starts beyond, which is how S3 answers a range on an empty object, it is
|
|
453
|
+
* the refusal spec 4.3 names; for any other it is a body the caller did not ask for.
|
|
656
454
|
*/
|
|
657
455
|
function wholeAnswerFailure(bucket, key, range, size) {
|
|
658
|
-
if (range
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
456
|
+
if (rangeCoversWhole(range, size)) return void 0;
|
|
457
|
+
const refusal = rangeStartRefusal(range, size, key);
|
|
458
|
+
if (refusal !== void 0) return s3Error(bucket, {
|
|
459
|
+
...refusal,
|
|
662
460
|
operation: "get",
|
|
663
461
|
key,
|
|
664
462
|
attempts: 1
|
|
@@ -671,53 +469,26 @@ function wholeAnswerFailure(bucket, key, range, size) {
|
|
|
671
469
|
attempts: 1
|
|
672
470
|
});
|
|
673
471
|
}
|
|
674
|
-
function isOffset(value) {
|
|
675
|
-
return Number.isInteger(value) && value >= 0;
|
|
676
|
-
}
|
|
677
472
|
//#endregion
|
|
678
473
|
//#region src/user-metadata.ts
|
|
679
474
|
const headerPrefix = "x-amz-meta-";
|
|
680
|
-
/** Spec 4.3 bounds the set at 2 KB of the header bytes it costs once it is encoded. */
|
|
681
|
-
const headerByteLimit = 2048;
|
|
682
|
-
/** The characters RFC 9110 allows in a field name, which is what a user metadata key is. */
|
|
683
|
-
const httpToken = /^[!#$%&'*+.^_`|~\dA-Za-z-]+$/u;
|
|
684
|
-
/**
|
|
685
|
-
* What survives a header field as it stands: printable ASCII with no space at either end,
|
|
686
|
-
* which HTTP trims, and no `=?`, which the reader would take for the start of an encoded
|
|
687
|
-
* word.
|
|
688
|
-
*/
|
|
689
|
-
const travelsAsWritten = /^(?:[\x21-\x7e](?:[\x20-\x7e]*[\x21-\x7e])?)?$/u;
|
|
690
|
-
const encodedWordStart = "=?UTF-8?B?";
|
|
691
|
-
const encodedWordEnd = "?=";
|
|
692
475
|
/**
|
|
693
|
-
*
|
|
694
|
-
*
|
|
476
|
+
* The header fields `userMetadata` travels in, refused before the request is signed in the
|
|
477
|
+
* order of spec 4.3, which reads the refusals off what the storage declares. A key outside
|
|
478
|
+
* ASCII is refused rather than sent, because R2 strips it on the way out and a write would
|
|
479
|
+
* lose it silently (ADR 0014).
|
|
695
480
|
*/
|
|
696
|
-
|
|
697
|
-
const
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
function userMetadataHeaders(bucket, userMetadata, key) {
|
|
705
|
-
const headers = [];
|
|
706
|
-
const held = Object.create(null);
|
|
707
|
-
let headerBytes = 0;
|
|
708
|
-
for (const [name, value] of Object.entries(userMetadata ?? {})) {
|
|
709
|
-
if (!httpToken.test(name)) throw refusal$1(bucket, `The user metadata key ${JSON.stringify(name)} is no ASCII HTTP token`, key);
|
|
710
|
-
const folded = name.toLowerCase();
|
|
711
|
-
if (folded in held) throw refusal$1(bucket, `The user metadata key ${JSON.stringify(folded)} is given more than once`, key);
|
|
712
|
-
const encoded = encodeValue(value);
|
|
713
|
-
held[folded] = value;
|
|
714
|
-
headers.push([`${headerPrefix}${folded}`, encoded]);
|
|
715
|
-
headerBytes += folded.length + encoded.length;
|
|
716
|
-
}
|
|
717
|
-
if (headerBytes > headerByteLimit) throw refusal$1(bucket, `The user metadata is ${headerBytes} encoded header bytes, above the limit of ${headerByteLimit}`, key);
|
|
481
|
+
function userMetadataHeaders(bucket, userMetadata, key, capabilities) {
|
|
482
|
+
const check = checkUserMetadata(userMetadata, capabilities);
|
|
483
|
+
if ("refusal" in check) throw s3Error(bucket, {
|
|
484
|
+
...check.refusal,
|
|
485
|
+
operation: "put",
|
|
486
|
+
key,
|
|
487
|
+
attempts: 0
|
|
488
|
+
});
|
|
718
489
|
return {
|
|
719
|
-
headers,
|
|
720
|
-
held:
|
|
490
|
+
headers: Object.entries(check.held).map(([name, value]) => [`${headerPrefix}${name}`, encodeUserMetadataValue(value)]),
|
|
491
|
+
held: check.held
|
|
721
492
|
};
|
|
722
493
|
}
|
|
723
494
|
/**
|
|
@@ -729,87 +500,10 @@ function readUserMetadata(headers) {
|
|
|
729
500
|
const held = Object.create(null);
|
|
730
501
|
for (const [name, value] of headers) {
|
|
731
502
|
if (!name.startsWith(headerPrefix)) continue;
|
|
732
|
-
held[name.slice(11)] =
|
|
503
|
+
held[name.slice(11)] = decodeUserMetadataValue(value);
|
|
733
504
|
}
|
|
734
505
|
return Object.freeze(held);
|
|
735
506
|
}
|
|
736
|
-
function encodeValue(value) {
|
|
737
|
-
if (travelsAsWritten.test(value) && !value.includes("=?")) return value;
|
|
738
|
-
return utf8PiecesOf(value).map((piece) => `${encodedWordStart}${btoa(String.fromCharCode(...piece))}${encodedWordEnd}`).join(" ");
|
|
739
|
-
}
|
|
740
|
-
/** The UTF-8 bytes in pieces of at most one encoded word, each ending on a character. */
|
|
741
|
-
function utf8PiecesOf(value) {
|
|
742
|
-
const pieces = [];
|
|
743
|
-
let pending = [];
|
|
744
|
-
for (const character of value) {
|
|
745
|
-
const bytes = utf8$4.encode(character);
|
|
746
|
-
if (pending.length + bytes.length > bytesPerEncodedWord) {
|
|
747
|
-
pieces.push(Uint8Array.from(pending));
|
|
748
|
-
pending = [];
|
|
749
|
-
}
|
|
750
|
-
pending.push(...bytes);
|
|
751
|
-
}
|
|
752
|
-
pieces.push(Uint8Array.from(pending));
|
|
753
|
-
return pieces;
|
|
754
|
-
}
|
|
755
|
-
const encodedWord = /=\?([^?\s]+)\?([BbQq])\?([^?\s]*)\?=/gu;
|
|
756
|
-
const whitespaceOnly = /^[ \t]+$/u;
|
|
757
|
-
const hexPair = /^[\dA-Fa-f]{2}$/u;
|
|
758
|
-
function decodeValue(value) {
|
|
759
|
-
let decoded = "";
|
|
760
|
-
let last = 0;
|
|
761
|
-
let previousWasEncoded = false;
|
|
762
|
-
for (const match of value.matchAll(encodedWord)) {
|
|
763
|
-
const between = value.slice(last, match.index);
|
|
764
|
-
const word = decodeWord(match[1] ?? "", match[2] ?? "", match[3] ?? "");
|
|
765
|
-
if (!(previousWasEncoded && word !== void 0 && whitespaceOnly.test(between))) decoded += between;
|
|
766
|
-
decoded += word ?? match[0];
|
|
767
|
-
previousWasEncoded = word !== void 0;
|
|
768
|
-
last = match.index + match[0].length;
|
|
769
|
-
}
|
|
770
|
-
return decoded + value.slice(last);
|
|
771
|
-
}
|
|
772
|
-
/** The text of one encoded word, or `undefined` where it names what cannot be read. */
|
|
773
|
-
function decodeWord(charset, encoding, text) {
|
|
774
|
-
const bytes = encoding.toUpperCase() === "B" ? base64Bytes(text) : quotedBytes(text);
|
|
775
|
-
if (bytes === void 0) return void 0;
|
|
776
|
-
try {
|
|
777
|
-
return new TextDecoder(charset, { fatal: true }).decode(bytes);
|
|
778
|
-
} catch {
|
|
779
|
-
return;
|
|
780
|
-
}
|
|
781
|
-
}
|
|
782
|
-
function base64Bytes(text) {
|
|
783
|
-
try {
|
|
784
|
-
return Uint8Array.from(atob(text), (character) => character.charCodeAt(0));
|
|
785
|
-
} catch {
|
|
786
|
-
return;
|
|
787
|
-
}
|
|
788
|
-
}
|
|
789
|
-
/** The `Q` encoding: `_` is a space and `=XX` one byte in hex. */
|
|
790
|
-
function quotedBytes(text) {
|
|
791
|
-
const bytes = [];
|
|
792
|
-
for (let index = 0; index < text.length; index += 1) {
|
|
793
|
-
const character = text[index];
|
|
794
|
-
if (character === "_") bytes.push(32);
|
|
795
|
-
else if (character === "=") {
|
|
796
|
-
const pair = text.slice(index + 1, index + 3);
|
|
797
|
-
if (!hexPair.test(pair)) return void 0;
|
|
798
|
-
bytes.push(Number.parseInt(pair, 16));
|
|
799
|
-
index += 2;
|
|
800
|
-
} else bytes.push(text.charCodeAt(index));
|
|
801
|
-
}
|
|
802
|
-
return Uint8Array.from(bytes);
|
|
803
|
-
}
|
|
804
|
-
function refusal$1(bucket, message, key) {
|
|
805
|
-
return s3Error(bucket, {
|
|
806
|
-
code: "InvalidRequest",
|
|
807
|
-
message,
|
|
808
|
-
operation: "put",
|
|
809
|
-
key,
|
|
810
|
-
attempts: 0
|
|
811
|
-
});
|
|
812
|
-
}
|
|
813
507
|
//#endregion
|
|
814
508
|
//#region src/description.ts
|
|
815
509
|
const defaultContentType = "application/octet-stream";
|
|
@@ -853,7 +547,7 @@ function unquotedEtag(etag) {
|
|
|
853
547
|
}
|
|
854
548
|
function sizeOf$1(bucket, key, operation, response) {
|
|
855
549
|
if (response.status === 206) {
|
|
856
|
-
const size = wholeSizeOf(response);
|
|
550
|
+
const size = wholeSizeOf(response.headers.get("content-range"));
|
|
857
551
|
if (size === void 0) throw incomplete(bucket, key, operation, "no size of the whole object");
|
|
858
552
|
return size;
|
|
859
553
|
}
|
|
@@ -916,21 +610,6 @@ function fromEnv(_options) {
|
|
|
916
610
|
};
|
|
917
611
|
}
|
|
918
612
|
/**
|
|
919
|
-
* ADR 0007: `process.env` is the one route through Node, Bun, Deno's compatibility layer
|
|
920
|
-
* and a Worker under `nodejs_compat`. A Worker without it has no `process` at all, and
|
|
921
|
-
* Deno without `--allow-env` throws `NotCapable` rather than answering `undefined`, so
|
|
922
|
-
* both leave the value empty. The three names are read one at a time, because
|
|
923
|
-
* enumerating `process.env` needs the unscoped permission in Deno.
|
|
924
|
-
*/
|
|
925
|
-
function readEnvironment(name) {
|
|
926
|
-
try {
|
|
927
|
-
if (typeof process === "undefined") return "";
|
|
928
|
-
return process?.env?.[name] ?? "";
|
|
929
|
-
} catch {
|
|
930
|
-
return "";
|
|
931
|
-
}
|
|
932
|
-
}
|
|
933
|
-
/**
|
|
934
613
|
* Spec 7.3: both required fields are non-empty strings and every key of the resolved
|
|
935
614
|
* object is one of the three, checked before signing rather than a round trip later.
|
|
936
615
|
*/
|
|
@@ -957,6 +636,30 @@ function refusal(message) {
|
|
|
957
636
|
});
|
|
958
637
|
}
|
|
959
638
|
//#endregion
|
|
639
|
+
//#region src/xml.ts
|
|
640
|
+
/**
|
|
641
|
+
* The five entities XML defines without a DTD. The core's parser keeps its table to itself,
|
|
642
|
+
* and `readErrorDocument` decodes a failure's message without the parser.
|
|
643
|
+
*/
|
|
644
|
+
const predefinedEntities = /* @__PURE__ */ new Map([
|
|
645
|
+
["amp", "&"],
|
|
646
|
+
["lt", "<"],
|
|
647
|
+
["gt", ">"],
|
|
648
|
+
["quot", "\""],
|
|
649
|
+
["apos", "'"]
|
|
650
|
+
]);
|
|
651
|
+
const xmlEscapes = {
|
|
652
|
+
"&": "&",
|
|
653
|
+
"<": "<",
|
|
654
|
+
">": ">",
|
|
655
|
+
"\"": """,
|
|
656
|
+
"'": "'"
|
|
657
|
+
};
|
|
658
|
+
/** Text as it stands inside an element of a request document stowage writes. */
|
|
659
|
+
function escapeXml(text) {
|
|
660
|
+
return text.replaceAll(/[&<>"']/gu, (character) => xmlEscapes[character] ?? character);
|
|
661
|
+
}
|
|
662
|
+
//#endregion
|
|
960
663
|
//#region src/error-document.ts
|
|
961
664
|
/**
|
|
962
665
|
* The `Code` and `Message` a provider answers a failed request with. The parser spec 7.4
|
|
@@ -1359,7 +1062,9 @@ function decodeCursor(cursor) {
|
|
|
1359
1062
|
}
|
|
1360
1063
|
if (!decoded.startsWith(cursorTag)) return void 0;
|
|
1361
1064
|
const position = decoded.slice(13);
|
|
1362
|
-
|
|
1065
|
+
if (position.length === 0 || !hexPosition.test(position)) return void 0;
|
|
1066
|
+
const token = tokenOf(position);
|
|
1067
|
+
return token.isWellFormed() ? token : void 0;
|
|
1363
1068
|
}
|
|
1364
1069
|
function hexOf(token) {
|
|
1365
1070
|
const units = [];
|
|
@@ -1382,16 +1087,33 @@ function tokenOf(position) {
|
|
|
1382
1087
|
function readListingDocument(answer, body) {
|
|
1383
1088
|
const root = parse(answer, body);
|
|
1384
1089
|
if (root.name !== "ListBucketResult") throw malformed(answer, `a <${root.name}> where a <ListBucketResult> belongs`);
|
|
1090
|
+
const readName = nameReader(answer, textOf$1(root, "EncodingType"));
|
|
1385
1091
|
return {
|
|
1386
|
-
objects: childrenNamed(root, "Contents").map((entry) => readEntry(answer, entry)),
|
|
1092
|
+
objects: childrenNamed(root, "Contents").map((entry) => readEntry(answer, entry, readName)),
|
|
1387
1093
|
prefixes: childrenNamed(root, "CommonPrefixes").map((prefix) => {
|
|
1388
1094
|
const text = textOf$1(prefix, "Prefix");
|
|
1389
1095
|
if (text === void 0 || text === "") throw malformed(answer, "a pseudo-directory with no prefix");
|
|
1390
|
-
return text;
|
|
1096
|
+
return readName(text);
|
|
1391
1097
|
}),
|
|
1392
1098
|
continuationToken: continuationOf(answer, root)
|
|
1393
1099
|
};
|
|
1394
1100
|
}
|
|
1101
|
+
/**
|
|
1102
|
+
* Spec 7.4: under `EncodingType` `url` a key and a prefix arrive as percent-encoded UTF-8,
|
|
1103
|
+
* a space in them possibly as `+`, which S3 never sends for a `+` of the key itself. The
|
|
1104
|
+
* continuation token and the elements the answer echoes are not names this reads.
|
|
1105
|
+
*/
|
|
1106
|
+
function nameReader(answer, encodingType) {
|
|
1107
|
+
if (encodingType !== "url") return (text) => text;
|
|
1108
|
+
return (text) => {
|
|
1109
|
+
try {
|
|
1110
|
+
return decodeURIComponent(text.replaceAll("+", " "));
|
|
1111
|
+
} catch (failure) {
|
|
1112
|
+
if (!(failure instanceof URIError)) throw failure;
|
|
1113
|
+
throw malformed(answer, `the name ${JSON.stringify(text)}, which does not decode`, failure);
|
|
1114
|
+
}
|
|
1115
|
+
};
|
|
1116
|
+
}
|
|
1395
1117
|
function parse(answer, body) {
|
|
1396
1118
|
try {
|
|
1397
1119
|
return parseXml(body);
|
|
@@ -1400,9 +1122,10 @@ function parse(answer, body) {
|
|
|
1400
1122
|
throw failure;
|
|
1401
1123
|
}
|
|
1402
1124
|
}
|
|
1403
|
-
function readEntry(answer, entry) {
|
|
1404
|
-
const
|
|
1405
|
-
if (
|
|
1125
|
+
function readEntry(answer, entry, readName) {
|
|
1126
|
+
const text = textOf$1(entry, "Key");
|
|
1127
|
+
if (text === void 0 || text === "") throw malformed(answer, "an object with no key");
|
|
1128
|
+
const key = readName(text);
|
|
1406
1129
|
const size = sizeOf(textOf$1(entry, "Size"));
|
|
1407
1130
|
if (size === void 0) throw malformed(answer, `the object under ${JSON.stringify(key)} with no size`);
|
|
1408
1131
|
const lastModified = Date.parse(textOf$1(entry, "LastModified") ?? "");
|
|
@@ -1516,7 +1239,11 @@ function readListRequest(bucket, options) {
|
|
|
1516
1239
|
}
|
|
1517
1240
|
async function requestPage(configuration, request) {
|
|
1518
1241
|
request.signal?.throwIfAborted();
|
|
1519
|
-
const query = [
|
|
1242
|
+
const query = [
|
|
1243
|
+
["list-type", "2"],
|
|
1244
|
+
["max-keys", String(request.pageSize)],
|
|
1245
|
+
["encoding-type", "url"]
|
|
1246
|
+
];
|
|
1520
1247
|
if (request.prefix !== "") query.push(["prefix", request.prefix]);
|
|
1521
1248
|
if (request.delimiter !== void 0) query.push(["delimiter", request.delimiter]);
|
|
1522
1249
|
if (request.continuationToken !== void 0) query.push(["continuation-token", request.continuationToken]);
|
|
@@ -1696,24 +1423,56 @@ function compress(state, words, offset) {
|
|
|
1696
1423
|
}
|
|
1697
1424
|
//#endregion
|
|
1698
1425
|
//#region src/delete.ts
|
|
1699
|
-
/** What one `DeleteObjects` names at most, and so what spec
|
|
1426
|
+
/** What one `DeleteObjects` names at most, and so what spec 7.1 sends one request per. */
|
|
1700
1427
|
const keysPerRequest = 1e3;
|
|
1428
|
+
/**
|
|
1429
|
+
* The characters of a key XML 1.0 carries neither raw nor as a reference, which is what
|
|
1430
|
+
* is left of those outside its `Char` once spec 4.8 refused the controls (ADR 0027).
|
|
1431
|
+
*/
|
|
1432
|
+
const outsideXmlChar = /[\uFFFE\uFFFF]/u;
|
|
1433
|
+
/**
|
|
1434
|
+
* Spec 4.7 rejects the call for a failure that says nothing about the key: no response, a
|
|
1435
|
+
* credential the provider refuses, a bucket that is absent or lives in another region. A
|
|
1436
|
+
* missing key answers `204`, so `NotFound` names the bucket. What remains is what
|
|
1437
|
+
* `DeleteObjects` reports per key, `AccessDenied` among it.
|
|
1438
|
+
*/
|
|
1439
|
+
const requestWideCodes = /* @__PURE__ */ new Set([
|
|
1440
|
+
"NetworkError",
|
|
1441
|
+
"InvalidCredentials",
|
|
1442
|
+
"Expired",
|
|
1443
|
+
"NotFound",
|
|
1444
|
+
"InvalidOption"
|
|
1445
|
+
]);
|
|
1446
|
+
/**
|
|
1447
|
+
* Spec 7.9 files a skewed clock under `InvalidRequest`, which otherwise concerns what one
|
|
1448
|
+
* request asked for, and every request after it would be signed against the same clock.
|
|
1449
|
+
*/
|
|
1450
|
+
const requestWideProviderCodes = /* @__PURE__ */ new Set(["RequestTimeTooSkewed"]);
|
|
1451
|
+
function failsTheRequestAsAWhole(failure) {
|
|
1452
|
+
return requestWideCodes.has(failure.code) || failure.providerCode !== void 0 && requestWideProviderCodes.has(failure.providerCode);
|
|
1453
|
+
}
|
|
1701
1454
|
const utf8$2 = new TextEncoder();
|
|
1702
1455
|
/**
|
|
1703
1456
|
* Spec 4.7: every key is reported, the invalid ones as `InvalidKey` without being sent,
|
|
1704
1457
|
* and a failure of a request as a whole rejects the call instead of filling the report.
|
|
1705
1458
|
*/
|
|
1706
|
-
async function deleteKeys(configuration, keys,
|
|
1459
|
+
async function deleteKeys(configuration, keys, call) {
|
|
1707
1460
|
const failed = [];
|
|
1708
|
-
const
|
|
1461
|
+
const batched = [];
|
|
1462
|
+
const alone = [];
|
|
1709
1463
|
for (const key of keys) {
|
|
1710
|
-
const refusal =
|
|
1711
|
-
if (refusal
|
|
1712
|
-
else
|
|
1464
|
+
const refusal = keyError(configuration.bucket, key, "addressable", call.operation);
|
|
1465
|
+
if (refusal !== void 0) failed.push(refusal);
|
|
1466
|
+
else if (outsideXmlChar.test(key)) alone.push(key);
|
|
1467
|
+
else batched.push(key);
|
|
1468
|
+
}
|
|
1469
|
+
for (let offset = 0; offset < batched.length; offset += keysPerRequest) {
|
|
1470
|
+
const slice = batched.slice(offset, offset + keysPerRequest);
|
|
1471
|
+
failed.push(...await deleteBatch(configuration, slice, call));
|
|
1713
1472
|
}
|
|
1714
|
-
for (
|
|
1715
|
-
const
|
|
1716
|
-
|
|
1473
|
+
for (const key of alone) {
|
|
1474
|
+
const failure = await deleteAlone(configuration, key, call);
|
|
1475
|
+
if (failure !== void 0) failed.push(failure);
|
|
1717
1476
|
}
|
|
1718
1477
|
return {
|
|
1719
1478
|
requested: keys.length,
|
|
@@ -1746,38 +1505,45 @@ async function deleteBelow(configuration, prefix, signal) {
|
|
|
1746
1505
|
failed
|
|
1747
1506
|
};
|
|
1748
1507
|
}
|
|
1749
|
-
function
|
|
1750
|
-
const invalid = keyError(bucket, key, "addressable", operation);
|
|
1751
|
-
if (invalid !== void 0) return invalid;
|
|
1752
|
-
if (key.isWellFormed()) return void 0;
|
|
1753
|
-
return s3Error(bucket, {
|
|
1754
|
-
code: "InvalidKey",
|
|
1755
|
-
message: `The key ${JSON.stringify(key)} holds a lone surrogate, which has no UTF-8 form`,
|
|
1756
|
-
operation,
|
|
1757
|
-
key,
|
|
1758
|
-
attempts: 0
|
|
1759
|
-
});
|
|
1760
|
-
}
|
|
1761
|
-
async function deleteBatch(configuration, keys, batch) {
|
|
1508
|
+
async function deleteBatch(configuration, keys, call) {
|
|
1762
1509
|
if (keys.length === 0) return [];
|
|
1763
|
-
|
|
1510
|
+
call.signal?.throwIfAborted();
|
|
1764
1511
|
const body = utf8$2.encode(deleteDocument(keys));
|
|
1765
1512
|
const response = await send(configuration, {
|
|
1766
1513
|
method: "POST",
|
|
1767
|
-
operation:
|
|
1514
|
+
operation: call.operation,
|
|
1768
1515
|
query: [["delete", ""]],
|
|
1769
1516
|
headers: [["content-type", "application/xml"], ["content-md5", md5Base64(body)]],
|
|
1770
1517
|
body,
|
|
1771
|
-
signal:
|
|
1518
|
+
signal: call.signal
|
|
1772
1519
|
});
|
|
1773
1520
|
const requestId = response.headers.get("x-amz-request-id") ?? void 0;
|
|
1774
1521
|
const answered = {
|
|
1775
1522
|
bucket: configuration.bucket,
|
|
1776
|
-
operation:
|
|
1523
|
+
operation: call.operation,
|
|
1777
1524
|
subject: "the deletion"
|
|
1778
1525
|
};
|
|
1779
1526
|
return (await readAnswerDocument(answered, response, "DeleteResult")).children.filter((child) => child.name === "Error").map((entry) => keyFailure(answered, entry, requestId));
|
|
1780
1527
|
}
|
|
1528
|
+
/**
|
|
1529
|
+
* Spec 7.4: a key no `DeleteObjects` body can carry travels percent-encoded in the path,
|
|
1530
|
+
* where no XML parser at the provider sees it, on the budget of a request of its own.
|
|
1531
|
+
*/
|
|
1532
|
+
async function deleteAlone(configuration, key, call) {
|
|
1533
|
+
call.signal?.throwIfAborted();
|
|
1534
|
+
try {
|
|
1535
|
+
await (await send(configuration, {
|
|
1536
|
+
method: "DELETE",
|
|
1537
|
+
operation: call.operation,
|
|
1538
|
+
key,
|
|
1539
|
+
signal: call.signal
|
|
1540
|
+
})).body?.cancel();
|
|
1541
|
+
return;
|
|
1542
|
+
} catch (failure) {
|
|
1543
|
+
if (isStorageError(failure) && !failsTheRequestAsAWhole(failure)) return failure;
|
|
1544
|
+
throw failure;
|
|
1545
|
+
}
|
|
1546
|
+
}
|
|
1781
1547
|
/** `Quiet` has the provider answer with the keys it failed alone. */
|
|
1782
1548
|
function deleteDocument(keys) {
|
|
1783
1549
|
return `<?xml version="1.0" encoding="UTF-8"?><Delete><Quiet>true</Quiet>${keys.map((key) => `<Object><Key>${escapeXml(key)}</Key></Object>`).join("")}</Delete>`;
|
|
@@ -1839,7 +1605,8 @@ async function presignGet(configuration, key, options) {
|
|
|
1839
1605
|
* Spec 7.10: `PutObject` on a writable key, with the content type and the content length
|
|
1840
1606
|
* bound through signed headers. Nothing else is signed in: no user metadata and no
|
|
1841
1607
|
* checksum, which the browser would have to match exactly for a `403` that names nothing
|
|
1842
|
-
* (ADR 0011).
|
|
1608
|
+
* (ADR 0011). Of the two, only the content type is handed back to send beside the body
|
|
1609
|
+
* (spec 4.13).
|
|
1843
1610
|
*/
|
|
1844
1611
|
async function presignPut(configuration, key, options) {
|
|
1845
1612
|
const operation = "presignPut";
|
|
@@ -1848,14 +1615,17 @@ async function presignPut(configuration, key, options) {
|
|
|
1848
1615
|
const expiresIn = readExpiresIn(configuration.bucket, given.expiresIn, operation);
|
|
1849
1616
|
const contentType = readText(configuration.bucket, given.contentType, "contentType", operation);
|
|
1850
1617
|
const contentLength = readContentLength(configuration.bucket, given.contentLength, operation);
|
|
1851
|
-
return
|
|
1852
|
-
|
|
1853
|
-
|
|
1854
|
-
|
|
1855
|
-
|
|
1856
|
-
|
|
1857
|
-
|
|
1858
|
-
|
|
1618
|
+
return {
|
|
1619
|
+
url: await presignedUrl(configuration, {
|
|
1620
|
+
method: "PUT",
|
|
1621
|
+
operation,
|
|
1622
|
+
key,
|
|
1623
|
+
query: [],
|
|
1624
|
+
headers: [["content-type", contentType], ["content-length", contentLength]],
|
|
1625
|
+
expiresIn
|
|
1626
|
+
}),
|
|
1627
|
+
headers: { "content-type": contentType }
|
|
1628
|
+
};
|
|
1859
1629
|
}
|
|
1860
1630
|
/**
|
|
1861
1631
|
* Spec 7.10: nothing is sent, so the one failure left after the options is the
|
|
@@ -2003,95 +1773,6 @@ function reportingFailure(body, asStorageError) {
|
|
|
2003
1773
|
});
|
|
2004
1774
|
}
|
|
2005
1775
|
//#endregion
|
|
2006
|
-
//#region src/part-reader.ts
|
|
2007
|
-
/** Where a buffer starts before the stream has shown whether it fills a part. */
|
|
2008
|
-
const initialCapacity = 65536;
|
|
2009
|
-
/**
|
|
2010
|
-
* A stream read into parts of one size (spec 7.6). A part is known to be the last only
|
|
2011
|
-
* once the stream has ended behind it, so a full part looks one chunk ahead; that chunk
|
|
2012
|
-
* is the stream's own and becomes the start of the next part.
|
|
2013
|
-
*
|
|
2014
|
-
* A part handed back through `recycle` lends its buffer to a later one. Spec 7.6 bounds
|
|
2015
|
-
* an upload at `partSize × concurrency` of buffers, and a fresh buffer per part would
|
|
2016
|
-
* keep the settled ones alive until the collector noticed them.
|
|
2017
|
-
*
|
|
2018
|
-
* Until a part has filled, the buffer grows by doubling, because ADR 0009 has a body
|
|
2019
|
-
* shorter than a part allocate only what it needs. Once one has, the stream is known to
|
|
2020
|
-
* go as a multipart upload, and every buffer after it starts at the full part size.
|
|
2021
|
-
*/
|
|
2022
|
-
var PartReader = class {
|
|
2023
|
-
#reader;
|
|
2024
|
-
#partSize;
|
|
2025
|
-
#signal;
|
|
2026
|
-
#cancelOnAbort = () => {
|
|
2027
|
-
this.#reader.cancel(this.#signal?.reason).catch(() => {});
|
|
2028
|
-
};
|
|
2029
|
-
#free = [];
|
|
2030
|
-
#ahead;
|
|
2031
|
-
#filledOnce = false;
|
|
2032
|
-
constructor(stream, partSize, signal) {
|
|
2033
|
-
this.#reader = stream.getReader();
|
|
2034
|
-
this.#partSize = partSize;
|
|
2035
|
-
this.#signal = signal;
|
|
2036
|
-
signal?.addEventListener("abort", this.#cancelOnAbort, { once: true });
|
|
2037
|
-
}
|
|
2038
|
-
async next() {
|
|
2039
|
-
let bytes = this.#freshBuffer();
|
|
2040
|
-
let filled = 0;
|
|
2041
|
-
while (filled < this.#partSize) {
|
|
2042
|
-
const chunk = this.#ahead ?? await this.#read();
|
|
2043
|
-
this.#ahead = void 0;
|
|
2044
|
-
if (chunk === void 0) return {
|
|
2045
|
-
bytes: bytes.subarray(0, filled),
|
|
2046
|
-
last: true
|
|
2047
|
-
};
|
|
2048
|
-
const taken = Math.min(chunk.byteLength, this.#partSize - filled);
|
|
2049
|
-
if (filled + taken > bytes.byteLength) bytes = this.#grown(bytes, filled + taken);
|
|
2050
|
-
bytes.set(chunk.subarray(0, taken), filled);
|
|
2051
|
-
filled += taken;
|
|
2052
|
-
if (taken < chunk.byteLength) this.#ahead = chunk.subarray(taken);
|
|
2053
|
-
}
|
|
2054
|
-
this.#filledOnce = true;
|
|
2055
|
-
this.#ahead ??= await this.#read();
|
|
2056
|
-
return {
|
|
2057
|
-
bytes,
|
|
2058
|
-
last: this.#ahead === void 0
|
|
2059
|
-
};
|
|
2060
|
-
}
|
|
2061
|
-
/** The part's request settled, so nothing reads its bytes any more. */
|
|
2062
|
-
recycle(part) {
|
|
2063
|
-
if (part.bytes.buffer.byteLength === this.#partSize) this.#free.push(part.bytes.buffer);
|
|
2064
|
-
}
|
|
2065
|
-
/** Spec 4.2 leaves the stream canceled where `put` rejects before reading it to its end. */
|
|
2066
|
-
async cancel(reason) {
|
|
2067
|
-
await this.#reader.cancel(reason).catch(() => {});
|
|
2068
|
-
}
|
|
2069
|
-
release() {
|
|
2070
|
-
this.#signal?.removeEventListener("abort", this.#cancelOnAbort);
|
|
2071
|
-
this.#reader.releaseLock();
|
|
2072
|
-
}
|
|
2073
|
-
#freshBuffer() {
|
|
2074
|
-
const pooled = this.#free.pop();
|
|
2075
|
-
if (pooled !== void 0) return new Uint8Array(pooled);
|
|
2076
|
-
return new Uint8Array(this.#filledOnce ? this.#partSize : Math.min(this.#partSize, initialCapacity));
|
|
2077
|
-
}
|
|
2078
|
-
#grown(bytes, needed) {
|
|
2079
|
-
let capacity = bytes.byteLength;
|
|
2080
|
-
while (capacity < needed) capacity *= 2;
|
|
2081
|
-
const larger = new Uint8Array(Math.min(capacity, this.#partSize));
|
|
2082
|
-
larger.set(bytes);
|
|
2083
|
-
return larger;
|
|
2084
|
-
}
|
|
2085
|
-
async #read() {
|
|
2086
|
-
for (;;) {
|
|
2087
|
-
const { done, value } = await this.#reader.read();
|
|
2088
|
-
this.#signal?.throwIfAborted();
|
|
2089
|
-
if (done) return void 0;
|
|
2090
|
-
if (value.byteLength > 0) return value;
|
|
2091
|
-
}
|
|
2092
|
-
}
|
|
2093
|
-
};
|
|
2094
|
-
//#endregion
|
|
2095
1776
|
//#region src/upload.ts
|
|
2096
1777
|
/** The object's own headers, which a multipart upload sends with the request that starts it. */
|
|
2097
1778
|
function objectHeaders(write) {
|
|
@@ -2114,18 +1795,19 @@ async function putObject(configuration, write, bytes) {
|
|
|
2114
1795
|
* Spec 7.6: a stream is read into parts of `partSize`, and one that ends within the first
|
|
2115
1796
|
* goes as the `PUT` held bytes get.
|
|
2116
1797
|
*/
|
|
2117
|
-
async function
|
|
2118
|
-
|
|
2119
|
-
|
|
2120
|
-
|
|
2121
|
-
|
|
2122
|
-
|
|
2123
|
-
|
|
2124
|
-
|
|
2125
|
-
|
|
2126
|
-
}
|
|
2127
|
-
|
|
2128
|
-
|
|
1798
|
+
async function putStream(configuration, write, stream) {
|
|
1799
|
+
return await uploadStream(stream, {
|
|
1800
|
+
partSize: configuration.partSize,
|
|
1801
|
+
concurrency: configuration.concurrency,
|
|
1802
|
+
maxParts,
|
|
1803
|
+
bucket: configuration.bucket,
|
|
1804
|
+
provider: "s3",
|
|
1805
|
+
key: write.key,
|
|
1806
|
+
signal: write.signal
|
|
1807
|
+
}, {
|
|
1808
|
+
whole: async (bytes) => await putObject(configuration, write, bytes),
|
|
1809
|
+
multipart: async (sendParts) => await multipartUpload(configuration, write, sendParts)
|
|
1810
|
+
});
|
|
2129
1811
|
}
|
|
2130
1812
|
const s3Namespace = "http://s3.amazonaws.com/doc/2006-03-01/";
|
|
2131
1813
|
/** The provider's limit on both sides, which ADR 0016 leaves no option to lift. */
|
|
@@ -2134,71 +1816,25 @@ const maxParts = 1e4;
|
|
|
2134
1816
|
* Spec 7.6: a multipart upload aborts itself once it failed, after the parts in flight
|
|
2135
1817
|
* and the source stream were canceled.
|
|
2136
1818
|
*/
|
|
2137
|
-
async function multipartUpload(configuration, write,
|
|
1819
|
+
async function multipartUpload(configuration, write, sendParts) {
|
|
2138
1820
|
const uploadId = await createUpload(configuration, write);
|
|
2139
1821
|
let sent;
|
|
2140
1822
|
try {
|
|
2141
|
-
sent = await sendParts(
|
|
1823
|
+
sent = await sendParts(async (index, bytes, signal) => {
|
|
1824
|
+
const number = index + 1;
|
|
1825
|
+
return {
|
|
1826
|
+
number,
|
|
1827
|
+
etag: await uploadPart(configuration, {
|
|
1828
|
+
...write,
|
|
1829
|
+
signal
|
|
1830
|
+
}, uploadId, number, bytes)
|
|
1831
|
+
};
|
|
1832
|
+
});
|
|
2142
1833
|
} catch (failure) {
|
|
2143
|
-
await parts.cancel(failure);
|
|
2144
1834
|
await abortUpload(configuration, write, uploadId);
|
|
2145
1835
|
throw failure;
|
|
2146
1836
|
}
|
|
2147
|
-
return await completeUpload(configuration, write, uploadId, sent);
|
|
2148
|
-
}
|
|
2149
|
-
/**
|
|
2150
|
-
* Spec 7.6: `concurrency` parts in flight, and the next part read only once one of them
|
|
2151
|
-
* settled, so the part buffers never outnumber the parts in flight. The first failure
|
|
2152
|
-
* stops the parts still in flight, and is what the upload rejects with once they settled.
|
|
2153
|
-
*/
|
|
2154
|
-
async function sendParts(configuration, write, uploadId, parts, first) {
|
|
2155
|
-
const stop = new AbortController();
|
|
2156
|
-
const partWrite = {
|
|
2157
|
-
...write,
|
|
2158
|
-
signal: write.signal === void 0 ? stop.signal : AbortSignal.any([write.signal, stop.signal])
|
|
2159
|
-
};
|
|
2160
|
-
const inFlight = /* @__PURE__ */ new Set();
|
|
2161
|
-
const uploaded = [];
|
|
2162
|
-
let failure;
|
|
2163
|
-
let size = 0;
|
|
2164
|
-
const fail = (reason) => {
|
|
2165
|
-
failure ??= { reason };
|
|
2166
|
-
stop.abort();
|
|
2167
|
-
parts.cancel(reason);
|
|
2168
|
-
};
|
|
2169
|
-
const sendPart = async (number, part) => {
|
|
2170
|
-
try {
|
|
2171
|
-
const etag = await uploadPart(configuration, partWrite, uploadId, number, part);
|
|
2172
|
-
uploaded.push({
|
|
2173
|
-
number,
|
|
2174
|
-
etag
|
|
2175
|
-
});
|
|
2176
|
-
} catch (reason) {
|
|
2177
|
-
fail(reason);
|
|
2178
|
-
} finally {
|
|
2179
|
-
parts.recycle(part);
|
|
2180
|
-
}
|
|
2181
|
-
};
|
|
2182
|
-
try {
|
|
2183
|
-
for (let number = 1, part = first; !stop.signal.aborted; number += 1) {
|
|
2184
|
-
if (number === maxParts && !part.last) throw tooManyParts(configuration, write);
|
|
2185
|
-
const sending = sendPart(number, part);
|
|
2186
|
-
inFlight.add(sending);
|
|
2187
|
-
sending.finally(() => inFlight.delete(sending));
|
|
2188
|
-
size += part.bytes.byteLength;
|
|
2189
|
-
if (part.last) break;
|
|
2190
|
-
while (inFlight.size >= configuration.concurrency) await Promise.race(inFlight);
|
|
2191
|
-
part = await parts.next();
|
|
2192
|
-
}
|
|
2193
|
-
} catch (reason) {
|
|
2194
|
-
fail(reason);
|
|
2195
|
-
}
|
|
2196
|
-
await Promise.all(inFlight);
|
|
2197
|
-
if (failure !== void 0) throw failure.reason;
|
|
2198
|
-
return {
|
|
2199
|
-
uploaded: uploaded.toSorted((one, other) => one.number - other.number),
|
|
2200
|
-
size
|
|
2201
|
-
};
|
|
1837
|
+
return await completeUpload(configuration, write, uploadId, sent.results, sent.size);
|
|
2202
1838
|
}
|
|
2203
1839
|
async function createUpload(configuration, write) {
|
|
2204
1840
|
const response = await send(configuration, {
|
|
@@ -2214,13 +1850,13 @@ async function createUpload(configuration, write) {
|
|
|
2214
1850
|
if (uploadId === void 0 || uploadId === "") throw incompleteAnswer(configuration, write, response, subject, "no upload id");
|
|
2215
1851
|
return uploadId;
|
|
2216
1852
|
}
|
|
2217
|
-
async function uploadPart(configuration, write, uploadId, number,
|
|
1853
|
+
async function uploadPart(configuration, write, uploadId, number, bytes) {
|
|
2218
1854
|
const response = await send(configuration, {
|
|
2219
1855
|
method: "PUT",
|
|
2220
1856
|
operation: "put",
|
|
2221
1857
|
key: write.key,
|
|
2222
1858
|
query: [["partNumber", String(number)], ["uploadId", uploadId]],
|
|
2223
|
-
body:
|
|
1859
|
+
body: bytes,
|
|
2224
1860
|
signal: write.signal
|
|
2225
1861
|
});
|
|
2226
1862
|
await response.body?.cancel();
|
|
@@ -2228,7 +1864,7 @@ async function uploadPart(configuration, write, uploadId, number, part) {
|
|
|
2228
1864
|
if (etag === null || etag === "") throw incompleteAnswer(configuration, write, response, `part ${number}`, "no entity tag");
|
|
2229
1865
|
return etag;
|
|
2230
1866
|
}
|
|
2231
|
-
async function completeUpload(configuration, write, uploadId,
|
|
1867
|
+
async function completeUpload(configuration, write, uploadId, uploaded, size) {
|
|
2232
1868
|
let response;
|
|
2233
1869
|
let document;
|
|
2234
1870
|
try {
|
|
@@ -2289,19 +1925,6 @@ function answeredRequest(configuration, write, subject) {
|
|
|
2289
1925
|
subject
|
|
2290
1926
|
};
|
|
2291
1927
|
}
|
|
2292
|
-
/**
|
|
2293
|
-
* Spec 7.6: known once the last part the provider takes is full and the stream goes on.
|
|
2294
|
-
* ADR 0016 fixes the part size before the first part, so the way past it is a larger one.
|
|
2295
|
-
*/
|
|
2296
|
-
function tooManyParts(configuration, write) {
|
|
2297
|
-
return s3Error(configuration.bucket, {
|
|
2298
|
-
code: "InvalidRequest",
|
|
2299
|
-
message: `The stream needs more than ${maxParts} parts of the configured \`partSize\` of ${configuration.partSize} bytes; a larger \`multipart.partSize\` carries it`,
|
|
2300
|
-
operation: "put",
|
|
2301
|
-
key: write.key,
|
|
2302
|
-
attempts: 0
|
|
2303
|
-
});
|
|
2304
|
-
}
|
|
2305
1928
|
function incompleteAnswer(configuration, write, response, subject, missing) {
|
|
2306
1929
|
return s3Error(configuration.bucket, {
|
|
2307
1930
|
code: "ProviderError",
|
|
@@ -2322,7 +1945,8 @@ function s3Storage(options) {
|
|
|
2322
1945
|
const s3Capabilities = Object.freeze([
|
|
2323
1946
|
"presignedUrls",
|
|
2324
1947
|
"rangeReads",
|
|
2325
|
-
"userMetadata"
|
|
1948
|
+
"userMetadata",
|
|
1949
|
+
"userMetadataTokenKeys"
|
|
2326
1950
|
]);
|
|
2327
1951
|
const utf8 = new TextEncoder();
|
|
2328
1952
|
var SimpleStorageServiceStorage = class {
|
|
@@ -2347,12 +1971,12 @@ var SimpleStorageServiceStorage = class {
|
|
|
2347
1971
|
requireKnownOptions(this.bucket, options, putOptionKeys, "put");
|
|
2348
1972
|
const write = {
|
|
2349
1973
|
key,
|
|
2350
|
-
userMetadata: userMetadataHeaders(this.bucket, options?.userMetadata, key),
|
|
1974
|
+
userMetadata: userMetadataHeaders(this.bucket, options?.userMetadata, key, this.capabilities),
|
|
2351
1975
|
contentType: this.#readContentType(options?.contentType),
|
|
2352
1976
|
signal: options?.signal
|
|
2353
1977
|
};
|
|
2354
1978
|
options?.signal?.throwIfAborted();
|
|
2355
|
-
if (isStream(body)) return await
|
|
1979
|
+
if (isStream(body)) return await putStream(this.#configuration, write, body);
|
|
2356
1980
|
return await putObject(this.#configuration, write, bytesOf(body));
|
|
2357
1981
|
}
|
|
2358
1982
|
async get(key, options) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@stowage/adapter-s3",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"description": "stowage adapter for one bucket of AWS S3 or Cloudflare R2, speaking the S3 wire protocol without an SDK.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"adapter",
|
|
@@ -35,7 +35,7 @@
|
|
|
35
35
|
"provenance": true
|
|
36
36
|
},
|
|
37
37
|
"dependencies": {
|
|
38
|
-
"@stowage/core": "^0.
|
|
38
|
+
"@stowage/core": "^0.2.0"
|
|
39
39
|
},
|
|
40
40
|
"engines": {
|
|
41
41
|
"node": ">=24"
|