@ultimat3/storage 29.3.2 → 30.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +27 -47
- package/README.md +130 -12
- package/package.json +12 -2
- package/src/accept-part.ts +30 -0
- package/src/accept.ts +7 -0
- package/src/attachment.ts +99 -6
- package/src/byte-pump.ts +46 -0
- package/src/driver-local-list.ts +75 -0
- package/src/driver-local-multipart.ts +133 -0
- package/src/driver-local-read.ts +13 -0
- package/src/driver-local-remove.ts +39 -0
- package/src/driver-local-stream.ts +104 -0
- package/src/driver-local-write.ts +28 -5
- package/src/driver-local.ts +257 -283
- package/src/driver-memory-multipart.ts +41 -0
- package/src/driver-memory.ts +205 -135
- package/src/driver-s3-client.ts +9 -0
- package/src/driver-s3-copy.ts +35 -0
- package/src/driver-s3-lock.ts +9 -2
- package/src/driver-s3-multipart-session.ts +87 -0
- package/src/driver-s3-multipart.ts +177 -0
- package/src/driver-s3-presign.ts +52 -0
- package/src/driver-s3-put-stream.ts +97 -0
- package/src/driver-s3-range.ts +99 -0
- package/src/driver-s3-settle.ts +150 -0
- package/src/driver-s3-signed.ts +69 -4
- package/src/driver-s3-sse.ts +29 -0
- package/src/driver-s3.ts +113 -65
- package/src/driver.ts +98 -38
- package/src/errors-multipart.ts +35 -0
- package/src/errors-promote.ts +14 -0
- package/src/errors-range.ts +47 -0
- package/src/errors-sniff.ts +35 -0
- package/src/errors-stream.ts +62 -0
- package/src/errors.ts +16 -1
- package/src/expire.ts +169 -0
- package/src/grant-multipart.ts +196 -0
- package/src/grant.ts +42 -25
- package/src/image-warm.ts +84 -0
- package/src/image.ts +57 -0
- package/src/index.ts +81 -2
- package/src/multipart-staged.ts +127 -0
- package/src/multipart.ts +119 -0
- package/src/put-stream.ts +52 -0
- package/src/range.ts +102 -0
- package/src/signed-url.ts +59 -2
- package/src/subject-target.ts +82 -0
- package/src/upload-client-multipart.ts +156 -0
- package/src/upload-client.ts +21 -0
- package/src/upload.ts +107 -21
package/src/driver-s3-signed.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// Single responsibility: the s3 disk's SIGNED requests — the ones `Bun.S3Client` has no option for
|
|
2
|
-
// (`x-amz-meta-*`, `Cache-Control`,
|
|
2
|
+
// (`x-amz-meta-*`, `Cache-Control`, object lock, a ranged GET read off its own `Content-Range`) — built,
|
|
3
3
|
// signed through core's one SigV4 signer and sent with `fetch`. The endpoint, the addressing style,
|
|
4
4
|
// the region and the credentials are read from the SAME `S3DriverOptions` the Bun client is built
|
|
5
5
|
// from, so the two transports of one disk cannot point at two buckets.
|
|
@@ -9,9 +9,11 @@ import {
|
|
|
9
9
|
type Clock,
|
|
10
10
|
ConfigInvalidError,
|
|
11
11
|
EnvMissingError,
|
|
12
|
+
presignAwsUrl,
|
|
12
13
|
signAwsRequest,
|
|
13
14
|
systemClock,
|
|
14
15
|
} from '@ultimat3/core';
|
|
16
|
+
import type { SignedPart } from './signed-url';
|
|
15
17
|
|
|
16
18
|
/** The slice of `fetch` this file uses; injected in tests so no socket is ever opened. */
|
|
17
19
|
export type S3FetchLike = (
|
|
@@ -51,10 +53,12 @@ export interface S3Refusal {
|
|
|
51
53
|
}
|
|
52
54
|
|
|
53
55
|
export interface S3SignedRequest {
|
|
54
|
-
readonly method: 'GET' | 'PUT';
|
|
56
|
+
readonly method: 'GET' | 'PUT' | 'POST' | 'DELETE';
|
|
55
57
|
readonly key: string;
|
|
56
58
|
/** A sub-resource such as `retention` — sent and signed as `?retention=`. */
|
|
57
59
|
readonly subresource?: string | undefined;
|
|
60
|
+
/** Query parameters of a multipart call, signed. An empty value is sent as `uploads=`. */
|
|
61
|
+
readonly query?: Readonly<Record<string, string>> | undefined;
|
|
58
62
|
readonly headers?: Readonly<Record<string, string>> | undefined;
|
|
59
63
|
readonly body?: Uint8Array | undefined;
|
|
60
64
|
}
|
|
@@ -122,6 +126,61 @@ export function resolveS3Target(options: S3WireOptions): S3Target {
|
|
|
122
126
|
};
|
|
123
127
|
}
|
|
124
128
|
|
|
129
|
+
/**
|
|
130
|
+
* A PUT URL that signs its `content-length`, which is what `Bun.S3Client.presign` has no option for.
|
|
131
|
+
* The provider recomputes the signature from the header the browser really sent, so the only body
|
|
132
|
+
* this URL lands is one of exactly `contentLength` bytes. A browser sets that header itself.
|
|
133
|
+
*/
|
|
134
|
+
export async function presignBoundPut(
|
|
135
|
+
options: S3WireOptions,
|
|
136
|
+
key: string,
|
|
137
|
+
contentLength: number,
|
|
138
|
+
expiresInSeconds: number,
|
|
139
|
+
part?: SignedPart | undefined,
|
|
140
|
+
): Promise<string> {
|
|
141
|
+
const url = s3ObjectUrl(options, key);
|
|
142
|
+
// A part of an open upload: S3's own UploadPart, its two parameters inside the signature.
|
|
143
|
+
if (part !== undefined) {
|
|
144
|
+
url.searchParams.set('partNumber', String(part.partNumber));
|
|
145
|
+
url.searchParams.set('uploadId', part.uploadId);
|
|
146
|
+
}
|
|
147
|
+
const presigned = await presignAwsUrl({
|
|
148
|
+
method: 'PUT',
|
|
149
|
+
url,
|
|
150
|
+
expiresInSeconds,
|
|
151
|
+
headers: { 'content-length': String(contentLength) },
|
|
152
|
+
credentials: s3Credentials(options),
|
|
153
|
+
region: signingRegion(options),
|
|
154
|
+
service: 's3',
|
|
155
|
+
clock: options.clock ?? systemClock,
|
|
156
|
+
});
|
|
157
|
+
return presigned.url;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/** The labels between `s3.` and `.amazonaws.com` of an AWS host: its region, or none. */
|
|
161
|
+
const AWS_HOST = /(?:^|\.)s3\.(?:(.+)\.)?amazonaws\.com$/i;
|
|
162
|
+
const REGION = /^[a-z0-9-]{1,64}$/;
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* The region a request is signed for, as `Bun.S3Client` derives it, measured on Bun 1.4.4 so the
|
|
166
|
+
* disk's two transports sign alike: a region set wins; no endpoint is AWS's `us-east-1`; an AWS host
|
|
167
|
+
* `s3.<region>.amazonaws.com` is that region, bare `s3.amazonaws.com` is `us-east-1`; any other host
|
|
168
|
+
* is `auto` (R2, a gateway). One deviation: Bun reads `s3.dualstack.<region>` as the whole
|
|
169
|
+
* `dualstack.<region>`, which is no region, and this reads the region.
|
|
170
|
+
*/
|
|
171
|
+
export function signingRegion(options: S3WireOptions): string {
|
|
172
|
+
const { region, endpoint } = resolveS3Target(options);
|
|
173
|
+
if (region !== undefined) return region;
|
|
174
|
+
if (endpoint === undefined) return 'us-east-1';
|
|
175
|
+
const host = URL.canParse(endpoint) ? new URL(endpoint).hostname : '';
|
|
176
|
+
const match = AWS_HOST.exec(host);
|
|
177
|
+
if (match === null) return 'auto';
|
|
178
|
+
const named = match[1];
|
|
179
|
+
if (named === undefined) return 'us-east-1';
|
|
180
|
+
const last = named.split('.').at(-1) ?? '';
|
|
181
|
+
return REGION.test(named) ? named : REGION.test(last) ? last : 'auto';
|
|
182
|
+
}
|
|
183
|
+
|
|
125
184
|
/** Read lazily, at the first signed request — importing or constructing the disk reads nothing. */
|
|
126
185
|
export function s3Credentials(options: S3WireOptions): AwsCredentials {
|
|
127
186
|
const env = options.env ?? process.env;
|
|
@@ -153,7 +212,10 @@ export function s3ObjectUrl(options: S3WireOptions, key: string): URL {
|
|
|
153
212
|
);
|
|
154
213
|
}
|
|
155
214
|
|
|
156
|
-
|
|
215
|
+
// Uncapped: a key is up to 1,024 bytes and S3 escapes it (`&` is `&`), so a 512-character
|
|
216
|
+
// cap dropped a long key's session from `listS3Uploads` and ended its paging at a long
|
|
217
|
+
// `NextKeyMarker`. `[^<]*` stops at the next tag, so the match stays linear with no cap.
|
|
218
|
+
const XML_FIELD = (name: string): RegExp => new RegExp(`<${name}>([^<]*)</${name}>`);
|
|
157
219
|
|
|
158
220
|
/** The first `<name>…</name>` of an S3 XML answer — the shapes read here are flat and fixed. */
|
|
159
221
|
export function xmlField(xml: string, name: string): string | undefined {
|
|
@@ -189,6 +251,9 @@ export async function sendSigned(
|
|
|
189
251
|
}
|
|
190
252
|
const url = s3ObjectUrl(options, request.key);
|
|
191
253
|
if (request.subresource !== undefined) url.search = `${request.subresource}=`;
|
|
254
|
+
for (const [name, value] of Object.entries(request.query ?? {})) {
|
|
255
|
+
url.searchParams.set(name, value);
|
|
256
|
+
}
|
|
192
257
|
const signed = await signAwsRequest({
|
|
193
258
|
method: request.method,
|
|
194
259
|
url,
|
|
@@ -197,7 +262,7 @@ export async function sendSigned(
|
|
|
197
262
|
// refuse a body that changed in flight; a read has none.
|
|
198
263
|
payload: { body: request.body ?? new Uint8Array() },
|
|
199
264
|
credentials: s3Credentials(options),
|
|
200
|
-
region:
|
|
265
|
+
region: signingRegion(options),
|
|
201
266
|
service: 's3',
|
|
202
267
|
clock: options.clock ?? systemClock,
|
|
203
268
|
});
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
// Single responsibility: the one `put()` option the s3 disk cannot honour, refused by name with the
|
|
2
|
+
// bucket-wide command that does the job instead.
|
|
3
|
+
|
|
4
|
+
import { NotImplementedError, renderFixLiteral, renderFixShellArg } from '@ultimat3/core';
|
|
5
|
+
import type { PutOptions } from './driver';
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Everything `put()` may be handed that this driver cannot honour, refused before a byte moves.
|
|
9
|
+
* A typed refusal at the call site is the whole point: `serverSideEncryption` exists on
|
|
10
|
+
* `PutOptions` so that "this disk cannot prove per-object encryption" is something an engineer
|
|
11
|
+
* meets while writing the call, not while answering a security review.
|
|
12
|
+
*/
|
|
13
|
+
export function refuseUnsupportedPut(bucket: string, putOptions?: PutOptions): void {
|
|
14
|
+
const sse = putOptions?.serverSideEncryption;
|
|
15
|
+
if (sse === undefined) return;
|
|
16
|
+
// The key id is the caller's own option, spliced into single-quoted JSON: `renderFixLiteral`
|
|
17
|
+
// escapes the `"` and `\` JSON needs, and a `'` — the one byte live inside single quotes — makes
|
|
18
|
+
// it the placeholder rather than ending the argument (plan 101 row S12).
|
|
19
|
+
const keyId = sse.kmsKeyId?.includes("'") === false ? sse.kmsKeyId : undefined;
|
|
20
|
+
const bucketArg = renderFixShellArg(bucket, "'<bucket>'");
|
|
21
|
+
throw new NotImplementedError({
|
|
22
|
+
cause:
|
|
23
|
+
'per-object server-side encryption on the s3 driver (Bun.S3Client exposes acl, storageClass and type, and nothing for x-amz-server-side-encryption) is not implemented by this driver — set it bucket-wide with the command in fix, then drop serverSideEncryption from put()',
|
|
24
|
+
fix:
|
|
25
|
+
sse.algorithm === 'aws:kms'
|
|
26
|
+
? `aws s3api put-bucket-encryption --bucket ${bucketArg} --server-side-encryption-configuration '{"Rules":[{"ApplyServerSideEncryptionByDefault":{"SSEAlgorithm":"aws:kms","KMSMasterKeyID":${renderFixLiteral(keyId, '"<key-arn>"')}},"BucketKeyEnabled":true}]}'`
|
|
27
|
+
: `aws s3api put-bucket-encryption --bucket ${bucketArg} --server-side-encryption-configuration '{"Rules":[{"ApplyServerSideEncryptionByDefault":{"SSEAlgorithm":"AES256"},"BucketKeyEnabled":true}]}'`,
|
|
28
|
+
});
|
|
29
|
+
}
|
package/src/driver-s3.ts
CHANGED
|
@@ -8,16 +8,15 @@ import {
|
|
|
8
8
|
type Clock,
|
|
9
9
|
finiteCount,
|
|
10
10
|
isUltimateError,
|
|
11
|
-
NotImplementedError,
|
|
12
11
|
type Page,
|
|
13
12
|
pageOf,
|
|
14
|
-
renderFixLiteral,
|
|
15
13
|
renderFixShellArg,
|
|
16
14
|
systemClock,
|
|
17
15
|
} from '@ultimat3/core';
|
|
18
16
|
import {
|
|
19
17
|
assertListOptions,
|
|
20
18
|
assertPutOptions,
|
|
19
|
+
type CopyOptions,
|
|
21
20
|
DEFAULT_CONTENT_TYPE,
|
|
22
21
|
type ListOptions,
|
|
23
22
|
type PutOptions,
|
|
@@ -32,10 +31,17 @@ import {
|
|
|
32
31
|
toBytes,
|
|
33
32
|
} from './driver';
|
|
34
33
|
import { isAbsentObject } from './driver-s3-absent';
|
|
35
|
-
import { buildClient, type S3ClientLike, type S3ListResultLike } from './driver-s3-client';
|
|
34
|
+
import { buildClient, dated, type S3ClientLike, type S3ListResultLike } from './driver-s3-client';
|
|
35
|
+
import { signedCopy } from './driver-s3-copy';
|
|
36
36
|
import { needsSignedPut, readObjectLock, signedPut } from './driver-s3-lock';
|
|
37
|
+
import { s3Multipart } from './driver-s3-multipart-session';
|
|
38
|
+
import { s3SignedUrl } from './driver-s3-presign';
|
|
39
|
+
import { putS3Stream } from './driver-s3-put-stream';
|
|
40
|
+
import { readS3Range } from './driver-s3-range';
|
|
37
41
|
import { regionMismatch } from './driver-s3-region';
|
|
42
|
+
import { settleLostPut } from './driver-s3-settle';
|
|
38
43
|
import type { S3FetchLike } from './driver-s3-signed';
|
|
44
|
+
import { refuseUnsupportedPut } from './driver-s3-sse';
|
|
39
45
|
import {
|
|
40
46
|
checksumMismatch,
|
|
41
47
|
deleteFailed,
|
|
@@ -46,9 +52,17 @@ import {
|
|
|
46
52
|
putFailed,
|
|
47
53
|
readFailed,
|
|
48
54
|
} from './errors';
|
|
55
|
+
import { objectChanged } from './errors-promote';
|
|
49
56
|
import { assertObjectLockOptions, type ObjectLock } from './object-lock';
|
|
50
57
|
import { assertSafeKey } from './path';
|
|
51
|
-
import {
|
|
58
|
+
import { assertPutStream, type PutStreamOptions } from './put-stream';
|
|
59
|
+
import {
|
|
60
|
+
assertGetOptions,
|
|
61
|
+
assertStreamRange,
|
|
62
|
+
type ByteRange,
|
|
63
|
+
type GetOptions,
|
|
64
|
+
streamedRange,
|
|
65
|
+
} from './range';
|
|
52
66
|
import { DEFAULT_MAX_UPLOAD_BYTES } from './upload';
|
|
53
67
|
|
|
54
68
|
const DRIVER_NAME = 's3';
|
|
@@ -90,44 +104,13 @@ export interface S3DriverOptions {
|
|
|
90
104
|
*/
|
|
91
105
|
readonly maxPutBytes?: number | undefined;
|
|
92
106
|
/**
|
|
93
|
-
* Ceiling on ONE `get()`, because `get()` buffers the whole object — and
|
|
94
|
-
* bounds
|
|
95
|
-
*
|
|
107
|
+
* Ceiling on ONE `get()`, because `get()` buffers the whole object — and a URL signed with no
|
|
108
|
+
* `contentLength` bounds nothing it stored. Defaults to `maxPutBytes`; an object past it is read
|
|
109
|
+
* with `stream()`.
|
|
96
110
|
*/
|
|
97
111
|
readonly maxGetBytes?: number | undefined;
|
|
98
112
|
}
|
|
99
113
|
|
|
100
|
-
/**
|
|
101
|
-
* A missing `LastModified` is ABSENT, never `new Date(0)`: epoch 0 is older than every window, so
|
|
102
|
-
* `sweepOrphans` deleted an upload on the strength of a field the provider never sent.
|
|
103
|
-
*/
|
|
104
|
-
const dated = (value: string | Date | undefined): { readonly lastModified?: Date } =>
|
|
105
|
-
value === undefined ? {} : { lastModified: value instanceof Date ? value : new Date(value) };
|
|
106
|
-
|
|
107
|
-
/**
|
|
108
|
-
* Everything `put()` may be handed that this driver cannot honour, refused before a byte moves.
|
|
109
|
-
* A typed refusal at the call site is the whole point: `serverSideEncryption` exists on
|
|
110
|
-
* `PutOptions` so that "this disk cannot prove per-object encryption" is something an engineer
|
|
111
|
-
* meets while writing the call, not while answering a security review.
|
|
112
|
-
*/
|
|
113
|
-
function refuseUnsupportedPut(bucket: string, putOptions?: PutOptions): void {
|
|
114
|
-
const sse = putOptions?.serverSideEncryption;
|
|
115
|
-
if (sse === undefined) return;
|
|
116
|
-
// The key id is the caller's own option, spliced into single-quoted JSON: `renderFixLiteral`
|
|
117
|
-
// escapes the `"` and `\` JSON needs, and a `'` — the one byte live inside single quotes — makes
|
|
118
|
-
// it the placeholder rather than ending the argument (plan 101 row S12).
|
|
119
|
-
const keyId = sse.kmsKeyId?.includes("'") === false ? sse.kmsKeyId : undefined;
|
|
120
|
-
const bucketArg = renderFixShellArg(bucket, "'<bucket>'");
|
|
121
|
-
throw new NotImplementedError({
|
|
122
|
-
cause:
|
|
123
|
-
'per-object server-side encryption on the s3 driver (Bun.S3Client exposes acl, storageClass and type, and nothing for x-amz-server-side-encryption) is not implemented by this driver — set it bucket-wide with the command in fix, then drop serverSideEncryption from put()',
|
|
124
|
-
fix:
|
|
125
|
-
sse.algorithm === 'aws:kms'
|
|
126
|
-
? `aws s3api put-bucket-encryption --bucket ${bucketArg} --server-side-encryption-configuration '{"Rules":[{"ApplyServerSideEncryptionByDefault":{"SSEAlgorithm":"aws:kms","KMSMasterKeyID":${renderFixLiteral(keyId, '"<key-arn>"')}},"BucketKeyEnabled":true}]}'`
|
|
127
|
-
: `aws s3api put-bucket-encryption --bucket ${bucketArg} --server-side-encryption-configuration '{"Rules":[{"ApplyServerSideEncryptionByDefault":{"SSEAlgorithm":"AES256"},"BucketKeyEnabled":true}]}'`,
|
|
128
|
-
});
|
|
129
|
-
}
|
|
130
|
-
|
|
131
114
|
export function s3Driver(options: S3DriverOptions): StorageDriver {
|
|
132
115
|
// `=== undefined`, never `??`: `??` coalesces on `null` too, so an explicitly blanked key in a
|
|
133
116
|
// decoded JSON config took the default instead of the refusal `finiteCount` is here to raise.
|
|
@@ -214,7 +197,12 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
|
|
|
214
197
|
};
|
|
215
198
|
};
|
|
216
199
|
|
|
217
|
-
|
|
200
|
+
/** The HEAD a lost write's answer is settled against, and the `.catch` of a single PUT. */
|
|
201
|
+
const statOf = async (key: string) => ((await present(key)) ? statObject(key) : undefined);
|
|
202
|
+
const lostPut = (key: string, bytes: Uint8Array, attributes: PutOptions | undefined) =>
|
|
203
|
+
settleLostPut(statOf, { driver: DRIVER_NAME, key, bytes, attributes }, refusedWrite(key));
|
|
204
|
+
|
|
205
|
+
const disk: StorageDriver = {
|
|
218
206
|
name: DRIVER_NAME,
|
|
219
207
|
registerAs(name: string): void {
|
|
220
208
|
diskName = name;
|
|
@@ -238,7 +226,9 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
|
|
|
238
226
|
if (putOptions !== undefined && needsSignedPut(putOptions)) {
|
|
239
227
|
// Off the Bun client only for what it cannot send: `S3File.write` takes a type, an ACL and
|
|
240
228
|
// a storage class and no header. One request either way — never a PUT then a second call.
|
|
241
|
-
const refusal = await signedPut(options, safe, bytes, putOptions).catch(
|
|
229
|
+
const refusal = await signedPut(options, safe, bytes, putOptions).catch(
|
|
230
|
+
lostPut(safe, bytes, putOptions),
|
|
231
|
+
);
|
|
242
232
|
if (refusal !== undefined) refusedWrite(safe)(refusal);
|
|
243
233
|
// Stored — the PUT succeeded with them — so reported, as the local disk reports its own.
|
|
244
234
|
return {
|
|
@@ -252,17 +242,66 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
|
|
|
252
242
|
await conn()
|
|
253
243
|
.file(safe)
|
|
254
244
|
.write(bytes, { type: putOptions?.contentType ?? DEFAULT_CONTENT_TYPE })
|
|
255
|
-
.catch(
|
|
245
|
+
.catch(lostPut(safe, bytes, putOptions));
|
|
256
246
|
return statObject(safe);
|
|
257
247
|
},
|
|
258
248
|
|
|
249
|
+
async putStream(
|
|
250
|
+
key: string,
|
|
251
|
+
body: ReadableStream<Uint8Array>,
|
|
252
|
+
streamOptions: PutStreamOptions,
|
|
253
|
+
): Promise<StorageObject> {
|
|
254
|
+
const safe = assertSafeKey(key);
|
|
255
|
+
const maxBytes = assertPutStream(DRIVER_NAME, body, streamOptions);
|
|
256
|
+
await putS3Stream(options, {
|
|
257
|
+
key: safe,
|
|
258
|
+
body,
|
|
259
|
+
options: streamOptions,
|
|
260
|
+
maxBytes,
|
|
261
|
+
driver: DRIVER_NAME,
|
|
262
|
+
refused: refusedWrite(safe),
|
|
263
|
+
stat: statOf,
|
|
264
|
+
}).catch(refusedWrite(safe));
|
|
265
|
+
return {
|
|
266
|
+
...(await statObject(safe)),
|
|
267
|
+
...(streamOptions.cacheControl === undefined
|
|
268
|
+
? {}
|
|
269
|
+
: { cacheControl: streamOptions.cacheControl }),
|
|
270
|
+
...(streamOptions.metadata === undefined
|
|
271
|
+
? {}
|
|
272
|
+
: { metadata: { ...streamOptions.metadata } }),
|
|
273
|
+
};
|
|
274
|
+
},
|
|
275
|
+
|
|
259
276
|
async stat(key: string): Promise<StorageObject | undefined> {
|
|
260
277
|
const safe = assertSafeKey(key);
|
|
261
278
|
return (await present(safe)) ? statObject(safe) : undefined;
|
|
262
279
|
},
|
|
263
280
|
|
|
264
|
-
async get(key: string): Promise<StorageRead> {
|
|
281
|
+
async get(key: string, getOptions?: GetOptions): Promise<StorageRead> {
|
|
265
282
|
const safe = assertSafeKey(key);
|
|
283
|
+
assertGetOptions(DRIVER_NAME, getOptions);
|
|
284
|
+
const range = getOptions?.range;
|
|
285
|
+
// ONE signed GET, bytes and description from the same response (`driver-s3-range.ts`).
|
|
286
|
+
if (range !== undefined) {
|
|
287
|
+
return readS3Range(options, safe, range, {
|
|
288
|
+
driver: DRIVER_NAME,
|
|
289
|
+
maxGetBytes,
|
|
290
|
+
refused: (key, refusal) => refusedRead(key)(refusal),
|
|
291
|
+
sizeOf: async (key) => {
|
|
292
|
+
if (!(await present(key))) throw objectNotFound(diskName, key);
|
|
293
|
+
return (await statObject(key)).size;
|
|
294
|
+
},
|
|
295
|
+
unreadable: (key, why) => {
|
|
296
|
+
throw readFailed(
|
|
297
|
+
DRIVER_NAME,
|
|
298
|
+
key,
|
|
299
|
+
why,
|
|
300
|
+
`aws s3api get-object --bucket ${renderFixShellArg(options.bucket, "'<bucket>'")} --key ${renderFixShellArg(key, "'<key>'")} --range bytes=0-0 /dev/null # what the provider answers a ranged GET`,
|
|
301
|
+
);
|
|
302
|
+
},
|
|
303
|
+
});
|
|
304
|
+
}
|
|
266
305
|
const file = conn().file(safe);
|
|
267
306
|
if (!(await present(safe))) throw objectNotFound(diskName, safe);
|
|
268
307
|
// The HEAD comes FIRST, and it is the whole point: a client PUT straight into the bucket is
|
|
@@ -281,6 +320,20 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
|
|
|
281
320
|
return file.stream();
|
|
282
321
|
},
|
|
283
322
|
|
|
323
|
+
async streamRange(key: string, range: ByteRange): Promise<ReadableStream<Uint8Array>> {
|
|
324
|
+
const safe = assertSafeKey(key);
|
|
325
|
+
assertStreamRange(DRIVER_NAME, range);
|
|
326
|
+
if (!(await present(safe))) throw objectNotFound(diskName, safe);
|
|
327
|
+
// The HEAD first: past the end, Bun's slice raises from inside the stream, after a caller has
|
|
328
|
+
// already answered 206. Refused here it is still a status.
|
|
329
|
+
const { size } = await statObject(safe);
|
|
330
|
+
const [first, last] = streamedRange(DRIVER_NAME, safe, range, size);
|
|
331
|
+
return conn()
|
|
332
|
+
.file(safe)
|
|
333
|
+
.slice(first, last + 1)
|
|
334
|
+
.stream();
|
|
335
|
+
},
|
|
336
|
+
|
|
284
337
|
/**
|
|
285
338
|
* Bytes from one key to another without a round trip through this process. Bun exposes no
|
|
286
339
|
* `CopyObject`, so the source `S3File` is handed to `write()` — Bun's own union accepts one —
|
|
@@ -288,9 +341,20 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
|
|
|
288
341
|
* win available today: promoting a 500MB attachment used to be `get()` + `put()`, a full GB
|
|
289
342
|
* resident in the pod. A true server-side copy needs a Bun API that does not exist in 1.3.
|
|
290
343
|
*/
|
|
291
|
-
async copy(from: string, to: string): Promise<StorageObject> {
|
|
344
|
+
async copy(from: string, to: string, copyOptions?: CopyOptions): Promise<StorageObject> {
|
|
292
345
|
const source = assertSafeKey(from);
|
|
293
346
|
const destination = assertSafeKey(to);
|
|
347
|
+
// Bun's `write` takes no condition, so a revision-bound copy is the signed CopyObject.
|
|
348
|
+
if (copyOptions?.ifMatch !== undefined) {
|
|
349
|
+
const refusal = await signedCopy(options, source, destination, copyOptions.ifMatch).catch(
|
|
350
|
+
refusedWrite(destination),
|
|
351
|
+
);
|
|
352
|
+
if (refusal?.statusCode === 404) throw objectNotFound(diskName, source);
|
|
353
|
+
if (refusal?.statusCode === 412)
|
|
354
|
+
throw objectChanged(source, copyOptions.ifMatch, 'another');
|
|
355
|
+
if (refusal !== undefined) refusedWrite(destination)(refusal);
|
|
356
|
+
return statObject(destination);
|
|
357
|
+
}
|
|
294
358
|
const file = conn().file(source);
|
|
295
359
|
if (!(await present(source))) throw objectNotFound(diskName, source);
|
|
296
360
|
const stat = await file.stat().catch(refusedRead(source));
|
|
@@ -397,29 +461,13 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
|
|
|
397
461
|
);
|
|
398
462
|
},
|
|
399
463
|
|
|
400
|
-
/**
|
|
401
|
-
* Provider presigning. The signature covers method, expiry and content type but NOT
|
|
402
|
-
* `maxBytes` — S3 has no header for it, so size stays a server-side policy check.
|
|
403
|
-
*/
|
|
464
|
+
/** Both presigners, one rule: `driver-s3-presign.ts`. */
|
|
404
465
|
async signedUrl(key: string, urlOptions?: SignedUrlOptions): Promise<string> {
|
|
405
|
-
|
|
406
|
-
// finite-bound path for both presigners, never a private copy per package: `Math.ceil(NaN /
|
|
407
|
-
// 1000)` is `NaN`, so `X-Amz-Expires=NaN` went to AWS and the app got a 403 on a link it
|
|
408
|
-
// believed it had just minted. `=== undefined` rather than `??`, so an explicit `null` is
|
|
409
|
-
// refused instead of silently taking the default.
|
|
410
|
-
const expiresInMs = finiteCount(
|
|
411
|
-
'the s3 driver',
|
|
412
|
-
'expiresInMs',
|
|
413
|
-
urlOptions?.expiresInMs === undefined ? DEFAULT_SIGNED_URL_TTL_MS : urlOptions.expiresInMs,
|
|
414
|
-
1,
|
|
415
|
-
);
|
|
416
|
-
return conn()
|
|
417
|
-
.file(assertSafeKey(key))
|
|
418
|
-
.presign({
|
|
419
|
-
method: urlOptions?.method ?? 'GET',
|
|
420
|
-
expiresIn: Math.ceil(expiresInMs / 1000),
|
|
421
|
-
...(urlOptions?.contentType === undefined ? {} : { type: urlOptions.contentType }),
|
|
422
|
-
});
|
|
466
|
+
return s3SignedUrl(options, (safe) => conn().file(safe), key, urlOptions);
|
|
423
467
|
},
|
|
424
468
|
};
|
|
469
|
+
// Attached in place: the capability presigns and stats through the disk it belongs to.
|
|
470
|
+
return Object.assign(disk, {
|
|
471
|
+
multipart: s3Multipart({ wire: options, driver: disk, refusedWrite }),
|
|
472
|
+
});
|
|
425
473
|
}
|
package/src/driver.ts
CHANGED
|
@@ -5,9 +5,13 @@
|
|
|
5
5
|
|
|
6
6
|
import type { Clock, Page } from '@ultimat3/core';
|
|
7
7
|
import { assert } from '@ultimat3/core';
|
|
8
|
+
import { pumpBounded } from './byte-pump';
|
|
8
9
|
import { putTooLarge } from './errors';
|
|
10
|
+
import type { MultipartCapability } from './multipart';
|
|
9
11
|
import type { ObjectLock, ObjectLockOptions } from './object-lock';
|
|
10
|
-
import type {
|
|
12
|
+
import type { PutStreamOptions } from './put-stream';
|
|
13
|
+
import type { ByteRange, GetOptions } from './range';
|
|
14
|
+
import type { SignedPart, SignedUrlVerification } from './signed-url';
|
|
11
15
|
|
|
12
16
|
export type StorageBody = Uint8Array | ReadableStream<Uint8Array> | Blob;
|
|
13
17
|
|
|
@@ -81,8 +85,15 @@ export interface ListOptions {
|
|
|
81
85
|
}
|
|
82
86
|
|
|
83
87
|
export interface StorageRead {
|
|
88
|
+
/** The WHOLE object's description — `size` is the object's, never the range's. */
|
|
84
89
|
readonly object: StorageObject;
|
|
85
90
|
readonly bytes: Uint8Array;
|
|
91
|
+
/**
|
|
92
|
+
* The bytes served, both inclusive, after the clamp — present exactly when `range` was asked.
|
|
93
|
+
* A third-party driver that ignored the option answers none, and that absence is how a caller
|
|
94
|
+
* tells a slice from the whole object rather than trusting `bytes.byteLength`.
|
|
95
|
+
*/
|
|
96
|
+
readonly range?: ByteRange | undefined;
|
|
86
97
|
}
|
|
87
98
|
|
|
88
99
|
export type SignedUrlMethod = 'GET' | 'PUT';
|
|
@@ -91,18 +102,30 @@ export interface SignedUrlOptions {
|
|
|
91
102
|
readonly method?: SignedUrlMethod | undefined;
|
|
92
103
|
readonly expiresInMs?: number | undefined;
|
|
93
104
|
/**
|
|
94
|
-
* Ceiling on a `PUT
|
|
95
|
-
*
|
|
96
|
-
* `
|
|
97
|
-
* `s3Driver` CANNOT: S3 has no request header for a size and Bun's `presign` covers method,
|
|
98
|
-
* expiry and content type only — so the client PUTs straight into the bucket and nothing between
|
|
99
|
-
* the grant and the object enforces this number. It is not refused there, because `grantUpload`
|
|
100
|
-
* supplies it on every grant and refusing would break every s3 upload an app mints; the ceiling
|
|
101
|
-
* a bucket-backed disk actually gets is a bucket rule or a post-upload check on `object.size`.
|
|
102
|
-
* `driver-parity.test.ts` pins both halves so neither moves alone.
|
|
105
|
+
* Ceiling on a `PUT`. `localDriver` signs it into the URL, so `acceptSignedUpload` refuses a client
|
|
106
|
+
* that widens it. A provider's URL has no ceiling to sign, so `s3Driver` refuses it without
|
|
107
|
+
* `contentLength` (`X_STORAGE_UPLOAD_SIZE_UNDECLARED`) rather than mint a URL that ignores it.
|
|
103
108
|
*/
|
|
104
109
|
readonly maxBytes?: number | undefined;
|
|
110
|
+
/**
|
|
111
|
+
* The exact size of a `PUT`, inside the signature. On `s3Driver` the URL then signs
|
|
112
|
+
* `content-length`, so the provider refuses any other length, a chunked body with none included
|
|
113
|
+
* (403 `SignatureDoesNotMatch`). The local disks hold it as a ceiling (`x-max`): they count what
|
|
114
|
+
* arrives, so a shorter body passes there and not on s3. `driver-parity.test.ts` pins both.
|
|
115
|
+
*/
|
|
116
|
+
readonly contentLength?: number | undefined;
|
|
105
117
|
readonly contentType?: string | undefined;
|
|
118
|
+
/**
|
|
119
|
+
* A part of an open upload, inside the signature: the URL then writes that part and nothing else.
|
|
120
|
+
* Only a disk whose `multipart.receivePart` stores parts signs one; `s3Driver` refuses it, since
|
|
121
|
+
* its parts are signed by `multipart.signPart` against the provider.
|
|
122
|
+
*/
|
|
123
|
+
readonly part?: SignedPart | undefined;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
export interface CopyOptions {
|
|
127
|
+
/** The source's `etag` as this disk reported it. */
|
|
128
|
+
readonly ifMatch?: string | undefined;
|
|
106
129
|
}
|
|
107
130
|
|
|
108
131
|
export interface StorageDriver {
|
|
@@ -125,6 +148,16 @@ export interface StorageDriver {
|
|
|
125
148
|
*/
|
|
126
149
|
registerAs?(diskName: string): void;
|
|
127
150
|
put(key: string, body: StorageBody, options?: PutOptions): Promise<StorageObject>;
|
|
151
|
+
/**
|
|
152
|
+
* A body of any length, without buffering it: counted against the call's required `maxBytes`, and
|
|
153
|
+
* past it the write is abandoned with the key as it was (`X_STORAGE_TOO_LARGE`). Takes no
|
|
154
|
+
* `checksum`, retention or legal hold, which need the bytes before the object exists.
|
|
155
|
+
*/
|
|
156
|
+
putStream(
|
|
157
|
+
key: string,
|
|
158
|
+
stream: ReadableStream<Uint8Array>,
|
|
159
|
+
options: PutStreamOptions,
|
|
160
|
+
): Promise<StorageObject>;
|
|
128
161
|
/**
|
|
129
162
|
* What a read would say about the object, WITHOUT its bytes — `undefined` when nothing is there.
|
|
130
163
|
* `promoteAttachment` decides on `stat().size`, because a presigned PUT lands on the disk
|
|
@@ -134,17 +167,26 @@ export interface StorageDriver {
|
|
|
134
167
|
stat(key: string): Promise<StorageObject | undefined>;
|
|
135
168
|
/**
|
|
136
169
|
* The whole object, BUFFERED — so bounded: an object over the driver's `maxGetBytes` is
|
|
137
|
-
* `X_STORAGE_TOO_LARGE`, exactly as `put()` refuses a body over `maxPutBytes`. Past it, `stream()
|
|
170
|
+
* `X_STORAGE_TOO_LARGE`, exactly as `put()` refuses a body over `maxPutBytes`. Past it, `stream()`
|
|
171
|
+
* — or `{ range: [first, last] }`, which reads only those bytes (inclusive) and is bounded by
|
|
172
|
+
* the range's length: s3 sends `Range:`, the local disk slices the file, memory slices its copy.
|
|
138
173
|
*/
|
|
139
|
-
get(key: string): Promise<StorageRead>;
|
|
174
|
+
get(key: string, options?: GetOptions): Promise<StorageRead>;
|
|
140
175
|
/** Bytes without buffering — the only safe path for anything over a few MB. */
|
|
141
176
|
stream(key: string): Promise<ReadableStream<Uint8Array>>;
|
|
177
|
+
/**
|
|
178
|
+
* The bytes `[first, last]` (inclusive, clamped, as `get`'s range) with none before them read, so
|
|
179
|
+
* a seek late in a film costs its window. `X_STORAGE_RANGE_UNSATISFIABLE` past the end. A method and
|
|
180
|
+
* not an option on `stream`, which a third-party driver would ignore and serve the wrong bytes.
|
|
181
|
+
*/
|
|
182
|
+
streamRange(key: string, range: ByteRange): Promise<ReadableStream<Uint8Array>>;
|
|
142
183
|
/**
|
|
143
184
|
* Bytes from one key to another without them passing through this process. Idempotent in the
|
|
144
185
|
* destination and non-destructive in the source: the caller deletes the source afterwards if
|
|
145
|
-
* it wanted a move. `X_STORAGE_NOT_FOUND` when `from` is absent.
|
|
186
|
+
* it wanted a move. `X_STORAGE_NOT_FOUND` when `from` is absent. `ifMatch` copies only that
|
|
187
|
+
* revision: any other is `X_STORAGE_OBJECT_CHANGED` and writes nothing.
|
|
146
188
|
*/
|
|
147
|
-
copy(from: string, to: string): Promise<StorageObject>;
|
|
189
|
+
copy(from: string, to: string, options?: CopyOptions): Promise<StorageObject>;
|
|
148
190
|
/**
|
|
149
191
|
* Deleting an ABSENT key is not an error. Anything else is: a denied `s3:DeleteObject`, a
|
|
150
192
|
* throttle, an expired credential and a read-only mount all raise `X_STORAGE_DELETE_FAILED`,
|
|
@@ -183,6 +225,12 @@ export interface StorageDriver {
|
|
|
183
225
|
* provider and never reaches this path, and a third-party driver that implements neither is
|
|
184
226
|
* refused by name (`X_STORAGE_URL_UNVERIFIABLE`) rather than reported as a bad signature.
|
|
185
227
|
*/
|
|
228
|
+
/**
|
|
229
|
+
* An upload in parts: resumable, and past S3's 5 GiB single-PUT ceiling. Optional, so a
|
|
230
|
+
* third-party driver still compiles; `grantMultipart` refuses a disk without it
|
|
231
|
+
* (`X_STORAGE_MULTIPART_UNSUPPORTED`). Every shipped disk has one.
|
|
232
|
+
*/
|
|
233
|
+
readonly multipart?: MultipartCapability | undefined;
|
|
186
234
|
verifySigned?(input: {
|
|
187
235
|
readonly url: string;
|
|
188
236
|
/** Overrides the driver's own, so a test's frozen clock still governs expiry. */
|
|
@@ -241,6 +289,36 @@ export function assertPutOptions(driver: string, options: PutOptions | undefined
|
|
|
241
289
|
);
|
|
242
290
|
}
|
|
243
291
|
|
|
292
|
+
/**
|
|
293
|
+
* The number a signed URL holds, screened on every disk: `contentLength` when there is one, since it
|
|
294
|
+
* is the tighter fact, and a `contentLength` past `maxBytes` or on anything but a `PUT` is a call
|
|
295
|
+
* that contradicts itself.
|
|
296
|
+
*/
|
|
297
|
+
export function signedByteBound(
|
|
298
|
+
driver: string,
|
|
299
|
+
options: SignedUrlOptions | undefined,
|
|
300
|
+
): number | undefined {
|
|
301
|
+
const length = options?.contentLength;
|
|
302
|
+
if (length === undefined) return options?.maxBytes;
|
|
303
|
+
const maxBytes = options?.maxBytes;
|
|
304
|
+
assert(
|
|
305
|
+
Number.isSafeInteger(length) && length >= 0,
|
|
306
|
+
`signedUrl() on the ${driver} disk was given a contentLength that is not a whole byte count`,
|
|
307
|
+
"signedUrl(key, { method: 'PUT', contentLength: file.size })",
|
|
308
|
+
);
|
|
309
|
+
assert(
|
|
310
|
+
options?.method === 'PUT',
|
|
311
|
+
`signedUrl() on the ${driver} disk was given a contentLength on a read, which has no body to bind`,
|
|
312
|
+
"signedUrl(key, { method: 'PUT', contentLength: file.size })",
|
|
313
|
+
);
|
|
314
|
+
assert(
|
|
315
|
+
maxBytes === undefined || length <= maxBytes,
|
|
316
|
+
`signedUrl() on the ${driver} disk was given a contentLength of ${length}B past its maxBytes of ${String(maxBytes)}B`,
|
|
317
|
+
"signedUrl(key, { method: 'PUT', contentLength: file.size }) # at or under maxBytes",
|
|
318
|
+
);
|
|
319
|
+
return length;
|
|
320
|
+
}
|
|
321
|
+
|
|
244
322
|
/** A `prefix` or a `cursor` that is not a string reached `startsWith` and `<=` as whatever it was. */
|
|
245
323
|
export function assertListOptions(options: ListOptions | undefined): void {
|
|
246
324
|
for (const name of ['prefix', 'cursor'] as const) {
|
|
@@ -299,31 +377,13 @@ async function readBounded(
|
|
|
299
377
|
body: ReadableStream<Uint8Array>,
|
|
300
378
|
limit: ByteLimit,
|
|
301
379
|
): Promise<Uint8Array> {
|
|
302
|
-
const reader = body.getReader();
|
|
303
380
|
const chunks: Uint8Array[] = [];
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
if (total > limit.maxBytes) {
|
|
311
|
-
throw putTooLarge(limit.driver, limit.key, total, limit.maxBytes);
|
|
312
|
-
}
|
|
313
|
-
chunks.push(next.value);
|
|
314
|
-
}
|
|
315
|
-
} finally {
|
|
316
|
-
// Cancel rather than merely release: a refused body must stop arriving, not keep filling a
|
|
317
|
-
// socket buffer nobody will read. Already-closed streams answer this with a no-op.
|
|
318
|
-
await reader.cancel().catch(() => undefined);
|
|
319
|
-
}
|
|
320
|
-
const bytes = new Uint8Array(total);
|
|
321
|
-
let offset = 0;
|
|
322
|
-
for (const chunk of chunks) {
|
|
323
|
-
bytes.set(chunk, offset);
|
|
324
|
-
offset += chunk.byteLength;
|
|
325
|
-
}
|
|
326
|
-
return bytes;
|
|
381
|
+
const { driver, key, maxBytes } = limit;
|
|
382
|
+
const refuse = (total: number): Error => putTooLarge(driver, key, total, maxBytes);
|
|
383
|
+
await pumpBounded(body, { driver, maxBytes, refuse }, (chunk) => {
|
|
384
|
+
chunks.push(chunk);
|
|
385
|
+
});
|
|
386
|
+
return new Uint8Array(Bun.concatArrayBuffers(chunks));
|
|
327
387
|
}
|
|
328
388
|
|
|
329
389
|
/** base64 SHA-256 — the wire form for `PutOptions.checksum` and `ValidatedUpload.checksum`. */
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
// The refusals of an upload in parts. Registered with the rest of the package's codes in
|
|
2
|
+
// `errors.ts`; each `fix:` names the call that goes next, with the key and upload id screened.
|
|
3
|
+
|
|
4
|
+
import { renderFixLiteral } from '@ultimat3/core';
|
|
5
|
+
import { StorageError } from './errors';
|
|
6
|
+
|
|
7
|
+
const keyArg = (key: string): string => renderFixLiteral(key, "'<key>'");
|
|
8
|
+
const idArg = (uploadId: string): string => renderFixLiteral(uploadId, "'<uploadId>'");
|
|
9
|
+
|
|
10
|
+
/** A disk whose driver declares no `multipart`: a third-party driver, most likely. */
|
|
11
|
+
export const multipartUnsupported = (disk: string, key: string): StorageError =>
|
|
12
|
+
new StorageError({
|
|
13
|
+
code: 'X_STORAGE_MULTIPART_UNSUPPORTED',
|
|
14
|
+
cause: `the ${disk} disk's driver declares no multipart capability, so "${key}" cannot be uploaded in parts`,
|
|
15
|
+
fix: 'grantUpload({ disk, orgId, request }) # one PUT; or implement StorageDriver.multipart on the driver',
|
|
16
|
+
meta: { disk, key },
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
/** No open session by that id for that key: never opened, completed, or aborted. */
|
|
20
|
+
export const uploadUnknown = (disk: string, key: string, uploadId: string): StorageError =>
|
|
21
|
+
new StorageError({
|
|
22
|
+
code: 'X_STORAGE_UPLOAD_UNKNOWN',
|
|
23
|
+
cause: `no upload ${idArg(uploadId)} is open for "${key}" on the ${disk} disk: it was never opened there, or it was already completed or aborted`,
|
|
24
|
+
fix: 'grantMultipart({ disk, orgId, request }) # a new session, then send every part under its uploadId',
|
|
25
|
+
meta: { disk, key, uploadId },
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
/** The parts a Complete named do not match what the disk holds. */
|
|
29
|
+
export const partsInvalid = (key: string, uploadId: string, problem: string): StorageError =>
|
|
30
|
+
new StorageError({
|
|
31
|
+
code: 'X_STORAGE_PARTS_INVALID',
|
|
32
|
+
cause: `the parts named to complete "${key}" do not match the parts stored: ${problem}`,
|
|
33
|
+
fix: `await listParts({ disk, orgId, key: ${keyArg(key)}, uploadId: ${idArg(uploadId)} }) # what landed; resend the missing part, then completeUpload again`,
|
|
34
|
+
meta: { key, uploadId, problem },
|
|
35
|
+
});
|