@ultimat3/storage 29.3.2 → 30.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/CLAUDE.md +27 -47
  2. package/README.md +130 -12
  3. package/package.json +12 -2
  4. package/src/accept-part.ts +30 -0
  5. package/src/accept.ts +7 -0
  6. package/src/attachment.ts +99 -6
  7. package/src/byte-pump.ts +46 -0
  8. package/src/driver-local-list.ts +75 -0
  9. package/src/driver-local-multipart.ts +133 -0
  10. package/src/driver-local-read.ts +13 -0
  11. package/src/driver-local-remove.ts +39 -0
  12. package/src/driver-local-stream.ts +104 -0
  13. package/src/driver-local-write.ts +28 -5
  14. package/src/driver-local.ts +257 -283
  15. package/src/driver-memory-multipart.ts +41 -0
  16. package/src/driver-memory.ts +205 -135
  17. package/src/driver-s3-client.ts +9 -0
  18. package/src/driver-s3-copy.ts +35 -0
  19. package/src/driver-s3-lock.ts +9 -2
  20. package/src/driver-s3-multipart-session.ts +87 -0
  21. package/src/driver-s3-multipart.ts +177 -0
  22. package/src/driver-s3-presign.ts +52 -0
  23. package/src/driver-s3-put-stream.ts +97 -0
  24. package/src/driver-s3-range.ts +99 -0
  25. package/src/driver-s3-settle.ts +150 -0
  26. package/src/driver-s3-signed.ts +69 -4
  27. package/src/driver-s3-sse.ts +29 -0
  28. package/src/driver-s3.ts +113 -65
  29. package/src/driver.ts +98 -38
  30. package/src/errors-multipart.ts +35 -0
  31. package/src/errors-promote.ts +14 -0
  32. package/src/errors-range.ts +47 -0
  33. package/src/errors-sniff.ts +35 -0
  34. package/src/errors-stream.ts +62 -0
  35. package/src/errors.ts +16 -1
  36. package/src/expire.ts +169 -0
  37. package/src/grant-multipart.ts +196 -0
  38. package/src/grant.ts +42 -25
  39. package/src/image-warm.ts +84 -0
  40. package/src/image.ts +57 -0
  41. package/src/index.ts +81 -2
  42. package/src/multipart-staged.ts +127 -0
  43. package/src/multipart.ts +119 -0
  44. package/src/put-stream.ts +52 -0
  45. package/src/range.ts +102 -0
  46. package/src/signed-url.ts +59 -2
  47. package/src/subject-target.ts +82 -0
  48. package/src/upload-client-multipart.ts +156 -0
  49. package/src/upload-client.ts +21 -0
  50. package/src/upload.ts +107 -21
@@ -1,5 +1,5 @@
1
1
  // Single responsibility: the s3 disk's SIGNED requests — the ones `Bun.S3Client` has no option for
2
- // (`x-amz-meta-*`, `Cache-Control`, `x-amz-object-lock-*`, `?retention`, `?legal-hold`) — built,
2
+ // (`x-amz-meta-*`, `Cache-Control`, object lock, a ranged GET read off its own `Content-Range`) — built,
3
3
  // signed through core's one SigV4 signer and sent with `fetch`. The endpoint, the addressing style,
4
4
  // the region and the credentials are read from the SAME `S3DriverOptions` the Bun client is built
5
5
  // from, so the two transports of one disk cannot point at two buckets.
@@ -9,9 +9,11 @@ import {
9
9
  type Clock,
10
10
  ConfigInvalidError,
11
11
  EnvMissingError,
12
+ presignAwsUrl,
12
13
  signAwsRequest,
13
14
  systemClock,
14
15
  } from '@ultimat3/core';
16
+ import type { SignedPart } from './signed-url';
15
17
 
16
18
  /** The slice of `fetch` this file uses; injected in tests so no socket is ever opened. */
17
19
  export type S3FetchLike = (
@@ -51,10 +53,12 @@ export interface S3Refusal {
51
53
  }
52
54
 
53
55
  export interface S3SignedRequest {
54
- readonly method: 'GET' | 'PUT';
56
+ readonly method: 'GET' | 'PUT' | 'POST' | 'DELETE';
55
57
  readonly key: string;
56
58
  /** A sub-resource such as `retention` — sent and signed as `?retention=`. */
57
59
  readonly subresource?: string | undefined;
60
+ /** Query parameters of a multipart call, signed. An empty value is sent as `uploads=`. */
61
+ readonly query?: Readonly<Record<string, string>> | undefined;
58
62
  readonly headers?: Readonly<Record<string, string>> | undefined;
59
63
  readonly body?: Uint8Array | undefined;
60
64
  }
@@ -122,6 +126,61 @@ export function resolveS3Target(options: S3WireOptions): S3Target {
122
126
  };
123
127
  }
124
128
 
129
+ /**
130
+ * A PUT URL that signs its `content-length`, which is what `Bun.S3Client.presign` has no option for.
131
+ * The provider recomputes the signature from the header the browser really sent, so the only body
132
+ * this URL lands is one of exactly `contentLength` bytes. A browser sets that header itself.
133
+ */
134
+ export async function presignBoundPut(
135
+ options: S3WireOptions,
136
+ key: string,
137
+ contentLength: number,
138
+ expiresInSeconds: number,
139
+ part?: SignedPart | undefined,
140
+ ): Promise<string> {
141
+ const url = s3ObjectUrl(options, key);
142
+ // A part of an open upload: S3's own UploadPart, its two parameters inside the signature.
143
+ if (part !== undefined) {
144
+ url.searchParams.set('partNumber', String(part.partNumber));
145
+ url.searchParams.set('uploadId', part.uploadId);
146
+ }
147
+ const presigned = await presignAwsUrl({
148
+ method: 'PUT',
149
+ url,
150
+ expiresInSeconds,
151
+ headers: { 'content-length': String(contentLength) },
152
+ credentials: s3Credentials(options),
153
+ region: signingRegion(options),
154
+ service: 's3',
155
+ clock: options.clock ?? systemClock,
156
+ });
157
+ return presigned.url;
158
+ }
159
+
160
+ /** The labels between `s3.` and `.amazonaws.com` of an AWS host: its region, or none. */
161
+ const AWS_HOST = /(?:^|\.)s3\.(?:(.+)\.)?amazonaws\.com$/i;
162
+ const REGION = /^[a-z0-9-]{1,64}$/;
163
+
164
+ /**
165
+ * The region a request is signed for, as `Bun.S3Client` derives it, measured on Bun 1.4.4 so the
166
+ * disk's two transports sign alike: a region set wins; no endpoint is AWS's `us-east-1`; an AWS host
167
+ * `s3.<region>.amazonaws.com` is that region, bare `s3.amazonaws.com` is `us-east-1`; any other host
168
+ * is `auto` (R2, a gateway). One deviation: Bun reads `s3.dualstack.<region>` as the whole
169
+ * `dualstack.<region>`, which is no region, and this reads the region.
170
+ */
171
+ export function signingRegion(options: S3WireOptions): string {
172
+ const { region, endpoint } = resolveS3Target(options);
173
+ if (region !== undefined) return region;
174
+ if (endpoint === undefined) return 'us-east-1';
175
+ const host = URL.canParse(endpoint) ? new URL(endpoint).hostname : '';
176
+ const match = AWS_HOST.exec(host);
177
+ if (match === null) return 'auto';
178
+ const named = match[1];
179
+ if (named === undefined) return 'us-east-1';
180
+ const last = named.split('.').at(-1) ?? '';
181
+ return REGION.test(named) ? named : REGION.test(last) ? last : 'auto';
182
+ }
183
+
125
184
  /** Read lazily, at the first signed request — importing or constructing the disk reads nothing. */
126
185
  export function s3Credentials(options: S3WireOptions): AwsCredentials {
127
186
  const env = options.env ?? process.env;
@@ -153,7 +212,10 @@ export function s3ObjectUrl(options: S3WireOptions, key: string): URL {
153
212
  );
154
213
  }
155
214
 
156
- const XML_FIELD = (name: string): RegExp => new RegExp(`<${name}>([^<]{0,512})</${name}>`);
215
+ // Uncapped: a key is up to 1,024 bytes and S3 escapes it (`&` is `&amp;`), so a 512-character
216
+ // cap dropped a long key's session from `listS3Uploads` and ended its paging at a long
217
+ // `NextKeyMarker`. `[^<]*` stops at the next tag, so the match stays linear with no cap.
218
+ const XML_FIELD = (name: string): RegExp => new RegExp(`<${name}>([^<]*)</${name}>`);
157
219
 
158
220
  /** The first `<name>…</name>` of an S3 XML answer — the shapes read here are flat and fixed. */
159
221
  export function xmlField(xml: string, name: string): string | undefined {
@@ -189,6 +251,9 @@ export async function sendSigned(
189
251
  }
190
252
  const url = s3ObjectUrl(options, request.key);
191
253
  if (request.subresource !== undefined) url.search = `${request.subresource}=`;
254
+ for (const [name, value] of Object.entries(request.query ?? {})) {
255
+ url.searchParams.set(name, value);
256
+ }
192
257
  const signed = await signAwsRequest({
193
258
  method: request.method,
194
259
  url,
@@ -197,7 +262,7 @@ export async function sendSigned(
197
262
  // refuse a body that changed in flight; a read has none.
198
263
  payload: { body: request.body ?? new Uint8Array() },
199
264
  credentials: s3Credentials(options),
200
- region: resolveS3Target(options).region ?? 'auto',
265
+ region: signingRegion(options),
201
266
  service: 's3',
202
267
  clock: options.clock ?? systemClock,
203
268
  });
@@ -0,0 +1,29 @@
1
+ // Single responsibility: the one `put()` option the s3 disk cannot honour, refused by name with the
2
+ // bucket-wide command that does the job instead.
3
+
4
+ import { NotImplementedError, renderFixLiteral, renderFixShellArg } from '@ultimat3/core';
5
+ import type { PutOptions } from './driver';
6
+
7
+ /**
8
+ * Everything `put()` may be handed that this driver cannot honour, refused before a byte moves.
9
+ * A typed refusal at the call site is the whole point: `serverSideEncryption` exists on
10
+ * `PutOptions` so that "this disk cannot prove per-object encryption" is something an engineer
11
+ * meets while writing the call, not while answering a security review.
12
+ */
13
+ export function refuseUnsupportedPut(bucket: string, putOptions?: PutOptions): void {
14
+ const sse = putOptions?.serverSideEncryption;
15
+ if (sse === undefined) return;
16
+ // The key id is the caller's own option, spliced into single-quoted JSON: `renderFixLiteral`
17
+ // escapes the `"` and `\` JSON needs, and a `'` — the one byte live inside single quotes — makes
18
+ // it the placeholder rather than ending the argument (plan 101 row S12).
19
+ const keyId = sse.kmsKeyId?.includes("'") === false ? sse.kmsKeyId : undefined;
20
+ const bucketArg = renderFixShellArg(bucket, "'<bucket>'");
21
+ throw new NotImplementedError({
22
+ cause:
23
+ 'per-object server-side encryption on the s3 driver (Bun.S3Client exposes acl, storageClass and type, and nothing for x-amz-server-side-encryption) is not implemented by this driver — set it bucket-wide with the command in fix, then drop serverSideEncryption from put()',
24
+ fix:
25
+ sse.algorithm === 'aws:kms'
26
+ ? `aws s3api put-bucket-encryption --bucket ${bucketArg} --server-side-encryption-configuration '{"Rules":[{"ApplyServerSideEncryptionByDefault":{"SSEAlgorithm":"aws:kms","KMSMasterKeyID":${renderFixLiteral(keyId, '"<key-arn>"')}},"BucketKeyEnabled":true}]}'`
27
+ : `aws s3api put-bucket-encryption --bucket ${bucketArg} --server-side-encryption-configuration '{"Rules":[{"ApplyServerSideEncryptionByDefault":{"SSEAlgorithm":"AES256"},"BucketKeyEnabled":true}]}'`,
28
+ });
29
+ }
package/src/driver-s3.ts CHANGED
@@ -8,16 +8,15 @@ import {
8
8
  type Clock,
9
9
  finiteCount,
10
10
  isUltimateError,
11
- NotImplementedError,
12
11
  type Page,
13
12
  pageOf,
14
- renderFixLiteral,
15
13
  renderFixShellArg,
16
14
  systemClock,
17
15
  } from '@ultimat3/core';
18
16
  import {
19
17
  assertListOptions,
20
18
  assertPutOptions,
19
+ type CopyOptions,
21
20
  DEFAULT_CONTENT_TYPE,
22
21
  type ListOptions,
23
22
  type PutOptions,
@@ -32,10 +31,17 @@ import {
32
31
  toBytes,
33
32
  } from './driver';
34
33
  import { isAbsentObject } from './driver-s3-absent';
35
- import { buildClient, type S3ClientLike, type S3ListResultLike } from './driver-s3-client';
34
+ import { buildClient, dated, type S3ClientLike, type S3ListResultLike } from './driver-s3-client';
35
+ import { signedCopy } from './driver-s3-copy';
36
36
  import { needsSignedPut, readObjectLock, signedPut } from './driver-s3-lock';
37
+ import { s3Multipart } from './driver-s3-multipart-session';
38
+ import { s3SignedUrl } from './driver-s3-presign';
39
+ import { putS3Stream } from './driver-s3-put-stream';
40
+ import { readS3Range } from './driver-s3-range';
37
41
  import { regionMismatch } from './driver-s3-region';
42
+ import { settleLostPut } from './driver-s3-settle';
38
43
  import type { S3FetchLike } from './driver-s3-signed';
44
+ import { refuseUnsupportedPut } from './driver-s3-sse';
39
45
  import {
40
46
  checksumMismatch,
41
47
  deleteFailed,
@@ -46,9 +52,17 @@ import {
46
52
  putFailed,
47
53
  readFailed,
48
54
  } from './errors';
55
+ import { objectChanged } from './errors-promote';
49
56
  import { assertObjectLockOptions, type ObjectLock } from './object-lock';
50
57
  import { assertSafeKey } from './path';
51
- import { DEFAULT_SIGNED_URL_TTL_MS } from './signed-url';
58
+ import { assertPutStream, type PutStreamOptions } from './put-stream';
59
+ import {
60
+ assertGetOptions,
61
+ assertStreamRange,
62
+ type ByteRange,
63
+ type GetOptions,
64
+ streamedRange,
65
+ } from './range';
52
66
  import { DEFAULT_MAX_UPLOAD_BYTES } from './upload';
53
67
 
54
68
  const DRIVER_NAME = 's3';
@@ -90,44 +104,13 @@ export interface S3DriverOptions {
90
104
  */
91
105
  readonly maxPutBytes?: number | undefined;
92
106
  /**
93
- * Ceiling on ONE `get()`, because `get()` buffers the whole object — and on this disk nothing
94
- * bounds what a presigned PUT stored (`SignedUrlOptions.maxBytes`). Defaults to `maxPutBytes`;
95
- * an object past it is read with `stream()`.
107
+ * Ceiling on ONE `get()`, because `get()` buffers the whole object — and a URL signed with no
108
+ * `contentLength` bounds nothing it stored. Defaults to `maxPutBytes`; an object past it is read
109
+ * with `stream()`.
96
110
  */
97
111
  readonly maxGetBytes?: number | undefined;
98
112
  }
99
113
 
100
- /**
101
- * A missing `LastModified` is ABSENT, never `new Date(0)`: epoch 0 is older than every window, so
102
- * `sweepOrphans` deleted an upload on the strength of a field the provider never sent.
103
- */
104
- const dated = (value: string | Date | undefined): { readonly lastModified?: Date } =>
105
- value === undefined ? {} : { lastModified: value instanceof Date ? value : new Date(value) };
106
-
107
- /**
108
- * Everything `put()` may be handed that this driver cannot honour, refused before a byte moves.
109
- * A typed refusal at the call site is the whole point: `serverSideEncryption` exists on
110
- * `PutOptions` so that "this disk cannot prove per-object encryption" is something an engineer
111
- * meets while writing the call, not while answering a security review.
112
- */
113
- function refuseUnsupportedPut(bucket: string, putOptions?: PutOptions): void {
114
- const sse = putOptions?.serverSideEncryption;
115
- if (sse === undefined) return;
116
- // The key id is the caller's own option, spliced into single-quoted JSON: `renderFixLiteral`
117
- // escapes the `"` and `\` JSON needs, and a `'` — the one byte live inside single quotes — makes
118
- // it the placeholder rather than ending the argument (plan 101 row S12).
119
- const keyId = sse.kmsKeyId?.includes("'") === false ? sse.kmsKeyId : undefined;
120
- const bucketArg = renderFixShellArg(bucket, "'<bucket>'");
121
- throw new NotImplementedError({
122
- cause:
123
- 'per-object server-side encryption on the s3 driver (Bun.S3Client exposes acl, storageClass and type, and nothing for x-amz-server-side-encryption) is not implemented by this driver — set it bucket-wide with the command in fix, then drop serverSideEncryption from put()',
124
- fix:
125
- sse.algorithm === 'aws:kms'
126
- ? `aws s3api put-bucket-encryption --bucket ${bucketArg} --server-side-encryption-configuration '{"Rules":[{"ApplyServerSideEncryptionByDefault":{"SSEAlgorithm":"aws:kms","KMSMasterKeyID":${renderFixLiteral(keyId, '"<key-arn>"')}},"BucketKeyEnabled":true}]}'`
127
- : `aws s3api put-bucket-encryption --bucket ${bucketArg} --server-side-encryption-configuration '{"Rules":[{"ApplyServerSideEncryptionByDefault":{"SSEAlgorithm":"AES256"},"BucketKeyEnabled":true}]}'`,
128
- });
129
- }
130
-
131
114
  export function s3Driver(options: S3DriverOptions): StorageDriver {
132
115
  // `=== undefined`, never `??`: `??` coalesces on `null` too, so an explicitly blanked key in a
133
116
  // decoded JSON config took the default instead of the refusal `finiteCount` is here to raise.
@@ -214,7 +197,12 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
214
197
  };
215
198
  };
216
199
 
217
- return {
200
+ /** The HEAD a lost write's answer is settled against, and the `.catch` of a single PUT. */
201
+ const statOf = async (key: string) => ((await present(key)) ? statObject(key) : undefined);
202
+ const lostPut = (key: string, bytes: Uint8Array, attributes: PutOptions | undefined) =>
203
+ settleLostPut(statOf, { driver: DRIVER_NAME, key, bytes, attributes }, refusedWrite(key));
204
+
205
+ const disk: StorageDriver = {
218
206
  name: DRIVER_NAME,
219
207
  registerAs(name: string): void {
220
208
  diskName = name;
@@ -238,7 +226,9 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
238
226
  if (putOptions !== undefined && needsSignedPut(putOptions)) {
239
227
  // Off the Bun client only for what it cannot send: `S3File.write` takes a type, an ACL and
240
228
  // a storage class and no header. One request either way — never a PUT then a second call.
241
- const refusal = await signedPut(options, safe, bytes, putOptions).catch(refusedWrite(safe));
229
+ const refusal = await signedPut(options, safe, bytes, putOptions).catch(
230
+ lostPut(safe, bytes, putOptions),
231
+ );
242
232
  if (refusal !== undefined) refusedWrite(safe)(refusal);
243
233
  // Stored — the PUT succeeded with them — so reported, as the local disk reports its own.
244
234
  return {
@@ -252,17 +242,66 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
252
242
  await conn()
253
243
  .file(safe)
254
244
  .write(bytes, { type: putOptions?.contentType ?? DEFAULT_CONTENT_TYPE })
255
- .catch(refusedWrite(safe));
245
+ .catch(lostPut(safe, bytes, putOptions));
256
246
  return statObject(safe);
257
247
  },
258
248
 
249
+ async putStream(
250
+ key: string,
251
+ body: ReadableStream<Uint8Array>,
252
+ streamOptions: PutStreamOptions,
253
+ ): Promise<StorageObject> {
254
+ const safe = assertSafeKey(key);
255
+ const maxBytes = assertPutStream(DRIVER_NAME, body, streamOptions);
256
+ await putS3Stream(options, {
257
+ key: safe,
258
+ body,
259
+ options: streamOptions,
260
+ maxBytes,
261
+ driver: DRIVER_NAME,
262
+ refused: refusedWrite(safe),
263
+ stat: statOf,
264
+ }).catch(refusedWrite(safe));
265
+ return {
266
+ ...(await statObject(safe)),
267
+ ...(streamOptions.cacheControl === undefined
268
+ ? {}
269
+ : { cacheControl: streamOptions.cacheControl }),
270
+ ...(streamOptions.metadata === undefined
271
+ ? {}
272
+ : { metadata: { ...streamOptions.metadata } }),
273
+ };
274
+ },
275
+
259
276
  async stat(key: string): Promise<StorageObject | undefined> {
260
277
  const safe = assertSafeKey(key);
261
278
  return (await present(safe)) ? statObject(safe) : undefined;
262
279
  },
263
280
 
264
- async get(key: string): Promise<StorageRead> {
281
+ async get(key: string, getOptions?: GetOptions): Promise<StorageRead> {
265
282
  const safe = assertSafeKey(key);
283
+ assertGetOptions(DRIVER_NAME, getOptions);
284
+ const range = getOptions?.range;
285
+ // ONE signed GET, bytes and description from the same response (`driver-s3-range.ts`).
286
+ if (range !== undefined) {
287
+ return readS3Range(options, safe, range, {
288
+ driver: DRIVER_NAME,
289
+ maxGetBytes,
290
+ refused: (key, refusal) => refusedRead(key)(refusal),
291
+ sizeOf: async (key) => {
292
+ if (!(await present(key))) throw objectNotFound(diskName, key);
293
+ return (await statObject(key)).size;
294
+ },
295
+ unreadable: (key, why) => {
296
+ throw readFailed(
297
+ DRIVER_NAME,
298
+ key,
299
+ why,
300
+ `aws s3api get-object --bucket ${renderFixShellArg(options.bucket, "'<bucket>'")} --key ${renderFixShellArg(key, "'<key>'")} --range bytes=0-0 /dev/null # what the provider answers a ranged GET`,
301
+ );
302
+ },
303
+ });
304
+ }
266
305
  const file = conn().file(safe);
267
306
  if (!(await present(safe))) throw objectNotFound(diskName, safe);
268
307
  // The HEAD comes FIRST, and it is the whole point: a client PUT straight into the bucket is
@@ -281,6 +320,20 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
281
320
  return file.stream();
282
321
  },
283
322
 
323
+ async streamRange(key: string, range: ByteRange): Promise<ReadableStream<Uint8Array>> {
324
+ const safe = assertSafeKey(key);
325
+ assertStreamRange(DRIVER_NAME, range);
326
+ if (!(await present(safe))) throw objectNotFound(diskName, safe);
327
+ // The HEAD first: past the end, Bun's slice raises from inside the stream, after a caller has
328
+ // already answered 206. Refused here it is still a status.
329
+ const { size } = await statObject(safe);
330
+ const [first, last] = streamedRange(DRIVER_NAME, safe, range, size);
331
+ return conn()
332
+ .file(safe)
333
+ .slice(first, last + 1)
334
+ .stream();
335
+ },
336
+
284
337
  /**
285
338
  * Bytes from one key to another without a round trip through this process. Bun exposes no
286
339
  * `CopyObject`, so the source `S3File` is handed to `write()` — Bun's own union accepts one —
@@ -288,9 +341,20 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
288
341
  * win available today: promoting a 500MB attachment used to be `get()` + `put()`, a full GB
289
342
  * resident in the pod. A true server-side copy needs a Bun API that does not exist in 1.3.
290
343
  */
291
- async copy(from: string, to: string): Promise<StorageObject> {
344
+ async copy(from: string, to: string, copyOptions?: CopyOptions): Promise<StorageObject> {
292
345
  const source = assertSafeKey(from);
293
346
  const destination = assertSafeKey(to);
347
+ // Bun's `write` takes no condition, so a revision-bound copy is the signed CopyObject.
348
+ if (copyOptions?.ifMatch !== undefined) {
349
+ const refusal = await signedCopy(options, source, destination, copyOptions.ifMatch).catch(
350
+ refusedWrite(destination),
351
+ );
352
+ if (refusal?.statusCode === 404) throw objectNotFound(diskName, source);
353
+ if (refusal?.statusCode === 412)
354
+ throw objectChanged(source, copyOptions.ifMatch, 'another');
355
+ if (refusal !== undefined) refusedWrite(destination)(refusal);
356
+ return statObject(destination);
357
+ }
294
358
  const file = conn().file(source);
295
359
  if (!(await present(source))) throw objectNotFound(diskName, source);
296
360
  const stat = await file.stat().catch(refusedRead(source));
@@ -397,29 +461,13 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
397
461
  );
398
462
  },
399
463
 
400
- /**
401
- * Provider presigning. The signature covers method, expiry and content type but NOT
402
- * `maxBytes` — S3 has no header for it, so size stays a server-side policy check.
403
- */
464
+ /** Both presigners, one rule: `driver-s3-presign.ts`. */
404
465
  async signedUrl(key: string, urlOptions?: SignedUrlOptions): Promise<string> {
405
- // Screened with the SAME function `buildSignedUrl` applies on the local disk — one
406
- // finite-bound path for both presigners, never a private copy per package: `Math.ceil(NaN /
407
- // 1000)` is `NaN`, so `X-Amz-Expires=NaN` went to AWS and the app got a 403 on a link it
408
- // believed it had just minted. `=== undefined` rather than `??`, so an explicit `null` is
409
- // refused instead of silently taking the default.
410
- const expiresInMs = finiteCount(
411
- 'the s3 driver',
412
- 'expiresInMs',
413
- urlOptions?.expiresInMs === undefined ? DEFAULT_SIGNED_URL_TTL_MS : urlOptions.expiresInMs,
414
- 1,
415
- );
416
- return conn()
417
- .file(assertSafeKey(key))
418
- .presign({
419
- method: urlOptions?.method ?? 'GET',
420
- expiresIn: Math.ceil(expiresInMs / 1000),
421
- ...(urlOptions?.contentType === undefined ? {} : { type: urlOptions.contentType }),
422
- });
466
+ return s3SignedUrl(options, (safe) => conn().file(safe), key, urlOptions);
423
467
  },
424
468
  };
469
+ // Attached in place: the capability presigns and stats through the disk it belongs to.
470
+ return Object.assign(disk, {
471
+ multipart: s3Multipart({ wire: options, driver: disk, refusedWrite }),
472
+ });
425
473
  }
package/src/driver.ts CHANGED
@@ -5,9 +5,13 @@
5
5
 
6
6
  import type { Clock, Page } from '@ultimat3/core';
7
7
  import { assert } from '@ultimat3/core';
8
+ import { pumpBounded } from './byte-pump';
8
9
  import { putTooLarge } from './errors';
10
+ import type { MultipartCapability } from './multipart';
9
11
  import type { ObjectLock, ObjectLockOptions } from './object-lock';
10
- import type { SignedUrlVerification } from './signed-url';
12
+ import type { PutStreamOptions } from './put-stream';
13
+ import type { ByteRange, GetOptions } from './range';
14
+ import type { SignedPart, SignedUrlVerification } from './signed-url';
11
15
 
12
16
  export type StorageBody = Uint8Array | ReadableStream<Uint8Array> | Blob;
13
17
 
@@ -81,8 +85,15 @@ export interface ListOptions {
81
85
  }
82
86
 
83
87
  export interface StorageRead {
88
+ /** The WHOLE object's description — `size` is the object's, never the range's. */
84
89
  readonly object: StorageObject;
85
90
  readonly bytes: Uint8Array;
91
+ /**
92
+ * The bytes served, both inclusive, after the clamp — present exactly when `range` was asked.
93
+ * A third-party driver that ignored the option answers none, and that absence is how a caller
94
+ * tells a slice from the whole object rather than trusting `bytes.byteLength`.
95
+ */
96
+ readonly range?: ByteRange | undefined;
86
97
  }
87
98
 
88
99
  export type SignedUrlMethod = 'GET' | 'PUT';
@@ -91,18 +102,30 @@ export interface SignedUrlOptions {
91
102
  readonly method?: SignedUrlMethod | undefined;
92
103
  readonly expiresInMs?: number | undefined;
93
104
  /**
94
- * Ceiling on a `PUT`, and **the one option whose enforcement is the driver's, not the caller's**.
95
- *
96
- * `localDriver` signs it into the URL, so `acceptSignedUpload` refuses a client that widens it.
97
- * `s3Driver` CANNOT: S3 has no request header for a size and Bun's `presign` covers method,
98
- * expiry and content type only — so the client PUTs straight into the bucket and nothing between
99
- * the grant and the object enforces this number. It is not refused there, because `grantUpload`
100
- * supplies it on every grant and refusing would break every s3 upload an app mints; the ceiling
101
- * a bucket-backed disk actually gets is a bucket rule or a post-upload check on `object.size`.
102
- * `driver-parity.test.ts` pins both halves so neither moves alone.
105
+ * Ceiling on a `PUT`. `localDriver` signs it into the URL, so `acceptSignedUpload` refuses a client
106
+ * that widens it. A provider's URL has no ceiling to sign, so `s3Driver` refuses it without
107
+ * `contentLength` (`X_STORAGE_UPLOAD_SIZE_UNDECLARED`) rather than mint a URL that ignores it.
103
108
  */
104
109
  readonly maxBytes?: number | undefined;
110
+ /**
111
+ * The exact size of a `PUT`, inside the signature. On `s3Driver` the URL then signs
112
+ * `content-length`, so the provider refuses any other length, a chunked body with none included
113
+ * (403 `SignatureDoesNotMatch`). The local disks hold it as a ceiling (`x-max`): they count what
114
+ * arrives, so a shorter body passes there and not on s3. `driver-parity.test.ts` pins both.
115
+ */
116
+ readonly contentLength?: number | undefined;
105
117
  readonly contentType?: string | undefined;
118
+ /**
119
+ * A part of an open upload, inside the signature: the URL then writes that part and nothing else.
120
+ * Only a disk whose `multipart.receivePart` stores parts signs one; `s3Driver` refuses it, since
121
+ * its parts are signed by `multipart.signPart` against the provider.
122
+ */
123
+ readonly part?: SignedPart | undefined;
124
+ }
125
+
126
+ export interface CopyOptions {
127
+ /** The source's `etag` as this disk reported it. */
128
+ readonly ifMatch?: string | undefined;
106
129
  }
107
130
 
108
131
  export interface StorageDriver {
@@ -125,6 +148,16 @@ export interface StorageDriver {
125
148
  */
126
149
  registerAs?(diskName: string): void;
127
150
  put(key: string, body: StorageBody, options?: PutOptions): Promise<StorageObject>;
151
+ /**
152
+ * A body of any length, without buffering it: counted against the call's required `maxBytes`, and
153
+ * past it the write is abandoned with the key as it was (`X_STORAGE_TOO_LARGE`). Takes no
154
+ * `checksum`, retention or legal hold, which need the bytes before the object exists.
155
+ */
156
+ putStream(
157
+ key: string,
158
+ stream: ReadableStream<Uint8Array>,
159
+ options: PutStreamOptions,
160
+ ): Promise<StorageObject>;
128
161
  /**
129
162
  * What a read would say about the object, WITHOUT its bytes — `undefined` when nothing is there.
130
163
  * `promoteAttachment` decides on `stat().size`, because a presigned PUT lands on the disk
@@ -134,17 +167,26 @@ export interface StorageDriver {
134
167
  stat(key: string): Promise<StorageObject | undefined>;
135
168
  /**
136
169
  * The whole object, BUFFERED — so bounded: an object over the driver's `maxGetBytes` is
137
- * `X_STORAGE_TOO_LARGE`, exactly as `put()` refuses a body over `maxPutBytes`. Past it, `stream()`.
170
+ * `X_STORAGE_TOO_LARGE`, exactly as `put()` refuses a body over `maxPutBytes`. Past it, `stream()`
171
+ * — or `{ range: [first, last] }`, which reads only those bytes (inclusive) and is bounded by
172
+ * the range's length: s3 sends `Range:`, the local disk slices the file, memory slices its copy.
138
173
  */
139
- get(key: string): Promise<StorageRead>;
174
+ get(key: string, options?: GetOptions): Promise<StorageRead>;
140
175
  /** Bytes without buffering — the only safe path for anything over a few MB. */
141
176
  stream(key: string): Promise<ReadableStream<Uint8Array>>;
177
+ /**
178
+ * The bytes `[first, last]` (inclusive, clamped, as `get`'s range) with none before them read, so
179
+ * a seek late in a film costs its window. `X_STORAGE_RANGE_UNSATISFIABLE` past the end. A method and
180
+ * not an option on `stream`, which a third-party driver would ignore and serve the wrong bytes.
181
+ */
182
+ streamRange(key: string, range: ByteRange): Promise<ReadableStream<Uint8Array>>;
142
183
  /**
143
184
  * Bytes from one key to another without them passing through this process. Idempotent in the
144
185
  * destination and non-destructive in the source: the caller deletes the source afterwards if
145
- * it wanted a move. `X_STORAGE_NOT_FOUND` when `from` is absent.
186
+ * it wanted a move. `X_STORAGE_NOT_FOUND` when `from` is absent. `ifMatch` copies only that
187
+ * revision: any other is `X_STORAGE_OBJECT_CHANGED` and writes nothing.
146
188
  */
147
- copy(from: string, to: string): Promise<StorageObject>;
189
+ copy(from: string, to: string, options?: CopyOptions): Promise<StorageObject>;
148
190
  /**
149
191
  * Deleting an ABSENT key is not an error. Anything else is: a denied `s3:DeleteObject`, a
150
192
  * throttle, an expired credential and a read-only mount all raise `X_STORAGE_DELETE_FAILED`,
@@ -183,6 +225,12 @@ export interface StorageDriver {
183
225
  * provider and never reaches this path, and a third-party driver that implements neither is
184
226
  * refused by name (`X_STORAGE_URL_UNVERIFIABLE`) rather than reported as a bad signature.
185
227
  */
228
+ /**
229
+ * An upload in parts: resumable, and past S3's 5 GiB single-PUT ceiling. Optional, so a
230
+ * third-party driver still compiles; `grantMultipart` refuses a disk without it
231
+ * (`X_STORAGE_MULTIPART_UNSUPPORTED`). Every shipped disk has one.
232
+ */
233
+ readonly multipart?: MultipartCapability | undefined;
186
234
  verifySigned?(input: {
187
235
  readonly url: string;
188
236
  /** Overrides the driver's own, so a test's frozen clock still governs expiry. */
@@ -241,6 +289,36 @@ export function assertPutOptions(driver: string, options: PutOptions | undefined
241
289
  );
242
290
  }
243
291
 
292
+ /**
293
+ * The number a signed URL holds, screened on every disk: `contentLength` when there is one, since it
294
+ * is the tighter fact, and a `contentLength` past `maxBytes` or on anything but a `PUT` is a call
295
+ * that contradicts itself.
296
+ */
297
+ export function signedByteBound(
298
+ driver: string,
299
+ options: SignedUrlOptions | undefined,
300
+ ): number | undefined {
301
+ const length = options?.contentLength;
302
+ if (length === undefined) return options?.maxBytes;
303
+ const maxBytes = options?.maxBytes;
304
+ assert(
305
+ Number.isSafeInteger(length) && length >= 0,
306
+ `signedUrl() on the ${driver} disk was given a contentLength that is not a whole byte count`,
307
+ "signedUrl(key, { method: 'PUT', contentLength: file.size })",
308
+ );
309
+ assert(
310
+ options?.method === 'PUT',
311
+ `signedUrl() on the ${driver} disk was given a contentLength on a read, which has no body to bind`,
312
+ "signedUrl(key, { method: 'PUT', contentLength: file.size })",
313
+ );
314
+ assert(
315
+ maxBytes === undefined || length <= maxBytes,
316
+ `signedUrl() on the ${driver} disk was given a contentLength of ${length}B past its maxBytes of ${String(maxBytes)}B`,
317
+ "signedUrl(key, { method: 'PUT', contentLength: file.size }) # at or under maxBytes",
318
+ );
319
+ return length;
320
+ }
321
+
244
322
  /** A `prefix` or a `cursor` that is not a string reached `startsWith` and `<=` as whatever it was. */
245
323
  export function assertListOptions(options: ListOptions | undefined): void {
246
324
  for (const name of ['prefix', 'cursor'] as const) {
@@ -299,31 +377,13 @@ async function readBounded(
299
377
  body: ReadableStream<Uint8Array>,
300
378
  limit: ByteLimit,
301
379
  ): Promise<Uint8Array> {
302
- const reader = body.getReader();
303
380
  const chunks: Uint8Array[] = [];
304
- let total = 0;
305
- try {
306
- for (;;) {
307
- const next = await reader.read();
308
- if (next.done) break;
309
- total += next.value.byteLength;
310
- if (total > limit.maxBytes) {
311
- throw putTooLarge(limit.driver, limit.key, total, limit.maxBytes);
312
- }
313
- chunks.push(next.value);
314
- }
315
- } finally {
316
- // Cancel rather than merely release: a refused body must stop arriving, not keep filling a
317
- // socket buffer nobody will read. Already-closed streams answer this with a no-op.
318
- await reader.cancel().catch(() => undefined);
319
- }
320
- const bytes = new Uint8Array(total);
321
- let offset = 0;
322
- for (const chunk of chunks) {
323
- bytes.set(chunk, offset);
324
- offset += chunk.byteLength;
325
- }
326
- return bytes;
381
+ const { driver, key, maxBytes } = limit;
382
+ const refuse = (total: number): Error => putTooLarge(driver, key, total, maxBytes);
383
+ await pumpBounded(body, { driver, maxBytes, refuse }, (chunk) => {
384
+ chunks.push(chunk);
385
+ });
386
+ return new Uint8Array(Bun.concatArrayBuffers(chunks));
327
387
  }
328
388
 
329
389
  /** base64 SHA-256 — the wire form for `PutOptions.checksum` and `ValidatedUpload.checksum`. */
@@ -0,0 +1,35 @@
1
+ // The refusals of an upload in parts. Registered with the rest of the package's codes in
2
+ // `errors.ts`; each `fix:` names the call that goes next, with the key and upload id screened.
3
+
4
+ import { renderFixLiteral } from '@ultimat3/core';
5
+ import { StorageError } from './errors';
6
+
7
+ const keyArg = (key: string): string => renderFixLiteral(key, "'<key>'");
8
+ const idArg = (uploadId: string): string => renderFixLiteral(uploadId, "'<uploadId>'");
9
+
10
+ /** A disk whose driver declares no `multipart`: a third-party driver, most likely. */
11
+ export const multipartUnsupported = (disk: string, key: string): StorageError =>
12
+ new StorageError({
13
+ code: 'X_STORAGE_MULTIPART_UNSUPPORTED',
14
+ cause: `the ${disk} disk's driver declares no multipart capability, so "${key}" cannot be uploaded in parts`,
15
+ fix: 'grantUpload({ disk, orgId, request }) # one PUT; or implement StorageDriver.multipart on the driver',
16
+ meta: { disk, key },
17
+ });
18
+
19
+ /** No open session by that id for that key: never opened, completed, or aborted. */
20
+ export const uploadUnknown = (disk: string, key: string, uploadId: string): StorageError =>
21
+ new StorageError({
22
+ code: 'X_STORAGE_UPLOAD_UNKNOWN',
23
+ cause: `no upload ${idArg(uploadId)} is open for "${key}" on the ${disk} disk: it was never opened there, or it was already completed or aborted`,
24
+ fix: 'grantMultipart({ disk, orgId, request }) # a new session, then send every part under its uploadId',
25
+ meta: { disk, key, uploadId },
26
+ });
27
+
28
+ /** The parts a Complete named do not match what the disk holds. */
29
+ export const partsInvalid = (key: string, uploadId: string, problem: string): StorageError =>
30
+ new StorageError({
31
+ code: 'X_STORAGE_PARTS_INVALID',
32
+ cause: `the parts named to complete "${key}" do not match the parts stored: ${problem}`,
33
+ fix: `await listParts({ disk, orgId, key: ${keyArg(key)}, uploadId: ${idArg(uploadId)} }) # what landed; resend the missing part, then completeUpload again`,
34
+ meta: { key, uploadId, problem },
35
+ });