@ultimat3/storage 29.4.0 → 30.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/CLAUDE.md +25 -45
  2. package/README.md +102 -12
  3. package/package.json +12 -2
  4. package/src/accept-part.ts +30 -0
  5. package/src/accept.ts +7 -0
  6. package/src/attachment.ts +99 -6
  7. package/src/byte-pump.ts +46 -0
  8. package/src/driver-local-list.ts +75 -0
  9. package/src/driver-local-multipart.ts +133 -0
  10. package/src/driver-local-remove.ts +39 -0
  11. package/src/driver-local-stream.ts +104 -0
  12. package/src/driver-local-write.ts +28 -5
  13. package/src/driver-local.ts +250 -295
  14. package/src/driver-memory-multipart.ts +41 -0
  15. package/src/driver-memory.ts +205 -143
  16. package/src/driver-s3-client.ts +9 -0
  17. package/src/driver-s3-copy.ts +35 -0
  18. package/src/driver-s3-lock.ts +9 -2
  19. package/src/driver-s3-multipart-session.ts +87 -0
  20. package/src/driver-s3-multipart.ts +177 -0
  21. package/src/driver-s3-presign.ts +52 -0
  22. package/src/driver-s3-put-stream.ts +97 -0
  23. package/src/driver-s3-settle.ts +150 -0
  24. package/src/driver-s3-signed.ts +68 -3
  25. package/src/driver-s3-sse.ts +29 -0
  26. package/src/driver-s3.ts +89 -65
  27. package/src/driver.ts +86 -36
  28. package/src/errors-multipart.ts +35 -0
  29. package/src/errors-promote.ts +14 -0
  30. package/src/errors-sniff.ts +35 -0
  31. package/src/errors-stream.ts +62 -0
  32. package/src/errors.ts +13 -0
  33. package/src/expire.ts +169 -0
  34. package/src/grant-multipart.ts +196 -0
  35. package/src/grant.ts +42 -25
  36. package/src/image-warm.ts +84 -0
  37. package/src/image.ts +57 -0
  38. package/src/index.ts +75 -3
  39. package/src/multipart-staged.ts +127 -0
  40. package/src/multipart.ts +119 -0
  41. package/src/put-stream.ts +52 -0
  42. package/src/range.ts +35 -4
  43. package/src/signed-url.ts +59 -2
  44. package/src/subject-target.ts +82 -0
  45. package/src/upload-client-multipart.ts +156 -0
  46. package/src/upload-client.ts +21 -0
  47. package/src/upload.ts +107 -21
package/src/driver-s3.ts CHANGED
@@ -8,16 +8,15 @@ import {
8
8
  type Clock,
9
9
  finiteCount,
10
10
  isUltimateError,
11
- NotImplementedError,
12
11
  type Page,
13
12
  pageOf,
14
- renderFixLiteral,
15
13
  renderFixShellArg,
16
14
  systemClock,
17
15
  } from '@ultimat3/core';
18
16
  import {
19
17
  assertListOptions,
20
18
  assertPutOptions,
19
+ type CopyOptions,
21
20
  DEFAULT_CONTENT_TYPE,
22
21
  type ListOptions,
23
22
  type PutOptions,
@@ -32,11 +31,17 @@ import {
32
31
  toBytes,
33
32
  } from './driver';
34
33
  import { isAbsentObject } from './driver-s3-absent';
35
- import { buildClient, type S3ClientLike, type S3ListResultLike } from './driver-s3-client';
34
+ import { buildClient, dated, type S3ClientLike, type S3ListResultLike } from './driver-s3-client';
35
+ import { signedCopy } from './driver-s3-copy';
36
36
  import { needsSignedPut, readObjectLock, signedPut } from './driver-s3-lock';
37
+ import { s3Multipart } from './driver-s3-multipart-session';
38
+ import { s3SignedUrl } from './driver-s3-presign';
39
+ import { putS3Stream } from './driver-s3-put-stream';
37
40
  import { readS3Range } from './driver-s3-range';
38
41
  import { regionMismatch } from './driver-s3-region';
42
+ import { settleLostPut } from './driver-s3-settle';
39
43
  import type { S3FetchLike } from './driver-s3-signed';
44
+ import { refuseUnsupportedPut } from './driver-s3-sse';
40
45
  import {
41
46
  checksumMismatch,
42
47
  deleteFailed,
@@ -47,10 +52,17 @@ import {
47
52
  putFailed,
48
53
  readFailed,
49
54
  } from './errors';
55
+ import { objectChanged } from './errors-promote';
50
56
  import { assertObjectLockOptions, type ObjectLock } from './object-lock';
51
57
  import { assertSafeKey } from './path';
52
- import { assertGetOptions, type GetOptions } from './range';
53
- import { DEFAULT_SIGNED_URL_TTL_MS } from './signed-url';
58
+ import { assertPutStream, type PutStreamOptions } from './put-stream';
59
+ import {
60
+ assertGetOptions,
61
+ assertStreamRange,
62
+ type ByteRange,
63
+ type GetOptions,
64
+ streamedRange,
65
+ } from './range';
54
66
  import { DEFAULT_MAX_UPLOAD_BYTES } from './upload';
55
67
 
56
68
  const DRIVER_NAME = 's3';
@@ -92,44 +104,13 @@ export interface S3DriverOptions {
92
104
  */
93
105
  readonly maxPutBytes?: number | undefined;
94
106
  /**
95
- * Ceiling on ONE `get()`, because `get()` buffers the whole object — and on this disk nothing
96
- * bounds what a presigned PUT stored (`SignedUrlOptions.maxBytes`). Defaults to `maxPutBytes`;
97
- * an object past it is read with `stream()`.
107
+ * Ceiling on ONE `get()`, because `get()` buffers the whole object — and a URL signed with no
108
+ * `contentLength` bounds nothing it stored. Defaults to `maxPutBytes`; an object past it is read
109
+ * with `stream()`.
98
110
  */
99
111
  readonly maxGetBytes?: number | undefined;
100
112
  }
101
113
 
102
- /**
103
- * A missing `LastModified` is ABSENT, never `new Date(0)`: epoch 0 is older than every window, so
104
- * `sweepOrphans` deleted an upload on the strength of a field the provider never sent.
105
- */
106
- const dated = (value: string | Date | undefined): { readonly lastModified?: Date } =>
107
- value === undefined ? {} : { lastModified: value instanceof Date ? value : new Date(value) };
108
-
109
- /**
110
- * Everything `put()` may be handed that this driver cannot honour, refused before a byte moves.
111
- * A typed refusal at the call site is the whole point: `serverSideEncryption` exists on
112
- * `PutOptions` so that "this disk cannot prove per-object encryption" is something an engineer
113
- * meets while writing the call, not while answering a security review.
114
- */
115
- function refuseUnsupportedPut(bucket: string, putOptions?: PutOptions): void {
116
- const sse = putOptions?.serverSideEncryption;
117
- if (sse === undefined) return;
118
- // The key id is the caller's own option, spliced into single-quoted JSON: `renderFixLiteral`
119
- // escapes the `"` and `\` JSON needs, and a `'` — the one byte live inside single quotes — makes
120
- // it the placeholder rather than ending the argument (plan 101 row S12).
121
- const keyId = sse.kmsKeyId?.includes("'") === false ? sse.kmsKeyId : undefined;
122
- const bucketArg = renderFixShellArg(bucket, "'<bucket>'");
123
- throw new NotImplementedError({
124
- cause:
125
- 'per-object server-side encryption on the s3 driver (Bun.S3Client exposes acl, storageClass and type, and nothing for x-amz-server-side-encryption) is not implemented by this driver — set it bucket-wide with the command in fix, then drop serverSideEncryption from put()',
126
- fix:
127
- sse.algorithm === 'aws:kms'
128
- ? `aws s3api put-bucket-encryption --bucket ${bucketArg} --server-side-encryption-configuration '{"Rules":[{"ApplyServerSideEncryptionByDefault":{"SSEAlgorithm":"aws:kms","KMSMasterKeyID":${renderFixLiteral(keyId, '"<key-arn>"')}},"BucketKeyEnabled":true}]}'`
129
- : `aws s3api put-bucket-encryption --bucket ${bucketArg} --server-side-encryption-configuration '{"Rules":[{"ApplyServerSideEncryptionByDefault":{"SSEAlgorithm":"AES256"},"BucketKeyEnabled":true}]}'`,
130
- });
131
- }
132
-
133
114
  export function s3Driver(options: S3DriverOptions): StorageDriver {
134
115
  // `=== undefined`, never `??`: `??` coalesces on `null` too, so an explicitly blanked key in a
135
116
  // decoded JSON config took the default instead of the refusal `finiteCount` is here to raise.
@@ -216,7 +197,12 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
216
197
  };
217
198
  };
218
199
 
219
- return {
200
+ /** The HEAD a lost write's answer is settled against, and the `.catch` of a single PUT. */
201
+ const statOf = async (key: string) => ((await present(key)) ? statObject(key) : undefined);
202
+ const lostPut = (key: string, bytes: Uint8Array, attributes: PutOptions | undefined) =>
203
+ settleLostPut(statOf, { driver: DRIVER_NAME, key, bytes, attributes }, refusedWrite(key));
204
+
205
+ const disk: StorageDriver = {
220
206
  name: DRIVER_NAME,
221
207
  registerAs(name: string): void {
222
208
  diskName = name;
@@ -240,7 +226,9 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
240
226
  if (putOptions !== undefined && needsSignedPut(putOptions)) {
241
227
  // Off the Bun client only for what it cannot send: `S3File.write` takes a type, an ACL and
242
228
  // a storage class and no header. One request either way — never a PUT then a second call.
243
- const refusal = await signedPut(options, safe, bytes, putOptions).catch(refusedWrite(safe));
229
+ const refusal = await signedPut(options, safe, bytes, putOptions).catch(
230
+ lostPut(safe, bytes, putOptions),
231
+ );
244
232
  if (refusal !== undefined) refusedWrite(safe)(refusal);
245
233
  // Stored — the PUT succeeded with them — so reported, as the local disk reports its own.
246
234
  return {
@@ -254,10 +242,37 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
254
242
  await conn()
255
243
  .file(safe)
256
244
  .write(bytes, { type: putOptions?.contentType ?? DEFAULT_CONTENT_TYPE })
257
- .catch(refusedWrite(safe));
245
+ .catch(lostPut(safe, bytes, putOptions));
258
246
  return statObject(safe);
259
247
  },
260
248
 
249
+ async putStream(
250
+ key: string,
251
+ body: ReadableStream<Uint8Array>,
252
+ streamOptions: PutStreamOptions,
253
+ ): Promise<StorageObject> {
254
+ const safe = assertSafeKey(key);
255
+ const maxBytes = assertPutStream(DRIVER_NAME, body, streamOptions);
256
+ await putS3Stream(options, {
257
+ key: safe,
258
+ body,
259
+ options: streamOptions,
260
+ maxBytes,
261
+ driver: DRIVER_NAME,
262
+ refused: refusedWrite(safe),
263
+ stat: statOf,
264
+ }).catch(refusedWrite(safe));
265
+ return {
266
+ ...(await statObject(safe)),
267
+ ...(streamOptions.cacheControl === undefined
268
+ ? {}
269
+ : { cacheControl: streamOptions.cacheControl }),
270
+ ...(streamOptions.metadata === undefined
271
+ ? {}
272
+ : { metadata: { ...streamOptions.metadata } }),
273
+ };
274
+ },
275
+
261
276
  async stat(key: string): Promise<StorageObject | undefined> {
262
277
  const safe = assertSafeKey(key);
263
278
  return (await present(safe)) ? statObject(safe) : undefined;
@@ -305,6 +320,20 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
305
320
  return file.stream();
306
321
  },
307
322
 
323
+ async streamRange(key: string, range: ByteRange): Promise<ReadableStream<Uint8Array>> {
324
+ const safe = assertSafeKey(key);
325
+ assertStreamRange(DRIVER_NAME, range);
326
+ if (!(await present(safe))) throw objectNotFound(diskName, safe);
327
+ // The HEAD first: past the end, Bun's slice raises from inside the stream, after a caller has
328
+ // already answered 206. Refused here it is still a status.
329
+ const { size } = await statObject(safe);
330
+ const [first, last] = streamedRange(DRIVER_NAME, safe, range, size);
331
+ return conn()
332
+ .file(safe)
333
+ .slice(first, last + 1)
334
+ .stream();
335
+ },
336
+
308
337
  /**
309
338
  * Bytes from one key to another without a round trip through this process. Bun exposes no
310
339
  * `CopyObject`, so the source `S3File` is handed to `write()` — Bun's own union accepts one —
@@ -312,9 +341,20 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
312
341
  * win available today: promoting a 500MB attachment used to be `get()` + `put()`, a full GB
313
342
  * resident in the pod. A true server-side copy needs a Bun API that does not exist in 1.3.
314
343
  */
315
- async copy(from: string, to: string): Promise<StorageObject> {
344
+ async copy(from: string, to: string, copyOptions?: CopyOptions): Promise<StorageObject> {
316
345
  const source = assertSafeKey(from);
317
346
  const destination = assertSafeKey(to);
347
+ // Bun's `write` takes no condition, so a revision-bound copy is the signed CopyObject.
348
+ if (copyOptions?.ifMatch !== undefined) {
349
+ const refusal = await signedCopy(options, source, destination, copyOptions.ifMatch).catch(
350
+ refusedWrite(destination),
351
+ );
352
+ if (refusal?.statusCode === 404) throw objectNotFound(diskName, source);
353
+ if (refusal?.statusCode === 412)
354
+ throw objectChanged(source, copyOptions.ifMatch, 'another');
355
+ if (refusal !== undefined) refusedWrite(destination)(refusal);
356
+ return statObject(destination);
357
+ }
318
358
  const file = conn().file(source);
319
359
  if (!(await present(source))) throw objectNotFound(diskName, source);
320
360
  const stat = await file.stat().catch(refusedRead(source));
@@ -421,29 +461,13 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
421
461
  );
422
462
  },
423
463
 
424
- /**
425
- * Provider presigning. The signature covers method, expiry and content type but NOT
426
- * `maxBytes` — S3 has no header for it, so size stays a server-side policy check.
427
- */
464
+ /** Both presigners, one rule: `driver-s3-presign.ts`. */
428
465
  async signedUrl(key: string, urlOptions?: SignedUrlOptions): Promise<string> {
429
- // Screened with the SAME function `buildSignedUrl` applies on the local disk — one
430
- // finite-bound path for both presigners, never a private copy per package: `Math.ceil(NaN /
431
- // 1000)` is `NaN`, so `X-Amz-Expires=NaN` went to AWS and the app got a 403 on a link it
432
- // believed it had just minted. `=== undefined` rather than `??`, so an explicit `null` is
433
- // refused instead of silently taking the default.
434
- const expiresInMs = finiteCount(
435
- 'the s3 driver',
436
- 'expiresInMs',
437
- urlOptions?.expiresInMs === undefined ? DEFAULT_SIGNED_URL_TTL_MS : urlOptions.expiresInMs,
438
- 1,
439
- );
440
- return conn()
441
- .file(assertSafeKey(key))
442
- .presign({
443
- method: urlOptions?.method ?? 'GET',
444
- expiresIn: Math.ceil(expiresInMs / 1000),
445
- ...(urlOptions?.contentType === undefined ? {} : { type: urlOptions.contentType }),
446
- });
466
+ return s3SignedUrl(options, (safe) => conn().file(safe), key, urlOptions);
447
467
  },
448
468
  };
469
+ // Attached in place: the capability presigns and stats through the disk it belongs to.
470
+ return Object.assign(disk, {
471
+ multipart: s3Multipart({ wire: options, driver: disk, refusedWrite }),
472
+ });
449
473
  }
package/src/driver.ts CHANGED
@@ -5,10 +5,13 @@
5
5
 
6
6
  import type { Clock, Page } from '@ultimat3/core';
7
7
  import { assert } from '@ultimat3/core';
8
+ import { pumpBounded } from './byte-pump';
8
9
  import { putTooLarge } from './errors';
10
+ import type { MultipartCapability } from './multipart';
9
11
  import type { ObjectLock, ObjectLockOptions } from './object-lock';
12
+ import type { PutStreamOptions } from './put-stream';
10
13
  import type { ByteRange, GetOptions } from './range';
11
- import type { SignedUrlVerification } from './signed-url';
14
+ import type { SignedPart, SignedUrlVerification } from './signed-url';
12
15
 
13
16
  export type StorageBody = Uint8Array | ReadableStream<Uint8Array> | Blob;
14
17
 
@@ -99,18 +102,30 @@ export interface SignedUrlOptions {
99
102
  readonly method?: SignedUrlMethod | undefined;
100
103
  readonly expiresInMs?: number | undefined;
101
104
  /**
102
- * Ceiling on a `PUT`, and **the one option whose enforcement is the driver's, not the caller's**.
103
- *
104
- * `localDriver` signs it into the URL, so `acceptSignedUpload` refuses a client that widens it.
105
- * `s3Driver` CANNOT: S3 has no request header for a size and Bun's `presign` covers method,
106
- * expiry and content type only — so the client PUTs straight into the bucket and nothing between
107
- * the grant and the object enforces this number. It is not refused there, because `grantUpload`
108
- * supplies it on every grant and refusing would break every s3 upload an app mints; the ceiling
109
- * a bucket-backed disk actually gets is a bucket rule or a post-upload check on `object.size`.
110
- * `driver-parity.test.ts` pins both halves so neither moves alone.
105
+ * Ceiling on a `PUT`. `localDriver` signs it into the URL, so `acceptSignedUpload` refuses a client
106
+ * that widens it. A provider's URL has no ceiling to sign, so `s3Driver` refuses it without
107
+ * `contentLength` (`X_STORAGE_UPLOAD_SIZE_UNDECLARED`) rather than mint a URL that ignores it.
111
108
  */
112
109
  readonly maxBytes?: number | undefined;
110
+ /**
111
+ * The exact size of a `PUT`, inside the signature. On `s3Driver` the URL then signs
112
+ * `content-length`, so the provider refuses any other length, a chunked body with none included
113
+ * (403 `SignatureDoesNotMatch`). The local disks hold it as a ceiling (`x-max`): they count what
114
+ * arrives, so a shorter body passes there and not on s3. `driver-parity.test.ts` pins both.
115
+ */
116
+ readonly contentLength?: number | undefined;
113
117
  readonly contentType?: string | undefined;
118
+ /**
119
+ * A part of an open upload, inside the signature: the URL then writes that part and nothing else.
120
+ * Only a disk whose `multipart.receivePart` stores parts signs one; `s3Driver` refuses it, since
121
+ * its parts are signed by `multipart.signPart` against the provider.
122
+ */
123
+ readonly part?: SignedPart | undefined;
124
+ }
125
+
126
+ export interface CopyOptions {
127
+ /** The source's `etag` as this disk reported it. */
128
+ readonly ifMatch?: string | undefined;
114
129
  }
115
130
 
116
131
  export interface StorageDriver {
@@ -133,6 +148,16 @@ export interface StorageDriver {
133
148
  */
134
149
  registerAs?(diskName: string): void;
135
150
  put(key: string, body: StorageBody, options?: PutOptions): Promise<StorageObject>;
151
+ /**
152
+ * A body of any length, without buffering it: counted against the call's required `maxBytes`, and
153
+ * past it the write is abandoned with the key as it was (`X_STORAGE_TOO_LARGE`). Takes no
154
+ * `checksum`, retention or legal hold, which need the bytes before the object exists.
155
+ */
156
+ putStream(
157
+ key: string,
158
+ stream: ReadableStream<Uint8Array>,
159
+ options: PutStreamOptions,
160
+ ): Promise<StorageObject>;
136
161
  /**
137
162
  * What a read would say about the object, WITHOUT its bytes — `undefined` when nothing is there.
138
163
  * `promoteAttachment` decides on `stat().size`, because a presigned PUT lands on the disk
@@ -149,12 +174,19 @@ export interface StorageDriver {
149
174
  get(key: string, options?: GetOptions): Promise<StorageRead>;
150
175
  /** Bytes without buffering — the only safe path for anything over a few MB. */
151
176
  stream(key: string): Promise<ReadableStream<Uint8Array>>;
177
+ /**
178
+ * The bytes `[first, last]` (inclusive, clamped, as `get`'s range) with none before them read, so
179
+ * a seek late in a film costs its window. `X_STORAGE_RANGE_UNSATISFIABLE` past the end. A method and
180
+ * not an option on `stream`, which a third-party driver would ignore and serve the wrong bytes.
181
+ */
182
+ streamRange(key: string, range: ByteRange): Promise<ReadableStream<Uint8Array>>;
152
183
  /**
153
184
  * Bytes from one key to another without them passing through this process. Idempotent in the
154
185
  * destination and non-destructive in the source: the caller deletes the source afterwards if
155
- * it wanted a move. `X_STORAGE_NOT_FOUND` when `from` is absent.
186
+ * it wanted a move. `X_STORAGE_NOT_FOUND` when `from` is absent. `ifMatch` copies only that
187
+ * revision: any other is `X_STORAGE_OBJECT_CHANGED` and writes nothing.
156
188
  */
157
- copy(from: string, to: string): Promise<StorageObject>;
189
+ copy(from: string, to: string, options?: CopyOptions): Promise<StorageObject>;
158
190
  /**
159
191
  * Deleting an ABSENT key is not an error. Anything else is: a denied `s3:DeleteObject`, a
160
192
  * throttle, an expired credential and a read-only mount all raise `X_STORAGE_DELETE_FAILED`,
@@ -193,6 +225,12 @@ export interface StorageDriver {
193
225
  * provider and never reaches this path, and a third-party driver that implements neither is
194
226
  * refused by name (`X_STORAGE_URL_UNVERIFIABLE`) rather than reported as a bad signature.
195
227
  */
228
+ /**
229
+ * An upload in parts: resumable, and past S3's 5 GiB single-PUT ceiling. Optional, so a
230
+ * third-party driver still compiles; `grantMultipart` refuses a disk without it
231
+ * (`X_STORAGE_MULTIPART_UNSUPPORTED`). Every shipped disk has one.
232
+ */
233
+ readonly multipart?: MultipartCapability | undefined;
196
234
  verifySigned?(input: {
197
235
  readonly url: string;
198
236
  /** Overrides the driver's own, so a test's frozen clock still governs expiry. */
@@ -251,6 +289,36 @@ export function assertPutOptions(driver: string, options: PutOptions | undefined
251
289
  );
252
290
  }
253
291
 
292
+ /**
293
+ * The number a signed URL holds, screened on every disk: `contentLength` when there is one, since it
294
+ * is the tighter fact, and a `contentLength` past `maxBytes` or on anything but a `PUT` is a call
295
+ * that contradicts itself.
296
+ */
297
+ export function signedByteBound(
298
+ driver: string,
299
+ options: SignedUrlOptions | undefined,
300
+ ): number | undefined {
301
+ const length = options?.contentLength;
302
+ if (length === undefined) return options?.maxBytes;
303
+ const maxBytes = options?.maxBytes;
304
+ assert(
305
+ Number.isSafeInteger(length) && length >= 0,
306
+ `signedUrl() on the ${driver} disk was given a contentLength that is not a whole byte count`,
307
+ "signedUrl(key, { method: 'PUT', contentLength: file.size })",
308
+ );
309
+ assert(
310
+ options?.method === 'PUT',
311
+ `signedUrl() on the ${driver} disk was given a contentLength on a read, which has no body to bind`,
312
+ "signedUrl(key, { method: 'PUT', contentLength: file.size })",
313
+ );
314
+ assert(
315
+ maxBytes === undefined || length <= maxBytes,
316
+ `signedUrl() on the ${driver} disk was given a contentLength of ${length}B past its maxBytes of ${String(maxBytes)}B`,
317
+ "signedUrl(key, { method: 'PUT', contentLength: file.size }) # at or under maxBytes",
318
+ );
319
+ return length;
320
+ }
321
+
254
322
  /** A `prefix` or a `cursor` that is not a string reached `startsWith` and `<=` as whatever it was. */
255
323
  export function assertListOptions(options: ListOptions | undefined): void {
256
324
  for (const name of ['prefix', 'cursor'] as const) {
@@ -309,31 +377,13 @@ async function readBounded(
309
377
  body: ReadableStream<Uint8Array>,
310
378
  limit: ByteLimit,
311
379
  ): Promise<Uint8Array> {
312
- const reader = body.getReader();
313
380
  const chunks: Uint8Array[] = [];
314
- let total = 0;
315
- try {
316
- for (;;) {
317
- const next = await reader.read();
318
- if (next.done) break;
319
- total += next.value.byteLength;
320
- if (total > limit.maxBytes) {
321
- throw putTooLarge(limit.driver, limit.key, total, limit.maxBytes);
322
- }
323
- chunks.push(next.value);
324
- }
325
- } finally {
326
- // Cancel rather than merely release: a refused body must stop arriving, not keep filling a
327
- // socket buffer nobody will read. Already-closed streams answer this with a no-op.
328
- await reader.cancel().catch(() => undefined);
329
- }
330
- const bytes = new Uint8Array(total);
331
- let offset = 0;
332
- for (const chunk of chunks) {
333
- bytes.set(chunk, offset);
334
- offset += chunk.byteLength;
335
- }
336
- return bytes;
381
+ const { driver, key, maxBytes } = limit;
382
+ const refuse = (total: number): Error => putTooLarge(driver, key, total, maxBytes);
383
+ await pumpBounded(body, { driver, maxBytes, refuse }, (chunk) => {
384
+ chunks.push(chunk);
385
+ });
386
+ return new Uint8Array(Bun.concatArrayBuffers(chunks));
337
387
  }
338
388
 
339
389
  /** base64 SHA-256 — the wire form for `PutOptions.checksum` and `ValidatedUpload.checksum`. */
@@ -0,0 +1,35 @@
1
+ // The refusals of an upload in parts. Registered with the rest of the package's codes in
2
+ // `errors.ts`; each `fix:` names the call that goes next, with the key and upload id screened.
3
+
4
+ import { renderFixLiteral } from '@ultimat3/core';
5
+ import { StorageError } from './errors';
6
+
7
+ const keyArg = (key: string): string => renderFixLiteral(key, "'<key>'");
8
+ const idArg = (uploadId: string): string => renderFixLiteral(uploadId, "'<uploadId>'");
9
+
10
+ /** A disk whose driver declares no `multipart`: a third-party driver, most likely. */
11
+ export const multipartUnsupported = (disk: string, key: string): StorageError =>
12
+ new StorageError({
13
+ code: 'X_STORAGE_MULTIPART_UNSUPPORTED',
14
+ cause: `the ${disk} disk's driver declares no multipart capability, so "${key}" cannot be uploaded in parts`,
15
+ fix: 'grantUpload({ disk, orgId, request }) # one PUT; or implement StorageDriver.multipart on the driver',
16
+ meta: { disk, key },
17
+ });
18
+
19
+ /** No open session by that id for that key: never opened, completed, or aborted. */
20
+ export const uploadUnknown = (disk: string, key: string, uploadId: string): StorageError =>
21
+ new StorageError({
22
+ code: 'X_STORAGE_UPLOAD_UNKNOWN',
23
+ cause: `no upload ${idArg(uploadId)} is open for "${key}" on the ${disk} disk: it was never opened there, or it was already completed or aborted`,
24
+ fix: 'grantMultipart({ disk, orgId, request }) # a new session, then send every part under its uploadId',
25
+ meta: { disk, key, uploadId },
26
+ });
27
+
28
+ /** The parts a Complete named do not match what the disk holds. */
29
+ export const partsInvalid = (key: string, uploadId: string, problem: string): StorageError =>
30
+ new StorageError({
31
+ code: 'X_STORAGE_PARTS_INVALID',
32
+ cause: `the parts named to complete "${key}" do not match the parts stored: ${problem}`,
33
+ fix: `await listParts({ disk, orgId, key: ${keyArg(key)}, uploadId: ${idArg(uploadId)} }) # what landed; resend the missing part, then completeUpload again`,
34
+ meta: { key, uploadId, problem },
35
+ });
@@ -2,6 +2,7 @@
2
2
  // still in quarantine, a key that is not a pending upload, and one an earlier call already moved.
3
3
  // Their codes are registered with the rest of the package's in `errors.ts`.
4
4
 
5
+ import { renderFixLiteral } from '@ultimat3/core';
5
6
  import { StorageError } from './errors';
6
7
 
7
8
  /**
@@ -42,3 +43,16 @@ export const alreadyPromoted = (key: string, attachedKey: string, orgId: string)
42
43
  fix: `await disk.stat('${attachedKey}') answers the moved object: answer the row that references that key if one exists, write it with that key if none does, and never delete the key on a failure of your own`,
43
44
  meta: { key, attachedKey, orgId },
44
45
  });
46
+
47
+ /**
48
+ * The object is no longer the revision that was judged. A presigned PUT stays valid until it
49
+ * expires, so an uploader can overwrite between the sniff and the copy, and what would land on the
50
+ * row's key is bytes nothing looked at.
51
+ */
52
+ export const objectChanged = (key: string, expected: string, actual: string): StorageError =>
53
+ new StorageError({
54
+ code: 'X_STORAGE_OBJECT_CHANGED',
55
+ cause: `"${key}" changed while it was being promoted: it was judged at ${expected} and is now ${actual}`,
56
+ fix: `promoteAttachment({ disk, key: ${renderFixLiteral(key, "'<pending key>'")}, orgId, target, policy }) once the upload has finished: it reads the object again`,
57
+ meta: { key, expected, actual },
58
+ });
@@ -0,0 +1,35 @@
1
+ // The two refusals only a bounded sniff can reach: markup that could sit past what was read. Their
2
+ // code is `X_STORAGE_TYPE_REJECTED`, registered with the rest of the package's in `errors.ts`; they
3
+ // are their own constructors because the sniff saw nothing to name, so `contentTypeMismatch`'s
4
+ // "re-upload with the sniffed type" would be advice about a type nobody found.
5
+
6
+ import { renderFixLiteral } from '@ultimat3/core';
7
+ import { StorageError } from './errors';
8
+
9
+ /**
10
+ * A body opening with more whitespace than `limit` bytes. Past a window nothing is seen, so padding
11
+ * is how markup would hide from a head read, and no real file opens this way.
12
+ */
13
+ export const leadingWhitespacePastLimit = (
14
+ key: string,
15
+ declared: string,
16
+ limit: number,
17
+ ): StorageError =>
18
+ new StorageError({
19
+ code: 'X_STORAGE_TYPE_REJECTED',
20
+ cause: `"${key}" declares ${declared} and opens with more than ${limit} bytes of whitespace, so its type cannot be read`,
21
+ fix: `validateUpload({ key: ${renderFixLiteral(key, "'a/file.txt'")}, declaredContentType: ${renderFixLiteral(declared, "'text/plain'")}, bytes: new TextEncoder().encode(text.trimStart()) }, policy) — where text is the file with its leading whitespace removed`,
22
+ meta: { key, declared, limit },
23
+ });
24
+
25
+ /**
26
+ * An XML document too long for a head window to screen: XML runs a script anywhere in the tree, so
27
+ * no prefix can clear it. The server-side path reads all of it, which is the way through.
28
+ */
29
+ export const xmlBodyPastWindow = (key: string, declared: string, window: number): StorageError =>
30
+ new StorageError({
31
+ code: 'X_STORAGE_TYPE_REJECTED',
32
+ cause: `"${key}" declares ${declared} and is over the ${window} bytes promotion reads, and an XML document can run a script anywhere in it`,
33
+ fix: `validateUpload({ key: ${renderFixLiteral(key, "'a/data.xml'")}, declaredContentType: ${renderFixLiteral(declared, "'application/xml'")}, bytes }, policy) — upload it through the server, which reads the whole body, then disk.put(key, bytes, { contentType })`,
34
+ meta: { key, declared, window },
35
+ });
@@ -0,0 +1,62 @@
1
+ // The refusals of the streaming calls and of a size a grant was not told. Registered with the rest
2
+ // of the package's codes in `errors.ts`; each `fix:` names the call that goes next, because these
3
+ // calls have their own option where `put()`'s refusal names the disk's ceiling.
4
+
5
+ import { describeValue, renderFixLiteral, renderThrowable } from '@ultimat3/core';
6
+ import { StorageError } from './errors';
7
+
8
+ const keyArg = (key: string): string => renderFixLiteral(key, "'<key>'");
9
+
10
+ /** A `putStream` body past the call's own `maxBytes`. Counted, so `bytes` is how far it had got. */
11
+ export const streamTooLarge = (
12
+ disk: string,
13
+ key: string,
14
+ bytes: number,
15
+ maxBytes: number,
16
+ ): StorageError =>
17
+ new StorageError({
18
+ code: 'X_STORAGE_TOO_LARGE',
19
+ cause: `the stream for "${key}" passed its maxBytes of ${maxBytes}B at ${bytes}B on the ${disk} disk, so the write was abandoned and nothing was stored`,
20
+ fix: `await disk.putStream(${keyArg(key)}, stream, { maxBytes: ${bytes * 2} }) # a higher maxBytes on the call, if a body this large is meant`,
21
+ meta: { disk, key, bytes, maxBytes },
22
+ });
23
+
24
+ /** `rangeUnsatisfiable` for a call that holds no size: the next step is learning it. */
25
+ export const streamRangeUnsatisfiable = (
26
+ disk: string,
27
+ key: string,
28
+ range: readonly [number, number],
29
+ size: number,
30
+ ): StorageError =>
31
+ new StorageError({
32
+ code: 'X_STORAGE_RANGE_UNSATISFIABLE',
33
+ cause: `streamRange() asked "${key}" for bytes ${range[0]}-${range[1]}, and the object on the ${disk} disk is ${size}B, so its first byte is past the end`,
34
+ fix: `const { size } = await disk.stat(${keyArg(key)}) # the object's size now; any first byte below it is a window`,
35
+ meta: { disk, key, first: range[0], last: range[1], size },
36
+ });
37
+
38
+ /**
39
+ * A grant with nothing to bind. A presign has one way to hold a size, the exact `Content-Length`
40
+ * inside the signature, so "any size up to the policy" is a URL that bounds nothing.
41
+ */
42
+ export const uploadSizeUndeclared = (key: string, size: unknown): StorageError =>
43
+ new StorageError({
44
+ code: 'X_STORAGE_UPLOAD_SIZE_UNDECLARED',
45
+ cause: `the upload for "${key}" declared its size as ${describeValue(size)}, not a whole byte count of at least 1, so the URL could not bind one`,
46
+ fix: 'grantUpload({ disk, orgId, request: { filename, contentType, size: file.size } })',
47
+ meta: { key },
48
+ });
49
+
50
+ /**
51
+ * A write sent whose answer was lost, and nothing could settle what it did. Never `putFailed`: that
52
+ * says the key is as it was, and the object may be the one sent. `X_DB_COMMIT_UNKNOWN`'s idea.
53
+ */
54
+ export const writeOutcomeUnknown = (disk: string, key: string, lost: unknown): StorageError => {
55
+ const reason = renderThrowable(lost);
56
+ return new StorageError({
57
+ code: 'X_STORAGE_WRITE_OUTCOME_UNKNOWN',
58
+ cause: `the write of "${key}" on the ${disk} disk was sent and its answer lost (${reason}), and neither a stat nor an abort could say whether it landed`,
59
+ fix: `await disk.stat(${keyArg(key)}) # the size and etag now at the key; the same as what was sent means it landed`,
60
+ meta: { disk, key, reason },
61
+ });
62
+ };
package/src/errors.ts CHANGED
@@ -33,6 +33,12 @@ export const STORAGE_OWNED_ERROR_CODES = [
33
33
  'X_STORAGE_KEY_UNSHARED',
34
34
  'X_STORAGE_ALREADY_PROMOTED',
35
35
  'X_STORAGE_RANGE_UNSATISFIABLE',
36
+ 'X_STORAGE_OBJECT_CHANGED',
37
+ 'X_STORAGE_UPLOAD_SIZE_UNDECLARED',
38
+ 'X_STORAGE_WRITE_OUTCOME_UNKNOWN',
39
+ 'X_STORAGE_MULTIPART_UNSUPPORTED',
40
+ 'X_STORAGE_UPLOAD_UNKNOWN',
41
+ 'X_STORAGE_PARTS_INVALID',
36
42
  ] as const;
37
43
 
38
44
  /**
@@ -76,6 +82,13 @@ export const STORAGE_ERROR_TITLES: Readonly<Record<StorageOwnedErrorCode, string
76
82
  X_STORAGE_KEY_UNSHARED: 'object key is outside every tenant and every shared prefix',
77
83
  X_STORAGE_ALREADY_PROMOTED: 'the upload was already promoted onto its row',
78
84
  X_STORAGE_RANGE_UNSATISFIABLE: 'the requested byte range starts past the end of the object',
85
+ X_STORAGE_OBJECT_CHANGED: 'the object changed while it was being promoted',
86
+ X_STORAGE_UPLOAD_SIZE_UNDECLARED: 'an upload grant was asked for without the size of the file',
87
+ X_STORAGE_WRITE_OUTCOME_UNKNOWN:
88
+ 'a write was sent and its answer lost, and the disk could not say whether it landed',
89
+ X_STORAGE_MULTIPART_UNSUPPORTED: 'this disk cannot take an upload in parts',
90
+ X_STORAGE_UPLOAD_UNKNOWN: 'no open multipart upload has that id',
91
+ X_STORAGE_PARTS_INVALID: 'the parts named do not match the parts stored',
79
92
  };
80
93
 
81
94
  // One unconditional call, so a second package claiming one of storage's codes throws