@ultimat3/storage 29.3.2 → 30.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/CLAUDE.md +27 -47
  2. package/README.md +130 -12
  3. package/package.json +12 -2
  4. package/src/accept-part.ts +30 -0
  5. package/src/accept.ts +7 -0
  6. package/src/attachment.ts +99 -6
  7. package/src/byte-pump.ts +46 -0
  8. package/src/driver-local-list.ts +75 -0
  9. package/src/driver-local-multipart.ts +133 -0
  10. package/src/driver-local-read.ts +13 -0
  11. package/src/driver-local-remove.ts +39 -0
  12. package/src/driver-local-stream.ts +104 -0
  13. package/src/driver-local-write.ts +28 -5
  14. package/src/driver-local.ts +257 -283
  15. package/src/driver-memory-multipart.ts +41 -0
  16. package/src/driver-memory.ts +205 -135
  17. package/src/driver-s3-client.ts +9 -0
  18. package/src/driver-s3-copy.ts +35 -0
  19. package/src/driver-s3-lock.ts +9 -2
  20. package/src/driver-s3-multipart-session.ts +87 -0
  21. package/src/driver-s3-multipart.ts +177 -0
  22. package/src/driver-s3-presign.ts +52 -0
  23. package/src/driver-s3-put-stream.ts +97 -0
  24. package/src/driver-s3-range.ts +99 -0
  25. package/src/driver-s3-settle.ts +150 -0
  26. package/src/driver-s3-signed.ts +69 -4
  27. package/src/driver-s3-sse.ts +29 -0
  28. package/src/driver-s3.ts +113 -65
  29. package/src/driver.ts +98 -38
  30. package/src/errors-multipart.ts +35 -0
  31. package/src/errors-promote.ts +14 -0
  32. package/src/errors-range.ts +47 -0
  33. package/src/errors-sniff.ts +35 -0
  34. package/src/errors-stream.ts +62 -0
  35. package/src/errors.ts +16 -1
  36. package/src/expire.ts +169 -0
  37. package/src/grant-multipart.ts +196 -0
  38. package/src/grant.ts +42 -25
  39. package/src/image-warm.ts +84 -0
  40. package/src/image.ts +57 -0
  41. package/src/index.ts +81 -2
  42. package/src/multipart-staged.ts +127 -0
  43. package/src/multipart.ts +119 -0
  44. package/src/put-stream.ts +52 -0
  45. package/src/range.ts +102 -0
  46. package/src/signed-url.ts +59 -2
  47. package/src/subject-target.ts +82 -0
  48. package/src/upload-client-multipart.ts +156 -0
  49. package/src/upload-client.ts +21 -0
  50. package/src/upload.ts +107 -21
@@ -0,0 +1,119 @@
1
+ // Single responsibility: the part arithmetic of an S3 multipart upload: the protocol's two limits
2
+ // and the part size that follows from a ceiling, plus the optional capability a disk declares for
3
+ // an upload in parts. No wire and no disk, so any caller may ask.
4
+
5
+ import type { StorageObject } from './driver';
6
+
7
+ /** S3's floor on every part but the last. */
8
+ export const MIN_PART_BYTES = 5 * 1024 * 1024;
9
+ /** S3's count of parts in one upload. */
10
+ export const MAX_PARTS = 10_000;
11
+
12
+ /**
13
+ * How much is held before a part is sent. The smallest size whose 10,000 parts still cover
14
+ * `maxBytes`, since what is held is what this process pays for the stream.
15
+ */
16
+ export const partBytesFor = (maxBytes: number): number =>
17
+ Math.max(MIN_PART_BYTES, Math.ceil(maxBytes / MAX_PARTS));
18
+
19
+ /** S3's ceiling on one part, and so on one part URL's signed length. */
20
+ export const MAX_PART_BYTES = 5 * 1024 * 1024 * 1024;
21
+
22
+ /** One part a disk holds for an open upload: what `listParts` answers, and what a resume reads. */
23
+ export interface UploadedPart {
24
+ readonly partNumber: number;
25
+ /** The disk's own etag of the part. S3's is quoted MD5; the local disks' is `etagOf`. */
26
+ readonly etag: string;
27
+ readonly size: number;
28
+ }
29
+
30
+ /** A part the caller asks `completeUpload` to assemble. The etag, when given, must be the stored one. */
31
+ export interface CompletedPart {
32
+ readonly partNumber: number;
33
+ readonly etag?: string | undefined;
34
+ }
35
+
36
+ /** One upload opened and neither completed nor aborted: what `abortStaleUploads` weighs. */
37
+ export interface MultipartSession {
38
+ readonly key: string;
39
+ readonly uploadId: string;
40
+ /** ABSENT when the disk did not say, as `StorageListEntry.lastModified` is — never epoch 0. */
41
+ readonly initiated?: Date | undefined;
42
+ }
43
+
44
+ export interface SignPartOptions {
45
+ /** The part's exact size, signed: the disk stores it at no other length. */
46
+ readonly contentLength: number;
47
+ readonly expiresInMs: number;
48
+ }
49
+
50
+ /**
51
+ * An upload in parts, on one disk. Optional on `StorageDriver`, so a third-party driver still
52
+ * compiles; `grantMultipart` refuses a disk without it (`X_STORAGE_MULTIPART_UNSUPPORTED`). Every
53
+ * method takes a key already held to the tenant by the caller in `grant-multipart.ts`.
54
+ */
55
+ export interface MultipartCapability {
56
+ /** Opens a session and answers its id. The object's content type travels here: a part has none. */
57
+ create(key: string, options: { readonly contentType: string }): Promise<string>;
58
+ /** A URL a browser PUTs one part to. Signs nothing about whether the upload is still open. */
59
+ signPart(
60
+ key: string,
61
+ uploadId: string,
62
+ partNumber: number,
63
+ options: SignPartOptions,
64
+ ): Promise<string>;
65
+ /** Ascending by part number. `X_STORAGE_UPLOAD_UNKNOWN` for a session that is not open. */
66
+ listParts(key: string, uploadId: string): Promise<readonly UploadedPart[]>;
67
+ /** Assembles `parts`, already checked against `listParts`; the object exists once it returns. */
68
+ complete(key: string, uploadId: string, parts: readonly UploadedPart[]): Promise<StorageObject>;
69
+ /** Drops the session and every part. An unknown session is already in the asked-for state. */
70
+ abort(key: string, uploadId: string): Promise<void>;
71
+ /** Every open session under `prefix`, so a sweep can reclaim what nobody finished. */
72
+ listUploads(prefix?: string): Promise<readonly MultipartSession[]>;
73
+ /**
74
+ * Stores one part's bytes — only on a disk that signs its own part URLs (`local`, `memory`),
75
+ * called by the `/_storage` PUT once the URL verified. On s3 the bytes go to the bucket.
76
+ */
77
+ receivePart?(
78
+ key: string,
79
+ uploadId: string,
80
+ partNumber: number,
81
+ bytes: Uint8Array,
82
+ ): Promise<UploadedPart>;
83
+ }
84
+
85
+ /**
86
+ * The parts asked for, held to the parts stored: each asked part is stored, in ascending order,
87
+ * under the stored etag when one is named, and none but the last is under S3's part floor.
88
+ * Answers the stored parts in the asked order, which is what `complete` assembles; `problem` names
89
+ * the first reason it cannot.
90
+ */
91
+ export function matchParts(
92
+ asked: readonly CompletedPart[],
93
+ stored: readonly UploadedPart[],
94
+ ): { readonly parts: readonly UploadedPart[] } | { readonly problem: string } {
95
+ if (asked.length === 0) return { problem: 'no parts were named' };
96
+ const byNumber = new Map(stored.map((part) => [part.partNumber, part]));
97
+ const parts: UploadedPart[] = [];
98
+ let previous = 0;
99
+ for (const { partNumber, etag } of asked) {
100
+ if (!Number.isSafeInteger(partNumber) || partNumber <= previous) {
101
+ return { problem: `part ${String(partNumber)} is not in ascending order after ${previous}` };
102
+ }
103
+ const held = byNumber.get(partNumber);
104
+ if (held === undefined) return { problem: `part ${partNumber} never arrived` };
105
+ if (etag !== undefined && etag !== held.etag) {
106
+ return { problem: `part ${partNumber}'s etag is not the one the disk stored` };
107
+ }
108
+ parts.push(held);
109
+ previous = partNumber;
110
+ }
111
+ // S3's EntityTooSmall, asked on every disk so the local ones cannot complete what s3 refuses.
112
+ const short = parts.slice(0, -1).find((part) => part.size < MIN_PART_BYTES);
113
+ if (short !== undefined) {
114
+ return {
115
+ problem: `part ${short.partNumber} is ${short.size}B, under the ${MIN_PART_BYTES}B every part but the last must reach`,
116
+ };
117
+ }
118
+ return { parts };
119
+ }
@@ -0,0 +1,52 @@
1
+ // Single responsibility: what `putStream` may be handed, and the limit it pumps under. The pump itself
2
+ // is `byte-pump.ts`, which `put()` reads through too.
3
+
4
+ import { assert, describeValue, finiteCount } from '@ultimat3/core';
5
+ import type { PumpLimit } from './byte-pump';
6
+ import { assertPutOptions } from './driver';
7
+ import { streamTooLarge } from './errors-stream';
8
+
9
+ export interface PutStreamOptions {
10
+ /**
11
+ * Required: a stream's length is the sender's until it is counted. Past it the write is abandoned
12
+ * (`X_STORAGE_TOO_LARGE`) and the key is as it was. Replaces the disk's `maxPutBytes`, which bounds
13
+ * what `put()` buffers, and nothing here is buffered past one s3 part.
14
+ */
15
+ readonly maxBytes: number;
16
+ readonly contentType?: string | undefined;
17
+ readonly cacheControl?: string | undefined;
18
+ readonly metadata?: Readonly<Record<string, string>> | undefined;
19
+ }
20
+
21
+ /** `put()` options that need the bytes before the object exists. The type omits them too. */
22
+ const NEEDS_THE_BYTES = ['checksum', 'retention', 'legalHold', 'serverSideEncryption'] as const;
23
+
24
+ /** Everything judged before a chunk is read. Answers the screened `maxBytes`. */
25
+ export function assertPutStream(
26
+ driver: string,
27
+ body: ReadableStream<Uint8Array>,
28
+ options: PutStreamOptions,
29
+ ): number {
30
+ assert(
31
+ body instanceof ReadableStream,
32
+ `putStream() on the ${driver} disk was given a body that is not a ReadableStream (${describeValue(body)})`,
33
+ 'await disk.putStream(key, request.body, { maxBytes }) # a Uint8Array or a Blob goes through disk.put(key, bytes)',
34
+ );
35
+ const asked = (options ?? {}) as unknown as Readonly<Record<string, unknown>>;
36
+ for (const name of NEEDS_THE_BYTES) {
37
+ assert(
38
+ asked[name] === undefined,
39
+ `putStream() on the ${driver} disk was given ${name}, which needs the whole body before the object exists`,
40
+ `await disk.put(key, bytes, { ${name} }) # buffered, so bounded by the disk's maxPutBytes`,
41
+ );
42
+ }
43
+ assertPutOptions(driver, options);
44
+ return finiteCount(`putStream() on the ${driver} disk`, 'maxBytes', options?.maxBytes, 1);
45
+ }
46
+
47
+ /** The limit `putStream` pumps under: its own `maxBytes`, and the refusal that names it. */
48
+ export const streamLimit = (driver: string, key: string, maxBytes: number): PumpLimit => ({
49
+ driver,
50
+ maxBytes,
51
+ refuse: (total) => streamTooLarge(driver, key, total, maxBytes),
52
+ });
package/src/range.ts ADDED
@@ -0,0 +1,102 @@
1
+ // Single responsibility: what `get(key, { range })` may ask for and what it is answered with — the
2
+ // screen of the tuple for every disk, and for memory and local the clamp, the 416 and the ceiling
3
+ // (which bounds the BYTES ASKED FOR, not the object). s3 learns the size from its one GET's answer
4
+ // and applies the same three rules there (`driver-s3-range.ts`), with the same refusals.
5
+
6
+ import { assert } from '@ultimat3/core';
7
+ import { rangeTooLarge, rangeUnsatisfiable } from './errors-range';
8
+ import { streamRangeUnsatisfiable } from './errors-stream';
9
+
10
+ /**
11
+ * The first and the last byte, BOTH INCLUSIVE — HTTP's `Range: bytes=first-last` and S3's, so a
12
+ * route copies a parsed header straight in and a `Content-Range` is read straight back out.
13
+ * `[0, 99]` is the first hundred bytes; `[0, 0]` is one.
14
+ */
15
+ export type ByteRange = readonly [first: number, last: number];
16
+
17
+ export interface GetOptions {
18
+ /**
19
+ * Read only these bytes. The last byte is clamped to the object (`[0, 1e12]` on a 10-byte object
20
+ * answers `[0, 9]`), as HTTP clamps it; a first byte at or past the end is
21
+ * `X_STORAGE_RANGE_UNSATISFIABLE`, which is a 416. `maxGetBytes` bounds the RANGE'S length, so a
22
+ * page of a large object is readable where `get()` of the whole would be refused.
23
+ *
24
+ * Meaningless on a SEALED object: `seal()`'s `x1` envelope is one AES-GCM ciphertext whose tag
25
+ * authenticates every byte, so a slice of it neither opens nor means anything. Range-read plain
26
+ * objects; page a sealed one by sealing each page as its own object.
27
+ */
28
+ readonly range?: ByteRange | undefined;
29
+ }
30
+
31
+ const isOffset = (value: unknown): value is number =>
32
+ typeof value === 'number' && Number.isSafeInteger(value) && value >= 0;
33
+
34
+ /**
35
+ * The tuple's SHAPE, before the disk is asked anything: two whole, non-negative offsets in order.
36
+ * Core's `assert` (`X_INVARIANT`), because a backwards or fractional range is the calling code's
37
+ * defect — no object makes it valid — while a first byte past the end depends on the object and
38
+ * is the coded, caller-facing refusal below.
39
+ */
40
+ export function assertGetOptions(driver: string, options: GetOptions | undefined): void {
41
+ const range = options?.range;
42
+ if (range === undefined) return;
43
+ assertRange(driver, 'get()', 'get(key, { range: ', ' })', range);
44
+ }
45
+
46
+ /** The same screen for `streamRange`, naming the call that was made. */
47
+ export function assertStreamRange(driver: string, range: ByteRange): void {
48
+ assertRange(driver, 'streamRange()', 'streamRange(key, ', ')', range);
49
+ }
50
+
51
+ function assertRange(
52
+ driver: string,
53
+ call: string,
54
+ open: string,
55
+ close: string,
56
+ range: ByteRange,
57
+ ): void {
58
+ const [first, last] = Array.isArray(range) ? range : [];
59
+ assert(
60
+ Array.isArray(range) && range.length === 2 && isOffset(first) && isOffset(last),
61
+ `${call} on the ${driver} disk was given a range that is not two whole, non-negative byte offsets`,
62
+ `${open}[first, last]${close} — inclusive offsets: [0, 99] reads the first 100 bytes`,
63
+ );
64
+ assert(
65
+ first <= last,
66
+ `${call} on the ${driver} disk was given the range [${first}, ${last}], whose first byte is past its last`,
67
+ `${open}[${last}, ${first}]${close} — first before last, both inclusive`,
68
+ );
69
+ }
70
+
71
+ /**
72
+ * The window `streamRange` serves: clamped as `servedRange` clamps. No ceiling, because nothing is
73
+ * buffered: the bytes asked for are the bytes pulled, and they flow at the reader's pace.
74
+ */
75
+ export function streamedRange(
76
+ disk: string,
77
+ key: string,
78
+ range: ByteRange,
79
+ size: number,
80
+ ): ByteRange {
81
+ const [first, wanted] = range;
82
+ if (first >= size) throw streamRangeUnsatisfiable(disk, key, range, size);
83
+ return [first, Math.min(wanted, size - 1)];
84
+ }
85
+
86
+ /**
87
+ * The range a disk SERVES for an object of `size` bytes — clamped, refused, or over the ceiling.
88
+ * Called with the object's own measured size, before a byte is read, by every driver.
89
+ */
90
+ export function servedRange(
91
+ disk: string,
92
+ key: string,
93
+ range: ByteRange,
94
+ size: number,
95
+ maxGetBytes: number,
96
+ ): ByteRange {
97
+ const [first, wanted] = range;
98
+ if (first >= size) throw rangeUnsatisfiable(disk, key, range, size);
99
+ const last = Math.min(wanted, size - 1);
100
+ if (last - first + 1 > maxGetBytes) throw rangeTooLarge(disk, key, [first, last], maxGetBytes);
101
+ return [first, last];
102
+ }
package/src/signed-url.ts CHANGED
@@ -4,7 +4,7 @@
4
4
  // a 2GB executable. Verification never throws and never short-circuits on expiry before the
5
5
  // signature, so a forged URL can never learn "the signature was fine, just late".
6
6
 
7
- import { type Clock, finiteCount, systemClock, timingSafeEqual } from '@ultimat3/core';
7
+ import { assert, type Clock, finiteCount, systemClock, timingSafeEqual } from '@ultimat3/core';
8
8
  import type { SignedUrlMethod } from './driver';
9
9
  import { assertSafeKey, isSafeKey } from './path';
10
10
 
@@ -31,8 +31,20 @@ export const SIGNED_URL_PARAMS = {
31
31
  maxBytes: 'x-max',
32
32
  contentType: 'x-ct',
33
33
  signature: 'x-sig',
34
+ uploadId: 'x-upload',
35
+ partNumber: 'x-part',
34
36
  } as const;
35
37
 
38
+ /** The session and part a part URL writes into — inside the signature, so neither can be edited. */
39
+ export interface SignedPart {
40
+ readonly uploadId: string;
41
+ readonly partNumber: number;
42
+ }
43
+
44
+ /** `MAX_PARTS` restated as the parse's bound: `multipart.ts` imports this module's types. */
45
+ const MAX_SIGNED_PART = 10_000;
46
+ const SIGNED_UPLOAD_ID = /^[A-Za-z0-9._~-]{1,128}$/;
47
+
36
48
  export interface SignedUrlConstraints {
37
49
  readonly key: string;
38
50
  readonly method: SignedUrlMethod;
@@ -40,6 +52,8 @@ export interface SignedUrlConstraints {
40
52
  readonly expiresAt: number;
41
53
  readonly maxBytes: number | undefined;
42
54
  readonly contentType: string | undefined;
55
+ /** Present on a part URL only: a single PUT and a part are two capabilities, never one. */
56
+ readonly part?: SignedPart | undefined;
43
57
  }
44
58
 
45
59
  export interface SignedUrlInput {
@@ -50,6 +64,8 @@ export interface SignedUrlInput {
50
64
  readonly expiresInMs?: number | undefined;
51
65
  readonly maxBytes?: number | undefined;
52
66
  readonly contentType?: string | undefined;
67
+ /** Mint a part URL of an open upload (`MultipartCapability.signPart` on the local disks). */
68
+ readonly part?: SignedPart | undefined;
53
69
  readonly baseUrl?: string | undefined;
54
70
  readonly clock?: Clock | undefined;
55
71
  }
@@ -92,14 +108,19 @@ export function signedUrlBasePath(baseUrl: string): string {
92
108
  * origin is deliberately outside it: the verifying route is handed a path.
93
109
  */
94
110
  export function canonicalRequest(constraints: SignedUrlConstraints, basePath: string): string {
111
+ const part = constraints.part;
112
+ // A part signs under its own version line, so no single-PUT string can ever read as a part's:
113
+ // the two domains share no canonical form, and a single URL's bytes are what they always were.
114
+ const tail = part === undefined ? [] : [part.uploadId, String(part.partNumber)];
95
115
  return [
96
- SIGNED_URL_VERSION,
116
+ part === undefined ? SIGNED_URL_VERSION : `${SIGNED_URL_VERSION}-part`,
97
117
  basePath,
98
118
  constraints.method,
99
119
  constraints.key,
100
120
  String(constraints.expiresAt),
101
121
  constraints.maxBytes === undefined ? '' : String(constraints.maxBytes),
102
122
  constraints.contentType ?? '',
123
+ ...tail,
103
124
  ].join('\n');
104
125
  }
105
126
 
@@ -163,6 +184,7 @@ export async function buildSignedUrl(input: SignedUrlInput): Promise<string> {
163
184
  // one while `acceptSignedUpload`'s `unconstrained` refusal (which tests `undefined`) stayed
164
185
  // silent. One spelling for "no content type", here and at the parse below.
165
186
  contentType: input.contentType === '' ? undefined : input.contentType,
187
+ ...(input.part === undefined ? {} : { part: assertSignedPart(input.part) }),
166
188
  };
167
189
  const params = new URLSearchParams();
168
190
  params.set(SIGNED_URL_PARAMS.method, constraints.method);
@@ -173,6 +195,10 @@ export async function buildSignedUrl(input: SignedUrlInput): Promise<string> {
173
195
  if (constraints.contentType !== undefined) {
174
196
  params.set(SIGNED_URL_PARAMS.contentType, constraints.contentType);
175
197
  }
198
+ if (constraints.part !== undefined) {
199
+ params.set(SIGNED_URL_PARAMS.uploadId, constraints.part.uploadId);
200
+ params.set(SIGNED_URL_PARAMS.partNumber, String(constraints.part.partNumber));
201
+ }
176
202
  const base = trimBase(input.baseUrl ?? DEFAULT_SIGNED_URL_BASE);
177
203
  params.set(
178
204
  SIGNED_URL_PARAMS.signature,
@@ -232,15 +258,46 @@ function parseConstraints(url: URL, base: string): SignedUrlConstraints | Signed
232
258
  // An EMPTY `x-ct` is malformed, never "no content type": absent and empty share one canonical
233
259
  // string, so accepting it let `&x-ct=` be appended to a URL signed with none and still verify.
234
260
  if (rawContentType === '') return 'malformed';
261
+ const part = parsePart(url);
262
+ if (part === 'malformed') return part;
235
263
  return {
236
264
  key,
237
265
  method,
238
266
  expiresAt,
239
267
  maxBytes,
240
268
  contentType: rawContentType ?? undefined,
269
+ ...(part === undefined ? {} : { part }),
241
270
  };
242
271
  }
243
272
 
273
+ const isSignedPart = (part: SignedPart): boolean =>
274
+ SIGNED_UPLOAD_ID.test(part.uploadId) &&
275
+ Number.isSafeInteger(part.partNumber) &&
276
+ part.partNumber >= 1 &&
277
+ part.partNumber <= MAX_SIGNED_PART;
278
+
279
+ /** The mint's screen: a part this module could not parse back is a URL dead on arrival. */
280
+ function assertSignedPart(part: SignedPart): SignedPart {
281
+ assert(
282
+ isSignedPart(part),
283
+ `buildSignedUrl was handed a part that is not an upload id and a part number 1..${MAX_SIGNED_PART}`,
284
+ "buildSignedUrl({ secret, key, method: 'PUT', part: { uploadId: crypto.randomUUID(), partNumber: 1 } })",
285
+ );
286
+ return { uploadId: part.uploadId, partNumber: part.partNumber };
287
+ }
288
+
289
+ /** Both parameters or neither; one alone is a URL this module never minted. */
290
+ function parsePart(url: URL): SignedPart | 'malformed' | undefined {
291
+ const uploadId = url.searchParams.get(SIGNED_URL_PARAMS.uploadId);
292
+ const rawNumber = url.searchParams.get(SIGNED_URL_PARAMS.partNumber);
293
+ if (uploadId === null && rawNumber === null) return undefined;
294
+ if (uploadId === null || rawNumber === null || !/^[1-9][0-9]{0,4}$/.test(rawNumber)) {
295
+ return 'malformed';
296
+ }
297
+ const part = { uploadId, partNumber: Number(rawNumber) };
298
+ return isSignedPart(part) ? part : 'malformed';
299
+ }
300
+
244
301
  export async function verifySignedUrl(input: VerifySignedUrlInput): Promise<SignedUrlVerification> {
245
302
  let url: URL;
246
303
  try {
@@ -0,0 +1,82 @@
1
+ // `storageSubjectTarget(disk, keysOf)`: the subject's stored objects as a subject target a
2
+ // `defineSubject({ targets })` lists — structural, so `@ultimat3/jobs` imports nothing from here.
3
+ // Which keys are the person's is the app's (`keysOf`); the export lists each object, the erase
4
+ // deletes them. A key the app does not name is not found: there is no listing by owner.
5
+
6
+ import type { StorageDriver } from './driver';
7
+
8
+ /** The subject as a target is handed it: the key asked for, and the subject's row read fresh. */
9
+ export interface StorageSubjectRef {
10
+ readonly key: string;
11
+ readonly row: Readonly<Record<string, unknown>>;
12
+ }
13
+
14
+ export interface StorageSubjectTargetOptions {
15
+ /** The step key and export folder. Default `storage`. Unique among one subject's targets. */
16
+ readonly name?: string;
17
+ }
18
+
19
+ type Cell = string | number | boolean | null;
20
+
21
+ export interface StorageSubjectTarget {
22
+ readonly name: string;
23
+ readonly columns: readonly string[];
24
+ exportPage(
25
+ subject: StorageSubjectRef,
26
+ cursor: string | null,
27
+ size: number,
28
+ ): Promise<{
29
+ readonly rows: readonly Readonly<Record<string, Cell>>[];
30
+ readonly cursor: string | null;
31
+ }>;
32
+ erase(subject: StorageSubjectRef, size: number): Promise<number>;
33
+ }
34
+
35
+ export const STORAGE_SUBJECT_COLUMNS = ['key', 'size', 'contentType', 'lastModified'] as const;
36
+
37
+ /**
38
+ * `disk` is a THUNK, read per call, as an export's sink is: a subject is declared at import and
39
+ * the app's disks exist after `defineStorage()`.
40
+ */
41
+ export function storageSubjectTarget(
42
+ disk: () => StorageDriver,
43
+ /** The person's keys — a list the app builds, or lists from a prefix it owns. */
44
+ keysOf: (subject: StorageSubjectRef) => readonly string[] | Promise<readonly string[]>,
45
+ options: StorageSubjectTargetOptions = {},
46
+ ): StorageSubjectTarget {
47
+ return {
48
+ name: options.name ?? 'storage',
49
+ columns: STORAGE_SUBJECT_COLUMNS,
50
+ async exportPage(subject, cursor, size) {
51
+ const keys = await keysOf(subject);
52
+ const start = cursor === null ? 0 : Number(cursor);
53
+ const page = keys.slice(start, start + size);
54
+ const rows: Record<string, Cell>[] = [];
55
+ for (const key of page) {
56
+ const found = await disk().stat(key);
57
+ if (found === undefined) continue;
58
+ rows.push({
59
+ key,
60
+ size: found.size,
61
+ contentType: found.contentType,
62
+ // An instant, UTC, its zone in the string — never formatted for a locale here. Absent
63
+ // when the provider sent none, and never invented.
64
+ lastModified: found.lastModified?.toISOString() ?? null,
65
+ });
66
+ }
67
+ const next = start + page.length;
68
+ return { rows, cursor: next < keys.length ? String(next) : null };
69
+ },
70
+ async erase(subject, size) {
71
+ // Only what still exists counts, so a replayed batch answers short and the loop ends.
72
+ let removed = 0;
73
+ for (const key of await keysOf(subject)) {
74
+ if (removed >= size) break;
75
+ if (!(await disk().exists(key))) continue;
76
+ await disk().delete(key);
77
+ removed += 1;
78
+ }
79
+ return removed;
80
+ },
81
+ };
82
+ }
@@ -0,0 +1,156 @@
1
+ // Single responsibility: the browser half of an upload in parts — grant (or resume), PUT each part
2
+ // at its own signed URL with bounded concurrency and core's retry, then complete. Browser-safe as
3
+ // `upload-client.ts` is: no `Bun.*`, no `node:`, no driver import, and every network call is either
4
+ // the app's typed action or the one `SignedPut` seam.
5
+
6
+ import { finiteCount, retry } from '@ultimat3/core';
7
+ import { uploadFailed } from './errors';
8
+ import type { UploadRequest } from './grant';
9
+ import type { MultipartGrant } from './grant-multipart';
10
+ import type { CompletedPart, UploadedPart } from './multipart';
11
+ import type { SignedPut, UploadProgress, UploadSource } from './upload-client';
12
+
13
+ /** One open session, as each of the app's calls names it. */
14
+ export interface UploadSessionRef {
15
+ readonly key: string;
16
+ readonly uploadId: string;
17
+ }
18
+
19
+ /**
20
+ * The app's typed action calls, each wrapping one server function of `grant-multipart.ts` under the
21
+ * app's own policy — the reason `grant` is the app's in `uploadFile`.
22
+ */
23
+ export interface MultipartUploadClient {
24
+ readonly grant: (request: UploadRequest) => Promise<MultipartGrant>;
25
+ readonly signPart: (
26
+ input: UploadSessionRef & { readonly partNumber: number; readonly size: number },
27
+ ) => Promise<string>;
28
+ readonly listParts: (input: UploadSessionRef) => Promise<readonly UploadedPart[]>;
29
+ /** Etags are read on the server (`completeUpload`), so no bucket CORS has to expose `ETag`. */
30
+ readonly complete: (
31
+ input: UploadSessionRef & { readonly parts: readonly CompletedPart[] },
32
+ ) => Promise<unknown>;
33
+ /** Called when the caller's `signal` aborts: a cancelled upload leaves no parts billing. */
34
+ readonly abort: (input: UploadSessionRef) => Promise<unknown>;
35
+ /** At or under this, one PUT as before. Default 64 MiB. */
36
+ readonly partThresholdBytes?: number | undefined;
37
+ /**
38
+ * A grant from an earlier, interrupted call for the same file: its parts are listed first and only
39
+ * the missing ones are sent. Persisting it is the caller's — this file keeps no browser storage.
40
+ */
41
+ readonly resume?: MultipartGrant | undefined;
42
+ /** Handed the grant before any part moves, so the caller can persist it for `resume`. */
43
+ readonly onGrant?: ((grant: MultipartGrant) => void) | undefined;
44
+ /** Test seam for the retry waits. Defaults to a timer. */
45
+ readonly sleep?: ((ms: number) => Promise<void>) | undefined;
46
+ }
47
+
48
+ export const DEFAULT_PART_THRESHOLD_BYTES = 64 * 1024 * 1024;
49
+ /** Parts in flight at once: enough to fill a phone's uplink, few enough to keep its memory. */
50
+ export const PART_CONCURRENCY = 4;
51
+ const PART_RETRY = { attempts: 4, base: 500, max: 8_000, jitter: 'full' } as const;
52
+
53
+ export const partThresholdOf = (client: MultipartUploadClient): number =>
54
+ finiteCount(
55
+ 'uploadFile',
56
+ 'multipart.partThresholdBytes',
57
+ client.partThresholdBytes === undefined
58
+ ? DEFAULT_PART_THRESHOLD_BYTES
59
+ : client.partThresholdBytes,
60
+ 1,
61
+ );
62
+
63
+ export interface MultipartUploadInput {
64
+ readonly file: UploadSource;
65
+ readonly client: MultipartUploadClient;
66
+ readonly put: SignedPut;
67
+ readonly onProgress?: ((progress: UploadProgress) => void) | undefined;
68
+ readonly signal?: AbortSignal | undefined;
69
+ }
70
+
71
+ /** A grant for this file, or the resumed one; a resume for another size is a different file. */
72
+ async function sessionFor(input: MultipartUploadInput): Promise<MultipartGrant> {
73
+ const { file, client } = input;
74
+ const resumed = client.resume;
75
+ if (resumed !== undefined && resumed.size === file.size) return resumed;
76
+ const grant = await client.grant({
77
+ filename: file.name,
78
+ contentType: file.type,
79
+ size: file.size,
80
+ });
81
+ client.onGrant?.(grant);
82
+ return grant;
83
+ }
84
+
85
+ /** Send every part not yet on the disk, then complete. Answers the grant the object landed under. */
86
+ export async function uploadInParts(input: MultipartUploadInput): Promise<MultipartGrant> {
87
+ const { file, client, put, signal } = input;
88
+ const grant = await sessionFor(input);
89
+ const session = { key: grant.key, uploadId: grant.uploadId };
90
+ const total = file.size;
91
+ const count = Math.ceil(total / grant.partBytes);
92
+ const landed = client.resume === grant ? await client.listParts(session) : [];
93
+ const have = new Set(
94
+ landed
95
+ .filter(
96
+ (part) =>
97
+ part.size === Math.min(grant.partBytes, total - (part.partNumber - 1) * grant.partBytes),
98
+ )
99
+ .map((part) => part.partNumber),
100
+ );
101
+ let confirmed = landed
102
+ .filter((part) => have.has(part.partNumber))
103
+ .reduce((sum, part) => sum + part.size, 0);
104
+ input.onProgress?.({ loaded: confirmed, total, ratio: total > 0 ? confirmed / total : 0 });
105
+
106
+ const missing: number[] = [];
107
+ for (let number = 1; number <= count; number += 1) if (!have.has(number)) missing.push(number);
108
+
109
+ const cancelled = () => uploadFailed(grant.key, 0, 'the upload was aborted');
110
+ // A rejected wait ends core's retry loop with that error: a cancelled upload stops retrying.
111
+ const waitUnlessAborted = (ms: number): Promise<void> =>
112
+ signal?.aborted === true
113
+ ? Promise.reject(cancelled())
114
+ : (client.sleep ?? ((wait) => new Promise<void>((done) => setTimeout(done, wait))))(ms);
115
+
116
+ const sendPart = async (partNumber: number): Promise<void> => {
117
+ const start = (partNumber - 1) * grant.partBytes;
118
+ const body = file.slice(start, Math.min(start + grant.partBytes, total));
119
+ await retry(
120
+ async () => {
121
+ const url = await client.signPart({ ...session, partNumber, size: body.size });
122
+ await put({
123
+ url,
124
+ contentType: grant.contentType,
125
+ body,
126
+ ...(signal === undefined ? {} : { signal }),
127
+ });
128
+ },
129
+ PART_RETRY,
130
+ { sleep: waitUnlessAborted },
131
+ );
132
+ confirmed += body.size;
133
+ input.onProgress?.({ loaded: confirmed, total, ratio: total > 0 ? confirmed / total : 0 });
134
+ };
135
+
136
+ const worker = async (): Promise<void> => {
137
+ for (let next = missing.shift(); next !== undefined; next = missing.shift()) {
138
+ if (signal?.aborted === true) return;
139
+ await sendPart(next);
140
+ }
141
+ };
142
+ try {
143
+ await Promise.all(Array.from({ length: Math.min(PART_CONCURRENCY, missing.length) }, worker));
144
+ if (signal?.aborted === true) throw cancelled();
145
+ } catch (error) {
146
+ // Cancelled is abandoned: the parts are dropped now rather than billing until a sweep. Any
147
+ // other failure keeps the session open, so the same grant passed as `resume` sends only what
148
+ // is missing. A refused abort is the sweep's to reclaim (`abortStaleUploads`), and the
149
+ // caller hears the failure that stopped the upload, not the clean-up's.
150
+ if (signal?.aborted === true) await client.abort(session).catch(() => undefined);
151
+ throw error;
152
+ }
153
+ const parts = Array.from({ length: count }, (_, at) => ({ partNumber: at + 1 }));
154
+ await client.complete({ ...session, parts });
155
+ return grant;
156
+ }