@ultimat3/storage 29.4.0 → 30.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +25 -45
- package/README.md +102 -12
- package/package.json +12 -2
- package/src/accept-part.ts +30 -0
- package/src/accept.ts +7 -0
- package/src/attachment.ts +99 -6
- package/src/byte-pump.ts +46 -0
- package/src/driver-local-list.ts +75 -0
- package/src/driver-local-multipart.ts +133 -0
- package/src/driver-local-remove.ts +39 -0
- package/src/driver-local-stream.ts +104 -0
- package/src/driver-local-write.ts +28 -5
- package/src/driver-local.ts +250 -295
- package/src/driver-memory-multipart.ts +41 -0
- package/src/driver-memory.ts +205 -143
- package/src/driver-s3-client.ts +9 -0
- package/src/driver-s3-copy.ts +35 -0
- package/src/driver-s3-lock.ts +9 -2
- package/src/driver-s3-multipart-session.ts +87 -0
- package/src/driver-s3-multipart.ts +177 -0
- package/src/driver-s3-presign.ts +52 -0
- package/src/driver-s3-put-stream.ts +97 -0
- package/src/driver-s3-settle.ts +150 -0
- package/src/driver-s3-signed.ts +68 -3
- package/src/driver-s3-sse.ts +29 -0
- package/src/driver-s3.ts +89 -65
- package/src/driver.ts +86 -36
- package/src/errors-multipart.ts +35 -0
- package/src/errors-promote.ts +14 -0
- package/src/errors-sniff.ts +35 -0
- package/src/errors-stream.ts +62 -0
- package/src/errors.ts +13 -0
- package/src/expire.ts +169 -0
- package/src/grant-multipart.ts +196 -0
- package/src/grant.ts +42 -25
- package/src/image-warm.ts +84 -0
- package/src/image.ts +57 -0
- package/src/index.ts +75 -3
- package/src/multipart-staged.ts +127 -0
- package/src/multipart.ts +119 -0
- package/src/put-stream.ts +52 -0
- package/src/range.ts +35 -4
- package/src/signed-url.ts +59 -2
- package/src/subject-target.ts +82 -0
- package/src/upload-client-multipart.ts +156 -0
- package/src/upload-client.ts +21 -0
- package/src/upload.ts +107 -21
package/src/index.ts
CHANGED
|
@@ -3,8 +3,11 @@
|
|
|
3
3
|
|
|
4
4
|
export type { AcceptSignedUploadInput, SignedRequestInput } from './accept';
|
|
5
5
|
export { acceptSignedUpload, readSignedObject, signedUploadConstraints } from './accept';
|
|
6
|
+
export type { AcceptSignedPartInput } from './accept-part';
|
|
7
|
+
export { acceptSignedPart } from './accept-part';
|
|
6
8
|
export type {
|
|
7
9
|
AttachmentTarget,
|
|
10
|
+
DeleteVariantsInput,
|
|
8
11
|
PromoteAttachmentInput,
|
|
9
12
|
ReleaseQuarantineInput,
|
|
10
13
|
SweepFailure,
|
|
@@ -14,6 +17,7 @@ export type {
|
|
|
14
17
|
export {
|
|
15
18
|
attachmentKey,
|
|
16
19
|
attachmentPrefix,
|
|
20
|
+
deleteVariants,
|
|
17
21
|
isPendingKey,
|
|
18
22
|
isQuarantinedKey,
|
|
19
23
|
PENDING_SEGMENT,
|
|
@@ -28,8 +32,11 @@ export {
|
|
|
28
32
|
uploadExtension,
|
|
29
33
|
uploadName,
|
|
30
34
|
} from './attachment';
|
|
35
|
+
export type { PumpLimit } from './byte-pump';
|
|
36
|
+
export { pumpBounded } from './byte-pump';
|
|
31
37
|
export type {
|
|
32
38
|
ByteLimit,
|
|
39
|
+
CopyOptions,
|
|
33
40
|
ListOptions,
|
|
34
41
|
PutOptions,
|
|
35
42
|
ServerSideEncryption,
|
|
@@ -50,6 +57,7 @@ export {
|
|
|
50
57
|
etagOf,
|
|
51
58
|
resolveListLimit,
|
|
52
59
|
sha256Base64,
|
|
60
|
+
signedByteBound,
|
|
53
61
|
toBytes,
|
|
54
62
|
} from './driver';
|
|
55
63
|
export type { LocalDriverOptions } from './driver-local';
|
|
@@ -96,10 +104,33 @@ export {
|
|
|
96
104
|
uploadFailed,
|
|
97
105
|
xmlBodyUnreadable,
|
|
98
106
|
} from './errors';
|
|
107
|
+
export { multipartUnsupported, partsInvalid, uploadUnknown } from './errors-multipart';
|
|
99
108
|
export { alreadyPromoted, quarantined } from './errors-promote';
|
|
100
109
|
export { rangeTooLarge, rangeUnsatisfiable } from './errors-range';
|
|
110
|
+
export type {
|
|
111
|
+
AbortResult,
|
|
112
|
+
AbortStaleUploadsInput,
|
|
113
|
+
ExpireObjectsInput,
|
|
114
|
+
ExpireResult,
|
|
115
|
+
ObjectPurgeTargetInput,
|
|
116
|
+
} from './expire';
|
|
117
|
+
export { abortStaleUploads, expireObjects, objectPurgeTarget } from './expire';
|
|
101
118
|
export type { GrantUploadInput, UploadGrant, UploadRequest } from './grant';
|
|
102
119
|
export { grantUpload } from './grant';
|
|
120
|
+
export type {
|
|
121
|
+
CompleteUploadInput,
|
|
122
|
+
GrantMultipartInput,
|
|
123
|
+
MultipartGrant,
|
|
124
|
+
SignPartInput,
|
|
125
|
+
UploadSessionInput,
|
|
126
|
+
} from './grant-multipart';
|
|
127
|
+
export {
|
|
128
|
+
abortUpload,
|
|
129
|
+
completeUpload,
|
|
130
|
+
grantMultipart,
|
|
131
|
+
listParts,
|
|
132
|
+
signPart,
|
|
133
|
+
} from './grant-multipart';
|
|
103
134
|
export type {
|
|
104
135
|
ImageFit,
|
|
105
136
|
ImageSize,
|
|
@@ -120,8 +151,21 @@ export {
|
|
|
120
151
|
srcsetDescriptors,
|
|
121
152
|
transformImage,
|
|
122
153
|
VARIANT_FORMATS,
|
|
154
|
+
variantFormatOf,
|
|
123
155
|
variantKey,
|
|
156
|
+
variantRevision,
|
|
157
|
+
variantSourceOf,
|
|
124
158
|
} from './image';
|
|
159
|
+
export type { WarmFailure, WarmVariantsInput, WarmVariantsResult } from './image-warm';
|
|
160
|
+
export { warmVariants } from './image-warm';
|
|
161
|
+
export type {
|
|
162
|
+
CompletedPart,
|
|
163
|
+
MultipartCapability,
|
|
164
|
+
MultipartSession,
|
|
165
|
+
SignPartOptions,
|
|
166
|
+
UploadedPart,
|
|
167
|
+
} from './multipart';
|
|
168
|
+
export { MAX_PART_BYTES, MAX_PARTS, MIN_PART_BYTES, matchParts, partBytesFor } from './multipart';
|
|
125
169
|
export type {
|
|
126
170
|
LockedAction,
|
|
127
171
|
ObjectLock,
|
|
@@ -130,7 +174,6 @@ export type {
|
|
|
130
174
|
RetentionMode,
|
|
131
175
|
} from './object-lock';
|
|
132
176
|
export { isLocked, ObjectLockedError, RETENTION_MODES } from './object-lock';
|
|
133
|
-
|
|
134
177
|
export {
|
|
135
178
|
assertSafeKey,
|
|
136
179
|
isSafeKey,
|
|
@@ -145,13 +188,16 @@ export {
|
|
|
145
188
|
orgPrefix,
|
|
146
189
|
scopedKey,
|
|
147
190
|
} from './path';
|
|
191
|
+
export type { PutStreamOptions } from './put-stream';
|
|
192
|
+
export { assertPutStream, streamLimit } from './put-stream';
|
|
148
193
|
/**
|
|
149
194
|
* `get(key, { range })`. A third-party driver screens and clamps through these two, or it is a
|
|
150
195
|
* fourth answer to what a range means.
|
|
151
196
|
*/
|
|
152
197
|
export type { ByteRange, GetOptions } from './range';
|
|
153
|
-
export { assertGetOptions, servedRange } from './range';
|
|
198
|
+
export { assertGetOptions, assertStreamRange, servedRange, streamedRange } from './range';
|
|
154
199
|
export type {
|
|
200
|
+
SignedPart,
|
|
155
201
|
SignedUrlConstraints,
|
|
156
202
|
SignedUrlFailure,
|
|
157
203
|
SignedUrlInput,
|
|
@@ -178,15 +224,31 @@ export {
|
|
|
178
224
|
} from './signing-secret';
|
|
179
225
|
export type { Storage, StorageConfig } from './storage';
|
|
180
226
|
export { definedStorage, defineStorage, disk, resetStorage, storage } from './storage';
|
|
181
|
-
export type {
|
|
227
|
+
export type {
|
|
228
|
+
StorageSubjectRef,
|
|
229
|
+
StorageSubjectTarget,
|
|
230
|
+
StorageSubjectTargetOptions,
|
|
231
|
+
} from './subject-target';
|
|
232
|
+
export { STORAGE_SUBJECT_COLUMNS, storageSubjectTarget } from './subject-target';
|
|
233
|
+
export type {
|
|
234
|
+
StoredHeadCandidate,
|
|
235
|
+
UploadCandidate,
|
|
236
|
+
UploadPolicy,
|
|
237
|
+
UploadPolicyInit,
|
|
238
|
+
ValidatedHead,
|
|
239
|
+
ValidatedUpload,
|
|
240
|
+
} from './upload';
|
|
182
241
|
export {
|
|
183
242
|
contentTypeMatches,
|
|
184
243
|
DEFAULT_MAX_UPLOAD_BYTES,
|
|
185
244
|
DOCUMENT_CONTENT_TYPES,
|
|
186
245
|
IMAGE_CONTENT_TYPES,
|
|
246
|
+
MAX_LEADING_WHITESPACE,
|
|
187
247
|
normalizeContentType,
|
|
248
|
+
SNIFF_WINDOW_BYTES,
|
|
188
249
|
sniffContentType,
|
|
189
250
|
uploadPolicy,
|
|
251
|
+
validateStoredHead,
|
|
190
252
|
validateUpload,
|
|
191
253
|
} from './upload';
|
|
192
254
|
export type {
|
|
@@ -203,3 +265,13 @@ export {
|
|
|
203
265
|
uploadFile,
|
|
204
266
|
xhrSignedPut,
|
|
205
267
|
} from './upload-client';
|
|
268
|
+
export type {
|
|
269
|
+
MultipartUploadClient,
|
|
270
|
+
MultipartUploadInput,
|
|
271
|
+
UploadSessionRef,
|
|
272
|
+
} from './upload-client-multipart';
|
|
273
|
+
export {
|
|
274
|
+
DEFAULT_PART_THRESHOLD_BYTES,
|
|
275
|
+
PART_CONCURRENCY,
|
|
276
|
+
uploadInParts,
|
|
277
|
+
} from './upload-client-multipart';
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
// Single responsibility: an upload in parts on a disk that signs its own URLs (`local`, `memory`):
|
|
2
|
+
// the session rules over a `PartStore` the disk supplies. Same contract as s3's, so dev and tests
|
|
3
|
+
// behave as production: a part URL is signed, a part lands only in an open session, a Complete
|
|
4
|
+
// assembles exactly the parts named and drops the session, and an abort leaves nothing.
|
|
5
|
+
|
|
6
|
+
import type { Clock } from '@ultimat3/core';
|
|
7
|
+
import { etagOf, type StorageDriver, type StorageObject } from './driver';
|
|
8
|
+
import { uploadUnknown } from './errors-multipart';
|
|
9
|
+
import type { MultipartCapability, MultipartSession, UploadedPart } from './multipart';
|
|
10
|
+
import { assertSafeKey } from './path';
|
|
11
|
+
|
|
12
|
+
/** What a staged session records at `create`. `initiated` is ISO, so a file store can hold it. */
|
|
13
|
+
export interface StagedSession {
|
|
14
|
+
readonly key: string;
|
|
15
|
+
readonly contentType: string;
|
|
16
|
+
readonly initiated: string;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/** Where a disk keeps open sessions and their parts: a directory, or a `Map`. */
|
|
20
|
+
export interface PartStore {
|
|
21
|
+
open(uploadId: string, session: StagedSession): Promise<void>;
|
|
22
|
+
session(uploadId: string): Promise<StagedSession | undefined>;
|
|
23
|
+
writePart(uploadId: string, part: UploadedPart, bytes: Uint8Array): Promise<void>;
|
|
24
|
+
/** Ascending by part number. */
|
|
25
|
+
parts(uploadId: string): Promise<readonly UploadedPart[]>;
|
|
26
|
+
/** The named parts' bytes, end to end, read as the stream is pulled. */
|
|
27
|
+
read(uploadId: string, parts: readonly UploadedPart[]): ReadableStream<Uint8Array>;
|
|
28
|
+
drop(uploadId: string): Promise<void>;
|
|
29
|
+
sessions(): Promise<readonly (readonly [string, StagedSession])[]>;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export interface StagedMultipartOptions {
|
|
33
|
+
readonly store: PartStore;
|
|
34
|
+
readonly clock: Clock;
|
|
35
|
+
/** The disk the parts belong to: it signs their URLs and takes the Complete's stream. */
|
|
36
|
+
readonly driver: StorageDriver;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* The disk with `multipart` attached IN PLACE, never spread: a spread reads `signedUrlBase` once,
|
|
41
|
+
* and `registerAs` re-points it after construction.
|
|
42
|
+
*/
|
|
43
|
+
export function withStagedMultipart<D extends StorageDriver>(
|
|
44
|
+
driver: D,
|
|
45
|
+
store: PartStore,
|
|
46
|
+
clock: Clock,
|
|
47
|
+
): D {
|
|
48
|
+
return Object.assign(driver, { multipart: stagedMultipart({ store, clock, driver }) });
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export function stagedMultipart(options: StagedMultipartOptions): MultipartCapability {
|
|
52
|
+
const { store } = options;
|
|
53
|
+
/** The session, held to the key it was opened for: an id is never a capability on its own. */
|
|
54
|
+
const openSession = async (key: string, uploadId: string): Promise<StagedSession> => {
|
|
55
|
+
const session = await store.session(uploadId);
|
|
56
|
+
if (session === undefined || session.key !== key) {
|
|
57
|
+
throw uploadUnknown(options.driver.name, key, uploadId);
|
|
58
|
+
}
|
|
59
|
+
return session;
|
|
60
|
+
};
|
|
61
|
+
|
|
62
|
+
return {
|
|
63
|
+
async create(key, { contentType }): Promise<string> {
|
|
64
|
+
const safe = assertSafeKey(key);
|
|
65
|
+
const uploadId = crypto.randomUUID();
|
|
66
|
+
await store.open(uploadId, {
|
|
67
|
+
key: safe,
|
|
68
|
+
contentType,
|
|
69
|
+
initiated: options.clock.now().toISOString(),
|
|
70
|
+
});
|
|
71
|
+
return uploadId;
|
|
72
|
+
},
|
|
73
|
+
|
|
74
|
+
signPart: (key, uploadId, partNumber, { contentLength, expiresInMs }) =>
|
|
75
|
+
options.driver.signedUrl(key, {
|
|
76
|
+
method: 'PUT',
|
|
77
|
+
contentLength,
|
|
78
|
+
maxBytes: contentLength,
|
|
79
|
+
expiresInMs,
|
|
80
|
+
part: { uploadId, partNumber },
|
|
81
|
+
}),
|
|
82
|
+
|
|
83
|
+
async listParts(key, uploadId): Promise<readonly UploadedPart[]> {
|
|
84
|
+
await openSession(assertSafeKey(key), uploadId);
|
|
85
|
+
return store.parts(uploadId);
|
|
86
|
+
},
|
|
87
|
+
|
|
88
|
+
async receivePart(key, uploadId, partNumber, bytes): Promise<UploadedPart> {
|
|
89
|
+
await openSession(assertSafeKey(key), uploadId);
|
|
90
|
+
const part = { partNumber, etag: etagOf(bytes), size: bytes.byteLength };
|
|
91
|
+
await store.writePart(uploadId, part, bytes);
|
|
92
|
+
return part;
|
|
93
|
+
},
|
|
94
|
+
|
|
95
|
+
async complete(key, uploadId, parts): Promise<StorageObject> {
|
|
96
|
+
const safe = assertSafeKey(key);
|
|
97
|
+
const session = await openSession(safe, uploadId);
|
|
98
|
+
const total = parts.reduce((sum, part) => sum + part.size, 0);
|
|
99
|
+
const object = await options.driver.putStream(safe, store.read(uploadId, parts), {
|
|
100
|
+
maxBytes: Math.max(total, 1),
|
|
101
|
+
contentType: session.contentType,
|
|
102
|
+
});
|
|
103
|
+
await store.drop(uploadId);
|
|
104
|
+
return object;
|
|
105
|
+
},
|
|
106
|
+
|
|
107
|
+
async abort(key, uploadId): Promise<void> {
|
|
108
|
+
const safe = assertSafeKey(key);
|
|
109
|
+
const session = await store.session(uploadId);
|
|
110
|
+
if (session?.key === safe) await store.drop(uploadId);
|
|
111
|
+
},
|
|
112
|
+
|
|
113
|
+
async listUploads(prefix): Promise<readonly MultipartSession[]> {
|
|
114
|
+
const open = await store.sessions();
|
|
115
|
+
return open
|
|
116
|
+
.filter(([, session]) => prefix === undefined || session.key.startsWith(prefix))
|
|
117
|
+
.map(([uploadId, session]) => {
|
|
118
|
+
const initiated = new Date(session.initiated);
|
|
119
|
+
return {
|
|
120
|
+
key: session.key,
|
|
121
|
+
uploadId,
|
|
122
|
+
...(Number.isNaN(initiated.getTime()) ? {} : { initiated }),
|
|
123
|
+
};
|
|
124
|
+
});
|
|
125
|
+
},
|
|
126
|
+
};
|
|
127
|
+
}
|
package/src/multipart.ts
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
// Single responsibility: the part arithmetic of an S3 multipart upload: the protocol's two limits
|
|
2
|
+
// and the part size that follows from a ceiling, plus the optional capability a disk declares for
|
|
3
|
+
// an upload in parts. No wire and no disk, so any caller may ask.
|
|
4
|
+
|
|
5
|
+
import type { StorageObject } from './driver';
|
|
6
|
+
|
|
7
|
+
/** S3's floor on every part but the last. */
|
|
8
|
+
export const MIN_PART_BYTES = 5 * 1024 * 1024;
|
|
9
|
+
/** S3's count of parts in one upload. */
|
|
10
|
+
export const MAX_PARTS = 10_000;
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* How much is held before a part is sent. The smallest size whose 10,000 parts still cover
|
|
14
|
+
* `maxBytes`, since what is held is what this process pays for the stream.
|
|
15
|
+
*/
|
|
16
|
+
export const partBytesFor = (maxBytes: number): number =>
|
|
17
|
+
Math.max(MIN_PART_BYTES, Math.ceil(maxBytes / MAX_PARTS));
|
|
18
|
+
|
|
19
|
+
/** S3's ceiling on one part, and so on one part URL's signed length. */
|
|
20
|
+
export const MAX_PART_BYTES = 5 * 1024 * 1024 * 1024;
|
|
21
|
+
|
|
22
|
+
/** One part a disk holds for an open upload: what `listParts` answers, and what a resume reads. */
|
|
23
|
+
export interface UploadedPart {
|
|
24
|
+
readonly partNumber: number;
|
|
25
|
+
/** The disk's own etag of the part. S3's is quoted MD5; the local disks' is `etagOf`. */
|
|
26
|
+
readonly etag: string;
|
|
27
|
+
readonly size: number;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** A part the caller asks `completeUpload` to assemble. The etag, when given, must be the stored one. */
|
|
31
|
+
export interface CompletedPart {
|
|
32
|
+
readonly partNumber: number;
|
|
33
|
+
readonly etag?: string | undefined;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** One upload opened and neither completed nor aborted: what `abortStaleUploads` weighs. */
|
|
37
|
+
export interface MultipartSession {
|
|
38
|
+
readonly key: string;
|
|
39
|
+
readonly uploadId: string;
|
|
40
|
+
/** ABSENT when the disk did not say, as `StorageListEntry.lastModified` is — never epoch 0. */
|
|
41
|
+
readonly initiated?: Date | undefined;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export interface SignPartOptions {
|
|
45
|
+
/** The part's exact size, signed: the disk stores it at no other length. */
|
|
46
|
+
readonly contentLength: number;
|
|
47
|
+
readonly expiresInMs: number;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* An upload in parts, on one disk. Optional on `StorageDriver`, so a third-party driver still
|
|
52
|
+
* compiles; `grantMultipart` refuses a disk without it (`X_STORAGE_MULTIPART_UNSUPPORTED`). Every
|
|
53
|
+
* method takes a key already held to the tenant by the caller in `grant-multipart.ts`.
|
|
54
|
+
*/
|
|
55
|
+
export interface MultipartCapability {
|
|
56
|
+
/** Opens a session and answers its id. The object's content type travels here: a part has none. */
|
|
57
|
+
create(key: string, options: { readonly contentType: string }): Promise<string>;
|
|
58
|
+
/** A URL a browser PUTs one part to. Signs nothing about whether the upload is still open. */
|
|
59
|
+
signPart(
|
|
60
|
+
key: string,
|
|
61
|
+
uploadId: string,
|
|
62
|
+
partNumber: number,
|
|
63
|
+
options: SignPartOptions,
|
|
64
|
+
): Promise<string>;
|
|
65
|
+
/** Ascending by part number. `X_STORAGE_UPLOAD_UNKNOWN` for a session that is not open. */
|
|
66
|
+
listParts(key: string, uploadId: string): Promise<readonly UploadedPart[]>;
|
|
67
|
+
/** Assembles `parts`, already checked against `listParts`; the object exists once it returns. */
|
|
68
|
+
complete(key: string, uploadId: string, parts: readonly UploadedPart[]): Promise<StorageObject>;
|
|
69
|
+
/** Drops the session and every part. An unknown session is already in the asked-for state. */
|
|
70
|
+
abort(key: string, uploadId: string): Promise<void>;
|
|
71
|
+
/** Every open session under `prefix`, so a sweep can reclaim what nobody finished. */
|
|
72
|
+
listUploads(prefix?: string): Promise<readonly MultipartSession[]>;
|
|
73
|
+
/**
|
|
74
|
+
* Stores one part's bytes — only on a disk that signs its own part URLs (`local`, `memory`),
|
|
75
|
+
* called by the `/_storage` PUT once the URL verified. On s3 the bytes go to the bucket.
|
|
76
|
+
*/
|
|
77
|
+
receivePart?(
|
|
78
|
+
key: string,
|
|
79
|
+
uploadId: string,
|
|
80
|
+
partNumber: number,
|
|
81
|
+
bytes: Uint8Array,
|
|
82
|
+
): Promise<UploadedPart>;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* The parts asked for, held to the parts stored: each asked part is stored, in ascending order,
|
|
87
|
+
* under the stored etag when one is named, and none but the last is under S3's part floor.
|
|
88
|
+
* Answers the stored parts in the asked order, which is what `complete` assembles; `problem` names
|
|
89
|
+
* the first reason it cannot.
|
|
90
|
+
*/
|
|
91
|
+
export function matchParts(
|
|
92
|
+
asked: readonly CompletedPart[],
|
|
93
|
+
stored: readonly UploadedPart[],
|
|
94
|
+
): { readonly parts: readonly UploadedPart[] } | { readonly problem: string } {
|
|
95
|
+
if (asked.length === 0) return { problem: 'no parts were named' };
|
|
96
|
+
const byNumber = new Map(stored.map((part) => [part.partNumber, part]));
|
|
97
|
+
const parts: UploadedPart[] = [];
|
|
98
|
+
let previous = 0;
|
|
99
|
+
for (const { partNumber, etag } of asked) {
|
|
100
|
+
if (!Number.isSafeInteger(partNumber) || partNumber <= previous) {
|
|
101
|
+
return { problem: `part ${String(partNumber)} is not in ascending order after ${previous}` };
|
|
102
|
+
}
|
|
103
|
+
const held = byNumber.get(partNumber);
|
|
104
|
+
if (held === undefined) return { problem: `part ${partNumber} never arrived` };
|
|
105
|
+
if (etag !== undefined && etag !== held.etag) {
|
|
106
|
+
return { problem: `part ${partNumber}'s etag is not the one the disk stored` };
|
|
107
|
+
}
|
|
108
|
+
parts.push(held);
|
|
109
|
+
previous = partNumber;
|
|
110
|
+
}
|
|
111
|
+
// S3's EntityTooSmall, asked on every disk so the local ones cannot complete what s3 refuses.
|
|
112
|
+
const short = parts.slice(0, -1).find((part) => part.size < MIN_PART_BYTES);
|
|
113
|
+
if (short !== undefined) {
|
|
114
|
+
return {
|
|
115
|
+
problem: `part ${short.partNumber} is ${short.size}B, under the ${MIN_PART_BYTES}B every part but the last must reach`,
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
return { parts };
|
|
119
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
// Single responsibility: what `putStream` may be handed, and the limit it pumps under. The pump itself
|
|
2
|
+
// is `byte-pump.ts`, which `put()` reads through too.
|
|
3
|
+
|
|
4
|
+
import { assert, describeValue, finiteCount } from '@ultimat3/core';
|
|
5
|
+
import type { PumpLimit } from './byte-pump';
|
|
6
|
+
import { assertPutOptions } from './driver';
|
|
7
|
+
import { streamTooLarge } from './errors-stream';
|
|
8
|
+
|
|
9
|
+
export interface PutStreamOptions {
|
|
10
|
+
/**
|
|
11
|
+
* Required: a stream's length is the sender's until it is counted. Past it the write is abandoned
|
|
12
|
+
* (`X_STORAGE_TOO_LARGE`) and the key is as it was. Replaces the disk's `maxPutBytes`, which bounds
|
|
13
|
+
* what `put()` buffers, and nothing here is buffered past one s3 part.
|
|
14
|
+
*/
|
|
15
|
+
readonly maxBytes: number;
|
|
16
|
+
readonly contentType?: string | undefined;
|
|
17
|
+
readonly cacheControl?: string | undefined;
|
|
18
|
+
readonly metadata?: Readonly<Record<string, string>> | undefined;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/** `put()` options that need the bytes before the object exists. The type omits them too. */
|
|
22
|
+
const NEEDS_THE_BYTES = ['checksum', 'retention', 'legalHold', 'serverSideEncryption'] as const;
|
|
23
|
+
|
|
24
|
+
/** Everything judged before a chunk is read. Answers the screened `maxBytes`. */
|
|
25
|
+
export function assertPutStream(
|
|
26
|
+
driver: string,
|
|
27
|
+
body: ReadableStream<Uint8Array>,
|
|
28
|
+
options: PutStreamOptions,
|
|
29
|
+
): number {
|
|
30
|
+
assert(
|
|
31
|
+
body instanceof ReadableStream,
|
|
32
|
+
`putStream() on the ${driver} disk was given a body that is not a ReadableStream (${describeValue(body)})`,
|
|
33
|
+
'await disk.putStream(key, request.body, { maxBytes }) # a Uint8Array or a Blob goes through disk.put(key, bytes)',
|
|
34
|
+
);
|
|
35
|
+
const asked = (options ?? {}) as unknown as Readonly<Record<string, unknown>>;
|
|
36
|
+
for (const name of NEEDS_THE_BYTES) {
|
|
37
|
+
assert(
|
|
38
|
+
asked[name] === undefined,
|
|
39
|
+
`putStream() on the ${driver} disk was given ${name}, which needs the whole body before the object exists`,
|
|
40
|
+
`await disk.put(key, bytes, { ${name} }) # buffered, so bounded by the disk's maxPutBytes`,
|
|
41
|
+
);
|
|
42
|
+
}
|
|
43
|
+
assertPutOptions(driver, options);
|
|
44
|
+
return finiteCount(`putStream() on the ${driver} disk`, 'maxBytes', options?.maxBytes, 1);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** The limit `putStream` pumps under: its own `maxBytes`, and the refusal that names it. */
|
|
48
|
+
export const streamLimit = (driver: string, key: string, maxBytes: number): PumpLimit => ({
|
|
49
|
+
driver,
|
|
50
|
+
maxBytes,
|
|
51
|
+
refuse: (total) => streamTooLarge(driver, key, total, maxBytes),
|
|
52
|
+
});
|
package/src/range.ts
CHANGED
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
|
|
6
6
|
import { assert } from '@ultimat3/core';
|
|
7
7
|
import { rangeTooLarge, rangeUnsatisfiable } from './errors-range';
|
|
8
|
+
import { streamRangeUnsatisfiable } from './errors-stream';
|
|
8
9
|
|
|
9
10
|
/**
|
|
10
11
|
* The first and the last byte, BOTH INCLUSIVE — HTTP's `Range: bytes=first-last` and S3's, so a
|
|
@@ -39,19 +40,49 @@ const isOffset = (value: unknown): value is number =>
|
|
|
39
40
|
export function assertGetOptions(driver: string, options: GetOptions | undefined): void {
|
|
40
41
|
const range = options?.range;
|
|
41
42
|
if (range === undefined) return;
|
|
43
|
+
assertRange(driver, 'get()', 'get(key, { range: ', ' })', range);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** The same screen for `streamRange`, naming the call that was made. */
|
|
47
|
+
export function assertStreamRange(driver: string, range: ByteRange): void {
|
|
48
|
+
assertRange(driver, 'streamRange()', 'streamRange(key, ', ')', range);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function assertRange(
|
|
52
|
+
driver: string,
|
|
53
|
+
call: string,
|
|
54
|
+
open: string,
|
|
55
|
+
close: string,
|
|
56
|
+
range: ByteRange,
|
|
57
|
+
): void {
|
|
42
58
|
const [first, last] = Array.isArray(range) ? range : [];
|
|
43
59
|
assert(
|
|
44
60
|
Array.isArray(range) && range.length === 2 && isOffset(first) && isOffset(last),
|
|
45
|
-
|
|
46
|
-
|
|
61
|
+
`${call} on the ${driver} disk was given a range that is not two whole, non-negative byte offsets`,
|
|
62
|
+
`${open}[first, last]${close} — inclusive offsets: [0, 99] reads the first 100 bytes`,
|
|
47
63
|
);
|
|
48
64
|
assert(
|
|
49
65
|
first <= last,
|
|
50
|
-
|
|
51
|
-
|
|
66
|
+
`${call} on the ${driver} disk was given the range [${first}, ${last}], whose first byte is past its last`,
|
|
67
|
+
`${open}[${last}, ${first}]${close} — first before last, both inclusive`,
|
|
52
68
|
);
|
|
53
69
|
}
|
|
54
70
|
|
|
71
|
+
/**
|
|
72
|
+
* The window `streamRange` serves: clamped as `servedRange` clamps. No ceiling, because nothing is
|
|
73
|
+
* buffered: the bytes asked for are the bytes pulled, and they flow at the reader's pace.
|
|
74
|
+
*/
|
|
75
|
+
export function streamedRange(
|
|
76
|
+
disk: string,
|
|
77
|
+
key: string,
|
|
78
|
+
range: ByteRange,
|
|
79
|
+
size: number,
|
|
80
|
+
): ByteRange {
|
|
81
|
+
const [first, wanted] = range;
|
|
82
|
+
if (first >= size) throw streamRangeUnsatisfiable(disk, key, range, size);
|
|
83
|
+
return [first, Math.min(wanted, size - 1)];
|
|
84
|
+
}
|
|
85
|
+
|
|
55
86
|
/**
|
|
56
87
|
* The range a disk SERVES for an object of `size` bytes — clamped, refused, or over the ceiling.
|
|
57
88
|
* Called with the object's own measured size, before a byte is read, by every driver.
|
package/src/signed-url.ts
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
// a 2GB executable. Verification never throws and never short-circuits on expiry before the
|
|
5
5
|
// signature, so a forged URL can never learn "the signature was fine, just late".
|
|
6
6
|
|
|
7
|
-
import { type Clock, finiteCount, systemClock, timingSafeEqual } from '@ultimat3/core';
|
|
7
|
+
import { assert, type Clock, finiteCount, systemClock, timingSafeEqual } from '@ultimat3/core';
|
|
8
8
|
import type { SignedUrlMethod } from './driver';
|
|
9
9
|
import { assertSafeKey, isSafeKey } from './path';
|
|
10
10
|
|
|
@@ -31,8 +31,20 @@ export const SIGNED_URL_PARAMS = {
|
|
|
31
31
|
maxBytes: 'x-max',
|
|
32
32
|
contentType: 'x-ct',
|
|
33
33
|
signature: 'x-sig',
|
|
34
|
+
uploadId: 'x-upload',
|
|
35
|
+
partNumber: 'x-part',
|
|
34
36
|
} as const;
|
|
35
37
|
|
|
38
|
+
/** The session and part a part URL writes into — inside the signature, so neither can be edited. */
|
|
39
|
+
export interface SignedPart {
|
|
40
|
+
readonly uploadId: string;
|
|
41
|
+
readonly partNumber: number;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** `MAX_PARTS` restated as the parse's bound: `multipart.ts` imports this module's types. */
|
|
45
|
+
const MAX_SIGNED_PART = 10_000;
|
|
46
|
+
const SIGNED_UPLOAD_ID = /^[A-Za-z0-9._~-]{1,128}$/;
|
|
47
|
+
|
|
36
48
|
export interface SignedUrlConstraints {
|
|
37
49
|
readonly key: string;
|
|
38
50
|
readonly method: SignedUrlMethod;
|
|
@@ -40,6 +52,8 @@ export interface SignedUrlConstraints {
|
|
|
40
52
|
readonly expiresAt: number;
|
|
41
53
|
readonly maxBytes: number | undefined;
|
|
42
54
|
readonly contentType: string | undefined;
|
|
55
|
+
/** Present on a part URL only: a single PUT and a part are two capabilities, never one. */
|
|
56
|
+
readonly part?: SignedPart | undefined;
|
|
43
57
|
}
|
|
44
58
|
|
|
45
59
|
export interface SignedUrlInput {
|
|
@@ -50,6 +64,8 @@ export interface SignedUrlInput {
|
|
|
50
64
|
readonly expiresInMs?: number | undefined;
|
|
51
65
|
readonly maxBytes?: number | undefined;
|
|
52
66
|
readonly contentType?: string | undefined;
|
|
67
|
+
/** Mint a part URL of an open upload (`MultipartCapability.signPart` on the local disks). */
|
|
68
|
+
readonly part?: SignedPart | undefined;
|
|
53
69
|
readonly baseUrl?: string | undefined;
|
|
54
70
|
readonly clock?: Clock | undefined;
|
|
55
71
|
}
|
|
@@ -92,14 +108,19 @@ export function signedUrlBasePath(baseUrl: string): string {
|
|
|
92
108
|
* origin is deliberately outside it: the verifying route is handed a path.
|
|
93
109
|
*/
|
|
94
110
|
export function canonicalRequest(constraints: SignedUrlConstraints, basePath: string): string {
|
|
111
|
+
const part = constraints.part;
|
|
112
|
+
// A part signs under its own version line, so no single-PUT string can ever read as a part's:
|
|
113
|
+
// the two domains share no canonical form, and a single URL's bytes are what they always were.
|
|
114
|
+
const tail = part === undefined ? [] : [part.uploadId, String(part.partNumber)];
|
|
95
115
|
return [
|
|
96
|
-
SIGNED_URL_VERSION
|
|
116
|
+
part === undefined ? SIGNED_URL_VERSION : `${SIGNED_URL_VERSION}-part`,
|
|
97
117
|
basePath,
|
|
98
118
|
constraints.method,
|
|
99
119
|
constraints.key,
|
|
100
120
|
String(constraints.expiresAt),
|
|
101
121
|
constraints.maxBytes === undefined ? '' : String(constraints.maxBytes),
|
|
102
122
|
constraints.contentType ?? '',
|
|
123
|
+
...tail,
|
|
103
124
|
].join('\n');
|
|
104
125
|
}
|
|
105
126
|
|
|
@@ -163,6 +184,7 @@ export async function buildSignedUrl(input: SignedUrlInput): Promise<string> {
|
|
|
163
184
|
// one while `acceptSignedUpload`'s `unconstrained` refusal (which tests `undefined`) stayed
|
|
164
185
|
// silent. One spelling for "no content type", here and at the parse below.
|
|
165
186
|
contentType: input.contentType === '' ? undefined : input.contentType,
|
|
187
|
+
...(input.part === undefined ? {} : { part: assertSignedPart(input.part) }),
|
|
166
188
|
};
|
|
167
189
|
const params = new URLSearchParams();
|
|
168
190
|
params.set(SIGNED_URL_PARAMS.method, constraints.method);
|
|
@@ -173,6 +195,10 @@ export async function buildSignedUrl(input: SignedUrlInput): Promise<string> {
|
|
|
173
195
|
if (constraints.contentType !== undefined) {
|
|
174
196
|
params.set(SIGNED_URL_PARAMS.contentType, constraints.contentType);
|
|
175
197
|
}
|
|
198
|
+
if (constraints.part !== undefined) {
|
|
199
|
+
params.set(SIGNED_URL_PARAMS.uploadId, constraints.part.uploadId);
|
|
200
|
+
params.set(SIGNED_URL_PARAMS.partNumber, String(constraints.part.partNumber));
|
|
201
|
+
}
|
|
176
202
|
const base = trimBase(input.baseUrl ?? DEFAULT_SIGNED_URL_BASE);
|
|
177
203
|
params.set(
|
|
178
204
|
SIGNED_URL_PARAMS.signature,
|
|
@@ -232,15 +258,46 @@ function parseConstraints(url: URL, base: string): SignedUrlConstraints | Signed
|
|
|
232
258
|
// An EMPTY `x-ct` is malformed, never "no content type": absent and empty share one canonical
|
|
233
259
|
// string, so accepting it let `&x-ct=` be appended to a URL signed with none and still verify.
|
|
234
260
|
if (rawContentType === '') return 'malformed';
|
|
261
|
+
const part = parsePart(url);
|
|
262
|
+
if (part === 'malformed') return part;
|
|
235
263
|
return {
|
|
236
264
|
key,
|
|
237
265
|
method,
|
|
238
266
|
expiresAt,
|
|
239
267
|
maxBytes,
|
|
240
268
|
contentType: rawContentType ?? undefined,
|
|
269
|
+
...(part === undefined ? {} : { part }),
|
|
241
270
|
};
|
|
242
271
|
}
|
|
243
272
|
|
|
273
|
+
const isSignedPart = (part: SignedPart): boolean =>
|
|
274
|
+
SIGNED_UPLOAD_ID.test(part.uploadId) &&
|
|
275
|
+
Number.isSafeInteger(part.partNumber) &&
|
|
276
|
+
part.partNumber >= 1 &&
|
|
277
|
+
part.partNumber <= MAX_SIGNED_PART;
|
|
278
|
+
|
|
279
|
+
/** The mint's screen: a part this module could not parse back is a URL dead on arrival. */
|
|
280
|
+
function assertSignedPart(part: SignedPart): SignedPart {
|
|
281
|
+
assert(
|
|
282
|
+
isSignedPart(part),
|
|
283
|
+
`buildSignedUrl was handed a part that is not an upload id and a part number 1..${MAX_SIGNED_PART}`,
|
|
284
|
+
"buildSignedUrl({ secret, key, method: 'PUT', part: { uploadId: crypto.randomUUID(), partNumber: 1 } })",
|
|
285
|
+
);
|
|
286
|
+
return { uploadId: part.uploadId, partNumber: part.partNumber };
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
/** Both parameters or neither; one alone is a URL this module never minted. */
|
|
290
|
+
function parsePart(url: URL): SignedPart | 'malformed' | undefined {
|
|
291
|
+
const uploadId = url.searchParams.get(SIGNED_URL_PARAMS.uploadId);
|
|
292
|
+
const rawNumber = url.searchParams.get(SIGNED_URL_PARAMS.partNumber);
|
|
293
|
+
if (uploadId === null && rawNumber === null) return undefined;
|
|
294
|
+
if (uploadId === null || rawNumber === null || !/^[1-9][0-9]{0,4}$/.test(rawNumber)) {
|
|
295
|
+
return 'malformed';
|
|
296
|
+
}
|
|
297
|
+
const part = { uploadId, partNumber: Number(rawNumber) };
|
|
298
|
+
return isSignedPart(part) ? part : 'malformed';
|
|
299
|
+
}
|
|
300
|
+
|
|
244
301
|
export async function verifySignedUrl(input: VerifySignedUrlInput): Promise<SignedUrlVerification> {
|
|
245
302
|
let url: URL;
|
|
246
303
|
try {
|