@ultimat3/storage 30.9.0 → 31.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +56 -113
- package/README.md +117 -24
- package/package.json +4 -3
- package/src/accept-stream.ts +125 -0
- package/src/accept.ts +24 -6
- package/src/adapter.ts +46 -0
- package/src/attachment.ts +43 -61
- package/src/driver-local-list.ts +1 -1
- package/src/driver-local-open.ts +112 -0
- package/src/driver-local-read.ts +1 -1
- package/src/driver-local-stream.ts +14 -3
- package/src/driver-local.ts +48 -16
- package/src/driver-memory.ts +39 -9
- package/src/driver-s3-copy-parts.ts +177 -0
- package/src/driver-s3-copy.ts +14 -7
- package/src/driver-s3-lock.ts +1 -3
- package/src/driver-s3-put-stream.ts +81 -17
- package/src/driver-s3-range.ts +14 -7
- package/src/driver-s3-signed.ts +1 -1
- package/src/driver-s3-stream.ts +36 -0
- package/src/driver-s3.ts +44 -27
- package/src/driver.ts +30 -5
- package/src/errors-promote.ts +3 -15
- package/src/errors-scan.ts +61 -0
- package/src/errors-stream.ts +13 -0
- package/src/errors-variant.ts +40 -0
- package/src/errors.ts +6 -0
- package/src/grant.ts +15 -10
- package/src/image-variant.ts +106 -0
- package/src/index.ts +21 -40
- package/src/path.ts +27 -1
- package/src/request-ceiling.ts +45 -0
- package/src/storage-instrument.ts +108 -0
- package/src/storage.ts +8 -2
- package/src/upload-scan.ts +91 -0
- package/src/upload-scanner-memory.ts +62 -0
- package/src/upload-scanner.ts +71 -0
- package/src/upload.ts +35 -2
package/src/driver-local.ts
CHANGED
|
@@ -25,12 +25,14 @@ import {
|
|
|
25
25
|
type StorageListEntry,
|
|
26
26
|
type StorageObject,
|
|
27
27
|
type StorageRead,
|
|
28
|
+
type StreamOptions,
|
|
28
29
|
sha256Base64,
|
|
29
30
|
signedByteBound,
|
|
30
31
|
toBytes,
|
|
31
32
|
} from './driver';
|
|
32
33
|
import { listLocalPage } from './driver-local-list';
|
|
33
34
|
import { localPartStore } from './driver-local-multipart';
|
|
35
|
+
import { streamPinned } from './driver-local-open';
|
|
34
36
|
import {
|
|
35
37
|
etagOfFile,
|
|
36
38
|
headObject,
|
|
@@ -60,7 +62,8 @@ import {
|
|
|
60
62
|
ObjectLockedError,
|
|
61
63
|
requestedLock,
|
|
62
64
|
} from './object-lock';
|
|
63
|
-
import { assertSafeKey } from './path';
|
|
65
|
+
import { assertLocalKey, assertSafeKey } from './path';
|
|
66
|
+
|
|
64
67
|
import { assertPutStream, type PutStreamOptions } from './put-stream';
|
|
65
68
|
import {
|
|
66
69
|
assertGetOptions,
|
|
@@ -70,6 +73,11 @@ import {
|
|
|
70
73
|
servedRange,
|
|
71
74
|
streamedRange,
|
|
72
75
|
} from './range';
|
|
76
|
+
import {
|
|
77
|
+
assertWithinRequestCeiling,
|
|
78
|
+
type RequestCeiling,
|
|
79
|
+
screenRequestCeiling,
|
|
80
|
+
} from './request-ceiling';
|
|
73
81
|
import type { SignedUrlVerification } from './signed-url';
|
|
74
82
|
import { buildSignedUrl, signedUrlBaseFor, verifySignedUrl } from './signed-url';
|
|
75
83
|
import { resolveSigningSecret } from './signing-secret';
|
|
@@ -77,6 +85,9 @@ import { DEFAULT_MAX_UPLOAD_BYTES } from './upload';
|
|
|
77
85
|
|
|
78
86
|
const DRIVER_NAME = 'local';
|
|
79
87
|
|
|
88
|
+
/** Every key this disk touches: the shared rule, then Windows' own (`assertLocalKey`). */
|
|
89
|
+
const safeKey = (key: string): string => assertLocalKey(assertSafeKey(key));
|
|
90
|
+
|
|
80
91
|
export interface LocalDriverOptions {
|
|
81
92
|
/** Directory the disk owns outright. Created on first write. */
|
|
82
93
|
readonly root: string;
|
|
@@ -112,6 +123,14 @@ export interface LocalDriverOptions {
|
|
|
112
123
|
* is the uploader's, not this process's. Defaults to `maxPutBytes`; past it, `stream()`.
|
|
113
124
|
*/
|
|
114
125
|
readonly maxGetBytes?: number | undefined;
|
|
126
|
+
/**
|
|
127
|
+
* The most ONE request to this server carries — the app's `http.maxRequestBodyBytes`, which this
|
|
128
|
+
* package cannot import. A signed PUT lands on this process, so a single-request grant past it is
|
|
129
|
+
* refused at signing (`X_STORAGE_TOO_LARGE`, naming `grantMultipart`). Unset, nothing is refused
|
|
130
|
+
* here and the route's own `X_BODY_TOO_LARGE` answers the upload. A function is read at each
|
|
131
|
+
* signing: the CLI's disk is built before the app's `configureHttp` runs.
|
|
132
|
+
*/
|
|
133
|
+
readonly maxRequestBytes?: RequestCeiling | undefined;
|
|
115
134
|
/**
|
|
116
135
|
* The per-key queue every read and write of this disk waits in. Defaults to the disk's own
|
|
117
136
|
* `keyedQueue()`; a test passes one it also holds, to park a key deterministically.
|
|
@@ -136,6 +155,7 @@ export function localDriver(options: LocalDriverOptions): StorageDriver {
|
|
|
136
155
|
1,
|
|
137
156
|
);
|
|
138
157
|
const clock = options.clock ?? systemClock;
|
|
158
|
+
const maxRequestBytes = screenRequestCeiling('local', options.maxRequestBytes);
|
|
139
159
|
const oneAtATime = options.queue ?? keyedQueue();
|
|
140
160
|
// What `defineStorage` registered this driver as — the name a refusal's `disk('…')` must use.
|
|
141
161
|
let registered = DRIVER_NAME;
|
|
@@ -189,7 +209,7 @@ export function localDriver(options: LocalDriverOptions): StorageDriver {
|
|
|
189
209
|
},
|
|
190
210
|
|
|
191
211
|
async put(key: string, body: StorageBody, putOptions?: PutOptions): Promise<StorageObject> {
|
|
192
|
-
const safe =
|
|
212
|
+
const safe = safeKey(key);
|
|
193
213
|
refuseUnsupportedPut(putOptions);
|
|
194
214
|
assertPutOptions(DRIVER_NAME, putOptions);
|
|
195
215
|
assertObjectLockOptions(DRIVER_NAME, putOptions, clock);
|
|
@@ -233,7 +253,7 @@ export function localDriver(options: LocalDriverOptions): StorageDriver {
|
|
|
233
253
|
body: ReadableStream<Uint8Array>,
|
|
234
254
|
streamOptions: PutStreamOptions,
|
|
235
255
|
): Promise<StorageObject> {
|
|
236
|
-
const safe =
|
|
256
|
+
const safe = safeKey(key);
|
|
237
257
|
const maxBytes = assertPutStream(DRIVER_NAME, body, streamOptions);
|
|
238
258
|
const input = { root, key: safe, disk: registered, body, maxBytes, clock };
|
|
239
259
|
return putLocalStream(input, streamOptions, (then) =>
|
|
@@ -245,12 +265,12 @@ export function localDriver(options: LocalDriverOptions): StorageDriver {
|
|
|
245
265
|
},
|
|
246
266
|
|
|
247
267
|
async stat(key: string): Promise<StorageObject | undefined> {
|
|
248
|
-
const entry = await measured(
|
|
268
|
+
const entry = await measured(safeKey(key));
|
|
249
269
|
return entry && { ...entry, contentType: entry.contentType ?? DEFAULT_CONTENT_TYPE };
|
|
250
270
|
},
|
|
251
271
|
|
|
252
272
|
async get(key: string, getOptions?: GetOptions): Promise<StorageRead> {
|
|
253
|
-
const safe =
|
|
273
|
+
const safe = safeKey(key);
|
|
254
274
|
assertGetOptions(DRIVER_NAME, getOptions);
|
|
255
275
|
const range = getOptions?.range;
|
|
256
276
|
// Queued behind this key's writers: an object is two files, and a read between a put()'s
|
|
@@ -281,16 +301,26 @@ export function localDriver(options: LocalDriverOptions): StorageDriver {
|
|
|
281
301
|
});
|
|
282
302
|
},
|
|
283
303
|
|
|
284
|
-
async stream(key: string): Promise<ReadableStream<Uint8Array>> {
|
|
285
|
-
const safe =
|
|
304
|
+
async stream(key: string, options?: StreamOptions): Promise<ReadableStream<Uint8Array>> {
|
|
305
|
+
const safe = safeKey(key);
|
|
306
|
+
const etag = options?.etag;
|
|
307
|
+
if (etag !== undefined) return streamPinned({ root, key: safe, disk: registered, etag });
|
|
286
308
|
const file = Bun.file(filePath(safe));
|
|
287
309
|
if (!(await file.exists())) throw objectNotFound(registered, safe);
|
|
288
310
|
return file.stream();
|
|
289
311
|
},
|
|
290
312
|
|
|
291
|
-
async streamRange(
|
|
292
|
-
|
|
313
|
+
async streamRange(
|
|
314
|
+
key: string,
|
|
315
|
+
range: ByteRange,
|
|
316
|
+
options?: StreamOptions,
|
|
317
|
+
): Promise<ReadableStream<Uint8Array>> {
|
|
318
|
+
const safe = safeKey(key);
|
|
293
319
|
assertStreamRange(DRIVER_NAME, range);
|
|
320
|
+
const etag = options?.etag;
|
|
321
|
+
if (etag !== undefined) {
|
|
322
|
+
return streamPinned({ root, key: safe, disk: registered, etag, range });
|
|
323
|
+
}
|
|
294
324
|
const file = Bun.file(filePath(safe));
|
|
295
325
|
if (!(await file.exists())) throw objectNotFound(registered, safe);
|
|
296
326
|
const [first, last] = streamedRange(DRIVER_NAME, safe, range, file.size);
|
|
@@ -299,8 +329,8 @@ export function localDriver(options: LocalDriverOptions): StorageDriver {
|
|
|
299
329
|
|
|
300
330
|
/** A real file copy — `Bun.write` from a `BunFile` never routes the bytes through the heap. */
|
|
301
331
|
async copy(from: string, to: string, options?: CopyOptions): Promise<StorageObject> {
|
|
302
|
-
const source =
|
|
303
|
-
const destination =
|
|
332
|
+
const source = safeKey(from);
|
|
333
|
+
const destination = safeKey(to);
|
|
304
334
|
// Measured AND read in one turn on both keys. Measured in the source's turn and read in the
|
|
305
335
|
// destination's, a put() of the source in between committed its NEW bytes under the OLD
|
|
306
336
|
// etag and type — a pair no marker flags, so every later `get()` of the copy trusted it.
|
|
@@ -338,7 +368,7 @@ export function localDriver(options: LocalDriverOptions): StorageDriver {
|
|
|
338
368
|
},
|
|
339
369
|
|
|
340
370
|
async delete(key: string): Promise<void> {
|
|
341
|
-
const safe =
|
|
371
|
+
const safe = safeKey(key);
|
|
342
372
|
// Idempotent by contract: a missing key is already in the desired state. A REFUSED unlink
|
|
343
373
|
// is not — a read-only mount or a root this process cannot write reports the bytes gone
|
|
344
374
|
// when they are still on disk, which is the one lie an erasure sweep must never repeat.
|
|
@@ -360,12 +390,12 @@ export function localDriver(options: LocalDriverOptions): StorageDriver {
|
|
|
360
390
|
},
|
|
361
391
|
|
|
362
392
|
async exists(key: string): Promise<boolean> {
|
|
363
|
-
return Bun.file(filePath(
|
|
393
|
+
return Bun.file(filePath(safeKey(key))).exists();
|
|
364
394
|
},
|
|
365
395
|
|
|
366
396
|
/** From the sidecar, as recorded — a lapsed retention is still reported, as S3 reports it. */
|
|
367
397
|
async retentionOf(key: string): Promise<ObjectLock> {
|
|
368
|
-
const safe =
|
|
398
|
+
const safe = safeKey(key);
|
|
369
399
|
return oneAtATime(safe, async () => {
|
|
370
400
|
if (!(await Bun.file(filePath(safe)).exists())) throw objectNotFound(registered, safe);
|
|
371
401
|
return lockOfSidecar(await readSidecar(root, safe));
|
|
@@ -384,12 +414,14 @@ export function localDriver(options: LocalDriverOptions): StorageDriver {
|
|
|
384
414
|
},
|
|
385
415
|
|
|
386
416
|
async signedUrl(key: string, urlOptions?: SignedUrlOptions): Promise<string> {
|
|
417
|
+
const bound = signedByteBound(DRIVER_NAME, urlOptions);
|
|
418
|
+
assertWithinRequestCeiling(DRIVER_NAME, maxRequestBytes(), { ...urlOptions, bound });
|
|
387
419
|
return buildSignedUrl({
|
|
388
420
|
secret,
|
|
389
|
-
key:
|
|
421
|
+
key: safeKey(key),
|
|
390
422
|
method: urlOptions?.method,
|
|
391
423
|
expiresInMs: urlOptions?.expiresInMs,
|
|
392
|
-
maxBytes:
|
|
424
|
+
maxBytes: bound,
|
|
393
425
|
contentType: urlOptions?.contentType,
|
|
394
426
|
part: urlOptions?.part,
|
|
395
427
|
baseUrl,
|
package/src/driver-memory.ts
CHANGED
|
@@ -31,6 +31,7 @@ import {
|
|
|
31
31
|
type StorageListEntry,
|
|
32
32
|
type StorageObject,
|
|
33
33
|
type StorageRead,
|
|
34
|
+
type StreamOptions,
|
|
34
35
|
sha256Base64,
|
|
35
36
|
signedByteBound,
|
|
36
37
|
toBytes,
|
|
@@ -40,6 +41,7 @@ import { keyInTheWay } from './driver-memory-conflict';
|
|
|
40
41
|
import { memoryPartStore } from './driver-memory-multipart';
|
|
41
42
|
import { checksumMismatch, getTooLarge, keyConflict, objectNotFound, putFailed } from './errors';
|
|
42
43
|
import { objectChanged } from './errors-promote';
|
|
44
|
+
import { revisionChanged } from './errors-stream';
|
|
43
45
|
import { withStagedMultipart } from './multipart-staged';
|
|
44
46
|
import {
|
|
45
47
|
assertObjectLockOptions,
|
|
@@ -58,6 +60,7 @@ import {
|
|
|
58
60
|
servedRange,
|
|
59
61
|
streamedRange,
|
|
60
62
|
} from './range';
|
|
63
|
+
import { assertWithinRequestCeiling, screenRequestCeiling } from './request-ceiling';
|
|
61
64
|
import type { SignedUrlVerification } from './signed-url';
|
|
62
65
|
import { buildSignedUrl, signedUrlBaseFor, verifySignedUrl } from './signed-url';
|
|
63
66
|
import { resolveSigningSecret } from './signing-secret';
|
|
@@ -110,6 +113,7 @@ export function memoryStorageDriver(options: MemoryStorageDriverOptions = {}): M
|
|
|
110
113
|
1,
|
|
111
114
|
);
|
|
112
115
|
const clock = options.clock ?? systemClock;
|
|
116
|
+
const maxRequestBytes = screenRequestCeiling(DRIVER_NAME, options.maxRequestBytes);
|
|
113
117
|
let baseUrl = options.baseUrl ?? signedUrlBaseFor(DRIVER_NAME);
|
|
114
118
|
const secret = resolveSigningSecret(DRIVER_NAME, options);
|
|
115
119
|
const stored = new Map<string, Stored>();
|
|
@@ -136,8 +140,16 @@ export function memoryStorageDriver(options: MemoryStorageDriverOptions = {}): M
|
|
|
136
140
|
if (blocking !== undefined) throw keyConflict(registered, key, blocking);
|
|
137
141
|
};
|
|
138
142
|
|
|
139
|
-
/**
|
|
140
|
-
|
|
143
|
+
/**
|
|
144
|
+
* Both writes end here, with no await between the checks and the write. `owned` is a buffer this
|
|
145
|
+
* disk built itself (`putStream`'s one concatenation), kept as is rather than copied a third time.
|
|
146
|
+
*/
|
|
147
|
+
const commit = (
|
|
148
|
+
safe: string,
|
|
149
|
+
bytes: Uint8Array,
|
|
150
|
+
putOptions?: PutOptions,
|
|
151
|
+
owned = false,
|
|
152
|
+
): StorageObject => {
|
|
141
153
|
const object: StorageObject = {
|
|
142
154
|
key: safe,
|
|
143
155
|
size: bytes.byteLength,
|
|
@@ -152,7 +164,7 @@ export function memoryStorageDriver(options: MemoryStorageDriverOptions = {}): M
|
|
|
152
164
|
refuseOverwrite(safe, 'overwrite');
|
|
153
165
|
// A copy in, as a file write is: the caller's buffer and metadata are the caller's to reuse.
|
|
154
166
|
stored.set(safe, {
|
|
155
|
-
bytes: bytes.slice(),
|
|
167
|
+
bytes: owned ? bytes : bytes.slice(),
|
|
156
168
|
object: snapshot(object),
|
|
157
169
|
lock: requestedLock(putOptions),
|
|
158
170
|
});
|
|
@@ -164,6 +176,14 @@ export function memoryStorageDriver(options: MemoryStorageDriverOptions = {}): M
|
|
|
164
176
|
if (hit === undefined) throw objectNotFound(registered, key);
|
|
165
177
|
return hit;
|
|
166
178
|
};
|
|
179
|
+
/** `found`, held to the revision a streamed read was pinned to. */
|
|
180
|
+
const pinned = (key: string, options: StreamOptions | undefined): Stored => {
|
|
181
|
+
const hit = found(key);
|
|
182
|
+
const etag = options?.etag;
|
|
183
|
+
if (etag !== undefined && hit.object.etag !== etag)
|
|
184
|
+
throw revisionChanged(registered, key, etag);
|
|
185
|
+
return hit;
|
|
186
|
+
};
|
|
167
187
|
|
|
168
188
|
return withStagedMultipart(
|
|
169
189
|
{
|
|
@@ -230,7 +250,10 @@ export function memoryStorageDriver(options: MemoryStorageDriverOptions = {}): M
|
|
|
230
250
|
'await disk.putStream(key, stream, { maxBytes }) # with a fresh stream',
|
|
231
251
|
);
|
|
232
252
|
});
|
|
233
|
-
|
|
253
|
+
// The chunks are dropped once concatenated: one copy of the body, owned by the map.
|
|
254
|
+
const bytes = new Uint8Array(Bun.concatArrayBuffers(chunks));
|
|
255
|
+
chunks.length = 0;
|
|
256
|
+
return commit(safe, bytes, streamOptions, true);
|
|
234
257
|
},
|
|
235
258
|
|
|
236
259
|
async stat(key: string): Promise<StorageObject | undefined> {
|
|
@@ -255,15 +278,20 @@ export function memoryStorageDriver(options: MemoryStorageDriverOptions = {}): M
|
|
|
255
278
|
return { object: snapshot(hit.object), bytes: hit.bytes.slice() };
|
|
256
279
|
},
|
|
257
280
|
|
|
258
|
-
async stream(key: string): Promise<ReadableStream<Uint8Array>> {
|
|
259
|
-
const
|
|
281
|
+
async stream(key: string, options?: StreamOptions): Promise<ReadableStream<Uint8Array>> {
|
|
282
|
+
const safe = assertSafeKey(key);
|
|
283
|
+
const hit = pinned(safe, options);
|
|
260
284
|
return new Blob([hit.bytes.slice()]).stream();
|
|
261
285
|
},
|
|
262
286
|
|
|
263
|
-
async streamRange(
|
|
287
|
+
async streamRange(
|
|
288
|
+
key: string,
|
|
289
|
+
range: ByteRange,
|
|
290
|
+
options?: StreamOptions,
|
|
291
|
+
): Promise<ReadableStream<Uint8Array>> {
|
|
264
292
|
const safe = assertSafeKey(key);
|
|
265
293
|
assertStreamRange(DRIVER_NAME, range);
|
|
266
|
-
const hit =
|
|
294
|
+
const hit = pinned(safe, options);
|
|
267
295
|
const [first, last] = streamedRange(DRIVER_NAME, safe, range, hit.bytes.byteLength);
|
|
268
296
|
return new Blob([hit.bytes.slice(first, last + 1)]).stream();
|
|
269
297
|
},
|
|
@@ -323,12 +351,14 @@ export function memoryStorageDriver(options: MemoryStorageDriverOptions = {}): M
|
|
|
323
351
|
},
|
|
324
352
|
|
|
325
353
|
async signedUrl(key: string, urlOptions?: SignedUrlOptions): Promise<string> {
|
|
354
|
+
const bound = signedByteBound(DRIVER_NAME, urlOptions);
|
|
355
|
+
assertWithinRequestCeiling(DRIVER_NAME, maxRequestBytes(), { ...urlOptions, bound });
|
|
326
356
|
return buildSignedUrl({
|
|
327
357
|
secret,
|
|
328
358
|
key: assertSafeKey(key),
|
|
329
359
|
method: urlOptions?.method,
|
|
330
360
|
expiresInMs: urlOptions?.expiresInMs,
|
|
331
|
-
maxBytes:
|
|
361
|
+
maxBytes: bound,
|
|
332
362
|
contentType: urlOptions?.contentType,
|
|
333
363
|
part: urlOptions?.part,
|
|
334
364
|
baseUrl,
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
// Single responsibility: an s3 copy past CopyObject's 5 GiB ceiling — a signed HEAD for the
|
|
2
|
+
// source's own headers and revision, CreateMultipartUpload carrying those headers, UploadPartCopy
|
|
3
|
+
// over byte ranges all pinned to that one revision, then Complete. Every exit but a landed Complete
|
|
4
|
+
// aborts the upload, so a failed copy leaves no parts billing.
|
|
5
|
+
|
|
6
|
+
import { copySourceOf, MAX_COPY_OBJECT_BYTES, signedCopy } from './driver-s3-copy';
|
|
7
|
+
import {
|
|
8
|
+
abortS3Upload,
|
|
9
|
+
completeS3Upload,
|
|
10
|
+
createS3Upload,
|
|
11
|
+
type S3UploadTarget,
|
|
12
|
+
xmlText,
|
|
13
|
+
} from './driver-s3-multipart';
|
|
14
|
+
import {
|
|
15
|
+
isAnswered,
|
|
16
|
+
multipartEtagOf,
|
|
17
|
+
type S3Stat,
|
|
18
|
+
sentObject,
|
|
19
|
+
settleComplete,
|
|
20
|
+
} from './driver-s3-settle';
|
|
21
|
+
import { type S3Refusal, type S3WireOptions, sendSigned, xmlField } from './driver-s3-signed';
|
|
22
|
+
|
|
23
|
+
/** The smallest part a large copy cuts: 512 MiB, so 5 TiB (S3's object ceiling) fits 10,000. */
|
|
24
|
+
export const COPY_PART_BYTES = 512 * 1024 ** 2;
|
|
25
|
+
const MAX_PARTS = 10_000;
|
|
26
|
+
/** UploadPartCopy is server-side, so this bounds requests in flight, never bytes held. */
|
|
27
|
+
const COPY_CONCURRENCY = 4;
|
|
28
|
+
|
|
29
|
+
/** The headers CopyObject's COPY directive carries, which a multipart Create must restate. */
|
|
30
|
+
const CARRIED = [
|
|
31
|
+
'content-type',
|
|
32
|
+
'cache-control',
|
|
33
|
+
'content-disposition',
|
|
34
|
+
'content-encoding',
|
|
35
|
+
'content-language',
|
|
36
|
+
'expires',
|
|
37
|
+
] as const;
|
|
38
|
+
|
|
39
|
+
export interface S3PartCopy {
|
|
40
|
+
readonly source: string;
|
|
41
|
+
/** The caller's revision; absent, the HEAD's own etag pins every part to one revision. */
|
|
42
|
+
readonly ifMatch?: string | undefined;
|
|
43
|
+
/** The disk's HEAD, asked only when the Complete's answer was lost. */
|
|
44
|
+
readonly stat: S3Stat;
|
|
45
|
+
/** Injected in tests; production cuts `COPY_PART_BYTES` or `size / 10,000`, the larger. */
|
|
46
|
+
readonly partBytes?: number | undefined;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** The part size: the floor, grown only when 10,000 of it cannot hold the object. */
|
|
50
|
+
export const copyPartBytes = (size: number, floor = COPY_PART_BYTES): number =>
|
|
51
|
+
Math.max(floor, Math.ceil(size / MAX_PARTS));
|
|
52
|
+
|
|
53
|
+
/** Copy `source` onto `target.key` in parts. A refusal goes to `target.refused`, never bare. */
|
|
54
|
+
export async function copyS3InParts(target: S3UploadTarget, copy: S3PartCopy): Promise<void> {
|
|
55
|
+
const { wire, refused } = target;
|
|
56
|
+
const head = await sendSigned(wire, {
|
|
57
|
+
method: 'HEAD',
|
|
58
|
+
key: copy.source,
|
|
59
|
+
headers: copy.ifMatch === undefined ? {} : { 'if-match': copy.ifMatch },
|
|
60
|
+
});
|
|
61
|
+
if (!head.ok) return refused(head.refusal);
|
|
62
|
+
const answer = head.response.headers;
|
|
63
|
+
const size = Number(answer.get('content-length') ?? Number.NaN);
|
|
64
|
+
if (!Number.isSafeInteger(size) || size < 0) {
|
|
65
|
+
return refused({
|
|
66
|
+
name: 'S3Error',
|
|
67
|
+
code: 'UnreadableAnswer',
|
|
68
|
+
message: 'the HEAD before a multipart copy carried no Content-Length',
|
|
69
|
+
statusCode: 200,
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
const etag = answer.get('etag') ?? copy.ifMatch;
|
|
73
|
+
const headers: Record<string, string> = {};
|
|
74
|
+
answer.forEach((value, name) => {
|
|
75
|
+
const lower = name.toLowerCase();
|
|
76
|
+
if (lower.startsWith('x-amz-meta-') || (CARRIED as readonly string[]).includes(lower)) {
|
|
77
|
+
headers[lower] = value;
|
|
78
|
+
}
|
|
79
|
+
});
|
|
80
|
+
const uploadId = await createS3Upload(target, headers);
|
|
81
|
+
const step = copyPartBytes(size, copy.partBytes);
|
|
82
|
+
const count = Math.max(1, Math.ceil(size / step));
|
|
83
|
+
const etags: string[] = new Array(count);
|
|
84
|
+
// Set by the first refused part: its siblings take no new range, and every one in flight settles
|
|
85
|
+
// before the abort — a part landing after it would bill against an upload nobody can see.
|
|
86
|
+
let stopped = false;
|
|
87
|
+
let next = 0;
|
|
88
|
+
const worker = async (): Promise<void> => {
|
|
89
|
+
for (let index = next++; index < count && !stopped; index = next++) {
|
|
90
|
+
const first = index * step;
|
|
91
|
+
const last = Math.min(size, first + step) - 1;
|
|
92
|
+
try {
|
|
93
|
+
etags[index] = await copyPart(target, uploadId, index + 1, {
|
|
94
|
+
'x-amz-copy-source': copySourceOf(wire, copy.source),
|
|
95
|
+
'x-amz-copy-source-range': `bytes=${first}-${last}`,
|
|
96
|
+
...(etag === undefined ? {} : { 'x-amz-copy-source-if-match': etag }),
|
|
97
|
+
});
|
|
98
|
+
} catch (error) {
|
|
99
|
+
stopped = true;
|
|
100
|
+
throw error;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
};
|
|
104
|
+
let settled = false;
|
|
105
|
+
try {
|
|
106
|
+
const workers = await Promise.allSettled(
|
|
107
|
+
Array.from({ length: Math.min(COPY_CONCURRENCY, count) }, worker),
|
|
108
|
+
);
|
|
109
|
+
const failed = workers.find((outcome) => outcome.status === 'rejected');
|
|
110
|
+
if (failed !== undefined) throw failed.reason;
|
|
111
|
+
const parts = etags.map((partEtag, index) => ({ partNumber: index + 1, etag: partEtag }));
|
|
112
|
+
await completeS3Upload(target, uploadId, parts).catch(async (lost: unknown) => {
|
|
113
|
+
if (isAnswered(lost)) throw lost;
|
|
114
|
+
// The answer was lost, not refused: the copy may have landed. Settled — abort included — by
|
|
115
|
+
// the disk's stat, exactly as a streamed put's lost Complete is.
|
|
116
|
+
settled = true;
|
|
117
|
+
const sent = sentObject(size, multipartEtagOf(etags), {
|
|
118
|
+
contentType: headers['content-type'],
|
|
119
|
+
cacheControl: headers['cache-control'],
|
|
120
|
+
metadata: Object.keys(headers).some((name) => name.startsWith('x-amz-meta-'))
|
|
121
|
+
? {}
|
|
122
|
+
: undefined,
|
|
123
|
+
});
|
|
124
|
+
await settleComplete(copy.stat, { ...target, driver: 's3' }, uploadId, sent, lost);
|
|
125
|
+
});
|
|
126
|
+
} catch (error) {
|
|
127
|
+
if (!settled) await abortS3Upload(target, uploadId).catch(() => undefined);
|
|
128
|
+
throw error;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/** One UploadPartCopy, answering its etag — read off the body, where S3 puts it for a copy. */
|
|
133
|
+
async function copyPart(
|
|
134
|
+
{ wire, key, refused }: S3UploadTarget,
|
|
135
|
+
uploadId: string,
|
|
136
|
+
partNumber: number,
|
|
137
|
+
headers: Readonly<Record<string, string>>,
|
|
138
|
+
): Promise<string> {
|
|
139
|
+
const result = await sendSigned(wire, {
|
|
140
|
+
method: 'PUT',
|
|
141
|
+
key,
|
|
142
|
+
query: { partNumber: String(partNumber), uploadId },
|
|
143
|
+
headers,
|
|
144
|
+
});
|
|
145
|
+
if (!result.ok) return refused(result.refusal);
|
|
146
|
+
const body = await result.response.text();
|
|
147
|
+
const etag = xmlField(body, 'ETag');
|
|
148
|
+
if (body.includes('<Error>') || etag === undefined) {
|
|
149
|
+
return refused({
|
|
150
|
+
name: 'S3Error',
|
|
151
|
+
code: xmlField(body, 'Code') ?? 'UnreadableAnswer',
|
|
152
|
+
message: xmlField(body, 'Message') ?? 'UploadPartCopy answered with no ETag',
|
|
153
|
+
statusCode: 200,
|
|
154
|
+
});
|
|
155
|
+
}
|
|
156
|
+
return xmlText(etag);
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/** One copy, either way: CopyObject up to its 5 GiB ceiling, parts past it. Refusals to `refused`. */
|
|
160
|
+
export async function copyS3(
|
|
161
|
+
wire: S3WireOptions,
|
|
162
|
+
copy: {
|
|
163
|
+
readonly source: string;
|
|
164
|
+
readonly destination: string;
|
|
165
|
+
readonly ifMatch: string | undefined;
|
|
166
|
+
readonly size: number;
|
|
167
|
+
readonly stat: S3Stat;
|
|
168
|
+
},
|
|
169
|
+
refused: (refusal: S3Refusal) => never,
|
|
170
|
+
): Promise<void> {
|
|
171
|
+
if (copy.size > MAX_COPY_OBJECT_BYTES) {
|
|
172
|
+
const target = { wire, key: copy.destination, refused };
|
|
173
|
+
return copyS3InParts(target, { source: copy.source, ifMatch: copy.ifMatch, stat: copy.stat });
|
|
174
|
+
}
|
|
175
|
+
const refusal = await signedCopy(wire, copy.source, copy.destination, copy.ifMatch);
|
|
176
|
+
if (refusal !== undefined) refused(refusal);
|
|
177
|
+
}
|
package/src/driver-s3-copy.ts
CHANGED
|
@@ -1,9 +1,17 @@
|
|
|
1
|
-
// Single responsibility: the s3 copy
|
|
2
|
-
//
|
|
3
|
-
//
|
|
1
|
+
// Single responsibility: the s3 copy — ONE signed CopyObject, directive COPY, so the provider
|
|
2
|
+
// carries the source's type, `Cache-Control` and `x-amz-meta-*` as local and memory do (Bun's
|
|
3
|
+
// `write(S3File)` dropped both); `ifMatch` sends `x-amz-copy-source-if-match`. Past 5 GiB, CopyObject
|
|
4
|
+
// refuses: `driver-s3-copy-parts.ts`.
|
|
4
5
|
|
|
5
6
|
import { type S3Refusal, type S3WireOptions, sendSigned, xmlField } from './driver-s3-signed';
|
|
6
7
|
|
|
8
|
+
/** CopyObject's own ceiling: S3 refuses a larger source with `InvalidRequest`. */
|
|
9
|
+
export const MAX_COPY_OBJECT_BYTES = 5 * 1024 ** 3;
|
|
10
|
+
|
|
11
|
+
/** `/<bucket>/<key>`, each segment escaped — the `x-amz-copy-source` CopyObject and UploadPartCopy name. */
|
|
12
|
+
export const copySourceOf = (options: S3WireOptions, source: string): string =>
|
|
13
|
+
`/${encodeURIComponent(options.bucket)}/${source.split('/').map(encodeURIComponent).join('/')}`;
|
|
14
|
+
|
|
7
15
|
/**
|
|
8
16
|
* The provider's refusal, or `undefined` when the copy landed. A 200 is not yet a verdict: S3 sends
|
|
9
17
|
* its headers before a long copy ends, so a failure can arrive as an `<Error>` inside a 200's body.
|
|
@@ -12,15 +20,14 @@ export async function signedCopy(
|
|
|
12
20
|
options: S3WireOptions,
|
|
13
21
|
source: string,
|
|
14
22
|
destination: string,
|
|
15
|
-
ifMatch
|
|
23
|
+
ifMatch?: string | undefined,
|
|
16
24
|
): Promise<S3Refusal | undefined> {
|
|
17
|
-
const path = source.split('/').map(encodeURIComponent).join('/');
|
|
18
25
|
const result = await sendSigned(options, {
|
|
19
26
|
method: 'PUT',
|
|
20
27
|
key: destination,
|
|
21
28
|
headers: {
|
|
22
|
-
'x-amz-copy-source':
|
|
23
|
-
'x-amz-copy-source-if-match': ifMatch,
|
|
29
|
+
'x-amz-copy-source': copySourceOf(options, source),
|
|
30
|
+
...(ifMatch === undefined ? {} : { 'x-amz-copy-source-if-match': ifMatch }),
|
|
24
31
|
},
|
|
25
32
|
});
|
|
26
33
|
if (!result.ok) return result.refusal;
|
package/src/driver-s3-lock.ts
CHANGED
|
@@ -4,14 +4,12 @@
|
|
|
4
4
|
// vocabulary. A plain `put()` never comes here: it stays on the Bun client.
|
|
5
5
|
|
|
6
6
|
import { assert, describeValue } from '@ultimat3/core';
|
|
7
|
-
import { DEFAULT_CONTENT_TYPE, type PutOptions } from './driver';
|
|
7
|
+
import { DEFAULT_CONTENT_TYPE, HEADER_VALUE, type PutOptions } from './driver';
|
|
8
8
|
import { type S3Refusal, type S3WireOptions, sendSigned, xmlField } from './driver-s3-signed';
|
|
9
9
|
import { isRetentionMode, type ObjectLock } from './object-lock';
|
|
10
10
|
|
|
11
11
|
/** RFC 9110 `token` — the only bytes an `x-amz-meta-<name>` header name may carry. */
|
|
12
12
|
const HEADER_TOKEN = /^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/;
|
|
13
|
-
/** Printable ASCII: S3 stores a metadata value as a header, and a CR or a non-ASCII byte is not one. */
|
|
14
|
-
const HEADER_VALUE = /^[\x20-\x7e]*$/;
|
|
15
13
|
|
|
16
14
|
/** The options only the signed PUT can carry. Any one of them routes the put off the Bun client. */
|
|
17
15
|
export function needsSignedPut(options: PutOptions | undefined): boolean {
|