@ultimat3/storage 29.3.2 → 30.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +27 -47
- package/README.md +130 -12
- package/package.json +12 -2
- package/src/accept-part.ts +30 -0
- package/src/accept.ts +7 -0
- package/src/attachment.ts +99 -6
- package/src/byte-pump.ts +46 -0
- package/src/driver-local-list.ts +75 -0
- package/src/driver-local-multipart.ts +133 -0
- package/src/driver-local-read.ts +13 -0
- package/src/driver-local-remove.ts +39 -0
- package/src/driver-local-stream.ts +104 -0
- package/src/driver-local-write.ts +28 -5
- package/src/driver-local.ts +257 -283
- package/src/driver-memory-multipart.ts +41 -0
- package/src/driver-memory.ts +205 -135
- package/src/driver-s3-client.ts +9 -0
- package/src/driver-s3-copy.ts +35 -0
- package/src/driver-s3-lock.ts +9 -2
- package/src/driver-s3-multipart-session.ts +87 -0
- package/src/driver-s3-multipart.ts +177 -0
- package/src/driver-s3-presign.ts +52 -0
- package/src/driver-s3-put-stream.ts +97 -0
- package/src/driver-s3-range.ts +99 -0
- package/src/driver-s3-settle.ts +150 -0
- package/src/driver-s3-signed.ts +69 -4
- package/src/driver-s3-sse.ts +29 -0
- package/src/driver-s3.ts +113 -65
- package/src/driver.ts +98 -38
- package/src/errors-multipart.ts +35 -0
- package/src/errors-promote.ts +14 -0
- package/src/errors-range.ts +47 -0
- package/src/errors-sniff.ts +35 -0
- package/src/errors-stream.ts +62 -0
- package/src/errors.ts +16 -1
- package/src/expire.ts +169 -0
- package/src/grant-multipart.ts +196 -0
- package/src/grant.ts +42 -25
- package/src/image-warm.ts +84 -0
- package/src/image.ts +57 -0
- package/src/index.ts +81 -2
- package/src/multipart-staged.ts +127 -0
- package/src/multipart.ts +119 -0
- package/src/put-stream.ts +52 -0
- package/src/range.ts +102 -0
- package/src/signed-url.ts +59 -2
- package/src/subject-target.ts +82 -0
- package/src/upload-client-multipart.ts +156 -0
- package/src/upload-client.ts +21 -0
- package/src/upload.ts +107 -21
package/src/driver-local.ts
CHANGED
|
@@ -6,17 +6,14 @@
|
|
|
6
6
|
import {
|
|
7
7
|
type Clock,
|
|
8
8
|
finiteCount,
|
|
9
|
-
NotImplementedError,
|
|
10
9
|
type Page,
|
|
11
|
-
pageOf,
|
|
12
10
|
type ResolveEnvironmentOptions,
|
|
13
|
-
renderFixShellArg,
|
|
14
|
-
stringField,
|
|
15
11
|
systemClock,
|
|
16
12
|
} from '@ultimat3/core';
|
|
17
13
|
import {
|
|
18
14
|
assertListOptions,
|
|
19
15
|
assertPutOptions,
|
|
16
|
+
type CopyOptions,
|
|
20
17
|
DEFAULT_CONTENT_TYPE,
|
|
21
18
|
etagOf,
|
|
22
19
|
type ListOptions,
|
|
@@ -29,19 +26,33 @@ import {
|
|
|
29
26
|
type StorageObject,
|
|
30
27
|
type StorageRead,
|
|
31
28
|
sha256Base64,
|
|
29
|
+
signedByteBound,
|
|
32
30
|
toBytes,
|
|
33
31
|
} from './driver';
|
|
34
|
-
import {
|
|
32
|
+
import { listLocalPage } from './driver-local-list';
|
|
33
|
+
import { localPartStore } from './driver-local-multipart';
|
|
34
|
+
import {
|
|
35
|
+
etagOfFile,
|
|
36
|
+
headObject,
|
|
37
|
+
readObjectBytes,
|
|
38
|
+
readObjectRange,
|
|
39
|
+
readSidecar,
|
|
40
|
+
} from './driver-local-read';
|
|
41
|
+
import { removeIfPresent } from './driver-local-remove';
|
|
35
42
|
import { lockOfSidecar, type Sidecar, sidecarLock } from './driver-local-sidecar';
|
|
43
|
+
import { putLocalStream } from './driver-local-stream';
|
|
36
44
|
import {
|
|
37
45
|
commitObject,
|
|
38
46
|
inTurnOfBoth,
|
|
39
47
|
type KeyedQueue,
|
|
40
48
|
keyedQueue,
|
|
41
49
|
pendingPathOf,
|
|
50
|
+
refuseUnsupportedPut,
|
|
42
51
|
sidecarPathOf,
|
|
43
52
|
} from './driver-local-write';
|
|
44
|
-
import { checksumMismatch,
|
|
53
|
+
import { checksumMismatch, getTooLarge, objectNotFound } from './errors';
|
|
54
|
+
import { objectChanged } from './errors-promote';
|
|
55
|
+
import { withStagedMultipart } from './multipart-staged';
|
|
45
56
|
import {
|
|
46
57
|
assertObjectLockOptions,
|
|
47
58
|
isLocked,
|
|
@@ -49,7 +60,16 @@ import {
|
|
|
49
60
|
ObjectLockedError,
|
|
50
61
|
requestedLock,
|
|
51
62
|
} from './object-lock';
|
|
52
|
-
import { assertSafeKey
|
|
63
|
+
import { assertSafeKey } from './path';
|
|
64
|
+
import { assertPutStream, type PutStreamOptions } from './put-stream';
|
|
65
|
+
import {
|
|
66
|
+
assertGetOptions,
|
|
67
|
+
assertStreamRange,
|
|
68
|
+
type ByteRange,
|
|
69
|
+
type GetOptions,
|
|
70
|
+
servedRange,
|
|
71
|
+
streamedRange,
|
|
72
|
+
} from './range';
|
|
53
73
|
import type { SignedUrlVerification } from './signed-url';
|
|
54
74
|
import { buildSignedUrl, signedUrlBaseFor, verifySignedUrl } from './signed-url';
|
|
55
75
|
import { resolveSigningSecret } from './signing-secret';
|
|
@@ -99,38 +119,6 @@ export interface LocalDriverOptions {
|
|
|
99
119
|
readonly queue?: KeyedQueue | undefined;
|
|
100
120
|
}
|
|
101
121
|
|
|
102
|
-
/**
|
|
103
|
-
* A POSIX file is not encrypted at rest by this driver, and recording the request in the sidecar
|
|
104
|
-
* would answer a security review with a field the disk never honoured. Refused on the DEV disk
|
|
105
|
-
* too, and deliberately: a `put()` that succeeds locally and throws in production is a gap an app
|
|
106
|
-
* meets on the worst day, and both drivers refusing is one rule instead of two.
|
|
107
|
-
*/
|
|
108
|
-
function refuseUnsupportedPut(putOptions?: PutOptions): void {
|
|
109
|
-
if (putOptions?.serverSideEncryption === undefined) return;
|
|
110
|
-
throw new NotImplementedError({
|
|
111
|
-
cause:
|
|
112
|
-
'server-side encryption on the local driver (it writes plain files under one root) is not implemented by this driver — drop serverSideEncryption from put() and encrypt the disk itself: an s3Driver over a bucket with a default KMS rule, or a LUKS/FileVault volume under root',
|
|
113
|
-
fix: 'put(key, body)',
|
|
114
|
-
});
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
/** `ENOENT` is the one delete failure that means "already in the desired state". */
|
|
118
|
-
const isMissingFile = (error: unknown): boolean => stringField(error, 'code') === 'ENOENT';
|
|
119
|
-
|
|
120
|
-
/**
|
|
121
|
-
* A directory at an object's path is OTHER keys' parent (`c` beside `c/d`), never an object: no
|
|
122
|
-
* bytes are stored at the key, so deleting it is deleting an absent key. Asked only after an
|
|
123
|
-
* unlink failed, because the errno for "that is a directory" is `EISDIR` on Linux and `EPERM` on
|
|
124
|
-
* macOS — the stat is the portable answer.
|
|
125
|
-
*/
|
|
126
|
-
const isDirectory = async (path: string): Promise<boolean> =>
|
|
127
|
-
Bun.file(path)
|
|
128
|
-
.stat()
|
|
129
|
-
.then(
|
|
130
|
-
(stats) => stats.isDirectory(),
|
|
131
|
-
() => false,
|
|
132
|
-
);
|
|
133
|
-
|
|
134
122
|
export function localDriver(options: LocalDriverOptions): StorageDriver {
|
|
135
123
|
const root = options.root.replace(/\/+$/, '');
|
|
136
124
|
// `=== undefined`, never `??`: `??` coalesces on `null` too, so an explicitly blanked key in a
|
|
@@ -186,266 +174,252 @@ export function localDriver(options: LocalDriverOptions): StorageDriver {
|
|
|
186
174
|
}
|
|
187
175
|
};
|
|
188
176
|
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
await Bun.file(path).delete();
|
|
193
|
-
} catch (error) {
|
|
194
|
-
if (isMissingFile(error) || (await isDirectory(path))) return;
|
|
195
|
-
throw deleteFailed(
|
|
196
|
-
DRIVER_NAME,
|
|
197
|
-
key,
|
|
198
|
-
error,
|
|
199
|
-
// Screened: a key may carry `$(…)` and a root is configuration, and a fix: is pasted.
|
|
200
|
-
`make the disk root writable by this process, then retry: ls -ld ${renderFixShellArg(root, "'<the disk root>'")} && rm -f ${renderFixShellArg(path, "'<the object file>'")}`,
|
|
201
|
-
);
|
|
202
|
-
}
|
|
203
|
-
};
|
|
177
|
+
return withStagedMultipart(
|
|
178
|
+
{
|
|
179
|
+
name: DRIVER_NAME,
|
|
204
180
|
|
|
205
|
-
|
|
206
|
-
|
|
181
|
+
/** A getter, not a captured string: `registerAs` runs after the driver is constructed. */
|
|
182
|
+
get signedUrlBase(): string {
|
|
183
|
+
return baseUrl;
|
|
184
|
+
},
|
|
207
185
|
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
186
|
+
registerAs(diskName: string): void {
|
|
187
|
+
registered = diskName;
|
|
188
|
+
if (options.baseUrl === undefined) baseUrl = signedUrlBaseFor(diskName);
|
|
189
|
+
},
|
|
212
190
|
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
191
|
+
async put(key: string, body: StorageBody, putOptions?: PutOptions): Promise<StorageObject> {
|
|
192
|
+
const safe = assertSafeKey(key);
|
|
193
|
+
refuseUnsupportedPut(putOptions);
|
|
194
|
+
assertPutOptions(DRIVER_NAME, putOptions);
|
|
195
|
+
assertObjectLockOptions(DRIVER_NAME, putOptions, clock);
|
|
196
|
+
const bytes = await toBytes(body, {
|
|
197
|
+
driver: DRIVER_NAME,
|
|
198
|
+
key: safe,
|
|
199
|
+
maxBytes: maxPutBytes,
|
|
200
|
+
});
|
|
201
|
+
const claimed = putOptions?.checksum;
|
|
202
|
+
if (claimed !== undefined) {
|
|
203
|
+
const actual = sha256Base64(bytes);
|
|
204
|
+
if (claimed !== actual) throw checksumMismatch(safe, claimed, actual);
|
|
205
|
+
}
|
|
206
|
+
const lastModified = clock.now();
|
|
207
|
+
const sidecar: Sidecar = {
|
|
208
|
+
contentType: putOptions?.contentType ?? DEFAULT_CONTENT_TYPE,
|
|
209
|
+
etag: etagOf(bytes),
|
|
210
|
+
cacheControl: putOptions?.cacheControl,
|
|
211
|
+
metadata: putOptions?.metadata,
|
|
212
|
+
// The write's instant, on the injected clock, recorded for every later read to report.
|
|
213
|
+
lastModified: lastModified.toISOString(),
|
|
214
|
+
...sidecarLock(requestedLock(putOptions)),
|
|
215
|
+
};
|
|
216
|
+
await oneAtATime(safe, async () => {
|
|
217
|
+
await refuseOverwrite(safe, 'overwrite');
|
|
218
|
+
await commitObject({ root, key: safe, disk: registered, body: bytes, sidecar });
|
|
219
|
+
});
|
|
220
|
+
return {
|
|
221
|
+
key: safe,
|
|
222
|
+
size: bytes.byteLength,
|
|
223
|
+
contentType: sidecar.contentType,
|
|
224
|
+
etag: sidecar.etag,
|
|
225
|
+
lastModified,
|
|
226
|
+
...(sidecar.cacheControl === undefined ? {} : { cacheControl: sidecar.cacheControl }),
|
|
227
|
+
...(sidecar.metadata === undefined ? {} : { metadata: sidecar.metadata }),
|
|
228
|
+
};
|
|
229
|
+
},
|
|
217
230
|
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
cacheControl: putOptions?.cacheControl,
|
|
234
|
-
metadata: putOptions?.metadata,
|
|
235
|
-
// The write's instant, on the injected clock, recorded for every later read to report.
|
|
236
|
-
lastModified: lastModified.toISOString(),
|
|
237
|
-
...sidecarLock(requestedLock(putOptions)),
|
|
238
|
-
};
|
|
239
|
-
await oneAtATime(safe, async () => {
|
|
240
|
-
await refuseOverwrite(safe, 'overwrite');
|
|
241
|
-
await commitObject({ root, key: safe, disk: registered, body: bytes, sidecar });
|
|
242
|
-
});
|
|
243
|
-
return {
|
|
244
|
-
key: safe,
|
|
245
|
-
size: bytes.byteLength,
|
|
246
|
-
contentType: sidecar.contentType,
|
|
247
|
-
etag: sidecar.etag,
|
|
248
|
-
lastModified,
|
|
249
|
-
...(sidecar.cacheControl === undefined ? {} : { cacheControl: sidecar.cacheControl }),
|
|
250
|
-
...(sidecar.metadata === undefined ? {} : { metadata: sidecar.metadata }),
|
|
251
|
-
};
|
|
252
|
-
},
|
|
231
|
+
async putStream(
|
|
232
|
+
key: string,
|
|
233
|
+
body: ReadableStream<Uint8Array>,
|
|
234
|
+
streamOptions: PutStreamOptions,
|
|
235
|
+
): Promise<StorageObject> {
|
|
236
|
+
const safe = assertSafeKey(key);
|
|
237
|
+
const maxBytes = assertPutStream(DRIVER_NAME, body, streamOptions);
|
|
238
|
+
const input = { root, key: safe, disk: registered, body, maxBytes, clock };
|
|
239
|
+
return putLocalStream(input, streamOptions, (then) =>
|
|
240
|
+
oneAtATime(safe, async () => {
|
|
241
|
+
await refuseOverwrite(safe, 'overwrite');
|
|
242
|
+
await then();
|
|
243
|
+
}),
|
|
244
|
+
);
|
|
245
|
+
},
|
|
253
246
|
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
247
|
+
async stat(key: string): Promise<StorageObject | undefined> {
|
|
248
|
+
const entry = await measured(assertSafeKey(key));
|
|
249
|
+
return entry && { ...entry, contentType: entry.contentType ?? DEFAULT_CONTENT_TYPE };
|
|
250
|
+
},
|
|
251
|
+
|
|
252
|
+
async get(key: string, getOptions?: GetOptions): Promise<StorageRead> {
|
|
253
|
+
const safe = assertSafeKey(key);
|
|
254
|
+
assertGetOptions(DRIVER_NAME, getOptions);
|
|
255
|
+
const range = getOptions?.range;
|
|
256
|
+
// Queued behind this key's writers: an object is two files, and a read between a put()'s
|
|
257
|
+
// renames is a pair in doubt this process has no need to meet.
|
|
258
|
+
return oneAtATime(safe, async () => {
|
|
259
|
+
const file = Bun.file(filePath(safe));
|
|
260
|
+
if (!(await file.exists())) throw objectNotFound(registered, safe);
|
|
261
|
+
if (range !== undefined) {
|
|
262
|
+
const served = servedRange(DRIVER_NAME, safe, range, file.size, maxGetBytes);
|
|
263
|
+
const bytes = await readObjectRange(root, safe, registered, served);
|
|
264
|
+
// A doubt is settled by streaming the WHOLE file through the hash, never by buffering it:
|
|
265
|
+
// the range is the only part of this object the read may hold.
|
|
266
|
+
const entry = await headObject(root, safe, () => etagOfFile(root, safe, registered));
|
|
267
|
+
if (entry === undefined) throw objectNotFound(registered, safe);
|
|
268
|
+
const object = { ...entry, contentType: entry.contentType ?? DEFAULT_CONTENT_TYPE };
|
|
269
|
+
return { object, bytes, range: served };
|
|
270
|
+
}
|
|
271
|
+
if (file.size > maxGetBytes) throw getTooLarge(DRIVER_NAME, safe, file.size, maxGetBytes);
|
|
272
|
+
const bytes = await readObjectBytes(root, safe, registered);
|
|
273
|
+
// Settled against the bytes this read ALREADY holds: hashed only for a sidecar-less object
|
|
274
|
+
// or a pair in doubt, and then exactly once.
|
|
275
|
+
const entry = await headObject(root, safe, () => etagOf(bytes));
|
|
276
|
+
if (entry === undefined) throw objectNotFound(registered, safe);
|
|
277
|
+
return {
|
|
278
|
+
object: { ...entry, contentType: entry.contentType ?? DEFAULT_CONTENT_TYPE },
|
|
279
|
+
bytes,
|
|
280
|
+
};
|
|
281
|
+
});
|
|
282
|
+
},
|
|
258
283
|
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
// Queued behind this key's writers: an object is two files, and a read between a put()'s
|
|
262
|
-
// renames is a pair in doubt this process has no need to meet.
|
|
263
|
-
return oneAtATime(safe, async () => {
|
|
284
|
+
async stream(key: string): Promise<ReadableStream<Uint8Array>> {
|
|
285
|
+
const safe = assertSafeKey(key);
|
|
264
286
|
const file = Bun.file(filePath(safe));
|
|
265
287
|
if (!(await file.exists())) throw objectNotFound(registered, safe);
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
// Settled against the bytes this read ALREADY holds: hashed only for a sidecar-less object
|
|
269
|
-
// or a pair in doubt, and then exactly once.
|
|
270
|
-
const entry = await headObject(root, safe, () => etagOf(bytes));
|
|
271
|
-
if (entry === undefined) throw objectNotFound(registered, safe);
|
|
272
|
-
return {
|
|
273
|
-
object: { ...entry, contentType: entry.contentType ?? DEFAULT_CONTENT_TYPE },
|
|
274
|
-
bytes,
|
|
275
|
-
};
|
|
276
|
-
});
|
|
277
|
-
},
|
|
278
|
-
|
|
279
|
-
async stream(key: string): Promise<ReadableStream<Uint8Array>> {
|
|
280
|
-
const safe = assertSafeKey(key);
|
|
281
|
-
const file = Bun.file(filePath(safe));
|
|
282
|
-
if (!(await file.exists())) throw objectNotFound(registered, safe);
|
|
283
|
-
return file.stream();
|
|
284
|
-
},
|
|
288
|
+
return file.stream();
|
|
289
|
+
},
|
|
285
290
|
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
// Measured, not merely read: a sidecar carrying `etag: ''` or a torn source's borrowed
|
|
295
|
-
// type is the same durable lie. The hash is streamed, and only for a pair in doubt.
|
|
296
|
-
const entry = await headObject(root, source, () => etagOfFile(root, source, registered));
|
|
297
|
-
if (entry === undefined) throw objectNotFound(registered, source);
|
|
298
|
-
const lastModified = clock.now();
|
|
299
|
-
const sidecar: Sidecar = {
|
|
300
|
-
contentType: entry.contentType ?? DEFAULT_CONTENT_TYPE,
|
|
301
|
-
etag: entry.etag,
|
|
302
|
-
cacheControl: entry.cacheControl,
|
|
303
|
-
metadata: entry.metadata,
|
|
304
|
-
lastModified: lastModified.toISOString(),
|
|
305
|
-
};
|
|
306
|
-
await refuseOverwrite(destination, 'copy onto');
|
|
307
|
-
await commitObject({
|
|
308
|
-
root,
|
|
309
|
-
key: destination,
|
|
310
|
-
disk: registered,
|
|
311
|
-
body: Bun.file(filePath(source)),
|
|
312
|
-
sidecar,
|
|
313
|
-
}).catch(async (error: unknown) => {
|
|
314
|
-
// The queue is this process's: another process can still take the source mid-copy.
|
|
315
|
-
if (!(await Bun.file(filePath(source)).exists()))
|
|
316
|
-
throw objectNotFound(registered, source);
|
|
317
|
-
throw error;
|
|
318
|
-
});
|
|
319
|
-
return { ...entry, key: destination, contentType: sidecar.contentType, lastModified };
|
|
320
|
-
});
|
|
321
|
-
},
|
|
291
|
+
async streamRange(key: string, range: ByteRange): Promise<ReadableStream<Uint8Array>> {
|
|
292
|
+
const safe = assertSafeKey(key);
|
|
293
|
+
assertStreamRange(DRIVER_NAME, range);
|
|
294
|
+
const file = Bun.file(filePath(safe));
|
|
295
|
+
if (!(await file.exists())) throw objectNotFound(registered, safe);
|
|
296
|
+
const [first, last] = streamedRange(DRIVER_NAME, safe, range, file.size);
|
|
297
|
+
return file.slice(first, last + 1).stream();
|
|
298
|
+
},
|
|
322
299
|
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
300
|
+
/** A real file copy — `Bun.write` from a `BunFile` never routes the bytes through the heap. */
|
|
301
|
+
async copy(from: string, to: string, options?: CopyOptions): Promise<StorageObject> {
|
|
302
|
+
const source = assertSafeKey(from);
|
|
303
|
+
const destination = assertSafeKey(to);
|
|
304
|
+
// Measured AND read in one turn on both keys. Measured in the source's turn and read in the
|
|
305
|
+
// destination's, a put() of the source in between committed its NEW bytes under the OLD
|
|
306
|
+
// etag and type — a pair no marker flags, so every later `get()` of the copy trusted it.
|
|
307
|
+
return inTurnOfBoth(oneAtATime, source, destination, async () => {
|
|
308
|
+
// Measured, not merely read: a sidecar carrying `etag: ''` or a torn source's borrowed
|
|
309
|
+
// type is the same durable lie. The hash is streamed, and only for a pair in doubt.
|
|
310
|
+
const entry = await headObject(root, source, () => etagOfFile(root, source, registered));
|
|
311
|
+
if (entry === undefined) throw objectNotFound(registered, source);
|
|
312
|
+
if (options?.ifMatch !== undefined && entry.etag !== options.ifMatch) {
|
|
313
|
+
throw objectChanged(source, options.ifMatch, entry.etag);
|
|
314
|
+
}
|
|
315
|
+
const lastModified = clock.now();
|
|
316
|
+
const sidecar: Sidecar = {
|
|
317
|
+
contentType: entry.contentType ?? DEFAULT_CONTENT_TYPE,
|
|
318
|
+
etag: entry.etag,
|
|
319
|
+
cacheControl: entry.cacheControl,
|
|
320
|
+
metadata: entry.metadata,
|
|
321
|
+
lastModified: lastModified.toISOString(),
|
|
322
|
+
};
|
|
323
|
+
await refuseOverwrite(destination, 'copy onto');
|
|
324
|
+
await commitObject({
|
|
325
|
+
root,
|
|
326
|
+
key: destination,
|
|
332
327
|
disk: registered,
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
328
|
+
body: Bun.file(filePath(source)),
|
|
329
|
+
sidecar,
|
|
330
|
+
}).catch(async (error: unknown) => {
|
|
331
|
+
// The queue is this process's: another process can still take the source mid-copy.
|
|
332
|
+
if (!(await Bun.file(filePath(source)).exists()))
|
|
333
|
+
throw objectNotFound(registered, source);
|
|
334
|
+
throw error;
|
|
337
335
|
});
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
await removeIfPresent(`${root}/${pendingPathOf(safe)}`, safe);
|
|
342
|
-
});
|
|
343
|
-
},
|
|
336
|
+
return { ...entry, key: destination, contentType: sidecar.contentType, lastModified };
|
|
337
|
+
});
|
|
338
|
+
},
|
|
344
339
|
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
340
|
+
async delete(key: string): Promise<void> {
|
|
341
|
+
const safe = assertSafeKey(key);
|
|
342
|
+
// Idempotent by contract: a missing key is already in the desired state. A REFUSED unlink
|
|
343
|
+
// is not — a read-only mount or a root this process cannot write reports the bytes gone
|
|
344
|
+
// when they are still on disk, which is the one lie an erasure sweep must never repeat.
|
|
345
|
+
await oneAtATime(safe, async () => {
|
|
346
|
+
const lock = await lockAt(safe);
|
|
347
|
+
if (lock !== undefined) {
|
|
348
|
+
throw new ObjectLockedError({
|
|
349
|
+
disk: registered,
|
|
350
|
+
key: safe,
|
|
351
|
+
action: 'delete',
|
|
352
|
+
lock,
|
|
353
|
+
sidecar: metaPath(safe),
|
|
354
|
+
});
|
|
355
|
+
}
|
|
356
|
+
await removeIfPresent(root, filePath(safe), safe);
|
|
357
|
+
await removeIfPresent(root, metaPath(safe), safe);
|
|
358
|
+
await removeIfPresent(root, `${root}/${pendingPathOf(safe)}`, safe);
|
|
359
|
+
});
|
|
360
|
+
},
|
|
348
361
|
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
return oneAtATime(safe, async () => {
|
|
353
|
-
if (!(await Bun.file(filePath(safe)).exists())) throw objectNotFound(registered, safe);
|
|
354
|
-
return lockOfSidecar(await readSidecar(root, safe));
|
|
355
|
-
});
|
|
356
|
-
},
|
|
362
|
+
async exists(key: string): Promise<boolean> {
|
|
363
|
+
return Bun.file(filePath(assertSafeKey(key))).exists();
|
|
364
|
+
},
|
|
357
365
|
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
// every object whose key has one — `.hidden.txt`, `org/o1/pending/.x.png`, the
|
|
367
|
-
// `.metadata/a.json` `path.test.ts` pins as legal — was absent from the listing while
|
|
368
|
-
// `put`/`get`/`exists` handled it normally and the s3 listing returned it. `sweepOrphans`
|
|
369
|
-
// pages through `list()`, so those objects were swept as if they did not exist: a false
|
|
370
|
-
// erasure report by omission, which is what the classification below exists to prevent.
|
|
371
|
-
for await (const entry of new Bun.Glob('**/*').scan({
|
|
372
|
-
cwd: root,
|
|
373
|
-
onlyFiles: true,
|
|
374
|
-
dot: true,
|
|
375
|
-
})) {
|
|
376
|
-
const key = entry.replaceAll('\\', '/');
|
|
377
|
-
// The real filter now, not a second line of defence: the glob above yields the sidecar
|
|
378
|
-
// tree, and this is the one thing keeping `.meta/<key>.json` out of the object
|
|
379
|
-
// namespace. Folded like `assertSafeKey`'s reservation — `.META/` and `.meta/` are one
|
|
380
|
-
// directory on APFS and NTFS, and listing a key `get()` would refuse is the worse half.
|
|
381
|
-
if (key.toLowerCase().startsWith(`${META_DIR}/`)) continue;
|
|
382
|
-
if (!key.startsWith(prefix)) continue;
|
|
383
|
-
// The cursor IS the last key of the previous page — lexicographic order keeps it stable.
|
|
384
|
-
if (cursor !== undefined && key <= cursor) continue;
|
|
385
|
-
keys.push(key);
|
|
386
|
-
}
|
|
387
|
-
} catch (error) {
|
|
388
|
-
// A disk nobody has written to yet has no directory: an empty listing, not an error.
|
|
389
|
-
if (isMissingFile(error)) return pageOf([], null);
|
|
390
|
-
// Everything else is a refusal, and a bare `catch` reported all of them as "this disk is
|
|
391
|
-
// empty" — `EACCES` on the root, `ENOTDIR` on a root that is a file, an I/O error on the
|
|
392
|
-
// mount. `sweepOrphans` walks `list()`, so that swallow certified an unreadable prefix as
|
|
393
|
-
// having no orphans: the same false report `delete()`'s `.catch(() => undefined)` used to
|
|
394
|
-
// make, one call to the left.
|
|
395
|
-
throw listFailed(
|
|
396
|
-
DRIVER_NAME,
|
|
397
|
-
prefix,
|
|
398
|
-
error,
|
|
399
|
-
`make the disk root readable by this process, then retry: ls -ld ${renderFixShellArg(root, "'<the disk root>'")}`,
|
|
400
|
-
);
|
|
401
|
-
}
|
|
402
|
-
keys.sort();
|
|
403
|
-
const page = keys.slice(0, limit);
|
|
404
|
-
const objects: StorageListEntry[] = [];
|
|
405
|
-
for (const key of page) {
|
|
406
|
-
const object = await head(key);
|
|
407
|
-
if (object !== undefined) objects.push(object);
|
|
408
|
-
}
|
|
409
|
-
// The cursor is minted only when a key PAST the page exists — `keys` is every match, so a
|
|
410
|
-
// full last page answers no cursor. `limit` is a positive integer, so a page with more after
|
|
411
|
-
// it always HAS a last key: the `?? null` is the type's, not a second condition. It used to
|
|
412
|
-
// be one, and `limit: 0` fell through it — an empty page reported as complete.
|
|
413
|
-
return pageOf(objects, keys.length > page.length ? (page.at(-1) ?? null) : null);
|
|
414
|
-
},
|
|
366
|
+
/** From the sidecar, as recorded — a lapsed retention is still reported, as S3 reports it. */
|
|
367
|
+
async retentionOf(key: string): Promise<ObjectLock> {
|
|
368
|
+
const safe = assertSafeKey(key);
|
|
369
|
+
return oneAtATime(safe, async () => {
|
|
370
|
+
if (!(await Bun.file(filePath(safe)).exists())) throw objectNotFound(registered, safe);
|
|
371
|
+
return lockOfSidecar(await readSidecar(root, safe));
|
|
372
|
+
});
|
|
373
|
+
},
|
|
415
374
|
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
375
|
+
async list(listOptions?: ListOptions): Promise<Page<StorageListEntry>> {
|
|
376
|
+
assertListOptions(listOptions);
|
|
377
|
+
return listLocalPage({
|
|
378
|
+
root,
|
|
379
|
+
prefix: listOptions?.prefix ?? '',
|
|
380
|
+
cursor: listOptions?.cursor ?? undefined,
|
|
381
|
+
limit: resolveListLimit(listOptions?.limit),
|
|
382
|
+
head,
|
|
383
|
+
});
|
|
384
|
+
},
|
|
385
|
+
|
|
386
|
+
async signedUrl(key: string, urlOptions?: SignedUrlOptions): Promise<string> {
|
|
387
|
+
return buildSignedUrl({
|
|
388
|
+
secret,
|
|
389
|
+
key: assertSafeKey(key),
|
|
390
|
+
method: urlOptions?.method,
|
|
391
|
+
expiresInMs: urlOptions?.expiresInMs,
|
|
392
|
+
maxBytes: signedByteBound(DRIVER_NAME, urlOptions),
|
|
393
|
+
contentType: urlOptions?.contentType,
|
|
394
|
+
part: urlOptions?.part,
|
|
395
|
+
baseUrl,
|
|
396
|
+
clock,
|
|
397
|
+
});
|
|
398
|
+
},
|
|
428
399
|
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
400
|
+
/**
|
|
401
|
+
* The mint above, run backwards, on the same three values from the same closure: `secret`,
|
|
402
|
+
* `baseUrl` and `clock`. That is the whole reason it belongs here rather than in a caller — a
|
|
403
|
+
* route holding this disk can now accept an upload without ever being handed the key, and
|
|
404
|
+
* before this member existed there was no way for one to verify at all, which is why the
|
|
405
|
+
* shipped `PUT` half of `/_storage` was never mounted.
|
|
406
|
+
*
|
|
407
|
+
* Never throws: `verifySignedUrl` returns a reason, and `accept.ts` owns which reason becomes
|
|
408
|
+
* which error. A driver that decided that here would be a second error taxonomy.
|
|
409
|
+
*/
|
|
410
|
+
async verifySigned(input: {
|
|
411
|
+
readonly url: string;
|
|
412
|
+
readonly clock?: Clock | undefined;
|
|
413
|
+
}): Promise<SignedUrlVerification> {
|
|
414
|
+
return verifySignedUrl({
|
|
415
|
+
url: input.url,
|
|
416
|
+
secret,
|
|
417
|
+
baseUrl,
|
|
418
|
+
clock: input.clock ?? clock,
|
|
419
|
+
});
|
|
420
|
+
},
|
|
449
421
|
},
|
|
450
|
-
|
|
422
|
+
localPartStore(root),
|
|
423
|
+
clock,
|
|
424
|
+
);
|
|
451
425
|
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
// Single responsibility: where the memory disk keeps an open upload in parts — a `Map` beside its
|
|
2
|
+
// objects, holding copies, so a test's disk answers the local disk's multipart contract with
|
|
3
|
+
// nothing on a filesystem.
|
|
4
|
+
|
|
5
|
+
import type { UploadedPart } from './multipart';
|
|
6
|
+
import type { PartStore, StagedSession } from './multipart-staged';
|
|
7
|
+
|
|
8
|
+
interface Held {
|
|
9
|
+
readonly session: StagedSession;
|
|
10
|
+
readonly parts: Map<number, { readonly part: UploadedPart; readonly bytes: Uint8Array }>;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export function memoryPartStore(): PartStore {
|
|
14
|
+
const held = new Map<string, Held>();
|
|
15
|
+
return {
|
|
16
|
+
async open(uploadId, session): Promise<void> {
|
|
17
|
+
held.set(uploadId, { session, parts: new Map() });
|
|
18
|
+
},
|
|
19
|
+
async session(uploadId): Promise<StagedSession | undefined> {
|
|
20
|
+
return held.get(uploadId)?.session;
|
|
21
|
+
},
|
|
22
|
+
async writePart(uploadId, part, bytes): Promise<void> {
|
|
23
|
+
held.get(uploadId)?.parts.set(part.partNumber, { part, bytes: bytes.slice() });
|
|
24
|
+
},
|
|
25
|
+
async parts(uploadId): Promise<readonly UploadedPart[]> {
|
|
26
|
+
const parts = [...(held.get(uploadId)?.parts.values() ?? [])].map(({ part }) => part);
|
|
27
|
+
return parts.sort((a, b) => a.partNumber - b.partNumber);
|
|
28
|
+
},
|
|
29
|
+
read(uploadId, parts): ReadableStream<Uint8Array> {
|
|
30
|
+
const stored = held.get(uploadId)?.parts;
|
|
31
|
+
const chunks = parts.map(({ partNumber }) => stored?.get(partNumber)?.bytes.slice());
|
|
32
|
+
return new Blob(chunks.filter((chunk) => chunk !== undefined)).stream();
|
|
33
|
+
},
|
|
34
|
+
async drop(uploadId): Promise<void> {
|
|
35
|
+
held.delete(uploadId);
|
|
36
|
+
},
|
|
37
|
+
async sessions(): Promise<readonly (readonly [string, StagedSession])[]> {
|
|
38
|
+
return [...held].map(([uploadId, { session }]) => [uploadId, session] as const);
|
|
39
|
+
},
|
|
40
|
+
};
|
|
41
|
+
}
|