@ultimat3/storage 24.0.0 → 25.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +36 -40
- package/README.md +86 -17
- package/package.json +3 -3
- package/src/attachment.ts +6 -9
- package/src/driver-local-read.ts +19 -9
- package/src/driver-local-sidecar.ts +61 -3
- package/src/driver-local.ts +98 -33
- package/src/driver-memory-conflict.ts +45 -0
- package/src/driver-memory.ts +75 -16
- package/src/driver-s3-absent.ts +32 -0
- package/src/driver-s3-client.ts +94 -0
- package/src/driver-s3-lock.ts +131 -0
- package/src/driver-s3-signed.ts +220 -0
- package/src/driver-s3.ts +95 -155
- package/src/driver.ts +36 -19
- package/src/errors.ts +23 -6
- package/src/index.ts +15 -7
- package/src/object-lock.ts +173 -0
- package/src/signed-url.ts +0 -4
- package/src/signing-secret.ts +12 -7
- package/src/upload.ts +51 -0
package/src/driver-local.ts
CHANGED
|
@@ -7,7 +7,10 @@ import {
|
|
|
7
7
|
type Clock,
|
|
8
8
|
finiteCount,
|
|
9
9
|
NotImplementedError,
|
|
10
|
+
type Page,
|
|
11
|
+
pageOf,
|
|
10
12
|
type ResolveEnvironmentOptions,
|
|
13
|
+
renderFixShellArg,
|
|
11
14
|
stringField,
|
|
12
15
|
systemClock,
|
|
13
16
|
} from '@ultimat3/core';
|
|
@@ -17,7 +20,6 @@ import {
|
|
|
17
20
|
DEFAULT_CONTENT_TYPE,
|
|
18
21
|
etagOf,
|
|
19
22
|
type ListOptions,
|
|
20
|
-
type ListPage,
|
|
21
23
|
type PutOptions,
|
|
22
24
|
resolveListLimit,
|
|
23
25
|
type SignedUrlOptions,
|
|
@@ -29,10 +31,17 @@ import {
|
|
|
29
31
|
sha256Base64,
|
|
30
32
|
toBytes,
|
|
31
33
|
} from './driver';
|
|
32
|
-
import { etagOfFile, headObject, readObjectBytes } from './driver-local-read';
|
|
33
|
-
import type
|
|
34
|
+
import { etagOfFile, headObject, readObjectBytes, readSidecar } from './driver-local-read';
|
|
35
|
+
import { lockOfSidecar, type Sidecar, sidecarLock } from './driver-local-sidecar';
|
|
34
36
|
import { commitObject, keyedQueue, pendingPathOf, sidecarPathOf } from './driver-local-write';
|
|
35
37
|
import { checksumMismatch, deleteFailed, getTooLarge, listFailed, objectNotFound } from './errors';
|
|
38
|
+
import {
|
|
39
|
+
assertObjectLockOptions,
|
|
40
|
+
isLocked,
|
|
41
|
+
type ObjectLock,
|
|
42
|
+
ObjectLockedError,
|
|
43
|
+
requestedLock,
|
|
44
|
+
} from './object-lock';
|
|
36
45
|
import { assertSafeKey, META_DIR } from './path';
|
|
37
46
|
import type { SignedUrlVerification } from './signed-url';
|
|
38
47
|
import { buildSignedUrl, signedUrlBaseFor, verifySignedUrl } from './signed-url';
|
|
@@ -96,6 +105,20 @@ function refuseUnsupportedPut(putOptions?: PutOptions): void {
|
|
|
96
105
|
/** `ENOENT` is the one delete failure that means "already in the desired state". */
|
|
97
106
|
const isMissingFile = (error: unknown): boolean => stringField(error, 'code') === 'ENOENT';
|
|
98
107
|
|
|
108
|
+
/**
|
|
109
|
+
* A directory at an object's path is OTHER keys' parent (`c` beside `c/d`), never an object: no
|
|
110
|
+
* bytes are stored at the key, so deleting it is deleting an absent key. Asked only after an
|
|
111
|
+
* unlink failed, because the errno for "that is a directory" is `EISDIR` on Linux and `EPERM` on
|
|
112
|
+
* macOS — the stat is the portable answer.
|
|
113
|
+
*/
|
|
114
|
+
const isDirectory = async (path: string): Promise<boolean> =>
|
|
115
|
+
Bun.file(path)
|
|
116
|
+
.stat()
|
|
117
|
+
.then(
|
|
118
|
+
(stats) => stats.isDirectory(),
|
|
119
|
+
() => false,
|
|
120
|
+
);
|
|
121
|
+
|
|
99
122
|
export function localDriver(options: LocalDriverOptions): StorageDriver {
|
|
100
123
|
const root = options.root.replace(/\/+$/, '');
|
|
101
124
|
// `=== undefined`, never `??`: `??` coalesces on `null` too, so an explicitly blanked key in a
|
|
@@ -133,19 +156,36 @@ export function localDriver(options: LocalDriverOptions): StorageDriver {
|
|
|
133
156
|
oneAtATime(key, () => headObject(root, key));
|
|
134
157
|
/** The same, settled against the bytes on disk — streamed through a hasher, never buffered. */
|
|
135
158
|
const measured = (key: string): Promise<StorageListEntry | undefined> =>
|
|
136
|
-
oneAtATime(key, () => headObject(root, key, () => etagOfFile(root, key)));
|
|
159
|
+
oneAtATime(key, () => headObject(root, key, () => etagOfFile(root, key, registered)));
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* The lock the sidecar RECORDS, read whole even when the pair is in doubt: a torn write never
|
|
163
|
+
* unlocks an object. `undefined` when nothing on disk locks it now.
|
|
164
|
+
*/
|
|
165
|
+
const lockAt = async (key: string): Promise<ObjectLock | undefined> => {
|
|
166
|
+
const lock = lockOfSidecar(await readSidecar(root, key));
|
|
167
|
+
return isLocked(lock, clock.now()) ? lock : undefined;
|
|
168
|
+
};
|
|
169
|
+
/** Called INSIDE the key's queue, so no writer of this process lands between check and commit. */
|
|
170
|
+
const refuseOverwrite = async (key: string, action: 'overwrite' | 'copy onto'): Promise<void> => {
|
|
171
|
+
const lock = await lockAt(key);
|
|
172
|
+
if (lock !== undefined) {
|
|
173
|
+
throw new ObjectLockedError({ disk: registered, key, action, lock, sidecar: metaPath(key) });
|
|
174
|
+
}
|
|
175
|
+
};
|
|
137
176
|
|
|
138
177
|
/** Removes one path, or reports WHY it could not — a swallowed refusal is a false erasure. */
|
|
139
178
|
const removeIfPresent = async (path: string, key: string): Promise<void> => {
|
|
140
179
|
try {
|
|
141
180
|
await Bun.file(path).delete();
|
|
142
181
|
} catch (error) {
|
|
143
|
-
if (isMissingFile(error)) return;
|
|
182
|
+
if (isMissingFile(error) || (await isDirectory(path))) return;
|
|
144
183
|
throw deleteFailed(
|
|
145
184
|
DRIVER_NAME,
|
|
146
185
|
key,
|
|
147
186
|
error,
|
|
148
|
-
|
|
187
|
+
// Screened: a key may carry `$(…)` and a root is configuration, and a fix: is pasted.
|
|
188
|
+
`make the disk root writable by this process, then retry: ls -ld ${renderFixShellArg(root, "'<the disk root>'")} && rm -f ${renderFixShellArg(path, "'<the object file>'")}`,
|
|
149
189
|
);
|
|
150
190
|
}
|
|
151
191
|
};
|
|
@@ -167,27 +207,33 @@ export function localDriver(options: LocalDriverOptions): StorageDriver {
|
|
|
167
207
|
const safe = assertSafeKey(key);
|
|
168
208
|
refuseUnsupportedPut(putOptions);
|
|
169
209
|
assertPutOptions(DRIVER_NAME, putOptions);
|
|
210
|
+
assertObjectLockOptions(DRIVER_NAME, putOptions, clock);
|
|
170
211
|
const bytes = await toBytes(body, { driver: DRIVER_NAME, key: safe, maxBytes: maxPutBytes });
|
|
171
212
|
const claimed = putOptions?.checksum;
|
|
172
213
|
if (claimed !== undefined) {
|
|
173
214
|
const actual = sha256Base64(bytes);
|
|
174
215
|
if (claimed !== actual) throw checksumMismatch(safe, claimed, actual);
|
|
175
216
|
}
|
|
217
|
+
const lastModified = clock.now();
|
|
176
218
|
const sidecar: Sidecar = {
|
|
177
219
|
contentType: putOptions?.contentType ?? DEFAULT_CONTENT_TYPE,
|
|
178
220
|
etag: etagOf(bytes),
|
|
179
221
|
cacheControl: putOptions?.cacheControl,
|
|
180
222
|
metadata: putOptions?.metadata,
|
|
223
|
+
// The write's instant, on the injected clock, recorded for every later read to report.
|
|
224
|
+
lastModified: lastModified.toISOString(),
|
|
225
|
+
...sidecarLock(requestedLock(putOptions)),
|
|
181
226
|
};
|
|
182
|
-
await oneAtATime(safe, () =>
|
|
183
|
-
|
|
184
|
-
|
|
227
|
+
await oneAtATime(safe, async () => {
|
|
228
|
+
await refuseOverwrite(safe, 'overwrite');
|
|
229
|
+
await commitObject({ root, key: safe, disk: registered, body: bytes, sidecar });
|
|
230
|
+
});
|
|
185
231
|
return {
|
|
186
232
|
key: safe,
|
|
187
233
|
size: bytes.byteLength,
|
|
188
234
|
contentType: sidecar.contentType,
|
|
189
235
|
etag: sidecar.etag,
|
|
190
|
-
lastModified
|
|
236
|
+
lastModified,
|
|
191
237
|
...(sidecar.cacheControl === undefined ? {} : { cacheControl: sidecar.cacheControl }),
|
|
192
238
|
...(sidecar.metadata === undefined ? {} : { metadata: sidecar.metadata }),
|
|
193
239
|
};
|
|
@@ -204,13 +250,13 @@ export function localDriver(options: LocalDriverOptions): StorageDriver {
|
|
|
204
250
|
// renames is a pair in doubt this process has no need to meet.
|
|
205
251
|
return oneAtATime(safe, async () => {
|
|
206
252
|
const file = Bun.file(filePath(safe));
|
|
207
|
-
if (!(await file.exists())) throw objectNotFound(
|
|
253
|
+
if (!(await file.exists())) throw objectNotFound(registered, safe);
|
|
208
254
|
if (file.size > maxGetBytes) throw getTooLarge(DRIVER_NAME, safe, file.size, maxGetBytes);
|
|
209
|
-
const bytes = await readObjectBytes(root, safe);
|
|
255
|
+
const bytes = await readObjectBytes(root, safe, registered);
|
|
210
256
|
// Settled against the bytes this read ALREADY holds: hashed only for a sidecar-less object
|
|
211
257
|
// or a pair in doubt, and then exactly once.
|
|
212
258
|
const entry = await headObject(root, safe, () => etagOf(bytes));
|
|
213
|
-
if (entry === undefined) throw objectNotFound(
|
|
259
|
+
if (entry === undefined) throw objectNotFound(registered, safe);
|
|
214
260
|
return {
|
|
215
261
|
object: { ...entry, contentType: entry.contentType ?? DEFAULT_CONTENT_TYPE },
|
|
216
262
|
bytes,
|
|
@@ -221,7 +267,7 @@ export function localDriver(options: LocalDriverOptions): StorageDriver {
|
|
|
221
267
|
async stream(key: string): Promise<ReadableStream<Uint8Array>> {
|
|
222
268
|
const safe = assertSafeKey(key);
|
|
223
269
|
const file = Bun.file(filePath(safe));
|
|
224
|
-
if (!(await file.exists())) throw objectNotFound(
|
|
270
|
+
if (!(await file.exists())) throw objectNotFound(registered, safe);
|
|
225
271
|
return file.stream();
|
|
226
272
|
},
|
|
227
273
|
|
|
@@ -233,31 +279,34 @@ export function localDriver(options: LocalDriverOptions): StorageDriver {
|
|
|
233
279
|
// a torn source's borrowed type is a durable lie every later `get()` of the copy would
|
|
234
280
|
// trust. The hash is streamed, and only for a sidecar-less source or a pair in doubt.
|
|
235
281
|
const entry = await measured(source);
|
|
236
|
-
if (entry === undefined) throw objectNotFound(
|
|
282
|
+
if (entry === undefined) throw objectNotFound(registered, source);
|
|
283
|
+
const lastModified = clock.now();
|
|
237
284
|
const sidecar: Sidecar = {
|
|
238
285
|
contentType: entry.contentType ?? DEFAULT_CONTENT_TYPE,
|
|
239
286
|
etag: entry.etag,
|
|
240
287
|
cacheControl: entry.cacheControl,
|
|
241
288
|
metadata: entry.metadata,
|
|
289
|
+
lastModified: lastModified.toISOString(),
|
|
242
290
|
};
|
|
243
|
-
await oneAtATime(destination, () =>
|
|
244
|
-
|
|
291
|
+
await oneAtATime(destination, async () => {
|
|
292
|
+
await refuseOverwrite(destination, 'copy onto');
|
|
293
|
+
await commitObject({
|
|
245
294
|
root,
|
|
246
295
|
key: destination,
|
|
247
296
|
disk: registered,
|
|
248
297
|
body: Bun.file(filePath(source)),
|
|
249
298
|
sidecar,
|
|
250
|
-
})
|
|
251
|
-
).catch(async (error: unknown) => {
|
|
299
|
+
});
|
|
300
|
+
}).catch(async (error: unknown) => {
|
|
252
301
|
// The source was deleted while this copy waited its turn on the destination.
|
|
253
|
-
if (!(await Bun.file(filePath(source)).exists())) throw objectNotFound(
|
|
302
|
+
if (!(await Bun.file(filePath(source)).exists())) throw objectNotFound(registered, source);
|
|
254
303
|
throw error;
|
|
255
304
|
});
|
|
256
305
|
return {
|
|
257
306
|
...entry,
|
|
258
307
|
key: destination,
|
|
259
308
|
contentType: sidecar.contentType,
|
|
260
|
-
lastModified
|
|
309
|
+
lastModified,
|
|
261
310
|
};
|
|
262
311
|
},
|
|
263
312
|
|
|
@@ -267,6 +316,16 @@ export function localDriver(options: LocalDriverOptions): StorageDriver {
|
|
|
267
316
|
// is not — a read-only mount or a root this process cannot write reports the bytes gone
|
|
268
317
|
// when they are still on disk, which is the one lie an erasure sweep must never repeat.
|
|
269
318
|
await oneAtATime(safe, async () => {
|
|
319
|
+
const lock = await lockAt(safe);
|
|
320
|
+
if (lock !== undefined) {
|
|
321
|
+
throw new ObjectLockedError({
|
|
322
|
+
disk: registered,
|
|
323
|
+
key: safe,
|
|
324
|
+
action: 'delete',
|
|
325
|
+
lock,
|
|
326
|
+
sidecar: metaPath(safe),
|
|
327
|
+
});
|
|
328
|
+
}
|
|
270
329
|
await removeIfPresent(filePath(safe), safe);
|
|
271
330
|
await removeIfPresent(metaPath(safe), safe);
|
|
272
331
|
await removeIfPresent(`${root}/${pendingPathOf(safe)}`, safe);
|
|
@@ -277,11 +336,20 @@ export function localDriver(options: LocalDriverOptions): StorageDriver {
|
|
|
277
336
|
return Bun.file(filePath(assertSafeKey(key))).exists();
|
|
278
337
|
},
|
|
279
338
|
|
|
280
|
-
|
|
339
|
+
/** From the sidecar, as recorded — a lapsed retention is still reported, as S3 reports it. */
|
|
340
|
+
async retentionOf(key: string): Promise<ObjectLock> {
|
|
341
|
+
const safe = assertSafeKey(key);
|
|
342
|
+
return oneAtATime(safe, async () => {
|
|
343
|
+
if (!(await Bun.file(filePath(safe)).exists())) throw objectNotFound(registered, safe);
|
|
344
|
+
return lockOfSidecar(await readSidecar(root, safe));
|
|
345
|
+
});
|
|
346
|
+
},
|
|
347
|
+
|
|
348
|
+
async list(listOptions?: ListOptions): Promise<Page<StorageListEntry>> {
|
|
281
349
|
const prefix = listOptions?.prefix ?? '';
|
|
282
350
|
assertListOptions(listOptions);
|
|
283
351
|
const limit = resolveListLimit(listOptions?.limit);
|
|
284
|
-
const cursor = listOptions?.cursor;
|
|
352
|
+
const cursor = listOptions?.cursor ?? undefined;
|
|
285
353
|
const keys: string[] = [];
|
|
286
354
|
try {
|
|
287
355
|
// `dot: true`, and it is load-bearing: without it a glob matches no dot-prefixed entry, so
|
|
@@ -308,7 +376,7 @@ export function localDriver(options: LocalDriverOptions): StorageDriver {
|
|
|
308
376
|
}
|
|
309
377
|
} catch (error) {
|
|
310
378
|
// A disk nobody has written to yet has no directory: an empty listing, not an error.
|
|
311
|
-
if (isMissingFile(error)) return
|
|
379
|
+
if (isMissingFile(error)) return pageOf([], null);
|
|
312
380
|
// Everything else is a refusal, and a bare `catch` reported all of them as "this disk is
|
|
313
381
|
// empty" — `EACCES` on the root, `ENOTDIR` on a root that is a file, an I/O error on the
|
|
314
382
|
// mount. `sweepOrphans` walks `list()`, so that swallow certified an unreadable prefix as
|
|
@@ -318,7 +386,7 @@ export function localDriver(options: LocalDriverOptions): StorageDriver {
|
|
|
318
386
|
DRIVER_NAME,
|
|
319
387
|
prefix,
|
|
320
388
|
error,
|
|
321
|
-
`make the disk root readable by this process, then retry: ls -ld ${root}`,
|
|
389
|
+
`make the disk root readable by this process, then retry: ls -ld ${renderFixShellArg(root, "'<the disk root>'")}`,
|
|
322
390
|
);
|
|
323
391
|
}
|
|
324
392
|
keys.sort();
|
|
@@ -328,14 +396,11 @@ export function localDriver(options: LocalDriverOptions): StorageDriver {
|
|
|
328
396
|
const object = await head(key);
|
|
329
397
|
if (object !== undefined) objects.push(object);
|
|
330
398
|
}
|
|
331
|
-
|
|
332
|
-
// `limit` is a positive integer, so a
|
|
333
|
-
//
|
|
334
|
-
// an empty page reported as complete
|
|
335
|
-
|
|
336
|
-
return truncated && last !== undefined
|
|
337
|
-
? { objects, truncated, cursor: last }
|
|
338
|
-
: { objects, truncated: false };
|
|
399
|
+
// The cursor is minted only when a key PAST the page exists — `keys` is every match, so a
|
|
400
|
+
// full last page answers no cursor. `limit` is a positive integer, so a page with more after
|
|
401
|
+
// it always HAS a last key: the `?? null` is the type's, not a second condition. It used to
|
|
402
|
+
// be one, and `limit: 0` fell through it — an empty page reported as complete.
|
|
403
|
+
return pageOf(objects, keys.length > page.length ? (page.at(-1) ?? null) : null);
|
|
339
404
|
},
|
|
340
405
|
|
|
341
406
|
async signedUrl(key: string, urlOptions?: SignedUrlOptions): Promise<string> {
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
// Single responsibility: which stored key stands in the way of a new one, answered over a set of
|
|
2
|
+
// keys exactly as `driver-local-write.ts`'s `claim()` answers it over a directory tree — so the
|
|
3
|
+
// memory disk refuses the keys the local disk it stands in for cannot hold.
|
|
4
|
+
|
|
5
|
+
import { pendingPathOf, sidecarPathOf } from './driver-local-write';
|
|
6
|
+
import { META_DIR } from './path';
|
|
7
|
+
|
|
8
|
+
/** `.meta/<key>.json` → `<key>.json`: the sidecar tree, read as keys. */
|
|
9
|
+
const inSidecarTree = (relative: string): string => relative.slice(META_DIR.length + 1);
|
|
10
|
+
|
|
11
|
+
/** Every proper ancestor directory of `path`, shallowest first: what `mkdir -p` would make. */
|
|
12
|
+
const ancestorsOf = (path: string): readonly string[] => {
|
|
13
|
+
const segments = path.split('/').slice(0, -1);
|
|
14
|
+
return segments.map((_, depth) => segments.slice(0, depth + 1).join('/'));
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* The key in the way of storing `key`, spelled as the local disk's `X_STORAGE_KEY_CONFLICT` spells
|
|
19
|
+
* it (`blocking`), or `undefined`. Checked in `claim()`'s order — the object path, its sidecar,
|
|
20
|
+
* its pending marker — each first for a FILE where a directory must be, then for a DIRECTORY
|
|
21
|
+
* where the file must go. A marker is transient (cleared on commit), so it is never in the way of
|
|
22
|
+
* a later key; a directory where one must be written still is.
|
|
23
|
+
*/
|
|
24
|
+
export function keyInTheWay(stored: ReadonlyMap<string, unknown>, key: string): string | undefined {
|
|
25
|
+
// Object tree: `a` stored blocks `a/b`; `c/d` stored blocks `c` (reported as `c/`).
|
|
26
|
+
for (const ancestor of ancestorsOf(key)) if (stored.has(ancestor)) return ancestor;
|
|
27
|
+
const beneath = (prefix: string): boolean => {
|
|
28
|
+
for (const other of stored.keys()) if (other.startsWith(prefix)) return true;
|
|
29
|
+
return false;
|
|
30
|
+
};
|
|
31
|
+
if (beneath(`${key}/`)) return `${key}/`;
|
|
32
|
+
// Sidecar tree: `.meta/<k>.json` is a FILE per stored key `k`, and its directories are the
|
|
33
|
+
// object tree's — so `s` stored blocks `s.json/x`, and `t.json/x` stored blocks `t`.
|
|
34
|
+
const sidecar = inSidecarTree(sidecarPathOf(key));
|
|
35
|
+
for (const ancestor of ancestorsOf(sidecar)) {
|
|
36
|
+
if (ancestor.endsWith('.json') && stored.has(ancestor.slice(0, -'.json'.length))) {
|
|
37
|
+
return ancestor.slice(0, -'.json'.length);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
if (beneath(`${sidecar}/`)) return `${sidecar}/`;
|
|
41
|
+
const pending = inSidecarTree(pendingPathOf(key));
|
|
42
|
+
// `<k>.json.pending/` is a directory exactly when a key's sidecar lives beneath it.
|
|
43
|
+
if (beneath(`${pending}/`)) return `${pending}/`;
|
|
44
|
+
return undefined;
|
|
45
|
+
}
|
package/src/driver-memory.ts
CHANGED
|
@@ -1,30 +1,46 @@
|
|
|
1
1
|
// Single responsibility: the disk a TEST holds — a `StorageDriver` over one `Map`, answering every
|
|
2
|
-
// question `localDriver` answers and refusing what it refuses
|
|
3
|
-
// create or clean up. Not a deployment's disk: a
|
|
2
|
+
// question `localDriver` answers and refusing what it refuses (a key that is another key's path
|
|
3
|
+
// included), with nothing on a filesystem to create or clean up. Not a deployment's disk: a
|
|
4
|
+
// restart is every object gone.
|
|
4
5
|
//
|
|
5
6
|
// It exists because the alternative was written by hand in every suite that needed a bucket: a
|
|
6
7
|
// fake with three methods and five `unsupported()` stubs, each a slightly different contract.
|
|
7
8
|
|
|
8
|
-
import {
|
|
9
|
+
import {
|
|
10
|
+
type Clock,
|
|
11
|
+
finiteCount,
|
|
12
|
+
NotImplementedError,
|
|
13
|
+
type Page,
|
|
14
|
+
pageOf,
|
|
15
|
+
systemClock,
|
|
16
|
+
} from '@ultimat3/core';
|
|
9
17
|
import {
|
|
10
18
|
assertListOptions,
|
|
11
19
|
assertPutOptions,
|
|
12
20
|
DEFAULT_CONTENT_TYPE,
|
|
13
21
|
etagOf,
|
|
14
22
|
type ListOptions,
|
|
15
|
-
type ListPage,
|
|
16
23
|
type PutOptions,
|
|
17
24
|
resolveListLimit,
|
|
18
25
|
type SignedUrlOptions,
|
|
19
26
|
type StorageBody,
|
|
20
27
|
type StorageDriver,
|
|
28
|
+
type StorageListEntry,
|
|
21
29
|
type StorageObject,
|
|
22
30
|
type StorageRead,
|
|
23
31
|
sha256Base64,
|
|
24
32
|
toBytes,
|
|
25
33
|
} from './driver';
|
|
26
34
|
import type { LocalDriverOptions } from './driver-local';
|
|
27
|
-
import {
|
|
35
|
+
import { keyInTheWay } from './driver-memory-conflict';
|
|
36
|
+
import { checksumMismatch, getTooLarge, keyConflict, objectNotFound } from './errors';
|
|
37
|
+
import {
|
|
38
|
+
assertObjectLockOptions,
|
|
39
|
+
isLocked,
|
|
40
|
+
type ObjectLock,
|
|
41
|
+
ObjectLockedError,
|
|
42
|
+
requestedLock,
|
|
43
|
+
} from './object-lock';
|
|
28
44
|
import { assertSafeKey } from './path';
|
|
29
45
|
import type { SignedUrlVerification } from './signed-url';
|
|
30
46
|
import { buildSignedUrl, signedUrlBaseFor, verifySignedUrl } from './signed-url';
|
|
@@ -34,7 +50,7 @@ import { DEFAULT_MAX_UPLOAD_BYTES } from './upload';
|
|
|
34
50
|
const DRIVER_NAME = 'memory';
|
|
35
51
|
|
|
36
52
|
/** `localDriver`'s options, minus the directory there is none of. */
|
|
37
|
-
export type
|
|
53
|
+
export type MemoryStorageDriverOptions = Omit<LocalDriverOptions, 'root'>;
|
|
38
54
|
|
|
39
55
|
export interface MemoryStorageDriver extends StorageDriver {
|
|
40
56
|
/**
|
|
@@ -47,6 +63,8 @@ export interface MemoryStorageDriver extends StorageDriver {
|
|
|
47
63
|
interface Stored {
|
|
48
64
|
readonly bytes: Uint8Array;
|
|
49
65
|
readonly object: StorageObject;
|
|
66
|
+
/** The local disk's sidecar lock, held here instead. */
|
|
67
|
+
readonly lock?: ObjectLock | undefined;
|
|
50
68
|
}
|
|
51
69
|
|
|
52
70
|
/**
|
|
@@ -62,7 +80,7 @@ const snapshot = (object: StorageObject): StorageObject => ({
|
|
|
62
80
|
...(object.metadata === undefined ? {} : { metadata: { ...object.metadata } }),
|
|
63
81
|
});
|
|
64
82
|
|
|
65
|
-
export function
|
|
83
|
+
export function memoryStorageDriver(options: MemoryStorageDriverOptions = {}): MemoryStorageDriver {
|
|
66
84
|
const maxPutBytes = finiteCount(
|
|
67
85
|
'the memory disk driver',
|
|
68
86
|
'maxPutBytes',
|
|
@@ -79,10 +97,32 @@ export function memoryDriver(options: MemoryDriverOptions = {}): MemoryStorageDr
|
|
|
79
97
|
let baseUrl = options.baseUrl ?? signedUrlBaseFor(DRIVER_NAME);
|
|
80
98
|
const secret = resolveSigningSecret(DRIVER_NAME, options);
|
|
81
99
|
const stored = new Map<string, Stored>();
|
|
100
|
+
// The name a registry gave this disk, so a miss names `disk('<it>')` — the call the author wrote.
|
|
101
|
+
let registered: string = DRIVER_NAME;
|
|
102
|
+
|
|
103
|
+
/** The lock that holds NOW, or `undefined` — the local disk's rule, read off the map. */
|
|
104
|
+
const lockAt = (key: string): ObjectLock | undefined => {
|
|
105
|
+
const lock = stored.get(key)?.lock;
|
|
106
|
+
return isLocked(lock, clock.now()) ? lock : undefined;
|
|
107
|
+
};
|
|
108
|
+
const refuseOverwrite = (key: string, action: 'overwrite' | 'copy onto'): void => {
|
|
109
|
+
const lock = lockAt(key);
|
|
110
|
+
if (lock !== undefined) throw new ObjectLockedError({ disk: registered, key, action, lock });
|
|
111
|
+
};
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* The local disk's path rule: a key cannot be another key's directory (`X_STORAGE_KEY_CONFLICT`).
|
|
115
|
+
* Emulated, not inherited — a `Map` holds both — because a suite on this disk stands in for the
|
|
116
|
+
* dev disk, and a key layout that passes here and fails in `x dev` is a gap found late.
|
|
117
|
+
*/
|
|
118
|
+
const refuseConflict = (key: string): void => {
|
|
119
|
+
const blocking = keyInTheWay(stored, key);
|
|
120
|
+
if (blocking !== undefined) throw keyConflict(registered, key, blocking);
|
|
121
|
+
};
|
|
82
122
|
|
|
83
123
|
const found = (key: string): Stored => {
|
|
84
124
|
const hit = stored.get(key);
|
|
85
|
-
if (hit === undefined) throw objectNotFound(
|
|
125
|
+
if (hit === undefined) throw objectNotFound(registered, key);
|
|
86
126
|
return hit;
|
|
87
127
|
};
|
|
88
128
|
|
|
@@ -94,6 +134,7 @@ export function memoryDriver(options: MemoryDriverOptions = {}): MemoryStorageDr
|
|
|
94
134
|
},
|
|
95
135
|
|
|
96
136
|
registerAs(diskName: string): void {
|
|
137
|
+
registered = diskName;
|
|
97
138
|
if (options.baseUrl === undefined) baseUrl = signedUrlBaseFor(diskName);
|
|
98
139
|
},
|
|
99
140
|
|
|
@@ -113,6 +154,7 @@ export function memoryDriver(options: MemoryDriverOptions = {}): MemoryStorageDr
|
|
|
113
154
|
});
|
|
114
155
|
}
|
|
115
156
|
assertPutOptions(DRIVER_NAME, putOptions);
|
|
157
|
+
assertObjectLockOptions(DRIVER_NAME, putOptions, clock);
|
|
116
158
|
const bytes = await toBytes(body, { driver: DRIVER_NAME, key: safe, maxBytes: maxPutBytes });
|
|
117
159
|
const claimed = putOptions?.checksum;
|
|
118
160
|
if (claimed !== undefined) {
|
|
@@ -130,8 +172,16 @@ export function memoryDriver(options: MemoryDriverOptions = {}): MemoryStorageDr
|
|
|
130
172
|
: { cacheControl: putOptions.cacheControl }),
|
|
131
173
|
...(putOptions?.metadata === undefined ? {} : { metadata: putOptions.metadata }),
|
|
132
174
|
};
|
|
175
|
+
// Checked AFTER the body is read, with no await between check and write: the map is the
|
|
176
|
+
// queue the local disk needs a `keyedQueue` for.
|
|
177
|
+
refuseConflict(safe);
|
|
178
|
+
refuseOverwrite(safe, 'overwrite');
|
|
133
179
|
// A copy in, as a file write is: the caller's buffer and metadata are the caller's to reuse.
|
|
134
|
-
stored.set(safe, {
|
|
180
|
+
stored.set(safe, {
|
|
181
|
+
bytes: bytes.slice(),
|
|
182
|
+
object: snapshot(object),
|
|
183
|
+
lock: requestedLock(putOptions),
|
|
184
|
+
});
|
|
135
185
|
return snapshot(object);
|
|
136
186
|
},
|
|
137
187
|
|
|
@@ -160,6 +210,8 @@ export function memoryDriver(options: MemoryDriverOptions = {}): MemoryStorageDr
|
|
|
160
210
|
// refusal whether or not the source exists.
|
|
161
211
|
const [from_, destination] = [assertSafeKey(from), assertSafeKey(to)];
|
|
162
212
|
const source = found(from_);
|
|
213
|
+
refuseConflict(destination);
|
|
214
|
+
refuseOverwrite(destination, 'copy onto');
|
|
163
215
|
const object: StorageObject = {
|
|
164
216
|
...source.object,
|
|
165
217
|
key: destination,
|
|
@@ -170,18 +222,28 @@ export function memoryDriver(options: MemoryDriverOptions = {}): MemoryStorageDr
|
|
|
170
222
|
},
|
|
171
223
|
|
|
172
224
|
async delete(key: string): Promise<void> {
|
|
173
|
-
|
|
225
|
+
const safe = assertSafeKey(key);
|
|
226
|
+
const lock = lockAt(safe);
|
|
227
|
+
if (lock !== undefined) {
|
|
228
|
+
throw new ObjectLockedError({ disk: registered, key: safe, action: 'delete', lock });
|
|
229
|
+
}
|
|
230
|
+
stored.delete(safe);
|
|
231
|
+
},
|
|
232
|
+
|
|
233
|
+
async retentionOf(key: string): Promise<ObjectLock> {
|
|
234
|
+
const lock = found(assertSafeKey(key)).lock;
|
|
235
|
+
return lock === undefined ? { legalHold: false } : structuredClone(lock);
|
|
174
236
|
},
|
|
175
237
|
|
|
176
238
|
async exists(key: string): Promise<boolean> {
|
|
177
239
|
return stored.has(assertSafeKey(key));
|
|
178
240
|
},
|
|
179
241
|
|
|
180
|
-
async list(listOptions?: ListOptions): Promise<
|
|
242
|
+
async list(listOptions?: ListOptions): Promise<Page<StorageListEntry>> {
|
|
181
243
|
const prefix = listOptions?.prefix ?? '';
|
|
182
244
|
assertListOptions(listOptions);
|
|
183
245
|
const limit = resolveListLimit(listOptions?.limit);
|
|
184
|
-
const cursor = listOptions?.cursor;
|
|
246
|
+
const cursor = listOptions?.cursor ?? undefined;
|
|
185
247
|
// Code-unit order and "the cursor is the last key of the page before" — the local disk's own.
|
|
186
248
|
const keys = [...stored.keys()]
|
|
187
249
|
.filter((key) => key.startsWith(prefix) && (cursor === undefined || key > cursor))
|
|
@@ -191,10 +253,7 @@ export function memoryDriver(options: MemoryDriverOptions = {}): MemoryStorageDr
|
|
|
191
253
|
const hit = stored.get(key);
|
|
192
254
|
return hit === undefined ? [] : [snapshot(hit.object)];
|
|
193
255
|
});
|
|
194
|
-
|
|
195
|
-
return keys.length > page.length && last !== undefined
|
|
196
|
-
? { objects, truncated: true, cursor: last }
|
|
197
|
-
: { objects, truncated: false };
|
|
256
|
+
return pageOf(objects, keys.length > page.length ? (page.at(-1) ?? null) : null);
|
|
198
257
|
},
|
|
199
258
|
|
|
200
259
|
async signedUrl(key: string, urlOptions?: SignedUrlOptions): Promise<string> {
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
// Single responsibility: does a provider failure mean "there is nothing at that key"? Split from
|
|
2
|
+
// `driver-s3.ts` at its line ceiling; the one reader is that driver's delete and read paths.
|
|
3
|
+
|
|
4
|
+
import { stringField } from '@ultimat3/core';
|
|
5
|
+
|
|
6
|
+
/** One numeric field off a value that may fight being read — `stringField`'s missing twin. */
|
|
7
|
+
function numberField(value: unknown, key: string): number | undefined {
|
|
8
|
+
if (typeof value !== 'object' || value === null) return undefined;
|
|
9
|
+
try {
|
|
10
|
+
const held = (value as Record<string, unknown>)[key];
|
|
11
|
+
return typeof held === 'number' ? held : undefined;
|
|
12
|
+
} catch {
|
|
13
|
+
return undefined;
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/** The provider codes that mean "there is nothing at that key", and nothing wider. */
|
|
18
|
+
const ABSENT_OBJECT_CODES: ReadonlySet<string> = new Set(['NoSuchKey', 'NotFound', 'ENOENT']);
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* The ONE delete failure the contract calls success. `AccessDenied`, `SlowDown`, an expired
|
|
22
|
+
* credential and a reset connection are none of them, and the previous `.catch(() => undefined)`
|
|
23
|
+
* reported all four as deleted — which is how an erasure sweep certifies data it never removed.
|
|
24
|
+
*
|
|
25
|
+
* Read structurally: an `S3Error` is `name: 'S3Error'` with a `code` the service returned, and
|
|
26
|
+
* every field of a value this process did not build is a getter that can throw.
|
|
27
|
+
*/
|
|
28
|
+
export function isAbsentObject(error: unknown): boolean {
|
|
29
|
+
const code = stringField(error, 'code');
|
|
30
|
+
if (code !== undefined && ABSENT_OBJECT_CODES.has(code)) return true;
|
|
31
|
+
return numberField(error, 'statusCode') === 404 || numberField(error, 'status') === 404;
|
|
32
|
+
}
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
// Single responsibility: the s3 disk's view of `Bun.S3Client` — the structural types its methods
|
|
2
|
+
// are called through (so `bun-types` stays out of the public contract and a test injects a fake),
|
|
3
|
+
// and the one place the real client is constructed, lazily, from the disk's options.
|
|
4
|
+
|
|
5
|
+
import { ConfigInvalidError } from '@ultimat3/core';
|
|
6
|
+
import type { S3DriverOptions } from './driver-s3';
|
|
7
|
+
import { requireEnv, resolveS3Target } from './driver-s3-signed';
|
|
8
|
+
|
|
9
|
+
/** Structural view of `Bun.S3Client` — typing it here keeps `bun-types` out of the contract. */
|
|
10
|
+
export interface S3FileLike {
|
|
11
|
+
/** `S3FileLike` is in the union because Bun's own `write` takes an `S3File` — that is `copy`. */
|
|
12
|
+
write(data: Uint8Array | Blob | S3FileLike, options?: { type?: string }): Promise<number>;
|
|
13
|
+
arrayBuffer(): Promise<ArrayBuffer>;
|
|
14
|
+
exists(): Promise<boolean>;
|
|
15
|
+
delete(): Promise<void>;
|
|
16
|
+
stream(): ReadableStream<Uint8Array>;
|
|
17
|
+
stat(): Promise<S3StatLike>;
|
|
18
|
+
presign(options: { method?: string; expiresIn?: number; type?: string }): string;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export interface S3StatLike {
|
|
22
|
+
readonly size: number;
|
|
23
|
+
readonly type?: string | undefined;
|
|
24
|
+
readonly etag?: string | undefined;
|
|
25
|
+
readonly lastModified?: string | Date | undefined;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export interface S3ListEntryLike {
|
|
29
|
+
readonly key?: string | undefined;
|
|
30
|
+
readonly size?: number | undefined;
|
|
31
|
+
readonly eTag?: string | undefined;
|
|
32
|
+
readonly lastModified?: string | Date | undefined;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export interface S3ListResultLike {
|
|
36
|
+
readonly contents?: readonly S3ListEntryLike[] | undefined;
|
|
37
|
+
readonly isTruncated?: boolean | undefined;
|
|
38
|
+
readonly nextContinuationToken?: string | undefined;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export interface S3ClientLike {
|
|
42
|
+
file(key: string): S3FileLike;
|
|
43
|
+
list(input: {
|
|
44
|
+
prefix?: string;
|
|
45
|
+
maxKeys?: number;
|
|
46
|
+
continuationToken?: string;
|
|
47
|
+
}): Promise<S3ListResultLike>;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
interface S3ClientConstructor {
|
|
51
|
+
new (options: Record<string, unknown>): S3ClientLike;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Everything but the key pair `Bun.S3Client` is built with — the SAME `resolveS3Target` the signed
|
|
56
|
+
* path reads, handed over explicitly so Bun never consults a second table.
|
|
57
|
+
*/
|
|
58
|
+
export function clientTarget(options: S3DriverOptions): Record<string, unknown> {
|
|
59
|
+
const target = resolveS3Target(options);
|
|
60
|
+
// Bun's flag is the inverse: path style means "not virtual hosted".
|
|
61
|
+
const pathStyle = options.forcePathStyle;
|
|
62
|
+
return {
|
|
63
|
+
bucket: options.bucket,
|
|
64
|
+
...(target.region === undefined ? {} : { region: target.region }),
|
|
65
|
+
...(target.endpoint === undefined ? {} : { endpoint: target.endpoint }),
|
|
66
|
+
...(pathStyle === undefined ? {} : { virtualHostedStyle: !pathStyle }),
|
|
67
|
+
...(target.sessionToken === undefined ? {} : { sessionToken: target.sessionToken }),
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export function buildClient(options: S3DriverOptions): S3ClientLike {
|
|
72
|
+
if (options.client !== undefined) return options.client;
|
|
73
|
+
if (options.bucket === '') {
|
|
74
|
+
throw new ConfigInvalidError({
|
|
75
|
+
cause: 's3 disk was defined without a bucket',
|
|
76
|
+
fix: 'set storage.disks.<name>.bucket in app.config.ts',
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
const env = options.env ?? process.env;
|
|
80
|
+
const idVar = options.accessKeyIdEnv ?? 'S3_ACCESS_KEY_ID';
|
|
81
|
+
const secretVar = options.secretAccessKeyEnv ?? 'S3_SECRET_ACCESS_KEY';
|
|
82
|
+
const Client = (Bun as unknown as { S3Client?: S3ClientConstructor }).S3Client;
|
|
83
|
+
if (Client === undefined) {
|
|
84
|
+
throw new ConfigInvalidError({
|
|
85
|
+
cause: 'Bun.S3Client is unavailable in this runtime',
|
|
86
|
+
fix: 'upgrade the runtime: bun upgrade # the s3 disk needs bun >= 1.3',
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
return new Client({
|
|
90
|
+
accessKeyId: requireEnv(env, idVar, secretVar),
|
|
91
|
+
secretAccessKey: requireEnv(env, secretVar, idVar),
|
|
92
|
+
...clientTarget(options),
|
|
93
|
+
});
|
|
94
|
+
}
|