@ultimat3/storage 23.0.0 → 25.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +69 -42
- package/README.md +165 -15
- package/package.json +3 -3
- package/src/accept.ts +3 -6
- package/src/attachment.ts +46 -16
- package/src/driver-local-read.ts +114 -0
- package/src/driver-local-sidecar.ts +96 -0
- package/src/driver-local-write.ts +185 -0
- package/src/driver-local.ts +166 -188
- package/src/driver-memory-conflict.ts +45 -0
- package/src/driver-memory.ts +109 -25
- package/src/driver-s3-absent.ts +32 -0
- package/src/driver-s3-client.ts +94 -0
- package/src/driver-s3-lock.ts +131 -0
- package/src/driver-s3-signed.ts +220 -0
- package/src/driver-s3.ts +180 -167
- package/src/driver.ts +98 -19
- package/src/errors.ts +112 -13
- package/src/grant.ts +2 -2
- package/src/image.ts +34 -8
- package/src/index.ts +29 -16
- package/src/iso-bmff.ts +50 -0
- package/src/object-lock.ts +173 -0
- package/src/path.ts +17 -2
- package/src/signed-url.ts +41 -12
- package/src/signing-secret.ts +76 -0
- package/src/upload.ts +82 -5
package/src/driver-local.ts
CHANGED
|
@@ -6,17 +6,20 @@
|
|
|
6
6
|
import {
|
|
7
7
|
type Clock,
|
|
8
8
|
finiteCount,
|
|
9
|
-
|
|
9
|
+
NotImplementedError,
|
|
10
|
+
type Page,
|
|
11
|
+
pageOf,
|
|
10
12
|
type ResolveEnvironmentOptions,
|
|
11
|
-
|
|
13
|
+
renderFixShellArg,
|
|
12
14
|
stringField,
|
|
13
15
|
systemClock,
|
|
14
16
|
} from '@ultimat3/core';
|
|
15
17
|
import {
|
|
18
|
+
assertListOptions,
|
|
19
|
+
assertPutOptions,
|
|
16
20
|
DEFAULT_CONTENT_TYPE,
|
|
17
21
|
etagOf,
|
|
18
22
|
type ListOptions,
|
|
19
|
-
type ListPage,
|
|
20
23
|
type PutOptions,
|
|
21
24
|
resolveListLimit,
|
|
22
25
|
type SignedUrlOptions,
|
|
@@ -28,52 +31,25 @@ import {
|
|
|
28
31
|
sha256Base64,
|
|
29
32
|
toBytes,
|
|
30
33
|
} from './driver';
|
|
34
|
+
import { etagOfFile, headObject, readObjectBytes, readSidecar } from './driver-local-read';
|
|
35
|
+
import { lockOfSidecar, type Sidecar, sidecarLock } from './driver-local-sidecar';
|
|
36
|
+
import { commitObject, keyedQueue, pendingPathOf, sidecarPathOf } from './driver-local-write';
|
|
37
|
+
import { checksumMismatch, deleteFailed, getTooLarge, listFailed, objectNotFound } from './errors';
|
|
31
38
|
import {
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
} from './errors';
|
|
39
|
+
assertObjectLockOptions,
|
|
40
|
+
isLocked,
|
|
41
|
+
type ObjectLock,
|
|
42
|
+
ObjectLockedError,
|
|
43
|
+
requestedLock,
|
|
44
|
+
} from './object-lock';
|
|
39
45
|
import { assertSafeKey, META_DIR } from './path';
|
|
40
46
|
import type { SignedUrlVerification } from './signed-url';
|
|
41
47
|
import { buildSignedUrl, signedUrlBaseFor, verifySignedUrl } from './signed-url';
|
|
48
|
+
import { resolveSigningSecret } from './signing-secret';
|
|
42
49
|
import { DEFAULT_MAX_UPLOAD_BYTES } from './upload';
|
|
43
50
|
|
|
44
51
|
const DRIVER_NAME = 'local';
|
|
45
52
|
|
|
46
|
-
/**
|
|
47
|
-
* The dev-only fallback signing key. A literal, not a per-process random one, so a restart does
|
|
48
|
-
* not invalidate every URL `x dev` handed out — and published in this repo, which is exactly why
|
|
49
|
-
* `localDriver` refuses to use it outside a development or test environment.
|
|
50
|
-
*/
|
|
51
|
-
export const DEV_SIGNING_SECRET = 'ultimate-dev-signing-secret';
|
|
52
|
-
|
|
53
|
-
/** The env key production must set. Named once, read by the driver and by the predicate below. */
|
|
54
|
-
export const STORAGE_SIGNING_SECRET_KEY = 'STORAGE_SIGNING_SECRET';
|
|
55
|
-
|
|
56
|
-
/**
|
|
57
|
-
* True while a local disk built without an explicit `signingSecret` would sign with the shipped
|
|
58
|
-
* development key — `x doctor` reports it, exactly as it reports `usesDevCursorSecret()`.
|
|
59
|
-
*
|
|
60
|
-
* Reads the environment, not a driver instance: this is the same question `x doctor` asks about
|
|
61
|
-
* the cursor secret, and a disk handed an explicit `signingSecret` in `app.config.ts` never
|
|
62
|
-
* consults the variable at all.
|
|
63
|
-
*
|
|
64
|
-
* `env` is core's own slot, so this half of the guard reads the SAME table its other half does:
|
|
65
|
-
* `dev-runtime.ts` asks `!isLocal({ env }) && usesDevStorageSecret({ env })`, and an embedding
|
|
66
|
-
* caller (`serveApp({ env })`, a test fixture) whose `env` is not `process.env` used to get one
|
|
67
|
-
* answer about the boot and one about the process — for the decision of whether a disk may be
|
|
68
|
-
* signed with the published development key. Defaulted to `process.env`, so a bare call is
|
|
69
|
-
* unchanged.
|
|
70
|
-
*/
|
|
71
|
-
export function usesDevStorageSecret(options?: Pick<ResolveEnvironmentOptions, 'env'>): boolean {
|
|
72
|
-
const source = options?.env ?? (process.env as Record<string, string | undefined>);
|
|
73
|
-
const configured = source[STORAGE_SIGNING_SECRET_KEY];
|
|
74
|
-
return configured === undefined || configured === '' || configured === DEV_SIGNING_SECRET;
|
|
75
|
-
}
|
|
76
|
-
|
|
77
53
|
export interface LocalDriverOptions {
|
|
78
54
|
/** Directory the disk owns outright. Created on first write. */
|
|
79
55
|
readonly root: string;
|
|
@@ -104,72 +80,11 @@ export interface LocalDriverOptions {
|
|
|
104
80
|
* discovers by being OOM-killed.
|
|
105
81
|
*/
|
|
106
82
|
readonly maxPutBytes?: number | undefined;
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
readonly
|
|
112
|
-
readonly cacheControl?: string | undefined;
|
|
113
|
-
readonly metadata?: Readonly<Record<string, string>> | undefined;
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
// `!Array.isArray` is the load-bearing clause, matching `isPlainObject` in
|
|
117
|
-
// `@ultimat3/schema`'s `builder.ts`: `typeof [] === 'object'` and every value of `['a','b']` is a
|
|
118
|
-
// string, so an array in the `metadata` slot was handed back through `head()`/`get()` as object
|
|
119
|
-
// metadata — against a `Record<string, string>` every reader downstream is typed on.
|
|
120
|
-
const isStringRecord = (value: unknown): value is Readonly<Record<string, string>> =>
|
|
121
|
-
typeof value === 'object' &&
|
|
122
|
-
value !== null &&
|
|
123
|
-
!Array.isArray(value) &&
|
|
124
|
-
Object.values(value as Record<string, unknown>).every((entry) => typeof entry === 'string');
|
|
125
|
-
|
|
126
|
-
function parseSidecar(raw: unknown): Sidecar | undefined {
|
|
127
|
-
if (typeof raw !== 'object' || raw === null) return undefined;
|
|
128
|
-
const record = raw as Record<string, unknown>;
|
|
129
|
-
const contentType = record['contentType'];
|
|
130
|
-
const etag = record['etag'];
|
|
131
|
-
if (typeof contentType !== 'string' || typeof etag !== 'string') return undefined;
|
|
132
|
-
// `put()` writes cacheControl/metadata into the same sidecar (below) — dropping them here
|
|
133
|
-
// silently truncated what was just written, even though `Sidecar` itself declares both.
|
|
134
|
-
const cacheControl = record['cacheControl'];
|
|
135
|
-
const metadata = record['metadata'];
|
|
136
|
-
return {
|
|
137
|
-
contentType,
|
|
138
|
-
etag,
|
|
139
|
-
...(typeof cacheControl === 'string' ? { cacheControl } : {}),
|
|
140
|
-
...(isStringRecord(metadata) ? { metadata } : {}),
|
|
141
|
-
};
|
|
142
|
-
}
|
|
143
|
-
|
|
144
|
-
/**
|
|
145
|
-
* The secret a disk that mints its OWN URLs signs with — `localDriver` and `memoryDriver` both.
|
|
146
|
-
*
|
|
147
|
-
* A dev disk must work with zero config. Outside development the fallback is refused rather than
|
|
148
|
-
* used: the literal is published, so signing with it hands every reader the power to mint a PUT
|
|
149
|
-
* for any key with any size and type limit — which `acceptSignedUpload` then trusts over the app's
|
|
150
|
-
* own `uploadPolicy`. Refused at construction, so the boot fails rather than the first upload.
|
|
151
|
-
*
|
|
152
|
-
* The published literal counts as no secret at all, whichever way it arrives: an env var or an
|
|
153
|
-
* `app.config.ts` that pasted it in signs exactly as weakly as the fallback does. One table for
|
|
154
|
-
* all three reads — the secret, the environment test and the environment the refusal names.
|
|
155
|
-
* Splitting them is how the guard and the disk came to answer about two different processes.
|
|
156
|
-
*/
|
|
157
|
-
export function resolveSigningSecret(
|
|
158
|
-
disk: string,
|
|
159
|
-
options: Pick<LocalDriverOptions, 'signingSecret' | 'env'>,
|
|
160
|
-
): string {
|
|
161
|
-
const env = options.env ?? (process.env as Record<string, string | undefined>);
|
|
162
|
-
const supplied = options.signingSecret ?? env[STORAGE_SIGNING_SECRET_KEY];
|
|
163
|
-
const configured =
|
|
164
|
-
supplied === undefined || supplied === '' || supplied === DEV_SIGNING_SECRET
|
|
165
|
-
? undefined
|
|
166
|
-
: supplied;
|
|
167
|
-
// FAILS CLOSED: a process that names no environment resolves as `production` here, the answer
|
|
168
|
-
// core's `assertNoDevSecretsOutsideLocal` gives. `isLocal`'s own fallback is `development`, so the
|
|
169
|
-
// process that forgot to say signed with the published key — exactly the one that must not.
|
|
170
|
-
if (configured === undefined && !isLocal({ env, fallback: 'production' }))
|
|
171
|
-
throw signingSecretMissing(resolveEnvironment({ env, fallback: 'production' }), disk);
|
|
172
|
-
return configured ?? DEV_SIGNING_SECRET;
|
|
83
|
+
/**
|
|
84
|
+
* Ceiling on ONE `get()`, because `get()` buffers the whole object too — and an object's size
|
|
85
|
+
* is the uploader's, not this process's. Defaults to `maxPutBytes`; past it, `stream()`.
|
|
86
|
+
*/
|
|
87
|
+
readonly maxGetBytes?: number | undefined;
|
|
173
88
|
}
|
|
174
89
|
|
|
175
90
|
/**
|
|
@@ -180,15 +95,30 @@ export function resolveSigningSecret(
|
|
|
180
95
|
*/
|
|
181
96
|
function refuseUnsupportedPut(putOptions?: PutOptions): void {
|
|
182
97
|
if (putOptions?.serverSideEncryption === undefined) return;
|
|
183
|
-
throw
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
98
|
+
throw new NotImplementedError({
|
|
99
|
+
cause:
|
|
100
|
+
'server-side encryption on the local driver (it writes plain files under one root) is not implemented by this driver — drop serverSideEncryption from put() and encrypt the disk itself: an s3Driver over a bucket with a default KMS rule, or a LUKS/FileVault volume under root',
|
|
101
|
+
fix: 'put(key, body)',
|
|
102
|
+
});
|
|
187
103
|
}
|
|
188
104
|
|
|
189
105
|
/** `ENOENT` is the one delete failure that means "already in the desired state". */
|
|
190
106
|
const isMissingFile = (error: unknown): boolean => stringField(error, 'code') === 'ENOENT';
|
|
191
107
|
|
|
108
|
+
/**
|
|
109
|
+
* A directory at an object's path is OTHER keys' parent (`c` beside `c/d`), never an object: no
|
|
110
|
+
* bytes are stored at the key, so deleting it is deleting an absent key. Asked only after an
|
|
111
|
+
* unlink failed, because the errno for "that is a directory" is `EISDIR` on Linux and `EPERM` on
|
|
112
|
+
* macOS — the stat is the portable answer.
|
|
113
|
+
*/
|
|
114
|
+
const isDirectory = async (path: string): Promise<boolean> =>
|
|
115
|
+
Bun.file(path)
|
|
116
|
+
.stat()
|
|
117
|
+
.then(
|
|
118
|
+
(stats) => stats.isDirectory(),
|
|
119
|
+
() => false,
|
|
120
|
+
);
|
|
121
|
+
|
|
192
122
|
export function localDriver(options: LocalDriverOptions): StorageDriver {
|
|
193
123
|
const root = options.root.replace(/\/+$/, '');
|
|
194
124
|
// `=== undefined`, never `??`: `??` coalesces on `null` too, so an explicitly blanked key in a
|
|
@@ -199,7 +129,16 @@ export function localDriver(options: LocalDriverOptions): StorageDriver {
|
|
|
199
129
|
options.maxPutBytes === undefined ? DEFAULT_MAX_UPLOAD_BYTES : options.maxPutBytes,
|
|
200
130
|
1,
|
|
201
131
|
);
|
|
132
|
+
const maxGetBytes = finiteCount(
|
|
133
|
+
'the local disk driver',
|
|
134
|
+
'maxGetBytes',
|
|
135
|
+
options.maxGetBytes === undefined ? maxPutBytes : options.maxGetBytes,
|
|
136
|
+
1,
|
|
137
|
+
);
|
|
202
138
|
const clock = options.clock ?? systemClock;
|
|
139
|
+
const oneAtATime = keyedQueue();
|
|
140
|
+
// What `defineStorage` registered this driver as — the name a refusal's `disk('…')` must use.
|
|
141
|
+
let registered = DRIVER_NAME;
|
|
203
142
|
// The segment is the disk's REGISTERED name, learned from `defineStorage` at boot — the driver
|
|
204
143
|
// kind is not a mount point, and minting under it made every disk not literally named `local`
|
|
205
144
|
// 404 its own URLs. An explicit `baseUrl` outranks the registration: that is the operator
|
|
@@ -208,44 +147,31 @@ export function localDriver(options: LocalDriverOptions): StorageDriver {
|
|
|
208
147
|
const secret = resolveSigningSecret(DRIVER_NAME, options);
|
|
209
148
|
|
|
210
149
|
const filePath = (key: string): string => `${root}/${key}`;
|
|
211
|
-
const metaPath = (key: string): string => `${root}/${
|
|
150
|
+
const metaPath = (key: string): string => `${root}/${sidecarPathOf(key)}`;
|
|
212
151
|
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
}
|
|
222
|
-
};
|
|
152
|
+
// Both QUEUED behind the key's writers, as `get()` is: unqueued, a measurement read the size of
|
|
153
|
+
// one generation before a commit and the sidecar of the next after it.
|
|
154
|
+
/** A listing's view: no bytes read, so a pair in doubt reports no type and no etag. */
|
|
155
|
+
const head = (key: string): Promise<StorageListEntry | undefined> =>
|
|
156
|
+
oneAtATime(key, () => headObject(root, key));
|
|
157
|
+
/** The same, settled against the bytes on disk — streamed through a hasher, never buffered. */
|
|
158
|
+
const measured = (key: string): Promise<StorageListEntry | undefined> =>
|
|
159
|
+
oneAtATime(key, () => headObject(root, key, () => etagOfFile(root, key, registered)));
|
|
223
160
|
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
const sidecar = await readSidecar(key);
|
|
239
|
-
const etag = sidecar?.etag ?? (hash ? etagOf(new Uint8Array(await file.arrayBuffer())) : '');
|
|
240
|
-
return {
|
|
241
|
-
key,
|
|
242
|
-
size: file.size,
|
|
243
|
-
etag,
|
|
244
|
-
lastModified: new Date(file.lastModified),
|
|
245
|
-
...(sidecar?.contentType === undefined ? {} : { contentType: sidecar.contentType }),
|
|
246
|
-
...(sidecar?.cacheControl === undefined ? {} : { cacheControl: sidecar.cacheControl }),
|
|
247
|
-
...(sidecar?.metadata === undefined ? {} : { metadata: sidecar.metadata }),
|
|
248
|
-
};
|
|
161
|
+
/**
|
|
162
|
+
* The lock the sidecar RECORDS, read whole even when the pair is in doubt: a torn write never
|
|
163
|
+
* unlocks an object. `undefined` when nothing on disk locks it now.
|
|
164
|
+
*/
|
|
165
|
+
const lockAt = async (key: string): Promise<ObjectLock | undefined> => {
|
|
166
|
+
const lock = lockOfSidecar(await readSidecar(root, key));
|
|
167
|
+
return isLocked(lock, clock.now()) ? lock : undefined;
|
|
168
|
+
};
|
|
169
|
+
/** Called INSIDE the key's queue, so no writer of this process lands between check and commit. */
|
|
170
|
+
const refuseOverwrite = async (key: string, action: 'overwrite' | 'copy onto'): Promise<void> => {
|
|
171
|
+
const lock = await lockAt(key);
|
|
172
|
+
if (lock !== undefined) {
|
|
173
|
+
throw new ObjectLockedError({ disk: registered, key, action, lock, sidecar: metaPath(key) });
|
|
174
|
+
}
|
|
249
175
|
};
|
|
250
176
|
|
|
251
177
|
/** Removes one path, or reports WHY it could not — a swallowed refusal is a false erasure. */
|
|
@@ -253,12 +179,13 @@ export function localDriver(options: LocalDriverOptions): StorageDriver {
|
|
|
253
179
|
try {
|
|
254
180
|
await Bun.file(path).delete();
|
|
255
181
|
} catch (error) {
|
|
256
|
-
if (isMissingFile(error)) return;
|
|
182
|
+
if (isMissingFile(error) || (await isDirectory(path))) return;
|
|
257
183
|
throw deleteFailed(
|
|
258
184
|
DRIVER_NAME,
|
|
259
185
|
key,
|
|
260
186
|
error,
|
|
261
|
-
|
|
187
|
+
// Screened: a key may carry `$(…)` and a root is configuration, and a fix: is pasted.
|
|
188
|
+
`make the disk root writable by this process, then retry: ls -ld ${renderFixShellArg(root, "'<the disk root>'")} && rm -f ${renderFixShellArg(path, "'<the object file>'")}`,
|
|
262
189
|
);
|
|
263
190
|
}
|
|
264
191
|
};
|
|
@@ -272,58 +199,75 @@ export function localDriver(options: LocalDriverOptions): StorageDriver {
|
|
|
272
199
|
},
|
|
273
200
|
|
|
274
201
|
registerAs(diskName: string): void {
|
|
202
|
+
registered = diskName;
|
|
275
203
|
if (options.baseUrl === undefined) baseUrl = signedUrlBaseFor(diskName);
|
|
276
204
|
},
|
|
277
205
|
|
|
278
206
|
async put(key: string, body: StorageBody, putOptions?: PutOptions): Promise<StorageObject> {
|
|
279
207
|
const safe = assertSafeKey(key);
|
|
280
208
|
refuseUnsupportedPut(putOptions);
|
|
209
|
+
assertPutOptions(DRIVER_NAME, putOptions);
|
|
210
|
+
assertObjectLockOptions(DRIVER_NAME, putOptions, clock);
|
|
281
211
|
const bytes = await toBytes(body, { driver: DRIVER_NAME, key: safe, maxBytes: maxPutBytes });
|
|
282
212
|
const claimed = putOptions?.checksum;
|
|
283
213
|
if (claimed !== undefined) {
|
|
284
214
|
const actual = sha256Base64(bytes);
|
|
285
215
|
if (claimed !== actual) throw checksumMismatch(safe, claimed, actual);
|
|
286
216
|
}
|
|
217
|
+
const lastModified = clock.now();
|
|
287
218
|
const sidecar: Sidecar = {
|
|
288
219
|
contentType: putOptions?.contentType ?? DEFAULT_CONTENT_TYPE,
|
|
289
220
|
etag: etagOf(bytes),
|
|
290
221
|
cacheControl: putOptions?.cacheControl,
|
|
291
222
|
metadata: putOptions?.metadata,
|
|
223
|
+
// The write's instant, on the injected clock, recorded for every later read to report.
|
|
224
|
+
lastModified: lastModified.toISOString(),
|
|
225
|
+
...sidecarLock(requestedLock(putOptions)),
|
|
292
226
|
};
|
|
293
|
-
await
|
|
294
|
-
|
|
227
|
+
await oneAtATime(safe, async () => {
|
|
228
|
+
await refuseOverwrite(safe, 'overwrite');
|
|
229
|
+
await commitObject({ root, key: safe, disk: registered, body: bytes, sidecar });
|
|
230
|
+
});
|
|
295
231
|
return {
|
|
296
232
|
key: safe,
|
|
297
233
|
size: bytes.byteLength,
|
|
298
234
|
contentType: sidecar.contentType,
|
|
299
235
|
etag: sidecar.etag,
|
|
300
|
-
lastModified
|
|
236
|
+
lastModified,
|
|
301
237
|
...(sidecar.cacheControl === undefined ? {} : { cacheControl: sidecar.cacheControl }),
|
|
302
238
|
...(sidecar.metadata === undefined ? {} : { metadata: sidecar.metadata }),
|
|
303
239
|
};
|
|
304
240
|
},
|
|
305
241
|
|
|
242
|
+
async stat(key: string): Promise<StorageObject | undefined> {
|
|
243
|
+
const entry = await measured(assertSafeKey(key));
|
|
244
|
+
return entry && { ...entry, contentType: entry.contentType ?? DEFAULT_CONTENT_TYPE };
|
|
245
|
+
},
|
|
246
|
+
|
|
306
247
|
async get(key: string): Promise<StorageRead> {
|
|
307
248
|
const safe = assertSafeKey(key);
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
249
|
+
// Queued behind this key's writers: an object is two files, and a read between a put()'s
|
|
250
|
+
// renames is a pair in doubt this process has no need to meet.
|
|
251
|
+
return oneAtATime(safe, async () => {
|
|
252
|
+
const file = Bun.file(filePath(safe));
|
|
253
|
+
if (!(await file.exists())) throw objectNotFound(registered, safe);
|
|
254
|
+
if (file.size > maxGetBytes) throw getTooLarge(DRIVER_NAME, safe, file.size, maxGetBytes);
|
|
255
|
+
const bytes = await readObjectBytes(root, safe, registered);
|
|
256
|
+
// Settled against the bytes this read ALREADY holds: hashed only for a sidecar-less object
|
|
257
|
+
// or a pair in doubt, and then exactly once.
|
|
258
|
+
const entry = await headObject(root, safe, () => etagOf(bytes));
|
|
259
|
+
if (entry === undefined) throw objectNotFound(registered, safe);
|
|
260
|
+
return {
|
|
261
|
+
object: { ...entry, contentType: entry.contentType ?? DEFAULT_CONTENT_TYPE },
|
|
262
|
+
bytes,
|
|
263
|
+
};
|
|
264
|
+
});
|
|
321
265
|
},
|
|
322
266
|
|
|
323
267
|
async stream(key: string): Promise<ReadableStream<Uint8Array>> {
|
|
324
268
|
const safe = assertSafeKey(key);
|
|
325
269
|
const file = Bun.file(filePath(safe));
|
|
326
|
-
if (!(await file.exists())) throw objectNotFound(
|
|
270
|
+
if (!(await file.exists())) throw objectNotFound(registered, safe);
|
|
327
271
|
return file.stream();
|
|
328
272
|
},
|
|
329
273
|
|
|
@@ -331,24 +275,38 @@ export function localDriver(options: LocalDriverOptions): StorageDriver {
|
|
|
331
275
|
async copy(from: string, to: string): Promise<StorageObject> {
|
|
332
276
|
const source = assertSafeKey(from);
|
|
333
277
|
const destination = assertSafeKey(to);
|
|
334
|
-
//
|
|
335
|
-
// durable lie every later `get()` of the copy would
|
|
336
|
-
//
|
|
337
|
-
const entry = await
|
|
338
|
-
if (entry === undefined) throw objectNotFound(
|
|
339
|
-
|
|
278
|
+
// Measured, not merely read: the destination gets a sidecar, and one carrying `etag: ''` or
|
|
279
|
+
// a torn source's borrowed type is a durable lie every later `get()` of the copy would
|
|
280
|
+
// trust. The hash is streamed, and only for a sidecar-less source or a pair in doubt.
|
|
281
|
+
const entry = await measured(source);
|
|
282
|
+
if (entry === undefined) throw objectNotFound(registered, source);
|
|
283
|
+
const lastModified = clock.now();
|
|
340
284
|
const sidecar: Sidecar = {
|
|
341
285
|
contentType: entry.contentType ?? DEFAULT_CONTENT_TYPE,
|
|
342
286
|
etag: entry.etag,
|
|
343
287
|
cacheControl: entry.cacheControl,
|
|
344
288
|
metadata: entry.metadata,
|
|
289
|
+
lastModified: lastModified.toISOString(),
|
|
345
290
|
};
|
|
346
|
-
await
|
|
291
|
+
await oneAtATime(destination, async () => {
|
|
292
|
+
await refuseOverwrite(destination, 'copy onto');
|
|
293
|
+
await commitObject({
|
|
294
|
+
root,
|
|
295
|
+
key: destination,
|
|
296
|
+
disk: registered,
|
|
297
|
+
body: Bun.file(filePath(source)),
|
|
298
|
+
sidecar,
|
|
299
|
+
});
|
|
300
|
+
}).catch(async (error: unknown) => {
|
|
301
|
+
// The source was deleted while this copy waited its turn on the destination.
|
|
302
|
+
if (!(await Bun.file(filePath(source)).exists())) throw objectNotFound(registered, source);
|
|
303
|
+
throw error;
|
|
304
|
+
});
|
|
347
305
|
return {
|
|
348
306
|
...entry,
|
|
349
307
|
key: destination,
|
|
350
308
|
contentType: sidecar.contentType,
|
|
351
|
-
lastModified
|
|
309
|
+
lastModified,
|
|
352
310
|
};
|
|
353
311
|
},
|
|
354
312
|
|
|
@@ -357,18 +315,41 @@ export function localDriver(options: LocalDriverOptions): StorageDriver {
|
|
|
357
315
|
// Idempotent by contract: a missing key is already in the desired state. A REFUSED unlink
|
|
358
316
|
// is not — a read-only mount or a root this process cannot write reports the bytes gone
|
|
359
317
|
// when they are still on disk, which is the one lie an erasure sweep must never repeat.
|
|
360
|
-
await
|
|
361
|
-
|
|
318
|
+
await oneAtATime(safe, async () => {
|
|
319
|
+
const lock = await lockAt(safe);
|
|
320
|
+
if (lock !== undefined) {
|
|
321
|
+
throw new ObjectLockedError({
|
|
322
|
+
disk: registered,
|
|
323
|
+
key: safe,
|
|
324
|
+
action: 'delete',
|
|
325
|
+
lock,
|
|
326
|
+
sidecar: metaPath(safe),
|
|
327
|
+
});
|
|
328
|
+
}
|
|
329
|
+
await removeIfPresent(filePath(safe), safe);
|
|
330
|
+
await removeIfPresent(metaPath(safe), safe);
|
|
331
|
+
await removeIfPresent(`${root}/${pendingPathOf(safe)}`, safe);
|
|
332
|
+
});
|
|
362
333
|
},
|
|
363
334
|
|
|
364
335
|
async exists(key: string): Promise<boolean> {
|
|
365
336
|
return Bun.file(filePath(assertSafeKey(key))).exists();
|
|
366
337
|
},
|
|
367
338
|
|
|
368
|
-
|
|
339
|
+
/** From the sidecar, as recorded — a lapsed retention is still reported, as S3 reports it. */
|
|
340
|
+
async retentionOf(key: string): Promise<ObjectLock> {
|
|
341
|
+
const safe = assertSafeKey(key);
|
|
342
|
+
return oneAtATime(safe, async () => {
|
|
343
|
+
if (!(await Bun.file(filePath(safe)).exists())) throw objectNotFound(registered, safe);
|
|
344
|
+
return lockOfSidecar(await readSidecar(root, safe));
|
|
345
|
+
});
|
|
346
|
+
},
|
|
347
|
+
|
|
348
|
+
async list(listOptions?: ListOptions): Promise<Page<StorageListEntry>> {
|
|
369
349
|
const prefix = listOptions?.prefix ?? '';
|
|
350
|
+
assertListOptions(listOptions);
|
|
370
351
|
const limit = resolveListLimit(listOptions?.limit);
|
|
371
|
-
const cursor = listOptions?.cursor;
|
|
352
|
+
const cursor = listOptions?.cursor ?? undefined;
|
|
372
353
|
const keys: string[] = [];
|
|
373
354
|
try {
|
|
374
355
|
// `dot: true`, and it is load-bearing: without it a glob matches no dot-prefixed entry, so
|
|
@@ -395,7 +376,7 @@ export function localDriver(options: LocalDriverOptions): StorageDriver {
|
|
|
395
376
|
}
|
|
396
377
|
} catch (error) {
|
|
397
378
|
// A disk nobody has written to yet has no directory: an empty listing, not an error.
|
|
398
|
-
if (isMissingFile(error)) return
|
|
379
|
+
if (isMissingFile(error)) return pageOf([], null);
|
|
399
380
|
// Everything else is a refusal, and a bare `catch` reported all of them as "this disk is
|
|
400
381
|
// empty" — `EACCES` on the root, `ENOTDIR` on a root that is a file, an I/O error on the
|
|
401
382
|
// mount. `sweepOrphans` walks `list()`, so that swallow certified an unreadable prefix as
|
|
@@ -405,7 +386,7 @@ export function localDriver(options: LocalDriverOptions): StorageDriver {
|
|
|
405
386
|
DRIVER_NAME,
|
|
406
387
|
prefix,
|
|
407
388
|
error,
|
|
408
|
-
`make the disk root readable by this process, then retry: ls -ld ${root}`,
|
|
389
|
+
`make the disk root readable by this process, then retry: ls -ld ${renderFixShellArg(root, "'<the disk root>'")}`,
|
|
409
390
|
);
|
|
410
391
|
}
|
|
411
392
|
keys.sort();
|
|
@@ -415,14 +396,11 @@ export function localDriver(options: LocalDriverOptions): StorageDriver {
|
|
|
415
396
|
const object = await head(key);
|
|
416
397
|
if (object !== undefined) objects.push(object);
|
|
417
398
|
}
|
|
418
|
-
|
|
419
|
-
// `limit` is a positive integer, so a
|
|
420
|
-
//
|
|
421
|
-
// an empty page reported as complete
|
|
422
|
-
|
|
423
|
-
return truncated && last !== undefined
|
|
424
|
-
? { objects, truncated, cursor: last }
|
|
425
|
-
: { objects, truncated: false };
|
|
399
|
+
// The cursor is minted only when a key PAST the page exists — `keys` is every match, so a
|
|
400
|
+
// full last page answers no cursor. `limit` is a positive integer, so a page with more after
|
|
401
|
+
// it always HAS a last key: the `?? null` is the type's, not a second condition. It used to
|
|
402
|
+
// be one, and `limit: 0` fell through it — an empty page reported as complete.
|
|
403
|
+
return pageOf(objects, keys.length > page.length ? (page.at(-1) ?? null) : null);
|
|
426
404
|
},
|
|
427
405
|
|
|
428
406
|
async signedUrl(key: string, urlOptions?: SignedUrlOptions): Promise<string> {
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
// Single responsibility: which stored key stands in the way of a new one, answered over a set of
|
|
2
|
+
// keys exactly as `driver-local-write.ts`'s `claim()` answers it over a directory tree — so the
|
|
3
|
+
// memory disk refuses the keys the local disk it stands in for cannot hold.
|
|
4
|
+
|
|
5
|
+
import { pendingPathOf, sidecarPathOf } from './driver-local-write';
|
|
6
|
+
import { META_DIR } from './path';
|
|
7
|
+
|
|
8
|
+
/** `.meta/<key>.json` → `<key>.json`: the sidecar tree, read as keys. */
|
|
9
|
+
const inSidecarTree = (relative: string): string => relative.slice(META_DIR.length + 1);
|
|
10
|
+
|
|
11
|
+
/** Every proper ancestor directory of `path`, shallowest first: what `mkdir -p` would make. */
|
|
12
|
+
const ancestorsOf = (path: string): readonly string[] => {
|
|
13
|
+
const segments = path.split('/').slice(0, -1);
|
|
14
|
+
return segments.map((_, depth) => segments.slice(0, depth + 1).join('/'));
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* The key in the way of storing `key`, spelled as the local disk's `X_STORAGE_KEY_CONFLICT` spells
|
|
19
|
+
* it (`blocking`), or `undefined`. Checked in `claim()`'s order — the object path, its sidecar,
|
|
20
|
+
* its pending marker — each first for a FILE where a directory must be, then for a DIRECTORY
|
|
21
|
+
* where the file must go. A marker is transient (cleared on commit), so it is never in the way of
|
|
22
|
+
* a later key; a directory where one must be written still is.
|
|
23
|
+
*/
|
|
24
|
+
export function keyInTheWay(stored: ReadonlyMap<string, unknown>, key: string): string | undefined {
|
|
25
|
+
// Object tree: `a` stored blocks `a/b`; `c/d` stored blocks `c` (reported as `c/`).
|
|
26
|
+
for (const ancestor of ancestorsOf(key)) if (stored.has(ancestor)) return ancestor;
|
|
27
|
+
const beneath = (prefix: string): boolean => {
|
|
28
|
+
for (const other of stored.keys()) if (other.startsWith(prefix)) return true;
|
|
29
|
+
return false;
|
|
30
|
+
};
|
|
31
|
+
if (beneath(`${key}/`)) return `${key}/`;
|
|
32
|
+
// Sidecar tree: `.meta/<k>.json` is a FILE per stored key `k`, and its directories are the
|
|
33
|
+
// object tree's — so `s` stored blocks `s.json/x`, and `t.json/x` stored blocks `t`.
|
|
34
|
+
const sidecar = inSidecarTree(sidecarPathOf(key));
|
|
35
|
+
for (const ancestor of ancestorsOf(sidecar)) {
|
|
36
|
+
if (ancestor.endsWith('.json') && stored.has(ancestor.slice(0, -'.json'.length))) {
|
|
37
|
+
return ancestor.slice(0, -'.json'.length);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
if (beneath(`${sidecar}/`)) return `${sidecar}/`;
|
|
41
|
+
const pending = inSidecarTree(pendingPathOf(key));
|
|
42
|
+
// `<k>.json.pending/` is a directory exactly when a key's sidecar lives beneath it.
|
|
43
|
+
if (beneath(`${pending}/`)) return `${pending}/`;
|
|
44
|
+
return undefined;
|
|
45
|
+
}
|