@ultimat3/storage 22.15.0 → 24.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +46 -7
- package/README.md +119 -8
- package/package.json +3 -2
- package/src/accept.ts +3 -6
- package/src/attachment.ts +40 -7
- package/src/driver-local-read.ts +104 -0
- package/src/driver-local-sidecar.ts +38 -0
- package/src/driver-local-write.ts +185 -0
- package/src/driver-local.ts +87 -164
- package/src/driver-memory.ts +220 -0
- package/src/driver-s3-region.ts +39 -0
- package/src/driver-s3.ts +129 -31
- package/src/driver.ts +63 -1
- package/src/errors.ts +99 -15
- package/src/grant.ts +2 -2
- package/src/image.ts +34 -8
- package/src/index.ts +16 -9
- package/src/iso-bmff.ts +50 -0
- package/src/path.ts +17 -2
- package/src/signed-url.ts +41 -8
- package/src/signing-secret.ts +71 -0
- package/src/upload.ts +31 -5
- package/src/driver-s3-fixture.ts +0 -164
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
// Single responsibility: how the local disk WRITES one object — the bytes, the sidecar and a
|
|
2
|
+
// pending marker, staged under `.meta/.tmp/` and renamed into place in the one order whose every
|
|
3
|
+
// crash point reads as absent, whole or untyped; refused by code when the disk says no.
|
|
4
|
+
|
|
5
|
+
// why: Bun has no `rename` and no `mkdir` — `Bun.write` creates parents and replaces in place,
|
|
6
|
+
// which is exactly the non-atomic write this file exists to stop. Delete both the day Bun ships them.
|
|
7
|
+
import { mkdir, rename } from 'node:fs/promises';
|
|
8
|
+
import { renderFixShellArg, stringField } from '@ultimat3/core';
|
|
9
|
+
import type { Sidecar } from './driver-local-sidecar';
|
|
10
|
+
import { isStorageError, keyConflict, putFailed } from './errors';
|
|
11
|
+
import { META_DIR } from './path';
|
|
12
|
+
|
|
13
|
+
/** Inside `.meta/`, so the listing's one filter already keeps a staged file out of the keys. */
|
|
14
|
+
const STAGING_DIR = `${META_DIR}/.tmp`;
|
|
15
|
+
|
|
16
|
+
export const sidecarPathOf = (key: string): string => `${META_DIR}/${key}.json`;
|
|
17
|
+
/** Beside the sidecar, and never one itself: a sidecar's name ends in `.json`. */
|
|
18
|
+
export const pendingPathOf = (key: string): string => `${META_DIR}/${key}.json.pending`;
|
|
19
|
+
|
|
20
|
+
const codeOf = (error: unknown): string | undefined => stringField(error, 'code');
|
|
21
|
+
/** `mkdir -p` through a file, a rename onto a directory, a write under a file. */
|
|
22
|
+
const isPathConflict = (error: unknown): boolean =>
|
|
23
|
+
['ENOTDIR', 'EEXIST', 'EISDIR', 'ENOTEMPTY'].includes(codeOf(error) ?? '');
|
|
24
|
+
const isAbsent = (error: unknown): boolean => ['ENOENT', 'ENOTDIR'].includes(codeOf(error) ?? '');
|
|
25
|
+
|
|
26
|
+
/** The object KEY a path inside the root stands for — a sidecar's path names its object. */
|
|
27
|
+
const keyAt = (relative: string): string =>
|
|
28
|
+
relative.startsWith(`${META_DIR}/`)
|
|
29
|
+
? relative.slice(META_DIR.length + 1).replace(/\.json(\.pending)?$/, '')
|
|
30
|
+
: relative;
|
|
31
|
+
|
|
32
|
+
/** The first ancestor of `relative` that is a FILE: the stored object standing where a directory must be. */
|
|
33
|
+
async function fileInTheWay(root: string, relative: string): Promise<string | undefined> {
|
|
34
|
+
const segments = relative.split('/').slice(0, -1);
|
|
35
|
+
for (let depth = 1; depth <= segments.length; depth += 1) {
|
|
36
|
+
const ancestor = segments.slice(0, depth).join('/');
|
|
37
|
+
if (await Bun.file(`${root}/${ancestor}`).exists()) return keyAt(ancestor);
|
|
38
|
+
}
|
|
39
|
+
// Nothing INSIDE the root is in the way: the root itself is the problem, and that is the
|
|
40
|
+
// disk refusing the write, not two keys colliding.
|
|
41
|
+
return undefined;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Make `relative` writable as a FILE, or refuse. Runs for both of an object's paths BEFORE either
|
|
46
|
+
* is touched: the sidecar tree collides where the object tree does not (`a` and `a.json/b`), and a
|
|
47
|
+
* put() that wrote its bytes and then failed on its sidecar left an object nobody had recorded.
|
|
48
|
+
*/
|
|
49
|
+
async function claim(write: ObjectWrite, relative: string): Promise<void> {
|
|
50
|
+
const { root, key } = write;
|
|
51
|
+
const target = `${root}/${relative}`;
|
|
52
|
+
try {
|
|
53
|
+
await mkdir(target.slice(0, target.lastIndexOf('/')), { recursive: true });
|
|
54
|
+
} catch (error) {
|
|
55
|
+
const blocking = isPathConflict(error) ? await fileInTheWay(root, relative) : undefined;
|
|
56
|
+
throw blocking === undefined ? error : keyConflict(write.disk, key, blocking);
|
|
57
|
+
}
|
|
58
|
+
let directory = false;
|
|
59
|
+
try {
|
|
60
|
+
directory = (await Bun.file(target).stat()).isDirectory();
|
|
61
|
+
} catch (error) {
|
|
62
|
+
if (!isAbsent(error)) throw error;
|
|
63
|
+
}
|
|
64
|
+
// The keys in the way live UNDER this path: `<key>/…`, or `<key>.json/…` in the sidecar tree.
|
|
65
|
+
const beneath = relative.startsWith(`${META_DIR}/`)
|
|
66
|
+
? relative.slice(META_DIR.length + 1)
|
|
67
|
+
: relative;
|
|
68
|
+
if (directory) throw keyConflict(write.disk, key, `${beneath}/`);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
const discard = async (path: string): Promise<void> => {
|
|
72
|
+
await Bun.file(path)
|
|
73
|
+
.delete()
|
|
74
|
+
.catch(() => undefined);
|
|
75
|
+
};
|
|
76
|
+
|
|
77
|
+
export interface ObjectWrite {
|
|
78
|
+
readonly root: string;
|
|
79
|
+
readonly key: string;
|
|
80
|
+
/** The name the disk was REGISTERED under — what a refusal's `disk('…')` call must name. */
|
|
81
|
+
readonly disk: string;
|
|
82
|
+
/** A `BunFile` is a `Blob`: a copy hands the source file over and no byte crosses the heap. */
|
|
83
|
+
readonly body: Uint8Array | Blob;
|
|
84
|
+
/** `etag` MUST be the body's own: it is what the pending marker announces. */
|
|
85
|
+
readonly sidecar: Sidecar;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** The four mutations a commit makes, in order — the seam a crash test dies at. */
|
|
89
|
+
export interface WriteSteps {
|
|
90
|
+
rename(from: string, to: string): Promise<void>;
|
|
91
|
+
unlink(path: string): Promise<void>;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
const REAL_STEPS: WriteSteps = {
|
|
95
|
+
rename: (from, to) => rename(from, to),
|
|
96
|
+
unlink: async (path) => {
|
|
97
|
+
await Bun.file(path).delete();
|
|
98
|
+
},
|
|
99
|
+
};
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Stage three files, then four steps: (1) rename the pending MARKER, which holds the new etag;
|
|
103
|
+
* (2) rename the sidecar; (3) rename the bytes; (4) remove the marker. Each rename replaces a
|
|
104
|
+
* directory entry whole, so no reader ever opens half a file.
|
|
105
|
+
*
|
|
106
|
+
* The order is chosen by what a crash leaves (`driver-local-crash.test.ts` dies at every step):
|
|
107
|
+
* - before (3), a FRESH key has no bytes file and so no object — absent, never half-present;
|
|
108
|
+
* - before (2), an overwrite still has the previous sidecar over the previous bytes — whole;
|
|
109
|
+
* - between (2) and (3), the new sidecar sits over the OLD bytes, and the marker naming that
|
|
110
|
+
* sidecar's etag is what tells `headObject` not to believe it;
|
|
111
|
+
* - between (3) and (4) the pair is whole and still marked, so a read re-checks it and a listing
|
|
112
|
+
* reports it untyped until the next put.
|
|
113
|
+
* Bytes-first has no such table: its torn overwrite is new bytes under the old type, undetectable.
|
|
114
|
+
*/
|
|
115
|
+
async function commit(write: ObjectWrite, steps: WriteSteps): Promise<void> {
|
|
116
|
+
const sidecarPath = sidecarPathOf(write.key);
|
|
117
|
+
const pendingPath = pendingPathOf(write.key);
|
|
118
|
+
await claim(write, write.key);
|
|
119
|
+
await claim(write, sidecarPath);
|
|
120
|
+
await claim(write, pendingPath);
|
|
121
|
+
const staged = `${write.root}/${STAGING_DIR}/${crypto.randomUUID()}`;
|
|
122
|
+
// A second writer can make the colliding key between `claim` and the rename; only a RENAME's
|
|
123
|
+
// refusal is that, never a failed staging write.
|
|
124
|
+
const settle = async (suffix: string, relative: string): Promise<void> => {
|
|
125
|
+
await steps.rename(`${staged}.${suffix}`, `${write.root}/${relative}`).catch((error) => {
|
|
126
|
+
throw isPathConflict(error) ? keyConflict(write.disk, write.key, write.key) : error;
|
|
127
|
+
});
|
|
128
|
+
};
|
|
129
|
+
try {
|
|
130
|
+
await Bun.write(`${staged}.object`, write.body);
|
|
131
|
+
await Bun.write(`${staged}.sidecar`, JSON.stringify(write.sidecar));
|
|
132
|
+
await Bun.write(`${staged}.pending`, write.sidecar.etag);
|
|
133
|
+
await settle('pending', pendingPath);
|
|
134
|
+
await settle('sidecar', sidecarPath);
|
|
135
|
+
await settle('object', write.key);
|
|
136
|
+
} catch (error) {
|
|
137
|
+
for (const suffix of ['object', 'sidecar', 'pending']) await discard(`${staged}.${suffix}`);
|
|
138
|
+
throw error;
|
|
139
|
+
}
|
|
140
|
+
// The object is COMMITTED: all three renames landed. A marker that will not clear is not a
|
|
141
|
+
// failed put — reporting one told the caller to retry a write every reader can already see.
|
|
142
|
+
// What it leaves is exactly the crash-at-step-4 state, which `headObject` re-checks.
|
|
143
|
+
await steps.unlink(`${write.root}/${pendingPath}`).catch(() => undefined);
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* The local write, coded. Whatever the filesystem refuses with — `EACCES`, `ENOSPC`, `EROFS`, a
|
|
148
|
+
* root that is a file — leaves here as `X_STORAGE_PUT_FAILED` carrying the errno, where it used to
|
|
149
|
+
* be the bare `Error` out of `Bun.write`. A refusal this package already named passes through.
|
|
150
|
+
*/
|
|
151
|
+
export async function commitObject(
|
|
152
|
+
write: ObjectWrite,
|
|
153
|
+
steps: WriteSteps = REAL_STEPS,
|
|
154
|
+
): Promise<void> {
|
|
155
|
+
try {
|
|
156
|
+
await commit(write, steps);
|
|
157
|
+
} catch (error) {
|
|
158
|
+
if (isStorageError(error)) throw error;
|
|
159
|
+
const at = renderFixShellArg(write.root, '<the disk root>');
|
|
160
|
+
throw putFailed(write.disk, write.key, error, `ls -ld ${at} && df -h ${at}`);
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/** Runs `task` after every earlier task queued under the same key has settled. */
|
|
165
|
+
export type KeyedQueue = <T>(key: string, task: () => Promise<T>) => Promise<T>;
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* One writer — or one reader — per key at a time, inside this process. An object is two files, so
|
|
169
|
+
* two unserialised `put()`s of one key interleaved their four writes and left one's bytes under the
|
|
170
|
+
* other's etag; a `get()` between a writer's two renames read the same mismatch.
|
|
171
|
+
*/
|
|
172
|
+
export function keyedQueue(): KeyedQueue {
|
|
173
|
+
const tails = new Map<string, Promise<unknown>>();
|
|
174
|
+
return async <T>(key: string, task: () => Promise<T>): Promise<T> => {
|
|
175
|
+
const result = (tails.get(key) ?? Promise.resolve()).then(task, task);
|
|
176
|
+
const tail = result.catch(() => undefined);
|
|
177
|
+
tails.set(key, tail);
|
|
178
|
+
try {
|
|
179
|
+
return await result;
|
|
180
|
+
} finally {
|
|
181
|
+
// Only the LAST task clears the entry, or a Map of every key ever written is a leak.
|
|
182
|
+
if (tails.get(key) === tail) tails.delete(key);
|
|
183
|
+
}
|
|
184
|
+
};
|
|
185
|
+
}
|
package/src/driver-local.ts
CHANGED
|
@@ -1,18 +1,19 @@
|
|
|
1
1
|
// Single responsibility: the dev-default disk — a real, working driver over `Bun.file` /
|
|
2
|
-
// `Bun.write` rooted at one directory, so `x dev` needs no
|
|
2
|
+
// `Bun.write` rooted at one directory, so `x dev` needs no S3 server and no cloud account.
|
|
3
3
|
// Content type, etag and user metadata live in a sidecar under `.meta/`: a POSIX file has
|
|
4
4
|
// nowhere to keep them, and `get` must round-trip exactly what `put` was handed.
|
|
5
5
|
|
|
6
6
|
import {
|
|
7
7
|
type Clock,
|
|
8
8
|
finiteCount,
|
|
9
|
-
|
|
9
|
+
NotImplementedError,
|
|
10
10
|
type ResolveEnvironmentOptions,
|
|
11
|
-
resolveEnvironment,
|
|
12
11
|
stringField,
|
|
13
12
|
systemClock,
|
|
14
13
|
} from '@ultimat3/core';
|
|
15
14
|
import {
|
|
15
|
+
assertListOptions,
|
|
16
|
+
assertPutOptions,
|
|
16
17
|
DEFAULT_CONTENT_TYPE,
|
|
17
18
|
etagOf,
|
|
18
19
|
type ListOptions,
|
|
@@ -28,52 +29,18 @@ import {
|
|
|
28
29
|
sha256Base64,
|
|
29
30
|
toBytes,
|
|
30
31
|
} from './driver';
|
|
31
|
-
import {
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
objectNotFound,
|
|
36
|
-
signingSecretMissing,
|
|
37
|
-
storageNotImplemented,
|
|
38
|
-
} from './errors';
|
|
32
|
+
import { etagOfFile, headObject, readObjectBytes } from './driver-local-read';
|
|
33
|
+
import type { Sidecar } from './driver-local-sidecar';
|
|
34
|
+
import { commitObject, keyedQueue, pendingPathOf, sidecarPathOf } from './driver-local-write';
|
|
35
|
+
import { checksumMismatch, deleteFailed, getTooLarge, listFailed, objectNotFound } from './errors';
|
|
39
36
|
import { assertSafeKey, META_DIR } from './path';
|
|
40
37
|
import type { SignedUrlVerification } from './signed-url';
|
|
41
38
|
import { buildSignedUrl, signedUrlBaseFor, verifySignedUrl } from './signed-url';
|
|
39
|
+
import { resolveSigningSecret } from './signing-secret';
|
|
42
40
|
import { DEFAULT_MAX_UPLOAD_BYTES } from './upload';
|
|
43
41
|
|
|
44
42
|
const DRIVER_NAME = 'local';
|
|
45
43
|
|
|
46
|
-
/**
|
|
47
|
-
* The dev-only fallback signing key. A literal, not a per-process random one, so a restart does
|
|
48
|
-
* not invalidate every URL `x dev` handed out — and published in this repo, which is exactly why
|
|
49
|
-
* `localDriver` refuses to use it outside a development or test environment.
|
|
50
|
-
*/
|
|
51
|
-
export const DEV_SIGNING_SECRET = 'ultimate-dev-signing-secret';
|
|
52
|
-
|
|
53
|
-
/** The env key production must set. Named once, read by the driver and by the predicate below. */
|
|
54
|
-
export const STORAGE_SIGNING_SECRET_KEY = 'STORAGE_SIGNING_SECRET';
|
|
55
|
-
|
|
56
|
-
/**
|
|
57
|
-
* True while a local disk built without an explicit `signingSecret` would sign with the shipped
|
|
58
|
-
* development key — `x doctor` reports it, exactly as it reports `usesDevCursorSecret()`.
|
|
59
|
-
*
|
|
60
|
-
* Reads the environment, not a driver instance: this is the same question `x doctor` asks about
|
|
61
|
-
* the cursor secret, and a disk handed an explicit `signingSecret` in `app.config.ts` never
|
|
62
|
-
* consults the variable at all.
|
|
63
|
-
*
|
|
64
|
-
* `env` is core's own slot, so this half of the guard reads the SAME table its other half does:
|
|
65
|
-
* `dev-runtime.ts` asks `!isLocal({ env }) && usesDevStorageSecret({ env })`, and an embedding
|
|
66
|
-
* caller (`serveApp({ env })`, a test fixture) whose `env` is not `process.env` used to get one
|
|
67
|
-
* answer about the boot and one about the process — for the decision of whether a disk may be
|
|
68
|
-
* signed with the published development key. Defaulted to `process.env`, so a bare call is
|
|
69
|
-
* unchanged.
|
|
70
|
-
*/
|
|
71
|
-
export function usesDevStorageSecret(options?: Pick<ResolveEnvironmentOptions, 'env'>): boolean {
|
|
72
|
-
const source = options?.env ?? (process.env as Record<string, string | undefined>);
|
|
73
|
-
const configured = source[STORAGE_SIGNING_SECRET_KEY];
|
|
74
|
-
return configured === undefined || configured === '' || configured === DEV_SIGNING_SECRET;
|
|
75
|
-
}
|
|
76
|
-
|
|
77
44
|
export interface LocalDriverOptions {
|
|
78
45
|
/** Directory the disk owns outright. Created on first write. */
|
|
79
46
|
readonly root: string;
|
|
@@ -104,41 +71,11 @@ export interface LocalDriverOptions {
|
|
|
104
71
|
* discovers by being OOM-killed.
|
|
105
72
|
*/
|
|
106
73
|
readonly maxPutBytes?: number | undefined;
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
readonly
|
|
112
|
-
readonly cacheControl?: string | undefined;
|
|
113
|
-
readonly metadata?: Readonly<Record<string, string>> | undefined;
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
// `!Array.isArray` is the load-bearing clause, matching `isPlainObject` in
|
|
117
|
-
// `@ultimat3/schema`'s `builder.ts`: `typeof [] === 'object'` and every value of `['a','b']` is a
|
|
118
|
-
// string, so an array in the `metadata` slot was handed back through `head()`/`get()` as object
|
|
119
|
-
// metadata — against a `Record<string, string>` every reader downstream is typed on.
|
|
120
|
-
const isStringRecord = (value: unknown): value is Readonly<Record<string, string>> =>
|
|
121
|
-
typeof value === 'object' &&
|
|
122
|
-
value !== null &&
|
|
123
|
-
!Array.isArray(value) &&
|
|
124
|
-
Object.values(value as Record<string, unknown>).every((entry) => typeof entry === 'string');
|
|
125
|
-
|
|
126
|
-
function parseSidecar(raw: unknown): Sidecar | undefined {
|
|
127
|
-
if (typeof raw !== 'object' || raw === null) return undefined;
|
|
128
|
-
const record = raw as Record<string, unknown>;
|
|
129
|
-
const contentType = record['contentType'];
|
|
130
|
-
const etag = record['etag'];
|
|
131
|
-
if (typeof contentType !== 'string' || typeof etag !== 'string') return undefined;
|
|
132
|
-
// `put()` writes cacheControl/metadata into the same sidecar (below) — dropping them here
|
|
133
|
-
// silently truncated what was just written, even though `Sidecar` itself declares both.
|
|
134
|
-
const cacheControl = record['cacheControl'];
|
|
135
|
-
const metadata = record['metadata'];
|
|
136
|
-
return {
|
|
137
|
-
contentType,
|
|
138
|
-
etag,
|
|
139
|
-
...(typeof cacheControl === 'string' ? { cacheControl } : {}),
|
|
140
|
-
...(isStringRecord(metadata) ? { metadata } : {}),
|
|
141
|
-
};
|
|
74
|
+
/**
|
|
75
|
+
* Ceiling on ONE `get()`, because `get()` buffers the whole object too — and an object's size
|
|
76
|
+
* is the uploader's, not this process's. Defaults to `maxPutBytes`; past it, `stream()`.
|
|
77
|
+
*/
|
|
78
|
+
readonly maxGetBytes?: number | undefined;
|
|
142
79
|
}
|
|
143
80
|
|
|
144
81
|
/**
|
|
@@ -149,10 +86,11 @@ function parseSidecar(raw: unknown): Sidecar | undefined {
|
|
|
149
86
|
*/
|
|
150
87
|
function refuseUnsupportedPut(putOptions?: PutOptions): void {
|
|
151
88
|
if (putOptions?.serverSideEncryption === undefined) return;
|
|
152
|
-
throw
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
89
|
+
throw new NotImplementedError({
|
|
90
|
+
cause:
|
|
91
|
+
'server-side encryption on the local driver (it writes plain files under one root) is not implemented by this driver — drop serverSideEncryption from put() and encrypt the disk itself: an s3Driver over a bucket with a default KMS rule, or a LUKS/FileVault volume under root',
|
|
92
|
+
fix: 'put(key, body)',
|
|
93
|
+
});
|
|
156
94
|
}
|
|
157
95
|
|
|
158
96
|
/** `ENOENT` is the one delete failure that means "already in the desired state". */
|
|
@@ -168,75 +106,34 @@ export function localDriver(options: LocalDriverOptions): StorageDriver {
|
|
|
168
106
|
options.maxPutBytes === undefined ? DEFAULT_MAX_UPLOAD_BYTES : options.maxPutBytes,
|
|
169
107
|
1,
|
|
170
108
|
);
|
|
109
|
+
const maxGetBytes = finiteCount(
|
|
110
|
+
'the local disk driver',
|
|
111
|
+
'maxGetBytes',
|
|
112
|
+
options.maxGetBytes === undefined ? maxPutBytes : options.maxGetBytes,
|
|
113
|
+
1,
|
|
114
|
+
);
|
|
171
115
|
const clock = options.clock ?? systemClock;
|
|
116
|
+
const oneAtATime = keyedQueue();
|
|
117
|
+
// What `defineStorage` registered this driver as — the name a refusal's `disk('…')` must use.
|
|
118
|
+
let registered = DRIVER_NAME;
|
|
172
119
|
// The segment is the disk's REGISTERED name, learned from `defineStorage` at boot — the driver
|
|
173
120
|
// kind is not a mount point, and minting under it made every disk not literally named `local`
|
|
174
121
|
// 404 its own URLs. An explicit `baseUrl` outranks the registration: that is the operator
|
|
175
122
|
// stating where the route is mounted, and inference must not overwrite a decision.
|
|
176
123
|
let baseUrl = options.baseUrl ?? signedUrlBaseFor(DRIVER_NAME);
|
|
177
|
-
|
|
178
|
-
// than used: the literal is published, so signing with it hands every reader the power to mint
|
|
179
|
-
// a PUT for any key with any size and type limit — which `acceptSignedUpload` then trusts over
|
|
180
|
-
// the app's own `uploadPolicy`. Refused HERE, at construction, so the boot fails rather than
|
|
181
|
-
// the first upload.
|
|
182
|
-
// The published literal counts as no secret at all, whichever way it arrives: an env var or an
|
|
183
|
-
// `app.config.ts` that pasted it in signs exactly as weakly as the fallback does.
|
|
184
|
-
// One table for all three reads — the secret, the environment test and the environment the
|
|
185
|
-
// refusal names. Splitting them is how the guard and the disk came to answer about two
|
|
186
|
-
// different processes.
|
|
187
|
-
const env = options.env ?? (process.env as Record<string, string | undefined>);
|
|
188
|
-
const supplied = options.signingSecret ?? env[STORAGE_SIGNING_SECRET_KEY];
|
|
189
|
-
const configured =
|
|
190
|
-
supplied === undefined || supplied === '' || supplied === DEV_SIGNING_SECRET
|
|
191
|
-
? undefined
|
|
192
|
-
: supplied;
|
|
193
|
-
// FAILS CLOSED: a process that names no environment resolves as `production` here, the answer
|
|
194
|
-
// core's `assertNoDevSecretsOutsideLocal` gives. `isLocal`'s own fallback is `development`, so the
|
|
195
|
-
// process that forgot to say signed with the published key — exactly the one that must not.
|
|
196
|
-
if (configured === undefined && !isLocal({ env, fallback: 'production' }))
|
|
197
|
-
throw signingSecretMissing(resolveEnvironment({ env, fallback: 'production' }));
|
|
198
|
-
const secret = configured ?? DEV_SIGNING_SECRET;
|
|
124
|
+
const secret = resolveSigningSecret(DRIVER_NAME, options);
|
|
199
125
|
|
|
200
126
|
const filePath = (key: string): string => `${root}/${key}`;
|
|
201
|
-
const metaPath = (key: string): string => `${root}/${
|
|
127
|
+
const metaPath = (key: string): string => `${root}/${sidecarPathOf(key)}`;
|
|
202
128
|
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
}
|
|
212
|
-
};
|
|
213
|
-
|
|
214
|
-
// No `contentType` fallback: the sidecar is the only thing that knows, so a missing one means
|
|
215
|
-
// this driver does not know either — exactly what the s3 driver's `list()` reports. `get()`
|
|
216
|
-
// fills the default below, because a `StorageObject` promises a type and a read has one.
|
|
217
|
-
//
|
|
218
|
-
// `hash` is the ONLY thing that reads the object's bytes, and it defaults off. The etag used to
|
|
219
|
-
// be computed unconditionally when the sidecar was missing, under a comment saying "`list()`
|
|
220
|
-
// must not read every file it lists" — which is exactly what `list()` then did, one whole
|
|
221
|
-
// object at a time, sequentially, for every sidecar-less key on the disk (a `put()` that died
|
|
222
|
-
// between its two writes leaves one). `copy()` inherited it too, so a copy documented as never
|
|
223
|
-
// routing bytes through the heap buffered the whole source. A listing that cannot know an etag
|
|
224
|
-
// reports `''`, which is what the s3 listing already answers for a provider that returns none.
|
|
225
|
-
const head = async (key: string, hash = false): Promise<StorageListEntry | undefined> => {
|
|
226
|
-
const file = Bun.file(filePath(key));
|
|
227
|
-
if (!(await file.exists())) return undefined;
|
|
228
|
-
const sidecar = await readSidecar(key);
|
|
229
|
-
const etag = sidecar?.etag ?? (hash ? etagOf(new Uint8Array(await file.arrayBuffer())) : '');
|
|
230
|
-
return {
|
|
231
|
-
key,
|
|
232
|
-
size: file.size,
|
|
233
|
-
etag,
|
|
234
|
-
lastModified: new Date(file.lastModified),
|
|
235
|
-
...(sidecar?.contentType === undefined ? {} : { contentType: sidecar.contentType }),
|
|
236
|
-
...(sidecar?.cacheControl === undefined ? {} : { cacheControl: sidecar.cacheControl }),
|
|
237
|
-
...(sidecar?.metadata === undefined ? {} : { metadata: sidecar.metadata }),
|
|
238
|
-
};
|
|
239
|
-
};
|
|
129
|
+
// Both QUEUED behind the key's writers, as `get()` is: unqueued, a measurement read the size of
|
|
130
|
+
// one generation before a commit and the sidecar of the next after it.
|
|
131
|
+
/** A listing's view: no bytes read, so a pair in doubt reports no type and no etag. */
|
|
132
|
+
const head = (key: string): Promise<StorageListEntry | undefined> =>
|
|
133
|
+
oneAtATime(key, () => headObject(root, key));
|
|
134
|
+
/** The same, settled against the bytes on disk — streamed through a hasher, never buffered. */
|
|
135
|
+
const measured = (key: string): Promise<StorageListEntry | undefined> =>
|
|
136
|
+
oneAtATime(key, () => headObject(root, key, () => etagOfFile(root, key)));
|
|
240
137
|
|
|
241
138
|
/** Removes one path, or reports WHY it could not — a swallowed refusal is a false erasure. */
|
|
242
139
|
const removeIfPresent = async (path: string, key: string): Promise<void> => {
|
|
@@ -262,12 +159,14 @@ export function localDriver(options: LocalDriverOptions): StorageDriver {
|
|
|
262
159
|
},
|
|
263
160
|
|
|
264
161
|
registerAs(diskName: string): void {
|
|
162
|
+
registered = diskName;
|
|
265
163
|
if (options.baseUrl === undefined) baseUrl = signedUrlBaseFor(diskName);
|
|
266
164
|
},
|
|
267
165
|
|
|
268
166
|
async put(key: string, body: StorageBody, putOptions?: PutOptions): Promise<StorageObject> {
|
|
269
167
|
const safe = assertSafeKey(key);
|
|
270
168
|
refuseUnsupportedPut(putOptions);
|
|
169
|
+
assertPutOptions(DRIVER_NAME, putOptions);
|
|
271
170
|
const bytes = await toBytes(body, { driver: DRIVER_NAME, key: safe, maxBytes: maxPutBytes });
|
|
272
171
|
const claimed = putOptions?.checksum;
|
|
273
172
|
if (claimed !== undefined) {
|
|
@@ -280,8 +179,9 @@ export function localDriver(options: LocalDriverOptions): StorageDriver {
|
|
|
280
179
|
cacheControl: putOptions?.cacheControl,
|
|
281
180
|
metadata: putOptions?.metadata,
|
|
282
181
|
};
|
|
283
|
-
await
|
|
284
|
-
|
|
182
|
+
await oneAtATime(safe, () =>
|
|
183
|
+
commitObject({ root, key: safe, disk: registered, body: bytes, sidecar }),
|
|
184
|
+
);
|
|
285
185
|
return {
|
|
286
186
|
key: safe,
|
|
287
187
|
size: bytes.byteLength,
|
|
@@ -293,21 +193,29 @@ export function localDriver(options: LocalDriverOptions): StorageDriver {
|
|
|
293
193
|
};
|
|
294
194
|
},
|
|
295
195
|
|
|
196
|
+
async stat(key: string): Promise<StorageObject | undefined> {
|
|
197
|
+
const entry = await measured(assertSafeKey(key));
|
|
198
|
+
return entry && { ...entry, contentType: entry.contentType ?? DEFAULT_CONTENT_TYPE };
|
|
199
|
+
},
|
|
200
|
+
|
|
296
201
|
async get(key: string): Promise<StorageRead> {
|
|
297
202
|
const safe = assertSafeKey(key);
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
203
|
+
// Queued behind this key's writers: an object is two files, and a read between a put()'s
|
|
204
|
+
// renames is a pair in doubt this process has no need to meet.
|
|
205
|
+
return oneAtATime(safe, async () => {
|
|
206
|
+
const file = Bun.file(filePath(safe));
|
|
207
|
+
if (!(await file.exists())) throw objectNotFound(DRIVER_NAME, safe);
|
|
208
|
+
if (file.size > maxGetBytes) throw getTooLarge(DRIVER_NAME, safe, file.size, maxGetBytes);
|
|
209
|
+
const bytes = await readObjectBytes(root, safe);
|
|
210
|
+
// Settled against the bytes this read ALREADY holds: hashed only for a sidecar-less object
|
|
211
|
+
// or a pair in doubt, and then exactly once.
|
|
212
|
+
const entry = await headObject(root, safe, () => etagOf(bytes));
|
|
213
|
+
if (entry === undefined) throw objectNotFound(DRIVER_NAME, safe);
|
|
214
|
+
return {
|
|
215
|
+
object: { ...entry, contentType: entry.contentType ?? DEFAULT_CONTENT_TYPE },
|
|
216
|
+
bytes,
|
|
217
|
+
};
|
|
218
|
+
});
|
|
311
219
|
},
|
|
312
220
|
|
|
313
221
|
async stream(key: string): Promise<ReadableStream<Uint8Array>> {
|
|
@@ -321,19 +229,30 @@ export function localDriver(options: LocalDriverOptions): StorageDriver {
|
|
|
321
229
|
async copy(from: string, to: string): Promise<StorageObject> {
|
|
322
230
|
const source = assertSafeKey(from);
|
|
323
231
|
const destination = assertSafeKey(to);
|
|
324
|
-
//
|
|
325
|
-
// durable lie every later `get()` of the copy would
|
|
326
|
-
//
|
|
327
|
-
const entry = await
|
|
232
|
+
// Measured, not merely read: the destination gets a sidecar, and one carrying `etag: ''` or
|
|
233
|
+
// a torn source's borrowed type is a durable lie every later `get()` of the copy would
|
|
234
|
+
// trust. The hash is streamed, and only for a sidecar-less source or a pair in doubt.
|
|
235
|
+
const entry = await measured(source);
|
|
328
236
|
if (entry === undefined) throw objectNotFound(DRIVER_NAME, source);
|
|
329
|
-
await Bun.write(filePath(destination), Bun.file(filePath(source)));
|
|
330
237
|
const sidecar: Sidecar = {
|
|
331
238
|
contentType: entry.contentType ?? DEFAULT_CONTENT_TYPE,
|
|
332
239
|
etag: entry.etag,
|
|
333
240
|
cacheControl: entry.cacheControl,
|
|
334
241
|
metadata: entry.metadata,
|
|
335
242
|
};
|
|
336
|
-
await
|
|
243
|
+
await oneAtATime(destination, () =>
|
|
244
|
+
commitObject({
|
|
245
|
+
root,
|
|
246
|
+
key: destination,
|
|
247
|
+
disk: registered,
|
|
248
|
+
body: Bun.file(filePath(source)),
|
|
249
|
+
sidecar,
|
|
250
|
+
}),
|
|
251
|
+
).catch(async (error: unknown) => {
|
|
252
|
+
// The source was deleted while this copy waited its turn on the destination.
|
|
253
|
+
if (!(await Bun.file(filePath(source)).exists())) throw objectNotFound(DRIVER_NAME, source);
|
|
254
|
+
throw error;
|
|
255
|
+
});
|
|
337
256
|
return {
|
|
338
257
|
...entry,
|
|
339
258
|
key: destination,
|
|
@@ -347,8 +266,11 @@ export function localDriver(options: LocalDriverOptions): StorageDriver {
|
|
|
347
266
|
// Idempotent by contract: a missing key is already in the desired state. A REFUSED unlink
|
|
348
267
|
// is not — a read-only mount or a root this process cannot write reports the bytes gone
|
|
349
268
|
// when they are still on disk, which is the one lie an erasure sweep must never repeat.
|
|
350
|
-
await
|
|
351
|
-
|
|
269
|
+
await oneAtATime(safe, async () => {
|
|
270
|
+
await removeIfPresent(filePath(safe), safe);
|
|
271
|
+
await removeIfPresent(metaPath(safe), safe);
|
|
272
|
+
await removeIfPresent(`${root}/${pendingPathOf(safe)}`, safe);
|
|
273
|
+
});
|
|
352
274
|
},
|
|
353
275
|
|
|
354
276
|
async exists(key: string): Promise<boolean> {
|
|
@@ -357,6 +279,7 @@ export function localDriver(options: LocalDriverOptions): StorageDriver {
|
|
|
357
279
|
|
|
358
280
|
async list(listOptions?: ListOptions): Promise<ListPage> {
|
|
359
281
|
const prefix = listOptions?.prefix ?? '';
|
|
282
|
+
assertListOptions(listOptions);
|
|
360
283
|
const limit = resolveListLimit(listOptions?.limit);
|
|
361
284
|
const cursor = listOptions?.cursor;
|
|
362
285
|
const keys: string[] = [];
|