@ultimat3/storage 22.15.0 → 23.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +8 -0
- package/README.md +32 -2
- package/package.json +3 -2
- package/src/driver-local.ts +33 -23
- package/src/driver-memory.ts +195 -0
- package/src/driver-s3-region.ts +39 -0
- package/src/driver-s3.ts +32 -7
- package/src/errors.ts +5 -3
- package/src/index.ts +2 -0
- package/src/driver-s3-fixture.ts +0 -164
package/CLAUDE.md
CHANGED
|
@@ -39,7 +39,9 @@ Tier 1. Object storage: named disks, safe keys, signed URLs, sniffed uploads.
|
|
|
39
39
|
|---|---|
|
|
40
40
|
| `driver.ts` | `StorageDriver` contract (8 methods) + bounded `toBytes`/`sha256Base64`/`etagOf` |
|
|
41
41
|
| `driver-local.ts` | dev default over `Bun.file`, `.meta/` sidecars, `Bun.Glob` listing |
|
|
42
|
+
| `driver-memory.ts` | `memoryDriver()` — a test's disk over a `Map`: `localDriver`'s contract and refusals, `objects()` for an assertion about the bucket. `driver-memory.test.ts` runs each claim on BOTH disks. Signs through `resolveSigningSecret` (`driver-local.ts`), the one rule both share |
|
|
42
43
|
| `driver-s3.ts` | `Bun.S3Client`, built lazily (import must never open a socket) |
|
|
44
|
+
| `driver-s3-region.ts` | a provider's wrong-region refusal → `X_CONFIG_INVALID` whose fix names `S3_REGION` |
|
|
43
45
|
| `path.ts` | key validation + `META_DIR` + `scopedKey`/`isWithinOrg`/`isTenantScoped` tenant boundary |
|
|
44
46
|
| `signed-url.ts` | HMAC over the constraint tuple, constant-time verify |
|
|
45
47
|
| `upload.ts` | magic-byte sniff + size/allowlist/checksum policy |
|
|
@@ -187,6 +189,12 @@ Gotchas:
|
|
|
187
189
|
whole path exists to prevent. That is why it derives the box itself and asks core for
|
|
188
190
|
`fit: 'cover'` — core's `contain` letterboxes to the requested box, this API fits inside it.
|
|
189
191
|
- Bun's S3 flag is `virtualHostedStyle`; our `forcePathStyle` is its inverse.
|
|
192
|
+
- **No `region` signs for `auto`.** R2 wants that; AWS and a gateway that checks the scope answer
|
|
193
|
+
`AuthorizationHeaderMalformed … expecting "<region>"`. `put`/`copy`/`list`/`delete` ask
|
|
194
|
+
`regionMismatch()` BEFORE their own verdict, so the code is `X_CONFIG_INVALID` and the fix is the
|
|
195
|
+
region — never `X_STORAGE_LIST_FAILED` with a fix about `s3:ListBucket`. The captured region is
|
|
196
|
+
held to `[A-Za-z0-9-]`: a provider's sentence may put nothing else into a `fix:`. A HEAD has no
|
|
197
|
+
body, so `exists()`/`stat()` never see it. `driver-s3.live.test.ts` runs with no region on purpose.
|
|
190
198
|
- The signature check runs BEFORE the expiry check. Do not reorder.
|
|
191
199
|
- `timingSafeEqual` is `@ultimat3/core`'s (`signed-url.ts` imports and re-exports it) — the same
|
|
192
200
|
implementation `@ultimat3/auth` uses, not a second copy. Add new secret comparisons through it.
|
package/README.md
CHANGED
|
@@ -16,14 +16,44 @@ defineStorage({
|
|
|
16
16
|
await disk('media').put(scopedKey(orgId, 'avatars', 'a.png'), bytes, { contentType: 'image/png' });
|
|
17
17
|
```
|
|
18
18
|
|
|
19
|
-
Swapping `local` for `s3` in `app.config.ts` changes no call site. `x dev` needs no
|
|
19
|
+
Swapping `local` for `s3` in `app.config.ts` changes no call site. `x dev` needs no S3 server.
|
|
20
20
|
|
|
21
21
|
## Drivers
|
|
22
22
|
|
|
23
23
|
| Driver | Backing | For | Signed URLs |
|
|
24
24
|
|---|---|---|---|
|
|
25
25
|
| `localDriver` | `Bun.file`/`Bun.write`, one root dir | dev, tests, single-node | HMAC + dev route |
|
|
26
|
-
| `s3Driver` | `Bun.s3` | prod:
|
|
26
|
+
| `s3Driver` | `Bun.s3` | prod: any S3-compatible endpoint — AWS, R2, a self-hosted gateway | provider presign |
|
|
27
|
+
| `memoryDriver` | a `Map` in this process | a TEST's disk — never a deployment's: a restart is every object gone | HMAC, the local disk's own |
|
|
28
|
+
|
|
29
|
+
`memoryDriver()` is what a suite holds instead of a temp directory or a hand-written fake: it
|
|
30
|
+
answers every method `localDriver` does, refuses what it refuses (an unsafe key, a wrong checksum,
|
|
31
|
+
a body past `maxPutBytes`, `serverSideEncryption`), and signs under the same rule — the published
|
|
32
|
+
development key in `development` and `test` only. `objects()` hands back a COPY of every stored
|
|
33
|
+
object's bytes by key, for the assertion a test makes about the bucket itself.
|
|
34
|
+
|
|
35
|
+
```ts
|
|
36
|
+
import { afterEach, beforeEach } from 'bun:test';
|
|
37
|
+
import { defineStorage, disk, memoryDriver, resetStorage } from '@ultimat3/storage';
|
|
38
|
+
|
|
39
|
+
// A fresh driver per test: `resetStorage()` forgets the REGISTRY, never a driver's objects, so a
|
|
40
|
+
// driver held across tests carries every earlier test's writes.
|
|
41
|
+
let sessions = memoryDriver();
|
|
42
|
+
beforeEach(() => {
|
|
43
|
+
sessions = memoryDriver();
|
|
44
|
+
defineStorage({ disks: { sessions } });
|
|
45
|
+
});
|
|
46
|
+
afterEach(() => resetStorage());
|
|
47
|
+
|
|
48
|
+
await disk('sessions').put('org/o1/session.json', new TextEncoder().encode('{"sealed":"x1.…"}'));
|
|
49
|
+
const stored = [...sessions.objects().values()].map((bytes) => new TextDecoder().decode(bytes));
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
Server-side only, like both other drivers: nothing an island imports reaches it.
|
|
53
|
+
|
|
54
|
+
`region` is what requests are signed for; unset, that is `auto`. An endpoint that expects another
|
|
55
|
+
region refuses the first write, list or delete with `X_CONFIG_INVALID`, and the fix names the
|
|
56
|
+
`S3_REGION` to set.
|
|
27
57
|
|
|
28
58
|
`copy(from, to)` is on the contract so a promotion is not a download-and-reupload:
|
|
29
59
|
`promoteAttachment` used to `get()` the whole object into the app and `put()` it back, a gigabyte
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ultimat3/storage",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "23.0.0",
|
|
4
4
|
"description": "Named disks over Bun.file and Bun.s3: safe keys, signed URLs, sniffed uploads",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -19,6 +19,7 @@
|
|
|
19
19
|
"files": [
|
|
20
20
|
"src",
|
|
21
21
|
"!src/**/*.test.ts",
|
|
22
|
+
"!src/**/*-fixture.ts",
|
|
22
23
|
"CLAUDE.md",
|
|
23
24
|
"README.md",
|
|
24
25
|
"LICENSE"
|
|
@@ -31,6 +32,6 @@
|
|
|
31
32
|
"test": "bun test"
|
|
32
33
|
},
|
|
33
34
|
"dependencies": {
|
|
34
|
-
"@ultimat3/core": "
|
|
35
|
+
"@ultimat3/core": "23.0.0"
|
|
35
36
|
}
|
|
36
37
|
}
|
package/src/driver-local.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// Single responsibility: the dev-default disk — a real, working driver over `Bun.file` /
|
|
2
|
-
// `Bun.write` rooted at one directory, so `x dev` needs no
|
|
2
|
+
// `Bun.write` rooted at one directory, so `x dev` needs no S3 server and no cloud account.
|
|
3
3
|
// Content type, etag and user metadata live in a sidecar under `.meta/`: a POSIX file has
|
|
4
4
|
// nowhere to keep them, and `get` must round-trip exactly what `put` was handed.
|
|
5
5
|
|
|
@@ -141,6 +141,37 @@ function parseSidecar(raw: unknown): Sidecar | undefined {
|
|
|
141
141
|
};
|
|
142
142
|
}
|
|
143
143
|
|
|
144
|
+
/**
|
|
145
|
+
* The secret a disk that mints its OWN URLs signs with — `localDriver` and `memoryDriver` both.
|
|
146
|
+
*
|
|
147
|
+
* A dev disk must work with zero config. Outside development the fallback is refused rather than
|
|
148
|
+
* used: the literal is published, so signing with it hands every reader the power to mint a PUT
|
|
149
|
+
* for any key with any size and type limit — which `acceptSignedUpload` then trusts over the app's
|
|
150
|
+
* own `uploadPolicy`. Refused at construction, so the boot fails rather than the first upload.
|
|
151
|
+
*
|
|
152
|
+
* The published literal counts as no secret at all, whichever way it arrives: an env var or an
|
|
153
|
+
* `app.config.ts` that pasted it in signs exactly as weakly as the fallback does. One table for
|
|
154
|
+
* all three reads — the secret, the environment test and the environment the refusal names.
|
|
155
|
+
* Splitting them is how the guard and the disk came to answer about two different processes.
|
|
156
|
+
*/
|
|
157
|
+
export function resolveSigningSecret(
|
|
158
|
+
disk: string,
|
|
159
|
+
options: Pick<LocalDriverOptions, 'signingSecret' | 'env'>,
|
|
160
|
+
): string {
|
|
161
|
+
const env = options.env ?? (process.env as Record<string, string | undefined>);
|
|
162
|
+
const supplied = options.signingSecret ?? env[STORAGE_SIGNING_SECRET_KEY];
|
|
163
|
+
const configured =
|
|
164
|
+
supplied === undefined || supplied === '' || supplied === DEV_SIGNING_SECRET
|
|
165
|
+
? undefined
|
|
166
|
+
: supplied;
|
|
167
|
+
// FAILS CLOSED: a process that names no environment resolves as `production` here, the answer
|
|
168
|
+
// core's `assertNoDevSecretsOutsideLocal` gives. `isLocal`'s own fallback is `development`, so the
|
|
169
|
+
// process that forgot to say signed with the published key — exactly the one that must not.
|
|
170
|
+
if (configured === undefined && !isLocal({ env, fallback: 'production' }))
|
|
171
|
+
throw signingSecretMissing(resolveEnvironment({ env, fallback: 'production' }), disk);
|
|
172
|
+
return configured ?? DEV_SIGNING_SECRET;
|
|
173
|
+
}
|
|
174
|
+
|
|
144
175
|
/**
|
|
145
176
|
* A POSIX file is not encrypted at rest by this driver, and recording the request in the sidecar
|
|
146
177
|
* would answer a security review with a field the disk never honoured. Refused on the DEV disk
|
|
@@ -174,28 +205,7 @@ export function localDriver(options: LocalDriverOptions): StorageDriver {
|
|
|
174
205
|
// 404 its own URLs. An explicit `baseUrl` outranks the registration: that is the operator
|
|
175
206
|
// stating where the route is mounted, and inference must not overwrite a decision.
|
|
176
207
|
let baseUrl = options.baseUrl ?? signedUrlBaseFor(DRIVER_NAME);
|
|
177
|
-
|
|
178
|
-
// than used: the literal is published, so signing with it hands every reader the power to mint
|
|
179
|
-
// a PUT for any key with any size and type limit — which `acceptSignedUpload` then trusts over
|
|
180
|
-
// the app's own `uploadPolicy`. Refused HERE, at construction, so the boot fails rather than
|
|
181
|
-
// the first upload.
|
|
182
|
-
// The published literal counts as no secret at all, whichever way it arrives: an env var or an
|
|
183
|
-
// `app.config.ts` that pasted it in signs exactly as weakly as the fallback does.
|
|
184
|
-
// One table for all three reads — the secret, the environment test and the environment the
|
|
185
|
-
// refusal names. Splitting them is how the guard and the disk came to answer about two
|
|
186
|
-
// different processes.
|
|
187
|
-
const env = options.env ?? (process.env as Record<string, string | undefined>);
|
|
188
|
-
const supplied = options.signingSecret ?? env[STORAGE_SIGNING_SECRET_KEY];
|
|
189
|
-
const configured =
|
|
190
|
-
supplied === undefined || supplied === '' || supplied === DEV_SIGNING_SECRET
|
|
191
|
-
? undefined
|
|
192
|
-
: supplied;
|
|
193
|
-
// FAILS CLOSED: a process that names no environment resolves as `production` here, the answer
|
|
194
|
-
// core's `assertNoDevSecretsOutsideLocal` gives. `isLocal`'s own fallback is `development`, so the
|
|
195
|
-
// process that forgot to say signed with the published key — exactly the one that must not.
|
|
196
|
-
if (configured === undefined && !isLocal({ env, fallback: 'production' }))
|
|
197
|
-
throw signingSecretMissing(resolveEnvironment({ env, fallback: 'production' }));
|
|
198
|
-
const secret = configured ?? DEV_SIGNING_SECRET;
|
|
208
|
+
const secret = resolveSigningSecret(DRIVER_NAME, options);
|
|
199
209
|
|
|
200
210
|
const filePath = (key: string): string => `${root}/${key}`;
|
|
201
211
|
const metaPath = (key: string): string => `${root}/${META_DIR}/${key}.json`;
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
// Single responsibility: the disk a TEST holds — a `StorageDriver` over one `Map`, answering every
|
|
2
|
+
// question `localDriver` answers and refusing what it refuses, with nothing on a filesystem to
|
|
3
|
+
// create or clean up. Not a deployment's disk: a restart is every object gone.
|
|
4
|
+
//
|
|
5
|
+
// It exists because the alternative was written by hand in every suite that needed a bucket: a
|
|
6
|
+
// fake with three methods and five `unsupported()` stubs, each a slightly different contract.
|
|
7
|
+
|
|
8
|
+
import { type Clock, finiteCount, systemClock } from '@ultimat3/core';
|
|
9
|
+
import {
|
|
10
|
+
DEFAULT_CONTENT_TYPE,
|
|
11
|
+
etagOf,
|
|
12
|
+
type ListOptions,
|
|
13
|
+
type ListPage,
|
|
14
|
+
type PutOptions,
|
|
15
|
+
resolveListLimit,
|
|
16
|
+
type SignedUrlOptions,
|
|
17
|
+
type StorageBody,
|
|
18
|
+
type StorageDriver,
|
|
19
|
+
type StorageObject,
|
|
20
|
+
type StorageRead,
|
|
21
|
+
sha256Base64,
|
|
22
|
+
toBytes,
|
|
23
|
+
} from './driver';
|
|
24
|
+
import type { LocalDriverOptions } from './driver-local';
|
|
25
|
+
import { resolveSigningSecret } from './driver-local';
|
|
26
|
+
import { checksumMismatch, objectNotFound, storageNotImplemented } from './errors';
|
|
27
|
+
import { assertSafeKey } from './path';
|
|
28
|
+
import type { SignedUrlVerification } from './signed-url';
|
|
29
|
+
import { buildSignedUrl, signedUrlBaseFor, verifySignedUrl } from './signed-url';
|
|
30
|
+
import { DEFAULT_MAX_UPLOAD_BYTES } from './upload';
|
|
31
|
+
|
|
32
|
+
const DRIVER_NAME = 'memory';
|
|
33
|
+
|
|
34
|
+
/** `localDriver`'s options, minus the directory there is none of. */
|
|
35
|
+
export type MemoryDriverOptions = Omit<LocalDriverOptions, 'root'>;
|
|
36
|
+
|
|
37
|
+
export interface MemoryStorageDriver extends StorageDriver {
|
|
38
|
+
/**
|
|
39
|
+
* Every stored object's bytes by key, COPIED — what an operator holding the bucket could read.
|
|
40
|
+
* For the assertion a test makes about the store itself: "nothing in it is a cookie value".
|
|
41
|
+
*/
|
|
42
|
+
objects(): ReadonlyMap<string, Uint8Array>;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
interface Stored {
|
|
46
|
+
readonly bytes: Uint8Array;
|
|
47
|
+
readonly object: StorageObject;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* An object no caller shares with the store. `metadata` and `lastModified` are mutable by anyone
|
|
52
|
+
* holding a reference, and the local disk's sidecar is a copy by construction — a test that
|
|
53
|
+
* mutated what `put()` was handed must not see a different object on the next `get()`.
|
|
54
|
+
*/
|
|
55
|
+
const snapshot = (object: StorageObject): StorageObject => ({
|
|
56
|
+
...object,
|
|
57
|
+
lastModified: new Date(object.lastModified.getTime()),
|
|
58
|
+
...(object.metadata === undefined ? {} : { metadata: { ...object.metadata } }),
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
export function memoryDriver(options: MemoryDriverOptions = {}): MemoryStorageDriver {
|
|
62
|
+
const maxPutBytes = finiteCount(
|
|
63
|
+
'the memory disk driver',
|
|
64
|
+
'maxPutBytes',
|
|
65
|
+
options.maxPutBytes === undefined ? DEFAULT_MAX_UPLOAD_BYTES : options.maxPutBytes,
|
|
66
|
+
1,
|
|
67
|
+
);
|
|
68
|
+
const clock = options.clock ?? systemClock;
|
|
69
|
+
let baseUrl = options.baseUrl ?? signedUrlBaseFor(DRIVER_NAME);
|
|
70
|
+
const secret = resolveSigningSecret(DRIVER_NAME, options);
|
|
71
|
+
const stored = new Map<string, Stored>();
|
|
72
|
+
|
|
73
|
+
const found = (key: string): Stored => {
|
|
74
|
+
const hit = stored.get(key);
|
|
75
|
+
if (hit === undefined) throw objectNotFound(DRIVER_NAME, key);
|
|
76
|
+
return hit;
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
return {
|
|
80
|
+
name: DRIVER_NAME,
|
|
81
|
+
|
|
82
|
+
get signedUrlBase(): string {
|
|
83
|
+
return baseUrl;
|
|
84
|
+
},
|
|
85
|
+
|
|
86
|
+
registerAs(diskName: string): void {
|
|
87
|
+
if (options.baseUrl === undefined) baseUrl = signedUrlBaseFor(diskName);
|
|
88
|
+
},
|
|
89
|
+
|
|
90
|
+
objects(): ReadonlyMap<string, Uint8Array> {
|
|
91
|
+
return new Map([...stored].map(([key, value]) => [key, value.bytes.slice()]));
|
|
92
|
+
},
|
|
93
|
+
|
|
94
|
+
async put(key: string, body: StorageBody, putOptions?: PutOptions): Promise<StorageObject> {
|
|
95
|
+
const safe = assertSafeKey(key);
|
|
96
|
+
// The same refusal both shipped disks make: a `put()` that succeeds in a test and throws in
|
|
97
|
+
// production is a gap an app meets on the worst day.
|
|
98
|
+
if (putOptions?.serverSideEncryption !== undefined) {
|
|
99
|
+
throw storageNotImplemented(
|
|
100
|
+
'server-side encryption on the memory driver (it holds bytes in a Map)',
|
|
101
|
+
'drop serverSideEncryption from put(), and encrypt the disk itself — an s3Driver over a bucket with a default KMS rule',
|
|
102
|
+
);
|
|
103
|
+
}
|
|
104
|
+
const bytes = await toBytes(body, { driver: DRIVER_NAME, key: safe, maxBytes: maxPutBytes });
|
|
105
|
+
const claimed = putOptions?.checksum;
|
|
106
|
+
if (claimed !== undefined) {
|
|
107
|
+
const actual = sha256Base64(bytes);
|
|
108
|
+
if (claimed !== actual) throw checksumMismatch(safe, claimed, actual);
|
|
109
|
+
}
|
|
110
|
+
const object: StorageObject = {
|
|
111
|
+
key: safe,
|
|
112
|
+
size: bytes.byteLength,
|
|
113
|
+
contentType: putOptions?.contentType ?? DEFAULT_CONTENT_TYPE,
|
|
114
|
+
etag: etagOf(bytes),
|
|
115
|
+
lastModified: clock.now(),
|
|
116
|
+
...(putOptions?.cacheControl === undefined
|
|
117
|
+
? {}
|
|
118
|
+
: { cacheControl: putOptions.cacheControl }),
|
|
119
|
+
...(putOptions?.metadata === undefined ? {} : { metadata: putOptions.metadata }),
|
|
120
|
+
};
|
|
121
|
+
// A copy in, as a file write is: the caller's buffer and metadata are the caller's to reuse.
|
|
122
|
+
stored.set(safe, { bytes: bytes.slice(), object: snapshot(object) });
|
|
123
|
+
return snapshot(object);
|
|
124
|
+
},
|
|
125
|
+
|
|
126
|
+
async get(key: string): Promise<StorageRead> {
|
|
127
|
+
const hit = found(assertSafeKey(key));
|
|
128
|
+
return { object: snapshot(hit.object), bytes: hit.bytes.slice() };
|
|
129
|
+
},
|
|
130
|
+
|
|
131
|
+
async stream(key: string): Promise<ReadableStream<Uint8Array>> {
|
|
132
|
+
const hit = found(assertSafeKey(key));
|
|
133
|
+
return new Blob([hit.bytes.slice()]).stream();
|
|
134
|
+
},
|
|
135
|
+
|
|
136
|
+
async copy(from: string, to: string): Promise<StorageObject> {
|
|
137
|
+
const source = found(assertSafeKey(from));
|
|
138
|
+
const destination = assertSafeKey(to);
|
|
139
|
+
const object: StorageObject = {
|
|
140
|
+
...source.object,
|
|
141
|
+
key: destination,
|
|
142
|
+
lastModified: clock.now(),
|
|
143
|
+
};
|
|
144
|
+
stored.set(destination, { bytes: source.bytes.slice(), object: snapshot(object) });
|
|
145
|
+
return snapshot(object);
|
|
146
|
+
},
|
|
147
|
+
|
|
148
|
+
async delete(key: string): Promise<void> {
|
|
149
|
+
stored.delete(assertSafeKey(key));
|
|
150
|
+
},
|
|
151
|
+
|
|
152
|
+
async exists(key: string): Promise<boolean> {
|
|
153
|
+
return stored.has(assertSafeKey(key));
|
|
154
|
+
},
|
|
155
|
+
|
|
156
|
+
async list(listOptions?: ListOptions): Promise<ListPage> {
|
|
157
|
+
const prefix = listOptions?.prefix ?? '';
|
|
158
|
+
const limit = resolveListLimit(listOptions?.limit);
|
|
159
|
+
const cursor = listOptions?.cursor;
|
|
160
|
+
// Code-unit order and "the cursor is the last key of the page before" — the local disk's own.
|
|
161
|
+
const keys = [...stored.keys()]
|
|
162
|
+
.filter((key) => key.startsWith(prefix) && (cursor === undefined || key > cursor))
|
|
163
|
+
.sort();
|
|
164
|
+
const page = keys.slice(0, limit);
|
|
165
|
+
const objects = page.flatMap((key) => {
|
|
166
|
+
const hit = stored.get(key);
|
|
167
|
+
return hit === undefined ? [] : [snapshot(hit.object)];
|
|
168
|
+
});
|
|
169
|
+
const last = page.at(-1);
|
|
170
|
+
return keys.length > page.length && last !== undefined
|
|
171
|
+
? { objects, truncated: true, cursor: last }
|
|
172
|
+
: { objects, truncated: false };
|
|
173
|
+
},
|
|
174
|
+
|
|
175
|
+
async signedUrl(key: string, urlOptions?: SignedUrlOptions): Promise<string> {
|
|
176
|
+
return buildSignedUrl({
|
|
177
|
+
secret,
|
|
178
|
+
key: assertSafeKey(key),
|
|
179
|
+
method: urlOptions?.method,
|
|
180
|
+
expiresInMs: urlOptions?.expiresInMs,
|
|
181
|
+
maxBytes: urlOptions?.maxBytes,
|
|
182
|
+
contentType: urlOptions?.contentType,
|
|
183
|
+
baseUrl,
|
|
184
|
+
clock,
|
|
185
|
+
});
|
|
186
|
+
},
|
|
187
|
+
|
|
188
|
+
async verifySigned(input: {
|
|
189
|
+
readonly url: string;
|
|
190
|
+
readonly clock?: Clock | undefined;
|
|
191
|
+
}): Promise<SignedUrlVerification> {
|
|
192
|
+
return verifySignedUrl({ url: input.url, secret, baseUrl, clock: input.clock ?? clock });
|
|
193
|
+
},
|
|
194
|
+
};
|
|
195
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
// Single responsibility: recognise the one provider refusal that names its own repair — a request
|
|
2
|
+
// signed for the wrong region — and turn it into the disk-misconfigured error it is. Split from
|
|
3
|
+
// driver-s3.ts because every call that sends a signed request asks this before its own verdict.
|
|
4
|
+
|
|
5
|
+
import { ConfigInvalidError, stringField } from '@ultimat3/core';
|
|
6
|
+
|
|
7
|
+
/** What S3 and every S3-compatible gateway answer a SigV4 scope with the wrong region in it. */
|
|
8
|
+
const REGION_MISMATCH_CODE = 'AuthorizationHeaderMalformed';
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* `the region "auto" is wrong; expecting "us-east-1"` — double quotes from a gateway, single from
|
|
12
|
+
* AWS. The captures are held to a region name's alphabet, so a sentence this process did not
|
|
13
|
+
* write can put nothing else into a `fix:` line.
|
|
14
|
+
*/
|
|
15
|
+
const REGION_SENTENCE =
|
|
16
|
+
/the region ['"]([A-Za-z0-9-]{1,64})['"] is wrong; expecting ['"]([A-Za-z0-9-]{1,64})['"]/;
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* The refusal as `X_CONFIG_INVALID`, or `undefined` when `error` is anything else. A region
|
|
20
|
+
* mismatch is the DISK misconfigured, exactly as a missing bucket is, so it carries that code from
|
|
21
|
+
* every call rather than `X_STORAGE_LIST_FAILED` with a fix about `s3:ListBucket` from one and a
|
|
22
|
+
* bare `S3Error` from another: `Bun.S3Client` signs for `auto` when no region is set, and a
|
|
23
|
+
* provider that checks the scope refuses every request the app ever sends.
|
|
24
|
+
*
|
|
25
|
+
* A HEAD carries no body, so `exists()` and `stat()` never see this sentence — the first PUT,
|
|
26
|
+
* LIST or DELETE does.
|
|
27
|
+
*/
|
|
28
|
+
export function regionMismatch(bucket: string, error: unknown): ConfigInvalidError | undefined {
|
|
29
|
+
if (stringField(error, 'code') !== REGION_MISMATCH_CODE) return undefined;
|
|
30
|
+
const match = REGION_SENTENCE.exec(stringField(error, 'message') ?? '');
|
|
31
|
+
const sent = match?.[1];
|
|
32
|
+
const expected = match?.[2];
|
|
33
|
+
if (sent === undefined || expected === undefined) return undefined;
|
|
34
|
+
return new ConfigInvalidError({
|
|
35
|
+
cause: `the s3 disk for bucket "${bucket}" signed a request for region "${sent}" and its endpoint expects "${expected}"`,
|
|
36
|
+
fix: `set S3_REGION=${expected} in .env (or pass region: '${expected}' to s3Driver), then re-run`,
|
|
37
|
+
meta: { bucket, sent, expected },
|
|
38
|
+
});
|
|
39
|
+
}
|
package/src/driver-s3.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
// Single responsibility: the production disk over Bun's native S3 client. One driver covers
|
|
2
|
-
//
|
|
1
|
+
// Single responsibility: the production disk over Bun's native S3 client. One driver covers AWS,
|
|
2
|
+
// Cloudflare R2 and any S3-compatible gateway — the difference is `endpoint`, `region` and
|
|
3
|
+
// `forcePathStyle`, nothing else.
|
|
3
4
|
// The client is built lazily on first use so importing this module never opens a socket, and
|
|
4
5
|
// credentials arrive as env var NAMES: a literal key in app.config.ts is a key in git.
|
|
5
6
|
|
|
@@ -19,6 +20,7 @@ import {
|
|
|
19
20
|
sha256Base64,
|
|
20
21
|
toBytes,
|
|
21
22
|
} from './driver';
|
|
23
|
+
import { regionMismatch } from './driver-s3-region';
|
|
22
24
|
import {
|
|
23
25
|
checksumMismatch,
|
|
24
26
|
deleteFailed,
|
|
@@ -75,10 +77,14 @@ export interface S3ClientLike {
|
|
|
75
77
|
|
|
76
78
|
export interface S3DriverOptions {
|
|
77
79
|
readonly bucket: string;
|
|
80
|
+
/**
|
|
81
|
+
* The region requests are signed for. Unset, `Bun.S3Client` signs for `auto` — which R2 wants,
|
|
82
|
+
* and which AWS and a gateway that checks the scope refuse (`X_CONFIG_INVALID`, naming theirs).
|
|
83
|
+
*/
|
|
78
84
|
readonly region?: string | undefined;
|
|
79
|
-
/**
|
|
85
|
+
/** A self-hosted gateway: `http://localhost:9000`. R2: `https://<account>.r2.cloudflarestorage.com`. */
|
|
80
86
|
readonly endpoint?: string | undefined;
|
|
81
|
-
/**
|
|
87
|
+
/** A gateway addressed by host and port needs `true`; AWS and R2 do not. */
|
|
82
88
|
readonly forcePathStyle?: boolean | undefined;
|
|
83
89
|
/** Env var NAME holding the key id. Default `S3_ACCESS_KEY_ID`. */
|
|
84
90
|
readonly accessKeyIdEnv?: string | undefined;
|
|
@@ -138,7 +144,7 @@ function buildClient(options: S3DriverOptions): S3ClientLike {
|
|
|
138
144
|
}
|
|
139
145
|
const tokenVar = options.sessionTokenEnv;
|
|
140
146
|
const sessionToken = tokenVar === undefined ? undefined : env[tokenVar];
|
|
141
|
-
// Bun's flag is the inverse:
|
|
147
|
+
// Bun's flag is the inverse: path style means "not virtual hosted".
|
|
142
148
|
const pathStyle = options.forcePathStyle;
|
|
143
149
|
return new Client({
|
|
144
150
|
bucket: options.bucket,
|
|
@@ -224,6 +230,21 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
|
|
|
224
230
|
return client;
|
|
225
231
|
};
|
|
226
232
|
|
|
233
|
+
/**
|
|
234
|
+
* A refused request as the disk-misconfigured error, when the provider named its own repair (a
|
|
235
|
+
* wrong region). Asked before any call's own verdict, so the fix an operator reads is the one
|
|
236
|
+
* that works rather than one about a grant they already have. `misconfigured` is the same
|
|
237
|
+
* question for a call with no verdict of its own: it rethrows whatever arrived.
|
|
238
|
+
*/
|
|
239
|
+
const throwIfMisconfigured = (error: unknown): void => {
|
|
240
|
+
const mismatch = regionMismatch(options.bucket, error);
|
|
241
|
+
if (mismatch !== undefined) throw mismatch;
|
|
242
|
+
};
|
|
243
|
+
const misconfigured = (error: unknown): never => {
|
|
244
|
+
throwIfMisconfigured(error);
|
|
245
|
+
throw error;
|
|
246
|
+
};
|
|
247
|
+
|
|
227
248
|
const statObject = async (key: string): Promise<StorageObject> => {
|
|
228
249
|
const stat = await conn().file(key).stat();
|
|
229
250
|
return {
|
|
@@ -253,7 +274,8 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
|
|
|
253
274
|
}
|
|
254
275
|
await conn()
|
|
255
276
|
.file(safe)
|
|
256
|
-
.write(bytes, { type: putOptions?.contentType ?? DEFAULT_CONTENT_TYPE })
|
|
277
|
+
.write(bytes, { type: putOptions?.contentType ?? DEFAULT_CONTENT_TYPE })
|
|
278
|
+
.catch(misconfigured);
|
|
257
279
|
return statObject(safe);
|
|
258
280
|
},
|
|
259
281
|
|
|
@@ -287,7 +309,8 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
|
|
|
287
309
|
const stat = await file.stat();
|
|
288
310
|
await conn()
|
|
289
311
|
.file(destination)
|
|
290
|
-
.write(file, { type: stat.type ?? DEFAULT_CONTENT_TYPE })
|
|
312
|
+
.write(file, { type: stat.type ?? DEFAULT_CONTENT_TYPE })
|
|
313
|
+
.catch(misconfigured);
|
|
291
314
|
return statObject(destination);
|
|
292
315
|
},
|
|
293
316
|
|
|
@@ -299,6 +322,7 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
|
|
|
299
322
|
// Idempotent means an ABSENT key, and nothing else. AWS answers DELETE on a missing key
|
|
300
323
|
// with 204, so this branch is for the providers that do not.
|
|
301
324
|
if (isAbsentObject(error)) return;
|
|
325
|
+
throwIfMisconfigured(error);
|
|
302
326
|
throw deleteFailed(
|
|
303
327
|
DRIVER_NAME,
|
|
304
328
|
safe,
|
|
@@ -331,6 +355,7 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
|
|
|
331
355
|
// A bare `S3Error` used to escape here, uncoded: no `X_*`, no `fix`, no `--json` shape, and
|
|
332
356
|
// `@ultimat3/http`'s error map has nothing to turn it into but a 500. A denied
|
|
333
357
|
// `s3:ListBucket` is the commonest one and reads to an operator as an app crash.
|
|
358
|
+
throwIfMisconfigured(error);
|
|
334
359
|
throw listFailed(
|
|
335
360
|
DRIVER_NAME,
|
|
336
361
|
prefix,
|
package/src/errors.ts
CHANGED
|
@@ -155,7 +155,9 @@ export const putTooLarge = (
|
|
|
155
155
|
`send it direct with grantUpload({ disk, orgId, request }) instead of put(), or raise the ceiling: ` +
|
|
156
156
|
(disk === 's3'
|
|
157
157
|
? `s3Driver({ bucket, maxPutBytes: ${bytes} })`
|
|
158
|
-
:
|
|
158
|
+
: disk === 'memory'
|
|
159
|
+
? `memoryDriver({ maxPutBytes: ${bytes} })`
|
|
160
|
+
: `localDriver({ root, maxPutBytes: ${bytes} })`),
|
|
159
161
|
meta: { disk, key, bytes, maxBytes },
|
|
160
162
|
});
|
|
161
163
|
|
|
@@ -352,12 +354,12 @@ export const signedUrlUnverifiable = (diskName: string): StorageError =>
|
|
|
352
354
|
meta: { disk: diskName },
|
|
353
355
|
});
|
|
354
356
|
|
|
355
|
-
export const signingSecretMissing = (environment: string): StorageError =>
|
|
357
|
+
export const signingSecretMissing = (environment: string, disk = 'local'): StorageError =>
|
|
356
358
|
new StorageError({
|
|
357
359
|
code: 'X_ENV_MISSING',
|
|
358
360
|
// The environment names what `resolveEnvironment()` resolved, which may have come from
|
|
359
361
|
// NODE_ENV — naming ULTIMATE_ENV here reported a variable the process never set.
|
|
360
|
-
cause: `the
|
|
362
|
+
cause: `the ${disk} disk has no usable signing secret (no signingSecret option, and STORAGE_SIGNING_SECRET is unset, empty or the published development key) and the resolved environment is "${environment}", so it would sign URLs with the shipped development key`,
|
|
361
363
|
fix: 'export STORAGE_SIGNING_SECRET="$(openssl rand -hex 32)"',
|
|
362
364
|
meta: { key: 'STORAGE_SIGNING_SECRET', environment },
|
|
363
365
|
});
|
package/src/index.ts
CHANGED
|
@@ -59,6 +59,8 @@ export {
|
|
|
59
59
|
STORAGE_SIGNING_SECRET_KEY,
|
|
60
60
|
usesDevStorageSecret,
|
|
61
61
|
} from './driver-local';
|
|
62
|
+
export type { MemoryDriverOptions, MemoryStorageDriver } from './driver-memory';
|
|
63
|
+
export { memoryDriver } from './driver-memory';
|
|
62
64
|
export type {
|
|
63
65
|
S3ClientLike,
|
|
64
66
|
S3DriverOptions,
|
package/src/driver-s3-fixture.ts
DELETED
|
@@ -1,164 +0,0 @@
|
|
|
1
|
-
// The in-memory `Bun.S3Client` stand-in the s3 driver's tests drive, and the helpers that read
|
|
2
|
-
// its answers. Shared rather than copied because `driver-s3.test.ts` (reads, copies, listings,
|
|
3
|
-
// credentials) and `driver-s3-put.test.ts` (everything `put()` refuses) must exercise the SAME
|
|
4
|
-
// fake — two fakes drifting apart would be two providers, agreeing only by construction.
|
|
5
|
-
|
|
6
|
-
import type { S3ClientLike, S3FileLike, S3ListResultLike, S3StatLike } from './driver-s3';
|
|
7
|
-
import { isStorageError, objectNotFound } from './errors';
|
|
8
|
-
|
|
9
|
-
/** The driver's private `DRIVER_NAME`; the fake reports failures against the same disk. */
|
|
10
|
-
export const FAKE_DISK = 's3';
|
|
11
|
-
|
|
12
|
-
export interface FakeObject {
|
|
13
|
-
bytes: Uint8Array;
|
|
14
|
-
type?: string | undefined;
|
|
15
|
-
etag?: string | undefined;
|
|
16
|
-
lastModified?: string | Date | undefined;
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
export interface PresignCall {
|
|
20
|
-
key: string;
|
|
21
|
-
options: {
|
|
22
|
-
method?: string | undefined;
|
|
23
|
-
expiresIn?: number | undefined;
|
|
24
|
-
type?: string | undefined;
|
|
25
|
-
};
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
export interface ListCall {
|
|
29
|
-
prefix?: string | undefined;
|
|
30
|
-
maxKeys?: number | undefined;
|
|
31
|
-
continuationToken?: string | undefined;
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
/**
|
|
35
|
-
* What Bun hands back when the SERVICE refuses: an ordinary `Error` named `S3Error`, carrying the
|
|
36
|
-
* provider's own code and status. The driver must read those structurally, because nothing in the
|
|
37
|
-
* type system says a caught value has them.
|
|
38
|
-
*/
|
|
39
|
-
export function s3Error(code: string, statusCode: number, key: string): Error {
|
|
40
|
-
return Object.assign(new Error(`${code}: the fake provider refused ${key}`), {
|
|
41
|
-
name: 'S3Error',
|
|
42
|
-
code,
|
|
43
|
-
statusCode,
|
|
44
|
-
path: key,
|
|
45
|
-
});
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
/** In-memory stand-in for `Bun.S3Client`, driven through `S3ClientLike` — no socket, ever. */
|
|
49
|
-
export class FakeS3Client implements S3ClientLike {
|
|
50
|
-
readonly store = new Map<string, FakeObject>();
|
|
51
|
-
readonly fileCalls: string[] = [];
|
|
52
|
-
readonly presignCalls: PresignCall[] = [];
|
|
53
|
-
readonly listCalls: ListCall[] = [];
|
|
54
|
-
listResult: S3ListResultLike = { contents: [] };
|
|
55
|
-
/** A provider REFUSAL (403/503/reset) — the failure `delete()` must surface, not swallow. */
|
|
56
|
-
failDeleteFor: string | undefined;
|
|
57
|
-
/** A provider "there is nothing there" — the one failure `delete()` may call success. */
|
|
58
|
-
absentDeleteFor: string | undefined;
|
|
59
|
-
/**
|
|
60
|
-
* A refused LISTING — a denied `s3:ListBucket`, a throttle, an expired credential. There is no
|
|
61
|
-
* "absent" counterpart: a prefix with no keys is an empty page, never a rejection, so every
|
|
62
|
-
* failure `list()` can meet is one it must surface.
|
|
63
|
-
*/
|
|
64
|
-
failListWith: Error | undefined;
|
|
65
|
-
|
|
66
|
-
/** Which key each handed-out `S3FileLike` stands for, so `write(sourceFile)` can read it. */
|
|
67
|
-
private readonly fileKeys = new WeakMap<object, string>();
|
|
68
|
-
|
|
69
|
-
/** The bytes behind an `S3FileLike` passed as `write`'s data, or `undefined` for raw bytes. */
|
|
70
|
-
sourceOf(data: unknown): Uint8Array | undefined {
|
|
71
|
-
if (typeof data !== 'object' || data === null) return undefined;
|
|
72
|
-
const sourceKey = this.fileKeys.get(data);
|
|
73
|
-
return sourceKey === undefined ? undefined : this.store.get(sourceKey)?.bytes;
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
file(key: string): S3FileLike {
|
|
77
|
-
this.fileCalls.push(key);
|
|
78
|
-
const store = this.store;
|
|
79
|
-
const client = this;
|
|
80
|
-
const handle: S3FileLike = {
|
|
81
|
-
async write(data, options) {
|
|
82
|
-
// `copy()` hands the SOURCE S3File to write(), exactly as Bun's own union allows, so the
|
|
83
|
-
// fake has to read one back out of its store rather than assume bytes.
|
|
84
|
-
const source = client.sourceOf(data);
|
|
85
|
-
const bytes =
|
|
86
|
-
source !== undefined
|
|
87
|
-
? source
|
|
88
|
-
: data instanceof Uint8Array
|
|
89
|
-
? data
|
|
90
|
-
: new Uint8Array(await (data as Blob).arrayBuffer());
|
|
91
|
-
store.set(key, { bytes, type: options?.type });
|
|
92
|
-
return bytes.byteLength;
|
|
93
|
-
},
|
|
94
|
-
async arrayBuffer() {
|
|
95
|
-
const entry = store.get(key);
|
|
96
|
-
// Reads the way the provider does: a GET on a key that is not there is a 404, and the
|
|
97
|
-
// driver is expected to have gated it behind exists() before ever getting here.
|
|
98
|
-
if (entry === undefined) throw objectNotFound(FAKE_DISK, key);
|
|
99
|
-
return new Uint8Array(entry.bytes).buffer;
|
|
100
|
-
},
|
|
101
|
-
async exists() {
|
|
102
|
-
return store.has(key);
|
|
103
|
-
},
|
|
104
|
-
async delete() {
|
|
105
|
-
// Shaped like the real thing: an `S3Error` is a plain Error with `name: 'S3Error'`, a
|
|
106
|
-
// provider `code` and a status. Deliberately NOT a StorageError — the driver has to
|
|
107
|
-
// classify a rejection it never authored, which is the only kind a provider hands back.
|
|
108
|
-
if (client.absentDeleteFor === key) throw s3Error('NoSuchKey', 404, key);
|
|
109
|
-
if (client.failDeleteFor === key) throw s3Error('AccessDenied', 403, key);
|
|
110
|
-
store.delete(key);
|
|
111
|
-
},
|
|
112
|
-
stream() {
|
|
113
|
-
const entry = store.get(key);
|
|
114
|
-
const bytes = entry?.bytes ?? new Uint8Array();
|
|
115
|
-
return new ReadableStream<Uint8Array>({
|
|
116
|
-
start(controller) {
|
|
117
|
-
controller.enqueue(bytes);
|
|
118
|
-
controller.close();
|
|
119
|
-
},
|
|
120
|
-
});
|
|
121
|
-
},
|
|
122
|
-
async stat(): Promise<S3StatLike> {
|
|
123
|
-
const entry = store.get(key);
|
|
124
|
-
if (entry === undefined) throw objectNotFound(FAKE_DISK, key);
|
|
125
|
-
return {
|
|
126
|
-
size: entry.bytes.byteLength,
|
|
127
|
-
type: entry.type,
|
|
128
|
-
etag: entry.etag,
|
|
129
|
-
lastModified: entry.lastModified,
|
|
130
|
-
};
|
|
131
|
-
},
|
|
132
|
-
presign(options) {
|
|
133
|
-
client.presignCalls.push({ key, options });
|
|
134
|
-
return `https://fake.example/${key}?signed`;
|
|
135
|
-
},
|
|
136
|
-
};
|
|
137
|
-
this.fileKeys.set(handle, key);
|
|
138
|
-
return handle;
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
async list(input: ListCall): Promise<S3ListResultLike> {
|
|
142
|
-
this.listCalls.push(input);
|
|
143
|
-
if (this.failListWith !== undefined) throw this.failListWith;
|
|
144
|
-
return this.listResult;
|
|
145
|
-
}
|
|
146
|
-
}
|
|
147
|
-
|
|
148
|
-
export const bytesOf = (text: string): Uint8Array => new TextEncoder().encode(text);
|
|
149
|
-
export const textOf = (bytes: Uint8Array): string => new TextDecoder().decode(bytes);
|
|
150
|
-
|
|
151
|
-
/** The error code a call answered with, or how it failed to answer with one. */
|
|
152
|
-
export function codeOf(caught: unknown): string {
|
|
153
|
-
return isStorageError(caught) ? caught.code : `not-a-storage-error: ${String(caught)}`;
|
|
154
|
-
}
|
|
155
|
-
|
|
156
|
-
/** The thrown value itself, where the assertion is about its `cause`/`fix` and not just its code. */
|
|
157
|
-
export async function catchError(fn: () => Promise<unknown>): Promise<unknown> {
|
|
158
|
-
try {
|
|
159
|
-
await fn();
|
|
160
|
-
return undefined;
|
|
161
|
-
} catch (error) {
|
|
162
|
-
return error;
|
|
163
|
-
}
|
|
164
|
-
}
|