@ultimat3/storage 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +89 -0
- package/package.json +35 -0
- package/src/driver-local.ts +196 -0
- package/src/driver-s3.ts +249 -0
- package/src/driver.ts +83 -0
- package/src/errors.ts +153 -0
- package/src/image.ts +148 -0
- package/src/index.ts +111 -0
- package/src/path.ts +99 -0
- package/src/signed-url.ts +185 -0
- package/src/storage.ts +76 -0
- package/src/upload.ts +193 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 developerz.ai
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
# @ultimat3/storage 🗄️
|
|
2
|
+
|
|
3
|
+
Named disks. **Call sites name a disk, never a driver.**
|
|
4
|
+
|
|
5
|
+
```ts
|
|
6
|
+
import { defineStorage, disk, localDriver, s3Driver, scopedKey } from '@ultimat3/storage';
|
|
7
|
+
|
|
8
|
+
defineStorage({
|
|
9
|
+
disks: {
|
|
10
|
+
uploads: localDriver({ root: '.storage/uploads' }),
|
|
11
|
+
media: s3Driver({ bucket: 'media', endpoint: process.env.S3_ENDPOINT, forcePathStyle: true }),
|
|
12
|
+
},
|
|
13
|
+
default: 'uploads', // omit and the first disk wins
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
await disk('media').put(scopedKey(orgId, 'avatars', 'a.png'), bytes, { contentType: 'image/png' });
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
Swapping `local` for `s3` in `app.config.ts` changes no call site. `x dev` needs no MinIO.
|
|
20
|
+
|
|
21
|
+
## Drivers
|
|
22
|
+
|
|
23
|
+
| Driver | Backing | For | Signed URLs |
|
|
24
|
+
|---|---|---|---|
|
|
25
|
+
| `localDriver` | `Bun.file`/`Bun.write`, one root dir | dev, tests, single-node | HMAC + dev route |
|
|
26
|
+
| `s3Driver` | `Bun.s3` | prod: MinIO, R2, AWS | provider presign |
|
|
27
|
+
|
|
28
|
+
One S3 driver covers all three backends — the difference is `endpoint` + `forcePathStyle`.
|
|
29
|
+
Credentials are **env var NAMES** (`accessKeyIdEnv`, default `S3_ACCESS_KEY_ID`), never
|
|
30
|
+
literals: a key in `app.config.ts` is a key in git. Missing ones throw `X_ENV_MISSING`.
|
|
31
|
+
`localDriver` keeps content type and etag in a `<root>/.meta/` sidecar so `get()` round-trips
|
|
32
|
+
`put()`; sidecars never appear in `list()`.
|
|
33
|
+
|
|
34
|
+
## Keys
|
|
35
|
+
|
|
36
|
+
`assertSafeKey()` runs on every key before it reaches a driver. Rejected: `..` segments,
|
|
37
|
+
absolute keys, backslashes, NUL/control bytes, percent-encoded separators (`%2e`, `%2f`),
|
|
38
|
+
empty segments, over 1024 chars. No sanitising — a key that needed fixing was built wrong.
|
|
39
|
+
`scopedKey('org-1', 'avatars', 'a.png')` is `org/org-1/avatars/a.png`; guard every
|
|
40
|
+
client-supplied key with `isWithinOrg(key, ctx.actor.orgId)`.
|
|
41
|
+
|
|
42
|
+
## Signed URLs
|
|
43
|
+
|
|
44
|
+
The HMAC covers the **constraints**, not just the key —
|
|
45
|
+
`v1 \n METHOD \n key \n expiresAt \n maxBytes \n contentType`.
|
|
46
|
+
A client that edits `?x-max=` invalidates the signature — it cannot widen what it was granted.
|
|
47
|
+
Verification is constant-time, checks the signature *before* the expiry (a forged URL never
|
|
48
|
+
learns it was merely late), takes a `Clock` so tests freeze time, and returns
|
|
49
|
+
`{ ok: false, reason }` rather than throwing — `malformed | unsafe-key | signature-mismatch |
|
|
50
|
+
expired`. S3 presign covers method, expiry and content type but **not** `maxBytes`: S3 has no
|
|
51
|
+
header for it, so size stays a server-side `validateUpload()` check.
|
|
52
|
+
|
|
53
|
+
## Uploads sniff the content type
|
|
54
|
+
|
|
55
|
+
`Content-Type` is attacker-controlled. A `.png` that is really an HTML document is stored XSS
|
|
56
|
+
the moment a surface serves it back with the declared type. `validateUpload()` reads the magic
|
|
57
|
+
bytes (PNG, JPEG, GIF, WebP, PDF, ZIP/OOXML, SVG, MP4, HTML, plain text) and rejects any
|
|
58
|
+
payload whose bytes contradict the declaration. Checks run cheapest-first: key → size →
|
|
59
|
+
allowlist → sniff → checksum.
|
|
60
|
+
|
|
61
|
+
`validateUpload({ key, declaredContentType, bytes }, uploadPolicy({ maxBytes: 5e6 }))`
|
|
62
|
+
|
|
63
|
+
## Errors
|
|
64
|
+
|
|
65
|
+
| Code | Fires when |
|
|
66
|
+
|---|---|
|
|
67
|
+
| `X_STORAGE_DISK_UNKNOWN` | `disk(name)` is not in `storage.disks`; cause lists the real ones |
|
|
68
|
+
| `X_STORAGE_NOT_FOUND` | `get`/`stream` on a key that does not exist |
|
|
69
|
+
| `X_STORAGE_PATH_UNSAFE` | traversal, absolute key, backslash, NUL, `%2e`, empty segment |
|
|
70
|
+
| `X_STORAGE_TOO_LARGE` | payload over the policy `maxBytes` |
|
|
71
|
+
| `X_STORAGE_TYPE_REJECTED` | declared type off the allowlist, or contradicted by magic bytes |
|
|
72
|
+
| `X_STORAGE_CHECKSUM_MISMATCH` | supplied base64 SHA-256 does not describe the bytes |
|
|
73
|
+
| `X_NOT_IMPLEMENTED` | S3 user metadata |
|
|
74
|
+
| `X_IMAGE_UNSUPPORTED` | core's: an `avif`/`webp` encode, or a source no built-in decoder reads |
|
|
75
|
+
| `X_IMAGE_DECODE_FAILED` | core's: truncated or corrupt image bytes |
|
|
76
|
+
|
|
77
|
+
## Images
|
|
78
|
+
|
|
79
|
+
`variantKey()`, `srcsetDescriptors()`, `fitDimensions()` are pure — `@ultimat3/seo` builds
|
|
80
|
+
`srcset` from them without decoding a byte.
|
|
81
|
+
|
|
82
|
+
`transformImage()` and `blurPlaceholder()` are real, over `@ultimat3/core`'s zero-dependency
|
|
83
|
+
pipeline. **It encodes `png` and `jpeg`, nothing else** — `avif`/`webp` remain key and `srcset`
|
|
84
|
+
math, and asking for their bytes rejects with core's `X_IMAGE_UNSUPPORTED` naming the two that
|
|
85
|
+
work; produce them through a CDN or a custom `ImageTransformDriver`. `png` is the only output
|
|
86
|
+
that keeps alpha. The encoded size is always exactly `fitDimensions()`, so the `width`/`height`
|
|
87
|
+
`@ultimat3/seo` already wrote into the tag match the bytes — `contain` fits inside the box, it
|
|
88
|
+
does not letterbox to it. `blurPlaceholder()` returns a real 16px-wide PNG `data:` URI.
|
|
89
|
+
`bun test` from `packages/storage`.
|
package/package.json
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@ultimat3/storage",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"description": "Named disks over Bun.file and Bun.s3: safe keys, signed URLs, sniffed uploads",
|
|
5
|
+
"license": "MIT",
|
|
6
|
+
"type": "module",
|
|
7
|
+
"repository": {
|
|
8
|
+
"type": "git",
|
|
9
|
+
"url": "git+https://github.com/developerz-ai/ultimate.git",
|
|
10
|
+
"directory": "packages/storage"
|
|
11
|
+
},
|
|
12
|
+
"publishConfig": {
|
|
13
|
+
"access": "public",
|
|
14
|
+
"provenance": true
|
|
15
|
+
},
|
|
16
|
+
"exports": {
|
|
17
|
+
".": "./src/index.ts"
|
|
18
|
+
},
|
|
19
|
+
"files": [
|
|
20
|
+
"src",
|
|
21
|
+
"!src/**/*.test.ts",
|
|
22
|
+
"README.md",
|
|
23
|
+
"LICENSE"
|
|
24
|
+
],
|
|
25
|
+
"engines": {
|
|
26
|
+
"bun": ">=1.3.0"
|
|
27
|
+
},
|
|
28
|
+
"scripts": {
|
|
29
|
+
"typecheck": "tsc --noEmit -p tsconfig.json",
|
|
30
|
+
"test": "bun test"
|
|
31
|
+
},
|
|
32
|
+
"dependencies": {
|
|
33
|
+
"@ultimat3/core": "1.0.0"
|
|
34
|
+
}
|
|
35
|
+
}
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
// Single responsibility: the dev-default disk — a real, working driver over `Bun.file` /
|
|
2
|
+
// `Bun.write` rooted at one directory, so `x dev` needs no MinIO and no cloud account.
|
|
3
|
+
// Content type, etag and user metadata live in a sidecar under `.meta/`: a POSIX file has
|
|
4
|
+
// nowhere to keep them, and `get` must round-trip exactly what `put` was handed.
|
|
5
|
+
|
|
6
|
+
import { type Clock, systemClock } from '@ultimat3/core';
|
|
7
|
+
import {
|
|
8
|
+
DEFAULT_CONTENT_TYPE,
|
|
9
|
+
DEFAULT_LIST_LIMIT,
|
|
10
|
+
etagOf,
|
|
11
|
+
type ListOptions,
|
|
12
|
+
type ListPage,
|
|
13
|
+
type PutOptions,
|
|
14
|
+
type SignedUrlOptions,
|
|
15
|
+
type StorageBody,
|
|
16
|
+
type StorageDriver,
|
|
17
|
+
type StorageObject,
|
|
18
|
+
type StorageRead,
|
|
19
|
+
sha256Base64,
|
|
20
|
+
toBytes,
|
|
21
|
+
} from './driver';
|
|
22
|
+
import { checksumMismatch, objectNotFound } from './errors';
|
|
23
|
+
import { assertSafeKey } from './path';
|
|
24
|
+
import { buildSignedUrl } from './signed-url';
|
|
25
|
+
|
|
26
|
+
const META_DIR = '.meta';
|
|
27
|
+
const DRIVER_NAME = 'local';
|
|
28
|
+
|
|
29
|
+
export interface LocalDriverOptions {
|
|
30
|
+
/** Directory the disk owns outright. Created on first write. */
|
|
31
|
+
readonly root: string;
|
|
32
|
+
/** HMAC secret for signed URLs. Production must pass one; dev falls back to a fixed string. */
|
|
33
|
+
readonly signingSecret?: string | undefined;
|
|
34
|
+
/** Route prefix the dev server serves signed URLs from. */
|
|
35
|
+
readonly baseUrl?: string | undefined;
|
|
36
|
+
readonly clock?: Clock | undefined;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
interface Sidecar {
|
|
40
|
+
readonly contentType: string;
|
|
41
|
+
readonly etag: string;
|
|
42
|
+
readonly cacheControl?: string | undefined;
|
|
43
|
+
readonly metadata?: Readonly<Record<string, string>> | undefined;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function parseSidecar(raw: unknown): Sidecar | undefined {
|
|
47
|
+
if (typeof raw !== 'object' || raw === null) return undefined;
|
|
48
|
+
const record = raw as Record<string, unknown>;
|
|
49
|
+
const contentType = record['contentType'];
|
|
50
|
+
const etag = record['etag'];
|
|
51
|
+
if (typeof contentType !== 'string' || typeof etag !== 'string') return undefined;
|
|
52
|
+
return { contentType, etag };
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export function localDriver(options: LocalDriverOptions): StorageDriver {
|
|
56
|
+
const root = options.root.replace(/\/+$/, '');
|
|
57
|
+
const clock = options.clock ?? systemClock;
|
|
58
|
+
const baseUrl = options.baseUrl ?? `/_storage/${DRIVER_NAME}`;
|
|
59
|
+
// A dev disk must work with zero config; a production disk that forgets the secret still
|
|
60
|
+
// gets a *shared* secret, never a per-process random one that breaks on restart.
|
|
61
|
+
const secret =
|
|
62
|
+
options.signingSecret ?? process.env['STORAGE_SIGNING_SECRET'] ?? 'ultimate-dev-signing-secret';
|
|
63
|
+
|
|
64
|
+
const filePath = (key: string): string => `${root}/${key}`;
|
|
65
|
+
const metaPath = (key: string): string => `${root}/${META_DIR}/${key}.json`;
|
|
66
|
+
|
|
67
|
+
const readSidecar = async (key: string): Promise<Sidecar | undefined> => {
|
|
68
|
+
const file = Bun.file(metaPath(key));
|
|
69
|
+
if (!(await file.exists())) return undefined;
|
|
70
|
+
try {
|
|
71
|
+
const raw: unknown = await file.json();
|
|
72
|
+
return parseSidecar(raw);
|
|
73
|
+
} catch {
|
|
74
|
+
return undefined;
|
|
75
|
+
}
|
|
76
|
+
};
|
|
77
|
+
|
|
78
|
+
const head = async (key: string): Promise<StorageObject | undefined> => {
|
|
79
|
+
const file = Bun.file(filePath(key));
|
|
80
|
+
if (!(await file.exists())) return undefined;
|
|
81
|
+
const sidecar = await readSidecar(key);
|
|
82
|
+
// Only hash when the sidecar is gone — `list()` must not read every file it lists.
|
|
83
|
+
const etag = sidecar?.etag ?? etagOf(new Uint8Array(await file.arrayBuffer()));
|
|
84
|
+
return {
|
|
85
|
+
key,
|
|
86
|
+
size: file.size,
|
|
87
|
+
contentType: sidecar?.contentType ?? DEFAULT_CONTENT_TYPE,
|
|
88
|
+
etag,
|
|
89
|
+
lastModified: new Date(file.lastModified),
|
|
90
|
+
};
|
|
91
|
+
};
|
|
92
|
+
|
|
93
|
+
return {
|
|
94
|
+
name: DRIVER_NAME,
|
|
95
|
+
|
|
96
|
+
async put(key: string, body: StorageBody, putOptions?: PutOptions): Promise<StorageObject> {
|
|
97
|
+
const safe = assertSafeKey(key);
|
|
98
|
+
const bytes = await toBytes(body);
|
|
99
|
+
const claimed = putOptions?.checksum;
|
|
100
|
+
if (claimed !== undefined) {
|
|
101
|
+
const actual = sha256Base64(bytes);
|
|
102
|
+
if (claimed !== actual) throw checksumMismatch(safe, claimed, actual);
|
|
103
|
+
}
|
|
104
|
+
const sidecar: Sidecar = {
|
|
105
|
+
contentType: putOptions?.contentType ?? DEFAULT_CONTENT_TYPE,
|
|
106
|
+
etag: etagOf(bytes),
|
|
107
|
+
cacheControl: putOptions?.cacheControl,
|
|
108
|
+
metadata: putOptions?.metadata,
|
|
109
|
+
};
|
|
110
|
+
await Bun.write(filePath(safe), bytes);
|
|
111
|
+
await Bun.write(metaPath(safe), JSON.stringify(sidecar));
|
|
112
|
+
return {
|
|
113
|
+
key: safe,
|
|
114
|
+
size: bytes.byteLength,
|
|
115
|
+
contentType: sidecar.contentType,
|
|
116
|
+
etag: sidecar.etag,
|
|
117
|
+
lastModified: clock.now(),
|
|
118
|
+
};
|
|
119
|
+
},
|
|
120
|
+
|
|
121
|
+
async get(key: string): Promise<StorageRead> {
|
|
122
|
+
const safe = assertSafeKey(key);
|
|
123
|
+
const object = await head(safe);
|
|
124
|
+
if (object === undefined) throw objectNotFound(DRIVER_NAME, safe);
|
|
125
|
+
const bytes = new Uint8Array(await Bun.file(filePath(safe)).arrayBuffer());
|
|
126
|
+
return { object, bytes };
|
|
127
|
+
},
|
|
128
|
+
|
|
129
|
+
async stream(key: string): Promise<ReadableStream<Uint8Array>> {
|
|
130
|
+
const safe = assertSafeKey(key);
|
|
131
|
+
const file = Bun.file(filePath(safe));
|
|
132
|
+
if (!(await file.exists())) throw objectNotFound(DRIVER_NAME, safe);
|
|
133
|
+
return file.stream();
|
|
134
|
+
},
|
|
135
|
+
|
|
136
|
+
async delete(key: string): Promise<void> {
|
|
137
|
+
const safe = assertSafeKey(key);
|
|
138
|
+
// Idempotent by contract: a missing key is already in the desired state.
|
|
139
|
+
await Bun.file(filePath(safe))
|
|
140
|
+
.delete()
|
|
141
|
+
.catch(() => undefined);
|
|
142
|
+
await Bun.file(metaPath(safe))
|
|
143
|
+
.delete()
|
|
144
|
+
.catch(() => undefined);
|
|
145
|
+
},
|
|
146
|
+
|
|
147
|
+
async exists(key: string): Promise<boolean> {
|
|
148
|
+
return Bun.file(filePath(assertSafeKey(key))).exists();
|
|
149
|
+
},
|
|
150
|
+
|
|
151
|
+
async list(listOptions?: ListOptions): Promise<ListPage> {
|
|
152
|
+
const prefix = listOptions?.prefix ?? '';
|
|
153
|
+
const limit = listOptions?.limit ?? DEFAULT_LIST_LIMIT;
|
|
154
|
+
const cursor = listOptions?.cursor;
|
|
155
|
+
const keys: string[] = [];
|
|
156
|
+
try {
|
|
157
|
+
for await (const entry of new Bun.Glob('**/*').scan({ cwd: root, onlyFiles: true })) {
|
|
158
|
+
const key = entry.replaceAll('\\', '/');
|
|
159
|
+
if (key.startsWith(`${META_DIR}/`)) continue;
|
|
160
|
+
if (!key.startsWith(prefix)) continue;
|
|
161
|
+
// The cursor IS the last key of the previous page — lexicographic order keeps it stable.
|
|
162
|
+
if (cursor !== undefined && key <= cursor) continue;
|
|
163
|
+
keys.push(key);
|
|
164
|
+
}
|
|
165
|
+
} catch {
|
|
166
|
+
// A disk nobody has written to yet has no directory: an empty listing, not an error.
|
|
167
|
+
return { objects: [], truncated: false };
|
|
168
|
+
}
|
|
169
|
+
keys.sort();
|
|
170
|
+
const page = keys.slice(0, limit);
|
|
171
|
+
const objects: StorageObject[] = [];
|
|
172
|
+
for (const key of page) {
|
|
173
|
+
const object = await head(key);
|
|
174
|
+
if (object !== undefined) objects.push(object);
|
|
175
|
+
}
|
|
176
|
+
const truncated = keys.length > page.length;
|
|
177
|
+
const last = page.at(-1);
|
|
178
|
+
return truncated && last !== undefined
|
|
179
|
+
? { objects, truncated, cursor: last }
|
|
180
|
+
: { objects, truncated: false };
|
|
181
|
+
},
|
|
182
|
+
|
|
183
|
+
async signedUrl(key: string, urlOptions?: SignedUrlOptions): Promise<string> {
|
|
184
|
+
return buildSignedUrl({
|
|
185
|
+
secret,
|
|
186
|
+
key: assertSafeKey(key),
|
|
187
|
+
method: urlOptions?.method,
|
|
188
|
+
expiresInMs: urlOptions?.expiresInMs,
|
|
189
|
+
maxBytes: urlOptions?.maxBytes,
|
|
190
|
+
contentType: urlOptions?.contentType,
|
|
191
|
+
baseUrl,
|
|
192
|
+
clock,
|
|
193
|
+
});
|
|
194
|
+
},
|
|
195
|
+
};
|
|
196
|
+
}
|
package/src/driver-s3.ts
ADDED
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
// Single responsibility: the production disk over Bun's native S3 client. One driver covers
|
|
2
|
+
// MinIO, Cloudflare R2 and AWS — the difference is `endpoint` + `forcePathStyle`, nothing else.
|
|
3
|
+
// The client is built lazily on first use so importing this module never opens a socket, and
|
|
4
|
+
// credentials arrive as env var NAMES: a literal key in app.config.ts is a key in git.
|
|
5
|
+
|
|
6
|
+
import { ConfigInvalidError, EnvMissingError } from '@ultimat3/core';
|
|
7
|
+
import {
|
|
8
|
+
DEFAULT_CONTENT_TYPE,
|
|
9
|
+
DEFAULT_LIST_LIMIT,
|
|
10
|
+
type ListOptions,
|
|
11
|
+
type ListPage,
|
|
12
|
+
type PutOptions,
|
|
13
|
+
type SignedUrlOptions,
|
|
14
|
+
type StorageBody,
|
|
15
|
+
type StorageDriver,
|
|
16
|
+
type StorageObject,
|
|
17
|
+
type StorageRead,
|
|
18
|
+
sha256Base64,
|
|
19
|
+
toBytes,
|
|
20
|
+
} from './driver';
|
|
21
|
+
import { checksumMismatch, objectNotFound, storageNotImplemented } from './errors';
|
|
22
|
+
import { assertSafeKey } from './path';
|
|
23
|
+
|
|
24
|
+
const DRIVER_NAME = 's3';
|
|
25
|
+
|
|
26
|
+
/** Structural view of `Bun.S3Client` — typing it here keeps `bun-types` out of the contract. */
|
|
27
|
+
export interface S3FileLike {
|
|
28
|
+
write(data: Uint8Array | Blob, options?: { type?: string }): Promise<number>;
|
|
29
|
+
arrayBuffer(): Promise<ArrayBuffer>;
|
|
30
|
+
exists(): Promise<boolean>;
|
|
31
|
+
delete(): Promise<void>;
|
|
32
|
+
stream(): ReadableStream<Uint8Array>;
|
|
33
|
+
stat(): Promise<S3StatLike>;
|
|
34
|
+
presign(options: { method?: string; expiresIn?: number; type?: string }): string;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export interface S3StatLike {
|
|
38
|
+
readonly size: number;
|
|
39
|
+
readonly type?: string | undefined;
|
|
40
|
+
readonly etag?: string | undefined;
|
|
41
|
+
readonly lastModified?: string | Date | undefined;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export interface S3ListEntryLike {
|
|
45
|
+
readonly key?: string | undefined;
|
|
46
|
+
readonly size?: number | undefined;
|
|
47
|
+
readonly eTag?: string | undefined;
|
|
48
|
+
readonly lastModified?: string | Date | undefined;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export interface S3ListResultLike {
|
|
52
|
+
readonly contents?: readonly S3ListEntryLike[] | undefined;
|
|
53
|
+
readonly isTruncated?: boolean | undefined;
|
|
54
|
+
readonly nextContinuationToken?: string | undefined;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export interface S3ClientLike {
|
|
58
|
+
file(key: string): S3FileLike;
|
|
59
|
+
list(input: {
|
|
60
|
+
prefix?: string;
|
|
61
|
+
maxKeys?: number;
|
|
62
|
+
continuationToken?: string;
|
|
63
|
+
}): Promise<S3ListResultLike>;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export interface S3DriverOptions {
|
|
67
|
+
readonly bucket: string;
|
|
68
|
+
readonly region?: string | undefined;
|
|
69
|
+
/** MinIO: `http://localhost:9000`. R2: `https://<account>.r2.cloudflarestorage.com`. */
|
|
70
|
+
readonly endpoint?: string | undefined;
|
|
71
|
+
/** MinIO needs `true`; AWS and R2 do not. */
|
|
72
|
+
readonly forcePathStyle?: boolean | undefined;
|
|
73
|
+
/** Env var NAME holding the key id. Default `S3_ACCESS_KEY_ID`. */
|
|
74
|
+
readonly accessKeyIdEnv?: string | undefined;
|
|
75
|
+
/** Env var NAME holding the secret. Default `S3_SECRET_ACCESS_KEY`. */
|
|
76
|
+
readonly secretAccessKeyEnv?: string | undefined;
|
|
77
|
+
readonly sessionTokenEnv?: string | undefined;
|
|
78
|
+
/** Injected in tests; production reads `process.env`. */
|
|
79
|
+
readonly env?: Readonly<Record<string, string | undefined>> | undefined;
|
|
80
|
+
/** Injected in tests; production constructs `Bun.S3Client`. */
|
|
81
|
+
readonly client?: S3ClientLike | undefined;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
interface S3ClientConstructor {
|
|
85
|
+
new (options: Record<string, unknown>): S3ClientLike;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function requireEnv(
|
|
89
|
+
env: Readonly<Record<string, string | undefined>>,
|
|
90
|
+
name: string,
|
|
91
|
+
partner: string,
|
|
92
|
+
): string {
|
|
93
|
+
const value = env[name];
|
|
94
|
+
if (value === undefined || value === '') {
|
|
95
|
+
throw new EnvMissingError({
|
|
96
|
+
cause: `${name} is not set, so the s3 disk cannot authenticate`,
|
|
97
|
+
fix: `set ${name} and ${partner} in .env (or the container's secret store), then re-run`,
|
|
98
|
+
meta: { missing: name },
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
return value;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
function buildClient(options: S3DriverOptions): S3ClientLike {
|
|
105
|
+
if (options.client !== undefined) return options.client;
|
|
106
|
+
if (options.bucket === '') {
|
|
107
|
+
throw new ConfigInvalidError({
|
|
108
|
+
cause: 's3 disk was defined without a bucket',
|
|
109
|
+
fix: 'set storage.disks.<name>.bucket in app.config.ts',
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
const env = options.env ?? process.env;
|
|
113
|
+
const idVar = options.accessKeyIdEnv ?? 'S3_ACCESS_KEY_ID';
|
|
114
|
+
const secretVar = options.secretAccessKeyEnv ?? 'S3_SECRET_ACCESS_KEY';
|
|
115
|
+
const Client = (Bun as unknown as { S3Client?: S3ClientConstructor }).S3Client;
|
|
116
|
+
if (Client === undefined) {
|
|
117
|
+
throw new ConfigInvalidError({
|
|
118
|
+
cause: 'Bun.S3Client is unavailable in this runtime',
|
|
119
|
+
fix: 'upgrade the runtime: bun upgrade # the s3 disk needs bun >= 1.3',
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
const tokenVar = options.sessionTokenEnv;
|
|
123
|
+
const sessionToken = tokenVar === undefined ? undefined : env[tokenVar];
|
|
124
|
+
// Bun's flag is the inverse: MinIO's path style means "not virtual hosted".
|
|
125
|
+
const pathStyle = options.forcePathStyle;
|
|
126
|
+
return new Client({
|
|
127
|
+
bucket: options.bucket,
|
|
128
|
+
accessKeyId: requireEnv(env, idVar, secretVar),
|
|
129
|
+
secretAccessKey: requireEnv(env, secretVar, idVar),
|
|
130
|
+
...(options.region === undefined ? {} : { region: options.region }),
|
|
131
|
+
...(options.endpoint === undefined ? {} : { endpoint: options.endpoint }),
|
|
132
|
+
...(pathStyle === undefined ? {} : { virtualHostedStyle: !pathStyle }),
|
|
133
|
+
...(sessionToken === undefined ? {} : { sessionToken }),
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
const toDate = (value: string | Date | undefined): Date =>
|
|
138
|
+
value === undefined ? new Date(0) : value instanceof Date ? value : new Date(value);
|
|
139
|
+
|
|
140
|
+
export function s3Driver(options: S3DriverOptions): StorageDriver {
|
|
141
|
+
let client: S3ClientLike | undefined;
|
|
142
|
+
const conn = (): S3ClientLike => {
|
|
143
|
+
client ??= buildClient(options);
|
|
144
|
+
return client;
|
|
145
|
+
};
|
|
146
|
+
|
|
147
|
+
const statObject = async (key: string): Promise<StorageObject> => {
|
|
148
|
+
const stat = await conn().file(key).stat();
|
|
149
|
+
return {
|
|
150
|
+
key,
|
|
151
|
+
size: stat.size,
|
|
152
|
+
contentType: stat.type ?? DEFAULT_CONTENT_TYPE,
|
|
153
|
+
etag: stat.etag ?? '',
|
|
154
|
+
lastModified: toDate(stat.lastModified),
|
|
155
|
+
};
|
|
156
|
+
};
|
|
157
|
+
|
|
158
|
+
return {
|
|
159
|
+
name: DRIVER_NAME,
|
|
160
|
+
|
|
161
|
+
async put(key: string, body: StorageBody, putOptions?: PutOptions): Promise<StorageObject> {
|
|
162
|
+
const safe = assertSafeKey(key);
|
|
163
|
+
if (putOptions?.metadata !== undefined || putOptions?.cacheControl !== undefined) {
|
|
164
|
+
const uri = `s3://${options.bucket}/${safe}`;
|
|
165
|
+
throw storageNotImplemented(
|
|
166
|
+
'user metadata and cache-control on the s3 driver (Bun exposes no header hook yet)',
|
|
167
|
+
`drop metadata/cacheControl from put(), or set them out of band: ` +
|
|
168
|
+
`aws s3 cp ${uri} ${uri} --metadata-directive REPLACE`,
|
|
169
|
+
);
|
|
170
|
+
}
|
|
171
|
+
// Buffered on purpose: size and checksum must be known before the object exists.
|
|
172
|
+
// Multipart streaming upload is a later optimisation, not a correctness change.
|
|
173
|
+
const bytes = await toBytes(body);
|
|
174
|
+
const claimed = putOptions?.checksum;
|
|
175
|
+
if (claimed !== undefined) {
|
|
176
|
+
const actual = sha256Base64(bytes);
|
|
177
|
+
if (claimed !== actual) throw checksumMismatch(safe, claimed, actual);
|
|
178
|
+
}
|
|
179
|
+
await conn()
|
|
180
|
+
.file(safe)
|
|
181
|
+
.write(bytes, { type: putOptions?.contentType ?? DEFAULT_CONTENT_TYPE });
|
|
182
|
+
return statObject(safe);
|
|
183
|
+
},
|
|
184
|
+
|
|
185
|
+
async get(key: string): Promise<StorageRead> {
|
|
186
|
+
const safe = assertSafeKey(key);
|
|
187
|
+
const file = conn().file(safe);
|
|
188
|
+
if (!(await file.exists())) throw objectNotFound(DRIVER_NAME, safe);
|
|
189
|
+
const bytes = new Uint8Array(await file.arrayBuffer());
|
|
190
|
+
return { object: await statObject(safe), bytes };
|
|
191
|
+
},
|
|
192
|
+
|
|
193
|
+
async stream(key: string): Promise<ReadableStream<Uint8Array>> {
|
|
194
|
+
const safe = assertSafeKey(key);
|
|
195
|
+
const file = conn().file(safe);
|
|
196
|
+
if (!(await file.exists())) throw objectNotFound(DRIVER_NAME, safe);
|
|
197
|
+
return file.stream();
|
|
198
|
+
},
|
|
199
|
+
|
|
200
|
+
async delete(key: string): Promise<void> {
|
|
201
|
+
await conn()
|
|
202
|
+
.file(assertSafeKey(key))
|
|
203
|
+
.delete()
|
|
204
|
+
.catch(() => undefined);
|
|
205
|
+
},
|
|
206
|
+
|
|
207
|
+
async exists(key: string): Promise<boolean> {
|
|
208
|
+
return conn().file(assertSafeKey(key)).exists();
|
|
209
|
+
},
|
|
210
|
+
|
|
211
|
+
async list(listOptions?: ListOptions): Promise<ListPage> {
|
|
212
|
+
const result = await conn().list({
|
|
213
|
+
maxKeys: listOptions?.limit ?? DEFAULT_LIST_LIMIT,
|
|
214
|
+
...(listOptions?.prefix === undefined ? {} : { prefix: listOptions.prefix }),
|
|
215
|
+
...(listOptions?.cursor === undefined ? {} : { continuationToken: listOptions.cursor }),
|
|
216
|
+
});
|
|
217
|
+
const objects: StorageObject[] = [];
|
|
218
|
+
for (const entry of result.contents ?? []) {
|
|
219
|
+
if (entry.key === undefined) continue;
|
|
220
|
+
objects.push({
|
|
221
|
+
key: entry.key,
|
|
222
|
+
size: entry.size ?? 0,
|
|
223
|
+
contentType: DEFAULT_CONTENT_TYPE,
|
|
224
|
+
etag: entry.eTag ?? '',
|
|
225
|
+
lastModified: toDate(entry.lastModified),
|
|
226
|
+
});
|
|
227
|
+
}
|
|
228
|
+
const cursor = result.nextContinuationToken;
|
|
229
|
+
return result.isTruncated === true && cursor !== undefined
|
|
230
|
+
? { objects, truncated: true, cursor }
|
|
231
|
+
: { objects, truncated: false };
|
|
232
|
+
},
|
|
233
|
+
|
|
234
|
+
/**
|
|
235
|
+
* Provider presigning. The signature covers method, expiry and content type but NOT
|
|
236
|
+
* `maxBytes` — S3 has no header for it, so size stays a server-side policy check.
|
|
237
|
+
*/
|
|
238
|
+
async signedUrl(key: string, urlOptions?: SignedUrlOptions): Promise<string> {
|
|
239
|
+
const expiresInMs = urlOptions?.expiresInMs ?? 900_000;
|
|
240
|
+
return conn()
|
|
241
|
+
.file(assertSafeKey(key))
|
|
242
|
+
.presign({
|
|
243
|
+
method: urlOptions?.method ?? 'GET',
|
|
244
|
+
expiresIn: Math.ceil(expiresInMs / 1000),
|
|
245
|
+
...(urlOptions?.contentType === undefined ? {} : { type: urlOptions.contentType }),
|
|
246
|
+
});
|
|
247
|
+
},
|
|
248
|
+
};
|
|
249
|
+
}
|
package/src/driver.ts
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
// Single responsibility: the driver contract every disk implements, plus the byte helpers all
|
|
2
|
+
// drivers share (body normalisation, SHA-256 etag/checksum). Bytes never travel as strings —
|
|
3
|
+
// a base64 round trip through JSON is how a "small" upload becomes a 33%-larger OOM.
|
|
4
|
+
|
|
5
|
+
export type StorageBody = Uint8Array | ReadableStream<Uint8Array> | Blob;
|
|
6
|
+
|
|
7
|
+
export interface StorageObject {
|
|
8
|
+
readonly key: string;
|
|
9
|
+
readonly size: number;
|
|
10
|
+
readonly contentType: string;
|
|
11
|
+
readonly etag: string;
|
|
12
|
+
readonly lastModified: Date;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export interface PutOptions {
|
|
16
|
+
readonly contentType?: string | undefined;
|
|
17
|
+
readonly cacheControl?: string | undefined;
|
|
18
|
+
readonly metadata?: Readonly<Record<string, string>> | undefined;
|
|
19
|
+
/** base64 SHA-256 of the body. Supplied means verified: a mismatch is a rejected write. */
|
|
20
|
+
readonly checksum?: string | undefined;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export interface ListOptions {
|
|
24
|
+
readonly prefix?: string | undefined;
|
|
25
|
+
/** Opaque; pass back the `cursor` of the previous page. */
|
|
26
|
+
readonly cursor?: string | undefined;
|
|
27
|
+
readonly limit?: number | undefined;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export interface ListPage {
|
|
31
|
+
readonly objects: readonly StorageObject[];
|
|
32
|
+
readonly truncated: boolean;
|
|
33
|
+
/** Absent when `truncated` is false. */
|
|
34
|
+
readonly cursor?: string | undefined;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export interface StorageRead {
|
|
38
|
+
readonly object: StorageObject;
|
|
39
|
+
readonly bytes: Uint8Array;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export type SignedUrlMethod = 'GET' | 'PUT';
|
|
43
|
+
|
|
44
|
+
export interface SignedUrlOptions {
|
|
45
|
+
readonly method?: SignedUrlMethod | undefined;
|
|
46
|
+
readonly expiresInMs?: number | undefined;
|
|
47
|
+
/** Constrains a `PUT`: the signature covers it, so a client cannot widen it. */
|
|
48
|
+
readonly maxBytes?: number | undefined;
|
|
49
|
+
readonly contentType?: string | undefined;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export interface StorageDriver {
|
|
53
|
+
/** Disk-independent driver name (`local`, `s3`) — appears in every error cause. */
|
|
54
|
+
readonly name: string;
|
|
55
|
+
put(key: string, body: StorageBody, options?: PutOptions): Promise<StorageObject>;
|
|
56
|
+
get(key: string): Promise<StorageRead>;
|
|
57
|
+
/** Bytes without buffering — the only safe path for anything over a few MB. */
|
|
58
|
+
stream(key: string): Promise<ReadableStream<Uint8Array>>;
|
|
59
|
+
/** Idempotent: deleting an absent key is not an error. */
|
|
60
|
+
delete(key: string): Promise<void>;
|
|
61
|
+
exists(key: string): Promise<boolean>;
|
|
62
|
+
list(options?: ListOptions): Promise<ListPage>;
|
|
63
|
+
signedUrl(key: string, options?: SignedUrlOptions): Promise<string>;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export const DEFAULT_CONTENT_TYPE = 'application/octet-stream';
|
|
67
|
+
export const DEFAULT_LIST_LIMIT = 1000;
|
|
68
|
+
|
|
69
|
+
export async function toBytes(body: StorageBody): Promise<Uint8Array> {
|
|
70
|
+
if (body instanceof Uint8Array) return body;
|
|
71
|
+
if (body instanceof Blob) return new Uint8Array(await body.arrayBuffer());
|
|
72
|
+
return new Uint8Array(await new Response(body).arrayBuffer());
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** base64 SHA-256 — the wire form for `PutOptions.checksum` and `ValidatedUpload.checksum`. */
|
|
76
|
+
export function sha256Base64(bytes: Uint8Array): string {
|
|
77
|
+
return new Bun.CryptoHasher('sha256').update(bytes).digest('base64');
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** Local etag. S3 returns the provider's etag instead, so never compare across drivers. */
|
|
81
|
+
export function etagOf(bytes: Uint8Array): string {
|
|
82
|
+
return new Bun.CryptoHasher('sha256').update(bytes).digest('hex').slice(0, 32);
|
|
83
|
+
}
|