@ultimat3/storage 22.15.0 → 24.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +46 -7
- package/README.md +119 -8
- package/package.json +3 -2
- package/src/accept.ts +3 -6
- package/src/attachment.ts +40 -7
- package/src/driver-local-read.ts +104 -0
- package/src/driver-local-sidecar.ts +38 -0
- package/src/driver-local-write.ts +185 -0
- package/src/driver-local.ts +87 -164
- package/src/driver-memory.ts +220 -0
- package/src/driver-s3-region.ts +39 -0
- package/src/driver-s3.ts +129 -31
- package/src/driver.ts +63 -1
- package/src/errors.ts +99 -15
- package/src/grant.ts +2 -2
- package/src/image.ts +34 -8
- package/src/index.ts +16 -9
- package/src/iso-bmff.ts +50 -0
- package/src/path.ts +17 -2
- package/src/signed-url.ts +41 -8
- package/src/signing-secret.ts +71 -0
- package/src/upload.ts +31 -5
- package/src/driver-s3-fixture.ts +0 -164
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
// Single responsibility: the disk a TEST holds — a `StorageDriver` over one `Map`, answering every
|
|
2
|
+
// question `localDriver` answers and refusing what it refuses, with nothing on a filesystem to
|
|
3
|
+
// create or clean up. Not a deployment's disk: a restart is every object gone.
|
|
4
|
+
//
|
|
5
|
+
// It exists because the alternative was written by hand in every suite that needed a bucket: a
|
|
6
|
+
// fake with three methods and five `unsupported()` stubs, each a slightly different contract.
|
|
7
|
+
|
|
8
|
+
import { type Clock, finiteCount, NotImplementedError, systemClock } from '@ultimat3/core';
|
|
9
|
+
import {
|
|
10
|
+
assertListOptions,
|
|
11
|
+
assertPutOptions,
|
|
12
|
+
DEFAULT_CONTENT_TYPE,
|
|
13
|
+
etagOf,
|
|
14
|
+
type ListOptions,
|
|
15
|
+
type ListPage,
|
|
16
|
+
type PutOptions,
|
|
17
|
+
resolveListLimit,
|
|
18
|
+
type SignedUrlOptions,
|
|
19
|
+
type StorageBody,
|
|
20
|
+
type StorageDriver,
|
|
21
|
+
type StorageObject,
|
|
22
|
+
type StorageRead,
|
|
23
|
+
sha256Base64,
|
|
24
|
+
toBytes,
|
|
25
|
+
} from './driver';
|
|
26
|
+
import type { LocalDriverOptions } from './driver-local';
|
|
27
|
+
import { checksumMismatch, getTooLarge, objectNotFound } from './errors';
|
|
28
|
+
import { assertSafeKey } from './path';
|
|
29
|
+
import type { SignedUrlVerification } from './signed-url';
|
|
30
|
+
import { buildSignedUrl, signedUrlBaseFor, verifySignedUrl } from './signed-url';
|
|
31
|
+
import { resolveSigningSecret } from './signing-secret';
|
|
32
|
+
import { DEFAULT_MAX_UPLOAD_BYTES } from './upload';
|
|
33
|
+
|
|
34
|
+
const DRIVER_NAME = 'memory';
|
|
35
|
+
|
|
36
|
+
/** `localDriver`'s options, minus the directory there is none of. */
|
|
37
|
+
export type MemoryDriverOptions = Omit<LocalDriverOptions, 'root'>;
|
|
38
|
+
|
|
39
|
+
export interface MemoryStorageDriver extends StorageDriver {
|
|
40
|
+
/**
|
|
41
|
+
* Every stored object's bytes by key, COPIED — what an operator holding the bucket could read.
|
|
42
|
+
* For the assertion a test makes about the store itself: "nothing in it is a cookie value".
|
|
43
|
+
*/
|
|
44
|
+
objects(): ReadonlyMap<string, Uint8Array>;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
interface Stored {
|
|
48
|
+
readonly bytes: Uint8Array;
|
|
49
|
+
readonly object: StorageObject;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* An object no caller shares with the store. `metadata` and `lastModified` are mutable by anyone
|
|
54
|
+
* holding a reference, and the local disk's sidecar is a copy by construction — a test that
|
|
55
|
+
* mutated what `put()` was handed must not see a different object on the next `get()`.
|
|
56
|
+
*/
|
|
57
|
+
const snapshot = (object: StorageObject): StorageObject => ({
|
|
58
|
+
...object,
|
|
59
|
+
...(object.lastModified === undefined
|
|
60
|
+
? {}
|
|
61
|
+
: { lastModified: new Date(object.lastModified.getTime()) }),
|
|
62
|
+
...(object.metadata === undefined ? {} : { metadata: { ...object.metadata } }),
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
export function memoryDriver(options: MemoryDriverOptions = {}): MemoryStorageDriver {
|
|
66
|
+
const maxPutBytes = finiteCount(
|
|
67
|
+
'the memory disk driver',
|
|
68
|
+
'maxPutBytes',
|
|
69
|
+
options.maxPutBytes === undefined ? DEFAULT_MAX_UPLOAD_BYTES : options.maxPutBytes,
|
|
70
|
+
1,
|
|
71
|
+
);
|
|
72
|
+
const maxGetBytes = finiteCount(
|
|
73
|
+
'the memory disk driver',
|
|
74
|
+
'maxGetBytes',
|
|
75
|
+
options.maxGetBytes === undefined ? maxPutBytes : options.maxGetBytes,
|
|
76
|
+
1,
|
|
77
|
+
);
|
|
78
|
+
const clock = options.clock ?? systemClock;
|
|
79
|
+
let baseUrl = options.baseUrl ?? signedUrlBaseFor(DRIVER_NAME);
|
|
80
|
+
const secret = resolveSigningSecret(DRIVER_NAME, options);
|
|
81
|
+
const stored = new Map<string, Stored>();
|
|
82
|
+
|
|
83
|
+
const found = (key: string): Stored => {
|
|
84
|
+
const hit = stored.get(key);
|
|
85
|
+
if (hit === undefined) throw objectNotFound(DRIVER_NAME, key);
|
|
86
|
+
return hit;
|
|
87
|
+
};
|
|
88
|
+
|
|
89
|
+
return {
|
|
90
|
+
name: DRIVER_NAME,
|
|
91
|
+
|
|
92
|
+
get signedUrlBase(): string {
|
|
93
|
+
return baseUrl;
|
|
94
|
+
},
|
|
95
|
+
|
|
96
|
+
registerAs(diskName: string): void {
|
|
97
|
+
if (options.baseUrl === undefined) baseUrl = signedUrlBaseFor(diskName);
|
|
98
|
+
},
|
|
99
|
+
|
|
100
|
+
objects(): ReadonlyMap<string, Uint8Array> {
|
|
101
|
+
return new Map([...stored].map(([key, value]) => [key, value.bytes.slice()]));
|
|
102
|
+
},
|
|
103
|
+
|
|
104
|
+
async put(key: string, body: StorageBody, putOptions?: PutOptions): Promise<StorageObject> {
|
|
105
|
+
const safe = assertSafeKey(key);
|
|
106
|
+
// The same refusal both shipped disks make: a `put()` that succeeds in a test and throws in
|
|
107
|
+
// production is a gap an app meets on the worst day.
|
|
108
|
+
if (putOptions?.serverSideEncryption !== undefined) {
|
|
109
|
+
throw new NotImplementedError({
|
|
110
|
+
cause:
|
|
111
|
+
'server-side encryption on the memory driver (it holds bytes in a Map) is not implemented by this driver — drop serverSideEncryption from put() and encrypt the disk itself: an s3Driver over a bucket with a default KMS rule',
|
|
112
|
+
fix: 'put(key, body)',
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
assertPutOptions(DRIVER_NAME, putOptions);
|
|
116
|
+
const bytes = await toBytes(body, { driver: DRIVER_NAME, key: safe, maxBytes: maxPutBytes });
|
|
117
|
+
const claimed = putOptions?.checksum;
|
|
118
|
+
if (claimed !== undefined) {
|
|
119
|
+
const actual = sha256Base64(bytes);
|
|
120
|
+
if (claimed !== actual) throw checksumMismatch(safe, claimed, actual);
|
|
121
|
+
}
|
|
122
|
+
const object: StorageObject = {
|
|
123
|
+
key: safe,
|
|
124
|
+
size: bytes.byteLength,
|
|
125
|
+
contentType: putOptions?.contentType ?? DEFAULT_CONTENT_TYPE,
|
|
126
|
+
etag: etagOf(bytes),
|
|
127
|
+
lastModified: clock.now(),
|
|
128
|
+
...(putOptions?.cacheControl === undefined
|
|
129
|
+
? {}
|
|
130
|
+
: { cacheControl: putOptions.cacheControl }),
|
|
131
|
+
...(putOptions?.metadata === undefined ? {} : { metadata: putOptions.metadata }),
|
|
132
|
+
};
|
|
133
|
+
// A copy in, as a file write is: the caller's buffer and metadata are the caller's to reuse.
|
|
134
|
+
stored.set(safe, { bytes: bytes.slice(), object: snapshot(object) });
|
|
135
|
+
return snapshot(object);
|
|
136
|
+
},
|
|
137
|
+
|
|
138
|
+
async stat(key: string): Promise<StorageObject | undefined> {
|
|
139
|
+
const hit = stored.get(assertSafeKey(key));
|
|
140
|
+
return hit && snapshot(hit.object);
|
|
141
|
+
},
|
|
142
|
+
|
|
143
|
+
async get(key: string): Promise<StorageRead> {
|
|
144
|
+
const safe = assertSafeKey(key);
|
|
145
|
+
const hit = found(safe);
|
|
146
|
+
// The local disk's refusal, so a suite on this disk meets the ceiling production has.
|
|
147
|
+
if (hit.bytes.byteLength > maxGetBytes) {
|
|
148
|
+
throw getTooLarge(DRIVER_NAME, safe, hit.bytes.byteLength, maxGetBytes);
|
|
149
|
+
}
|
|
150
|
+
return { object: snapshot(hit.object), bytes: hit.bytes.slice() };
|
|
151
|
+
},
|
|
152
|
+
|
|
153
|
+
async stream(key: string): Promise<ReadableStream<Uint8Array>> {
|
|
154
|
+
const hit = found(assertSafeKey(key));
|
|
155
|
+
return new Blob([hit.bytes.slice()]).stream();
|
|
156
|
+
},
|
|
157
|
+
|
|
158
|
+
async copy(from: string, to: string): Promise<StorageObject> {
|
|
159
|
+
// Both keys before either lookup, as the other two disks do: an unsafe destination is the
|
|
160
|
+
// refusal whether or not the source exists.
|
|
161
|
+
const [from_, destination] = [assertSafeKey(from), assertSafeKey(to)];
|
|
162
|
+
const source = found(from_);
|
|
163
|
+
const object: StorageObject = {
|
|
164
|
+
...source.object,
|
|
165
|
+
key: destination,
|
|
166
|
+
lastModified: clock.now(),
|
|
167
|
+
};
|
|
168
|
+
stored.set(destination, { bytes: source.bytes.slice(), object: snapshot(object) });
|
|
169
|
+
return snapshot(object);
|
|
170
|
+
},
|
|
171
|
+
|
|
172
|
+
async delete(key: string): Promise<void> {
|
|
173
|
+
stored.delete(assertSafeKey(key));
|
|
174
|
+
},
|
|
175
|
+
|
|
176
|
+
async exists(key: string): Promise<boolean> {
|
|
177
|
+
return stored.has(assertSafeKey(key));
|
|
178
|
+
},
|
|
179
|
+
|
|
180
|
+
async list(listOptions?: ListOptions): Promise<ListPage> {
|
|
181
|
+
const prefix = listOptions?.prefix ?? '';
|
|
182
|
+
assertListOptions(listOptions);
|
|
183
|
+
const limit = resolveListLimit(listOptions?.limit);
|
|
184
|
+
const cursor = listOptions?.cursor;
|
|
185
|
+
// Code-unit order and "the cursor is the last key of the page before" — the local disk's own.
|
|
186
|
+
const keys = [...stored.keys()]
|
|
187
|
+
.filter((key) => key.startsWith(prefix) && (cursor === undefined || key > cursor))
|
|
188
|
+
.sort();
|
|
189
|
+
const page = keys.slice(0, limit);
|
|
190
|
+
const objects = page.flatMap((key) => {
|
|
191
|
+
const hit = stored.get(key);
|
|
192
|
+
return hit === undefined ? [] : [snapshot(hit.object)];
|
|
193
|
+
});
|
|
194
|
+
const last = page.at(-1);
|
|
195
|
+
return keys.length > page.length && last !== undefined
|
|
196
|
+
? { objects, truncated: true, cursor: last }
|
|
197
|
+
: { objects, truncated: false };
|
|
198
|
+
},
|
|
199
|
+
|
|
200
|
+
async signedUrl(key: string, urlOptions?: SignedUrlOptions): Promise<string> {
|
|
201
|
+
return buildSignedUrl({
|
|
202
|
+
secret,
|
|
203
|
+
key: assertSafeKey(key),
|
|
204
|
+
method: urlOptions?.method,
|
|
205
|
+
expiresInMs: urlOptions?.expiresInMs,
|
|
206
|
+
maxBytes: urlOptions?.maxBytes,
|
|
207
|
+
contentType: urlOptions?.contentType,
|
|
208
|
+
baseUrl,
|
|
209
|
+
clock,
|
|
210
|
+
});
|
|
211
|
+
},
|
|
212
|
+
|
|
213
|
+
async verifySigned(input: {
|
|
214
|
+
readonly url: string;
|
|
215
|
+
readonly clock?: Clock | undefined;
|
|
216
|
+
}): Promise<SignedUrlVerification> {
|
|
217
|
+
return verifySignedUrl({ url: input.url, secret, baseUrl, clock: input.clock ?? clock });
|
|
218
|
+
},
|
|
219
|
+
};
|
|
220
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
// Single responsibility: recognise the one provider refusal that names its own repair — a request
|
|
2
|
+
// signed for the wrong region — and turn it into the disk-misconfigured error it is. Split from
|
|
3
|
+
// driver-s3.ts because every call that sends a signed request asks this before its own verdict.
|
|
4
|
+
|
|
5
|
+
import { ConfigInvalidError, stringField } from '@ultimat3/core';
|
|
6
|
+
|
|
7
|
+
/** What S3 and every S3-compatible gateway answer a SigV4 scope with the wrong region in it. */
|
|
8
|
+
const REGION_MISMATCH_CODE = 'AuthorizationHeaderMalformed';
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* `the region "auto" is wrong; expecting "us-east-1"` — double quotes from a gateway, single from
|
|
12
|
+
* AWS. The captures are held to a region name's alphabet, so a sentence this process did not
|
|
13
|
+
* write can put nothing else into a `fix:` line.
|
|
14
|
+
*/
|
|
15
|
+
const REGION_SENTENCE =
|
|
16
|
+
/the region ['"]([A-Za-z0-9-]{1,64})['"] is wrong; expecting ['"]([A-Za-z0-9-]{1,64})['"]/;
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* The refusal as `X_CONFIG_INVALID`, or `undefined` when `error` is anything else. A region
|
|
20
|
+
* mismatch is the DISK misconfigured, exactly as a missing bucket is, so it carries that code from
|
|
21
|
+
* every call rather than `X_STORAGE_LIST_FAILED` with a fix about `s3:ListBucket` from one and a
|
|
22
|
+
* bare `S3Error` from another: `Bun.S3Client` signs for `auto` when no region is set, and a
|
|
23
|
+
* provider that checks the scope refuses every request the app ever sends.
|
|
24
|
+
*
|
|
25
|
+
* A HEAD carries no body, so `exists()` and `stat()` never see this sentence — the first PUT,
|
|
26
|
+
* LIST or DELETE does.
|
|
27
|
+
*/
|
|
28
|
+
export function regionMismatch(bucket: string, error: unknown): ConfigInvalidError | undefined {
|
|
29
|
+
if (stringField(error, 'code') !== REGION_MISMATCH_CODE) return undefined;
|
|
30
|
+
const match = REGION_SENTENCE.exec(stringField(error, 'message') ?? '');
|
|
31
|
+
const sent = match?.[1];
|
|
32
|
+
const expected = match?.[2];
|
|
33
|
+
if (sent === undefined || expected === undefined) return undefined;
|
|
34
|
+
return new ConfigInvalidError({
|
|
35
|
+
cause: `the s3 disk for bucket "${bucket}" signed a request for region "${sent}" and its endpoint expects "${expected}"`,
|
|
36
|
+
fix: `set S3_REGION=${expected} in .env (or pass region: '${expected}' to s3Driver), then re-run`,
|
|
37
|
+
meta: { bucket, sent, expected },
|
|
38
|
+
});
|
|
39
|
+
}
|
package/src/driver-s3.ts
CHANGED
|
@@ -1,10 +1,20 @@
|
|
|
1
|
-
// Single responsibility: the production disk over Bun's native S3 client. One driver covers
|
|
2
|
-
//
|
|
1
|
+
// Single responsibility: the production disk over Bun's native S3 client. One driver covers AWS,
|
|
2
|
+
// Cloudflare R2 and any S3-compatible gateway — the difference is `endpoint`, `region` and
|
|
3
|
+
// `forcePathStyle`, nothing else.
|
|
3
4
|
// The client is built lazily on first use so importing this module never opens a socket, and
|
|
4
5
|
// credentials arrive as env var NAMES: a literal key in app.config.ts is a key in git.
|
|
5
6
|
|
|
6
|
-
import { ConfigInvalidError, EnvMissingError, finiteCount, stringField } from '@ultimat3/core';
|
|
7
7
|
import {
|
|
8
|
+
ConfigInvalidError,
|
|
9
|
+
EnvMissingError,
|
|
10
|
+
finiteCount,
|
|
11
|
+
NotImplementedError,
|
|
12
|
+
renderFixShellArg,
|
|
13
|
+
stringField,
|
|
14
|
+
} from '@ultimat3/core';
|
|
15
|
+
import {
|
|
16
|
+
assertListOptions,
|
|
17
|
+
assertPutOptions,
|
|
8
18
|
DEFAULT_CONTENT_TYPE,
|
|
9
19
|
type ListOptions,
|
|
10
20
|
type ListPage,
|
|
@@ -19,12 +29,16 @@ import {
|
|
|
19
29
|
sha256Base64,
|
|
20
30
|
toBytes,
|
|
21
31
|
} from './driver';
|
|
32
|
+
import { regionMismatch } from './driver-s3-region';
|
|
22
33
|
import {
|
|
23
34
|
checksumMismatch,
|
|
24
35
|
deleteFailed,
|
|
36
|
+
getTooLarge,
|
|
37
|
+
isStorageError,
|
|
25
38
|
listFailed,
|
|
26
39
|
objectNotFound,
|
|
27
|
-
|
|
40
|
+
putFailed,
|
|
41
|
+
readFailed,
|
|
28
42
|
} from './errors';
|
|
29
43
|
import { assertSafeKey } from './path';
|
|
30
44
|
import { DEFAULT_SIGNED_URL_TTL_MS } from './signed-url';
|
|
@@ -75,10 +89,14 @@ export interface S3ClientLike {
|
|
|
75
89
|
|
|
76
90
|
export interface S3DriverOptions {
|
|
77
91
|
readonly bucket: string;
|
|
92
|
+
/**
|
|
93
|
+
* The region requests are signed for. Unset, `Bun.S3Client` signs for `auto` — which R2 wants,
|
|
94
|
+
* and which AWS and a gateway that checks the scope refuse (`X_CONFIG_INVALID`, naming theirs).
|
|
95
|
+
*/
|
|
78
96
|
readonly region?: string | undefined;
|
|
79
|
-
/**
|
|
97
|
+
/** A self-hosted gateway: `http://localhost:9000`. R2: `https://<account>.r2.cloudflarestorage.com`. */
|
|
80
98
|
readonly endpoint?: string | undefined;
|
|
81
|
-
/**
|
|
99
|
+
/** A gateway addressed by host and port needs `true`; AWS and R2 do not. */
|
|
82
100
|
readonly forcePathStyle?: boolean | undefined;
|
|
83
101
|
/** Env var NAME holding the key id. Default `S3_ACCESS_KEY_ID`. */
|
|
84
102
|
readonly accessKeyIdEnv?: string | undefined;
|
|
@@ -96,6 +114,12 @@ export interface S3DriverOptions {
|
|
|
96
114
|
* and S3's single-PUT limit is 5GB regardless of what this says.
|
|
97
115
|
*/
|
|
98
116
|
readonly maxPutBytes?: number | undefined;
|
|
117
|
+
/**
|
|
118
|
+
* Ceiling on ONE `get()`, because `get()` buffers the whole object — and on this disk nothing
|
|
119
|
+
* bounds what a presigned PUT stored (`SignedUrlOptions.maxBytes`). Defaults to `maxPutBytes`;
|
|
120
|
+
* an object past it is read with `stream()`.
|
|
121
|
+
*/
|
|
122
|
+
readonly maxGetBytes?: number | undefined;
|
|
99
123
|
}
|
|
100
124
|
|
|
101
125
|
interface S3ClientConstructor {
|
|
@@ -138,7 +162,7 @@ function buildClient(options: S3DriverOptions): S3ClientLike {
|
|
|
138
162
|
}
|
|
139
163
|
const tokenVar = options.sessionTokenEnv;
|
|
140
164
|
const sessionToken = tokenVar === undefined ? undefined : env[tokenVar];
|
|
141
|
-
// Bun's flag is the inverse:
|
|
165
|
+
// Bun's flag is the inverse: path style means "not virtual hosted".
|
|
142
166
|
const pathStyle = options.forcePathStyle;
|
|
143
167
|
return new Client({
|
|
144
168
|
bucket: options.bucket,
|
|
@@ -151,8 +175,12 @@ function buildClient(options: S3DriverOptions): S3ClientLike {
|
|
|
151
175
|
});
|
|
152
176
|
}
|
|
153
177
|
|
|
154
|
-
|
|
155
|
-
|
|
178
|
+
/**
|
|
179
|
+
* A missing `LastModified` is ABSENT, never `new Date(0)`: epoch 0 is older than every window, so
|
|
180
|
+
* `sweepOrphans` deleted an upload on the strength of a field the provider never sent.
|
|
181
|
+
*/
|
|
182
|
+
const dated = (value: string | Date | undefined): { readonly lastModified?: Date } =>
|
|
183
|
+
value === undefined ? {} : { lastModified: value instanceof Date ? value : new Date(value) };
|
|
156
184
|
|
|
157
185
|
/** One numeric field off a value that may fight being read — `stringField`'s missing twin. */
|
|
158
186
|
function numberField(value: unknown, key: string): number | undefined {
|
|
@@ -190,12 +218,12 @@ function isAbsentObject(error: unknown): boolean {
|
|
|
190
218
|
*/
|
|
191
219
|
function refuseUnsupportedPut(bucket: string, key: string, putOptions?: PutOptions): void {
|
|
192
220
|
if (putOptions?.metadata !== undefined || putOptions?.cacheControl !== undefined) {
|
|
193
|
-
const uri = `s3://${bucket}/${key}
|
|
194
|
-
throw
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
);
|
|
221
|
+
const uri = renderFixShellArg(`s3://${bucket}/${key}`, '<s3-uri>');
|
|
222
|
+
throw new NotImplementedError({
|
|
223
|
+
cause:
|
|
224
|
+
'user metadata and cache-control on the s3 driver (Bun exposes no header hook yet) is not implemented by this driver — drop metadata/cacheControl from put(), or set them out of band with the command in fix',
|
|
225
|
+
fix: `aws s3 cp ${uri} ${uri} --metadata-directive REPLACE`,
|
|
226
|
+
});
|
|
199
227
|
}
|
|
200
228
|
const sse = putOptions?.serverSideEncryption;
|
|
201
229
|
if (sse === undefined) return;
|
|
@@ -203,10 +231,11 @@ function refuseUnsupportedPut(bucket: string, key: string, putOptions?: PutOptio
|
|
|
203
231
|
sse.algorithm === 'aws:kms'
|
|
204
232
|
? `{"SSEAlgorithm":"aws:kms","KMSMasterKeyID":"${sse.kmsKeyId ?? '<key-arn>'}"}`
|
|
205
233
|
: '{"SSEAlgorithm":"AES256"}';
|
|
206
|
-
throw
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
234
|
+
throw new NotImplementedError({
|
|
235
|
+
cause:
|
|
236
|
+
'per-object server-side encryption on the s3 driver (Bun.S3Client exposes acl, storageClass and type, and nothing for x-amz-server-side-encryption) is not implemented by this driver — set it bucket-wide with the command in fix, then drop serverSideEncryption from put()',
|
|
237
|
+
fix: `aws s3api put-bucket-encryption --bucket ${renderFixShellArg(bucket, '<bucket>')} --server-side-encryption-configuration '{"Rules":[{"ApplyServerSideEncryptionByDefault":${rule},"BucketKeyEnabled":true}]}'`,
|
|
238
|
+
});
|
|
210
239
|
}
|
|
211
240
|
|
|
212
241
|
export function s3Driver(options: S3DriverOptions): StorageDriver {
|
|
@@ -218,20 +247,73 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
|
|
|
218
247
|
options.maxPutBytes === undefined ? DEFAULT_MAX_UPLOAD_BYTES : options.maxPutBytes,
|
|
219
248
|
1,
|
|
220
249
|
);
|
|
250
|
+
const maxGetBytes = finiteCount(
|
|
251
|
+
'the s3 driver',
|
|
252
|
+
'maxGetBytes',
|
|
253
|
+
options.maxGetBytes === undefined ? maxPutBytes : options.maxGetBytes,
|
|
254
|
+
1,
|
|
255
|
+
);
|
|
221
256
|
let client: S3ClientLike | undefined;
|
|
222
257
|
const conn = (): S3ClientLike => {
|
|
223
258
|
client ??= buildClient(options);
|
|
224
259
|
return client;
|
|
225
260
|
};
|
|
226
261
|
|
|
262
|
+
/**
|
|
263
|
+
* A refused request as the disk-misconfigured error, when the provider named its own repair (a
|
|
264
|
+
* wrong region). Asked before any call's own verdict, so the fix an operator reads is the one
|
|
265
|
+
* that works rather than one about a grant they already have. `misconfigured` is the same
|
|
266
|
+
* question for a call with no verdict of its own: it rethrows whatever arrived.
|
|
267
|
+
*/
|
|
268
|
+
const throwIfMisconfigured = (error: unknown): void => {
|
|
269
|
+
const mismatch = regionMismatch(options.bucket, error);
|
|
270
|
+
if (mismatch !== undefined) throw mismatch;
|
|
271
|
+
};
|
|
272
|
+
/**
|
|
273
|
+
* A write the provider refused, coded: a denied `s3:PutObject`, a throttle, an expired
|
|
274
|
+
* credential. The bare `S3Error` used to escape `put()` and `copy()` exactly as it once escaped
|
|
275
|
+
* `list()` — no code, no fix, an anonymous 500.
|
|
276
|
+
*/
|
|
277
|
+
const refusedWrite =
|
|
278
|
+
(key: string) =>
|
|
279
|
+
(error: unknown): never => {
|
|
280
|
+
throwIfMisconfigured(error);
|
|
281
|
+
throw putFailed(
|
|
282
|
+
DRIVER_NAME,
|
|
283
|
+
key,
|
|
284
|
+
error,
|
|
285
|
+
`aws s3api get-bucket-policy --bucket ${renderFixShellArg(options.bucket, '<bucket>')}`,
|
|
286
|
+
);
|
|
287
|
+
};
|
|
288
|
+
|
|
289
|
+
/**
|
|
290
|
+
* A read the provider refused, coded — `refusedWrite`'s twin. A 404 is "not found" wherever it
|
|
291
|
+
* arrives: an object deleted between `exists()` and the read is absent, not unreadable.
|
|
292
|
+
*/
|
|
293
|
+
const refusedRead =
|
|
294
|
+
(key: string) =>
|
|
295
|
+
(error: unknown): never => {
|
|
296
|
+
if (isStorageError(error)) throw error;
|
|
297
|
+
if (isAbsentObject(error)) throw objectNotFound(DRIVER_NAME, key);
|
|
298
|
+
throwIfMisconfigured(error);
|
|
299
|
+
throw readFailed(
|
|
300
|
+
DRIVER_NAME,
|
|
301
|
+
key,
|
|
302
|
+
error,
|
|
303
|
+
`aws s3api head-object --bucket ${renderFixShellArg(options.bucket, '<bucket>')} --key ${renderFixShellArg(key, '<key>')}`,
|
|
304
|
+
);
|
|
305
|
+
};
|
|
306
|
+
const present = (key: string): Promise<boolean> =>
|
|
307
|
+
conn().file(key).exists().catch(refusedRead(key));
|
|
308
|
+
|
|
227
309
|
const statObject = async (key: string): Promise<StorageObject> => {
|
|
228
|
-
const stat = await conn().file(key).stat();
|
|
310
|
+
const stat = await conn().file(key).stat().catch(refusedRead(key));
|
|
229
311
|
return {
|
|
230
312
|
key,
|
|
231
313
|
size: stat.size,
|
|
232
314
|
contentType: stat.type ?? DEFAULT_CONTENT_TYPE,
|
|
233
315
|
etag: stat.etag ?? '',
|
|
234
|
-
|
|
316
|
+
...dated(stat.lastModified),
|
|
235
317
|
};
|
|
236
318
|
};
|
|
237
319
|
|
|
@@ -241,6 +323,7 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
|
|
|
241
323
|
async put(key: string, body: StorageBody, putOptions?: PutOptions): Promise<StorageObject> {
|
|
242
324
|
const safe = assertSafeKey(key);
|
|
243
325
|
refuseUnsupportedPut(options.bucket, safe, putOptions);
|
|
326
|
+
assertPutOptions(DRIVER_NAME, putOptions);
|
|
244
327
|
// Buffered on purpose: size and checksum must be known before the object exists — so this
|
|
245
328
|
// path is for objects that FIT IN MEMORY, and `maxPutBytes` is what makes that a contract
|
|
246
329
|
// rather than a hope. User uploads never come through here: they go direct to the bucket
|
|
@@ -253,22 +336,33 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
|
|
|
253
336
|
}
|
|
254
337
|
await conn()
|
|
255
338
|
.file(safe)
|
|
256
|
-
.write(bytes, { type: putOptions?.contentType ?? DEFAULT_CONTENT_TYPE })
|
|
339
|
+
.write(bytes, { type: putOptions?.contentType ?? DEFAULT_CONTENT_TYPE })
|
|
340
|
+
.catch(refusedWrite(safe));
|
|
257
341
|
return statObject(safe);
|
|
258
342
|
},
|
|
259
343
|
|
|
344
|
+
async stat(key: string): Promise<StorageObject | undefined> {
|
|
345
|
+
const safe = assertSafeKey(key);
|
|
346
|
+
return (await present(safe)) ? statObject(safe) : undefined;
|
|
347
|
+
},
|
|
348
|
+
|
|
260
349
|
async get(key: string): Promise<StorageRead> {
|
|
261
350
|
const safe = assertSafeKey(key);
|
|
262
351
|
const file = conn().file(safe);
|
|
263
|
-
if (!(await
|
|
264
|
-
|
|
265
|
-
|
|
352
|
+
if (!(await present(safe))) throw objectNotFound(DRIVER_NAME, safe);
|
|
353
|
+
// The HEAD comes FIRST, and it is the whole point: a client PUT straight into the bucket is
|
|
354
|
+
// bounded by nothing on this disk, so `arrayBuffer()` on whatever is there was heap growth
|
|
355
|
+
// the uploader chose. Refused on the provider's own size before a byte is read.
|
|
356
|
+
const object = await statObject(safe);
|
|
357
|
+
if (object.size > maxGetBytes) throw getTooLarge(DRIVER_NAME, safe, object.size, maxGetBytes);
|
|
358
|
+
const bytes = new Uint8Array(await file.arrayBuffer().catch(refusedRead(safe)));
|
|
359
|
+
return { object, bytes };
|
|
266
360
|
},
|
|
267
361
|
|
|
268
362
|
async stream(key: string): Promise<ReadableStream<Uint8Array>> {
|
|
269
363
|
const safe = assertSafeKey(key);
|
|
270
364
|
const file = conn().file(safe);
|
|
271
|
-
if (!(await
|
|
365
|
+
if (!(await present(safe))) throw objectNotFound(DRIVER_NAME, safe);
|
|
272
366
|
return file.stream();
|
|
273
367
|
},
|
|
274
368
|
|
|
@@ -283,11 +377,12 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
|
|
|
283
377
|
const source = assertSafeKey(from);
|
|
284
378
|
const destination = assertSafeKey(to);
|
|
285
379
|
const file = conn().file(source);
|
|
286
|
-
if (!(await
|
|
287
|
-
const stat = await file.stat();
|
|
380
|
+
if (!(await present(source))) throw objectNotFound(DRIVER_NAME, source);
|
|
381
|
+
const stat = await file.stat().catch(refusedRead(source));
|
|
288
382
|
await conn()
|
|
289
383
|
.file(destination)
|
|
290
|
-
.write(file, { type: stat.type ?? DEFAULT_CONTENT_TYPE })
|
|
384
|
+
.write(file, { type: stat.type ?? DEFAULT_CONTENT_TYPE })
|
|
385
|
+
.catch(refusedWrite(destination));
|
|
291
386
|
return statObject(destination);
|
|
292
387
|
},
|
|
293
388
|
|
|
@@ -299,6 +394,7 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
|
|
|
299
394
|
// Idempotent means an ABSENT key, and nothing else. AWS answers DELETE on a missing key
|
|
300
395
|
// with 204, so this branch is for the providers that do not.
|
|
301
396
|
if (isAbsentObject(error)) return;
|
|
397
|
+
throwIfMisconfigured(error);
|
|
302
398
|
throw deleteFailed(
|
|
303
399
|
DRIVER_NAME,
|
|
304
400
|
safe,
|
|
@@ -309,13 +405,14 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
|
|
|
309
405
|
},
|
|
310
406
|
|
|
311
407
|
async exists(key: string): Promise<boolean> {
|
|
312
|
-
return
|
|
408
|
+
return present(assertSafeKey(key));
|
|
313
409
|
},
|
|
314
410
|
|
|
315
411
|
async list(listOptions?: ListOptions): Promise<ListPage> {
|
|
316
412
|
const prefix = listOptions?.prefix ?? '';
|
|
317
413
|
// Refused at the seam both drivers share, before the provider is asked: `maxKeys: 0` used to
|
|
318
414
|
// go straight through, while the local disk answered a complete-looking empty page.
|
|
415
|
+
assertListOptions(listOptions);
|
|
319
416
|
const maxKeys = resolveListLimit(listOptions?.limit);
|
|
320
417
|
// `conn()` OUTSIDE the try: a missing credential or an absent `Bun.S3Client` is this disk
|
|
321
418
|
// misconfigured, and it already answers with its own code and its own fix.
|
|
@@ -331,6 +428,7 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
|
|
|
331
428
|
// A bare `S3Error` used to escape here, uncoded: no `X_*`, no `fix`, no `--json` shape, and
|
|
332
429
|
// `@ultimat3/http`'s error map has nothing to turn it into but a 500. A denied
|
|
333
430
|
// `s3:ListBucket` is the commonest one and reads to an operator as an app crash.
|
|
431
|
+
throwIfMisconfigured(error);
|
|
334
432
|
throw listFailed(
|
|
335
433
|
DRIVER_NAME,
|
|
336
434
|
prefix,
|
|
@@ -350,7 +448,7 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
|
|
|
350
448
|
key: entry.key,
|
|
351
449
|
size: entry.size ?? 0,
|
|
352
450
|
etag: entry.eTag ?? '',
|
|
353
|
-
|
|
451
|
+
...dated(entry.lastModified),
|
|
354
452
|
});
|
|
355
453
|
}
|
|
356
454
|
const cursor = result.nextContinuationToken;
|
package/src/driver.ts
CHANGED
|
@@ -23,7 +23,12 @@ export interface StorageListEntry {
|
|
|
23
23
|
/** Absent means the driver's listing does not carry it. Never a fabricated default. */
|
|
24
24
|
readonly contentType?: string | undefined;
|
|
25
25
|
readonly etag: string;
|
|
26
|
-
|
|
26
|
+
/**
|
|
27
|
+
* ABSENT when the provider did not say — never epoch 0, which every age test reads as "older
|
|
28
|
+
* than any window", and never an invalid `Date`, which is a value a reader has to remember to
|
|
29
|
+
* test. A reader that needs an age handles `undefined`; `sweepOrphans` spares it.
|
|
30
|
+
*/
|
|
31
|
+
readonly lastModified?: Date | undefined;
|
|
27
32
|
/** Present only when the driver actually stored what `put()` was handed — never invented. */
|
|
28
33
|
readonly cacheControl?: string | undefined;
|
|
29
34
|
readonly metadata?: Readonly<Record<string, string>> | undefined;
|
|
@@ -115,6 +120,17 @@ export interface StorageDriver {
|
|
|
115
120
|
*/
|
|
116
121
|
registerAs?(diskName: string): void;
|
|
117
122
|
put(key: string, body: StorageBody, options?: PutOptions): Promise<StorageObject>;
|
|
123
|
+
/**
|
|
124
|
+
* What a read would say about the object, WITHOUT its bytes — `undefined` when nothing is there.
|
|
125
|
+
* `promoteAttachment` decides on `stat().size`, because a presigned PUT lands on the disk
|
|
126
|
+
* unmeasured and `get()` is the one call that must not be how its size is learned. Required: a
|
|
127
|
+
* driver that cannot measure cannot be promoted from, and that is a compile error, not a refusal.
|
|
128
|
+
*/
|
|
129
|
+
stat(key: string): Promise<StorageObject | undefined>;
|
|
130
|
+
/**
|
|
131
|
+
* The whole object, BUFFERED — so bounded: an object over the driver's `maxGetBytes` is
|
|
132
|
+
* `X_STORAGE_TOO_LARGE`, exactly as `put()` refuses a body over `maxPutBytes`. Past it, `stream()`.
|
|
133
|
+
*/
|
|
118
134
|
get(key: string): Promise<StorageRead>;
|
|
119
135
|
/** Bytes without buffering — the only safe path for anything over a few MB. */
|
|
120
136
|
stream(key: string): Promise<ReadableStream<Uint8Array>>;
|
|
@@ -181,6 +197,45 @@ export function resolveListLimit(limit: number | undefined): number {
|
|
|
181
197
|
return limit;
|
|
182
198
|
}
|
|
183
199
|
|
|
200
|
+
const isStringRecord = (value: unknown): boolean =>
|
|
201
|
+
typeof value === 'object' &&
|
|
202
|
+
value !== null &&
|
|
203
|
+
!Array.isArray(value) &&
|
|
204
|
+
Object.values(value).every((entry) => typeof entry === 'string');
|
|
205
|
+
|
|
206
|
+
/**
|
|
207
|
+
* What `put()` was told ABOUT the bytes, held to its type before a byte moves. An untyped caller's
|
|
208
|
+
* `metadata: { n: 1n }` reached `JSON.stringify` on the local disk as a bare `TypeError`, and the
|
|
209
|
+
* memory disk stored it — against the `Record<string, string>` every reader is typed on.
|
|
210
|
+
*/
|
|
211
|
+
export function assertPutOptions(driver: string, options: PutOptions | undefined): void {
|
|
212
|
+
if (options === undefined) return;
|
|
213
|
+
for (const name of ['contentType', 'cacheControl'] as const) {
|
|
214
|
+
assert(
|
|
215
|
+
options[name] === undefined || typeof options[name] === 'string',
|
|
216
|
+
`put() on the ${driver} disk was given a ${name} that is not a string`,
|
|
217
|
+
`put(key, body, { ${name}: 'a string' }) — or omit ${name}`,
|
|
218
|
+
);
|
|
219
|
+
}
|
|
220
|
+
assert(
|
|
221
|
+
options.metadata === undefined || isStringRecord(options.metadata),
|
|
222
|
+
`put() on the ${driver} disk was given metadata that is not a plain object of string values`,
|
|
223
|
+
"put(key, body, { metadata: { name: 'value' } }) — every value a string: String(n) a number, JSON.stringify anything nested",
|
|
224
|
+
);
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
/** A `prefix` or a `cursor` that is not a string reached `startsWith` and `<=` as whatever it was. */
|
|
228
|
+
export function assertListOptions(options: ListOptions | undefined): void {
|
|
229
|
+
for (const name of ['prefix', 'cursor'] as const) {
|
|
230
|
+
const value = options?.[name];
|
|
231
|
+
assert(
|
|
232
|
+
value === undefined || typeof value === 'string',
|
|
233
|
+
`list() was given a ${name} that is not a string`,
|
|
234
|
+
`disk.list({ ${name}: 'a string' }) — or omit ${name}`,
|
|
235
|
+
);
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
|
|
184
239
|
/** What `toBytes` needs to refuse a body: the ceiling, and the key to name in the refusal. */
|
|
185
240
|
export interface ByteLimit {
|
|
186
241
|
readonly driver: string;
|
|
@@ -212,6 +267,13 @@ export async function toBytes(body: StorageBody, limit: ByteLimit): Promise<Uint
|
|
|
212
267
|
}
|
|
213
268
|
return new Uint8Array(await body.arrayBuffer());
|
|
214
269
|
}
|
|
270
|
+
// The type says so; an untyped caller's `put(key, 'a string')` reached `getReader` as a
|
|
271
|
+
// bare `TypeError`.
|
|
272
|
+
assert(
|
|
273
|
+
body instanceof ReadableStream,
|
|
274
|
+
`put() on the ${limit.driver} disk was given a body that is not a Uint8Array, a Blob or a ReadableStream`,
|
|
275
|
+
"put(key, new TextEncoder().encode(text)) — bytes, never a string: encode it, or pass the request's own body stream",
|
|
276
|
+
);
|
|
215
277
|
return readBounded(body, limit);
|
|
216
278
|
}
|
|
217
279
|
|