@knowledgefutures/sdk 0.0.0-stage → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +71 -2
- package/dist/storage/binding.d.ts +76 -0
- package/dist/storage/binding.js +97 -0
- package/dist/storage/brokered.d.ts +21 -0
- package/dist/storage/brokered.js +94 -0
- package/dist/storage/errors.d.ts +27 -0
- package/dist/storage/errors.js +21 -0
- package/dist/storage/index.d.ts +34 -0
- package/dist/storage/index.js +77 -0
- package/dist/storage/s3.d.ts +22 -0
- package/dist/storage/s3.js +254 -0
- package/dist/storage/shared.d.ts +45 -0
- package/dist/storage/shared.js +167 -0
- package/dist/storage/sigv4.d.ts +50 -0
- package/dist/storage/sigv4.js +128 -0
- package/dist/storage/testing.d.ts +25 -0
- package/dist/storage/testing.js +242 -0
- package/dist/storage/types.d.ts +122 -0
- package/dist/storage/types.js +13 -0
- package/dist/whistle/hono.d.ts +42 -0
- package/dist/whistle/hono.js +89 -0
- package/dist/whistle/index.d.ts +135 -0
- package/dist/whistle/index.js +249 -0
- package/dist/whistle/wire.d.ts +64 -0
- package/dist/whistle/wire.js +98 -0
- package/package.json +34 -3
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
import { StorageError } from './errors.js';
|
|
2
|
+
import { attachment, checkExpiresIn, DOWNLOAD_EXPIRES_IN, UPLOAD_EXPIRES_IN, checkKey, checkMetadata, checkPrefix, checkSignUpload, DEFAULT_CONTENT_TYPE, IMMUTABLE, publicDisposition, SHORT_PUBLIC, toBytes, uploadAsPut, } from './shared.js';
|
|
3
|
+
import { encodeKey, EMPTY_SHA256, presign, sha256Hex, signRequest, } from './sigv4.js';
|
|
4
|
+
export function createS3Store(binding, options = {}) {
|
|
5
|
+
const doFetch = options.fetch ?? ((input, init) => fetch(input, init));
|
|
6
|
+
const credentials = {
|
|
7
|
+
accessKeyId: binding.accessKeyId,
|
|
8
|
+
secretAccessKey: binding.secretAccessKey,
|
|
9
|
+
region: binding.region,
|
|
10
|
+
...(options.sessionToken ? { sessionToken: options.sessionToken } : {}),
|
|
11
|
+
};
|
|
12
|
+
const isPublic = binding.publicBase !== undefined;
|
|
13
|
+
const objectUrl = (key) => new URL(`${binding.endpoint}/${binding.bucket}/${encodeKey(key)}`);
|
|
14
|
+
async function send(method, url, o = {}) {
|
|
15
|
+
const headers = await signRequest({
|
|
16
|
+
method,
|
|
17
|
+
url,
|
|
18
|
+
headers: o.headers ?? {},
|
|
19
|
+
payloadHash: o.payloadHash ?? EMPTY_SHA256,
|
|
20
|
+
credentials,
|
|
21
|
+
});
|
|
22
|
+
try {
|
|
23
|
+
return await doFetch(url, {
|
|
24
|
+
method,
|
|
25
|
+
headers,
|
|
26
|
+
...(o.body ? { body: o.body } : {}),
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
catch (e) {
|
|
30
|
+
throw new StorageError('unavailable', `storage is unreachable: ${e.message}`);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
async function put(key, body, opts = {}) {
|
|
34
|
+
checkKey(key);
|
|
35
|
+
const metadata = checkMetadata(opts.metadata);
|
|
36
|
+
const { bytes, contentType } = await toBytes(body, opts.contentType);
|
|
37
|
+
const cacheControl = opts.cacheControl ?? (isPublic ? SHORT_PUBLIC : undefined);
|
|
38
|
+
const disposition = isPublic
|
|
39
|
+
? publicDisposition(contentType, opts.contentDisposition)
|
|
40
|
+
: opts.contentDisposition;
|
|
41
|
+
const headers = { 'content-type': contentType };
|
|
42
|
+
if (cacheControl)
|
|
43
|
+
headers['cache-control'] = cacheControl;
|
|
44
|
+
if (disposition)
|
|
45
|
+
headers['content-disposition'] = disposition;
|
|
46
|
+
for (const [name, value] of Object.entries(metadata))
|
|
47
|
+
headers[`x-amz-meta-${name}`] = value;
|
|
48
|
+
const res = await send('PUT', objectUrl(key), {
|
|
49
|
+
headers,
|
|
50
|
+
body: bytes,
|
|
51
|
+
payloadHash: await sha256Hex(bytes),
|
|
52
|
+
});
|
|
53
|
+
if (!res.ok)
|
|
54
|
+
throw await failure(res, `put ${key}`);
|
|
55
|
+
await res.body?.cancel();
|
|
56
|
+
const stored = {
|
|
57
|
+
key,
|
|
58
|
+
size: bytes.byteLength,
|
|
59
|
+
etag: unquote(res.headers.get('etag') ?? ''),
|
|
60
|
+
uploaded: new Date(res.headers.get('date') ?? Date.now()),
|
|
61
|
+
contentType,
|
|
62
|
+
metadata,
|
|
63
|
+
};
|
|
64
|
+
if (cacheControl)
|
|
65
|
+
stored.cacheControl = cacheControl;
|
|
66
|
+
if (disposition)
|
|
67
|
+
stored.contentDisposition = disposition;
|
|
68
|
+
return stored;
|
|
69
|
+
}
|
|
70
|
+
async function head(key) {
|
|
71
|
+
checkKey(key);
|
|
72
|
+
const res = await send('HEAD', objectUrl(key));
|
|
73
|
+
if (res.status === 404)
|
|
74
|
+
return null;
|
|
75
|
+
if (!res.ok)
|
|
76
|
+
throw await failure(res, `head ${key}`);
|
|
77
|
+
return describe(key, res);
|
|
78
|
+
}
|
|
79
|
+
async function get(key) {
|
|
80
|
+
checkKey(key);
|
|
81
|
+
const res = await send('GET', objectUrl(key));
|
|
82
|
+
if (res.status === 404) {
|
|
83
|
+
await res.body?.cancel();
|
|
84
|
+
return null;
|
|
85
|
+
}
|
|
86
|
+
if (!res.ok)
|
|
87
|
+
throw await failure(res, `get ${key}`);
|
|
88
|
+
const meta = describe(key, res);
|
|
89
|
+
const body = res.body ?? new Response('').body;
|
|
90
|
+
return {
|
|
91
|
+
...meta,
|
|
92
|
+
body,
|
|
93
|
+
arrayBuffer: () => new Response(body).arrayBuffer(),
|
|
94
|
+
text: () => new Response(body).text(),
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
async function del(key) {
|
|
98
|
+
checkKey(key);
|
|
99
|
+
const res = await send('DELETE', objectUrl(key));
|
|
100
|
+
// 204 for a delete; 404 from some backends for a key that was never there — both are success.
|
|
101
|
+
if (!res.ok && res.status !== 404)
|
|
102
|
+
throw await failure(res, `delete ${key}`);
|
|
103
|
+
await res.body?.cancel();
|
|
104
|
+
}
|
|
105
|
+
async function list(opts = {}) {
|
|
106
|
+
checkPrefix(opts.prefix);
|
|
107
|
+
const limit = opts.limit ?? 1000;
|
|
108
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > 1000) {
|
|
109
|
+
throw new StorageError('invalid', 'limit is 1–1000');
|
|
110
|
+
}
|
|
111
|
+
const url = new URL(`${binding.endpoint}/${binding.bucket}`);
|
|
112
|
+
url.searchParams.set('list-type', '2');
|
|
113
|
+
url.searchParams.set('max-keys', String(limit));
|
|
114
|
+
if (opts.prefix)
|
|
115
|
+
url.searchParams.set('prefix', opts.prefix);
|
|
116
|
+
if (opts.cursor)
|
|
117
|
+
url.searchParams.set('continuation-token', opts.cursor);
|
|
118
|
+
const res = await send('GET', url);
|
|
119
|
+
if (!res.ok)
|
|
120
|
+
throw await failure(res, 'list');
|
|
121
|
+
return parseList(await res.text());
|
|
122
|
+
}
|
|
123
|
+
async function signUpload(key, o) {
|
|
124
|
+
checkKey(key);
|
|
125
|
+
checkSignUpload(o);
|
|
126
|
+
const expiresIn = checkExpiresIn(o.expiresIn, UPLOAD_EXPIRES_IN);
|
|
127
|
+
const cacheControl = o.cacheControl ?? (isPublic ? SHORT_PUBLIC : undefined);
|
|
128
|
+
// Sent by the browser. content-length is signed too, but the browser sets
|
|
129
|
+
// it from the body, so it is not in what the caller must send.
|
|
130
|
+
const sent = { 'content-type': o.contentType };
|
|
131
|
+
if (cacheControl)
|
|
132
|
+
sent['cache-control'] = cacheControl;
|
|
133
|
+
// Signed, so a browser cannot upload HTML to the public store as inline.
|
|
134
|
+
const disposition = isPublic ? publicDisposition(o.contentType, undefined) : undefined;
|
|
135
|
+
if (disposition)
|
|
136
|
+
sent['content-disposition'] = disposition;
|
|
137
|
+
const url = await presign({
|
|
138
|
+
method: 'PUT',
|
|
139
|
+
url: objectUrl(key),
|
|
140
|
+
headers: { ...sent, 'content-length': String(o.size) },
|
|
141
|
+
expiresIn,
|
|
142
|
+
credentials,
|
|
143
|
+
});
|
|
144
|
+
return {
|
|
145
|
+
key,
|
|
146
|
+
url,
|
|
147
|
+
method: 'PUT',
|
|
148
|
+
headers: sent,
|
|
149
|
+
expiresAt: new Date(Date.now() + expiresIn * 1000),
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
async function signDownload(key, o = {}) {
|
|
153
|
+
checkKey(key);
|
|
154
|
+
const expiresIn = checkExpiresIn(o.expiresIn, DOWNLOAD_EXPIRES_IN);
|
|
155
|
+
const url = objectUrl(key);
|
|
156
|
+
if (o.filename)
|
|
157
|
+
url.searchParams.set('response-content-disposition', attachment(o.filename));
|
|
158
|
+
return presign({ method: 'GET', url, expiresIn, credentials });
|
|
159
|
+
}
|
|
160
|
+
const store = {
|
|
161
|
+
put,
|
|
162
|
+
get,
|
|
163
|
+
head,
|
|
164
|
+
delete: del,
|
|
165
|
+
list,
|
|
166
|
+
signUpload,
|
|
167
|
+
signDownload,
|
|
168
|
+
upload: async (file, options) => {
|
|
169
|
+
const p = uploadAsPut(file, options, isPublic ? { cacheControl: IMMUTABLE } : {});
|
|
170
|
+
return put(p.key, p.body, p.options);
|
|
171
|
+
},
|
|
172
|
+
};
|
|
173
|
+
if (!isPublic)
|
|
174
|
+
return store;
|
|
175
|
+
const base = binding.publicBase;
|
|
176
|
+
return {
|
|
177
|
+
...store,
|
|
178
|
+
url: (key) => {
|
|
179
|
+
checkKey(key);
|
|
180
|
+
return `${base}/${encodeKey(key)}`;
|
|
181
|
+
},
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
/* ------------------------------ responses ------------------------------ */
|
|
185
|
+
function describe(key, res) {
|
|
186
|
+
const metadata = {};
|
|
187
|
+
for (const [name, value] of res.headers) {
|
|
188
|
+
if (name.startsWith('x-amz-meta-'))
|
|
189
|
+
metadata[name.slice('x-amz-meta-'.length)] = value;
|
|
190
|
+
}
|
|
191
|
+
const o = {
|
|
192
|
+
key,
|
|
193
|
+
size: Number(res.headers.get('content-length') ?? 0),
|
|
194
|
+
etag: unquote(res.headers.get('etag') ?? ''),
|
|
195
|
+
uploaded: new Date(res.headers.get('last-modified') ?? 0),
|
|
196
|
+
contentType: res.headers.get('content-type') ?? DEFAULT_CONTENT_TYPE,
|
|
197
|
+
metadata,
|
|
198
|
+
};
|
|
199
|
+
const cacheControl = res.headers.get('cache-control');
|
|
200
|
+
const disposition = res.headers.get('content-disposition');
|
|
201
|
+
if (cacheControl)
|
|
202
|
+
o.cacheControl = cacheControl;
|
|
203
|
+
if (disposition)
|
|
204
|
+
o.contentDisposition = disposition;
|
|
205
|
+
return o;
|
|
206
|
+
}
|
|
207
|
+
/** Every backend failure as a StorageError — the vendor's XML never reaches the caller. */
|
|
208
|
+
async function failure(res, what) {
|
|
209
|
+
const text = res.body ? await res.text().catch(() => '') : '';
|
|
210
|
+
const code = tag(text, 'Code') ?? '';
|
|
211
|
+
const message = tag(text, 'Message') ?? res.statusText;
|
|
212
|
+
const detail = `${what}: ${code || res.status}${message ? ` — ${message}` : ''}`;
|
|
213
|
+
if (res.status === 413 || code === 'EntityTooLarge') {
|
|
214
|
+
return new StorageError('too_large', detail, res.status);
|
|
215
|
+
}
|
|
216
|
+
if (res.status === 401 || res.status === 403)
|
|
217
|
+
return new StorageError('forbidden', detail, res.status);
|
|
218
|
+
if (code === 'NoSuchBucket') {
|
|
219
|
+
return new StorageError('not_configured', `${detail} (the binding names a bucket that does not exist)`, res.status);
|
|
220
|
+
}
|
|
221
|
+
if (res.status === 429 || res.status >= 500)
|
|
222
|
+
return new StorageError('unavailable', detail, res.status);
|
|
223
|
+
return new StorageError('invalid', detail, res.status);
|
|
224
|
+
}
|
|
225
|
+
/**
|
|
226
|
+
* ListObjectsV2 by hand: Workers have no DOMParser, and the response is a
|
|
227
|
+
* flat, well-known shape.
|
|
228
|
+
*/
|
|
229
|
+
export function parseList(xml) {
|
|
230
|
+
const objects = [...xml.matchAll(/<Contents>([\s\S]*?)<\/Contents>/g)].map(([, c]) => ({
|
|
231
|
+
key: tag(c, 'Key') ?? '',
|
|
232
|
+
size: Number(tag(c, 'Size') ?? 0),
|
|
233
|
+
etag: unquote(tag(c, 'ETag') ?? ''),
|
|
234
|
+
uploaded: new Date(tag(c, 'LastModified') ?? 0),
|
|
235
|
+
}));
|
|
236
|
+
const truncated = tag(xml, 'IsTruncated') === 'true';
|
|
237
|
+
const cursor = tag(xml, 'NextContinuationToken');
|
|
238
|
+
return truncated && cursor ? { objects, cursor } : { objects };
|
|
239
|
+
}
|
|
240
|
+
function tag(xml, name) {
|
|
241
|
+
const m = new RegExp(`<${name}>([\\s\\S]*?)</${name}>`).exec(xml);
|
|
242
|
+
return m ? unescapeXml(m[1]) : undefined;
|
|
243
|
+
}
|
|
244
|
+
function unescapeXml(s) {
|
|
245
|
+
return s.replace(/&(#x[0-9a-fA-F]+|#\d+|amp|lt|gt|quot|apos);/g, (_, e) => {
|
|
246
|
+
if (e[0] === '#') {
|
|
247
|
+
return String.fromCodePoint(e[1] === 'x' ? parseInt(e.slice(2), 16) : parseInt(e.slice(1), 10));
|
|
248
|
+
}
|
|
249
|
+
return { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" }[e];
|
|
250
|
+
});
|
|
251
|
+
}
|
|
252
|
+
function unquote(etag) {
|
|
253
|
+
return etag.replace(/^W\//, '').replace(/^"|"$/g, '');
|
|
254
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import type { Body, PutOptions, SignUploadOptions, Upload, UploadOptions } from './types.js';
|
|
2
|
+
export declare const MAX_KEY_BYTES = 1024;
|
|
3
|
+
export declare const MAX_EXPIRES_IN = 604800;
|
|
4
|
+
/** Default lifetimes of a signed upload and a signed download, in seconds. */
|
|
5
|
+
export declare const UPLOAD_EXPIRES_IN = 600;
|
|
6
|
+
export declare const DOWNLOAD_EXPIRES_IN = 300;
|
|
7
|
+
/**
|
|
8
|
+
* A key is 1–1024 UTF-8 bytes with no leading `/`, no `.` or `..` segment and
|
|
9
|
+
* no control characters. The segment rule is not S3's: a URL path normalizes
|
|
10
|
+
* `a/../b` to `b` before it is sent, so such a key would silently name a
|
|
11
|
+
* different object.
|
|
12
|
+
*/
|
|
13
|
+
export declare function checkKey(key: string): void;
|
|
14
|
+
export declare function checkPrefix(prefix: string | undefined): void;
|
|
15
|
+
export declare function checkMetadata(metadata: Record<string, string> | undefined): Record<string, string>;
|
|
16
|
+
export declare function checkExpiresIn(seconds: number | undefined, fallback: number): number;
|
|
17
|
+
export declare function checkSignUpload(o: SignUploadOptions): void;
|
|
18
|
+
export declare const DEFAULT_CONTENT_TYPE = "application/octet-stream";
|
|
19
|
+
/** Bytes and a content type, from anything `put` accepts. */
|
|
20
|
+
export declare function toBytes(body: Body, contentType: string | undefined): Promise<{
|
|
21
|
+
bytes: Uint8Array;
|
|
22
|
+
contentType: string;
|
|
23
|
+
}>;
|
|
24
|
+
/** `upload`'s arguments as a `put`: a generated key under the prefix, the file's extension kept. */
|
|
25
|
+
export declare function uploadAsPut(file: Blob | Upload, options: UploadOptions | undefined, defaults: {
|
|
26
|
+
cacheControl?: string;
|
|
27
|
+
}): {
|
|
28
|
+
key: string;
|
|
29
|
+
body: Body;
|
|
30
|
+
options: PutOptions;
|
|
31
|
+
};
|
|
32
|
+
/** `.jpg` from `Cover Art.JPG`; nothing when there is no sensible extension. */
|
|
33
|
+
export declare function extensionOf(name: string): string;
|
|
34
|
+
/** How the public store caches: generated keys never change, chosen keys might. */
|
|
35
|
+
export declare const IMMUTABLE = "public, max-age=31536000, immutable";
|
|
36
|
+
export declare const SHORT_PUBLIC = "public, max-age=300";
|
|
37
|
+
export declare function rendersInline(contentType: string): boolean;
|
|
38
|
+
/**
|
|
39
|
+
* The disposition a public object is stored with. A caller's own `attachment;
|
|
40
|
+
* filename=…` is kept; asking for `inline` on a type outside the list is not
|
|
41
|
+
* honoured.
|
|
42
|
+
*/
|
|
43
|
+
export declare function publicDisposition(contentType: string, requested: string | undefined): string | undefined;
|
|
44
|
+
/** An attachment disposition, safe for any file name (RFC 6266 + RFC 5987). */
|
|
45
|
+
export declare function attachment(filename: string): string;
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
import { StorageError } from './errors.js';
|
|
2
|
+
/**
|
|
3
|
+
* Rules every implementation applies before it talks to a backend. They live
|
|
4
|
+
* here, once, so "what is invalid" cannot differ between R2, RustFS and the
|
|
5
|
+
* in-memory store — that difference would be exactly the staging/production
|
|
6
|
+
* surprise the conformance suite exists to prevent.
|
|
7
|
+
*/
|
|
8
|
+
const utf8 = new TextEncoder();
|
|
9
|
+
export const MAX_KEY_BYTES = 1024;
|
|
10
|
+
export const MAX_EXPIRES_IN = 604_800;
|
|
11
|
+
/** Default lifetimes of a signed upload and a signed download, in seconds. */
|
|
12
|
+
export const UPLOAD_EXPIRES_IN = 600;
|
|
13
|
+
export const DOWNLOAD_EXPIRES_IN = 300;
|
|
14
|
+
const METADATA_NAME = /^[a-z0-9][a-z0-9-]{0,63}$/;
|
|
15
|
+
const MAX_METADATA_BYTES = 2048;
|
|
16
|
+
/**
|
|
17
|
+
* A key is 1–1024 UTF-8 bytes with no leading `/`, no `.` or `..` segment and
|
|
18
|
+
* no control characters. The segment rule is not S3's: a URL path normalizes
|
|
19
|
+
* `a/../b` to `b` before it is sent, so such a key would silently name a
|
|
20
|
+
* different object.
|
|
21
|
+
*/
|
|
22
|
+
export function checkKey(key) {
|
|
23
|
+
if (typeof key !== 'string' || key.length === 0)
|
|
24
|
+
throw invalid('a key must be a non-empty string');
|
|
25
|
+
if (utf8.encode(key).length > MAX_KEY_BYTES)
|
|
26
|
+
throw invalid(`a key is at most ${MAX_KEY_BYTES} bytes`);
|
|
27
|
+
if (key.startsWith('/'))
|
|
28
|
+
throw invalid(`"${key}": a key must not start with "/"`);
|
|
29
|
+
// oxlint-disable-next-line no-control-regex
|
|
30
|
+
if (/[\u0000-\u001f\u007f]/.test(key))
|
|
31
|
+
throw invalid(`"${key}": a key must not contain control characters`);
|
|
32
|
+
for (const segment of key.split('/')) {
|
|
33
|
+
if (segment === '.' || segment === '..') {
|
|
34
|
+
throw invalid(`"${key}": a key must not contain a "." or ".." segment`);
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
export function checkPrefix(prefix) {
|
|
39
|
+
if (prefix === undefined || prefix === '')
|
|
40
|
+
return;
|
|
41
|
+
checkKey(prefix);
|
|
42
|
+
}
|
|
43
|
+
export function checkMetadata(metadata) {
|
|
44
|
+
const out = {};
|
|
45
|
+
let bytes = 0;
|
|
46
|
+
for (const [name, value] of Object.entries(metadata ?? {})) {
|
|
47
|
+
if (!METADATA_NAME.test(name)) {
|
|
48
|
+
throw invalid(`metadata name "${name}": lowercase letters, digits and "-", at most 64`);
|
|
49
|
+
}
|
|
50
|
+
if (typeof value !== 'string')
|
|
51
|
+
throw invalid(`metadata "${name}" must be a string`);
|
|
52
|
+
// Header values: printable ASCII only, which is what every backend round-trips intact.
|
|
53
|
+
if (!/^[\u0020-\u007e]*$/.test(value)) {
|
|
54
|
+
throw invalid(`metadata "${name}": printable ASCII only — encode anything else`);
|
|
55
|
+
}
|
|
56
|
+
bytes += name.length + value.length;
|
|
57
|
+
out[name] = value;
|
|
58
|
+
}
|
|
59
|
+
if (bytes > MAX_METADATA_BYTES)
|
|
60
|
+
throw invalid(`metadata is at most ${MAX_METADATA_BYTES} bytes`);
|
|
61
|
+
return out;
|
|
62
|
+
}
|
|
63
|
+
export function checkExpiresIn(seconds, fallback) {
|
|
64
|
+
const s = seconds ?? fallback;
|
|
65
|
+
if (!Number.isInteger(s) || s < 1 || s > MAX_EXPIRES_IN) {
|
|
66
|
+
throw invalid(`expiresIn is whole seconds, 1–${MAX_EXPIRES_IN}`);
|
|
67
|
+
}
|
|
68
|
+
return s;
|
|
69
|
+
}
|
|
70
|
+
export function checkSignUpload(o) {
|
|
71
|
+
if (!Number.isSafeInteger(o.size) || o.size < 0)
|
|
72
|
+
throw invalid('size is the exact byte length');
|
|
73
|
+
if (typeof o.contentType !== 'string' || o.contentType.length === 0) {
|
|
74
|
+
throw invalid('contentType is required: it is signed into the URL');
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
export const DEFAULT_CONTENT_TYPE = 'application/octet-stream';
|
|
78
|
+
/** Bytes and a content type, from anything `put` accepts. */
|
|
79
|
+
export async function toBytes(body, contentType) {
|
|
80
|
+
if (typeof body === 'string') {
|
|
81
|
+
return { bytes: utf8.encode(body), contentType: contentType ?? 'text/plain; charset=utf-8' };
|
|
82
|
+
}
|
|
83
|
+
if (body instanceof Blob) {
|
|
84
|
+
return {
|
|
85
|
+
bytes: new Uint8Array(await body.arrayBuffer()),
|
|
86
|
+
contentType: contentType ?? (body.type || DEFAULT_CONTENT_TYPE),
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
if (body instanceof ArrayBuffer) {
|
|
90
|
+
return {
|
|
91
|
+
bytes: new Uint8Array(body.slice(0)),
|
|
92
|
+
contentType: contentType ?? DEFAULT_CONTENT_TYPE,
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
if (ArrayBuffer.isView(body)) {
|
|
96
|
+
const copy = new Uint8Array(body.byteLength);
|
|
97
|
+
copy.set(new Uint8Array(body.buffer, body.byteOffset, body.byteLength));
|
|
98
|
+
return { bytes: copy, contentType: contentType ?? DEFAULT_CONTENT_TYPE };
|
|
99
|
+
}
|
|
100
|
+
if (body instanceof ReadableStream) {
|
|
101
|
+
const bytes = new Uint8Array(await new Response(body).arrayBuffer());
|
|
102
|
+
return { bytes, contentType: contentType ?? DEFAULT_CONTENT_TYPE };
|
|
103
|
+
}
|
|
104
|
+
throw invalid('a body is a string, ArrayBuffer, typed array, Blob or ReadableStream');
|
|
105
|
+
}
|
|
106
|
+
/** `upload`'s arguments as a `put`: a generated key under the prefix, the file's extension kept. */
|
|
107
|
+
export function uploadAsPut(file, options, defaults) {
|
|
108
|
+
checkPrefix(options?.prefix);
|
|
109
|
+
const isBlob = file instanceof Blob;
|
|
110
|
+
const name = isBlob ? file.name : file.name;
|
|
111
|
+
const ext = typeof name === 'string' ? extensionOf(name) : '';
|
|
112
|
+
const prefix = options?.prefix ? `${options.prefix.replace(/\/+$/, '')}/` : '';
|
|
113
|
+
const key = `${prefix}${crypto.randomUUID()}${ext}`;
|
|
114
|
+
const contentType = isBlob ? file.type || undefined : file.contentType;
|
|
115
|
+
const cacheControl = options?.cacheControl ?? defaults.cacheControl;
|
|
116
|
+
return {
|
|
117
|
+
key,
|
|
118
|
+
body: isBlob ? file : file.body,
|
|
119
|
+
options: {
|
|
120
|
+
...(contentType ? { contentType } : {}),
|
|
121
|
+
...(cacheControl ? { cacheControl } : {}),
|
|
122
|
+
...(options?.metadata ? { metadata: options.metadata } : {}),
|
|
123
|
+
},
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
/** `.jpg` from `Cover Art.JPG`; nothing when there is no sensible extension. */
|
|
127
|
+
export function extensionOf(name) {
|
|
128
|
+
const m = /\.([A-Za-z0-9]{1,10})$/.exec(name);
|
|
129
|
+
return m ? `.${m[1].toLowerCase()}` : '';
|
|
130
|
+
}
|
|
131
|
+
/** How the public store caches: generated keys never change, chosen keys might. */
|
|
132
|
+
export const IMMUTABLE = 'public, max-age=31536000, immutable';
|
|
133
|
+
export const SHORT_PUBLIC = 'public, max-age=300';
|
|
134
|
+
/**
|
|
135
|
+
* Public objects render in a browser tab only if they are media: raster
|
|
136
|
+
* images, audio, video. Everything else — HTML, SVG, PDF, text — is served as
|
|
137
|
+
* an attachment, because a static scam page on a KF-held domain needs no
|
|
138
|
+
* script to do harm (earlier KF projects drew takedown notices for exactly
|
|
139
|
+
* that). Only direct navigation is affected: <img>, <video>, fonts, CSS and
|
|
140
|
+
* fetch() ignore the header, so embeds keep working.
|
|
141
|
+
*
|
|
142
|
+
* The console's assets zone enforces the same rule on responses, for anyone
|
|
143
|
+
* writing around the SDK; this makes every backend store it, so local and
|
|
144
|
+
* staging behave as production does.
|
|
145
|
+
*/
|
|
146
|
+
const INLINE_PUBLIC = /^(image\/(png|jpeg|gif|webp|avif)|audio\/[a-z0-9.+-]+|video\/[a-z0-9.+-]+)$/;
|
|
147
|
+
export function rendersInline(contentType) {
|
|
148
|
+
return INLINE_PUBLIC.test(contentType.split(';')[0].trim().toLowerCase());
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* The disposition a public object is stored with. A caller's own `attachment;
|
|
152
|
+
* filename=…` is kept; asking for `inline` on a type outside the list is not
|
|
153
|
+
* honoured.
|
|
154
|
+
*/
|
|
155
|
+
export function publicDisposition(contentType, requested) {
|
|
156
|
+
if (rendersInline(contentType))
|
|
157
|
+
return requested;
|
|
158
|
+
return requested?.trim().toLowerCase().startsWith('attachment') ? requested : 'attachment';
|
|
159
|
+
}
|
|
160
|
+
/** An attachment disposition, safe for any file name (RFC 6266 + RFC 5987). */
|
|
161
|
+
export function attachment(filename) {
|
|
162
|
+
const ascii = filename.replace(/[^\u0020-\u007e]/g, '_').replace(/["\\]/g, '_');
|
|
163
|
+
return `attachment; filename="${ascii}"; filename*=UTF-8''${encodeURIComponent(filename)}`;
|
|
164
|
+
}
|
|
165
|
+
function invalid(message) {
|
|
166
|
+
return new StorageError('invalid', message);
|
|
167
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AWS Signature Version 4 on WebCrypto — the dialect R2 and RustFS speak.
|
|
3
|
+
*
|
|
4
|
+
* Hand-written rather than aws4fetch because a presigned upload must sign
|
|
5
|
+
* `content-type` and `content-length`: that is what makes `signUpload` exact
|
|
6
|
+
* on every backend (R2 has no POST policies, so no `content-length-range`),
|
|
7
|
+
* and aws4fetch refuses to sign either header. Pure: credentials, time and
|
|
8
|
+
* request are arguments, so the tests check it against AWS's published
|
|
9
|
+
* examples byte for byte.
|
|
10
|
+
*/
|
|
11
|
+
export interface Credentials {
|
|
12
|
+
accessKeyId: string;
|
|
13
|
+
secretAccessKey: string;
|
|
14
|
+
region: string;
|
|
15
|
+
/**
|
|
16
|
+
* Temporary credentials' third part (R2 temporary credentials, STS). Signed
|
|
17
|
+
* as `x-amz-security-token` on a request and carried as
|
|
18
|
+
* `X-Amz-Security-Token` in a presigned URL.
|
|
19
|
+
*/
|
|
20
|
+
sessionToken?: string;
|
|
21
|
+
}
|
|
22
|
+
export interface SignInput {
|
|
23
|
+
method: string;
|
|
24
|
+
/** Its path must already be encoded exactly as it will be sent (`encodeKey`). */
|
|
25
|
+
url: URL;
|
|
26
|
+
/** Headers to sign and send. `host` is signed from the URL and never sent (fetch sets it). */
|
|
27
|
+
headers?: Record<string, string>;
|
|
28
|
+
credentials: Credentials;
|
|
29
|
+
/** Defaults to now. */
|
|
30
|
+
date?: Date;
|
|
31
|
+
}
|
|
32
|
+
export declare const EMPTY_SHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
|
|
33
|
+
export declare const UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD";
|
|
34
|
+
/** RFC 3986 unreserved characters pass; everything else is %XX of its UTF-8 bytes. */
|
|
35
|
+
export declare function encodeRfc3986(s: string): string;
|
|
36
|
+
/** An object key as a URL path: each segment encoded, the slashes kept. */
|
|
37
|
+
export declare function encodeKey(key: string): string;
|
|
38
|
+
/** The headers to send, `authorization` included, for a request signed in the Authorization header. */
|
|
39
|
+
export declare function signRequest(o: SignInput & {
|
|
40
|
+
payloadHash: string;
|
|
41
|
+
}): Promise<Record<string, string>>;
|
|
42
|
+
/**
|
|
43
|
+
* A presigned URL: the signature travels in the query string, and whoever
|
|
44
|
+
* holds the URL must send exactly the signed headers with exactly these
|
|
45
|
+
* values, before `expiresIn` runs out.
|
|
46
|
+
*/
|
|
47
|
+
export declare function presign(o: SignInput & {
|
|
48
|
+
expiresIn: number;
|
|
49
|
+
}): Promise<string>;
|
|
50
|
+
export declare function sha256Hex(data: Uint8Array | string): Promise<string>;
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AWS Signature Version 4 on WebCrypto — the dialect R2 and RustFS speak.
|
|
3
|
+
*
|
|
4
|
+
* Hand-written rather than aws4fetch because a presigned upload must sign
|
|
5
|
+
* `content-type` and `content-length`: that is what makes `signUpload` exact
|
|
6
|
+
* on every backend (R2 has no POST policies, so no `content-length-range`),
|
|
7
|
+
* and aws4fetch refuses to sign either header. Pure: credentials, time and
|
|
8
|
+
* request are arguments, so the tests check it against AWS's published
|
|
9
|
+
* examples byte for byte.
|
|
10
|
+
*/
|
|
11
|
+
const utf8 = new TextEncoder();
|
|
12
|
+
const SERVICE = 's3';
|
|
13
|
+
export const EMPTY_SHA256 = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855';
|
|
14
|
+
export const UNSIGNED_PAYLOAD = 'UNSIGNED-PAYLOAD';
|
|
15
|
+
/** RFC 3986 unreserved characters pass; everything else is %XX of its UTF-8 bytes. */
|
|
16
|
+
export function encodeRfc3986(s) {
|
|
17
|
+
return encodeURIComponent(s).replace(/[!'()*]/g, (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`);
|
|
18
|
+
}
|
|
19
|
+
/** An object key as a URL path: each segment encoded, the slashes kept. */
|
|
20
|
+
export function encodeKey(key) {
|
|
21
|
+
return key.split('/').map(encodeRfc3986).join('/');
|
|
22
|
+
}
|
|
23
|
+
/** The headers to send, `authorization` included, for a request signed in the Authorization header. */
|
|
24
|
+
export async function signRequest(o) {
|
|
25
|
+
const { amzDate, day } = stamps(o.date ?? new Date());
|
|
26
|
+
const headers = lowercase({
|
|
27
|
+
...o.headers,
|
|
28
|
+
'x-amz-date': amzDate,
|
|
29
|
+
'x-amz-content-sha256': o.payloadHash,
|
|
30
|
+
...(o.credentials.sessionToken ? { 'x-amz-security-token': o.credentials.sessionToken } : {}),
|
|
31
|
+
});
|
|
32
|
+
const { canonical, signed } = canonicalHeaders(o.url, headers);
|
|
33
|
+
const request = [
|
|
34
|
+
o.method,
|
|
35
|
+
o.url.pathname,
|
|
36
|
+
canonicalQuery(o.url),
|
|
37
|
+
canonical,
|
|
38
|
+
signed,
|
|
39
|
+
o.payloadHash,
|
|
40
|
+
].join('\n');
|
|
41
|
+
const signature = await sign(o.credentials, day, amzDate, request);
|
|
42
|
+
return {
|
|
43
|
+
...headers,
|
|
44
|
+
authorization: `AWS4-HMAC-SHA256 Credential=${o.credentials.accessKeyId}/${scope(o.credentials, day)}, SignedHeaders=${signed}, Signature=${signature}`,
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* A presigned URL: the signature travels in the query string, and whoever
|
|
49
|
+
* holds the URL must send exactly the signed headers with exactly these
|
|
50
|
+
* values, before `expiresIn` runs out.
|
|
51
|
+
*/
|
|
52
|
+
export async function presign(o) {
|
|
53
|
+
const { amzDate, day } = stamps(o.date ?? new Date());
|
|
54
|
+
const headers = lowercase(o.headers ?? {});
|
|
55
|
+
const { canonical, signed } = canonicalHeaders(o.url, headers);
|
|
56
|
+
const url = new URL(o.url);
|
|
57
|
+
url.searchParams.set('X-Amz-Algorithm', 'AWS4-HMAC-SHA256');
|
|
58
|
+
if (o.credentials.sessionToken) {
|
|
59
|
+
url.searchParams.set('X-Amz-Security-Token', o.credentials.sessionToken);
|
|
60
|
+
}
|
|
61
|
+
url.searchParams.set('X-Amz-Credential', `${o.credentials.accessKeyId}/${scope(o.credentials, day)}`);
|
|
62
|
+
url.searchParams.set('X-Amz-Date', amzDate);
|
|
63
|
+
url.searchParams.set('X-Amz-Expires', String(o.expiresIn));
|
|
64
|
+
url.searchParams.set('X-Amz-SignedHeaders', signed);
|
|
65
|
+
const request = [
|
|
66
|
+
o.method,
|
|
67
|
+
url.pathname,
|
|
68
|
+
canonicalQuery(url),
|
|
69
|
+
canonical,
|
|
70
|
+
signed,
|
|
71
|
+
UNSIGNED_PAYLOAD,
|
|
72
|
+
].join('\n');
|
|
73
|
+
const signature = await sign(o.credentials, day, amzDate, request);
|
|
74
|
+
// Appended by hand: URLSearchParams would re-encode the query we just signed.
|
|
75
|
+
return `${url.origin}${url.pathname}?${canonicalQuery(url)}&X-Amz-Signature=${signature}`;
|
|
76
|
+
}
|
|
77
|
+
export async function sha256Hex(data) {
|
|
78
|
+
const bytes = typeof data === 'string' ? utf8.encode(data) : data;
|
|
79
|
+
return hex(new Uint8Array(await crypto.subtle.digest('SHA-256', bytes)));
|
|
80
|
+
}
|
|
81
|
+
/* ------------------------------ internals ------------------------------ */
|
|
82
|
+
function stamps(date) {
|
|
83
|
+
const amzDate = date
|
|
84
|
+
.toISOString()
|
|
85
|
+
.replace(/[-:]/g, '')
|
|
86
|
+
.replace(/\.\d{3}/, '');
|
|
87
|
+
return { amzDate, day: amzDate.slice(0, 8) };
|
|
88
|
+
}
|
|
89
|
+
function scope(c, day) {
|
|
90
|
+
return `${day}/${c.region}/${SERVICE}/aws4_request`;
|
|
91
|
+
}
|
|
92
|
+
function lowercase(h) {
|
|
93
|
+
const out = {};
|
|
94
|
+
for (const [k, v] of Object.entries(h))
|
|
95
|
+
out[k.toLowerCase()] = v;
|
|
96
|
+
return out;
|
|
97
|
+
}
|
|
98
|
+
function canonicalHeaders(url, headers) {
|
|
99
|
+
const all = { ...headers, host: url.host };
|
|
100
|
+
const names = Object.keys(all).sort();
|
|
101
|
+
const canonical = names.map((n) => `${n}:${all[n].trim().replace(/\s+/g, ' ')}\n`).join('');
|
|
102
|
+
return { canonical, signed: names.join(';') };
|
|
103
|
+
}
|
|
104
|
+
/** Every parameter, RFC 3986-encoded, sorted by encoded name then value. */
|
|
105
|
+
function canonicalQuery(url) {
|
|
106
|
+
return [...url.searchParams]
|
|
107
|
+
.map(([k, v]) => [encodeRfc3986(k), encodeRfc3986(v)])
|
|
108
|
+
.sort(([ak, av], [bk, bv]) => (ak < bk ? -1 : ak > bk ? 1 : av < bv ? -1 : av > bv ? 1 : 0))
|
|
109
|
+
.map(([k, v]) => `${k}=${v}`)
|
|
110
|
+
.join('&');
|
|
111
|
+
}
|
|
112
|
+
async function sign(c, day, amzDate, request) {
|
|
113
|
+
const toSign = ['AWS4-HMAC-SHA256', amzDate, scope(c, day), await sha256Hex(request)].join('\n');
|
|
114
|
+
let key = utf8.encode(`AWS4${c.secretAccessKey}`);
|
|
115
|
+
for (const part of [day, c.region, SERVICE, 'aws4_request'])
|
|
116
|
+
key = await hmac(key, part);
|
|
117
|
+
return hex(await hmac(key, toSign));
|
|
118
|
+
}
|
|
119
|
+
async function hmac(key, data) {
|
|
120
|
+
const k = await crypto.subtle.importKey('raw', key, { name: 'HMAC', hash: 'SHA-256' }, false, ['sign']);
|
|
121
|
+
return new Uint8Array(await crypto.subtle.sign('HMAC', k, utf8.encode(data)));
|
|
122
|
+
}
|
|
123
|
+
function hex(bytes) {
|
|
124
|
+
let s = '';
|
|
125
|
+
for (const b of bytes)
|
|
126
|
+
s += b.toString(16).padStart(2, '0');
|
|
127
|
+
return s;
|
|
128
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { Storage } from './types.js';
|
|
2
|
+
/**
|
|
3
|
+
* The storage interface with no network: for a project's unit tests, and
|
|
4
|
+
* the reference the conformance suite is written against first. It enforces
|
|
5
|
+
* everything the real backends do — the same key rules (shared.ts), signed
|
|
6
|
+
* URLs that expire and check their signed headers, a public store that
|
|
7
|
+
* serves by URL — so a test that passes here is not passing by accident.
|
|
8
|
+
*
|
|
9
|
+
* Signed and public URLs are served by `fetch` on the returned object:
|
|
10
|
+
*
|
|
11
|
+
* const storage = memoryStorage()
|
|
12
|
+
* const up = await storage.signUpload('a.txt', { contentType: 'text/plain', size: 2 })
|
|
13
|
+
* await storage.fetch(up.url, { method: 'PUT', headers: up.headers, body: 'hi' })
|
|
14
|
+
*/
|
|
15
|
+
export interface MemoryStorageOptions {
|
|
16
|
+
/** The public store: its base URL, or false for an environment that doesn't declare `public`. Default on. */
|
|
17
|
+
public?: false | {
|
|
18
|
+
base?: string;
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
export interface MemoryStorage extends Storage {
|
|
22
|
+
/** Serves the URLs this store hands out: signed uploads, signed downloads, public objects. */
|
|
23
|
+
fetch(input: string | URL | Request, init?: RequestInit): Promise<Response>;
|
|
24
|
+
}
|
|
25
|
+
export declare function memoryStorage(options?: MemoryStorageOptions): MemoryStorage;
|