@knowledgefutures/sdk 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +24 -2
- package/dist/analytics/beacon.d.ts +25 -0
- package/dist/analytics/beacon.js +86 -0
- package/dist/analytics/binding.d.ts +24 -0
- package/dist/analytics/binding.js +46 -0
- package/dist/analytics/collector.d.ts +10 -0
- package/dist/analytics/collector.js +10 -0
- package/dist/analytics/cubes.d.ts +96 -0
- package/dist/analytics/cubes.js +213 -0
- package/dist/analytics/envelope.d.ts +135 -0
- package/dist/analytics/envelope.js +231 -0
- package/dist/analytics/errors.d.ts +24 -0
- package/dist/analytics/errors.js +20 -0
- package/dist/analytics/index.d.ts +38 -0
- package/dist/analytics/index.js +98 -0
- package/dist/analytics/memory.d.ts +35 -0
- package/dist/analytics/memory.js +66 -0
- package/dist/analytics/testing.d.ts +6 -0
- package/dist/analytics/testing.js +6 -0
- package/dist/analytics/types.d.ts +22 -0
- package/dist/analytics/types.js +1 -0
- package/dist/storage/binding.d.ts +76 -0
- package/dist/storage/binding.js +97 -0
- package/dist/storage/brokered.d.ts +21 -0
- package/dist/storage/brokered.js +94 -0
- package/dist/storage/conformance.d.ts +24 -0
- package/dist/storage/conformance.js +311 -0
- package/dist/storage/errors.d.ts +27 -0
- package/dist/storage/errors.js +21 -0
- package/dist/storage/index.d.ts +34 -0
- package/dist/storage/index.js +77 -0
- package/dist/storage/local.d.ts +26 -0
- package/dist/storage/local.js +57 -0
- package/dist/storage/s3.d.ts +22 -0
- package/dist/storage/s3.js +254 -0
- package/dist/storage/shared.d.ts +45 -0
- package/dist/storage/shared.js +167 -0
- package/dist/storage/sigv4.d.ts +50 -0
- package/dist/storage/sigv4.js +128 -0
- package/dist/storage/testing.d.ts +25 -0
- package/dist/storage/testing.js +242 -0
- package/dist/storage/types.d.ts +122 -0
- package/dist/storage/types.js +13 -0
- package/package.json +30 -3
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { Cube, Query, QueryResult } from './cubes.js';
|
|
2
|
+
import type { ServerHint } from './envelope.js';
|
|
3
|
+
/** What project code holds. Every implementation — the collector, the memory double, `kf dev`'s log — is this. */
|
|
4
|
+
export interface Analytics {
|
|
5
|
+
/**
|
|
6
|
+
* A server event, signed with the project's secret, so it counts as
|
|
7
|
+
* `source = 'server'`. For what matters: a download the project's Worker
|
|
8
|
+
* served, a sign-up. Pass the reader's `country` and `userAgent` when the
|
|
9
|
+
* server knows them.
|
|
10
|
+
*/
|
|
11
|
+
track(event: ServerHint): Promise<void>;
|
|
12
|
+
/** The project's own numbers. Server-side only: the secret signs it. */
|
|
13
|
+
query<C extends Cube>(q: Query<C>): Promise<QueryResult>;
|
|
14
|
+
/**
|
|
15
|
+
* The `<script>` tag for the project's pages. `keyed` adds the public site
|
|
16
|
+
* id, for a project whose domains the console can't list. Empty under
|
|
17
|
+
* `kf dev` and in the memory double: nothing collects there.
|
|
18
|
+
*/
|
|
19
|
+
beaconTag(options?: {
|
|
20
|
+
keyed?: boolean;
|
|
21
|
+
}): string;
|
|
22
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `KF_STORAGE_URL` and `KF_STORAGE_PUBLIC_URL` — the whole contract between
|
|
3
|
+
* the console, which provisions a bucket and writes the binding into a
|
|
4
|
+
* project's environment, and the SDK, which reads it. Both import this file,
|
|
5
|
+
* so the format has exactly one definition.
|
|
6
|
+
*
|
|
7
|
+
* s3+https://<accessKeyId>:<secretAccessKey>@<host>[:port]/<bucket>?region=<r>[&base=<public origin>]
|
|
8
|
+
*
|
|
9
|
+
* `s3+http` only for a local server. Credentials are percent-encoded (secrets
|
|
10
|
+
* contain `/` and `+`). `base` is present on the public store only: where
|
|
11
|
+
* `url(key)` points. One variable per store, so a store can never be half
|
|
12
|
+
* configured.
|
|
13
|
+
*
|
|
14
|
+
* Not for project code, and not published: projects call `createStorage` /
|
|
15
|
+
* `storage`, which read these variables. Only what WRITES them imports this
|
|
16
|
+
* entry point — the console's adapter and `kf dev`, both from the workspace.
|
|
17
|
+
*/
|
|
18
|
+
export interface StorageBinding {
|
|
19
|
+
/** `https://<account>.r2.cloudflarestorage.com`, or `http://localhost:9000` for a local server. */
|
|
20
|
+
endpoint: string;
|
|
21
|
+
bucket: string;
|
|
22
|
+
accessKeyId: string;
|
|
23
|
+
secretAccessKey: string;
|
|
24
|
+
/** `auto` for R2; a local server accepts any, conventionally `us-east-1`. */
|
|
25
|
+
region: string;
|
|
26
|
+
/** The public store's origin, e.g. `https://my-project.kfassets.example`. */
|
|
27
|
+
publicBase?: string;
|
|
28
|
+
}
|
|
29
|
+
export declare const STORAGE_ENV = "KF_STORAGE_URL";
|
|
30
|
+
export declare const PUBLIC_STORAGE_ENV = "KF_STORAGE_PUBLIC_URL";
|
|
31
|
+
export declare function encodeStorageBinding(b: StorageBinding): string;
|
|
32
|
+
export declare function decodeStorageBinding(value: string, name?: string): StorageBinding;
|
|
33
|
+
/**
|
|
34
|
+
* The deployed form (storage plan §8): no vendor credential at all, only the
|
|
35
|
+
* instance's id and secret at the storage service, which signs short-lived
|
|
36
|
+
* R2 credentials for its bucket on request.
|
|
37
|
+
*
|
|
38
|
+
* https://<instance>:<secret>@<storage service host>/[?store=public&base=<public origin>]
|
|
39
|
+
*
|
|
40
|
+
* The static `s3+…` form above stays for `kf dev` (RustFS) and tests.
|
|
41
|
+
*/
|
|
42
|
+
export interface BrokeredBinding {
|
|
43
|
+
/** The storage service's origin, e.g. `https://console-storage-service.kfutures.org`. */
|
|
44
|
+
service: string;
|
|
45
|
+
instance: string;
|
|
46
|
+
secret: string;
|
|
47
|
+
store: 'private' | 'public';
|
|
48
|
+
/** The public store's origin, where `url(key)` points. */
|
|
49
|
+
publicBase?: string;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* What the storage service's `POST /v1/credentials` answers: R2 temporary
|
|
53
|
+
* credentials for one bucket. The service (its own repo) defines the response
|
|
54
|
+
* and this is what the SDK reads; `brokered.test.ts` and the service's tests
|
|
55
|
+
* hold the same example response, so a change to either side fails a test.
|
|
56
|
+
*/
|
|
57
|
+
export interface IssuedCredentials {
|
|
58
|
+
endpoint: string;
|
|
59
|
+
bucket: string;
|
|
60
|
+
region: string;
|
|
61
|
+
accessKeyId: string;
|
|
62
|
+
secretAccessKey: string;
|
|
63
|
+
sessionToken: string;
|
|
64
|
+
/** ISO time. */
|
|
65
|
+
expiresAt: string;
|
|
66
|
+
}
|
|
67
|
+
export type ParsedBinding = {
|
|
68
|
+
kind: 'static';
|
|
69
|
+
binding: StorageBinding;
|
|
70
|
+
} | {
|
|
71
|
+
kind: 'brokered';
|
|
72
|
+
binding: BrokeredBinding;
|
|
73
|
+
};
|
|
74
|
+
export declare function encodeBrokeredBinding(b: BrokeredBinding): string;
|
|
75
|
+
/** Either form; the scheme says which. */
|
|
76
|
+
export declare function parseBinding(value: string, name?: string): ParsedBinding;
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
import { StorageError } from './errors.js';
|
|
2
|
+
export const STORAGE_ENV = 'KF_STORAGE_URL';
|
|
3
|
+
export const PUBLIC_STORAGE_ENV = 'KF_STORAGE_PUBLIC_URL';
|
|
4
|
+
const BUCKET = /^[a-z0-9][a-z0-9-]{1,61}[a-z0-9]$/;
|
|
5
|
+
export function encodeStorageBinding(b) {
|
|
6
|
+
const endpoint = new URL(b.endpoint);
|
|
7
|
+
if (endpoint.protocol !== 'https:' && endpoint.protocol !== 'http:') {
|
|
8
|
+
throw new StorageError('invalid', `endpoint must be http(s): ${b.endpoint}`);
|
|
9
|
+
}
|
|
10
|
+
if (!BUCKET.test(b.bucket))
|
|
11
|
+
throw new StorageError('invalid', `not a bucket name: ${b.bucket}`);
|
|
12
|
+
const query = new URLSearchParams({ region: b.region });
|
|
13
|
+
if (b.publicBase)
|
|
14
|
+
query.set('base', trimSlash(b.publicBase));
|
|
15
|
+
return (`s3+${endpoint.protocol}//` +
|
|
16
|
+
`${encodeURIComponent(b.accessKeyId)}:${encodeURIComponent(b.secretAccessKey)}@` +
|
|
17
|
+
`${endpoint.host}/${b.bucket}?${query}`);
|
|
18
|
+
}
|
|
19
|
+
export function decodeStorageBinding(value, name = STORAGE_ENV) {
|
|
20
|
+
let url;
|
|
21
|
+
try {
|
|
22
|
+
url = new URL(value.trim());
|
|
23
|
+
}
|
|
24
|
+
catch {
|
|
25
|
+
throw bad(name, 'not a URL');
|
|
26
|
+
}
|
|
27
|
+
const scheme = /^s3\+(https?):$/.exec(url.protocol)?.[1];
|
|
28
|
+
if (!scheme)
|
|
29
|
+
throw bad(name, 'the scheme is s3+https (or s3+http for a local server)');
|
|
30
|
+
const bucket = url.pathname.replace(/^\/+|\/+$/g, '');
|
|
31
|
+
if (!BUCKET.test(bucket))
|
|
32
|
+
throw bad(name, 'the path is the bucket name');
|
|
33
|
+
if (!url.username || !url.password)
|
|
34
|
+
throw bad(name, 'credentials are missing');
|
|
35
|
+
const region = url.searchParams.get('region');
|
|
36
|
+
if (!region)
|
|
37
|
+
throw bad(name, 'region is missing');
|
|
38
|
+
const base = url.searchParams.get('base');
|
|
39
|
+
return {
|
|
40
|
+
endpoint: `${scheme}://${url.host}`,
|
|
41
|
+
bucket,
|
|
42
|
+
accessKeyId: decodeURIComponent(url.username),
|
|
43
|
+
secretAccessKey: decodeURIComponent(url.password),
|
|
44
|
+
region,
|
|
45
|
+
...(base ? { publicBase: trimSlash(base) } : {}),
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
function trimSlash(s) {
|
|
49
|
+
return s.replace(/\/+$/, '');
|
|
50
|
+
}
|
|
51
|
+
function bad(name, why) {
|
|
52
|
+
// Never echo the value: it carries the secret.
|
|
53
|
+
return new StorageError('not_configured', `${name} is malformed: ${why}`);
|
|
54
|
+
}
|
|
55
|
+
export function encodeBrokeredBinding(b) {
|
|
56
|
+
const service = new URL(b.service);
|
|
57
|
+
if (service.protocol !== 'https:' && service.protocol !== 'http:') {
|
|
58
|
+
throw new StorageError('invalid', `service must be http(s): ${b.service}`);
|
|
59
|
+
}
|
|
60
|
+
const query = new URLSearchParams();
|
|
61
|
+
if (b.store === 'public')
|
|
62
|
+
query.set('store', 'public');
|
|
63
|
+
if (b.publicBase)
|
|
64
|
+
query.set('base', trimSlash(b.publicBase));
|
|
65
|
+
const q = query.toString();
|
|
66
|
+
return (`${service.protocol}//${encodeURIComponent(b.instance)}:${encodeURIComponent(b.secret)}@` +
|
|
67
|
+
`${service.host}/${q ? `?${q}` : ''}`);
|
|
68
|
+
}
|
|
69
|
+
/** Either form; the scheme says which. */
|
|
70
|
+
export function parseBinding(value, name = STORAGE_ENV) {
|
|
71
|
+
const v = value.trim();
|
|
72
|
+
if (/^s3\+https?:/.test(v))
|
|
73
|
+
return { kind: 'static', binding: decodeStorageBinding(v, name) };
|
|
74
|
+
let url;
|
|
75
|
+
try {
|
|
76
|
+
url = new URL(v);
|
|
77
|
+
}
|
|
78
|
+
catch {
|
|
79
|
+
throw bad(name, 'not a URL');
|
|
80
|
+
}
|
|
81
|
+
if (url.protocol !== 'https:' && url.protocol !== 'http:') {
|
|
82
|
+
throw bad(name, 'the scheme is s3+https (static) or https (the storage service)');
|
|
83
|
+
}
|
|
84
|
+
if (!url.username || !url.password)
|
|
85
|
+
throw bad(name, 'the instance id and secret are missing');
|
|
86
|
+
const base = url.searchParams.get('base');
|
|
87
|
+
return {
|
|
88
|
+
kind: 'brokered',
|
|
89
|
+
binding: {
|
|
90
|
+
service: `${url.protocol}//${url.host}`,
|
|
91
|
+
instance: decodeURIComponent(url.username),
|
|
92
|
+
secret: decodeURIComponent(url.password),
|
|
93
|
+
store: url.searchParams.get('store') === 'public' ? 'public' : 'private',
|
|
94
|
+
...(base ? { publicBase: trimSlash(base) } : {}),
|
|
95
|
+
},
|
|
96
|
+
};
|
|
97
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { BrokeredBinding, IssuedCredentials } from './binding.js';
|
|
2
|
+
import { type S3StoreOptions } from './s3.js';
|
|
3
|
+
import type { PublicStore, Store } from './types.js';
|
|
4
|
+
/**
|
|
5
|
+
* A store whose credentials come from the storage service (storage plan §8):
|
|
6
|
+
* the binding holds only the instance's id and secret, and the service signs
|
|
7
|
+
* an hour of R2 temporary credentials for this instance's bucket on request.
|
|
8
|
+
* They are cached and refreshed five minutes before they expire, one exchange
|
|
9
|
+
* at a time, so a busy process asks about once an hour and every call after
|
|
10
|
+
* that goes straight to R2.
|
|
11
|
+
*
|
|
12
|
+
* A signed URL carries the session token, so it stops working when the
|
|
13
|
+
* credentials that signed it expire. Signing therefore uses credentials that
|
|
14
|
+
* outlive the URL, exchanging early if the cached ones don't, and refuses a
|
|
15
|
+
* lifetime no credentials from the service can cover.
|
|
16
|
+
*/
|
|
17
|
+
export type { IssuedCredentials };
|
|
18
|
+
export declare function createBrokeredStore(b: BrokeredBinding & {
|
|
19
|
+
store: 'public';
|
|
20
|
+
}, options?: S3StoreOptions): PublicStore;
|
|
21
|
+
export declare function createBrokeredStore(b: BrokeredBinding, options?: S3StoreOptions): Store;
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import { StorageError } from './errors.js';
|
|
2
|
+
import { createS3Store } from './s3.js';
|
|
3
|
+
import { checkExpiresIn, checkKey, DOWNLOAD_EXPIRES_IN, UPLOAD_EXPIRES_IN } from './shared.js';
|
|
4
|
+
import { encodeKey } from './sigv4.js';
|
|
5
|
+
const REFRESH_EARLY_MS = 5 * 60_000;
|
|
6
|
+
export function createBrokeredStore(b, options = {}) {
|
|
7
|
+
const doFetch = options.fetch ?? ((input, init) => fetch(input, init));
|
|
8
|
+
let cached;
|
|
9
|
+
let inflight;
|
|
10
|
+
async function exchange() {
|
|
11
|
+
let res;
|
|
12
|
+
try {
|
|
13
|
+
res = await doFetch(`${b.service}/v1/credentials`, {
|
|
14
|
+
method: 'POST',
|
|
15
|
+
headers: {
|
|
16
|
+
authorization: `Basic ${btoa(`${b.instance}:${b.secret}`)}`,
|
|
17
|
+
'content-type': 'application/json',
|
|
18
|
+
},
|
|
19
|
+
body: JSON.stringify({ store: b.store }),
|
|
20
|
+
});
|
|
21
|
+
}
|
|
22
|
+
catch (e) {
|
|
23
|
+
throw new StorageError('unavailable', `the storage service is unreachable: ${e.message}`);
|
|
24
|
+
}
|
|
25
|
+
if (!res.ok) {
|
|
26
|
+
const why = (await res.text().catch(() => '')).slice(0, 200);
|
|
27
|
+
if (res.status === 401 || res.status === 403) {
|
|
28
|
+
throw new StorageError('forbidden', `the storage service refused this instance: ${why}`, res.status);
|
|
29
|
+
}
|
|
30
|
+
if (res.status === 404 || res.status === 410) {
|
|
31
|
+
throw new StorageError('not_configured', `the storage service has no such store: ${why}`, res.status);
|
|
32
|
+
}
|
|
33
|
+
throw new StorageError('unavailable', `the storage service answered ${res.status}: ${why}`, res.status);
|
|
34
|
+
}
|
|
35
|
+
const c = (await res.json());
|
|
36
|
+
const store = createS3Store({
|
|
37
|
+
endpoint: c.endpoint,
|
|
38
|
+
bucket: c.bucket,
|
|
39
|
+
region: c.region,
|
|
40
|
+
accessKeyId: c.accessKeyId,
|
|
41
|
+
secretAccessKey: c.secretAccessKey,
|
|
42
|
+
...(b.publicBase ? { publicBase: b.publicBase } : {}),
|
|
43
|
+
}, { ...options, sessionToken: c.sessionToken });
|
|
44
|
+
const expiresAt = Date.parse(c.expiresAt);
|
|
45
|
+
cached = { store, refreshAt: expiresAt - REFRESH_EARLY_MS, expiresAt };
|
|
46
|
+
return store;
|
|
47
|
+
}
|
|
48
|
+
async function current() {
|
|
49
|
+
if (cached && Date.now() < cached.refreshAt)
|
|
50
|
+
return cached.store;
|
|
51
|
+
inflight ??= exchange().finally(() => {
|
|
52
|
+
inflight = undefined;
|
|
53
|
+
});
|
|
54
|
+
return inflight;
|
|
55
|
+
}
|
|
56
|
+
/** A store whose credentials last at least `seconds` more. */
|
|
57
|
+
async function lasting(seconds) {
|
|
58
|
+
// A function, not a narrowed read: `cached` changes across the awaits.
|
|
59
|
+
const leftMs = () => (cached ? cached.expiresAt - Date.now() : 0);
|
|
60
|
+
let s = await current();
|
|
61
|
+
if (leftMs() >= seconds * 1000)
|
|
62
|
+
return s;
|
|
63
|
+
cached = undefined;
|
|
64
|
+
s = await current();
|
|
65
|
+
if (leftMs() >= seconds * 1000)
|
|
66
|
+
return s;
|
|
67
|
+
throw new StorageError('invalid', `expiresIn is at most ${Math.max(0, Math.floor(leftMs() / 1000))} seconds here: a ` +
|
|
68
|
+
'signed URL stops working when the storage service’s short-lived credentials that ' +
|
|
69
|
+
'signed it expire');
|
|
70
|
+
}
|
|
71
|
+
const store = {
|
|
72
|
+
put: async (...a) => (await current()).put(...a),
|
|
73
|
+
get: async (...a) => (await current()).get(...a),
|
|
74
|
+
head: async (...a) => (await current()).head(...a),
|
|
75
|
+
delete: async (...a) => (await current()).delete(...a),
|
|
76
|
+
list: async (...a) => (await current()).list(...a),
|
|
77
|
+
signUpload: async (key, o) => (await lasting(checkExpiresIn(o.expiresIn, UPLOAD_EXPIRES_IN))).signUpload(key, o),
|
|
78
|
+
signDownload: async (key, o = {}) => (await lasting(checkExpiresIn(o.expiresIn, DOWNLOAD_EXPIRES_IN))).signDownload(key, o),
|
|
79
|
+
upload: async (...a) => (await current()).upload(...a),
|
|
80
|
+
};
|
|
81
|
+
if (b.store !== 'public')
|
|
82
|
+
return store;
|
|
83
|
+
if (!b.publicBase)
|
|
84
|
+
throw new StorageError('not_configured', 'a public store binding has no base URL');
|
|
85
|
+
const base = b.publicBase;
|
|
86
|
+
// url() needs no credentials: the public origin plus the encoded key, as the static store builds it.
|
|
87
|
+
return {
|
|
88
|
+
...store,
|
|
89
|
+
url: (key) => {
|
|
90
|
+
checkKey(key);
|
|
91
|
+
return `${base}/${encodeKey(key)}`;
|
|
92
|
+
},
|
|
93
|
+
};
|
|
94
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { Storage } from './types.js';
|
|
2
|
+
/**
|
|
3
|
+
* The storage conformance suite: what "same interface" means, as tests.
|
|
4
|
+
* Every implementation — `memoryStorage`, RustFS under `kf dev`, R2 behind the
|
|
5
|
+
* console — must pass all of it, or it is not offered. Project code written
|
|
6
|
+
* against one therefore runs the same against the others.
|
|
7
|
+
*
|
|
8
|
+
* describeStorageConformance('rustfs', async () => ({ storage, fetch }))
|
|
9
|
+
*
|
|
10
|
+
* Everything is written under a fresh `conformance/<uuid>/` prefix and
|
|
11
|
+
* deleted afterwards, so the suite is safe against a real, shared bucket.
|
|
12
|
+
*/
|
|
13
|
+
export interface ConformanceTarget {
|
|
14
|
+
storage: Storage;
|
|
15
|
+
/**
|
|
16
|
+
* How a browser would reach the URLs the store hands out (signed uploads,
|
|
17
|
+
* signed downloads, public objects). The global fetch for a real backend;
|
|
18
|
+
* `memoryStorage().fetch` for the in-memory store.
|
|
19
|
+
*/
|
|
20
|
+
fetch: (input: string, init?: RequestInit) => Promise<Response>;
|
|
21
|
+
/** false when the environment doesn't declare `public`: the public tests then check it refuses. */
|
|
22
|
+
hasPublic?: boolean;
|
|
23
|
+
}
|
|
24
|
+
export declare function describeStorageConformance(name: string, setup: () => Promise<ConformanceTarget>): void;
|
|
@@ -0,0 +1,311 @@
|
|
|
1
|
+
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
|
2
|
+
import { isStorageError } from './errors.js';
|
|
3
|
+
export function describeStorageConformance(name, setup) {
|
|
4
|
+
describe(`storage conformance: ${name}`, () => {
|
|
5
|
+
let t;
|
|
6
|
+
const run = `conformance/${crypto.randomUUID()}/`;
|
|
7
|
+
const k = (key) => `${run}${key}`;
|
|
8
|
+
beforeAll(async () => {
|
|
9
|
+
t = await setup();
|
|
10
|
+
});
|
|
11
|
+
afterAll(async () => {
|
|
12
|
+
if (!t)
|
|
13
|
+
return;
|
|
14
|
+
const stores = [t.storage];
|
|
15
|
+
if (t.hasPublic !== false)
|
|
16
|
+
stores.push(t.storage.public);
|
|
17
|
+
for (const store of stores) {
|
|
18
|
+
let cursor;
|
|
19
|
+
do {
|
|
20
|
+
const page = await store.list({ prefix: run, ...(cursor ? { cursor } : {}) });
|
|
21
|
+
for (const o of page.objects)
|
|
22
|
+
await store.delete(o.key);
|
|
23
|
+
cursor = page.cursor;
|
|
24
|
+
} while (cursor);
|
|
25
|
+
}
|
|
26
|
+
});
|
|
27
|
+
describe('objects', () => {
|
|
28
|
+
it('put then get returns the bytes, type and metadata', async () => {
|
|
29
|
+
const put = await t.storage.put(k('a.bin'), new Uint8Array([0, 1, 2, 255]), {
|
|
30
|
+
contentType: 'application/x-test',
|
|
31
|
+
metadata: { owner: 'atlas', 'page-count': '3' },
|
|
32
|
+
cacheControl: 'private, max-age=60',
|
|
33
|
+
});
|
|
34
|
+
expect(put).toMatchObject({ key: k('a.bin'), size: 4, contentType: 'application/x-test' });
|
|
35
|
+
expect(put.etag).not.toBe('');
|
|
36
|
+
const got = await t.storage.get(k('a.bin'));
|
|
37
|
+
expect(got).not.toBeNull();
|
|
38
|
+
expect([...new Uint8Array(await got.arrayBuffer())]).toEqual([0, 1, 2, 255]);
|
|
39
|
+
expect(got).toMatchObject({
|
|
40
|
+
size: 4,
|
|
41
|
+
contentType: 'application/x-test',
|
|
42
|
+
cacheControl: 'private, max-age=60',
|
|
43
|
+
metadata: { owner: 'atlas', 'page-count': '3' },
|
|
44
|
+
etag: put.etag,
|
|
45
|
+
});
|
|
46
|
+
});
|
|
47
|
+
it('head agrees with put', async () => {
|
|
48
|
+
const put = await t.storage.put(k('head.txt'), 'hello');
|
|
49
|
+
const head = await t.storage.head(k('head.txt'));
|
|
50
|
+
expect(head).toMatchObject({
|
|
51
|
+
key: k('head.txt'),
|
|
52
|
+
size: 5,
|
|
53
|
+
etag: put.etag,
|
|
54
|
+
contentType: 'text/plain; charset=utf-8',
|
|
55
|
+
});
|
|
56
|
+
expect(Math.abs(head.uploaded.getTime() - put.uploaded.getTime())).toBeLessThan(5000);
|
|
57
|
+
});
|
|
58
|
+
it('accepts a Blob, taking its type, and a stream', async () => {
|
|
59
|
+
await t.storage.put(k('blob.json'), new Blob(['{"a":1}'], { type: 'application/json' }));
|
|
60
|
+
expect((await t.storage.head(k('blob.json')))?.contentType).toBe('application/json');
|
|
61
|
+
const stream = new Blob(['streamed']).stream();
|
|
62
|
+
await t.storage.put(k('stream.txt'), stream, { contentType: 'text/plain' });
|
|
63
|
+
expect(await (await t.storage.get(k('stream.txt'))).text()).toBe('streamed');
|
|
64
|
+
});
|
|
65
|
+
it('get and head of a missing key are null; deleting it is success', async () => {
|
|
66
|
+
expect(await t.storage.get(k('missing'))).toBeNull();
|
|
67
|
+
expect(await t.storage.head(k('missing'))).toBeNull();
|
|
68
|
+
await t.storage.delete(k('missing'));
|
|
69
|
+
});
|
|
70
|
+
it('delete removes; overwrite changes the etag', async () => {
|
|
71
|
+
const first = await t.storage.put(k('over.txt'), 'one');
|
|
72
|
+
const second = await t.storage.put(k('over.txt'), 'two!');
|
|
73
|
+
expect(second.etag).not.toBe(first.etag);
|
|
74
|
+
expect(await (await t.storage.get(k('over.txt'))).text()).toBe('two!');
|
|
75
|
+
await t.storage.delete(k('over.txt'));
|
|
76
|
+
expect(await t.storage.head(k('over.txt'))).toBeNull();
|
|
77
|
+
});
|
|
78
|
+
it('keys that need encoding round-trip exactly', async () => {
|
|
79
|
+
const keys = [
|
|
80
|
+
'sp ace.txt',
|
|
81
|
+
'plus+sign',
|
|
82
|
+
'pct%20lit',
|
|
83
|
+
'q?mark#hash',
|
|
84
|
+
'ünï/cødé/文字',
|
|
85
|
+
"quote'(paren)*",
|
|
86
|
+
];
|
|
87
|
+
for (const key of keys)
|
|
88
|
+
await t.storage.put(k(`enc/${key}`), key);
|
|
89
|
+
for (const key of keys)
|
|
90
|
+
expect(await (await t.storage.get(k(`enc/${key}`))).text()).toBe(key);
|
|
91
|
+
const listed = (await t.storage.list({ prefix: k('enc/') })).objects.map((o) => o.key);
|
|
92
|
+
expect(listed.sort()).toEqual(keys.map((key) => k(`enc/${key}`)).sort());
|
|
93
|
+
});
|
|
94
|
+
});
|
|
95
|
+
describe('list', () => {
|
|
96
|
+
it('lists a prefix in key order, paged by cursor', async () => {
|
|
97
|
+
const names = ['e', 'a', 'c', 'b', 'd'];
|
|
98
|
+
for (const n of names)
|
|
99
|
+
await t.storage.put(k(`list/${n}`), n);
|
|
100
|
+
await t.storage.put(k('listing-not-this'), 'x');
|
|
101
|
+
const seen = [];
|
|
102
|
+
let cursor;
|
|
103
|
+
let pages = 0;
|
|
104
|
+
do {
|
|
105
|
+
const page = await t.storage.list({
|
|
106
|
+
prefix: k('list/'),
|
|
107
|
+
limit: 2,
|
|
108
|
+
...(cursor ? { cursor } : {}),
|
|
109
|
+
});
|
|
110
|
+
expect(page.objects.length).toBeLessThanOrEqual(2);
|
|
111
|
+
seen.push(...page.objects.map((o) => o.key));
|
|
112
|
+
cursor = page.cursor;
|
|
113
|
+
pages++;
|
|
114
|
+
} while (cursor && pages < 10);
|
|
115
|
+
expect(seen).toEqual(['a', 'b', 'c', 'd', 'e'].map((n) => k(`list/${n}`)));
|
|
116
|
+
expect(pages).toBe(3);
|
|
117
|
+
});
|
|
118
|
+
it('reports size and etag', async () => {
|
|
119
|
+
const put = await t.storage.put(k('sized/one'), '12345');
|
|
120
|
+
const [o] = (await t.storage.list({ prefix: k('sized/') })).objects;
|
|
121
|
+
expect(o).toMatchObject({ key: k('sized/one'), size: 5, etag: put.etag });
|
|
122
|
+
expect(o.uploaded).toBeInstanceOf(Date);
|
|
123
|
+
});
|
|
124
|
+
});
|
|
125
|
+
describe('refusals are the same everywhere', () => {
|
|
126
|
+
it.each([
|
|
127
|
+
['empty', ''],
|
|
128
|
+
['a leading slash', '/abs'],
|
|
129
|
+
['a .. segment', 'a/../b'],
|
|
130
|
+
['a . segment', 'a/./b'],
|
|
131
|
+
['a control character', 'a\u0001b'],
|
|
132
|
+
['more than 1024 bytes', 'x'.repeat(1025)],
|
|
133
|
+
])('a key with %s is invalid', async (_why, key) => {
|
|
134
|
+
await expect(t.storage.put(key, 'x')).rejects.toSatisfy((e) => isStorageError(e, 'invalid'));
|
|
135
|
+
await expect(t.storage.get(key)).rejects.toSatisfy((e) => isStorageError(e, 'invalid'));
|
|
136
|
+
});
|
|
137
|
+
it('every failure is a rejected promise, never a synchronous throw', async () => {
|
|
138
|
+
const calls = [
|
|
139
|
+
() => t.storage.put('', 'x'),
|
|
140
|
+
() => t.storage.upload(new Blob(['x']), { prefix: '/abs' }),
|
|
141
|
+
() => t.storage.signUpload('', { contentType: 'text/plain', size: 1 }),
|
|
142
|
+
() => t.storage.list({ limit: 0 }),
|
|
143
|
+
];
|
|
144
|
+
for (const call of calls) {
|
|
145
|
+
let promise;
|
|
146
|
+
expect(() => {
|
|
147
|
+
promise = call();
|
|
148
|
+
}).not.toThrow();
|
|
149
|
+
await expect(promise).rejects.toSatisfy((e) => isStorageError(e, 'invalid'));
|
|
150
|
+
}
|
|
151
|
+
});
|
|
152
|
+
it('a metadata name outside [a-z0-9-] is invalid', async () => {
|
|
153
|
+
await expect(t.storage.put(k('meta'), 'x', { metadata: { Bad_Name: 'x' } })).rejects.toSatisfy((e) => isStorageError(e, 'invalid'));
|
|
154
|
+
});
|
|
155
|
+
});
|
|
156
|
+
describe('upload', () => {
|
|
157
|
+
it('generates a key under the prefix, keeping the extension and type', async () => {
|
|
158
|
+
const file = new File(['cover'], 'Cover Art.JPG', { type: 'image/jpeg' });
|
|
159
|
+
const o = await t.storage.upload(file, { prefix: k('covers') });
|
|
160
|
+
expect(o.key).toMatch(new RegExp(`^${k('covers/')}[0-9a-f-]{36}\\.jpg$`));
|
|
161
|
+
expect(o.contentType).toBe('image/jpeg');
|
|
162
|
+
expect(await (await t.storage.get(o.key)).text()).toBe('cover');
|
|
163
|
+
});
|
|
164
|
+
});
|
|
165
|
+
describe('signed URLs', () => {
|
|
166
|
+
it('signUpload: a PUT with the signed headers and exact size lands', async () => {
|
|
167
|
+
const body = 'uploaded from a browser';
|
|
168
|
+
const up = await t.storage.signUpload(k('signed/ok.txt'), {
|
|
169
|
+
contentType: 'text/plain',
|
|
170
|
+
size: new TextEncoder().encode(body).length,
|
|
171
|
+
});
|
|
172
|
+
expect(up.method).toBe('PUT');
|
|
173
|
+
const res = await t.fetch(up.url, { method: 'PUT', headers: up.headers, body });
|
|
174
|
+
expect(res.status).toBeLessThan(300);
|
|
175
|
+
const got = await t.storage.get(k('signed/ok.txt'));
|
|
176
|
+
expect(await got.text()).toBe(body);
|
|
177
|
+
expect(got.contentType).toBe('text/plain');
|
|
178
|
+
});
|
|
179
|
+
it('signUpload: a different size or type is refused', async () => {
|
|
180
|
+
const up = await t.storage.signUpload(k('signed/strict.txt'), {
|
|
181
|
+
contentType: 'text/plain',
|
|
182
|
+
size: 5,
|
|
183
|
+
});
|
|
184
|
+
const tooBig = await t.fetch(up.url, { method: 'PUT', headers: up.headers, body: '123456' });
|
|
185
|
+
expect(tooBig.ok).toBe(false);
|
|
186
|
+
const wrongType = await t.fetch(up.url, {
|
|
187
|
+
method: 'PUT',
|
|
188
|
+
headers: { ...up.headers, 'content-type': 'text/html' },
|
|
189
|
+
body: '12345',
|
|
190
|
+
});
|
|
191
|
+
expect(wrongType.ok).toBe(false);
|
|
192
|
+
expect(await t.storage.head(k('signed/strict.txt'))).toBeNull();
|
|
193
|
+
});
|
|
194
|
+
it('signDownload: GET returns the object, as an attachment when named', async () => {
|
|
195
|
+
await t.storage.put(k('signed/dl.txt'), 'download me');
|
|
196
|
+
const res = await t.fetch(await t.storage.signDownload(k('signed/dl.txt')));
|
|
197
|
+
expect(res.status).toBe(200);
|
|
198
|
+
expect(await res.text()).toBe('download me');
|
|
199
|
+
const named = await t.fetch(await t.storage.signDownload(k('signed/dl.txt'), { filename: 'Réport.txt' }));
|
|
200
|
+
expect(named.headers.get('content-disposition')).toContain('attachment');
|
|
201
|
+
expect(named.headers.get('content-disposition')).toContain('R%C3%A9port.txt');
|
|
202
|
+
});
|
|
203
|
+
it('an expired signed URL is refused', async () => {
|
|
204
|
+
await t.storage.put(k('signed/expiring.txt'), 'soon gone');
|
|
205
|
+
const url = await t.storage.signDownload(k('signed/expiring.txt'), { expiresIn: 2 });
|
|
206
|
+
expect((await t.fetch(url)).ok).toBe(true);
|
|
207
|
+
// The server judges expiry by its own clock, which can drift from
|
|
208
|
+
// ours (a container's VM, say): poll for the refusal rather than
|
|
209
|
+
// assume exactly when it comes.
|
|
210
|
+
let refused = false;
|
|
211
|
+
for (let waited = 0; !refused && waited < 10_000; waited += 500) {
|
|
212
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
213
|
+
refused = !(await t.fetch(url)).ok;
|
|
214
|
+
}
|
|
215
|
+
expect(refused).toBe(true);
|
|
216
|
+
}, 15_000);
|
|
217
|
+
it('expiresIn outside 1–604800 seconds is invalid', async () => {
|
|
218
|
+
await expect(t.storage.signDownload(k('x'), { expiresIn: 604_801 })).rejects.toSatisfy((e) => isStorageError(e, 'invalid'));
|
|
219
|
+
});
|
|
220
|
+
});
|
|
221
|
+
describe('public store', () => {
|
|
222
|
+
it('is refused with not_configured when the environment has none', async () => {
|
|
223
|
+
if (t.hasPublic !== false)
|
|
224
|
+
return;
|
|
225
|
+
expect(() => t.storage.public).toThrow(expect.objectContaining({ code: 'not_configured' }));
|
|
226
|
+
});
|
|
227
|
+
it('url(key) serves the object to anyone', async () => {
|
|
228
|
+
if (t.hasPublic === false)
|
|
229
|
+
return;
|
|
230
|
+
await t.storage.public.put(k('pub/hello.txt'), 'public bytes', {
|
|
231
|
+
contentType: 'text/plain',
|
|
232
|
+
});
|
|
233
|
+
const res = await t.fetch(t.storage.public.url(k('pub/hello.txt')));
|
|
234
|
+
expect(res.status).toBe(200);
|
|
235
|
+
expect(await res.text()).toBe('public bytes');
|
|
236
|
+
expect(res.headers.get('cache-control')).toBe('public, max-age=300');
|
|
237
|
+
});
|
|
238
|
+
it('upload() writes an immutable object at a fresh key', async () => {
|
|
239
|
+
if (t.hasPublic === false)
|
|
240
|
+
return;
|
|
241
|
+
const o = await t.storage.public.upload(new Blob(['img'], { type: 'image/png' }), {
|
|
242
|
+
prefix: k('pub/covers'),
|
|
243
|
+
});
|
|
244
|
+
expect(o.cacheControl).toBe('public, max-age=31536000, immutable');
|
|
245
|
+
const res = await t.fetch(t.storage.public.url(o.key));
|
|
246
|
+
expect(await res.text()).toBe('img');
|
|
247
|
+
expect(res.headers.get('cache-control')).toBe('public, max-age=31536000, immutable');
|
|
248
|
+
});
|
|
249
|
+
it('serves anything but raster images, audio and video as an attachment', async () => {
|
|
250
|
+
if (t.hasPublic === false)
|
|
251
|
+
return;
|
|
252
|
+
const page = '<html><body>not a scam</body></html>';
|
|
253
|
+
const html = await t.storage.public.put(k('pub/page.html'), page, {
|
|
254
|
+
contentType: 'text/html',
|
|
255
|
+
});
|
|
256
|
+
expect(html.contentDisposition).toMatch(/^attachment/);
|
|
257
|
+
const res = await t.fetch(t.storage.public.url(k('pub/page.html')));
|
|
258
|
+
expect(res.headers.get('content-disposition')).toMatch(/^attachment/);
|
|
259
|
+
expect(await res.text()).toBe(page);
|
|
260
|
+
// Asking for inline is not honoured for a type outside the list.
|
|
261
|
+
const svg = await t.storage.public.put(k('pub/logo.svg'), '<svg/>', {
|
|
262
|
+
contentType: 'image/svg+xml',
|
|
263
|
+
contentDisposition: 'inline',
|
|
264
|
+
});
|
|
265
|
+
expect(svg.contentDisposition).toMatch(/^attachment/);
|
|
266
|
+
// A caller's own attachment, with a file name, is kept.
|
|
267
|
+
const named = await t.storage.public.put(k('pub/r.pdf'), '%PDF', {
|
|
268
|
+
contentType: 'application/pdf',
|
|
269
|
+
contentDisposition: 'attachment; filename="report.pdf"',
|
|
270
|
+
});
|
|
271
|
+
expect(named.contentDisposition).toBe('attachment; filename="report.pdf"');
|
|
272
|
+
const png = await t.storage.public.put(k('pub/cover.png'), new Uint8Array([137, 80]), {
|
|
273
|
+
contentType: 'image/png',
|
|
274
|
+
});
|
|
275
|
+
expect(png.contentDisposition).toBeUndefined();
|
|
276
|
+
const img = await t.fetch(t.storage.public.url(k('pub/cover.png')));
|
|
277
|
+
expect(img.headers.get('content-disposition')).toBeNull();
|
|
278
|
+
});
|
|
279
|
+
it('signs the attachment into a public upload, so a browser cannot drop it', async () => {
|
|
280
|
+
if (t.hasPublic === false)
|
|
281
|
+
return;
|
|
282
|
+
const body = '<html></html>';
|
|
283
|
+
const up = await t.storage.public.signUpload(k('pub/up.html'), {
|
|
284
|
+
contentType: 'text/html',
|
|
285
|
+
size: body.length,
|
|
286
|
+
});
|
|
287
|
+
expect(up.headers['content-disposition']).toBe('attachment');
|
|
288
|
+
const { 'content-disposition': _dropped, ...without } = up.headers;
|
|
289
|
+
expect((await t.fetch(up.url, { method: 'PUT', headers: without, body })).ok).toBe(false);
|
|
290
|
+
expect((await t.fetch(up.url, { method: 'PUT', headers: up.headers, body })).ok).toBe(true);
|
|
291
|
+
expect((await t.storage.public.head(k('pub/up.html')))?.contentDisposition).toBe('attachment');
|
|
292
|
+
const img = await t.storage.public.signUpload(k('pub/up.webp'), {
|
|
293
|
+
contentType: 'image/webp',
|
|
294
|
+
size: 3,
|
|
295
|
+
});
|
|
296
|
+
expect(img.headers['content-disposition']).toBeUndefined();
|
|
297
|
+
});
|
|
298
|
+
it('leaves the private store alone', async () => {
|
|
299
|
+
const o = await t.storage.put(k('priv.html'), '<p>', { contentType: 'text/html' });
|
|
300
|
+
expect(o.contentDisposition).toBeUndefined();
|
|
301
|
+
});
|
|
302
|
+
it('the private and public stores are separate', async () => {
|
|
303
|
+
if (t.hasPublic === false)
|
|
304
|
+
return;
|
|
305
|
+
await t.storage.put(k('pub/secret.txt'), 'private');
|
|
306
|
+
expect(await t.storage.public.head(k('pub/secret.txt'))).toBeNull();
|
|
307
|
+
expect((await t.fetch(t.storage.public.url(k('pub/secret.txt')))).ok).toBe(false);
|
|
308
|
+
});
|
|
309
|
+
});
|
|
310
|
+
});
|
|
311
|
+
}
|