@knowledgefutures/sdk 0.0.0-stage → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +71 -2
- package/dist/storage/binding.d.ts +76 -0
- package/dist/storage/binding.js +97 -0
- package/dist/storage/brokered.d.ts +21 -0
- package/dist/storage/brokered.js +94 -0
- package/dist/storage/errors.d.ts +27 -0
- package/dist/storage/errors.js +21 -0
- package/dist/storage/index.d.ts +34 -0
- package/dist/storage/index.js +77 -0
- package/dist/storage/s3.d.ts +22 -0
- package/dist/storage/s3.js +254 -0
- package/dist/storage/shared.d.ts +45 -0
- package/dist/storage/shared.js +167 -0
- package/dist/storage/sigv4.d.ts +50 -0
- package/dist/storage/sigv4.js +128 -0
- package/dist/storage/testing.d.ts +25 -0
- package/dist/storage/testing.js +242 -0
- package/dist/storage/types.d.ts +122 -0
- package/dist/storage/types.js +13 -0
- package/dist/whistle/hono.d.ts +42 -0
- package/dist/whistle/hono.js +89 -0
- package/dist/whistle/index.d.ts +135 -0
- package/dist/whistle/index.js +249 -0
- package/dist/whistle/wire.d.ts +64 -0
- package/dist/whistle/wire.js +98 -0
- package/package.json +34 -3
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
import { StorageError } from './errors.js';
|
|
2
|
+
import { attachment, checkExpiresIn, DOWNLOAD_EXPIRES_IN, UPLOAD_EXPIRES_IN, checkKey, checkMetadata, checkPrefix, checkSignUpload, IMMUTABLE, publicDisposition, SHORT_PUBLIC, toBytes, uploadAsPut, } from './shared.js';
|
|
3
|
+
import { encodeKey } from './sigv4.js';
|
|
4
|
+
const PRIVATE_ORIGIN = 'https://signed.storage.test';
|
|
5
|
+
const DEFAULT_PUBLIC_BASE = 'https://public.storage.test';
|
|
6
|
+
export function memoryStorage(options = {}) {
|
|
7
|
+
const grants = new Map();
|
|
8
|
+
const priv = new MemoryStore(grants, false);
|
|
9
|
+
const pub = options.public === false
|
|
10
|
+
? null
|
|
11
|
+
: new MemoryStore(grants, true, (options.public?.base ?? DEFAULT_PUBLIC_BASE).replace(/\/+$/, ''));
|
|
12
|
+
async function serve(input, init) {
|
|
13
|
+
const req = new Request(input, init);
|
|
14
|
+
const url = new URL(req.url);
|
|
15
|
+
if (pub &&
|
|
16
|
+
req.url.startsWith(`${pub.base}/`) &&
|
|
17
|
+
(req.method === 'GET' || req.method === 'HEAD')) {
|
|
18
|
+
const key = decodeKey(url.pathname);
|
|
19
|
+
const e = pub.objects.get(key);
|
|
20
|
+
if (!e)
|
|
21
|
+
return new Response('not found', { status: 404 });
|
|
22
|
+
return objectResponse(e, req.method === 'HEAD');
|
|
23
|
+
}
|
|
24
|
+
if (url.origin !== PRIVATE_ORIGIN)
|
|
25
|
+
return new Response('not a URL this store handed out', { status: 404 });
|
|
26
|
+
const grant = grants.get(url.searchParams.get('kf-signature') ?? '');
|
|
27
|
+
if (!grant || grant.method !== req.method || grant.key !== decodeKey(url.pathname)) {
|
|
28
|
+
return new Response('SignatureDoesNotMatch', { status: 403 });
|
|
29
|
+
}
|
|
30
|
+
if (Date.now() > grant.expires)
|
|
31
|
+
return new Response('Request has expired', { status: 403 });
|
|
32
|
+
if (grant.method === 'GET') {
|
|
33
|
+
const e = grant.store.objects.get(grant.key);
|
|
34
|
+
if (!e)
|
|
35
|
+
return new Response('not found', { status: 404 });
|
|
36
|
+
const res = objectResponse(e, false);
|
|
37
|
+
if (grant.disposition)
|
|
38
|
+
res.headers.set('content-disposition', grant.disposition);
|
|
39
|
+
return res;
|
|
40
|
+
}
|
|
41
|
+
// PUT: exactly the signed headers, exactly the signed size.
|
|
42
|
+
const bytes = new Uint8Array(await req.arrayBuffer());
|
|
43
|
+
if (req.headers.get('content-type') !== grant.contentType ||
|
|
44
|
+
(req.headers.get('cache-control') ?? undefined) !== grant.cacheControl ||
|
|
45
|
+
(req.headers.get('content-disposition') ?? undefined) !== grant.disposition ||
|
|
46
|
+
bytes.byteLength !== grant.size) {
|
|
47
|
+
return new Response('SignatureDoesNotMatch', { status: 403 });
|
|
48
|
+
}
|
|
49
|
+
await grant.store.put(grant.key, bytes, {
|
|
50
|
+
contentType: grant.contentType,
|
|
51
|
+
...(grant.cacheControl ? { cacheControl: grant.cacheControl } : {}),
|
|
52
|
+
...(grant.disposition ? { contentDisposition: grant.disposition } : {}),
|
|
53
|
+
});
|
|
54
|
+
return new Response(null, { status: 200 });
|
|
55
|
+
}
|
|
56
|
+
return {
|
|
57
|
+
put: (key, body, o) => priv.put(key, body, o),
|
|
58
|
+
get: (key) => priv.get(key),
|
|
59
|
+
head: (key) => priv.head(key),
|
|
60
|
+
delete: (key) => priv.delete(key),
|
|
61
|
+
list: (o) => priv.list(o),
|
|
62
|
+
signUpload: (key, o) => priv.signUpload(key, o),
|
|
63
|
+
signDownload: (key, o) => priv.signDownload(key, o),
|
|
64
|
+
upload: (file, o) => priv.upload(file, o),
|
|
65
|
+
get public() {
|
|
66
|
+
if (!pub) {
|
|
67
|
+
throw new StorageError('not_configured', 'this store has no public store (memoryStorage({ public: false }))');
|
|
68
|
+
}
|
|
69
|
+
return pub;
|
|
70
|
+
},
|
|
71
|
+
fetch: serve,
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
class MemoryStore {
|
|
75
|
+
grants;
|
|
76
|
+
isPublic;
|
|
77
|
+
base;
|
|
78
|
+
objects = new Map();
|
|
79
|
+
constructor(grants, isPublic, base) {
|
|
80
|
+
this.grants = grants;
|
|
81
|
+
this.isPublic = isPublic;
|
|
82
|
+
this.base = base;
|
|
83
|
+
}
|
|
84
|
+
async put(key, body, opts = {}) {
|
|
85
|
+
checkKey(key);
|
|
86
|
+
const metadata = checkMetadata(opts.metadata);
|
|
87
|
+
const { bytes, contentType } = await toBytes(body, opts.contentType);
|
|
88
|
+
const cacheControl = opts.cacheControl ?? (this.isPublic ? SHORT_PUBLIC : undefined);
|
|
89
|
+
const disposition = this.isPublic
|
|
90
|
+
? publicDisposition(contentType, opts.contentDisposition)
|
|
91
|
+
: opts.contentDisposition;
|
|
92
|
+
const e = {
|
|
93
|
+
key,
|
|
94
|
+
size: bytes.byteLength,
|
|
95
|
+
etag: crypto.randomUUID().replaceAll('-', ''),
|
|
96
|
+
uploaded: new Date(Math.floor(Date.now() / 1000) * 1000),
|
|
97
|
+
contentType,
|
|
98
|
+
metadata,
|
|
99
|
+
bytes,
|
|
100
|
+
};
|
|
101
|
+
if (cacheControl)
|
|
102
|
+
e.cacheControl = cacheControl;
|
|
103
|
+
if (disposition)
|
|
104
|
+
e.contentDisposition = disposition;
|
|
105
|
+
this.objects.set(key, e);
|
|
106
|
+
return describe(e);
|
|
107
|
+
}
|
|
108
|
+
async get(key) {
|
|
109
|
+
checkKey(key);
|
|
110
|
+
const e = this.objects.get(key);
|
|
111
|
+
if (!e)
|
|
112
|
+
return null;
|
|
113
|
+
const copy = e.bytes.slice();
|
|
114
|
+
return {
|
|
115
|
+
...describe(e),
|
|
116
|
+
body: new Response(copy).body,
|
|
117
|
+
arrayBuffer: async () => copy.buffer.slice(0),
|
|
118
|
+
text: async () => new TextDecoder().decode(copy),
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
async head(key) {
|
|
122
|
+
checkKey(key);
|
|
123
|
+
const e = this.objects.get(key);
|
|
124
|
+
return e ? describe(e) : null;
|
|
125
|
+
}
|
|
126
|
+
async delete(key) {
|
|
127
|
+
checkKey(key);
|
|
128
|
+
this.objects.delete(key);
|
|
129
|
+
}
|
|
130
|
+
async list(opts = {}) {
|
|
131
|
+
checkPrefix(opts.prefix);
|
|
132
|
+
const limit = opts.limit ?? 1000;
|
|
133
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > 1000) {
|
|
134
|
+
throw new StorageError('invalid', 'limit is 1–1000');
|
|
135
|
+
}
|
|
136
|
+
const after = opts.cursor ? decodeCursor(opts.cursor) : undefined;
|
|
137
|
+
const keys = [...this.objects.keys()]
|
|
138
|
+
.filter((k) => k.startsWith(opts.prefix ?? '') && (after === undefined || byteCompare(k, after) > 0))
|
|
139
|
+
.sort(byteCompare);
|
|
140
|
+
const page = keys.slice(0, limit);
|
|
141
|
+
const objects = page.map((k) => {
|
|
142
|
+
const e = this.objects.get(k);
|
|
143
|
+
return { key: e.key, size: e.size, etag: e.etag, uploaded: e.uploaded };
|
|
144
|
+
});
|
|
145
|
+
return keys.length > limit
|
|
146
|
+
? { objects, cursor: btoa(encodeURIComponent(page.at(-1))) }
|
|
147
|
+
: { objects };
|
|
148
|
+
}
|
|
149
|
+
async signUpload(key, o) {
|
|
150
|
+
checkKey(key);
|
|
151
|
+
checkSignUpload(o);
|
|
152
|
+
const expiresIn = checkExpiresIn(o.expiresIn, UPLOAD_EXPIRES_IN);
|
|
153
|
+
const cacheControl = o.cacheControl ?? (this.isPublic ? SHORT_PUBLIC : undefined);
|
|
154
|
+
const headers = { 'content-type': o.contentType };
|
|
155
|
+
if (cacheControl)
|
|
156
|
+
headers['cache-control'] = cacheControl;
|
|
157
|
+
const disposition = this.isPublic ? publicDisposition(o.contentType, undefined) : undefined;
|
|
158
|
+
if (disposition)
|
|
159
|
+
headers['content-disposition'] = disposition;
|
|
160
|
+
const url = this.grant({
|
|
161
|
+
method: 'PUT',
|
|
162
|
+
key,
|
|
163
|
+
expiresIn,
|
|
164
|
+
contentType: o.contentType,
|
|
165
|
+
size: o.size,
|
|
166
|
+
...(cacheControl ? { cacheControl } : {}),
|
|
167
|
+
...(disposition ? { disposition } : {}),
|
|
168
|
+
});
|
|
169
|
+
return {
|
|
170
|
+
key,
|
|
171
|
+
url,
|
|
172
|
+
method: 'PUT',
|
|
173
|
+
headers,
|
|
174
|
+
expiresAt: new Date(Date.now() + expiresIn * 1000),
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
async signDownload(key, o = {}) {
|
|
178
|
+
checkKey(key);
|
|
179
|
+
const expiresIn = checkExpiresIn(o.expiresIn, DOWNLOAD_EXPIRES_IN);
|
|
180
|
+
return this.grant({
|
|
181
|
+
method: 'GET',
|
|
182
|
+
key,
|
|
183
|
+
expiresIn,
|
|
184
|
+
...(o.filename ? { disposition: attachment(o.filename) } : {}),
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
async upload(file, options) {
|
|
188
|
+
const p = uploadAsPut(file, options, this.isPublic ? { cacheControl: IMMUTABLE } : {});
|
|
189
|
+
return this.put(p.key, p.body, p.options);
|
|
190
|
+
}
|
|
191
|
+
url(key) {
|
|
192
|
+
checkKey(key);
|
|
193
|
+
if (!this.base)
|
|
194
|
+
throw new StorageError('not_configured', 'the private store has no public URLs');
|
|
195
|
+
return `${this.base}/${encodeKey(key)}`;
|
|
196
|
+
}
|
|
197
|
+
grant(g) {
|
|
198
|
+
const token = crypto.randomUUID();
|
|
199
|
+
const { expiresIn, ...rest } = g;
|
|
200
|
+
this.grants.set(token, { ...rest, store: this, expires: Date.now() + expiresIn * 1000 });
|
|
201
|
+
return `${PRIVATE_ORIGIN}/${encodeKey(g.key)}?kf-signature=${token}`;
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
function describe(e) {
|
|
205
|
+
const { bytes: _bytes, ...meta } = e;
|
|
206
|
+
return { ...meta, metadata: { ...meta.metadata } };
|
|
207
|
+
}
|
|
208
|
+
function objectResponse(e, headOnly) {
|
|
209
|
+
const headers = new Headers({
|
|
210
|
+
'content-type': e.contentType,
|
|
211
|
+
'content-length': String(e.size),
|
|
212
|
+
etag: `"${e.etag}"`,
|
|
213
|
+
'last-modified': e.uploaded.toUTCString(),
|
|
214
|
+
});
|
|
215
|
+
if (e.cacheControl)
|
|
216
|
+
headers.set('cache-control', e.cacheControl);
|
|
217
|
+
if (e.contentDisposition)
|
|
218
|
+
headers.set('content-disposition', e.contentDisposition);
|
|
219
|
+
return new Response(headOnly ? null : e.bytes.slice(), { status: 200, headers });
|
|
220
|
+
}
|
|
221
|
+
function decodeKey(pathname) {
|
|
222
|
+
return pathname.slice(1).split('/').map(decodeURIComponent).join('/');
|
|
223
|
+
}
|
|
224
|
+
function decodeCursor(cursor) {
|
|
225
|
+
try {
|
|
226
|
+
return decodeURIComponent(atob(cursor));
|
|
227
|
+
}
|
|
228
|
+
catch {
|
|
229
|
+
throw new StorageError('invalid', 'not a cursor this store handed out');
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
const utf8 = new TextEncoder();
|
|
233
|
+
/** S3 lists in UTF-8 byte order, which differs from UTF-16 `<` above the BMP. */
|
|
234
|
+
function byteCompare(a, b) {
|
|
235
|
+
const x = utf8.encode(a);
|
|
236
|
+
const y = utf8.encode(b);
|
|
237
|
+
for (let i = 0; i < Math.min(x.length, y.length); i++) {
|
|
238
|
+
if (x[i] !== y[i])
|
|
239
|
+
return x[i] - y[i];
|
|
240
|
+
}
|
|
241
|
+
return x.length - y.length;
|
|
242
|
+
}
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The storage interface every implementation honours: R2 behind the console,
|
|
3
|
+
* RustFS under `kf dev`, and `memoryStorage` in tests. Project code is written
|
|
4
|
+
* against these types only, and the conformance suite (conformance.ts) holds
|
|
5
|
+
* each implementation to them, so code that passes against one runs the same
|
|
6
|
+
* against the others.
|
|
7
|
+
*
|
|
8
|
+
* The surface is deliberately the subset all of them can honour: no POST
|
|
9
|
+
* policies (R2 has none), no per-object ACLs (R2 has none — public is a
|
|
10
|
+
* separate store), no multipart. Large files go browser → bucket through
|
|
11
|
+
* `signUpload`, never through the project's server.
|
|
12
|
+
*/
|
|
13
|
+
/** What `put` accepts. A stream is read to the end before upload; use `signUpload` for large files. */
|
|
14
|
+
export type Body = string | ArrayBuffer | ArrayBufferView | Blob | ReadableStream<Uint8Array>;
|
|
15
|
+
export interface PutOptions {
|
|
16
|
+
contentType?: string;
|
|
17
|
+
/** Sent back verbatim on every read — and, on the public store, what Cloudflare's cache obeys. */
|
|
18
|
+
cacheControl?: string;
|
|
19
|
+
contentDisposition?: string;
|
|
20
|
+
/** Lowercase `[a-z0-9-]` keys. Small: S3 caps all metadata at 2 KB. */
|
|
21
|
+
metadata?: Record<string, string>;
|
|
22
|
+
}
|
|
23
|
+
/** One object as a `list` returns it: only what every backend's listing reports. */
|
|
24
|
+
export interface ListedObject {
|
|
25
|
+
key: string;
|
|
26
|
+
size: number;
|
|
27
|
+
/** Opaque. Stable while the object is unchanged; changes when it is overwritten. */
|
|
28
|
+
etag: string;
|
|
29
|
+
uploaded: Date;
|
|
30
|
+
}
|
|
31
|
+
/** One object as `put` and `head` report it. */
|
|
32
|
+
export interface StoredObject extends ListedObject {
|
|
33
|
+
contentType: string;
|
|
34
|
+
cacheControl?: string;
|
|
35
|
+
contentDisposition?: string;
|
|
36
|
+
metadata: Record<string, string>;
|
|
37
|
+
}
|
|
38
|
+
export interface ObjectBody extends StoredObject {
|
|
39
|
+
body: ReadableStream<Uint8Array>;
|
|
40
|
+
arrayBuffer(): Promise<ArrayBuffer>;
|
|
41
|
+
text(): Promise<string>;
|
|
42
|
+
}
|
|
43
|
+
export interface ListOptions {
|
|
44
|
+
prefix?: string;
|
|
45
|
+
/** From the previous page's `cursor`. Opaque. */
|
|
46
|
+
cursor?: string;
|
|
47
|
+
/** 1–1000. Default 1000. */
|
|
48
|
+
limit?: number;
|
|
49
|
+
}
|
|
50
|
+
export interface ListResult {
|
|
51
|
+
/** In key order (UTF-8 byte order). */
|
|
52
|
+
objects: ListedObject[];
|
|
53
|
+
/** Present when there is another page. */
|
|
54
|
+
cursor?: string;
|
|
55
|
+
}
|
|
56
|
+
export interface SignUploadOptions {
|
|
57
|
+
contentType: string;
|
|
58
|
+
/** The exact byte length. It is signed, so the upload must be exactly this size. */
|
|
59
|
+
size: number;
|
|
60
|
+
/**
|
|
61
|
+
* Seconds. Default 600. Up to 604800 with fixed credentials (`kf dev`, tests);
|
|
62
|
+
* on the console, at most the storage service's credential lifetime (an hour),
|
|
63
|
+
* because a signed URL stops working when the credentials that signed it expire.
|
|
64
|
+
*/
|
|
65
|
+
expiresIn?: number;
|
|
66
|
+
cacheControl?: string;
|
|
67
|
+
}
|
|
68
|
+
/** What a browser needs to upload: PUT the file to `url` with exactly these headers. */
|
|
69
|
+
export interface SignedUpload {
|
|
70
|
+
key: string;
|
|
71
|
+
url: string;
|
|
72
|
+
method: 'PUT';
|
|
73
|
+
headers: Record<string, string>;
|
|
74
|
+
expiresAt: Date;
|
|
75
|
+
}
|
|
76
|
+
export interface SignDownloadOptions {
|
|
77
|
+
/** Seconds. Default 300. The same limits as `SignUploadOptions.expiresIn`. */
|
|
78
|
+
expiresIn?: number;
|
|
79
|
+
/** Served as an attachment with this file name. */
|
|
80
|
+
filename?: string;
|
|
81
|
+
}
|
|
82
|
+
/** A file with a name, for `upload`. A `File` is one. */
|
|
83
|
+
export interface Upload {
|
|
84
|
+
body: Body;
|
|
85
|
+
contentType?: string;
|
|
86
|
+
/** Its extension carries over to the generated key. */
|
|
87
|
+
name?: string;
|
|
88
|
+
}
|
|
89
|
+
export interface UploadOptions {
|
|
90
|
+
/** The key is `<prefix>/<uuid><ext>`. */
|
|
91
|
+
prefix?: string;
|
|
92
|
+
cacheControl?: string;
|
|
93
|
+
metadata?: Record<string, string>;
|
|
94
|
+
}
|
|
95
|
+
export interface Store {
|
|
96
|
+
put(key: string, body: Body, options?: PutOptions): Promise<StoredObject>;
|
|
97
|
+
/** null when there is no such object. */
|
|
98
|
+
get(key: string): Promise<ObjectBody | null>;
|
|
99
|
+
/** null when there is no such object. */
|
|
100
|
+
head(key: string): Promise<StoredObject | null>;
|
|
101
|
+
/** Deleting what is not there is success. */
|
|
102
|
+
delete(key: string): Promise<void>;
|
|
103
|
+
list(options?: ListOptions): Promise<ListResult>;
|
|
104
|
+
signUpload(key: string, options: SignUploadOptions): Promise<SignedUpload>;
|
|
105
|
+
signDownload(key: string, options?: SignDownloadOptions): Promise<string>;
|
|
106
|
+
/** A generated key, so nothing is ever overwritten by accident. */
|
|
107
|
+
upload(file: Blob | Upload, options?: UploadOptions): Promise<StoredObject>;
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* The public store: everything in it is readable by anyone at `url(key)`,
|
|
111
|
+
* through Cloudflare's cache and with no signature. Treat its keys as
|
|
112
|
+
* immutable — an overwrite or delete is seen only after the object's
|
|
113
|
+
* `max-age`, since nothing purges the cache.
|
|
114
|
+
*/
|
|
115
|
+
export interface PublicStore extends Store {
|
|
116
|
+
/** Where anyone can GET the object. Synchronous: no network, no signing. */
|
|
117
|
+
url(key: string): string;
|
|
118
|
+
}
|
|
119
|
+
export interface Storage extends Store {
|
|
120
|
+
/** Throws `not_configured` unless the manifest declares `"public": true`. */
|
|
121
|
+
readonly public: PublicStore;
|
|
122
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The storage interface every implementation honours: R2 behind the console,
|
|
3
|
+
* RustFS under `kf dev`, and `memoryStorage` in tests. Project code is written
|
|
4
|
+
* against these types only, and the conformance suite (conformance.ts) holds
|
|
5
|
+
* each implementation to them, so code that passes against one runs the same
|
|
6
|
+
* against the others.
|
|
7
|
+
*
|
|
8
|
+
* The surface is deliberately the subset all of them can honour: no POST
|
|
9
|
+
* policies (R2 has none), no per-object ACLs (R2 has none — public is a
|
|
10
|
+
* separate store), no multipart. Large files go browser → bucket through
|
|
11
|
+
* `signUpload`, never through the project's server.
|
|
12
|
+
*/
|
|
13
|
+
export {};
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `app.use(whistle())`: Hono middleware for what a tailed Worker's runs don't
|
|
3
|
+
* say by themselves. Typed by shape, so the SDK doesn't depend on Hono.
|
|
4
|
+
*
|
|
5
|
+
* 403s, and 401s to a request that presented a credential → `access.denied`
|
|
6
|
+
* writes under /admin that succeed → `admin.action`
|
|
7
|
+
* an error a custom onError turned into a 5xx → console.error, for the tail
|
|
8
|
+
*
|
|
9
|
+
* The user comes from `c.var.user.id` unless `user` says otherwise.
|
|
10
|
+
*/
|
|
11
|
+
import { type Sink } from './index.js';
|
|
12
|
+
/**
|
|
13
|
+
* The parts of Hono's Context this reads. `var` is loose so a `user` option
|
|
14
|
+
* reads the app's variables untyped; `whistle<Context<AppEnv>>()` types them.
|
|
15
|
+
*/
|
|
16
|
+
export interface HonoContextLike {
|
|
17
|
+
req: {
|
|
18
|
+
raw: Request;
|
|
19
|
+
method: string;
|
|
20
|
+
path: string;
|
|
21
|
+
routePath: string;
|
|
22
|
+
};
|
|
23
|
+
res: Response;
|
|
24
|
+
env: unknown;
|
|
25
|
+
error: Error | undefined;
|
|
26
|
+
var: any;
|
|
27
|
+
}
|
|
28
|
+
export interface WhistleOptions<C extends HonoContextLike = HonoContextLike> {
|
|
29
|
+
/** The console project key, e.g. `kf/pilot`. Default: `KF_WHISTLE_PROJECT`, which the console sets. */
|
|
30
|
+
project?: string;
|
|
31
|
+
/** Default: `KF_WHISTLE_ENVIRONMENT`, which the console sets, else `production`. */
|
|
32
|
+
environment?: string;
|
|
33
|
+
/** The signed-in user's id, if any; asked only when there's an event. Default: `c.var.user?.id`. */
|
|
34
|
+
user?: (c: C) => string | null | undefined | Promise<string | null | undefined>;
|
|
35
|
+
/** Path prefixes whose successful writes are recorded. Default `/admin`, `/api/admin`; false for none. */
|
|
36
|
+
admin?: string[] | false;
|
|
37
|
+
/** Where `KF_*` variables come from. Default: `c.env`, then `process.env`. */
|
|
38
|
+
env?: Record<string, unknown>;
|
|
39
|
+
/** Tests: where events go instead of Whistle. */
|
|
40
|
+
sink?: Sink;
|
|
41
|
+
}
|
|
42
|
+
export declare function whistle<C extends HonoContextLike = HonoContextLike>(options?: WhistleOptions<C>): (c: C, next: () => Promise<void>) => Promise<void>;
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `app.use(whistle())`: Hono middleware for what a tailed Worker's runs don't
|
|
3
|
+
* say by themselves. Typed by shape, so the SDK doesn't depend on Hono.
|
|
4
|
+
*
|
|
5
|
+
* 403s, and 401s to a request that presented a credential → `access.denied`
|
|
6
|
+
* writes under /admin that succeed → `admin.action`
|
|
7
|
+
* an error a custom onError turned into a 5xx → console.error, for the tail
|
|
8
|
+
*
|
|
9
|
+
* The user comes from `c.var.user.id` unless `user` says otherwise.
|
|
10
|
+
*/
|
|
11
|
+
import { defined, requestFields, telemetryFromEnv } from './index.js';
|
|
12
|
+
const WRITES = new Set(['POST', 'PUT', 'PATCH', 'DELETE']);
|
|
13
|
+
const NAMES = [
|
|
14
|
+
'KF_WHISTLE_URL',
|
|
15
|
+
'KF_WHISTLE_PROJECT',
|
|
16
|
+
'KF_WHISTLE_ENVIRONMENT',
|
|
17
|
+
'KF_DEPLOYMENT_ID',
|
|
18
|
+
'KF_COMMIT',
|
|
19
|
+
];
|
|
20
|
+
const record = (v) => v && typeof v === 'object' ? v : undefined;
|
|
21
|
+
const processEnv = () => record(globalThis.process?.env) ?? {};
|
|
22
|
+
function variables(c, given) {
|
|
23
|
+
const out = {};
|
|
24
|
+
const sources = given ? [given] : [record(c.env) ?? {}, processEnv()];
|
|
25
|
+
for (const name of NAMES) {
|
|
26
|
+
const found = sources.map((s) => s[name]).find((v) => typeof v === 'string' && v);
|
|
27
|
+
if (found !== undefined)
|
|
28
|
+
out[name] = found;
|
|
29
|
+
}
|
|
30
|
+
return out;
|
|
31
|
+
}
|
|
32
|
+
/** `c.executionCtx` throws outside Workers. */
|
|
33
|
+
function waitUntilOf(c) {
|
|
34
|
+
try {
|
|
35
|
+
const ctx = c.executionCtx;
|
|
36
|
+
return ctx ? (p) => ctx.waitUntil(p) : undefined;
|
|
37
|
+
}
|
|
38
|
+
catch {
|
|
39
|
+
return undefined;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
function defaultUser(c) {
|
|
43
|
+
const id = record(record(c.var)?.user)?.id;
|
|
44
|
+
return typeof id === 'string' || typeof id === 'number' ? String(id) : undefined;
|
|
45
|
+
}
|
|
46
|
+
const under = (path, prefixes) => prefixes.some((p) => path === p || path.startsWith(`${p}/`));
|
|
47
|
+
let warned = false;
|
|
48
|
+
export function whistle(options = {}) {
|
|
49
|
+
const admin = options.admin === false ? [] : (options.admin ?? ['/admin', '/api/admin']);
|
|
50
|
+
return async (c, next) => {
|
|
51
|
+
await next();
|
|
52
|
+
try {
|
|
53
|
+
const status = c.res.status;
|
|
54
|
+
const waitUntil = waitUntilOf(c);
|
|
55
|
+
// On a Worker the tail reads this; it drops a second copy of an error in the same run.
|
|
56
|
+
if (c.error && status >= 500 && waitUntil)
|
|
57
|
+
console.error(c.error);
|
|
58
|
+
const method = c.req.method.toUpperCase();
|
|
59
|
+
const denied = status === 403 || (status === 401 && c.req.raw.headers.has('authorization'));
|
|
60
|
+
const adminWrite = status < 400 && WRITES.has(method) && under(c.req.path, admin);
|
|
61
|
+
if (!denied && !adminWrite)
|
|
62
|
+
return;
|
|
63
|
+
const env = variables(c, options.env);
|
|
64
|
+
if (!options.project && !env.KF_WHISTLE_PROJECT) {
|
|
65
|
+
if (!warned) {
|
|
66
|
+
warned = true;
|
|
67
|
+
console.warn('whistle(): no project; pass { project } or redeploy through the console');
|
|
68
|
+
}
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
const t = telemetryFromEnv(env, defined({
|
|
72
|
+
project: options.project,
|
|
73
|
+
environment: options.environment,
|
|
74
|
+
sink: options.sink,
|
|
75
|
+
waitUntil,
|
|
76
|
+
}));
|
|
77
|
+
const id = await (options.user ?? defaultUser)(c);
|
|
78
|
+
t.securityEvent(denied ? 'access.denied' : 'admin.action', {
|
|
79
|
+
...defined({ actor: id ? `user:${id}` : undefined }),
|
|
80
|
+
...requestFields(c.req.raw),
|
|
81
|
+
...(denied ? { outcome: 'denied' } : {}),
|
|
82
|
+
detail: { method, route: c.req.routePath, status },
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
catch {
|
|
86
|
+
// Telemetry never breaks the request it describes.
|
|
87
|
+
}
|
|
88
|
+
};
|
|
89
|
+
}
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@knowledgefutures/sdk/whistle` — what a project that declares
|
|
3
|
+
* `kf/whistle@1` tells Whistle beyond what its Worker's runs already say.
|
|
4
|
+
* On a Worker the console hosts, Whistle tails every run, so uncaught
|
|
5
|
+
* exceptions, `console.error(err)` and cron runs need nothing from here.
|
|
6
|
+
* This is for security events, errors the app handles without logging, and
|
|
7
|
+
* check-ins from jobs that aren't Worker crons:
|
|
8
|
+
*
|
|
9
|
+
* app.use(whistle()) // Hono: denied requests and admin writes (hono.ts)
|
|
10
|
+
* const t = telemetryFromEnv(env, { waitUntil })
|
|
11
|
+
* t.securityEvent('auth.failed', { actor: `user:${id}`, ...requestFields(request) })
|
|
12
|
+
* await checkIn(env.KF_WHISTLE_URL, 'nightly-backup', { every: 1440 })
|
|
13
|
+
*
|
|
14
|
+
* With `KF_WHISTLE_URL` (the console hands it out) events go straight to
|
|
15
|
+
* Whistle; without it, to the console as JSON lines. Moved here from
|
|
16
|
+
* kf-whistle's `@kf/telemetry`, which Whistle keeps for parsing until it
|
|
17
|
+
* reads this package instead: the wire format (wire.ts) must not drift
|
|
18
|
+
* between the two.
|
|
19
|
+
*/
|
|
20
|
+
import { type SecurityEvent, type SecurityKind, type WireEvent } from './wire.js';
|
|
21
|
+
export * from './wire.js';
|
|
22
|
+
export interface Sink {
|
|
23
|
+
emit(event: WireEvent): void;
|
|
24
|
+
}
|
|
25
|
+
/** Workers (and anything whose stdout is collected): one JSON line per event. */
|
|
26
|
+
export declare const consoleSink: Sink;
|
|
27
|
+
/** Every event to each sink. */
|
|
28
|
+
export declare function teeSink(...sinks: Sink[]): Sink;
|
|
29
|
+
export interface WhistleTarget {
|
|
30
|
+
/** Whistle's origin, e.g. https://whistle.kfutures.org. */
|
|
31
|
+
origin: string;
|
|
32
|
+
/** The ingest key, sent as a bearer. */
|
|
33
|
+
key: string;
|
|
34
|
+
}
|
|
35
|
+
/** `KF_WHISTLE_URL` (`https://<instance>:<key>@<host>/`) as an origin and a key, or null. */
|
|
36
|
+
export declare function parseWhistleUrl(url: string | undefined): WhistleTarget | null;
|
|
37
|
+
/**
|
|
38
|
+
* A Sentry DSN that sends to Whistle, from `KF_WHISTLE_URL`: how an app on a
|
|
39
|
+
* Sentry SDK moves without code changes beyond `dsn` (and `release:
|
|
40
|
+
* KF_DEPLOYMENT_ID`, so errors group by deployment). Null without a URL, or
|
|
41
|
+
* when the key isn't one a DSN can carry (Sentry takes only [A-Za-z0-9_]).
|
|
42
|
+
* The project id is a placeholder: Sentry wants a number, Whistle goes by the key.
|
|
43
|
+
*/
|
|
44
|
+
export declare function sentryDsn(url: string | undefined): string | null;
|
|
45
|
+
/**
|
|
46
|
+
* Reports one run of a scheduled job to Whistle: `every` (minutes) says how
|
|
47
|
+
* often it should run, so Whistle can say when it stops. Never throws; false
|
|
48
|
+
* when there's no KF_WHISTLE_URL or the send failed.
|
|
49
|
+
*
|
|
50
|
+
* await checkIn(env.KF_WHISTLE_URL, 'nightly-backup', { every: 1440 })
|
|
51
|
+
* await checkIn(env.KF_WHISTLE_URL, 'nightly-backup', { status: 'error', message: String(e) })
|
|
52
|
+
*/
|
|
53
|
+
export declare function checkIn(url: string | undefined, name: string, run?: {
|
|
54
|
+
status?: 'ok' | 'error';
|
|
55
|
+
every?: number;
|
|
56
|
+
grace?: number;
|
|
57
|
+
message?: string;
|
|
58
|
+
}, doFetch?: typeof fetch): Promise<boolean>;
|
|
59
|
+
export interface WhistleSinkOptions {
|
|
60
|
+
fetch?: typeof fetch;
|
|
61
|
+
/** On Workers, the request's `ctx.waitUntil`, so the send outlives the response. */
|
|
62
|
+
waitUntil?: (promise: Promise<unknown>) => void;
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Posts events to Whistle's `/v1/events` as NDJSON, batched per tick. `flush()`
|
|
66
|
+
* sends what's queued now; failures are dropped, never thrown.
|
|
67
|
+
*/
|
|
68
|
+
export declare function whistleSink(target: WhistleTarget, options?: WhistleSinkOptions): Sink & {
|
|
69
|
+
flush(): Promise<void>;
|
|
70
|
+
};
|
|
71
|
+
export interface TelemetryOptions {
|
|
72
|
+
project: string;
|
|
73
|
+
environment: string;
|
|
74
|
+
sink?: Sink;
|
|
75
|
+
now?: () => Date;
|
|
76
|
+
/** The console's deployment id and commit, stamped on every event. */
|
|
77
|
+
deployment?: string;
|
|
78
|
+
commit?: string;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Telemetry set up from what the console hands a project: `KF_WHISTLE_URL`
|
|
82
|
+
* (sends straight to Whistle), `KF_WHISTLE_PROJECT` and `KF_WHISTLE_ENVIRONMENT`
|
|
83
|
+
* (what Whistle files events under), `KF_DEPLOYMENT_ID` and `KF_COMMIT`.
|
|
84
|
+
* Without `KF_WHISTLE_URL`, console output. Not both: on Workers, Cloudflare's
|
|
85
|
+
* export would deliver the console line as well, and every event would count twice.
|
|
86
|
+
*/
|
|
87
|
+
export declare function telemetryFromEnv(env: Record<string, unknown>, options?: Partial<Omit<TelemetryOptions, 'deployment' | 'commit'>> & WhistleSinkOptions): Telemetry;
|
|
88
|
+
export interface SecurityFields {
|
|
89
|
+
actor?: string;
|
|
90
|
+
ip?: string;
|
|
91
|
+
country?: string;
|
|
92
|
+
role?: string;
|
|
93
|
+
outcome?: SecurityEvent['outcome'];
|
|
94
|
+
rayId?: string;
|
|
95
|
+
traceId?: string;
|
|
96
|
+
detail?: Record<string, unknown>;
|
|
97
|
+
}
|
|
98
|
+
export interface ErrorContext {
|
|
99
|
+
route?: string;
|
|
100
|
+
rayId?: string;
|
|
101
|
+
traceId?: string;
|
|
102
|
+
detail?: Record<string, unknown>;
|
|
103
|
+
}
|
|
104
|
+
export interface Telemetry {
|
|
105
|
+
securityEvent(kind: SecurityKind, fields?: SecurityFields): void;
|
|
106
|
+
captureError(error: unknown, context?: ErrorContext): void;
|
|
107
|
+
/** Runs `fn`; if it throws, records the error with `name` as its route, then rethrows. */
|
|
108
|
+
span<T>(name: string, fn: () => T | Promise<T>, context?: ErrorContext): Promise<T>;
|
|
109
|
+
}
|
|
110
|
+
/** `T` with its possibly-undefined properties made optional and non-undefined, as `defined` returns it. */
|
|
111
|
+
export type Defined<T> = {
|
|
112
|
+
[K in keyof T as undefined extends T[K] ? never : K]: T[K];
|
|
113
|
+
} & {
|
|
114
|
+
[K in keyof T as undefined extends T[K] ? K : never]?: Exclude<T[K], undefined>;
|
|
115
|
+
};
|
|
116
|
+
/** `{ ...o }` without the undefined values, so events stay small and exact. */
|
|
117
|
+
export declare function defined<T extends object>(o: T): Defined<T>;
|
|
118
|
+
/** An Error, or whatever was thrown, as the three exception fields. */
|
|
119
|
+
export declare function describeError(error: unknown): {
|
|
120
|
+
type: string;
|
|
121
|
+
message: string;
|
|
122
|
+
stack?: string;
|
|
123
|
+
};
|
|
124
|
+
export declare function createTelemetry(options: TelemetryOptions): Telemetry;
|
|
125
|
+
/**
|
|
126
|
+
* What a request carries that events want, on Cloudflare: the client IP, its
|
|
127
|
+
* country, and the Ray ID that joins an event to its edge log row. Elsewhere,
|
|
128
|
+
* pass them yourself.
|
|
129
|
+
*/
|
|
130
|
+
export declare function requestFields(request: Request): {
|
|
131
|
+
ip?: string;
|
|
132
|
+
country?: string;
|
|
133
|
+
rayId?: string;
|
|
134
|
+
};
|
|
135
|
+
export { whistle, type HonoContextLike, type WhistleOptions } from './hono.js';
|