@knowledgefutures/sdk 0.1.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -0
- package/dist/whistle/hono.d.ts +42 -0
- package/dist/whistle/hono.js +89 -0
- package/dist/whistle/index.d.ts +135 -0
- package/dist/whistle/index.js +249 -0
- package/dist/whistle/wire.d.ts +64 -0
- package/dist/whistle/wire.js +98 -0
- package/package.json +5 -1
- package/dist/analytics/beacon.d.ts +0 -25
- package/dist/analytics/beacon.js +0 -86
- package/dist/analytics/binding.d.ts +0 -24
- package/dist/analytics/binding.js +0 -46
- package/dist/analytics/collector.d.ts +0 -10
- package/dist/analytics/collector.js +0 -10
- package/dist/analytics/cubes.d.ts +0 -96
- package/dist/analytics/cubes.js +0 -213
- package/dist/analytics/envelope.d.ts +0 -135
- package/dist/analytics/envelope.js +0 -231
- package/dist/analytics/errors.d.ts +0 -24
- package/dist/analytics/errors.js +0 -20
- package/dist/analytics/index.d.ts +0 -38
- package/dist/analytics/index.js +0 -98
- package/dist/analytics/memory.d.ts +0 -35
- package/dist/analytics/memory.js +0 -66
- package/dist/analytics/testing.d.ts +0 -6
- package/dist/analytics/testing.js +0 -6
- package/dist/analytics/types.d.ts +0 -22
- package/dist/analytics/types.js +0 -1
- package/dist/storage/conformance.d.ts +0 -24
- package/dist/storage/conformance.js +0 -311
- package/dist/storage/local.d.ts +0 -26
- package/dist/storage/local.js +0 -57
package/README.md
CHANGED
|
@@ -20,6 +20,53 @@ import { memoryStorage } from '@knowledgefutures/sdk/storage/testing'
|
|
|
20
20
|
`createStorage(env)` builds a client from an explicit environment, such as a
|
|
21
21
|
Worker's `env` when `nodejs_compat` is off.
|
|
22
22
|
|
|
23
|
+
## Whistle (`kf/whistle@1`)
|
|
24
|
+
|
|
25
|
+
On a Worker the console hosts, declaring `kf/whistle@1` is enough for errors:
|
|
26
|
+
Whistle tails every run, so uncaught exceptions, `console.error(err)` and cron
|
|
27
|
+
runs reach it with no code. Log `console.error(err)`, not `err.message`, to keep
|
|
28
|
+
the stack. This entry point is for what a run can't say by itself.
|
|
29
|
+
|
|
30
|
+
On Hono, one line, registered before the routes:
|
|
31
|
+
|
|
32
|
+
```ts
|
|
33
|
+
import { whistle } from '@knowledgefutures/sdk/whistle'
|
|
34
|
+
|
|
35
|
+
app.use(whistle())
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
It records a 403, or a 401 to a request that sent an `Authorization` header,
|
|
39
|
+
as `access.denied` (Whistle alerts on bursts per client and per user), and a
|
|
40
|
+
successful write under `/admin` or `/api/admin` as `admin.action`, with the
|
|
41
|
+
route pattern and the user. The user is `c.var.user.id` unless
|
|
42
|
+
`user: (c) => …` says otherwise; `admin` sets the prefixes. On a Worker it
|
|
43
|
+
also logs an error a custom `onError` turned into a 5xx, which the tail would
|
|
44
|
+
otherwise never see.
|
|
45
|
+
|
|
46
|
+
The rest by hand:
|
|
47
|
+
|
|
48
|
+
```ts
|
|
49
|
+
import { checkIn, requestFields, telemetryFromEnv } from '@knowledgefutures/sdk/whistle'
|
|
50
|
+
|
|
51
|
+
const t = telemetryFromEnv(env, { waitUntil })
|
|
52
|
+
t.securityEvent('auth.failed', {
|
|
53
|
+
actor: `user:${id}`,
|
|
54
|
+
outcome: 'failure',
|
|
55
|
+
...requestFields(request),
|
|
56
|
+
})
|
|
57
|
+
|
|
58
|
+
// A job that isn't a Worker cron: Whistle says when it fails or stops running.
|
|
59
|
+
await checkIn(env.KF_WHISTLE_URL, 'nightly-backup', { every: 1440 })
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
`captureError(err)` is for an error the app handles and doesn't log; on a
|
|
63
|
+
tailed Worker, one it also logs or throws would count twice. `sentryDsn(url)`
|
|
64
|
+
turns `KF_WHISTLE_URL` into a Sentry DSN, for apps that keep a Sentry SDK.
|
|
65
|
+
`actor` is an id (`user:<id>`), never an email or a name, and `detail` is
|
|
66
|
+
scrubbed of anything named like a credential. Events go under
|
|
67
|
+
`KF_WHISTLE_PROJECT` and `KF_WHISTLE_ENVIRONMENT`, which the console and
|
|
68
|
+
`kf dev` set; pass `project` and `environment` elsewhere.
|
|
69
|
+
|
|
23
70
|
What a project declares and what the services give it:
|
|
24
71
|
[console.knowledgefutures.org/llms.txt](https://console.knowledgefutures.org/llms.txt).
|
|
25
72
|
Source: `packages/sdk/` in [knowledgefutures/kf-console](https://github.com/knowledgefutures/kf-console).
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `app.use(whistle())`: Hono middleware for what a tailed Worker's runs don't
|
|
3
|
+
* say by themselves. Typed by shape, so the SDK doesn't depend on Hono.
|
|
4
|
+
*
|
|
5
|
+
* 403s, and 401s to a request that presented a credential → `access.denied`
|
|
6
|
+
* writes under /admin that succeed → `admin.action`
|
|
7
|
+
* an error a custom onError turned into a 5xx → console.error, for the tail
|
|
8
|
+
*
|
|
9
|
+
* The user comes from `c.var.user.id` unless `user` says otherwise.
|
|
10
|
+
*/
|
|
11
|
+
import { type Sink } from './index.js';
|
|
12
|
+
/**
|
|
13
|
+
* The parts of Hono's Context this reads. `var` is loose so a `user` option
|
|
14
|
+
* reads the app's variables untyped; `whistle<Context<AppEnv>>()` types them.
|
|
15
|
+
*/
|
|
16
|
+
export interface HonoContextLike {
|
|
17
|
+
req: {
|
|
18
|
+
raw: Request;
|
|
19
|
+
method: string;
|
|
20
|
+
path: string;
|
|
21
|
+
routePath: string;
|
|
22
|
+
};
|
|
23
|
+
res: Response;
|
|
24
|
+
env: unknown;
|
|
25
|
+
error: Error | undefined;
|
|
26
|
+
var: any;
|
|
27
|
+
}
|
|
28
|
+
export interface WhistleOptions<C extends HonoContextLike = HonoContextLike> {
|
|
29
|
+
/** The console project key, e.g. `kf/pilot`. Default: `KF_WHISTLE_PROJECT`, which the console sets. */
|
|
30
|
+
project?: string;
|
|
31
|
+
/** Default: `KF_WHISTLE_ENVIRONMENT`, which the console sets, else `production`. */
|
|
32
|
+
environment?: string;
|
|
33
|
+
/** The signed-in user's id, if any; asked only when there's an event. Default: `c.var.user?.id`. */
|
|
34
|
+
user?: (c: C) => string | null | undefined | Promise<string | null | undefined>;
|
|
35
|
+
/** Path prefixes whose successful writes are recorded. Default `/admin`, `/api/admin`; false for none. */
|
|
36
|
+
admin?: string[] | false;
|
|
37
|
+
/** Where `KF_*` variables come from. Default: `c.env`, then `process.env`. */
|
|
38
|
+
env?: Record<string, unknown>;
|
|
39
|
+
/** Tests: where events go instead of Whistle. */
|
|
40
|
+
sink?: Sink;
|
|
41
|
+
}
|
|
42
|
+
export declare function whistle<C extends HonoContextLike = HonoContextLike>(options?: WhistleOptions<C>): (c: C, next: () => Promise<void>) => Promise<void>;
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `app.use(whistle())`: Hono middleware for what a tailed Worker's runs don't
|
|
3
|
+
* say by themselves. Typed by shape, so the SDK doesn't depend on Hono.
|
|
4
|
+
*
|
|
5
|
+
* 403s, and 401s to a request that presented a credential → `access.denied`
|
|
6
|
+
* writes under /admin that succeed → `admin.action`
|
|
7
|
+
* an error a custom onError turned into a 5xx → console.error, for the tail
|
|
8
|
+
*
|
|
9
|
+
* The user comes from `c.var.user.id` unless `user` says otherwise.
|
|
10
|
+
*/
|
|
11
|
+
import { defined, requestFields, telemetryFromEnv } from './index.js';
|
|
12
|
+
const WRITES = new Set(['POST', 'PUT', 'PATCH', 'DELETE']);
|
|
13
|
+
const NAMES = [
|
|
14
|
+
'KF_WHISTLE_URL',
|
|
15
|
+
'KF_WHISTLE_PROJECT',
|
|
16
|
+
'KF_WHISTLE_ENVIRONMENT',
|
|
17
|
+
'KF_DEPLOYMENT_ID',
|
|
18
|
+
'KF_COMMIT',
|
|
19
|
+
];
|
|
20
|
+
const record = (v) => v && typeof v === 'object' ? v : undefined;
|
|
21
|
+
const processEnv = () => record(globalThis.process?.env) ?? {};
|
|
22
|
+
function variables(c, given) {
|
|
23
|
+
const out = {};
|
|
24
|
+
const sources = given ? [given] : [record(c.env) ?? {}, processEnv()];
|
|
25
|
+
for (const name of NAMES) {
|
|
26
|
+
const found = sources.map((s) => s[name]).find((v) => typeof v === 'string' && v);
|
|
27
|
+
if (found !== undefined)
|
|
28
|
+
out[name] = found;
|
|
29
|
+
}
|
|
30
|
+
return out;
|
|
31
|
+
}
|
|
32
|
+
/** `c.executionCtx` throws outside Workers. */
|
|
33
|
+
function waitUntilOf(c) {
|
|
34
|
+
try {
|
|
35
|
+
const ctx = c.executionCtx;
|
|
36
|
+
return ctx ? (p) => ctx.waitUntil(p) : undefined;
|
|
37
|
+
}
|
|
38
|
+
catch {
|
|
39
|
+
return undefined;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
function defaultUser(c) {
|
|
43
|
+
const id = record(record(c.var)?.user)?.id;
|
|
44
|
+
return typeof id === 'string' || typeof id === 'number' ? String(id) : undefined;
|
|
45
|
+
}
|
|
46
|
+
const under = (path, prefixes) => prefixes.some((p) => path === p || path.startsWith(`${p}/`));
|
|
47
|
+
let warned = false;
|
|
48
|
+
export function whistle(options = {}) {
|
|
49
|
+
const admin = options.admin === false ? [] : (options.admin ?? ['/admin', '/api/admin']);
|
|
50
|
+
return async (c, next) => {
|
|
51
|
+
await next();
|
|
52
|
+
try {
|
|
53
|
+
const status = c.res.status;
|
|
54
|
+
const waitUntil = waitUntilOf(c);
|
|
55
|
+
// On a Worker the tail reads this; it drops a second copy of an error in the same run.
|
|
56
|
+
if (c.error && status >= 500 && waitUntil)
|
|
57
|
+
console.error(c.error);
|
|
58
|
+
const method = c.req.method.toUpperCase();
|
|
59
|
+
const denied = status === 403 || (status === 401 && c.req.raw.headers.has('authorization'));
|
|
60
|
+
const adminWrite = status < 400 && WRITES.has(method) && under(c.req.path, admin);
|
|
61
|
+
if (!denied && !adminWrite)
|
|
62
|
+
return;
|
|
63
|
+
const env = variables(c, options.env);
|
|
64
|
+
if (!options.project && !env.KF_WHISTLE_PROJECT) {
|
|
65
|
+
if (!warned) {
|
|
66
|
+
warned = true;
|
|
67
|
+
console.warn('whistle(): no project; pass { project } or redeploy through the console');
|
|
68
|
+
}
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
const t = telemetryFromEnv(env, defined({
|
|
72
|
+
project: options.project,
|
|
73
|
+
environment: options.environment,
|
|
74
|
+
sink: options.sink,
|
|
75
|
+
waitUntil,
|
|
76
|
+
}));
|
|
77
|
+
const id = await (options.user ?? defaultUser)(c);
|
|
78
|
+
t.securityEvent(denied ? 'access.denied' : 'admin.action', {
|
|
79
|
+
...defined({ actor: id ? `user:${id}` : undefined }),
|
|
80
|
+
...requestFields(c.req.raw),
|
|
81
|
+
...(denied ? { outcome: 'denied' } : {}),
|
|
82
|
+
detail: { method, route: c.req.routePath, status },
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
catch {
|
|
86
|
+
// Telemetry never breaks the request it describes.
|
|
87
|
+
}
|
|
88
|
+
};
|
|
89
|
+
}
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@knowledgefutures/sdk/whistle` — what a project that declares
|
|
3
|
+
* `kf/whistle@1` tells Whistle beyond what its Worker's runs already say.
|
|
4
|
+
* On a Worker the console hosts, Whistle tails every run, so uncaught
|
|
5
|
+
* exceptions, `console.error(err)` and cron runs need nothing from here.
|
|
6
|
+
* This is for security events, errors the app handles without logging, and
|
|
7
|
+
* check-ins from jobs that aren't Worker crons:
|
|
8
|
+
*
|
|
9
|
+
* app.use(whistle()) // Hono: denied requests and admin writes (hono.ts)
|
|
10
|
+
* const t = telemetryFromEnv(env, { waitUntil })
|
|
11
|
+
* t.securityEvent('auth.failed', { actor: `user:${id}`, ...requestFields(request) })
|
|
12
|
+
* await checkIn(env.KF_WHISTLE_URL, 'nightly-backup', { every: 1440 })
|
|
13
|
+
*
|
|
14
|
+
* With `KF_WHISTLE_URL` (the console hands it out) events go straight to
|
|
15
|
+
* Whistle; without it, to the console as JSON lines. Moved here from
|
|
16
|
+
* kf-whistle's `@kf/telemetry`, which Whistle keeps for parsing until it
|
|
17
|
+
* reads this package instead: the wire format (wire.ts) must not drift
|
|
18
|
+
* between the two.
|
|
19
|
+
*/
|
|
20
|
+
import { type SecurityEvent, type SecurityKind, type WireEvent } from './wire.js';
|
|
21
|
+
export * from './wire.js';
|
|
22
|
+
export interface Sink {
|
|
23
|
+
emit(event: WireEvent): void;
|
|
24
|
+
}
|
|
25
|
+
/** Workers (and anything whose stdout is collected): one JSON line per event. */
|
|
26
|
+
export declare const consoleSink: Sink;
|
|
27
|
+
/** Every event to each sink. */
|
|
28
|
+
export declare function teeSink(...sinks: Sink[]): Sink;
|
|
29
|
+
export interface WhistleTarget {
|
|
30
|
+
/** Whistle's origin, e.g. https://whistle.kfutures.org. */
|
|
31
|
+
origin: string;
|
|
32
|
+
/** The ingest key, sent as a bearer. */
|
|
33
|
+
key: string;
|
|
34
|
+
}
|
|
35
|
+
/** `KF_WHISTLE_URL` (`https://<instance>:<key>@<host>/`) as an origin and a key, or null. */
|
|
36
|
+
export declare function parseWhistleUrl(url: string | undefined): WhistleTarget | null;
|
|
37
|
+
/**
|
|
38
|
+
* A Sentry DSN that sends to Whistle, from `KF_WHISTLE_URL`: how an app on a
|
|
39
|
+
* Sentry SDK moves without code changes beyond `dsn` (and `release:
|
|
40
|
+
* KF_DEPLOYMENT_ID`, so errors group by deployment). Null without a URL, or
|
|
41
|
+
* when the key isn't one a DSN can carry (Sentry takes only [A-Za-z0-9_]).
|
|
42
|
+
* The project id is a placeholder: Sentry wants a number, Whistle goes by the key.
|
|
43
|
+
*/
|
|
44
|
+
export declare function sentryDsn(url: string | undefined): string | null;
|
|
45
|
+
/**
|
|
46
|
+
* Reports one run of a scheduled job to Whistle: `every` (minutes) says how
|
|
47
|
+
* often it should run, so Whistle can say when it stops. Never throws; false
|
|
48
|
+
* when there's no KF_WHISTLE_URL or the send failed.
|
|
49
|
+
*
|
|
50
|
+
* await checkIn(env.KF_WHISTLE_URL, 'nightly-backup', { every: 1440 })
|
|
51
|
+
* await checkIn(env.KF_WHISTLE_URL, 'nightly-backup', { status: 'error', message: String(e) })
|
|
52
|
+
*/
|
|
53
|
+
export declare function checkIn(url: string | undefined, name: string, run?: {
|
|
54
|
+
status?: 'ok' | 'error';
|
|
55
|
+
every?: number;
|
|
56
|
+
grace?: number;
|
|
57
|
+
message?: string;
|
|
58
|
+
}, doFetch?: typeof fetch): Promise<boolean>;
|
|
59
|
+
export interface WhistleSinkOptions {
|
|
60
|
+
fetch?: typeof fetch;
|
|
61
|
+
/** On Workers, the request's `ctx.waitUntil`, so the send outlives the response. */
|
|
62
|
+
waitUntil?: (promise: Promise<unknown>) => void;
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Posts events to Whistle's `/v1/events` as NDJSON, batched per tick. `flush()`
|
|
66
|
+
* sends what's queued now; failures are dropped, never thrown.
|
|
67
|
+
*/
|
|
68
|
+
export declare function whistleSink(target: WhistleTarget, options?: WhistleSinkOptions): Sink & {
|
|
69
|
+
flush(): Promise<void>;
|
|
70
|
+
};
|
|
71
|
+
export interface TelemetryOptions {
|
|
72
|
+
project: string;
|
|
73
|
+
environment: string;
|
|
74
|
+
sink?: Sink;
|
|
75
|
+
now?: () => Date;
|
|
76
|
+
/** The console's deployment id and commit, stamped on every event. */
|
|
77
|
+
deployment?: string;
|
|
78
|
+
commit?: string;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Telemetry set up from what the console hands a project: `KF_WHISTLE_URL`
|
|
82
|
+
* (sends straight to Whistle), `KF_WHISTLE_PROJECT` and `KF_WHISTLE_ENVIRONMENT`
|
|
83
|
+
* (what Whistle files events under), `KF_DEPLOYMENT_ID` and `KF_COMMIT`.
|
|
84
|
+
* Without `KF_WHISTLE_URL`, console output. Not both: on Workers, Cloudflare's
|
|
85
|
+
* export would deliver the console line as well, and every event would count twice.
|
|
86
|
+
*/
|
|
87
|
+
export declare function telemetryFromEnv(env: Record<string, unknown>, options?: Partial<Omit<TelemetryOptions, 'deployment' | 'commit'>> & WhistleSinkOptions): Telemetry;
|
|
88
|
+
export interface SecurityFields {
|
|
89
|
+
actor?: string;
|
|
90
|
+
ip?: string;
|
|
91
|
+
country?: string;
|
|
92
|
+
role?: string;
|
|
93
|
+
outcome?: SecurityEvent['outcome'];
|
|
94
|
+
rayId?: string;
|
|
95
|
+
traceId?: string;
|
|
96
|
+
detail?: Record<string, unknown>;
|
|
97
|
+
}
|
|
98
|
+
export interface ErrorContext {
|
|
99
|
+
route?: string;
|
|
100
|
+
rayId?: string;
|
|
101
|
+
traceId?: string;
|
|
102
|
+
detail?: Record<string, unknown>;
|
|
103
|
+
}
|
|
104
|
+
export interface Telemetry {
|
|
105
|
+
securityEvent(kind: SecurityKind, fields?: SecurityFields): void;
|
|
106
|
+
captureError(error: unknown, context?: ErrorContext): void;
|
|
107
|
+
/** Runs `fn`; if it throws, records the error with `name` as its route, then rethrows. */
|
|
108
|
+
span<T>(name: string, fn: () => T | Promise<T>, context?: ErrorContext): Promise<T>;
|
|
109
|
+
}
|
|
110
|
+
/** `T` with its possibly-undefined properties made optional and non-undefined, as `defined` returns it. */
|
|
111
|
+
export type Defined<T> = {
|
|
112
|
+
[K in keyof T as undefined extends T[K] ? never : K]: T[K];
|
|
113
|
+
} & {
|
|
114
|
+
[K in keyof T as undefined extends T[K] ? K : never]?: Exclude<T[K], undefined>;
|
|
115
|
+
};
|
|
116
|
+
/** `{ ...o }` without the undefined values, so events stay small and exact. */
|
|
117
|
+
export declare function defined<T extends object>(o: T): Defined<T>;
|
|
118
|
+
/** An Error, or whatever was thrown, as the three exception fields. */
|
|
119
|
+
export declare function describeError(error: unknown): {
|
|
120
|
+
type: string;
|
|
121
|
+
message: string;
|
|
122
|
+
stack?: string;
|
|
123
|
+
};
|
|
124
|
+
export declare function createTelemetry(options: TelemetryOptions): Telemetry;
|
|
125
|
+
/**
|
|
126
|
+
* What a request carries that events want, on Cloudflare: the client IP, its
|
|
127
|
+
* country, and the Ray ID that joins an event to its edge log row. Elsewhere,
|
|
128
|
+
* pass them yourself.
|
|
129
|
+
*/
|
|
130
|
+
export declare function requestFields(request: Request): {
|
|
131
|
+
ip?: string;
|
|
132
|
+
country?: string;
|
|
133
|
+
rayId?: string;
|
|
134
|
+
};
|
|
135
|
+
export { whistle, type HonoContextLike, type WhistleOptions } from './hono.js';
|
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@knowledgefutures/sdk/whistle` — what a project that declares
|
|
3
|
+
* `kf/whistle@1` tells Whistle beyond what its Worker's runs already say.
|
|
4
|
+
* On a Worker the console hosts, Whistle tails every run, so uncaught
|
|
5
|
+
* exceptions, `console.error(err)` and cron runs need nothing from here.
|
|
6
|
+
* This is for security events, errors the app handles without logging, and
|
|
7
|
+
* check-ins from jobs that aren't Worker crons:
|
|
8
|
+
*
|
|
9
|
+
* app.use(whistle()) // Hono: denied requests and admin writes (hono.ts)
|
|
10
|
+
* const t = telemetryFromEnv(env, { waitUntil })
|
|
11
|
+
* t.securityEvent('auth.failed', { actor: `user:${id}`, ...requestFields(request) })
|
|
12
|
+
* await checkIn(env.KF_WHISTLE_URL, 'nightly-backup', { every: 1440 })
|
|
13
|
+
*
|
|
14
|
+
* With `KF_WHISTLE_URL` (the console hands it out) events go straight to
|
|
15
|
+
* Whistle; without it, to the console as JSON lines. Moved here from
|
|
16
|
+
* kf-whistle's `@kf/telemetry`, which Whistle keeps for parsing until it
|
|
17
|
+
* reads this package instead: the wire format (wire.ts) must not drift
|
|
18
|
+
* between the two.
|
|
19
|
+
*/
|
|
20
|
+
import { scrub, } from './wire.js';
|
|
21
|
+
export * from './wire.js';
|
|
22
|
+
/** Workers (and anything whose stdout is collected): one JSON line per event. */
|
|
23
|
+
export const consoleSink = {
|
|
24
|
+
emit(event) {
|
|
25
|
+
const line = JSON.stringify(event);
|
|
26
|
+
// console.error so Workers Logs, and the OTLP export, mark it as an error.
|
|
27
|
+
if (event.kf === 'error')
|
|
28
|
+
console.error(line);
|
|
29
|
+
else
|
|
30
|
+
console.log(line);
|
|
31
|
+
},
|
|
32
|
+
};
|
|
33
|
+
/** Every event to each sink. */
|
|
34
|
+
export function teeSink(...sinks) {
|
|
35
|
+
return {
|
|
36
|
+
emit(event) {
|
|
37
|
+
for (const s of sinks) {
|
|
38
|
+
try {
|
|
39
|
+
s.emit(event);
|
|
40
|
+
}
|
|
41
|
+
catch {
|
|
42
|
+
// One sink failing never stops the others.
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
},
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
/** `KF_WHISTLE_URL` (`https://<instance>:<key>@<host>/`) as an origin and a key, or null. */
|
|
49
|
+
export function parseWhistleUrl(url) {
|
|
50
|
+
if (!url)
|
|
51
|
+
return null;
|
|
52
|
+
try {
|
|
53
|
+
const u = new URL(url);
|
|
54
|
+
const key = decodeURIComponent(u.password);
|
|
55
|
+
if (!key || !/^https?:$/.test(u.protocol))
|
|
56
|
+
return null;
|
|
57
|
+
return { origin: `${u.protocol}//${u.host}`, key };
|
|
58
|
+
}
|
|
59
|
+
catch {
|
|
60
|
+
return null;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* A Sentry DSN that sends to Whistle, from `KF_WHISTLE_URL`: how an app on a
|
|
65
|
+
* Sentry SDK moves without code changes beyond `dsn` (and `release:
|
|
66
|
+
* KF_DEPLOYMENT_ID`, so errors group by deployment). Null without a URL, or
|
|
67
|
+
* when the key isn't one a DSN can carry (Sentry takes only [A-Za-z0-9_]).
|
|
68
|
+
* The project id is a placeholder: Sentry wants a number, Whistle goes by the key.
|
|
69
|
+
*/
|
|
70
|
+
export function sentryDsn(url) {
|
|
71
|
+
const target = parseWhistleUrl(url);
|
|
72
|
+
if (!target || !/^\w+$/.test(target.key))
|
|
73
|
+
return null;
|
|
74
|
+
const [scheme, host] = target.origin.split('//');
|
|
75
|
+
return `${scheme}//${target.key}@${host}/1`;
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Reports one run of a scheduled job to Whistle: `every` (minutes) says how
|
|
79
|
+
* often it should run, so Whistle can say when it stops. Never throws; false
|
|
80
|
+
* when there's no KF_WHISTLE_URL or the send failed.
|
|
81
|
+
*
|
|
82
|
+
* await checkIn(env.KF_WHISTLE_URL, 'nightly-backup', { every: 1440 })
|
|
83
|
+
* await checkIn(env.KF_WHISTLE_URL, 'nightly-backup', { status: 'error', message: String(e) })
|
|
84
|
+
*/
|
|
85
|
+
export async function checkIn(url, name, run = {}, doFetch = fetch) {
|
|
86
|
+
const target = parseWhistleUrl(url);
|
|
87
|
+
if (!target)
|
|
88
|
+
return false;
|
|
89
|
+
try {
|
|
90
|
+
const res = await doFetch(`${target.origin}/v1/checkins/${encodeURIComponent(name)}`, {
|
|
91
|
+
method: 'POST',
|
|
92
|
+
headers: { authorization: `Bearer ${target.key}`, 'content-type': 'application/json' },
|
|
93
|
+
body: JSON.stringify(run),
|
|
94
|
+
});
|
|
95
|
+
return res.ok;
|
|
96
|
+
}
|
|
97
|
+
catch {
|
|
98
|
+
return false;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Posts events to Whistle's `/v1/events` as NDJSON, batched per tick. `flush()`
|
|
103
|
+
* sends what's queued now; failures are dropped, never thrown.
|
|
104
|
+
*/
|
|
105
|
+
export function whistleSink(target, options = {}) {
|
|
106
|
+
const doFetch = options.fetch ?? fetch;
|
|
107
|
+
let queue = [];
|
|
108
|
+
let scheduled = false;
|
|
109
|
+
const flush = async () => {
|
|
110
|
+
scheduled = false;
|
|
111
|
+
if (queue.length === 0)
|
|
112
|
+
return;
|
|
113
|
+
const body = queue.map((e) => JSON.stringify(e)).join('\n');
|
|
114
|
+
queue = [];
|
|
115
|
+
try {
|
|
116
|
+
await doFetch(`${target.origin}/v1/events`, {
|
|
117
|
+
method: 'POST',
|
|
118
|
+
headers: { authorization: `Bearer ${target.key}`, 'content-type': 'application/x-ndjson' },
|
|
119
|
+
body,
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
catch {
|
|
123
|
+
// Telemetry never breaks the request it describes.
|
|
124
|
+
}
|
|
125
|
+
};
|
|
126
|
+
return {
|
|
127
|
+
emit(event) {
|
|
128
|
+
queue.push(event);
|
|
129
|
+
if (scheduled)
|
|
130
|
+
return;
|
|
131
|
+
scheduled = true;
|
|
132
|
+
const sent = new Promise((resolve) => queueMicrotask(() => void flush().then(resolve)));
|
|
133
|
+
options.waitUntil?.(sent);
|
|
134
|
+
},
|
|
135
|
+
flush,
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* Telemetry set up from what the console hands a project: `KF_WHISTLE_URL`
|
|
140
|
+
* (sends straight to Whistle), `KF_WHISTLE_PROJECT` and `KF_WHISTLE_ENVIRONMENT`
|
|
141
|
+
* (what Whistle files events under), `KF_DEPLOYMENT_ID` and `KF_COMMIT`.
|
|
142
|
+
* Without `KF_WHISTLE_URL`, console output. Not both: on Workers, Cloudflare's
|
|
143
|
+
* export would deliver the console line as well, and every event would count twice.
|
|
144
|
+
*/
|
|
145
|
+
export function telemetryFromEnv(env, options = {}) {
|
|
146
|
+
const read = (k) => typeof env[k] === 'string' && env[k] ? env[k] : undefined;
|
|
147
|
+
const target = parseWhistleUrl(read('KF_WHISTLE_URL'));
|
|
148
|
+
const sink = options.sink ??
|
|
149
|
+
(target
|
|
150
|
+
? whistleSink(target, defined({ fetch: options.fetch, waitUntil: options.waitUntil }))
|
|
151
|
+
: consoleSink);
|
|
152
|
+
return createTelemetry(defined({
|
|
153
|
+
// Whistle refuses events for a project their key isn't for.
|
|
154
|
+
project: options.project ?? read('KF_WHISTLE_PROJECT') ?? 'unknown',
|
|
155
|
+
environment: options.environment ?? read('KF_WHISTLE_ENVIRONMENT') ?? 'production',
|
|
156
|
+
sink,
|
|
157
|
+
now: options.now,
|
|
158
|
+
deployment: read('KF_DEPLOYMENT_ID'),
|
|
159
|
+
commit: read('KF_COMMIT'),
|
|
160
|
+
}));
|
|
161
|
+
}
|
|
162
|
+
/** `{ ...o }` without the undefined values, so events stay small and exact. */
|
|
163
|
+
export function defined(o) {
|
|
164
|
+
return Object.fromEntries(Object.entries(o).filter(([, v]) => v !== undefined));
|
|
165
|
+
}
|
|
166
|
+
/** An Error, or whatever was thrown, as the three exception fields. */
|
|
167
|
+
export function describeError(error) {
|
|
168
|
+
if (error instanceof Error) {
|
|
169
|
+
return defined({ type: error.name || 'Error', message: error.message, stack: error.stack });
|
|
170
|
+
}
|
|
171
|
+
if (typeof error === 'string')
|
|
172
|
+
return { type: 'string', message: error };
|
|
173
|
+
try {
|
|
174
|
+
return { type: typeof error, message: JSON.stringify(error) ?? String(error) };
|
|
175
|
+
}
|
|
176
|
+
catch {
|
|
177
|
+
return { type: typeof error, message: String(error) };
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
export function createTelemetry(options) {
|
|
181
|
+
const sink = options.sink ?? consoleSink;
|
|
182
|
+
const now = options.now ?? (() => new Date());
|
|
183
|
+
const base = () => defined({
|
|
184
|
+
v: 1,
|
|
185
|
+
project: options.project,
|
|
186
|
+
environment: options.environment,
|
|
187
|
+
at: now().toISOString(),
|
|
188
|
+
deployment: options.deployment,
|
|
189
|
+
commit: options.commit,
|
|
190
|
+
});
|
|
191
|
+
// Building the event is guarded too: `detail` is whatever the app passed.
|
|
192
|
+
const emit = (build) => {
|
|
193
|
+
try {
|
|
194
|
+
sink.emit(build());
|
|
195
|
+
}
|
|
196
|
+
catch {
|
|
197
|
+
// Telemetry never breaks the request it describes.
|
|
198
|
+
}
|
|
199
|
+
};
|
|
200
|
+
const captureError = (error, context = {}) => emit(() => defined({
|
|
201
|
+
kf: 'error',
|
|
202
|
+
...base(),
|
|
203
|
+
...describeError(error),
|
|
204
|
+
route: context.route,
|
|
205
|
+
rayId: context.rayId,
|
|
206
|
+
traceId: context.traceId,
|
|
207
|
+
detail: scrub(context.detail),
|
|
208
|
+
}));
|
|
209
|
+
return {
|
|
210
|
+
securityEvent(kind, fields = {}) {
|
|
211
|
+
emit(() => defined({
|
|
212
|
+
kf: 'security',
|
|
213
|
+
...base(),
|
|
214
|
+
kind,
|
|
215
|
+
actor: fields.actor,
|
|
216
|
+
ip: fields.ip,
|
|
217
|
+
country: fields.country,
|
|
218
|
+
role: fields.role,
|
|
219
|
+
outcome: fields.outcome,
|
|
220
|
+
rayId: fields.rayId,
|
|
221
|
+
traceId: fields.traceId,
|
|
222
|
+
detail: scrub(fields.detail),
|
|
223
|
+
}));
|
|
224
|
+
},
|
|
225
|
+
captureError,
|
|
226
|
+
async span(name, fn, context = {}) {
|
|
227
|
+
try {
|
|
228
|
+
return await fn();
|
|
229
|
+
}
|
|
230
|
+
catch (error) {
|
|
231
|
+
captureError(error, { route: name, ...context });
|
|
232
|
+
throw error;
|
|
233
|
+
}
|
|
234
|
+
},
|
|
235
|
+
};
|
|
236
|
+
}
|
|
237
|
+
/**
|
|
238
|
+
* What a request carries that events want, on Cloudflare: the client IP, its
|
|
239
|
+
* country, and the Ray ID that joins an event to its edge log row. Elsewhere,
|
|
240
|
+
* pass them yourself.
|
|
241
|
+
*/
|
|
242
|
+
export function requestFields(request) {
|
|
243
|
+
return defined({
|
|
244
|
+
ip: request.headers.get('cf-connecting-ip') ?? undefined,
|
|
245
|
+
country: request.headers.get('cf-ipcountry') ?? undefined,
|
|
246
|
+
rayId: request.headers.get('cf-ray')?.split('-')[0],
|
|
247
|
+
});
|
|
248
|
+
}
|
|
249
|
+
export { whistle } from './hono.js';
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The wire format: one JSON object per event, the same on every runtime.
|
|
3
|
+
*
|
|
4
|
+
* Sent to Whistle's `/v1/events` as NDJSON, or written as the body of a
|
|
5
|
+
* `console.log`/`console.error` line. Box agents send the same objects.
|
|
6
|
+
* Whistle parses it with `parseWire`, so this file is the contract between
|
|
7
|
+
* apps and Whistle; kf-whistle's `packages/telemetry/src/wire.ts` must say
|
|
8
|
+
* the same until Whistle imports it from here.
|
|
9
|
+
*/
|
|
10
|
+
/** Security events: what a careful attacker can't avoid causing. Add kinds here, nowhere else. */
|
|
11
|
+
export declare const SECURITY_KINDS: readonly ["auth.failed", "auth.success", "auth.mfa_failed", "auth.reset_requested", "token.created", "token.used_new_ip", "access.denied", "admin.action", "secret.read", "member.added", "export.large", "ssh.failed", "ssh.success", "sudo"];
|
|
12
|
+
export type SecurityKind = (typeof SECURITY_KINDS)[number];
|
|
13
|
+
interface Common {
|
|
14
|
+
v: 1;
|
|
15
|
+
/** `<account>/<project>`, the console key. */
|
|
16
|
+
project: string;
|
|
17
|
+
environment: string;
|
|
18
|
+
/** ISO 8601. */
|
|
19
|
+
at: string;
|
|
20
|
+
/** The edge request this happened in, when there is one: joins to the edge log lake. */
|
|
21
|
+
rayId?: string;
|
|
22
|
+
traceId?: string;
|
|
23
|
+
/** The console's deployment id (KF_DEPLOYMENT_ID) and its commit (KF_COMMIT). */
|
|
24
|
+
deployment?: string;
|
|
25
|
+
commit?: string;
|
|
26
|
+
detail?: Record<string, unknown>;
|
|
27
|
+
}
|
|
28
|
+
export interface SecurityEvent extends Common {
|
|
29
|
+
kf: 'security';
|
|
30
|
+
kind: SecurityKind;
|
|
31
|
+
/** Who: `user:<id>`, `token:<id>`, `ip`, or a box user. Never an email or a name. */
|
|
32
|
+
actor?: string;
|
|
33
|
+
ip?: string;
|
|
34
|
+
/** ISO 3166 alpha-2, from `cf-ipcountry` on Cloudflare. */
|
|
35
|
+
country?: string;
|
|
36
|
+
/** Staff, admin, member: lets rules treat privileged sign-ins differently. */
|
|
37
|
+
role?: string;
|
|
38
|
+
outcome?: 'success' | 'failure' | 'denied';
|
|
39
|
+
}
|
|
40
|
+
export interface ErrorEvent extends Common {
|
|
41
|
+
kf: 'error';
|
|
42
|
+
/** `exception.type`: TypeError, D1_ERROR… */
|
|
43
|
+
type: string;
|
|
44
|
+
/** `exception.message`. */
|
|
45
|
+
message: string;
|
|
46
|
+
/** `exception.stacktrace`. */
|
|
47
|
+
stack?: string;
|
|
48
|
+
/** The route pattern (`/api/projects/:id`), not the URL: keeps ids out of grouping. */
|
|
49
|
+
route?: string;
|
|
50
|
+
}
|
|
51
|
+
export type WireEvent = SecurityEvent | ErrorEvent;
|
|
52
|
+
/**
|
|
53
|
+
* `detail` without anything that looks like a credential, and bounded in size.
|
|
54
|
+
* Apps pass objects they didn't fully build (a request's params, an error's
|
|
55
|
+
* cause), so this is the last line before a secret ends up in the lake.
|
|
56
|
+
*/
|
|
57
|
+
export declare function scrub(detail: Record<string, unknown> | undefined): Record<string, unknown> | undefined;
|
|
58
|
+
/**
|
|
59
|
+
* A wire event from a log body (a JSON string or an already-parsed object), or
|
|
60
|
+
* null when it isn't one. Tolerant of extra fields; strict about the ones rules
|
|
61
|
+
* depend on, so a malformed event never matches a rule by accident.
|
|
62
|
+
*/
|
|
63
|
+
export declare function parseWire(input: unknown): WireEvent | null;
|
|
64
|
+
export {};
|