cursedbelt-server 1.0.2 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/server/index.d.ts +1 -0
- package/dist/server/index.js +1 -0
- package/dist/server/metrics/telemetrySink.d.ts +119 -0
- package/dist/server/metrics/telemetrySink.js +76 -0
- package/dist/server/middleware/index.d.ts +1 -0
- package/dist/server/middleware/index.js +3 -0
- package/dist/server/middleware/requestLogger.d.ts +30 -11
- package/dist/server/middleware/requestLogger.js +15 -6
- package/package.json +1 -1
- package/src/server/index.ts +9 -0
- package/src/server/metrics/telemetrySink.spec.ts +288 -0
- package/src/server/metrics/telemetrySink.ts +181 -0
- package/src/server/middleware/index.ts +9 -0
- package/src/server/middleware/requestLogger.ts +43 -17
- package/src/shippedFilesAreTracked.spec.ts +69 -0
package/dist/server/index.d.ts
CHANGED
|
@@ -10,6 +10,7 @@ export { type CreateJobRunnerOptions, createFireQueue, createJobRunner, createPo
|
|
|
10
10
|
export { buildProbeArgs, buildThumbnailArgs, buildTrimArgs, extractThumbnail, ffmpegTimeToSeconds, isFfmpegAvailable, isFfprobeAvailable, type ProbeResult, parseFfmpegProgress, parseFfprobe, probeMedia, renderImage, trimVideo, type VideoCodecPlan, videoCodecPlan, } from './media-bun';
|
|
11
11
|
export { createMetricsBuffer, type MetricRow, type MetricsBuffer, type MetricsBufferOptions, } from './metrics/metricsBuffer';
|
|
12
12
|
export { normalizeRoute } from './metrics/normalizeRoute';
|
|
13
|
+
export { type AnalyticsEngineDataset, createTelemetrySink, TELEMETRY_POINT_VERSION, type TelemetryEvent, type TelemetrySink, type TelemetrySinkOptions, toDataPoint, } from './metrics/telemetrySink';
|
|
13
14
|
export { correlationId } from './middleware/correlationId';
|
|
14
15
|
export { type CorsOpts, corsAllowlist, corsOptsFromEnv } from './middleware/corsAllowlist';
|
|
15
16
|
export { errorEnvelope } from './middleware/errorEnvelope';
|
package/dist/server/index.js
CHANGED
|
@@ -27,6 +27,7 @@ export { buildProbeArgs, buildThumbnailArgs, buildTrimArgs, extractThumbnail, ff
|
|
|
27
27
|
// ── Metrics ───────────────────────────────────────────────────────────────────
|
|
28
28
|
export { createMetricsBuffer, } from './metrics/metricsBuffer';
|
|
29
29
|
export { normalizeRoute } from './metrics/normalizeRoute';
|
|
30
|
+
export { createTelemetrySink, TELEMETRY_POINT_VERSION, toDataPoint, } from './metrics/telemetrySink';
|
|
30
31
|
// ── Middleware ────────────────────────────────────────────────────────────────
|
|
31
32
|
export { correlationId } from './middleware/correlationId';
|
|
32
33
|
export { corsAllowlist, corsOptsFromEnv } from './middleware/corsAllowlist';
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
import type { Database } from 'bun:sqlite';
|
|
2
|
+
import { type MetricRow } from './metricsBuffer';
|
|
3
|
+
/**
|
|
4
|
+
* ONE seam for "this app served a request", with two backends and no third.
|
|
5
|
+
*
|
|
6
|
+
* ── Why this exists ─────────────────────────────────────────────────────────
|
|
7
|
+
* `request_metrics` is a row per request. On D1 that is the one line in
|
|
8
|
+
* Cloudflare's pricing with a dollar-scale overage — **$1.00 per million rows
|
|
9
|
+
* written** — and a row per request is precisely the shape that buys it. Workers
|
|
10
|
+
* Analytics Engine is purpose-built for the same job: `writeDataPoint()` per
|
|
11
|
+
* event, SQL to query, unlimited cardinality, **10M data points and 1M read
|
|
12
|
+
* queries included per month**, and **zero** database writes.
|
|
13
|
+
*
|
|
14
|
+
* 🔴 **Adopt it for the ALLOWANCE, not because it is unbilled today.**
|
|
15
|
+
* Cloudflare's own wording is that you "will not be billed for your use of
|
|
16
|
+
* Workers Analytics Engine" today and that the published prices are "shared in
|
|
17
|
+
* advance… once Cloudflare starts billing for usage in the coming months". The
|
|
18
|
+
* free-today part is temporary; the allowance is not. At this fleet's measured
|
|
19
|
+
* ~1M requests/month the bill is $0 either way, so building against the
|
|
20
|
+
* allowance makes the billing switch a non-event.
|
|
21
|
+
*
|
|
22
|
+
* ── Why not `prom-client` / a `/metrics` scrape ──────────────────────────────
|
|
23
|
+
* A scrape genuinely does turn many writes into one, and on a long-lived Bun
|
|
24
|
+
* process it would work. A Worker is **a stateless V8 isolate with no
|
|
25
|
+
* long-lived process**: isolates are created and destroyed between requests and
|
|
26
|
+
* run concurrently in many datacentres, so a `/metrics` scrape reaches ONE
|
|
27
|
+
* arbitrary isolate holding a random fraction of the counters. The in-process
|
|
28
|
+
* registry has nowhere to live. (A Durable Object does have a durable identity
|
|
29
|
+
* *and* memory, and is the real `prom-client` analogue — but it is a whole
|
|
30
|
+
* stateful object to operate for counters Analytics Engine already aggregates.
|
|
31
|
+
* Prometheus itself also needs an always-on host to scrape FROM, and that host
|
|
32
|
+
* is the Mac this fleet is trying to switch off.)
|
|
33
|
+
*
|
|
34
|
+
* ── The contract ────────────────────────────────────────────────────────────
|
|
35
|
+
* · `analytics` bound → one `writeDataPoint` per event, **zero** SQLite/D1 rows.
|
|
36
|
+
* · no `analytics` → the existing {@link MetricsBuffer}, byte-for-byte
|
|
37
|
+
* unchanged, so the gate and a Mac-hosted app keep working.
|
|
38
|
+
* Both are proven by `telemetrySink.spec.ts`, which is the point of the seam:
|
|
39
|
+
* the choice is made ONCE here rather than per app.
|
|
40
|
+
*/
|
|
41
|
+
/**
|
|
42
|
+
* The Workers Analytics Engine binding, typed locally so this package needs no
|
|
43
|
+
* dependency on `@cloudflare/workers-types` (it is a Bun/Hono package, and a
|
|
44
|
+
* Worker supplies the real binding at runtime).
|
|
45
|
+
*
|
|
46
|
+
* Platform limits, which {@link toDataPoint} respects:
|
|
47
|
+
* **1** index of ≤96 bytes, ≤**20** blobs totalling ≤5120 bytes, ≤**20** doubles.
|
|
48
|
+
*/
|
|
49
|
+
export interface AnalyticsEngineDataset {
|
|
50
|
+
writeDataPoint(point: {
|
|
51
|
+
indexes?: (ArrayBuffer | string | null)[];
|
|
52
|
+
blobs?: (ArrayBuffer | string | null)[];
|
|
53
|
+
doubles?: number[];
|
|
54
|
+
}): void;
|
|
55
|
+
}
|
|
56
|
+
/** One request's telemetry. Identical to {@link MetricRow} — the seam does not re-shape it. */
|
|
57
|
+
export type TelemetryEvent = MetricRow;
|
|
58
|
+
/**
|
|
59
|
+
* A superset-compatible shape: a {@link MetricsBuffer} IS a `TelemetrySink`, so
|
|
60
|
+
* an app that already owns a buffer can keep passing it.
|
|
61
|
+
*/
|
|
62
|
+
export interface TelemetrySink {
|
|
63
|
+
/** Enqueue one request's telemetry. Never blocks the response. */
|
|
64
|
+
record(event: TelemetryEvent): void;
|
|
65
|
+
/** Force whatever is pending out now (shutdown + tests). No-op on Analytics Engine. */
|
|
66
|
+
flush(): void;
|
|
67
|
+
/** Stop any timer and drain. No-op on Analytics Engine. */
|
|
68
|
+
stop(): void;
|
|
69
|
+
/** Events pending locally. Always 0 on Analytics Engine — it has no buffer to hold. */
|
|
70
|
+
readonly size: number;
|
|
71
|
+
}
|
|
72
|
+
export interface TelemetrySinkOptions {
|
|
73
|
+
/**
|
|
74
|
+
* The Analytics Engine binding, when the platform supplies one. Present ⇒ it
|
|
75
|
+
* WINS, and `db` is never written. `null`/`undefined` ⇒ the SQLite path.
|
|
76
|
+
*/
|
|
77
|
+
analytics?: AnalyticsEngineDataset | null;
|
|
78
|
+
/** The `request_metrics` database. Required unless `analytics` is bound. */
|
|
79
|
+
db?: Database | null;
|
|
80
|
+
/** Flush cadence for the SQLite path, in ms. Default: 1500. Ignored by Analytics Engine. */
|
|
81
|
+
flushIntervalMs?: number;
|
|
82
|
+
/** Eager-flush threshold for the SQLite path. Default: 5000. Ignored by Analytics Engine. */
|
|
83
|
+
maxSize?: number;
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* 🔴 **The data-point layout is a WIRE FORMAT.** Analytics Engine columns are
|
|
87
|
+
* positional (`blob1`, `double1`, …), so every saved SQL query breaks if a
|
|
88
|
+
* position moves. Append only; never reorder, never repurpose.
|
|
89
|
+
*
|
|
90
|
+
* ```
|
|
91
|
+
* index1 = route, truncated to 96 bytes — the sampling key, so sampling is per route
|
|
92
|
+
* blob1 = method blob2 = route (untruncated) blob3 = userId ('' when anonymous)
|
|
93
|
+
* double1 = status double2 = durationMs double3 = bytesOut
|
|
94
|
+
* ```
|
|
95
|
+
*
|
|
96
|
+
* `ts` is deliberately absent: Analytics Engine stamps its own `timestamp`
|
|
97
|
+
* column, so sending ours would store the same instant twice.
|
|
98
|
+
*
|
|
99
|
+
* `-1` is the "unknown" marker for all three doubles, because a positional
|
|
100
|
+
* doubles array cannot hold a null and `0` is a real value for every one of them
|
|
101
|
+
* (a 0-byte 204, notably). Read it as `NULL`, not as a measurement.
|
|
102
|
+
*/
|
|
103
|
+
export declare const TELEMETRY_POINT_VERSION = 1;
|
|
104
|
+
/** The documented layout above, as data. Exported so the spec asserts the wire format itself. */
|
|
105
|
+
export declare function toDataPoint(event: TelemetryEvent): {
|
|
106
|
+
indexes: string[];
|
|
107
|
+
blobs: string[];
|
|
108
|
+
doubles: number[];
|
|
109
|
+
};
|
|
110
|
+
/**
|
|
111
|
+
* Build the sink for whatever platform this process is on.
|
|
112
|
+
*
|
|
113
|
+
* Throws when NEITHER backend is available, on purpose. "An app with no
|
|
114
|
+
* `request_metrics` is not quiet; it is unmeasured" — and a sink that silently
|
|
115
|
+
* drops every event is exactly the failure this seam exists to end. A misbound
|
|
116
|
+
* Worker should fail at construction, in the deploy, rather than serve traffic
|
|
117
|
+
* that nothing can see.
|
|
118
|
+
*/
|
|
119
|
+
export declare function createTelemetrySink(opts: TelemetrySinkOptions): TelemetrySink;
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { createMetricsBuffer } from './metricsBuffer';
|
|
2
|
+
/**
|
|
3
|
+
* 🔴 **The data-point layout is a WIRE FORMAT.** Analytics Engine columns are
|
|
4
|
+
* positional (`blob1`, `double1`, …), so every saved SQL query breaks if a
|
|
5
|
+
* position moves. Append only; never reorder, never repurpose.
|
|
6
|
+
*
|
|
7
|
+
* ```
|
|
8
|
+
* index1 = route, truncated to 96 bytes — the sampling key, so sampling is per route
|
|
9
|
+
* blob1 = method blob2 = route (untruncated) blob3 = userId ('' when anonymous)
|
|
10
|
+
* double1 = status double2 = durationMs double3 = bytesOut
|
|
11
|
+
* ```
|
|
12
|
+
*
|
|
13
|
+
* `ts` is deliberately absent: Analytics Engine stamps its own `timestamp`
|
|
14
|
+
* column, so sending ours would store the same instant twice.
|
|
15
|
+
*
|
|
16
|
+
* `-1` is the "unknown" marker for all three doubles, because a positional
|
|
17
|
+
* doubles array cannot hold a null and `0` is a real value for every one of them
|
|
18
|
+
* (a 0-byte 204, notably). Read it as `NULL`, not as a measurement.
|
|
19
|
+
*/
|
|
20
|
+
export const TELEMETRY_POINT_VERSION = 1;
|
|
21
|
+
const INDEX_MAX_BYTES = 96;
|
|
22
|
+
/**
|
|
23
|
+
* Truncate to at most `maxBytes` UTF-8 bytes without splitting a code point —
|
|
24
|
+
* `String.slice` counts UTF-16 units, so a route with one multi-byte character
|
|
25
|
+
* can be ≤96 chars and still exceed the 96-BYTE index limit.
|
|
26
|
+
*/
|
|
27
|
+
function truncateUtf8(value, maxBytes) {
|
|
28
|
+
const encoded = new TextEncoder().encode(value);
|
|
29
|
+
if (encoded.length <= maxBytes)
|
|
30
|
+
return value;
|
|
31
|
+
return new TextDecoder('utf-8', { fatal: false }).decode(encoded.subarray(0, maxBytes)).replace(
|
|
32
|
+
// A cut through a multi-byte sequence decodes to U+FFFD; drop that trailing artefact.
|
|
33
|
+
/�+$/, '');
|
|
34
|
+
}
|
|
35
|
+
const num = (value) => typeof value === 'number' && Number.isFinite(value) ? value : -1;
|
|
36
|
+
/** The documented layout above, as data. Exported so the spec asserts the wire format itself. */
|
|
37
|
+
export function toDataPoint(event) {
|
|
38
|
+
const route = event.route ?? '';
|
|
39
|
+
return {
|
|
40
|
+
indexes: [truncateUtf8(route, INDEX_MAX_BYTES)],
|
|
41
|
+
blobs: [event.method ?? '', route, event.userId ?? ''],
|
|
42
|
+
doubles: [num(event.status), num(event.durationMs), num(event.bytesOut)],
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Build the sink for whatever platform this process is on.
|
|
47
|
+
*
|
|
48
|
+
* Throws when NEITHER backend is available, on purpose. "An app with no
|
|
49
|
+
* `request_metrics` is not quiet; it is unmeasured" — and a sink that silently
|
|
50
|
+
* drops every event is exactly the failure this seam exists to end. A misbound
|
|
51
|
+
* Worker should fail at construction, in the deploy, rather than serve traffic
|
|
52
|
+
* that nothing can see.
|
|
53
|
+
*/
|
|
54
|
+
export function createTelemetrySink(opts) {
|
|
55
|
+
const { analytics, db, flushIntervalMs, maxSize } = opts;
|
|
56
|
+
if (analytics) {
|
|
57
|
+
// Analytics Engine writes are fire-and-forget and already off the response
|
|
58
|
+
// path — there is nothing to buffer, so flush/stop are honestly no-ops.
|
|
59
|
+
return {
|
|
60
|
+
record(event) {
|
|
61
|
+
analytics.writeDataPoint(toDataPoint(event));
|
|
62
|
+
},
|
|
63
|
+
flush() { },
|
|
64
|
+
stop() { },
|
|
65
|
+
get size() {
|
|
66
|
+
return 0;
|
|
67
|
+
},
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
if (!db) {
|
|
71
|
+
throw new Error('createTelemetrySink: neither an Analytics Engine binding (`analytics`) nor a `db` was ' +
|
|
72
|
+
'provided. Telemetry would be silently dropped; bind one of the two.');
|
|
73
|
+
}
|
|
74
|
+
// The unchanged SQLite path — same module, same batching, same rows.
|
|
75
|
+
return createMetricsBuffer({ db, flushIntervalMs, maxSize });
|
|
76
|
+
}
|
|
@@ -4,6 +4,7 @@ export { errorEnvelope } from './errorEnvelope';
|
|
|
4
4
|
export { crossOriginRefusal } from './localOrigin';
|
|
5
5
|
export { clientIp, createRateLimiter, type RateLimitOpts } from './rateLimit';
|
|
6
6
|
export { type RequestLoggerOpts, requestLogger } from './requestLogger';
|
|
7
|
+
export { type AnalyticsEngineDataset, createTelemetrySink, type TelemetryEvent, type TelemetrySink, type TelemetrySinkOptions, } from '../metrics/telemetrySink';
|
|
7
8
|
export { type CspDirectives, type SecurityHeadersOpts, securityHeaders, } from './securityHeaders';
|
|
8
9
|
export { toValidationIssues, VALIDATION_ERROR_CODE, type ValidationIssue, validationEnvelope, } from '../errors';
|
|
9
10
|
export { type ZValidatorTarget, zValidator } from './zValidator';
|
|
@@ -5,6 +5,9 @@ export { errorEnvelope } from './errorEnvelope';
|
|
|
5
5
|
export { crossOriginRefusal } from './localOrigin';
|
|
6
6
|
export { clientIp, createRateLimiter } from './rateLimit';
|
|
7
7
|
export { requestLogger } from './requestLogger';
|
|
8
|
+
// The telemetry seam `requestLogger` writes through — re-exported here so a caller
|
|
9
|
+
// that reaches `./middleware` for the logger can bind Analytics Engine in the same import.
|
|
10
|
+
export { createTelemetrySink, } from '../metrics/telemetrySink';
|
|
8
11
|
export { securityHeaders, } from './securityHeaders';
|
|
9
12
|
export { toValidationIssues, VALIDATION_ERROR_CODE, validationEnvelope, } from '../errors';
|
|
10
13
|
export { zValidator } from './zValidator';
|
|
@@ -1,29 +1,48 @@
|
|
|
1
1
|
import type { Database } from 'bun:sqlite';
|
|
2
2
|
import type { MiddlewareHandler } from 'hono';
|
|
3
3
|
import type { CursedbeltEnv } from '../context';
|
|
4
|
-
import {
|
|
4
|
+
import type { MetricsBuffer } from '../metrics/metricsBuffer';
|
|
5
|
+
import { type AnalyticsEngineDataset, type TelemetrySink } from '../metrics/telemetrySink';
|
|
5
6
|
/**
|
|
6
|
-
* Structured request/response logging. After each request it records one
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
7
|
+
* Structured request/response logging. After each request it records one
|
|
8
|
+
* telemetry event through the {@link TelemetrySink} (off the hot path) and, for
|
|
9
|
+
* errors (status ≥ 400) or slow requests, one `event_logs` row tagged with the
|
|
10
|
+
* correlation id. Attributes rows to `c.get('userId')` when the app's auth
|
|
10
11
|
* middleware has set it.
|
|
11
12
|
*
|
|
13
|
+
* 🔴 **Where the telemetry LANDS is the sink's decision, not this file's.** With
|
|
14
|
+
* an Analytics Engine binding it is one `writeDataPoint` and **zero** database
|
|
15
|
+
* rows; without one it is the same batched `request_metrics` insert it has always
|
|
16
|
+
* been. See `../metrics/telemetrySink.ts` for why that is the only choice offered.
|
|
17
|
+
*
|
|
12
18
|
* It records metrics even when a downstream handler THROWS: the error is caught,
|
|
13
|
-
* the would-be status derived, the
|
|
19
|
+
* the would-be status derived, the event recorded, then the error re-thrown so the
|
|
14
20
|
* error-envelope `onError` still produces the response.
|
|
15
21
|
*/
|
|
16
22
|
export interface RequestLoggerOpts {
|
|
17
23
|
/**
|
|
18
|
-
* The
|
|
19
|
-
*
|
|
20
|
-
|
|
21
|
-
|
|
24
|
+
* The Analytics Engine binding, when the platform supplies one. Present ⇒ the
|
|
25
|
+
* telemetry goes there and NOTHING is written to `request_metrics`.
|
|
26
|
+
*/
|
|
27
|
+
analytics?: AnalyticsEngineDataset | null;
|
|
28
|
+
/**
|
|
29
|
+
* The sink to write through. Provide one (and own its lifecycle via
|
|
30
|
+
* `sink.stop()`) to share it / flush deterministically; omit to build one from
|
|
31
|
+
* `db` + `analytics`. A {@link MetricsBuffer} is a valid sink.
|
|
22
32
|
*/
|
|
33
|
+
sink?: TelemetrySink;
|
|
34
|
+
/** @deprecated Use {@link RequestLoggerOpts.sink} — a `MetricsBuffer` is one. Kept for callers that predate the sink. */
|
|
23
35
|
buffer?: MetricsBuffer;
|
|
24
36
|
/** Duration (ms) at/above which a 2xx request also logs an `event_logs` row. Default: 2000. */
|
|
25
37
|
slowMs?: number;
|
|
26
38
|
/** Also write `event_logs` rows for status ≥ 400. Default: true. */
|
|
27
39
|
logErrors?: boolean;
|
|
28
40
|
}
|
|
29
|
-
|
|
41
|
+
/**
|
|
42
|
+
* @param db The `request_metrics` / `event_logs` database. Pass `null` on a
|
|
43
|
+
* platform that has no `bun:sqlite` — a Worker with `analytics` bound — in
|
|
44
|
+
* which case `event_logs` is not written either, because there is nowhere to
|
|
45
|
+
* put it. `analytics` or `sink` is then mandatory; `createTelemetrySink`
|
|
46
|
+
* throws rather than drop telemetry silently.
|
|
47
|
+
*/
|
|
48
|
+
export declare function requestLogger(db: Database | null, opts?: RequestLoggerOpts): MiddlewareHandler<CursedbeltEnv>;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { HTTPException } from 'hono/http-exception';
|
|
2
2
|
import { isApiError } from '../errors';
|
|
3
|
-
import { createMetricsBuffer } from '../metrics/metricsBuffer';
|
|
4
3
|
import { normalizeRoute } from '../metrics/normalizeRoute';
|
|
4
|
+
import { createTelemetrySink, } from '../metrics/telemetrySink';
|
|
5
5
|
function statusOfThrown(err) {
|
|
6
6
|
if (isApiError(err))
|
|
7
7
|
return err.status;
|
|
@@ -9,13 +9,22 @@ function statusOfThrown(err) {
|
|
|
9
9
|
return err.status;
|
|
10
10
|
return 500;
|
|
11
11
|
}
|
|
12
|
+
/**
|
|
13
|
+
* @param db The `request_metrics` / `event_logs` database. Pass `null` on a
|
|
14
|
+
* platform that has no `bun:sqlite` — a Worker with `analytics` bound — in
|
|
15
|
+
* which case `event_logs` is not written either, because there is nowhere to
|
|
16
|
+
* put it. `analytics` or `sink` is then mandatory; `createTelemetrySink`
|
|
17
|
+
* throws rather than drop telemetry silently.
|
|
18
|
+
*/
|
|
12
19
|
export function requestLogger(db, opts = {}) {
|
|
13
|
-
const
|
|
20
|
+
const sink = opts.sink ?? opts.buffer ?? createTelemetrySink({ db, analytics: opts.analytics ?? null });
|
|
14
21
|
const slowMs = opts.slowMs ?? 2000;
|
|
15
22
|
const logErrors = opts.logErrors ?? true;
|
|
16
|
-
const insertEvent = db
|
|
23
|
+
const insertEvent = db
|
|
24
|
+
? db.prepare(`INSERT INTO event_logs
|
|
17
25
|
(id, correlation_id, event_name, type, message, details, duration_ms, user_id, created_at)
|
|
18
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
26
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
27
|
+
: null;
|
|
19
28
|
return async (c, next) => {
|
|
20
29
|
const start = performance.now();
|
|
21
30
|
let thrown;
|
|
@@ -33,10 +42,10 @@ export function requestLogger(db, opts = {}) {
|
|
|
33
42
|
const lenHeader = didThrow ? null : c.res.headers.get('content-length');
|
|
34
43
|
const bytesOut = lenHeader ? Number(lenHeader) : null;
|
|
35
44
|
const userId = c.get('userId') ?? null;
|
|
36
|
-
|
|
45
|
+
sink.record({ ts: Date.now(), method, route, status, durationMs, bytesOut, userId });
|
|
37
46
|
const isError = status >= 400;
|
|
38
47
|
const isSlow = durationMs >= slowMs;
|
|
39
|
-
if ((logErrors && isError) || isSlow) {
|
|
48
|
+
if (insertEvent && ((logErrors && isError) || isSlow)) {
|
|
40
49
|
const type = status >= 500 ? 'error' : isError ? 'warn' : 'info';
|
|
41
50
|
insertEvent.run(crypto.randomUUID(), c.get('correlationId') ?? null, isError ? 'http_error' : 'http_slow', type, `${method} ${route} → ${status}`, JSON.stringify({ method, route, status, durationMs: Math.round(durationMs), bytesOut }), Math.round(durationMs), userId, new Date().toISOString());
|
|
42
51
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "cursedbelt-server",
|
|
3
|
-
"version": "1.0
|
|
3
|
+
"version": "1.1.0",
|
|
4
4
|
"license": "ISC",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"description": "The app-facing Bun/Hono server tier of the cursedbelt split \u2014 storage, sharing, activity, guard, sync. React-free; cursedbelt-core below it.",
|
package/src/server/index.ts
CHANGED
|
@@ -144,6 +144,15 @@ export {
|
|
|
144
144
|
type MetricsBufferOptions,
|
|
145
145
|
} from './metrics/metricsBuffer';
|
|
146
146
|
export { normalizeRoute } from './metrics/normalizeRoute';
|
|
147
|
+
export {
|
|
148
|
+
type AnalyticsEngineDataset,
|
|
149
|
+
createTelemetrySink,
|
|
150
|
+
TELEMETRY_POINT_VERSION,
|
|
151
|
+
type TelemetryEvent,
|
|
152
|
+
type TelemetrySink,
|
|
153
|
+
type TelemetrySinkOptions,
|
|
154
|
+
toDataPoint,
|
|
155
|
+
} from './metrics/telemetrySink';
|
|
147
156
|
// ── Middleware ────────────────────────────────────────────────────────────────
|
|
148
157
|
export { correlationId } from './middleware/correlationId';
|
|
149
158
|
export { type CorsOpts, corsAllowlist, corsOptsFromEnv } from './middleware/corsAllowlist';
|
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
import { Database } from 'bun:sqlite';
|
|
2
|
+
import { beforeEach, describe, expect, it } from 'bun:test';
|
|
3
|
+
import { Hono } from 'hono';
|
|
4
|
+
import { type RequestLoggerOpts, requestLogger } from '../middleware/requestLogger';
|
|
5
|
+
import { createMetricsBuffer } from './metricsBuffer';
|
|
6
|
+
import {
|
|
7
|
+
type AnalyticsEngineDataset,
|
|
8
|
+
createTelemetrySink,
|
|
9
|
+
type TelemetrySink,
|
|
10
|
+
toDataPoint,
|
|
11
|
+
} from './telemetrySink';
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* The two claims the seam makes, and nothing else:
|
|
15
|
+
*
|
|
16
|
+
* 1. Analytics Engine bound → a request writes **ZERO** `request_metrics` rows.
|
|
17
|
+
* This is the claim with money behind it: a row per request on D1 is billed
|
|
18
|
+
* at $1.00 per million rows written, and "we moved to Analytics Engine" is
|
|
19
|
+
* worth nothing if the insert is still firing beside it.
|
|
20
|
+
* 2. Analytics Engine ABSENT → the SQLite path is byte-for-byte what it is
|
|
21
|
+
* today. Proven by running the same request through the sink-backed logger
|
|
22
|
+
* and through a hand-built `createMetricsBuffer` — the pre-seam call — and
|
|
23
|
+
* comparing every column. The gate and every Mac-hosted app ride on this.
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
/** The `request_metrics` / `event_logs` DDL `metricsBuffer` + `requestLogger` write. */
|
|
27
|
+
function schema(db: Database): void {
|
|
28
|
+
db.run(`CREATE TABLE request_metrics (
|
|
29
|
+
id TEXT PRIMARY KEY, ts INTEGER NOT NULL, method TEXT, route TEXT,
|
|
30
|
+
status INTEGER, duration_ms REAL, bytes_out INTEGER, user_id TEXT)`);
|
|
31
|
+
db.run(`CREATE TABLE event_logs (
|
|
32
|
+
id TEXT PRIMARY KEY, correlation_id TEXT, event_name TEXT, type TEXT, message TEXT,
|
|
33
|
+
details TEXT, duration_ms INTEGER, user_id TEXT, created_at TEXT)`);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
type Point = Parameters<AnalyticsEngineDataset['writeDataPoint']>[0];
|
|
37
|
+
|
|
38
|
+
/** A recording stand-in for the platform binding. */
|
|
39
|
+
function fakeDataset(): AnalyticsEngineDataset & { points: Point[] } {
|
|
40
|
+
const points: Point[] = [];
|
|
41
|
+
return {
|
|
42
|
+
points,
|
|
43
|
+
writeDataPoint(point) {
|
|
44
|
+
points.push(point);
|
|
45
|
+
},
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
const rows = (db: Database) =>
|
|
50
|
+
db.query('SELECT * FROM request_metrics ORDER BY ts').all() as Record<string, unknown>[];
|
|
51
|
+
|
|
52
|
+
describe('createTelemetrySink — Analytics Engine bound means ZERO database rows', () => {
|
|
53
|
+
let db: Database;
|
|
54
|
+
beforeEach(() => {
|
|
55
|
+
db = new Database(':memory:');
|
|
56
|
+
schema(db);
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
it('writes no request_metrics row for a served request, and one data point instead', async () => {
|
|
60
|
+
const ae = fakeDataset();
|
|
61
|
+
const app = new Hono();
|
|
62
|
+
// 🔴 `db` is passed as well: the binding must WIN over an available database,
|
|
63
|
+
// not merely work in its absence. A Worker mid-migration has both.
|
|
64
|
+
app.use('*', requestLogger(db, { analytics: ae }));
|
|
65
|
+
app.get('/api/people/:id/portrait', (c) => c.text('bytes'));
|
|
66
|
+
|
|
67
|
+
expect((await app.request('/api/people/42/portrait')).status).toBe(200);
|
|
68
|
+
|
|
69
|
+
// The flush timer would be the only other writer; force it and re-check.
|
|
70
|
+
await Bun.sleep(5);
|
|
71
|
+
expect(rows(db)).toHaveLength(0);
|
|
72
|
+
expect(db.query('SELECT COUNT(*) AS n FROM event_logs').get()).toEqual({ n: 0 });
|
|
73
|
+
|
|
74
|
+
expect(ae.points).toHaveLength(1);
|
|
75
|
+
const [point] = ae.points;
|
|
76
|
+
expect(point.indexes).toEqual(['/api/people/:id/portrait']);
|
|
77
|
+
expect(point.blobs?.[0]).toBe('GET');
|
|
78
|
+
expect(point.blobs?.[1]).toBe('/api/people/:id/portrait');
|
|
79
|
+
expect(point.doubles?.[0]).toBe(200);
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
it('writes no row for a THROWN request either — the error path is the tempting leak', async () => {
|
|
83
|
+
const ae = fakeDataset();
|
|
84
|
+
const app = new Hono();
|
|
85
|
+
app.use('*', requestLogger(db, { analytics: ae }));
|
|
86
|
+
app.get('/boom', () => {
|
|
87
|
+
throw new Error('nope');
|
|
88
|
+
});
|
|
89
|
+
app.onError((_e, c) => c.text('handled', 500));
|
|
90
|
+
|
|
91
|
+
expect((await app.request('/boom')).status).toBe(500);
|
|
92
|
+
await Bun.sleep(5);
|
|
93
|
+
|
|
94
|
+
expect(rows(db)).toHaveLength(0);
|
|
95
|
+
expect(ae.points).toHaveLength(1);
|
|
96
|
+
expect(ae.points[0].doubles?.[0]).toBe(500);
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
it('needs no database at all — the Worker shape, where bun:sqlite does not exist', async () => {
|
|
100
|
+
const ae = fakeDataset();
|
|
101
|
+
const app = new Hono();
|
|
102
|
+
app.use('*', requestLogger(null, { analytics: ae }));
|
|
103
|
+
app.get('/', (c) => c.text('ok'));
|
|
104
|
+
|
|
105
|
+
expect((await app.request('/')).status).toBe(200);
|
|
106
|
+
expect(ae.points).toHaveLength(1);
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
it('holds nothing locally, so there is no buffer to lose on isolate teardown', () => {
|
|
110
|
+
const sink = createTelemetrySink({ analytics: fakeDataset(), db: null });
|
|
111
|
+
sink.record({
|
|
112
|
+
ts: 1,
|
|
113
|
+
method: 'GET',
|
|
114
|
+
route: '/',
|
|
115
|
+
status: 200,
|
|
116
|
+
durationMs: 1,
|
|
117
|
+
bytesOut: 2,
|
|
118
|
+
userId: null,
|
|
119
|
+
});
|
|
120
|
+
expect(sink.size).toBe(0);
|
|
121
|
+
// flush/stop are honest no-ops, not silent drops — calling them changes nothing.
|
|
122
|
+
sink.flush();
|
|
123
|
+
sink.stop();
|
|
124
|
+
expect(sink.size).toBe(0);
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
it('refuses to construct with neither backend rather than drop telemetry silently', () => {
|
|
128
|
+
expect(() => createTelemetrySink({ db: null, analytics: null })).toThrow(
|
|
129
|
+
/neither an Analytics Engine binding/,
|
|
130
|
+
);
|
|
131
|
+
});
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
describe('toDataPoint — the layout is a wire format', () => {
|
|
135
|
+
it('maps every column to its documented position, with -1 for unknown', () => {
|
|
136
|
+
expect(
|
|
137
|
+
toDataPoint({
|
|
138
|
+
ts: 123,
|
|
139
|
+
method: 'POST',
|
|
140
|
+
route: '/api/v1/sync/requested',
|
|
141
|
+
status: 204,
|
|
142
|
+
durationMs: 12.5,
|
|
143
|
+
bytesOut: null,
|
|
144
|
+
userId: 'u-1',
|
|
145
|
+
}),
|
|
146
|
+
).toEqual({
|
|
147
|
+
indexes: ['/api/v1/sync/requested'],
|
|
148
|
+
blobs: ['POST', '/api/v1/sync/requested', 'u-1'],
|
|
149
|
+
// bytesOut null → -1. `0` is a real value for a 204, so it cannot be the marker.
|
|
150
|
+
doubles: [204, 12.5, -1],
|
|
151
|
+
});
|
|
152
|
+
});
|
|
153
|
+
|
|
154
|
+
it('truncates the index to 96 BYTES, not 96 characters', () => {
|
|
155
|
+
// 60 three-byte characters = 180 bytes, but only 60 UTF-16 units — `slice(0, 96)`
|
|
156
|
+
// would leave 180 bytes and the platform would reject the data point.
|
|
157
|
+
const route = `/${'路'.repeat(60)}`;
|
|
158
|
+
const { indexes, blobs } = toDataPoint({
|
|
159
|
+
ts: 1,
|
|
160
|
+
method: 'GET',
|
|
161
|
+
route,
|
|
162
|
+
status: 200,
|
|
163
|
+
durationMs: 1,
|
|
164
|
+
bytesOut: 1,
|
|
165
|
+
userId: null,
|
|
166
|
+
});
|
|
167
|
+
const bytes = new TextEncoder().encode(indexes[0]).length;
|
|
168
|
+
expect(bytes).toBeLessThanOrEqual(96);
|
|
169
|
+
// Not split mid-character: it round-trips, so no U+FFFD was left behind.
|
|
170
|
+
expect(indexes[0]).toBe(new TextDecoder().decode(new TextEncoder().encode(indexes[0])));
|
|
171
|
+
expect(indexes[0].includes('�')).toBe(false);
|
|
172
|
+
// …and blob2 keeps the FULL route, so nothing is actually lost.
|
|
173
|
+
expect(blobs[1]).toBe(route);
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
it('an anonymous request sends an empty userId blob, never a null hole', () => {
|
|
177
|
+
const { blobs } = toDataPoint({
|
|
178
|
+
ts: 1,
|
|
179
|
+
method: 'GET',
|
|
180
|
+
route: '/',
|
|
181
|
+
status: 200,
|
|
182
|
+
durationMs: 1,
|
|
183
|
+
bytesOut: 0,
|
|
184
|
+
userId: null,
|
|
185
|
+
});
|
|
186
|
+
expect(blobs[2]).toBe('');
|
|
187
|
+
});
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
describe('createTelemetrySink — no binding means the SQLite path, unchanged', () => {
|
|
191
|
+
/** Columns that are allowed to differ between two identical requests. */
|
|
192
|
+
const stable = (row: Record<string, unknown>) => {
|
|
193
|
+
const { id: _id, ts: _ts, duration_ms: _d, ...rest } = row;
|
|
194
|
+
return rest;
|
|
195
|
+
};
|
|
196
|
+
|
|
197
|
+
it('produces the same row the pre-seam createMetricsBuffer call produces', async () => {
|
|
198
|
+
const viaSink = new Database(':memory:');
|
|
199
|
+
const viaBuffer = new Database(':memory:');
|
|
200
|
+
schema(viaSink);
|
|
201
|
+
schema(viaBuffer);
|
|
202
|
+
|
|
203
|
+
const build = (db: Database, logger: ReturnType<typeof requestLogger>) => {
|
|
204
|
+
const app = new Hono();
|
|
205
|
+
app.use('*', logger);
|
|
206
|
+
// 🔴 `content-length` is set EXPLICITLY. `requestLogger` reads `bytes_out` off
|
|
207
|
+
// that header and Hono does not add one for a small in-process response, so
|
|
208
|
+
// without it this test would assert `bytes_out: null` and prove nothing about
|
|
209
|
+
// the column that actually carries bytes.
|
|
210
|
+
app.get('/api/player', (c) => c.json({ playing: false }, 200, { 'content-length': '18' }));
|
|
211
|
+
return { app, db };
|
|
212
|
+
};
|
|
213
|
+
|
|
214
|
+
// The new path: no `analytics`, so the sink resolves to the SQLite buffer.
|
|
215
|
+
const a = build(viaSink, requestLogger(viaSink));
|
|
216
|
+
// The pre-seam path, spelled exactly as a caller spelled it before this change.
|
|
217
|
+
const priorBuffer = createMetricsBuffer({ db: viaBuffer });
|
|
218
|
+
const b = build(viaBuffer, requestLogger(viaBuffer, { buffer: priorBuffer }));
|
|
219
|
+
|
|
220
|
+
expect((await a.app.request('/api/player')).status).toBe(200);
|
|
221
|
+
expect((await b.app.request('/api/player')).status).toBe(200);
|
|
222
|
+
|
|
223
|
+
// Both buffer; both must flush the same way. 1500ms is the shared cadence, so
|
|
224
|
+
// force it rather than sleep through it.
|
|
225
|
+
priorBuffer.flush();
|
|
226
|
+
await Bun.sleep(1700);
|
|
227
|
+
|
|
228
|
+
const sinkRows = rows(viaSink);
|
|
229
|
+
const bufferRows = rows(viaBuffer);
|
|
230
|
+
expect(sinkRows).toHaveLength(1);
|
|
231
|
+
expect(bufferRows).toHaveLength(1);
|
|
232
|
+
expect(stable(sinkRows[0])).toEqual(stable(bufferRows[0]));
|
|
233
|
+
// Column-by-column, so a renamed or dropped column cannot hide behind a deep-equal.
|
|
234
|
+
expect(Object.keys(sinkRows[0])).toEqual([
|
|
235
|
+
'id',
|
|
236
|
+
'ts',
|
|
237
|
+
'method',
|
|
238
|
+
'route',
|
|
239
|
+
'status',
|
|
240
|
+
'duration_ms',
|
|
241
|
+
'bytes_out',
|
|
242
|
+
'user_id',
|
|
243
|
+
]);
|
|
244
|
+
expect(stable(sinkRows[0])).toEqual({
|
|
245
|
+
method: 'GET',
|
|
246
|
+
route: '/api/player',
|
|
247
|
+
status: 200,
|
|
248
|
+
bytes_out: 18,
|
|
249
|
+
user_id: null,
|
|
250
|
+
});
|
|
251
|
+
expect(typeof sinkRows[0].duration_ms).toBe('number');
|
|
252
|
+
});
|
|
253
|
+
|
|
254
|
+
it('still buffers rather than writing per request — the latency property is not lost', () => {
|
|
255
|
+
const db = new Database(':memory:');
|
|
256
|
+
schema(db);
|
|
257
|
+
const sink = createTelemetrySink({ db });
|
|
258
|
+
sink.record({
|
|
259
|
+
ts: 1,
|
|
260
|
+
method: 'GET',
|
|
261
|
+
route: '/',
|
|
262
|
+
status: 200,
|
|
263
|
+
durationMs: 1,
|
|
264
|
+
bytesOut: 0,
|
|
265
|
+
userId: null,
|
|
266
|
+
});
|
|
267
|
+
// Buffered, NOT written: `size` is 1 and the table is still empty.
|
|
268
|
+
expect(sink.size).toBe(1);
|
|
269
|
+
expect(rows(db)).toHaveLength(0);
|
|
270
|
+
sink.flush();
|
|
271
|
+
expect(sink.size).toBe(0);
|
|
272
|
+
expect(rows(db)).toHaveLength(1);
|
|
273
|
+
sink.stop();
|
|
274
|
+
});
|
|
275
|
+
|
|
276
|
+
it('a MetricsBuffer satisfies TelemetrySink, so an app can keep passing its own', () => {
|
|
277
|
+
const db = new Database(':memory:');
|
|
278
|
+
schema(db);
|
|
279
|
+
const buffer = createMetricsBuffer({ db });
|
|
280
|
+
// Structural, and the ASSIGNMENT is the assertion — this line is what would stop
|
|
281
|
+
// compiling if the two shapes ever diverged, which is why it is typed explicitly.
|
|
282
|
+
const sink: TelemetrySink = buffer;
|
|
283
|
+
expect(sink.size).toBe(0);
|
|
284
|
+
const opts: RequestLoggerOpts = { sink: buffer };
|
|
285
|
+
expect(opts.sink).toBe(buffer);
|
|
286
|
+
buffer.stop();
|
|
287
|
+
});
|
|
288
|
+
});
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
import type { Database } from 'bun:sqlite';
|
|
2
|
+
import { createMetricsBuffer, type MetricRow, type MetricsBuffer } from './metricsBuffer';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* ONE seam for "this app served a request", with two backends and no third.
|
|
6
|
+
*
|
|
7
|
+
* ── Why this exists ─────────────────────────────────────────────────────────
|
|
8
|
+
* `request_metrics` is a row per request. On D1 that is the one line in
|
|
9
|
+
* Cloudflare's pricing with a dollar-scale overage — **$1.00 per million rows
|
|
10
|
+
* written** — and a row per request is precisely the shape that buys it. Workers
|
|
11
|
+
* Analytics Engine is purpose-built for the same job: `writeDataPoint()` per
|
|
12
|
+
* event, SQL to query, unlimited cardinality, **10M data points and 1M read
|
|
13
|
+
* queries included per month**, and **zero** database writes.
|
|
14
|
+
*
|
|
15
|
+
* 🔴 **Adopt it for the ALLOWANCE, not because it is unbilled today.**
|
|
16
|
+
* Cloudflare's own wording is that you "will not be billed for your use of
|
|
17
|
+
* Workers Analytics Engine" today and that the published prices are "shared in
|
|
18
|
+
* advance… once Cloudflare starts billing for usage in the coming months". The
|
|
19
|
+
* free-today part is temporary; the allowance is not. At this fleet's measured
|
|
20
|
+
* ~1M requests/month the bill is $0 either way, so building against the
|
|
21
|
+
* allowance makes the billing switch a non-event.
|
|
22
|
+
*
|
|
23
|
+
* ── Why not `prom-client` / a `/metrics` scrape ──────────────────────────────
|
|
24
|
+
* A scrape genuinely does turn many writes into one, and on a long-lived Bun
|
|
25
|
+
* process it would work. A Worker is **a stateless V8 isolate with no
|
|
26
|
+
* long-lived process**: isolates are created and destroyed between requests and
|
|
27
|
+
* run concurrently in many datacentres, so a `/metrics` scrape reaches ONE
|
|
28
|
+
* arbitrary isolate holding a random fraction of the counters. The in-process
|
|
29
|
+
* registry has nowhere to live. (A Durable Object does have a durable identity
|
|
30
|
+
* *and* memory, and is the real `prom-client` analogue — but it is a whole
|
|
31
|
+
* stateful object to operate for counters Analytics Engine already aggregates.
|
|
32
|
+
* Prometheus itself also needs an always-on host to scrape FROM, and that host
|
|
33
|
+
* is the Mac this fleet is trying to switch off.)
|
|
34
|
+
*
|
|
35
|
+
* ── The contract ────────────────────────────────────────────────────────────
|
|
36
|
+
* · `analytics` bound → one `writeDataPoint` per event, **zero** SQLite/D1 rows.
|
|
37
|
+
* · no `analytics` → the existing {@link MetricsBuffer}, byte-for-byte
|
|
38
|
+
* unchanged, so the gate and a Mac-hosted app keep working.
|
|
39
|
+
* Both are proven by `telemetrySink.spec.ts`, which is the point of the seam:
|
|
40
|
+
* the choice is made ONCE here rather than per app.
|
|
41
|
+
*/
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* The Workers Analytics Engine binding, typed locally so this package needs no
|
|
45
|
+
* dependency on `@cloudflare/workers-types` (it is a Bun/Hono package, and a
|
|
46
|
+
* Worker supplies the real binding at runtime).
|
|
47
|
+
*
|
|
48
|
+
* Platform limits, which {@link toDataPoint} respects:
|
|
49
|
+
* **1** index of ≤96 bytes, ≤**20** blobs totalling ≤5120 bytes, ≤**20** doubles.
|
|
50
|
+
*/
|
|
51
|
+
export interface AnalyticsEngineDataset {
|
|
52
|
+
writeDataPoint(point: {
|
|
53
|
+
indexes?: (ArrayBuffer | string | null)[];
|
|
54
|
+
blobs?: (ArrayBuffer | string | null)[];
|
|
55
|
+
doubles?: number[];
|
|
56
|
+
}): void;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** One request's telemetry. Identical to {@link MetricRow} — the seam does not re-shape it. */
|
|
60
|
+
export type TelemetryEvent = MetricRow;
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* A superset-compatible shape: a {@link MetricsBuffer} IS a `TelemetrySink`, so
|
|
64
|
+
* an app that already owns a buffer can keep passing it.
|
|
65
|
+
*/
|
|
66
|
+
export interface TelemetrySink {
|
|
67
|
+
/** Enqueue one request's telemetry. Never blocks the response. */
|
|
68
|
+
record(event: TelemetryEvent): void;
|
|
69
|
+
/** Force whatever is pending out now (shutdown + tests). No-op on Analytics Engine. */
|
|
70
|
+
flush(): void;
|
|
71
|
+
/** Stop any timer and drain. No-op on Analytics Engine. */
|
|
72
|
+
stop(): void;
|
|
73
|
+
/** Events pending locally. Always 0 on Analytics Engine — it has no buffer to hold. */
|
|
74
|
+
readonly size: number;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export interface TelemetrySinkOptions {
|
|
78
|
+
/**
|
|
79
|
+
* The Analytics Engine binding, when the platform supplies one. Present ⇒ it
|
|
80
|
+
* WINS, and `db` is never written. `null`/`undefined` ⇒ the SQLite path.
|
|
81
|
+
*/
|
|
82
|
+
analytics?: AnalyticsEngineDataset | null;
|
|
83
|
+
/** The `request_metrics` database. Required unless `analytics` is bound. */
|
|
84
|
+
db?: Database | null;
|
|
85
|
+
/** Flush cadence for the SQLite path, in ms. Default: 1500. Ignored by Analytics Engine. */
|
|
86
|
+
flushIntervalMs?: number;
|
|
87
|
+
/** Eager-flush threshold for the SQLite path. Default: 5000. Ignored by Analytics Engine. */
|
|
88
|
+
maxSize?: number;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* 🔴 **The data-point layout is a WIRE FORMAT.** Analytics Engine columns are
|
|
93
|
+
* positional (`blob1`, `double1`, …), so every saved SQL query breaks if a
|
|
94
|
+
* position moves. Append only; never reorder, never repurpose.
|
|
95
|
+
*
|
|
96
|
+
* ```
|
|
97
|
+
* index1 = route, truncated to 96 bytes — the sampling key, so sampling is per route
|
|
98
|
+
* blob1 = method blob2 = route (untruncated) blob3 = userId ('' when anonymous)
|
|
99
|
+
* double1 = status double2 = durationMs double3 = bytesOut
|
|
100
|
+
* ```
|
|
101
|
+
*
|
|
102
|
+
* `ts` is deliberately absent: Analytics Engine stamps its own `timestamp`
|
|
103
|
+
* column, so sending ours would store the same instant twice.
|
|
104
|
+
*
|
|
105
|
+
* `-1` is the "unknown" marker for all three doubles, because a positional
|
|
106
|
+
* doubles array cannot hold a null and `0` is a real value for every one of them
|
|
107
|
+
* (a 0-byte 204, notably). Read it as `NULL`, not as a measurement.
|
|
108
|
+
*/
|
|
109
|
+
export const TELEMETRY_POINT_VERSION = 1;
|
|
110
|
+
|
|
111
|
+
const INDEX_MAX_BYTES = 96;
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Truncate to at most `maxBytes` UTF-8 bytes without splitting a code point —
|
|
115
|
+
* `String.slice` counts UTF-16 units, so a route with one multi-byte character
|
|
116
|
+
* can be ≤96 chars and still exceed the 96-BYTE index limit.
|
|
117
|
+
*/
|
|
118
|
+
function truncateUtf8(value: string, maxBytes: number): string {
|
|
119
|
+
const encoded = new TextEncoder().encode(value);
|
|
120
|
+
if (encoded.length <= maxBytes) return value;
|
|
121
|
+
return new TextDecoder('utf-8', { fatal: false }).decode(encoded.subarray(0, maxBytes)).replace(
|
|
122
|
+
// A cut through a multi-byte sequence decodes to U+FFFD; drop that trailing artefact.
|
|
123
|
+
/�+$/,
|
|
124
|
+
'',
|
|
125
|
+
);
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
const num = (value: number | null | undefined): number =>
|
|
129
|
+
typeof value === 'number' && Number.isFinite(value) ? value : -1;
|
|
130
|
+
|
|
131
|
+
/** The documented layout above, as data. Exported so the spec asserts the wire format itself. */
|
|
132
|
+
export function toDataPoint(event: TelemetryEvent): {
|
|
133
|
+
indexes: string[];
|
|
134
|
+
blobs: string[];
|
|
135
|
+
doubles: number[];
|
|
136
|
+
} {
|
|
137
|
+
const route = event.route ?? '';
|
|
138
|
+
return {
|
|
139
|
+
indexes: [truncateUtf8(route, INDEX_MAX_BYTES)],
|
|
140
|
+
blobs: [event.method ?? '', route, event.userId ?? ''],
|
|
141
|
+
doubles: [num(event.status), num(event.durationMs), num(event.bytesOut)],
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* Build the sink for whatever platform this process is on.
|
|
147
|
+
*
|
|
148
|
+
* Throws when NEITHER backend is available, on purpose. "An app with no
|
|
149
|
+
* `request_metrics` is not quiet; it is unmeasured" — and a sink that silently
|
|
150
|
+
* drops every event is exactly the failure this seam exists to end. A misbound
|
|
151
|
+
* Worker should fail at construction, in the deploy, rather than serve traffic
|
|
152
|
+
* that nothing can see.
|
|
153
|
+
*/
|
|
154
|
+
export function createTelemetrySink(opts: TelemetrySinkOptions): TelemetrySink {
|
|
155
|
+
const { analytics, db, flushIntervalMs, maxSize } = opts;
|
|
156
|
+
|
|
157
|
+
if (analytics) {
|
|
158
|
+
// Analytics Engine writes are fire-and-forget and already off the response
|
|
159
|
+
// path — there is nothing to buffer, so flush/stop are honestly no-ops.
|
|
160
|
+
return {
|
|
161
|
+
record(event) {
|
|
162
|
+
analytics.writeDataPoint(toDataPoint(event));
|
|
163
|
+
},
|
|
164
|
+
flush() {},
|
|
165
|
+
stop() {},
|
|
166
|
+
get size() {
|
|
167
|
+
return 0;
|
|
168
|
+
},
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
if (!db) {
|
|
173
|
+
throw new Error(
|
|
174
|
+
'createTelemetrySink: neither an Analytics Engine binding (`analytics`) nor a `db` was ' +
|
|
175
|
+
'provided. Telemetry would be silently dropped; bind one of the two.',
|
|
176
|
+
);
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
// The unchanged SQLite path — same module, same batching, same rows.
|
|
180
|
+
return createMetricsBuffer({ db, flushIntervalMs, maxSize });
|
|
181
|
+
}
|
|
@@ -5,6 +5,15 @@ export { errorEnvelope } from './errorEnvelope';
|
|
|
5
5
|
export { crossOriginRefusal } from './localOrigin';
|
|
6
6
|
export { clientIp, createRateLimiter, type RateLimitOpts } from './rateLimit';
|
|
7
7
|
export { type RequestLoggerOpts, requestLogger } from './requestLogger';
|
|
8
|
+
// The telemetry seam `requestLogger` writes through — re-exported here so a caller
|
|
9
|
+
// that reaches `./middleware` for the logger can bind Analytics Engine in the same import.
|
|
10
|
+
export {
|
|
11
|
+
type AnalyticsEngineDataset,
|
|
12
|
+
createTelemetrySink,
|
|
13
|
+
type TelemetryEvent,
|
|
14
|
+
type TelemetrySink,
|
|
15
|
+
type TelemetrySinkOptions,
|
|
16
|
+
} from '../metrics/telemetrySink';
|
|
8
17
|
export {
|
|
9
18
|
type CspDirectives,
|
|
10
19
|
type SecurityHeadersOpts,
|
|
@@ -3,28 +3,44 @@ import type { MiddlewareHandler } from 'hono';
|
|
|
3
3
|
import { HTTPException } from 'hono/http-exception';
|
|
4
4
|
import type { CursedbeltEnv } from '../context';
|
|
5
5
|
import { isApiError } from '../errors';
|
|
6
|
-
import {
|
|
6
|
+
import type { MetricsBuffer } from '../metrics/metricsBuffer';
|
|
7
7
|
import { normalizeRoute } from '../metrics/normalizeRoute';
|
|
8
|
+
import {
|
|
9
|
+
type AnalyticsEngineDataset,
|
|
10
|
+
createTelemetrySink,
|
|
11
|
+
type TelemetrySink,
|
|
12
|
+
} from '../metrics/telemetrySink';
|
|
8
13
|
|
|
9
14
|
/**
|
|
10
|
-
* Structured request/response logging. After each request it records one
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
15
|
+
* Structured request/response logging. After each request it records one
|
|
16
|
+
* telemetry event through the {@link TelemetrySink} (off the hot path) and, for
|
|
17
|
+
* errors (status ≥ 400) or slow requests, one `event_logs` row tagged with the
|
|
18
|
+
* correlation id. Attributes rows to `c.get('userId')` when the app's auth
|
|
14
19
|
* middleware has set it.
|
|
15
20
|
*
|
|
21
|
+
* 🔴 **Where the telemetry LANDS is the sink's decision, not this file's.** With
|
|
22
|
+
* an Analytics Engine binding it is one `writeDataPoint` and **zero** database
|
|
23
|
+
* rows; without one it is the same batched `request_metrics` insert it has always
|
|
24
|
+
* been. See `../metrics/telemetrySink.ts` for why that is the only choice offered.
|
|
25
|
+
*
|
|
16
26
|
* It records metrics even when a downstream handler THROWS: the error is caught,
|
|
17
|
-
* the would-be status derived, the
|
|
27
|
+
* the would-be status derived, the event recorded, then the error re-thrown so the
|
|
18
28
|
* error-envelope `onError` still produces the response.
|
|
19
29
|
*/
|
|
20
30
|
|
|
21
31
|
export interface RequestLoggerOpts {
|
|
22
32
|
/**
|
|
23
|
-
* The
|
|
24
|
-
*
|
|
25
|
-
|
|
26
|
-
|
|
33
|
+
* The Analytics Engine binding, when the platform supplies one. Present ⇒ the
|
|
34
|
+
* telemetry goes there and NOTHING is written to `request_metrics`.
|
|
35
|
+
*/
|
|
36
|
+
analytics?: AnalyticsEngineDataset | null;
|
|
37
|
+
/**
|
|
38
|
+
* The sink to write through. Provide one (and own its lifecycle via
|
|
39
|
+
* `sink.stop()`) to share it / flush deterministically; omit to build one from
|
|
40
|
+
* `db` + `analytics`. A {@link MetricsBuffer} is a valid sink.
|
|
27
41
|
*/
|
|
42
|
+
sink?: TelemetrySink;
|
|
43
|
+
/** @deprecated Use {@link RequestLoggerOpts.sink} — a `MetricsBuffer` is one. Kept for callers that predate the sink. */
|
|
28
44
|
buffer?: MetricsBuffer;
|
|
29
45
|
/** Duration (ms) at/above which a 2xx request also logs an `event_logs` row. Default: 2000. */
|
|
30
46
|
slowMs?: number;
|
|
@@ -38,19 +54,29 @@ function statusOfThrown(err: unknown): number {
|
|
|
38
54
|
return 500;
|
|
39
55
|
}
|
|
40
56
|
|
|
57
|
+
/**
|
|
58
|
+
* @param db The `request_metrics` / `event_logs` database. Pass `null` on a
|
|
59
|
+
* platform that has no `bun:sqlite` — a Worker with `analytics` bound — in
|
|
60
|
+
* which case `event_logs` is not written either, because there is nowhere to
|
|
61
|
+
* put it. `analytics` or `sink` is then mandatory; `createTelemetrySink`
|
|
62
|
+
* throws rather than drop telemetry silently.
|
|
63
|
+
*/
|
|
41
64
|
export function requestLogger(
|
|
42
|
-
db: Database,
|
|
65
|
+
db: Database | null,
|
|
43
66
|
opts: RequestLoggerOpts = {},
|
|
44
67
|
): MiddlewareHandler<CursedbeltEnv> {
|
|
45
|
-
const
|
|
68
|
+
const sink =
|
|
69
|
+
opts.sink ?? opts.buffer ?? createTelemetrySink({ db, analytics: opts.analytics ?? null });
|
|
46
70
|
const slowMs = opts.slowMs ?? 2000;
|
|
47
71
|
const logErrors = opts.logErrors ?? true;
|
|
48
72
|
|
|
49
|
-
const insertEvent = db
|
|
50
|
-
|
|
73
|
+
const insertEvent = db
|
|
74
|
+
? db.prepare(
|
|
75
|
+
`INSERT INTO event_logs
|
|
51
76
|
(id, correlation_id, event_name, type, message, details, duration_ms, user_id, created_at)
|
|
52
77
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
53
|
-
|
|
78
|
+
)
|
|
79
|
+
: null;
|
|
54
80
|
|
|
55
81
|
return async (c, next) => {
|
|
56
82
|
const start = performance.now();
|
|
@@ -70,11 +96,11 @@ export function requestLogger(
|
|
|
70
96
|
const bytesOut = lenHeader ? Number(lenHeader) : null;
|
|
71
97
|
const userId = c.get('userId') ?? null;
|
|
72
98
|
|
|
73
|
-
|
|
99
|
+
sink.record({ ts: Date.now(), method, route, status, durationMs, bytesOut, userId });
|
|
74
100
|
|
|
75
101
|
const isError = status >= 400;
|
|
76
102
|
const isSlow = durationMs >= slowMs;
|
|
77
|
-
if ((logErrors && isError) || isSlow) {
|
|
103
|
+
if (insertEvent && ((logErrors && isError) || isSlow)) {
|
|
78
104
|
const type = status >= 500 ? 'error' : isError ? 'warn' : 'info';
|
|
79
105
|
insertEvent.run(
|
|
80
106
|
crypto.randomUUID(),
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 🔴 `bun publish` packs the WORKING DIRECTORY, not the git index — so a package can
|
|
3
|
+
* ship a file the repo does not have.
|
|
4
|
+
*
|
|
5
|
+
* ── What this cost, 2026-09-15 ─────────────────────────────────────────────────────
|
|
6
|
+
* The cursedbelt split (task 148) moved two integration specs into this package from
|
|
7
|
+
* `cursedbelt`. `git add -u .` stages only files git already knows about, so both were
|
|
8
|
+
* committed nowhere — and `bun run verify` was green *because they were present on disk*,
|
|
9
|
+
* which is the same reason nobody noticed. `1.0.2` went to the registry carrying two
|
|
10
|
+
* `src/` files that existed in no commit, and `git status` said `tracked-dirty=0`.
|
|
11
|
+
*
|
|
12
|
+
* "The tree is clean" and "everything that ships is committed" are different claims, and
|
|
13
|
+
* only the first one is what people actually check. npm versions are immutable, so the
|
|
14
|
+
* window to notice closes at publish time.
|
|
15
|
+
*
|
|
16
|
+
* ── Why this shape ─────────────────────────────────────────────────────────────────
|
|
17
|
+
* It asks git, for exactly the directories `package.json#files` promises to ship, which
|
|
18
|
+
* of those paths git does not track. That is narrower than "any untracked file" on
|
|
19
|
+
* purpose: scratch files, `.agent.noindex/`, a half-written script and an uncommitted
|
|
20
|
+
* lockfile are all legitimate states for a working tree and reddening on them would teach
|
|
21
|
+
* people to skip this. A file INSIDE the published surface is the only case where
|
|
22
|
+
* untracked means "about to ship something no one can review or revert".
|
|
23
|
+
*
|
|
24
|
+
* 🔴 It must run from a real checkout to mean anything, so it refuses rather than passes
|
|
25
|
+
* when `git` cannot answer — a check that silently degrades to green is worse than none.
|
|
26
|
+
*/
|
|
27
|
+
import { describe, expect, test } from 'bun:test';
|
|
28
|
+
import { spawnSync } from 'node:child_process';
|
|
29
|
+
import { existsSync } from 'node:fs';
|
|
30
|
+
import { fileURLToPath } from 'node:url';
|
|
31
|
+
import pkg from '../package.json';
|
|
32
|
+
|
|
33
|
+
const repoRoot = fileURLToPath(new URL('..', import.meta.url));
|
|
34
|
+
|
|
35
|
+
/** The directories `files` promises to ship, minus build output nobody commits. */
|
|
36
|
+
const shippedDirs = (): string[] =>
|
|
37
|
+
((pkg as { files?: string[] }).files ?? []).filter((f) => f !== 'dist' && existsSync(`${repoRoot}/${f}`));
|
|
38
|
+
|
|
39
|
+
const git = (...args: string[]): { ok: boolean; out: string } => {
|
|
40
|
+
const r = spawnSync('git', ['-C', repoRoot, ...args], { encoding: 'utf8' });
|
|
41
|
+
return { ok: r.status === 0, out: (r.stdout ?? '').trim() };
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
describe('every file inside the published surface is tracked', () => {
|
|
45
|
+
test('git can answer at all — a degraded check must refuse, not pass', () => {
|
|
46
|
+
expect(git('rev-parse', '--is-inside-work-tree').ok, 'not a git checkout, so this check proves nothing').toBe(true);
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
test('package.json names a shippable directory to scan', () => {
|
|
50
|
+
// Vacuity guard: an empty `files`, or one naming only `dist`, makes the assertion
|
|
51
|
+
// below pass while scanning nothing at all.
|
|
52
|
+
expect(shippedDirs().length, '`files` names no committed directory — nothing would be scanned').toBeGreaterThan(0);
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
test('no untracked file sits inside a directory `files` ships', () => {
|
|
56
|
+
// --others = untracked; --exclude-standard honours .gitignore, so deliberately
|
|
57
|
+
// ignored paths (.agent.noindex, node_modules) are not offences.
|
|
58
|
+
const { ok, out } = git('ls-files', '--others', '--exclude-standard', '--', ...shippedDirs());
|
|
59
|
+
expect(ok, 'git ls-files failed').toBe(true);
|
|
60
|
+
const untracked = out === '' ? [] : out.split('\n');
|
|
61
|
+
expect(
|
|
62
|
+
untracked,
|
|
63
|
+
'these would be PUBLISHED but exist in no commit — `bun publish` packs the working directory, not the index,\n' +
|
|
64
|
+
'so the registry would carry code nobody can review, diff or revert, and npm versions are immutable.\n' +
|
|
65
|
+
'Commit them (or add them to .gitignore if they genuinely must not ship):\n ' +
|
|
66
|
+
untracked.join('\n '),
|
|
67
|
+
).toEqual([]);
|
|
68
|
+
});
|
|
69
|
+
});
|