cursedbelt-server 1.0.2 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/server/index.d.ts +1 -0
- package/dist/server/index.js +1 -0
- package/dist/server/metrics/telemetrySink.d.ts +119 -0
- package/dist/server/metrics/telemetrySink.js +76 -0
- package/dist/server/middleware/index.d.ts +1 -0
- package/dist/server/middleware/index.js +3 -0
- package/dist/server/middleware/requestLogger.d.ts +30 -11
- package/dist/server/middleware/requestLogger.js +15 -6
- package/dist/server/sync/http.d.ts +20 -3
- package/dist/server/sync/http.js +20 -14
- package/dist/server/sync/index.d.ts +10 -2
- package/dist/server/sync/index.js +9 -1
- package/dist/server/sync/planner.d.ts +38 -8
- package/dist/server/sync/planner.js +32 -8
- package/dist/server/sync/signal.d.ts +161 -0
- package/dist/server/sync/signal.js +348 -0
- package/dist/server/sync/timer.d.ts +63 -19
- package/dist/server/sync/timer.js +104 -45
- package/dist/server/sync/types.d.ts +0 -2
- package/package.json +1 -1
- package/src/noTimerDialsAPeer.spec.ts +469 -0
- package/src/server/index.ts +9 -0
- package/src/server/metrics/telemetrySink.spec.ts +288 -0
- package/src/server/metrics/telemetrySink.ts +181 -0
- package/src/server/middleware/index.ts +9 -0
- package/src/server/middleware/requestLogger.ts +43 -17
- package/src/server/sync/http.ts +31 -16
- package/src/server/sync/index.ts +23 -1
- package/src/server/sync/planner.spec.ts +33 -16
- package/src/server/sync/planner.ts +48 -11
- package/src/server/sync/signal.spec.ts +306 -0
- package/src/server/sync/signal.ts +422 -0
- package/src/server/sync/timer.spec.ts +97 -16
- package/src/server/sync/timer.ts +124 -47
- package/src/server/sync/types.ts +0 -2
- package/src/shippedFilesAreTracked.spec.ts +69 -0
package/dist/server/index.d.ts
CHANGED
|
@@ -10,6 +10,7 @@ export { type CreateJobRunnerOptions, createFireQueue, createJobRunner, createPo
|
|
|
10
10
|
export { buildProbeArgs, buildThumbnailArgs, buildTrimArgs, extractThumbnail, ffmpegTimeToSeconds, isFfmpegAvailable, isFfprobeAvailable, type ProbeResult, parseFfmpegProgress, parseFfprobe, probeMedia, renderImage, trimVideo, type VideoCodecPlan, videoCodecPlan, } from './media-bun';
|
|
11
11
|
export { createMetricsBuffer, type MetricRow, type MetricsBuffer, type MetricsBufferOptions, } from './metrics/metricsBuffer';
|
|
12
12
|
export { normalizeRoute } from './metrics/normalizeRoute';
|
|
13
|
+
export { type AnalyticsEngineDataset, createTelemetrySink, TELEMETRY_POINT_VERSION, type TelemetryEvent, type TelemetrySink, type TelemetrySinkOptions, toDataPoint, } from './metrics/telemetrySink';
|
|
13
14
|
export { correlationId } from './middleware/correlationId';
|
|
14
15
|
export { type CorsOpts, corsAllowlist, corsOptsFromEnv } from './middleware/corsAllowlist';
|
|
15
16
|
export { errorEnvelope } from './middleware/errorEnvelope';
|
package/dist/server/index.js
CHANGED
|
@@ -27,6 +27,7 @@ export { buildProbeArgs, buildThumbnailArgs, buildTrimArgs, extractThumbnail, ff
|
|
|
27
27
|
// ── Metrics ───────────────────────────────────────────────────────────────────
|
|
28
28
|
export { createMetricsBuffer, } from './metrics/metricsBuffer';
|
|
29
29
|
export { normalizeRoute } from './metrics/normalizeRoute';
|
|
30
|
+
export { createTelemetrySink, TELEMETRY_POINT_VERSION, toDataPoint, } from './metrics/telemetrySink';
|
|
30
31
|
// ── Middleware ────────────────────────────────────────────────────────────────
|
|
31
32
|
export { correlationId } from './middleware/correlationId';
|
|
32
33
|
export { corsAllowlist, corsOptsFromEnv } from './middleware/corsAllowlist';
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
import type { Database } from 'bun:sqlite';
|
|
2
|
+
import { type MetricRow } from './metricsBuffer';
|
|
3
|
+
/**
|
|
4
|
+
* ONE seam for "this app served a request", with two backends and no third.
|
|
5
|
+
*
|
|
6
|
+
* ── Why this exists ─────────────────────────────────────────────────────────
|
|
7
|
+
* `request_metrics` is a row per request. On D1 that is the one line in
|
|
8
|
+
* Cloudflare's pricing with a dollar-scale overage — **$1.00 per million rows
|
|
9
|
+
* written** — and a row per request is precisely the shape that buys it. Workers
|
|
10
|
+
* Analytics Engine is purpose-built for the same job: `writeDataPoint()` per
|
|
11
|
+
* event, SQL to query, unlimited cardinality, **10M data points and 1M read
|
|
12
|
+
* queries included per month**, and **zero** database writes.
|
|
13
|
+
*
|
|
14
|
+
* 🔴 **Adopt it for the ALLOWANCE, not because it is unbilled today.**
|
|
15
|
+
* Cloudflare's own wording is that you "will not be billed for your use of
|
|
16
|
+
* Workers Analytics Engine" today and that the published prices are "shared in
|
|
17
|
+
* advance… once Cloudflare starts billing for usage in the coming months". The
|
|
18
|
+
* free-today part is temporary; the allowance is not. At this fleet's measured
|
|
19
|
+
* ~1M requests/month the bill is $0 either way, so building against the
|
|
20
|
+
* allowance makes the billing switch a non-event.
|
|
21
|
+
*
|
|
22
|
+
* ── Why not `prom-client` / a `/metrics` scrape ──────────────────────────────
|
|
23
|
+
* A scrape genuinely does turn many writes into one, and on a long-lived Bun
|
|
24
|
+
* process it would work. A Worker is **a stateless V8 isolate with no
|
|
25
|
+
* long-lived process**: isolates are created and destroyed between requests and
|
|
26
|
+
* run concurrently in many datacentres, so a `/metrics` scrape reaches ONE
|
|
27
|
+
* arbitrary isolate holding a random fraction of the counters. The in-process
|
|
28
|
+
* registry has nowhere to live. (A Durable Object does have a durable identity
|
|
29
|
+
* *and* memory, and is the real `prom-client` analogue — but it is a whole
|
|
30
|
+
* stateful object to operate for counters Analytics Engine already aggregates.
|
|
31
|
+
* Prometheus itself also needs an always-on host to scrape FROM, and that host
|
|
32
|
+
* is the Mac this fleet is trying to switch off.)
|
|
33
|
+
*
|
|
34
|
+
* ── The contract ────────────────────────────────────────────────────────────
|
|
35
|
+
* · `analytics` bound → one `writeDataPoint` per event, **zero** SQLite/D1 rows.
|
|
36
|
+
* · no `analytics` → the existing {@link MetricsBuffer}, byte-for-byte
|
|
37
|
+
* unchanged, so the gate and a Mac-hosted app keep working.
|
|
38
|
+
* Both are proven by `telemetrySink.spec.ts`, which is the point of the seam:
|
|
39
|
+
* the choice is made ONCE here rather than per app.
|
|
40
|
+
*/
|
|
41
|
+
/**
|
|
42
|
+
* The Workers Analytics Engine binding, typed locally so this package needs no
|
|
43
|
+
* dependency on `@cloudflare/workers-types` (it is a Bun/Hono package, and a
|
|
44
|
+
* Worker supplies the real binding at runtime).
|
|
45
|
+
*
|
|
46
|
+
* Platform limits, which {@link toDataPoint} respects:
|
|
47
|
+
* **1** index of ≤96 bytes, ≤**20** blobs totalling ≤5120 bytes, ≤**20** doubles.
|
|
48
|
+
*/
|
|
49
|
+
export interface AnalyticsEngineDataset {
|
|
50
|
+
writeDataPoint(point: {
|
|
51
|
+
indexes?: (ArrayBuffer | string | null)[];
|
|
52
|
+
blobs?: (ArrayBuffer | string | null)[];
|
|
53
|
+
doubles?: number[];
|
|
54
|
+
}): void;
|
|
55
|
+
}
|
|
56
|
+
/** One request's telemetry. Identical to {@link MetricRow} — the seam does not re-shape it. */
|
|
57
|
+
export type TelemetryEvent = MetricRow;
|
|
58
|
+
/**
|
|
59
|
+
* A superset-compatible shape: a {@link MetricsBuffer} IS a `TelemetrySink`, so
|
|
60
|
+
* an app that already owns a buffer can keep passing it.
|
|
61
|
+
*/
|
|
62
|
+
export interface TelemetrySink {
|
|
63
|
+
/** Enqueue one request's telemetry. Never blocks the response. */
|
|
64
|
+
record(event: TelemetryEvent): void;
|
|
65
|
+
/** Force whatever is pending out now (shutdown + tests). No-op on Analytics Engine. */
|
|
66
|
+
flush(): void;
|
|
67
|
+
/** Stop any timer and drain. No-op on Analytics Engine. */
|
|
68
|
+
stop(): void;
|
|
69
|
+
/** Events pending locally. Always 0 on Analytics Engine — it has no buffer to hold. */
|
|
70
|
+
readonly size: number;
|
|
71
|
+
}
|
|
72
|
+
export interface TelemetrySinkOptions {
|
|
73
|
+
/**
|
|
74
|
+
* The Analytics Engine binding, when the platform supplies one. Present ⇒ it
|
|
75
|
+
* WINS, and `db` is never written. `null`/`undefined` ⇒ the SQLite path.
|
|
76
|
+
*/
|
|
77
|
+
analytics?: AnalyticsEngineDataset | null;
|
|
78
|
+
/** The `request_metrics` database. Required unless `analytics` is bound. */
|
|
79
|
+
db?: Database | null;
|
|
80
|
+
/** Flush cadence for the SQLite path, in ms. Default: 1500. Ignored by Analytics Engine. */
|
|
81
|
+
flushIntervalMs?: number;
|
|
82
|
+
/** Eager-flush threshold for the SQLite path. Default: 5000. Ignored by Analytics Engine. */
|
|
83
|
+
maxSize?: number;
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* 🔴 **The data-point layout is a WIRE FORMAT.** Analytics Engine columns are
|
|
87
|
+
* positional (`blob1`, `double1`, …), so every saved SQL query breaks if a
|
|
88
|
+
* position moves. Append only; never reorder, never repurpose.
|
|
89
|
+
*
|
|
90
|
+
* ```
|
|
91
|
+
* index1 = route, truncated to 96 bytes — the sampling key, so sampling is per route
|
|
92
|
+
* blob1 = method blob2 = route (untruncated) blob3 = userId ('' when anonymous)
|
|
93
|
+
* double1 = status double2 = durationMs double3 = bytesOut
|
|
94
|
+
* ```
|
|
95
|
+
*
|
|
96
|
+
* `ts` is deliberately absent: Analytics Engine stamps its own `timestamp`
|
|
97
|
+
* column, so sending ours would store the same instant twice.
|
|
98
|
+
*
|
|
99
|
+
* `-1` is the "unknown" marker for all three doubles, because a positional
|
|
100
|
+
* doubles array cannot hold a null and `0` is a real value for every one of them
|
|
101
|
+
* (a 0-byte 204, notably). Read it as `NULL`, not as a measurement.
|
|
102
|
+
*/
|
|
103
|
+
export declare const TELEMETRY_POINT_VERSION = 1;
|
|
104
|
+
/** The documented layout above, as data. Exported so the spec asserts the wire format itself. */
|
|
105
|
+
export declare function toDataPoint(event: TelemetryEvent): {
|
|
106
|
+
indexes: string[];
|
|
107
|
+
blobs: string[];
|
|
108
|
+
doubles: number[];
|
|
109
|
+
};
|
|
110
|
+
/**
|
|
111
|
+
* Build the sink for whatever platform this process is on.
|
|
112
|
+
*
|
|
113
|
+
* Throws when NEITHER backend is available, on purpose. "An app with no
|
|
114
|
+
* `request_metrics` is not quiet; it is unmeasured" — and a sink that silently
|
|
115
|
+
* drops every event is exactly the failure this seam exists to end. A misbound
|
|
116
|
+
* Worker should fail at construction, in the deploy, rather than serve traffic
|
|
117
|
+
* that nothing can see.
|
|
118
|
+
*/
|
|
119
|
+
export declare function createTelemetrySink(opts: TelemetrySinkOptions): TelemetrySink;
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { createMetricsBuffer } from './metricsBuffer';
|
|
2
|
+
/**
|
|
3
|
+
* 🔴 **The data-point layout is a WIRE FORMAT.** Analytics Engine columns are
|
|
4
|
+
* positional (`blob1`, `double1`, …), so every saved SQL query breaks if a
|
|
5
|
+
* position moves. Append only; never reorder, never repurpose.
|
|
6
|
+
*
|
|
7
|
+
* ```
|
|
8
|
+
* index1 = route, truncated to 96 bytes — the sampling key, so sampling is per route
|
|
9
|
+
* blob1 = method blob2 = route (untruncated) blob3 = userId ('' when anonymous)
|
|
10
|
+
* double1 = status double2 = durationMs double3 = bytesOut
|
|
11
|
+
* ```
|
|
12
|
+
*
|
|
13
|
+
* `ts` is deliberately absent: Analytics Engine stamps its own `timestamp`
|
|
14
|
+
* column, so sending ours would store the same instant twice.
|
|
15
|
+
*
|
|
16
|
+
* `-1` is the "unknown" marker for all three doubles, because a positional
|
|
17
|
+
* doubles array cannot hold a null and `0` is a real value for every one of them
|
|
18
|
+
* (a 0-byte 204, notably). Read it as `NULL`, not as a measurement.
|
|
19
|
+
*/
|
|
20
|
+
export const TELEMETRY_POINT_VERSION = 1;
|
|
21
|
+
const INDEX_MAX_BYTES = 96;
|
|
22
|
+
/**
|
|
23
|
+
* Truncate to at most `maxBytes` UTF-8 bytes without splitting a code point —
|
|
24
|
+
* `String.slice` counts UTF-16 units, so a route with one multi-byte character
|
|
25
|
+
* can be ≤96 chars and still exceed the 96-BYTE index limit.
|
|
26
|
+
*/
|
|
27
|
+
function truncateUtf8(value, maxBytes) {
|
|
28
|
+
const encoded = new TextEncoder().encode(value);
|
|
29
|
+
if (encoded.length <= maxBytes)
|
|
30
|
+
return value;
|
|
31
|
+
return new TextDecoder('utf-8', { fatal: false }).decode(encoded.subarray(0, maxBytes)).replace(
|
|
32
|
+
// A cut through a multi-byte sequence decodes to U+FFFD; drop that trailing artefact.
|
|
33
|
+
/�+$/, '');
|
|
34
|
+
}
|
|
35
|
+
const num = (value) => typeof value === 'number' && Number.isFinite(value) ? value : -1;
|
|
36
|
+
/** The documented layout above, as data. Exported so the spec asserts the wire format itself. */
|
|
37
|
+
export function toDataPoint(event) {
|
|
38
|
+
const route = event.route ?? '';
|
|
39
|
+
return {
|
|
40
|
+
indexes: [truncateUtf8(route, INDEX_MAX_BYTES)],
|
|
41
|
+
blobs: [event.method ?? '', route, event.userId ?? ''],
|
|
42
|
+
doubles: [num(event.status), num(event.durationMs), num(event.bytesOut)],
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Build the sink for whatever platform this process is on.
|
|
47
|
+
*
|
|
48
|
+
* Throws when NEITHER backend is available, on purpose. "An app with no
|
|
49
|
+
* `request_metrics` is not quiet; it is unmeasured" — and a sink that silently
|
|
50
|
+
* drops every event is exactly the failure this seam exists to end. A misbound
|
|
51
|
+
* Worker should fail at construction, in the deploy, rather than serve traffic
|
|
52
|
+
* that nothing can see.
|
|
53
|
+
*/
|
|
54
|
+
export function createTelemetrySink(opts) {
|
|
55
|
+
const { analytics, db, flushIntervalMs, maxSize } = opts;
|
|
56
|
+
if (analytics) {
|
|
57
|
+
// Analytics Engine writes are fire-and-forget and already off the response
|
|
58
|
+
// path — there is nothing to buffer, so flush/stop are honestly no-ops.
|
|
59
|
+
return {
|
|
60
|
+
record(event) {
|
|
61
|
+
analytics.writeDataPoint(toDataPoint(event));
|
|
62
|
+
},
|
|
63
|
+
flush() { },
|
|
64
|
+
stop() { },
|
|
65
|
+
get size() {
|
|
66
|
+
return 0;
|
|
67
|
+
},
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
if (!db) {
|
|
71
|
+
throw new Error('createTelemetrySink: neither an Analytics Engine binding (`analytics`) nor a `db` was ' +
|
|
72
|
+
'provided. Telemetry would be silently dropped; bind one of the two.');
|
|
73
|
+
}
|
|
74
|
+
// The unchanged SQLite path — same module, same batching, same rows.
|
|
75
|
+
return createMetricsBuffer({ db, flushIntervalMs, maxSize });
|
|
76
|
+
}
|
|
@@ -4,6 +4,7 @@ export { errorEnvelope } from './errorEnvelope';
|
|
|
4
4
|
export { crossOriginRefusal } from './localOrigin';
|
|
5
5
|
export { clientIp, createRateLimiter, type RateLimitOpts } from './rateLimit';
|
|
6
6
|
export { type RequestLoggerOpts, requestLogger } from './requestLogger';
|
|
7
|
+
export { type AnalyticsEngineDataset, createTelemetrySink, type TelemetryEvent, type TelemetrySink, type TelemetrySinkOptions, } from '../metrics/telemetrySink';
|
|
7
8
|
export { type CspDirectives, type SecurityHeadersOpts, securityHeaders, } from './securityHeaders';
|
|
8
9
|
export { toValidationIssues, VALIDATION_ERROR_CODE, type ValidationIssue, validationEnvelope, } from '../errors';
|
|
9
10
|
export { type ZValidatorTarget, zValidator } from './zValidator';
|
|
@@ -5,6 +5,9 @@ export { errorEnvelope } from './errorEnvelope';
|
|
|
5
5
|
export { crossOriginRefusal } from './localOrigin';
|
|
6
6
|
export { clientIp, createRateLimiter } from './rateLimit';
|
|
7
7
|
export { requestLogger } from './requestLogger';
|
|
8
|
+
// The telemetry seam `requestLogger` writes through — re-exported here so a caller
|
|
9
|
+
// that reaches `./middleware` for the logger can bind Analytics Engine in the same import.
|
|
10
|
+
export { createTelemetrySink, } from '../metrics/telemetrySink';
|
|
8
11
|
export { securityHeaders, } from './securityHeaders';
|
|
9
12
|
export { toValidationIssues, VALIDATION_ERROR_CODE, validationEnvelope, } from '../errors';
|
|
10
13
|
export { zValidator } from './zValidator';
|
|
@@ -1,29 +1,48 @@
|
|
|
1
1
|
import type { Database } from 'bun:sqlite';
|
|
2
2
|
import type { MiddlewareHandler } from 'hono';
|
|
3
3
|
import type { CursedbeltEnv } from '../context';
|
|
4
|
-
import {
|
|
4
|
+
import type { MetricsBuffer } from '../metrics/metricsBuffer';
|
|
5
|
+
import { type AnalyticsEngineDataset, type TelemetrySink } from '../metrics/telemetrySink';
|
|
5
6
|
/**
|
|
6
|
-
* Structured request/response logging. After each request it records one
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
7
|
+
* Structured request/response logging. After each request it records one
|
|
8
|
+
* telemetry event through the {@link TelemetrySink} (off the hot path) and, for
|
|
9
|
+
* errors (status ≥ 400) or slow requests, one `event_logs` row tagged with the
|
|
10
|
+
* correlation id. Attributes rows to `c.get('userId')` when the app's auth
|
|
10
11
|
* middleware has set it.
|
|
11
12
|
*
|
|
13
|
+
* 🔴 **Where the telemetry LANDS is the sink's decision, not this file's.** With
|
|
14
|
+
* an Analytics Engine binding it is one `writeDataPoint` and **zero** database
|
|
15
|
+
* rows; without one it is the same batched `request_metrics` insert it has always
|
|
16
|
+
* been. See `../metrics/telemetrySink.ts` for why that is the only choice offered.
|
|
17
|
+
*
|
|
12
18
|
* It records metrics even when a downstream handler THROWS: the error is caught,
|
|
13
|
-
* the would-be status derived, the
|
|
19
|
+
* the would-be status derived, the event recorded, then the error re-thrown so the
|
|
14
20
|
* error-envelope `onError` still produces the response.
|
|
15
21
|
*/
|
|
16
22
|
export interface RequestLoggerOpts {
|
|
17
23
|
/**
|
|
18
|
-
* The
|
|
19
|
-
*
|
|
20
|
-
|
|
21
|
-
|
|
24
|
+
* The Analytics Engine binding, when the platform supplies one. Present ⇒ the
|
|
25
|
+
* telemetry goes there and NOTHING is written to `request_metrics`.
|
|
26
|
+
*/
|
|
27
|
+
analytics?: AnalyticsEngineDataset | null;
|
|
28
|
+
/**
|
|
29
|
+
* The sink to write through. Provide one (and own its lifecycle via
|
|
30
|
+
* `sink.stop()`) to share it / flush deterministically; omit to build one from
|
|
31
|
+
* `db` + `analytics`. A {@link MetricsBuffer} is a valid sink.
|
|
22
32
|
*/
|
|
33
|
+
sink?: TelemetrySink;
|
|
34
|
+
/** @deprecated Use {@link RequestLoggerOpts.sink} — a `MetricsBuffer` is one. Kept for callers that predate the sink. */
|
|
23
35
|
buffer?: MetricsBuffer;
|
|
24
36
|
/** Duration (ms) at/above which a 2xx request also logs an `event_logs` row. Default: 2000. */
|
|
25
37
|
slowMs?: number;
|
|
26
38
|
/** Also write `event_logs` rows for status ≥ 400. Default: true. */
|
|
27
39
|
logErrors?: boolean;
|
|
28
40
|
}
|
|
29
|
-
|
|
41
|
+
/**
|
|
42
|
+
* @param db The `request_metrics` / `event_logs` database. Pass `null` on a
|
|
43
|
+
* platform that has no `bun:sqlite` — a Worker with `analytics` bound — in
|
|
44
|
+
* which case `event_logs` is not written either, because there is nowhere to
|
|
45
|
+
* put it. `analytics` or `sink` is then mandatory; `createTelemetrySink`
|
|
46
|
+
* throws rather than drop telemetry silently.
|
|
47
|
+
*/
|
|
48
|
+
export declare function requestLogger(db: Database | null, opts?: RequestLoggerOpts): MiddlewareHandler<CursedbeltEnv>;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { HTTPException } from 'hono/http-exception';
|
|
2
2
|
import { isApiError } from '../errors';
|
|
3
|
-
import { createMetricsBuffer } from '../metrics/metricsBuffer';
|
|
4
3
|
import { normalizeRoute } from '../metrics/normalizeRoute';
|
|
4
|
+
import { createTelemetrySink, } from '../metrics/telemetrySink';
|
|
5
5
|
function statusOfThrown(err) {
|
|
6
6
|
if (isApiError(err))
|
|
7
7
|
return err.status;
|
|
@@ -9,13 +9,22 @@ function statusOfThrown(err) {
|
|
|
9
9
|
return err.status;
|
|
10
10
|
return 500;
|
|
11
11
|
}
|
|
12
|
+
/**
|
|
13
|
+
* @param db The `request_metrics` / `event_logs` database. Pass `null` on a
|
|
14
|
+
* platform that has no `bun:sqlite` — a Worker with `analytics` bound — in
|
|
15
|
+
* which case `event_logs` is not written either, because there is nowhere to
|
|
16
|
+
* put it. `analytics` or `sink` is then mandatory; `createTelemetrySink`
|
|
17
|
+
* throws rather than drop telemetry silently.
|
|
18
|
+
*/
|
|
12
19
|
export function requestLogger(db, opts = {}) {
|
|
13
|
-
const
|
|
20
|
+
const sink = opts.sink ?? opts.buffer ?? createTelemetrySink({ db, analytics: opts.analytics ?? null });
|
|
14
21
|
const slowMs = opts.slowMs ?? 2000;
|
|
15
22
|
const logErrors = opts.logErrors ?? true;
|
|
16
|
-
const insertEvent = db
|
|
23
|
+
const insertEvent = db
|
|
24
|
+
? db.prepare(`INSERT INTO event_logs
|
|
17
25
|
(id, correlation_id, event_name, type, message, details, duration_ms, user_id, created_at)
|
|
18
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
26
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
27
|
+
: null;
|
|
19
28
|
return async (c, next) => {
|
|
20
29
|
const start = performance.now();
|
|
21
30
|
let thrown;
|
|
@@ -33,10 +42,10 @@ export function requestLogger(db, opts = {}) {
|
|
|
33
42
|
const lenHeader = didThrow ? null : c.res.headers.get('content-length');
|
|
34
43
|
const bytesOut = lenHeader ? Number(lenHeader) : null;
|
|
35
44
|
const userId = c.get('userId') ?? null;
|
|
36
|
-
|
|
45
|
+
sink.record({ ts: Date.now(), method, route, status, durationMs, bytesOut, userId });
|
|
37
46
|
const isError = status >= 400;
|
|
38
47
|
const isSlow = durationMs >= slowMs;
|
|
39
|
-
if ((logErrors && isError) || isSlow) {
|
|
48
|
+
if (insertEvent && ((logErrors && isError) || isSlow)) {
|
|
40
49
|
const type = status >= 500 ? 'error' : isError ? 'warn' : 'info';
|
|
41
50
|
insertEvent.run(crypto.randomUUID(), c.get('correlationId') ?? null, isError ? 'http_error' : 'http_slow', type, `${method} ${route} → ${status}`, JSON.stringify({ method, route, status, durationMs: Math.round(durationMs), bytesOut }), Math.round(durationMs), userId, new Date().toISOString());
|
|
42
51
|
}
|
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
import { Hono } from "hono";
|
|
12
12
|
import type { OpLog } from "./opLog";
|
|
13
13
|
import type { ApplyOne, ApplyReport, RemoteApi } from "./types";
|
|
14
|
+
import { type SyncSignalHub } from "./signal";
|
|
14
15
|
/**
|
|
15
16
|
* Trim a page to the byte budget, keeping it a PREFIX so `cursor` stays the seq
|
|
16
17
|
* of the last op actually returned. Anything else silently drops ops: the caller
|
|
@@ -26,8 +27,6 @@ export interface ReceiverHooks {
|
|
|
26
27
|
/** Every authenticated hit — the only evidence a non-dialing half has that it is
|
|
27
28
|
* in step (vault's `lastPeerContactAt` lesson). */
|
|
28
29
|
onPeerContact?: (peerId: string) => void;
|
|
29
|
-
/** The "Sync now was pressed HERE" stamp the dialer polls. */
|
|
30
|
-
readRequest?: () => number | null;
|
|
31
30
|
}
|
|
32
31
|
export interface ReceiverOptions {
|
|
33
32
|
log: OpLog;
|
|
@@ -58,8 +57,26 @@ export interface ReceiverOptions {
|
|
|
58
57
|
*/
|
|
59
58
|
afterBatch?: (report: ApplyReport) => void | Promise<void>;
|
|
60
59
|
hooks?: ReceiverHooks;
|
|
60
|
+
/**
|
|
61
|
+
* Mount `GET /signal` — the long-lived stream that replaced the dialer's
|
|
62
|
+
* 20-second `GET /requested` probe. Omitted ⇒ the route does not exist, the
|
|
63
|
+
* orch-companion idiom: nothing to probe, nothing to authenticate against.
|
|
64
|
+
*
|
|
65
|
+
* Call {@link SyncSignalHub.announce} after a local write on THIS half and the
|
|
66
|
+
* dialing half reconciles within the round trip. See `./signal.ts`.
|
|
67
|
+
*/
|
|
68
|
+
signal?: SyncSignalHub;
|
|
61
69
|
}
|
|
62
|
-
/**
|
|
70
|
+
/**
|
|
71
|
+
* Build the receiver: GET /info, GET /pull, POST /push.
|
|
72
|
+
*
|
|
73
|
+
* 🔴 There was a fourth route, `GET /requested`, and it is gone (2026-09-15). It
|
|
74
|
+
* served a single integer — "was Sync now pressed here?" — and existed only to be
|
|
75
|
+
* asked, every 20 seconds, by a dialer that otherwise had nothing to say. That made
|
|
76
|
+
* it **68 % of `vault`'s entire traffic**. A receiver with news now says so over
|
|
77
|
+
* {@link import("./signal")} instead of waiting to be asked; deleting the route is
|
|
78
|
+
* what stops a stale consumer from keeping the poll alive against a new build.
|
|
79
|
+
*/
|
|
63
80
|
export declare function createSyncReceiver(options: ReceiverOptions): Hono;
|
|
64
81
|
/**
|
|
65
82
|
* The fetch-backed remote the dialer binds to. `basePath` is where the peer
|
package/dist/server/sync/http.js
CHANGED
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
*/
|
|
11
11
|
import { Hono } from "hono";
|
|
12
12
|
import { createSyncApplier } from "./engine";
|
|
13
|
+
import { signalResponse } from "./signal";
|
|
13
14
|
const noStore = { "cache-control": "no-store" };
|
|
14
15
|
const SYNC_PAGE = 500;
|
|
15
16
|
/**
|
|
@@ -52,7 +53,16 @@ function peerIdOf(header, query) {
|
|
|
52
53
|
const raw = query ?? header ?? "";
|
|
53
54
|
return /^[0-9A-Za-z_-]{1,64}$/.test(raw) ? raw : "peer";
|
|
54
55
|
}
|
|
55
|
-
/**
|
|
56
|
+
/**
|
|
57
|
+
* Build the receiver: GET /info, GET /pull, POST /push.
|
|
58
|
+
*
|
|
59
|
+
* 🔴 There was a fourth route, `GET /requested`, and it is gone (2026-09-15). It
|
|
60
|
+
* served a single integer — "was Sync now pressed here?" — and existed only to be
|
|
61
|
+
* asked, every 20 seconds, by a dialer that otherwise had nothing to say. That made
|
|
62
|
+
* it **68 % of `vault`'s entire traffic**. A receiver with news now says so over
|
|
63
|
+
* {@link import("./signal")} instead of waiting to be asked; deleting the route is
|
|
64
|
+
* what stops a stale consumer from keeping the poll alive against a new build.
|
|
65
|
+
*/
|
|
56
66
|
export function createSyncReceiver(options) {
|
|
57
67
|
const { log, verifyToken } = options;
|
|
58
68
|
if ((options.applyOne === undefined) === (options.beginBatch === undefined)) {
|
|
@@ -84,7 +94,15 @@ export function createSyncReceiver(options) {
|
|
|
84
94
|
};
|
|
85
95
|
return c.json(body, 200, noStore);
|
|
86
96
|
});
|
|
87
|
-
|
|
97
|
+
const signal = options.signal;
|
|
98
|
+
if (signal !== undefined) {
|
|
99
|
+
api.get("/signal", (c) => signalResponse(signal, {
|
|
100
|
+
// The catch-up frame: a client reconnecting after a sleep learns the
|
|
101
|
+
// current head without a round trip of its own.
|
|
102
|
+
initial: { head: log.head(), instanceId: log.instanceId() },
|
|
103
|
+
signal: c.req.raw.signal,
|
|
104
|
+
}));
|
|
105
|
+
}
|
|
88
106
|
api.get("/pull", (c) => {
|
|
89
107
|
const afterRaw = c.req.query("after");
|
|
90
108
|
const after = afterRaw !== undefined && /^\d+$/.test(afterRaw) ? Number(afterRaw) : 0;
|
|
@@ -227,17 +245,5 @@ export function createHttpRemote(options) {
|
|
|
227
245
|
throw new Error(`sync/push: ${r.status}`);
|
|
228
246
|
return await readJson(r, "sync/push");
|
|
229
247
|
},
|
|
230
|
-
async requestedAt() {
|
|
231
|
-
try {
|
|
232
|
-
const r = await request("/requested");
|
|
233
|
-
if (!r.ok)
|
|
234
|
-
return null;
|
|
235
|
-
const body = await readJson(r, "sync/requested");
|
|
236
|
-
return typeof body.requestedAt === "number" ? body.requestedAt : null;
|
|
237
|
-
}
|
|
238
|
-
catch {
|
|
239
|
-
return null;
|
|
240
|
-
}
|
|
241
|
-
},
|
|
242
248
|
};
|
|
243
249
|
}
|
|
@@ -34,8 +34,16 @@ export { createOpLog, type OpLog } from "./opLog";
|
|
|
34
34
|
export { createTokenStore, type DeviceToken, type TokenStore } from "./tokens";
|
|
35
35
|
export { createSyncApplier, pushToRemote, pullFromRemote, runSync, type EngineOptions, type ExportPolicy, type PeerRefusal, } from "./engine";
|
|
36
36
|
export { capPageBytes, createHttpRemote, createSyncReceiver, type ReceiverHooks, type ReceiverOptions, } from "./http";
|
|
37
|
-
export { DEFAULT_LOOP, isUnreachable, planNextSync, type SyncLoopConfig, type SyncLoopState, } from "./planner";
|
|
37
|
+
export { DEFAULT_LOOP, isUnreachable, planNextSync, type SyncLoopConfig, type SyncLoopState, type SyncPlan, } from "./planner";
|
|
38
|
+
export { SIGNAL_SSE_HEADERS, connectSyncSignal, createSyncSignalHub, encodeNewsFrame, parseNewsFrame, signalResponse, signalStream, type SignalClient, type SignalClientOptions, type SignalStreamOptions, type SyncNews, type SyncSignalHub, } from "./signal";
|
|
38
39
|
export { OFFLINE_STATUS, PEER_CONTACT_WRITE_MS, createSyncStatusReporter, type SyncStatus, type SyncStatusReporter, type SyncStatusStore, } from "./status";
|
|
39
|
-
|
|
40
|
+
/**
|
|
41
|
+
* 🔴 `REQUEST_POLL_MS` was exported from here until 2026-09-15 and is GONE — that
|
|
42
|
+
* removal is why this package went to 2.0.0. It was a 20-second dial at a configured
|
|
43
|
+
* peer, and being *public API* is what made it dangerous: one more consumer and
|
|
44
|
+
* deleting it would have been a breaking change rather than an edit. `./timer`'s
|
|
45
|
+
* header has the ruling and the measurement; `./signal` is what replaced it.
|
|
46
|
+
*/
|
|
47
|
+
export { startSyncTimer, type SyncTimerDeps, type SyncTimerHandle, type WakeReason } from "./timer";
|
|
40
48
|
export { createSyncAlarm, type AlarmOutcome, type SyncAlarm, type SyncAlarmOptions } from "./alarm";
|
|
41
49
|
export { DEFAULT_HOLD_MS, createCommandQueue, createCommandReceiver, createCommandRemote, startCommandWorker, type CommandQueue, type CommandRemote, type CommandRoutesOptions, type CommandStatus, type CommandWorkerDeps, type CommandWorkerHandle, type SyncCommand, } from "./commands";
|
|
@@ -35,7 +35,15 @@ export { createTokenStore } from "./tokens";
|
|
|
35
35
|
export { createSyncApplier, pushToRemote, pullFromRemote, runSync, } from "./engine";
|
|
36
36
|
export { capPageBytes, createHttpRemote, createSyncReceiver, } from "./http";
|
|
37
37
|
export { DEFAULT_LOOP, isUnreachable, planNextSync, } from "./planner";
|
|
38
|
+
export { SIGNAL_SSE_HEADERS, connectSyncSignal, createSyncSignalHub, encodeNewsFrame, parseNewsFrame, signalResponse, signalStream, } from "./signal";
|
|
38
39
|
export { OFFLINE_STATUS, PEER_CONTACT_WRITE_MS, createSyncStatusReporter, } from "./status";
|
|
39
|
-
|
|
40
|
+
/**
|
|
41
|
+
* 🔴 `REQUEST_POLL_MS` was exported from here until 2026-09-15 and is GONE — that
|
|
42
|
+
* removal is why this package went to 2.0.0. It was a 20-second dial at a configured
|
|
43
|
+
* peer, and being *public API* is what made it dangerous: one more consumer and
|
|
44
|
+
* deleting it would have been a breaking change rather than an edit. `./timer`'s
|
|
45
|
+
* header has the ruling and the measurement; `./signal` is what replaced it.
|
|
46
|
+
*/
|
|
47
|
+
export { startSyncTimer } from "./timer";
|
|
40
48
|
export { createSyncAlarm } from "./alarm";
|
|
41
49
|
export { DEFAULT_HOLD_MS, createCommandQueue, createCommandReceiver, createCommandRemote, startCommandWorker, } from "./commands";
|
|
@@ -1,11 +1,30 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* The pure loop planner
|
|
3
|
-
*
|
|
4
|
-
*
|
|
2
|
+
* The pure loop planner: back off after a failure, else honor a debounced dirty
|
|
3
|
+
* flag, else **do not wake at all**. Pure so the timer is a thin wrapper and this
|
|
4
|
+
* is fake-clock testable.
|
|
5
|
+
*
|
|
6
|
+
* ── 🔴 There is no steady interval, and that is the whole point ──────────────────
|
|
7
|
+
*
|
|
8
|
+
* This used to end with `else: the steady interval` — `intervalMs: 5 * 60_000`, a
|
|
9
|
+
* dial at a configured peer every five minutes whether or not either side had
|
|
10
|
+
* anything to say. The owner ruled that out on 2026-09-15: *"There should be no
|
|
11
|
+
* polling in cb unless you can make some good case for it that beats the api
|
|
12
|
+
* option."* No such case was found, so the field is gone rather than defaulted to
|
|
13
|
+
* zero — a config key that reintroduces a poll is a config key somebody sets.
|
|
14
|
+
*
|
|
15
|
+
* What is left are the two wake-ups that have a REASON, and neither is a poll:
|
|
16
|
+
*
|
|
17
|
+
* · **debounce** — there are local ops to push. The side that has news says so;
|
|
18
|
+
* the wait only coalesces a burst.
|
|
19
|
+
* · **backoff** — the last attempt failed. This runs *only while disconnected*,
|
|
20
|
+
* never against a healthy peer, which is exactly the carve-out the ruling
|
|
21
|
+
* names for a reconnect timer.
|
|
22
|
+
*
|
|
23
|
+
* A clean, healthy loop returns `waitMs: null` — "nothing to do, do not book a
|
|
24
|
+
* timer" — and the process goes quiet until something happens. News from the far
|
|
25
|
+
* side arrives over `./signal`, not by asking.
|
|
5
26
|
*/
|
|
6
27
|
export interface SyncLoopConfig {
|
|
7
|
-
/** Steady poll cadence when reachable and idle. */
|
|
8
|
-
intervalMs: number;
|
|
9
28
|
/** How long after a local write to wait before syncing, so a burst coalesces. */
|
|
10
29
|
debounceMs: number;
|
|
11
30
|
/** First backoff step after a failure. */
|
|
@@ -22,10 +41,21 @@ export interface SyncLoopState {
|
|
|
22
41
|
/** When the oldest un-synced local write happened, or null if clean. */
|
|
23
42
|
dirtySince: number | null;
|
|
24
43
|
}
|
|
25
|
-
|
|
44
|
+
/**
|
|
45
|
+
* What the loop should do next.
|
|
46
|
+
*
|
|
47
|
+
* 🔴 `waitMs: null` means **do not schedule anything** — not "wait zero" and not
|
|
48
|
+
* "wait forever". It is the idle state, and a caller that turns it into a number
|
|
49
|
+
* has put the poll back.
|
|
50
|
+
*/
|
|
51
|
+
export interface SyncPlan {
|
|
26
52
|
runNow: boolean;
|
|
27
|
-
|
|
28
|
-
|
|
53
|
+
/** Milliseconds until the next wake-up, or `null` when there is no reason to wake. */
|
|
54
|
+
waitMs: number | null;
|
|
55
|
+
/** Why the loop will wake — `null` alongside `waitMs: null`. */
|
|
56
|
+
reason: "retry" | "dirty" | null;
|
|
57
|
+
}
|
|
58
|
+
export declare function planNextSync(state: SyncLoopState, now: number, cfg?: SyncLoopConfig): SyncPlan;
|
|
29
59
|
/** A peer that is asleep (the nightly EC2 window) or an internet-less Mac is NORMAL,
|
|
30
60
|
* not an error. It still feeds the backoff. */
|
|
31
61
|
export declare function isUnreachable(err: unknown): boolean;
|
|
@@ -1,10 +1,30 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* The pure loop planner
|
|
3
|
-
*
|
|
4
|
-
*
|
|
2
|
+
* The pure loop planner: back off after a failure, else honor a debounced dirty
|
|
3
|
+
* flag, else **do not wake at all**. Pure so the timer is a thin wrapper and this
|
|
4
|
+
* is fake-clock testable.
|
|
5
|
+
*
|
|
6
|
+
* ── 🔴 There is no steady interval, and that is the whole point ──────────────────
|
|
7
|
+
*
|
|
8
|
+
* This used to end with `else: the steady interval` — `intervalMs: 5 * 60_000`, a
|
|
9
|
+
* dial at a configured peer every five minutes whether or not either side had
|
|
10
|
+
* anything to say. The owner ruled that out on 2026-09-15: *"There should be no
|
|
11
|
+
* polling in cb unless you can make some good case for it that beats the api
|
|
12
|
+
* option."* No such case was found, so the field is gone rather than defaulted to
|
|
13
|
+
* zero — a config key that reintroduces a poll is a config key somebody sets.
|
|
14
|
+
*
|
|
15
|
+
* What is left are the two wake-ups that have a REASON, and neither is a poll:
|
|
16
|
+
*
|
|
17
|
+
* · **debounce** — there are local ops to push. The side that has news says so;
|
|
18
|
+
* the wait only coalesces a burst.
|
|
19
|
+
* · **backoff** — the last attempt failed. This runs *only while disconnected*,
|
|
20
|
+
* never against a healthy peer, which is exactly the carve-out the ruling
|
|
21
|
+
* names for a reconnect timer.
|
|
22
|
+
*
|
|
23
|
+
* A clean, healthy loop returns `waitMs: null` — "nothing to do, do not book a
|
|
24
|
+
* timer" — and the process goes quiet until something happens. News from the far
|
|
25
|
+
* side arrives over `./signal`, not by asking.
|
|
5
26
|
*/
|
|
6
27
|
export const DEFAULT_LOOP = {
|
|
7
|
-
intervalMs: 5 * 60_000,
|
|
8
28
|
debounceMs: 3_000,
|
|
9
29
|
backoffBaseMs: 30_000,
|
|
10
30
|
backoffMaxMs: 30 * 60_000,
|
|
@@ -13,14 +33,18 @@ export function planNextSync(state, now, cfg = DEFAULT_LOOP) {
|
|
|
13
33
|
if (state.consecutiveFailures > 0) {
|
|
14
34
|
const step = Math.min(cfg.backoffMaxMs, cfg.backoffBaseMs * 2 ** (state.consecutiveFailures - 1));
|
|
15
35
|
const due = state.lastAttempt + step;
|
|
16
|
-
return now >= due
|
|
36
|
+
return now >= due
|
|
37
|
+
? { runNow: true, waitMs: 0, reason: "retry" }
|
|
38
|
+
: { runNow: false, waitMs: due - now, reason: "retry" };
|
|
17
39
|
}
|
|
18
40
|
if (state.dirtySince !== null) {
|
|
19
41
|
const due = state.dirtySince + cfg.debounceMs;
|
|
20
|
-
return now >= due
|
|
42
|
+
return now >= due
|
|
43
|
+
? { runNow: true, waitMs: 0, reason: "dirty" }
|
|
44
|
+
: { runNow: false, waitMs: due - now, reason: "dirty" };
|
|
21
45
|
}
|
|
22
|
-
|
|
23
|
-
return
|
|
46
|
+
// Clean and healthy: nothing to push, nothing to retry. Stay silent.
|
|
47
|
+
return { runNow: false, waitMs: null, reason: null };
|
|
24
48
|
}
|
|
25
49
|
/** A peer that is asleep (the nightly EC2 window) or an internet-less Mac is NORMAL,
|
|
26
50
|
* not an error. It still feeds the backoff. */
|