@dwk/websub 0.1.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +15 -0
- package/README.md +155 -0
- package/dist/config.d.ts +122 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +96 -0
- package/dist/config.js.map +1 -0
- package/dist/consumer.d.ts +39 -0
- package/dist/consumer.d.ts.map +1 -0
- package/dist/consumer.js +146 -0
- package/dist/consumer.js.map +1 -0
- package/dist/distribute.d.ts +109 -0
- package/dist/distribute.d.ts.map +1 -0
- package/dist/distribute.js +140 -0
- package/dist/distribute.js.map +1 -0
- package/dist/fetch.d.ts +28 -0
- package/dist/fetch.d.ts.map +1 -0
- package/dist/fetch.js +73 -0
- package/dist/fetch.js.map +1 -0
- package/dist/handler.d.ts +43 -0
- package/dist/handler.d.ts.map +1 -0
- package/dist/handler.js +127 -0
- package/dist/handler.js.map +1 -0
- package/dist/index.d.ts +32 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +29 -0
- package/dist/index.js.map +1 -0
- package/dist/log.d.ts +54 -0
- package/dist/log.d.ts.map +1 -0
- package/dist/log.js +52 -0
- package/dist/log.js.map +1 -0
- package/dist/queue.d.ts +38 -0
- package/dist/queue.d.ts.map +1 -0
- package/dist/queue.js +12 -0
- package/dist/queue.js.map +1 -0
- package/dist/safe-fetch.d.ts +101 -0
- package/dist/safe-fetch.d.ts.map +1 -0
- package/dist/safe-fetch.js +354 -0
- package/dist/safe-fetch.js.map +1 -0
- package/dist/store.d.ts +61 -0
- package/dist/store.d.ts.map +1 -0
- package/dist/store.js +110 -0
- package/dist/store.js.map +1 -0
- package/dist/validate.d.ts +67 -0
- package/dist/validate.d.ts.map +1 -0
- package/dist/validate.js +106 -0
- package/dist/validate.js.map +1 -0
- package/dist/verify.d.ts +85 -0
- package/dist/verify.d.ts.map +1 -0
- package/dist/verify.js +149 -0
- package/dist/verify.js.map +1 -0
- package/package.json +46 -0
- package/src/config.ts +199 -0
- package/src/consumer.ts +187 -0
- package/src/distribute.ts +257 -0
- package/src/fetch.ts +84 -0
- package/src/handler.ts +163 -0
- package/src/index.ts +98 -0
- package/src/log.ts +56 -0
- package/src/queue.ts +40 -0
- package/src/safe-fetch.ts +412 -0
- package/src/store.ts +190 -0
- package/src/validate.ts +179 -0
- package/src/verify.ts +229 -0
package/package.json
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@dwk/websub",
|
|
3
|
+
"version": "0.1.0-beta.0",
|
|
4
|
+
"description": "WebSub (W3C) hub: subscription store, intent verification, and signed content distribution.",
|
|
5
|
+
"keywords": [
|
|
6
|
+
"websub",
|
|
7
|
+
"pubsubhubbub",
|
|
8
|
+
"indieweb",
|
|
9
|
+
"feeds",
|
|
10
|
+
"cloudflare-workers"
|
|
11
|
+
],
|
|
12
|
+
"type": "module",
|
|
13
|
+
"license": "ISC",
|
|
14
|
+
"author": "David W. Keith <me@dwk.io>",
|
|
15
|
+
"homepage": "https://github.com/davidwkeith/workers/tree/main/packages/websub#readme",
|
|
16
|
+
"repository": {
|
|
17
|
+
"type": "git",
|
|
18
|
+
"url": "git+https://github.com/davidwkeith/workers.git",
|
|
19
|
+
"directory": "packages/websub"
|
|
20
|
+
},
|
|
21
|
+
"sideEffects": false,
|
|
22
|
+
"main": "./dist/index.js",
|
|
23
|
+
"types": "./dist/index.d.ts",
|
|
24
|
+
"exports": {
|
|
25
|
+
".": {
|
|
26
|
+
"types": "./dist/index.d.ts",
|
|
27
|
+
"import": "./dist/index.js"
|
|
28
|
+
}
|
|
29
|
+
},
|
|
30
|
+
"files": [
|
|
31
|
+
"dist",
|
|
32
|
+
"src",
|
|
33
|
+
"!src/**/*.test.ts"
|
|
34
|
+
],
|
|
35
|
+
"publishConfig": {
|
|
36
|
+
"access": "public"
|
|
37
|
+
},
|
|
38
|
+
"dependencies": {
|
|
39
|
+
"@dwk/log": "0.1.0-beta.0"
|
|
40
|
+
},
|
|
41
|
+
"scripts": {
|
|
42
|
+
"build": "tsc -p tsconfig.build.json",
|
|
43
|
+
"typecheck": "tsc -p tsconfig.json",
|
|
44
|
+
"clean": "rm -rf dist"
|
|
45
|
+
}
|
|
46
|
+
}
|
package/src/config.ts
ADDED
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@dwk/websub` — injected configuration and the Cloudflare `Env` fragment.
|
|
3
|
+
*
|
|
4
|
+
* Per the composition contract, a package never reads the global environment
|
|
5
|
+
* directly: all config (base URL, the topics this hub serves, lease bounds) is
|
|
6
|
+
* passed into {@link createWebSub}, so the hub can be instantiated multiple times
|
|
7
|
+
* and unit-tested in isolation. The Cloudflare bindings the hub needs are
|
|
8
|
+
* declared as a TypeScript `Env` fragment that the composed Worker's `Env` is a
|
|
9
|
+
* superset of. See `spec/composition-contract.md` and `spec/packages/websub.md`.
|
|
10
|
+
*
|
|
11
|
+
* @packageDocumentation
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { noopLogger, noopMetrics, type Logger, type Metrics } from "@dwk/log";
|
|
15
|
+
import type { D1Database, Queue } from "@cloudflare/workers-types";
|
|
16
|
+
import {
|
|
17
|
+
DEFAULT_SIGNATURE_ALGORITHM,
|
|
18
|
+
type SignatureAlgorithm,
|
|
19
|
+
} from "./distribute";
|
|
20
|
+
import type { FetchLike } from "./fetch";
|
|
21
|
+
import type { WebSubJob } from "./queue";
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Cloudflare bindings required by the hub handler and its queue consumer.
|
|
25
|
+
*
|
|
26
|
+
* The subscription store **MUST** be strongly consistent — D1 (session
|
|
27
|
+
* consistency), never KV: a stale or lost subscription is a correctness bug, not
|
|
28
|
+
* a safe-to-be-stale cache (`spec/non-functional-requirements.md`).
|
|
29
|
+
*/
|
|
30
|
+
export interface WebSubEnv {
|
|
31
|
+
/** D1 database backing the subscription table. */
|
|
32
|
+
readonly WEBSUB_DB: D1Database;
|
|
33
|
+
/** Queue for intent verification and content-distribution fan-out + retries. */
|
|
34
|
+
readonly WEBSUB_QUEUE: Queue<WebSubJob>;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** Configuration passed to {@link createWebSub}. */
|
|
38
|
+
export interface WebSubConfig {
|
|
39
|
+
/** Base URL of this hub; informational and used for self-advertisement. */
|
|
40
|
+
readonly baseUrl: string;
|
|
41
|
+
/**
|
|
42
|
+
* Absolute URL of the hub endpoint itself, advertised to subscribers in the
|
|
43
|
+
* `rel="hub"` `Link` header on each delivery. Defaults to {@link baseUrl};
|
|
44
|
+
* set it when the hub is mounted under a path prefix.
|
|
45
|
+
*/
|
|
46
|
+
readonly hubUrl?: string;
|
|
47
|
+
/**
|
|
48
|
+
* The topic URLs this hub will serve (the user's own feeds). A subscribe or
|
|
49
|
+
* publish request for any other topic is rejected. Compared by normalized URL
|
|
50
|
+
* (default ports and a trailing-slash-only path are folded). Provide either
|
|
51
|
+
* this or {@link isAllowedTopic}.
|
|
52
|
+
*/
|
|
53
|
+
readonly allowedTopics?: readonly string[];
|
|
54
|
+
/**
|
|
55
|
+
* Predicate deciding whether a topic URL is one this hub serves, for callers
|
|
56
|
+
* whose feed set is dynamic. Takes precedence over {@link allowedTopics}.
|
|
57
|
+
*/
|
|
58
|
+
readonly isAllowedTopic?: (topic: string) => boolean;
|
|
59
|
+
/** Minimum lease (seconds) the hub will grant. Default 300 (5 minutes). */
|
|
60
|
+
readonly minLeaseSeconds?: number;
|
|
61
|
+
/** Maximum lease (seconds) the hub will grant. Default 864000 (10 days). */
|
|
62
|
+
readonly maxLeaseSeconds?: number;
|
|
63
|
+
/**
|
|
64
|
+
* Lease granted when a subscriber omits `hub.lease_seconds`. Default 864000
|
|
65
|
+
* (10 days). Clamped into `[minLeaseSeconds, maxLeaseSeconds]`.
|
|
66
|
+
*/
|
|
67
|
+
readonly defaultLeaseSeconds?: number;
|
|
68
|
+
/**
|
|
69
|
+
* HMAC digest method for the `X-Hub-Signature` on signed deliveries (WebSub
|
|
70
|
+
* §8 permits `sha1`/`sha256`/`sha384`/`sha512`). WebSub has no per-request
|
|
71
|
+
* method parameter, so this is a hub-level choice; it defaults to the secure
|
|
72
|
+
* `sha256`. Set it to `sha1` **only** when a subscriber requires the legacy
|
|
73
|
+
* method for interop — SHA-1 is weaker and not the default for that reason.
|
|
74
|
+
*/
|
|
75
|
+
readonly signatureAlgorithm?: SignatureAlgorithm;
|
|
76
|
+
/**
|
|
77
|
+
* Media type to forward on distribution when the topic response declares no
|
|
78
|
+
* `Content-Type`. WebSub §7 requires the distribution `Content-Type` to
|
|
79
|
+
* correspond to the topic's, so the hub never fabricates a generic
|
|
80
|
+
* `application/octet-stream`. Set this to the type the hub's feeds are served
|
|
81
|
+
* as (e.g. `application/atom+xml`) so a topic that omits the header is still
|
|
82
|
+
* distributable; left unset, such a topic is refused rather than mislabeled.
|
|
83
|
+
*/
|
|
84
|
+
readonly defaultContentType?: string;
|
|
85
|
+
/** `fetch` implementation for verification/distribution; defaults to global `fetch`. */
|
|
86
|
+
readonly fetch?: FetchLike;
|
|
87
|
+
/** Logger; defaults to a no-op (see `@dwk/log`). */
|
|
88
|
+
readonly logger?: Logger;
|
|
89
|
+
/** Metrics sink; defaults to a no-op (see `@dwk/log`). */
|
|
90
|
+
readonly metrics?: Metrics;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/** Default lower bound on a granted lease (5 minutes). */
|
|
94
|
+
export const DEFAULT_MIN_LEASE_SECONDS = 300;
|
|
95
|
+
/** Default upper bound on a granted lease (10 days). */
|
|
96
|
+
export const DEFAULT_MAX_LEASE_SECONDS = 864_000;
|
|
97
|
+
|
|
98
|
+
/** Config with defaults resolved and the topic check normalized to a predicate. */
|
|
99
|
+
export interface ResolvedConfig {
|
|
100
|
+
readonly baseUrl: string;
|
|
101
|
+
readonly hubUrl: string;
|
|
102
|
+
readonly isAllowedTopic: (topic: string) => boolean;
|
|
103
|
+
readonly minLeaseSeconds: number;
|
|
104
|
+
readonly maxLeaseSeconds: number;
|
|
105
|
+
readonly defaultLeaseSeconds: number;
|
|
106
|
+
readonly signatureAlgorithm: SignatureAlgorithm;
|
|
107
|
+
/** Fallback distribution `Content-Type`; `undefined` refuses to mislabel. */
|
|
108
|
+
readonly defaultContentType?: string;
|
|
109
|
+
readonly fetch: FetchLike;
|
|
110
|
+
readonly logger: Logger;
|
|
111
|
+
readonly metrics: Metrics;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Normalize a topic URL for comparison: lowercase host, default port dropped,
|
|
116
|
+
* a bare `/` path elided, and fragment removed. Query strings are preserved
|
|
117
|
+
* (a feed may legitimately carry one). Returns `null` when `raw` is unparseable
|
|
118
|
+
* or not `http(s)`.
|
|
119
|
+
*/
|
|
120
|
+
export function normalizeTopic(raw: string): string | null {
|
|
121
|
+
let url: URL;
|
|
122
|
+
try {
|
|
123
|
+
url = new URL(raw);
|
|
124
|
+
} catch {
|
|
125
|
+
return null;
|
|
126
|
+
}
|
|
127
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
|
128
|
+
return null;
|
|
129
|
+
}
|
|
130
|
+
url.hash = "";
|
|
131
|
+
const path = url.pathname === "/" ? "" : url.pathname;
|
|
132
|
+
return `${url.protocol}//${url.host}${path}${url.search}`;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* Resolve {@link WebSubConfig} into {@link ResolvedConfig}, filling defaults and
|
|
137
|
+
* turning the topic allowlist into a single predicate.
|
|
138
|
+
*
|
|
139
|
+
* @throws when neither `allowedTopics` nor `isAllowedTopic` is supplied — a hub
|
|
140
|
+
* with no topics would accept subscriptions it can never serve.
|
|
141
|
+
*/
|
|
142
|
+
export function resolveConfig(config: WebSubConfig): ResolvedConfig {
|
|
143
|
+
const min = config.minLeaseSeconds ?? DEFAULT_MIN_LEASE_SECONDS;
|
|
144
|
+
const max = config.maxLeaseSeconds ?? DEFAULT_MAX_LEASE_SECONDS;
|
|
145
|
+
if (min > max) {
|
|
146
|
+
throw new Error(
|
|
147
|
+
`@dwk/websub: minLeaseSeconds (${min}) exceeds maxLeaseSeconds (${max}).`,
|
|
148
|
+
);
|
|
149
|
+
}
|
|
150
|
+
const fallback = config.defaultLeaseSeconds ?? DEFAULT_MAX_LEASE_SECONDS;
|
|
151
|
+
const defaultLeaseSeconds = Math.min(Math.max(fallback, min), max);
|
|
152
|
+
|
|
153
|
+
let isAllowedTopic: (topic: string) => boolean;
|
|
154
|
+
if (config.isAllowedTopic !== undefined) {
|
|
155
|
+
isAllowedTopic = config.isAllowedTopic;
|
|
156
|
+
} else if (config.allowedTopics !== undefined) {
|
|
157
|
+
const allowed = new Set<string>();
|
|
158
|
+
for (const topic of config.allowedTopics) {
|
|
159
|
+
const normalized = normalizeTopic(topic);
|
|
160
|
+
if (normalized !== null) {
|
|
161
|
+
allowed.add(normalized);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
isAllowedTopic = (topic) => {
|
|
165
|
+
const normalized = normalizeTopic(topic);
|
|
166
|
+
return normalized !== null && allowed.has(normalized);
|
|
167
|
+
};
|
|
168
|
+
} else {
|
|
169
|
+
throw new Error(
|
|
170
|
+
"@dwk/websub: configure allowedTopics or isAllowedTopic — a hub must " +
|
|
171
|
+
"know which topics it serves.",
|
|
172
|
+
);
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
return {
|
|
176
|
+
baseUrl: config.baseUrl,
|
|
177
|
+
hubUrl: config.hubUrl ?? config.baseUrl,
|
|
178
|
+
isAllowedTopic,
|
|
179
|
+
minLeaseSeconds: min,
|
|
180
|
+
maxLeaseSeconds: max,
|
|
181
|
+
defaultLeaseSeconds,
|
|
182
|
+
signatureAlgorithm:
|
|
183
|
+
config.signatureAlgorithm ?? DEFAULT_SIGNATURE_ALGORITHM,
|
|
184
|
+
...(config.defaultContentType !== undefined
|
|
185
|
+
? { defaultContentType: config.defaultContentType }
|
|
186
|
+
: {}),
|
|
187
|
+
fetch: config.fetch ?? ((input, init) => fetch(input, init)),
|
|
188
|
+
logger: config.logger ?? noopLogger,
|
|
189
|
+
metrics: config.metrics ?? noopMetrics,
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/** Clamp a requested lease into the hub's `[min, max]` bounds. */
|
|
194
|
+
export function clampLease(requested: number, config: ResolvedConfig): number {
|
|
195
|
+
return Math.min(
|
|
196
|
+
Math.max(requested, config.minLeaseSeconds),
|
|
197
|
+
config.maxLeaseSeconds,
|
|
198
|
+
);
|
|
199
|
+
}
|
package/src/consumer.ts
ADDED
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@dwk/websub` — the queue consumer.
|
|
3
|
+
*
|
|
4
|
+
* The hub's slow work runs here, off the request path, with the queue providing
|
|
5
|
+
* retries and backoff. Two job kinds flow through:
|
|
6
|
+
*
|
|
7
|
+
* - **verify** — issue the verification-of-intent GET; on a confirmed subscribe,
|
|
8
|
+
* write the subscription to the D1 store with its lease expiry, and on a
|
|
9
|
+
* confirmed unsubscribe, remove it. A subscription row is created only after
|
|
10
|
+
* verification succeeds, so an unverified callback never lands in the store.
|
|
11
|
+
* - **distribute** — prune expired leases, fetch the topic's current content, and
|
|
12
|
+
* fan it out (signed per-subscriber when a secret is set) to every active
|
|
13
|
+
* subscriber.
|
|
14
|
+
*
|
|
15
|
+
* A job whose store/fetch work throws — or a distribution that cannot fetch the
|
|
16
|
+
* topic — is retried; everything else is acked. See `spec/packages/websub.md`.
|
|
17
|
+
*
|
|
18
|
+
* @packageDocumentation
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
import { hostFromUrl, type LogFields } from "@dwk/log";
|
|
22
|
+
import type { ExecutionContext, MessageBatch } from "@cloudflare/workers-types";
|
|
23
|
+
import {
|
|
24
|
+
resolveConfig,
|
|
25
|
+
type ResolvedConfig,
|
|
26
|
+
type WebSubConfig,
|
|
27
|
+
type WebSubEnv,
|
|
28
|
+
} from "./config";
|
|
29
|
+
import { deliverToSubscriber, fetchTopicContent } from "./distribute";
|
|
30
|
+
import { WebSubLogEvent } from "./log";
|
|
31
|
+
import type { WebSubJob } from "./queue";
|
|
32
|
+
import { createD1SubscriptionStore, type SubscriptionStore } from "./store";
|
|
33
|
+
import { notifyDenial, verifyIntent } from "./verify";
|
|
34
|
+
|
|
35
|
+
/** A Queue consumer for WebSub verification and distribution jobs. */
|
|
36
|
+
export type WebSubQueueConsumer = (
|
|
37
|
+
batch: MessageBatch<WebSubJob>,
|
|
38
|
+
env: WebSubEnv,
|
|
39
|
+
ctx: ExecutionContext,
|
|
40
|
+
) => Promise<void>;
|
|
41
|
+
|
|
42
|
+
/** Extra wiring for the consumer, primarily to inject a store in tests. */
|
|
43
|
+
export interface ConsumerOptions {
|
|
44
|
+
/** Override the subscription store; defaults to a D1 store over `WEBSUB_DB`. */
|
|
45
|
+
readonly store?: SubscriptionStore;
|
|
46
|
+
/** Clock injection for deterministic tests; defaults to `Date.now`. */
|
|
47
|
+
readonly now?: () => number;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function emit(
|
|
51
|
+
config: ResolvedConfig,
|
|
52
|
+
level: "info" | "warn",
|
|
53
|
+
event: string,
|
|
54
|
+
fields?: LogFields,
|
|
55
|
+
): void {
|
|
56
|
+
config.logger[level](event, fields);
|
|
57
|
+
config.metrics.count(event, fields);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function resolveStore(
|
|
61
|
+
options: ConsumerOptions | undefined,
|
|
62
|
+
env: WebSubEnv,
|
|
63
|
+
): SubscriptionStore {
|
|
64
|
+
if (options?.store !== undefined) {
|
|
65
|
+
return options.store;
|
|
66
|
+
}
|
|
67
|
+
if (env.WEBSUB_DB === undefined) {
|
|
68
|
+
throw new Error("@dwk/websub: missing required binding WEBSUB_DB.");
|
|
69
|
+
}
|
|
70
|
+
return createD1SubscriptionStore(env.WEBSUB_DB);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Build the Queue consumer that performs intent verification and content
|
|
75
|
+
* distribution. Fails loudly if no store is configured (neither
|
|
76
|
+
* `options.store` nor the `WEBSUB_DB` binding).
|
|
77
|
+
*/
|
|
78
|
+
export function createWebSubQueueConsumer(
|
|
79
|
+
config: WebSubConfig,
|
|
80
|
+
options?: ConsumerOptions,
|
|
81
|
+
): WebSubQueueConsumer {
|
|
82
|
+
const resolved = resolveConfig(config);
|
|
83
|
+
const clock = options?.now ?? (() => Date.now());
|
|
84
|
+
|
|
85
|
+
return async (batch, env, _ctx) => {
|
|
86
|
+
const store = resolveStore(options, env);
|
|
87
|
+
|
|
88
|
+
for (const message of batch.messages) {
|
|
89
|
+
const job = message.body;
|
|
90
|
+
try {
|
|
91
|
+
if (job.kind === "verify") {
|
|
92
|
+
const result = await verifyIntent(job.callback, job.topic, {
|
|
93
|
+
mode: job.mode,
|
|
94
|
+
leaseSeconds:
|
|
95
|
+
job.mode === "subscribe" ? job.leaseSeconds : undefined,
|
|
96
|
+
fetch: resolved.fetch,
|
|
97
|
+
logger: resolved.logger,
|
|
98
|
+
metrics: resolved.metrics,
|
|
99
|
+
});
|
|
100
|
+
if (result.confirmed) {
|
|
101
|
+
if (job.mode === "subscribe") {
|
|
102
|
+
await store.upsert({
|
|
103
|
+
callback: job.callback,
|
|
104
|
+
topic: job.topic,
|
|
105
|
+
secret: job.secret,
|
|
106
|
+
leaseSeconds: job.leaseSeconds,
|
|
107
|
+
now: clock(),
|
|
108
|
+
});
|
|
109
|
+
emit(resolved, "info", WebSubLogEvent.SubscriptionActivated, {
|
|
110
|
+
callbackHost: hostFromUrl(job.callback),
|
|
111
|
+
topicHost: hostFromUrl(job.topic),
|
|
112
|
+
leaseSeconds: job.leaseSeconds,
|
|
113
|
+
});
|
|
114
|
+
} else {
|
|
115
|
+
await store.remove(job.callback, job.topic);
|
|
116
|
+
emit(resolved, "info", WebSubLogEvent.SubscriptionRemoved, {
|
|
117
|
+
callbackHost: hostFromUrl(job.callback),
|
|
118
|
+
topicHost: hostFromUrl(job.topic),
|
|
119
|
+
reason: "unsubscribed",
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
} else if (job.mode === "subscribe") {
|
|
123
|
+
// Intent unconfirmed: signal denial to the subscriber (WebSub §5.2)
|
|
124
|
+
// instead of silently dropping the request. (An unconfirmed
|
|
125
|
+
// unsubscribe simply leaves the existing subscription in place.)
|
|
126
|
+
await notifyDenial(job.callback, job.topic, {
|
|
127
|
+
reason: "verification_failed",
|
|
128
|
+
fetch: resolved.fetch,
|
|
129
|
+
logger: resolved.logger,
|
|
130
|
+
metrics: resolved.metrics,
|
|
131
|
+
});
|
|
132
|
+
}
|
|
133
|
+
message.ack();
|
|
134
|
+
continue;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
// kind === "distribute"
|
|
138
|
+
const now = clock();
|
|
139
|
+
await store.pruneExpired(now);
|
|
140
|
+
const fetched = await fetchTopicContent(job.topic, {
|
|
141
|
+
fetch: resolved.fetch,
|
|
142
|
+
logger: resolved.logger,
|
|
143
|
+
metrics: resolved.metrics,
|
|
144
|
+
defaultContentType: resolved.defaultContentType,
|
|
145
|
+
});
|
|
146
|
+
if (fetched.kind === "retry") {
|
|
147
|
+
// The topic was unreachable / non-2xx — retry the whole job later
|
|
148
|
+
// rather than dropping the push.
|
|
149
|
+
message.retry();
|
|
150
|
+
continue;
|
|
151
|
+
}
|
|
152
|
+
if (fetched.kind === "drop") {
|
|
153
|
+
// A permanent, deterministic refusal (e.g. an unlabelable topic):
|
|
154
|
+
// retrying can't fix it, so ack and move on rather than clog the
|
|
155
|
+
// queue and re-hammer the topic. fetchTopicContent already logged why.
|
|
156
|
+
message.ack();
|
|
157
|
+
continue;
|
|
158
|
+
}
|
|
159
|
+
const content = fetched.content;
|
|
160
|
+
const subscribers = await store.listActive(job.topic, now);
|
|
161
|
+
// Fan out in parallel: deliverToSubscriber never throws (it reports a
|
|
162
|
+
// failed/blocked POST as delivered:false), so one slow or dead callback
|
|
163
|
+
// must not head-of-line block the rest and risk timing out the whole
|
|
164
|
+
// consumer invocation.
|
|
165
|
+
await Promise.all(
|
|
166
|
+
subscribers.map((subscriber) =>
|
|
167
|
+
deliverToSubscriber(subscriber, content, resolved.hubUrl, {
|
|
168
|
+
fetch: resolved.fetch,
|
|
169
|
+
logger: resolved.logger,
|
|
170
|
+
metrics: resolved.metrics,
|
|
171
|
+
signatureAlgorithm: resolved.signatureAlgorithm,
|
|
172
|
+
}),
|
|
173
|
+
),
|
|
174
|
+
);
|
|
175
|
+
message.ack();
|
|
176
|
+
} catch (err) {
|
|
177
|
+
// A store/queue failure must not retry silently — name the kind and
|
|
178
|
+
// error so an operator can tell a transient blip from a wedged job.
|
|
179
|
+
emit(resolved, "warn", WebSubLogEvent.QueueRetry, {
|
|
180
|
+
kind: job.kind,
|
|
181
|
+
error: err instanceof Error ? err.name : "unknown",
|
|
182
|
+
});
|
|
183
|
+
message.retry();
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
};
|
|
187
|
+
}
|
|
@@ -0,0 +1,257 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@dwk/websub` — signed content distribution.
|
|
3
|
+
*
|
|
4
|
+
* On publish, the hub fetches the topic's current content and `POST`s it to every
|
|
5
|
+
* active subscriber's callback (WebSub §7). When a subscriber registered a
|
|
6
|
+
* `hub.secret`, the body is authenticated with an HMAC signature in the
|
|
7
|
+
* `X-Hub-Signature: <method>=<hex>` header so the subscriber can verify the
|
|
8
|
+
* delivery came from this hub (§8). The digest method is a hub-level config
|
|
9
|
+
* option (`sha256` by default; `sha1`/`sha384`/`sha512` are also permitted by
|
|
10
|
+
* §8 and selected via `signatureAlgorithm`). Deliveries carry `Link` headers
|
|
11
|
+
* advertising
|
|
12
|
+
* the hub (`rel="hub"`) and the topic (`rel="self"`). Every POST goes through
|
|
13
|
+
* {@link safeFetch}. See `spec/packages/websub.md`.
|
|
14
|
+
*
|
|
15
|
+
* @packageDocumentation
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
import {
|
|
19
|
+
hostFromUrl,
|
|
20
|
+
noopLogger,
|
|
21
|
+
noopMetrics,
|
|
22
|
+
type Logger,
|
|
23
|
+
type Metrics,
|
|
24
|
+
} from "@dwk/log";
|
|
25
|
+
import type { FetchLike } from "./fetch";
|
|
26
|
+
import { readBytesCapped } from "./fetch";
|
|
27
|
+
import { WebSubLogEvent } from "./log";
|
|
28
|
+
import { safeFetch } from "./safe-fetch";
|
|
29
|
+
import type { Subscription } from "./store";
|
|
30
|
+
|
|
31
|
+
/** A topic's current content, as fetched from the topic URL. */
|
|
32
|
+
export interface TopicContent {
|
|
33
|
+
/** The raw body bytes to forward to subscribers. */
|
|
34
|
+
readonly body: Uint8Array;
|
|
35
|
+
/** The topic's `Content-Type`, forwarded verbatim to subscribers. */
|
|
36
|
+
readonly contentType: string;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* The HMAC digest methods WebSub §8 permits for `X-Hub-Signature`. The method
|
|
41
|
+
* name is emitted verbatim as the header's `<method>=` prefix, so it must match
|
|
42
|
+
* the WebSub spelling (`sha1`, not `sha-1`).
|
|
43
|
+
*/
|
|
44
|
+
export type SignatureAlgorithm = "sha1" | "sha256" | "sha384" | "sha512";
|
|
45
|
+
|
|
46
|
+
/** Map a WebSub signature method name to its WebCrypto SHA hash name. */
|
|
47
|
+
const HASH_FOR_METHOD: Record<SignatureAlgorithm, string> = {
|
|
48
|
+
sha1: "SHA-1",
|
|
49
|
+
sha256: "SHA-256",
|
|
50
|
+
sha384: "SHA-384",
|
|
51
|
+
sha512: "SHA-512",
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
/** WebSub's secure default signature method; SHA-1 interop is opt-in only. */
|
|
55
|
+
export const DEFAULT_SIGNATURE_ALGORITHM: SignatureAlgorithm = "sha256";
|
|
56
|
+
|
|
57
|
+
/** Outcome of delivering content to one subscriber. */
|
|
58
|
+
export interface DeliveryResult {
|
|
59
|
+
readonly callback: string;
|
|
60
|
+
/** Whether the callback accepted the delivery (2xx). */
|
|
61
|
+
readonly delivered: boolean;
|
|
62
|
+
/** The delivery's HTTP status (`0` when the POST threw or was blocked). */
|
|
63
|
+
readonly status: number;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Compute the `X-Hub-Signature` value for `body` under `secret`:
|
|
68
|
+
* `<method>=<lowercase hex HMAC>` (WebSub §8). `method` defaults to the secure
|
|
69
|
+
* `sha256`; WebSub also permits `sha1`/`sha384`/`sha512`. The header prefix is
|
|
70
|
+
* the WebSub method name verbatim (e.g. `sha1=`, not `sha-1=`).
|
|
71
|
+
*/
|
|
72
|
+
export async function contentSignature(
|
|
73
|
+
secret: string,
|
|
74
|
+
body: Uint8Array,
|
|
75
|
+
method: SignatureAlgorithm = DEFAULT_SIGNATURE_ALGORITHM,
|
|
76
|
+
): Promise<string> {
|
|
77
|
+
const key = await crypto.subtle.importKey(
|
|
78
|
+
"raw",
|
|
79
|
+
new TextEncoder().encode(secret),
|
|
80
|
+
{ name: "HMAC", hash: HASH_FOR_METHOD[method] },
|
|
81
|
+
false,
|
|
82
|
+
["sign"],
|
|
83
|
+
);
|
|
84
|
+
const mac = await crypto.subtle.sign("HMAC", key, body as BufferSource);
|
|
85
|
+
const hex = [...new Uint8Array(mac)]
|
|
86
|
+
.map((b) => b.toString(16).padStart(2, "0"))
|
|
87
|
+
.join("");
|
|
88
|
+
return `${method}=${hex}`;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** Build the `Link` header advertising this hub and the topic (WebSub §5.1). */
|
|
92
|
+
export function buildLinkHeader(hubUrl: string, topic: string): string {
|
|
93
|
+
return `<${hubUrl}>; rel="hub", <${topic}>; rel="self"`;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** Inputs shared by distribution helpers. */
|
|
97
|
+
export interface DistributeOptions {
|
|
98
|
+
readonly fetch?: FetchLike;
|
|
99
|
+
readonly logger?: Logger;
|
|
100
|
+
readonly metrics?: Metrics;
|
|
101
|
+
/**
|
|
102
|
+
* HMAC method used for the `X-Hub-Signature` header. WebSub §8 has no
|
|
103
|
+
* per-request method parameter, so this is a hub-level choice; it defaults to
|
|
104
|
+
* the secure {@link DEFAULT_SIGNATURE_ALGORITHM} (`sha256`). Set it to `sha1`
|
|
105
|
+
* only for interop with subscribers that require the legacy method.
|
|
106
|
+
*/
|
|
107
|
+
readonly signatureAlgorithm?: SignatureAlgorithm;
|
|
108
|
+
/**
|
|
109
|
+
* Media type to forward when the topic response declares no `Content-Type`.
|
|
110
|
+
* WebSub §7 requires the distribution `Content-Type` to correspond to the
|
|
111
|
+
* topic's, so the hub never fabricates a generic `application/octet-stream`:
|
|
112
|
+
* when the topic omits the header and no fallback is configured here, the
|
|
113
|
+
* content is refused rather than mislabeled.
|
|
114
|
+
*/
|
|
115
|
+
readonly defaultContentType?: string;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Outcome of {@link fetchTopicContent}, telling the caller what to do with the
|
|
120
|
+
* queue message:
|
|
121
|
+
*
|
|
122
|
+
* - **`ok`** — the topic's current content, ready to fan out.
|
|
123
|
+
* - **`retry`** — a transient failure (topic unreachable, non-2xx, or an
|
|
124
|
+
* over-cap body that may be a truncated response): re-enqueue and try later.
|
|
125
|
+
* - **`drop`** — a deterministic refusal that re-fetching cannot fix, so the
|
|
126
|
+
* caller acks rather than burning retries and re-hammering the topic. Today
|
|
127
|
+
* this is a topic that declares no `Content-Type` and for which no
|
|
128
|
+
* {@link DistributeOptions.defaultContentType} fallback is configured, since
|
|
129
|
+
* forwarding it would mislabel the feed (WebSub §7).
|
|
130
|
+
*/
|
|
131
|
+
export type TopicFetchResult =
|
|
132
|
+
| { readonly kind: "ok"; readonly content: TopicContent }
|
|
133
|
+
| { readonly kind: "retry" }
|
|
134
|
+
| { readonly kind: "drop" };
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* Fetch the topic's current content through {@link safeFetch}, classifying the
|
|
138
|
+
* outcome as `ok` / `retry` / `drop` (see {@link TopicFetchResult}). A missing,
|
|
139
|
+
* unlabelable `Content-Type` is a `drop` — a permanent format/config error that
|
|
140
|
+
* retrying would only turn into a self-inflicted hammering of the topic — while
|
|
141
|
+
* unreachable/non-2xx/over-cap responses are transient `retry`s.
|
|
142
|
+
*/
|
|
143
|
+
export async function fetchTopicContent(
|
|
144
|
+
topic: string,
|
|
145
|
+
options?: DistributeOptions,
|
|
146
|
+
): Promise<TopicFetchResult> {
|
|
147
|
+
const doFetch: FetchLike =
|
|
148
|
+
options?.fetch ?? ((input, init) => fetch(input, init));
|
|
149
|
+
const logger = options?.logger ?? noopLogger;
|
|
150
|
+
const metrics = options?.metrics ?? noopMetrics;
|
|
151
|
+
|
|
152
|
+
let response: Response;
|
|
153
|
+
try {
|
|
154
|
+
const result = await safeFetch(
|
|
155
|
+
doFetch,
|
|
156
|
+
topic,
|
|
157
|
+
{ method: "GET" },
|
|
158
|
+
{ logger, metrics },
|
|
159
|
+
);
|
|
160
|
+
response = result.response;
|
|
161
|
+
} catch {
|
|
162
|
+
const fields = { topicHost: hostFromUrl(topic), status: 0 };
|
|
163
|
+
logger.warn(WebSubLogEvent.TopicFetchFailed, fields);
|
|
164
|
+
metrics.count(WebSubLogEvent.TopicFetchFailed, fields);
|
|
165
|
+
return { kind: "retry" };
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
if (!response.ok) {
|
|
169
|
+
await response.body?.cancel().catch(() => undefined);
|
|
170
|
+
const fields = { topicHost: hostFromUrl(topic), status: response.status };
|
|
171
|
+
logger.warn(WebSubLogEvent.TopicFetchFailed, fields);
|
|
172
|
+
metrics.count(WebSubLogEvent.TopicFetchFailed, fields);
|
|
173
|
+
return { kind: "retry" };
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
const contentType =
|
|
177
|
+
response.headers.get("content-type") ?? options?.defaultContentType;
|
|
178
|
+
if (contentType === undefined || contentType === "") {
|
|
179
|
+
// WebSub §7: the distribution Content-Type MUST correspond to the topic's.
|
|
180
|
+
// With neither a topic header nor a configured fallback, forwarding would
|
|
181
|
+
// mislabel the feed. Re-fetching can't conjure a Content-Type, so drop the
|
|
182
|
+
// job (the caller acks) rather than retry — retrying would only clog the
|
|
183
|
+
// queue and re-hammer the topic for a permanent configuration error.
|
|
184
|
+
await response.body?.cancel().catch(() => undefined);
|
|
185
|
+
const fields = { topicHost: hostFromUrl(topic), status: response.status };
|
|
186
|
+
logger.warn(WebSubLogEvent.TopicContentTypeMissing, fields);
|
|
187
|
+
metrics.count(WebSubLogEvent.TopicContentTypeMissing, fields);
|
|
188
|
+
return { kind: "drop" };
|
|
189
|
+
}
|
|
190
|
+
const body = await readBytesCapped(response);
|
|
191
|
+
if (body === null) {
|
|
192
|
+
const fields = { topicHost: hostFromUrl(topic), status: response.status };
|
|
193
|
+
logger.warn(WebSubLogEvent.TopicFetchFailed, fields);
|
|
194
|
+
metrics.count(WebSubLogEvent.TopicFetchFailed, fields);
|
|
195
|
+
return { kind: "retry" };
|
|
196
|
+
}
|
|
197
|
+
return { kind: "ok", content: { body, contentType } };
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/**
|
|
201
|
+
* Deliver `content` to one subscriber. POSTs the body with the topic's
|
|
202
|
+
* `Content-Type`, the hub/self `Link` header, and — when the subscription
|
|
203
|
+
* carries a secret — the `X-Hub-Signature`. Never throws: a failed or blocked
|
|
204
|
+
* POST is reported as `delivered: false`.
|
|
205
|
+
*/
|
|
206
|
+
export async function deliverToSubscriber(
|
|
207
|
+
subscription: Subscription,
|
|
208
|
+
content: TopicContent,
|
|
209
|
+
hubUrl: string,
|
|
210
|
+
options?: DistributeOptions,
|
|
211
|
+
): Promise<DeliveryResult> {
|
|
212
|
+
const doFetch: FetchLike =
|
|
213
|
+
options?.fetch ?? ((input, init) => fetch(input, init));
|
|
214
|
+
const logger = options?.logger ?? noopLogger;
|
|
215
|
+
const metrics = options?.metrics ?? noopMetrics;
|
|
216
|
+
|
|
217
|
+
const headers: Record<string, string> = {
|
|
218
|
+
"content-type": content.contentType,
|
|
219
|
+
link: buildLinkHeader(hubUrl, subscription.topic),
|
|
220
|
+
};
|
|
221
|
+
if (subscription.secret !== null) {
|
|
222
|
+
headers["x-hub-signature"] = await contentSignature(
|
|
223
|
+
subscription.secret,
|
|
224
|
+
content.body,
|
|
225
|
+
options?.signatureAlgorithm ?? DEFAULT_SIGNATURE_ALGORITHM,
|
|
226
|
+
);
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
const finish = (delivered: boolean, status: number): DeliveryResult => {
|
|
230
|
+
const fields = {
|
|
231
|
+
callbackHost: hostFromUrl(subscription.callback),
|
|
232
|
+
delivered,
|
|
233
|
+
status,
|
|
234
|
+
};
|
|
235
|
+
logger.info(WebSubLogEvent.DeliveryCompleted, fields);
|
|
236
|
+
metrics.count(WebSubLogEvent.DeliveryCompleted, fields);
|
|
237
|
+
return { callback: subscription.callback, delivered, status };
|
|
238
|
+
};
|
|
239
|
+
|
|
240
|
+
try {
|
|
241
|
+
const result = await safeFetch(
|
|
242
|
+
doFetch,
|
|
243
|
+
subscription.callback,
|
|
244
|
+
{
|
|
245
|
+
method: "POST",
|
|
246
|
+
headers,
|
|
247
|
+
// `content.body` is a Uint8Array; pass its backing buffer as the body.
|
|
248
|
+
body: content.body as BodyInit,
|
|
249
|
+
},
|
|
250
|
+
{ logger, metrics },
|
|
251
|
+
);
|
|
252
|
+
await result.response.body?.cancel().catch(() => undefined);
|
|
253
|
+
return finish(result.response.ok, result.response.status);
|
|
254
|
+
} catch {
|
|
255
|
+
return finish(false, 0);
|
|
256
|
+
}
|
|
257
|
+
}
|