@dwk/microsub 0.1.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +15 -0
- package/README.md +92 -0
- package/dist/auth.d.ts +53 -0
- package/dist/auth.d.ts.map +1 -0
- package/dist/auth.js +102 -0
- package/dist/auth.js.map +1 -0
- package/dist/config.d.ts +102 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +64 -0
- package/dist/config.js.map +1 -0
- package/dist/consumer.d.ts +40 -0
- package/dist/consumer.d.ts.map +1 -0
- package/dist/consumer.js +87 -0
- package/dist/consumer.js.map +1 -0
- package/dist/discovery.d.ts +59 -0
- package/dist/discovery.d.ts.map +1 -0
- package/dist/discovery.js +190 -0
- package/dist/discovery.js.map +1 -0
- package/dist/fetch.d.ts +28 -0
- package/dist/fetch.d.ts.map +1 -0
- package/dist/fetch.js +72 -0
- package/dist/fetch.js.map +1 -0
- package/dist/handler.d.ts +24 -0
- package/dist/handler.d.ts.map +1 -0
- package/dist/handler.js +434 -0
- package/dist/handler.js.map +1 -0
- package/dist/hfeed.d.ts +25 -0
- package/dist/hfeed.d.ts.map +1 -0
- package/dist/hfeed.js +252 -0
- package/dist/hfeed.js.map +1 -0
- package/dist/index.d.ts +39 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +32 -0
- package/dist/index.js.map +1 -0
- package/dist/jf2.d.ts +69 -0
- package/dist/jf2.d.ts.map +1 -0
- package/dist/jf2.js +295 -0
- package/dist/jf2.js.map +1 -0
- package/dist/log.d.ts +44 -0
- package/dist/log.d.ts.map +1 -0
- package/dist/log.js +42 -0
- package/dist/log.js.map +1 -0
- package/dist/poll.d.ts +22 -0
- package/dist/poll.d.ts.map +1 -0
- package/dist/poll.js +39 -0
- package/dist/poll.js.map +1 -0
- package/dist/queue.d.ts +25 -0
- package/dist/queue.d.ts.map +1 -0
- package/dist/queue.js +13 -0
- package/dist/queue.js.map +1 -0
- package/dist/replay.d.ts +34 -0
- package/dist/replay.d.ts.map +1 -0
- package/dist/replay.js +49 -0
- package/dist/replay.js.map +1 -0
- package/dist/safe-fetch.d.ts +86 -0
- package/dist/safe-fetch.d.ts.map +1 -0
- package/dist/safe-fetch.js +311 -0
- package/dist/safe-fetch.js.map +1 -0
- package/dist/store.d.ts +131 -0
- package/dist/store.d.ts.map +1 -0
- package/dist/store.js +393 -0
- package/dist/store.js.map +1 -0
- package/dist/xml.d.ts +51 -0
- package/dist/xml.d.ts.map +1 -0
- package/dist/xml.js +196 -0
- package/dist/xml.js.map +1 -0
- package/package.json +49 -0
- package/src/auth.ts +184 -0
- package/src/config.ts +156 -0
- package/src/consumer.ts +140 -0
- package/src/discovery.ts +270 -0
- package/src/fetch.ts +82 -0
- package/src/handler.ts +594 -0
- package/src/hfeed.ts +287 -0
- package/src/index.ts +86 -0
- package/src/jf2.ts +394 -0
- package/src/log.ts +46 -0
- package/src/poll.ts +72 -0
- package/src/queue.ts +26 -0
- package/src/replay.ts +68 -0
- package/src/safe-fetch.ts +346 -0
- package/src/store.ts +644 -0
- package/src/xml.ts +229 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
ISC License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 David W. Keith
|
|
4
|
+
|
|
5
|
+
Permission to use, copy, modify, and/or distribute this software for any
|
|
6
|
+
purpose with or without fee is hereby granted, provided that the above
|
|
7
|
+
copyright notice and this permission notice appear in all copies.
|
|
8
|
+
|
|
9
|
+
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
|
10
|
+
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
|
11
|
+
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
|
12
|
+
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
|
13
|
+
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
|
14
|
+
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
|
15
|
+
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
# `@dwk/microsub`
|
|
2
|
+
|
|
3
|
+
> Microsub server: channel/feed subscriptions, server-side polling, and a normalised JF2 timeline. Consumes IndieAuth tokens.
|
|
4
|
+
|
|
5
|
+
Part of the [`@dwk` IndieWeb + Solid cohort](../../README.md). See the
|
|
6
|
+
[package specification](../../spec/packages/microsub.md) for the full requirements.
|
|
7
|
+
|
|
8
|
+
A [Microsub](https://indieweb.org/Microsub-spec) server that runs as a
|
|
9
|
+
Cloudflare Worker — the IndieWeb's **read side**, completing the loop alongside
|
|
10
|
+
[`@dwk/micropub`](../micropub) (write), [`@dwk/webmention`](../webmention)
|
|
11
|
+
(interaction), [`@dwk/indieauth`](../indieauth) (identity), and
|
|
12
|
+
[`@dwk/websub`](../websub) (push).
|
|
13
|
+
|
|
14
|
+
It manages feed subscriptions organised into channels, polls and parses sources
|
|
15
|
+
server-side (Atom / RSS / JSON Feed / `h-feed`), and serves a normalised
|
|
16
|
+
[JF2](https://jf2.spec.indieweb.org/) timeline to reader clients (Monocle,
|
|
17
|
+
Together, Indigenous). The user's reading state lives on infrastructure they
|
|
18
|
+
own, not in a hosted aggregator.
|
|
19
|
+
|
|
20
|
+
## Usage
|
|
21
|
+
|
|
22
|
+
```ts
|
|
23
|
+
import {
|
|
24
|
+
createMicrosub,
|
|
25
|
+
createMicrosubPoller,
|
|
26
|
+
createMicrosubQueueConsumer,
|
|
27
|
+
} from "@dwk/microsub";
|
|
28
|
+
|
|
29
|
+
const config = {
|
|
30
|
+
baseUrl: "https://example.com",
|
|
31
|
+
// the owner's IndieAuth profile URL; tokens minted for any other `me` are
|
|
32
|
+
// rejected even if they carry the right scope
|
|
33
|
+
me: "https://example.com/",
|
|
34
|
+
// optional: defaults to `${origin}/microsub`
|
|
35
|
+
microsubEndpoint: "https://example.com/microsub",
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
const microsub = createMicrosub(config);
|
|
39
|
+
const poll = createMicrosubPoller(config);
|
|
40
|
+
const consume = createMicrosubQueueConsumer(config);
|
|
41
|
+
|
|
42
|
+
export default {
|
|
43
|
+
fetch(request, env, ctx) {
|
|
44
|
+
return microsub(request, env, ctx);
|
|
45
|
+
},
|
|
46
|
+
// Cron Trigger: enqueue a poll job per followed feed.
|
|
47
|
+
scheduled(controller, env, ctx) {
|
|
48
|
+
return poll(controller, env, ctx);
|
|
49
|
+
},
|
|
50
|
+
// Queue consumer: fetch + parse + append to channel timelines.
|
|
51
|
+
queue(batch, env, ctx) {
|
|
52
|
+
return consume(batch, env, ctx);
|
|
53
|
+
},
|
|
54
|
+
};
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
### Bindings (declared `Env` fragment)
|
|
58
|
+
|
|
59
|
+
The handler fails loudly at startup if any of these are missing:
|
|
60
|
+
|
|
61
|
+
- `MICROSUB_DB` — D1 database for channels, follows, timeline items, and the
|
|
62
|
+
per-feed poll cache.
|
|
63
|
+
- `MICROSUB_QUEUE` — Queue for feed-poll fan-out and retries.
|
|
64
|
+
- `AUTH_DB` — the [`@dwk/indieauth`](../indieauth) issued-token store, consulted
|
|
65
|
+
for revocation.
|
|
66
|
+
- `TOKEN_SIGNING_KEY` — the secret the IndieAuth token endpoint signs tokens with.
|
|
67
|
+
|
|
68
|
+
### What it implements
|
|
69
|
+
|
|
70
|
+
The single endpoint dispatches on the `action` (and `method`) parameter:
|
|
71
|
+
|
|
72
|
+
- **Channels** (`action=channels`): list, create, rename, delete
|
|
73
|
+
(`method=delete`), and reorder (`method=order`). A reserved `notifications`
|
|
74
|
+
channel always exists and cannot be deleted or renamed.
|
|
75
|
+
- **Following** (`action=follow` / `action=unfollow`): subscribe with feed
|
|
76
|
+
discovery (Atom / RSS / JSON Feed / `h-feed`); a follow populates the timeline
|
|
77
|
+
immediately and primes the poll cache.
|
|
78
|
+
- **Timeline** (`action=timeline`): JF2 entries with `before` / `after` opaque
|
|
79
|
+
cursors, `mark_read` / `mark_unread` (`entry`, `entry[]`, or
|
|
80
|
+
`last_read_entry`), `remove`, and per-channel unread counts.
|
|
81
|
+
- **Search / preview** (`action=search` / `action=preview`): discover or preview
|
|
82
|
+
a feed's entries without subscribing.
|
|
83
|
+
|
|
84
|
+
Every request is authorized by a DPoP-bound IndieAuth access token whose subject
|
|
85
|
+
must match the configured `me`, with the proof-of-possession binding completed
|
|
86
|
+
via [`@dwk/dpop`](../dpop) and revocation checked against the strongly-consistent
|
|
87
|
+
token store. Polling runs off the read path on a Cron-triggered queue; the read
|
|
88
|
+
path serves stored entries only. Every outbound fetch is SSRF-guarded.
|
|
89
|
+
|
|
90
|
+
## License
|
|
91
|
+
|
|
92
|
+
[ISC](../../LICENSE)
|
package/dist/auth.d.ts
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Request authorization: validate the IndieAuth access token, complete its DPoP
|
|
3
|
+
* proof-of-possession binding, honour revocation, and gate the action on the
|
|
4
|
+
* token's scope.
|
|
5
|
+
*
|
|
6
|
+
* Mirrors `@dwk/micropub`'s authorization exactly — Microsub is the read side of
|
|
7
|
+
* the same identity layer, so it accepts the same DPoP-bound HS256 access tokens
|
|
8
|
+
* `@dwk/indieauth` mints and applies the same single-owner subject check: a
|
|
9
|
+
* token minted by this issuer for a *different* `me` cannot read here. The DPoP
|
|
10
|
+
* proof binds the token to this exact request (RFC 9449), so a stolen bearer
|
|
11
|
+
* token alone is useless; revocation is checked against the strongly-consistent
|
|
12
|
+
* issued-token store, never a cache. Replay detection (for state-changing
|
|
13
|
+
* `POST`s) is the caller's, recorded in the strongly-consistent `MICROSUB_DB`.
|
|
14
|
+
*
|
|
15
|
+
* @packageDocumentation
|
|
16
|
+
*/
|
|
17
|
+
import { type AccessTokenClaims, type IndieAuthStoreEnv } from "@dwk/indieauth";
|
|
18
|
+
import type { ResolvedConfig } from "./config";
|
|
19
|
+
import type { MicrosubStoreEnv } from "./store";
|
|
20
|
+
/** Bindings the authorization path needs. */
|
|
21
|
+
export interface AuthEnv extends IndieAuthStoreEnv, MicrosubStoreEnv {
|
|
22
|
+
/** HMAC key the IndieAuth token endpoint signed access tokens with. */
|
|
23
|
+
readonly TOKEN_SIGNING_KEY: string;
|
|
24
|
+
}
|
|
25
|
+
/** A failed authorization: an OAuth-style error to surface to the client. */
|
|
26
|
+
export interface AuthFailure {
|
|
27
|
+
readonly ok: false;
|
|
28
|
+
readonly error: string;
|
|
29
|
+
readonly description: string;
|
|
30
|
+
readonly status: number;
|
|
31
|
+
}
|
|
32
|
+
/** A successful authorization: the verified token claims. */
|
|
33
|
+
export interface AuthSuccess {
|
|
34
|
+
readonly ok: true;
|
|
35
|
+
readonly claims: AccessTokenClaims;
|
|
36
|
+
}
|
|
37
|
+
export type AuthResult = AuthSuccess | AuthFailure;
|
|
38
|
+
/**
|
|
39
|
+
* Extract the bearer token from the `Authorization` header. Both the RFC 9449
|
|
40
|
+
* `DPoP` scheme (which our tokens use) and the legacy `Bearer` scheme are
|
|
41
|
+
* accepted. Returns `null` when the header is absent or malformed.
|
|
42
|
+
*/
|
|
43
|
+
export declare function tokenFromHeader(request: Request): string | null;
|
|
44
|
+
/** Whether the granted scope string contains any of the acceptable scopes. */
|
|
45
|
+
export declare function hasScope(scope: string, acceptable: readonly string[]): boolean;
|
|
46
|
+
/**
|
|
47
|
+
* Authorize a request. Verifies the access token, completes the DPoP binding
|
|
48
|
+
* against this exact request (`htm`/`htu`/`ath`/`cnf.jkt`), checks revocation,
|
|
49
|
+
* records the proof for replay detection (when `recordReplay`), and — when
|
|
50
|
+
* `requiredScopes` is non-empty — enforces scope.
|
|
51
|
+
*/
|
|
52
|
+
export declare function authorize(request: Request, env: AuthEnv, config: ResolvedConfig, token: string | null, requiredScopes: readonly string[], recordReplay: boolean): Promise<AuthResult>;
|
|
53
|
+
//# sourceMappingURL=auth.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"auth.d.ts","sourceRoot":"","sources":["../src/auth.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAGH,OAAO,EAGL,KAAK,iBAAiB,EACtB,KAAK,iBAAiB,EACvB,MAAM,gBAAgB,CAAC;AAExB,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAE/C,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAEhD,6CAA6C;AAC7C,MAAM,WAAW,OAAQ,SAAQ,iBAAiB,EAAE,gBAAgB;IAClE,uEAAuE;IACvE,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;CACpC;AAED,6EAA6E;AAC7E,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAC;IACnB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,6DAA6D;AAC7D,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;IAClB,QAAQ,CAAC,MAAM,EAAE,iBAAiB,CAAC;CACpC;AAED,MAAM,MAAM,UAAU,GAAG,WAAW,GAAG,WAAW,CAAC;AAUnD;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI,CAK/D;AAED,8EAA8E;AAC9E,wBAAgB,QAAQ,CACtB,KAAK,EAAE,MAAM,EACb,UAAU,EAAE,SAAS,MAAM,EAAE,GAC5B,OAAO,CAGT;AAED;;;;;GAKG;AACH,wBAAsB,SAAS,CAC7B,OAAO,EAAE,OAAO,EAChB,GAAG,EAAE,OAAO,EACZ,MAAM,EAAE,cAAc,EACtB,KAAK,EAAE,MAAM,GAAG,IAAI,EACpB,cAAc,EAAE,SAAS,MAAM,EAAE,EACjC,YAAY,EAAE,OAAO,GACpB,OAAO,CAAC,UAAU,CAAC,CA0FrB"}
|
package/dist/auth.js
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Request authorization: validate the IndieAuth access token, complete its DPoP
|
|
3
|
+
* proof-of-possession binding, honour revocation, and gate the action on the
|
|
4
|
+
* token's scope.
|
|
5
|
+
*
|
|
6
|
+
* Mirrors `@dwk/micropub`'s authorization exactly — Microsub is the read side of
|
|
7
|
+
* the same identity layer, so it accepts the same DPoP-bound HS256 access tokens
|
|
8
|
+
* `@dwk/indieauth` mints and applies the same single-owner subject check: a
|
|
9
|
+
* token minted by this issuer for a *different* `me` cannot read here. The DPoP
|
|
10
|
+
* proof binds the token to this exact request (RFC 9449), so a stolen bearer
|
|
11
|
+
* token alone is useless; revocation is checked against the strongly-consistent
|
|
12
|
+
* issued-token store, never a cache. Replay detection (for state-changing
|
|
13
|
+
* `POST`s) is the caller's, recorded in the strongly-consistent `MICROSUB_DB`.
|
|
14
|
+
*
|
|
15
|
+
* @packageDocumentation
|
|
16
|
+
*/
|
|
17
|
+
import { DEFAULT_MAX_AGE_SECONDS, verifyDpopProof } from "@dwk/dpop";
|
|
18
|
+
import { createIndieAuthStore, verifyAccessToken, } from "@dwk/indieauth";
|
|
19
|
+
import { createDpopReplayStore } from "./replay";
|
|
20
|
+
function failure(error, description, status) {
|
|
21
|
+
return { ok: false, error, description, status };
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Extract the bearer token from the `Authorization` header. Both the RFC 9449
|
|
25
|
+
* `DPoP` scheme (which our tokens use) and the legacy `Bearer` scheme are
|
|
26
|
+
* accepted. Returns `null` when the header is absent or malformed.
|
|
27
|
+
*/
|
|
28
|
+
export function tokenFromHeader(request) {
|
|
29
|
+
const header = request.headers.get("Authorization");
|
|
30
|
+
if (!header)
|
|
31
|
+
return null;
|
|
32
|
+
const match = /^(DPoP|Bearer)\s+(.+)$/i.exec(header.trim());
|
|
33
|
+
return match ? match[2] : null;
|
|
34
|
+
}
|
|
35
|
+
/** Whether the granted scope string contains any of the acceptable scopes. */
|
|
36
|
+
export function hasScope(scope, acceptable) {
|
|
37
|
+
const granted = scope.split(/\s+/).filter(Boolean);
|
|
38
|
+
return acceptable.some((s) => granted.includes(s));
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Authorize a request. Verifies the access token, completes the DPoP binding
|
|
42
|
+
* against this exact request (`htm`/`htu`/`ath`/`cnf.jkt`), checks revocation,
|
|
43
|
+
* records the proof for replay detection (when `recordReplay`), and — when
|
|
44
|
+
* `requiredScopes` is non-empty — enforces scope.
|
|
45
|
+
*/
|
|
46
|
+
export async function authorize(request, env, config, token, requiredScopes, recordReplay) {
|
|
47
|
+
if (!token) {
|
|
48
|
+
return failure("unauthorized", "a bearer access token is required", 401);
|
|
49
|
+
}
|
|
50
|
+
const verified = await verifyAccessToken(token, env.TOKEN_SIGNING_KEY, {
|
|
51
|
+
issuer: config.tokenIssuer,
|
|
52
|
+
});
|
|
53
|
+
if (!verified.valid) {
|
|
54
|
+
return failure("invalid_token", `access token rejected: ${verified.reason}`, 401);
|
|
55
|
+
}
|
|
56
|
+
const claims = verified.claims;
|
|
57
|
+
// The token's subject (`sub`) is the canonical `me` it was minted for. A
|
|
58
|
+
// Microsub endpoint serves a single user, so reject any token whose subject is
|
|
59
|
+
// not this user — otherwise any token from the same issuer (for any `me`)
|
|
60
|
+
// could read this timeline. Both sides are canonicalized, so this is exact.
|
|
61
|
+
if (claims.sub !== config.me) {
|
|
62
|
+
return failure("invalid_token", "access token subject is not the owner of this server", 403);
|
|
63
|
+
}
|
|
64
|
+
// Complete the DPoP proof-of-possession binding for this request.
|
|
65
|
+
const proof = request.headers.get("DPoP");
|
|
66
|
+
if (!proof) {
|
|
67
|
+
return failure("invalid_request", "a DPoP proof is required for token-bound requests", 401);
|
|
68
|
+
}
|
|
69
|
+
const dpop = await verifyDpopProof({
|
|
70
|
+
proof,
|
|
71
|
+
htm: request.method,
|
|
72
|
+
htu: request.url,
|
|
73
|
+
accessToken: token,
|
|
74
|
+
expectedJkt: claims.cnf.jkt,
|
|
75
|
+
});
|
|
76
|
+
if (!dpop.valid) {
|
|
77
|
+
return failure("invalid_token", `DPoP proof verification failed: ${dpop.reason}`, 401);
|
|
78
|
+
}
|
|
79
|
+
// Replay: only for state-changing requests. A captured GET proof is far less
|
|
80
|
+
// valuable, and recording every read's proof would write on the read path.
|
|
81
|
+
if (recordReplay && config.checkDpopReplay && dpop.jti) {
|
|
82
|
+
const now = Math.floor(Date.now() / 1000);
|
|
83
|
+
const fresh = await createDpopReplayStore(env).recordProof(dpop.jti, now + 2 * DEFAULT_MAX_AGE_SECONDS, now);
|
|
84
|
+
if (!fresh) {
|
|
85
|
+
return failure("invalid_token", "DPoP proof has already been used (replay detected)", 401);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
// Revocation: staleness here is a security bug, so hit the strongly-consistent
|
|
89
|
+
// issued-token store rather than any cache.
|
|
90
|
+
if (config.checkRevocation) {
|
|
91
|
+
const store = createIndieAuthStore(env);
|
|
92
|
+
const now = Math.floor(Date.now() / 1000);
|
|
93
|
+
if (!(await store.isTokenActive(claims.jti, now))) {
|
|
94
|
+
return failure("invalid_token", "access token has been revoked", 401);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
if (requiredScopes.length > 0 && !hasScope(claims.scope, requiredScopes)) {
|
|
98
|
+
return failure("insufficient_scope", `this action requires one of the scopes: ${requiredScopes.join(", ")}`, 403);
|
|
99
|
+
}
|
|
100
|
+
return { ok: true, claims };
|
|
101
|
+
}
|
|
102
|
+
//# sourceMappingURL=auth.js.map
|
package/dist/auth.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"auth.js","sourceRoot":"","sources":["../src/auth.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,uBAAuB,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACrE,OAAO,EACL,oBAAoB,EACpB,iBAAiB,GAGlB,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;AAyBjD,SAAS,OAAO,CACd,KAAa,EACb,WAAmB,EACnB,MAAc;IAEd,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC;AACnD,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,OAAgB;IAC9C,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;IACpD,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IACzB,MAAM,KAAK,GAAG,yBAAyB,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IAC5D,OAAO,KAAK,CAAC,CAAC,CAAE,KAAK,CAAC,CAAC,CAAY,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7C,CAAC;AAED,8EAA8E;AAC9E,MAAM,UAAU,QAAQ,CACtB,KAAa,EACb,UAA6B;IAE7B,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACnD,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;AACrD,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,OAAgB,EAChB,GAAY,EACZ,MAAsB,EACtB,KAAoB,EACpB,cAAiC,EACjC,YAAqB;IAErB,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,OAAO,CAAC,cAAc,EAAE,mCAAmC,EAAE,GAAG,CAAC,CAAC;IAC3E,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAAC,KAAK,EAAE,GAAG,CAAC,iBAAiB,EAAE;QACrE,MAAM,EAAE,MAAM,CAAC,WAAW;KAC3B,CAAC,CAAC;IACH,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;QACpB,OAAO,OAAO,CACZ,eAAe,EACf,0BAA0B,QAAQ,CAAC,MAAM,EAAE,EAC3C,GAAG,CACJ,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;IAE/B,yEAAyE;IACzE,+EAA+E;IAC/E,0EAA0E;IAC1E,4EAA4E;IAC5E,IAAI,MAAM,CAAC,GAAG,KAAK,MAAM,CAAC,EAAE,EAAE,CAAC;QAC7B,OAAO,OAAO,CACZ,eAAe,EACf,sDAAsD,EACtD,GAAG,CACJ,CAAC;IACJ,CAAC;IAED,kEAAkE;IAClE,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAC1C,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,OAAO,CACZ,iBAAiB,EACjB,mDAAmD,EACnD,GAAG,CACJ,CAAC;IACJ,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,eAAe,CAAC;QACjC,KAAK;QACL,GAAG,EAAE,OAAO,CAAC,MAAM;QACnB,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,WAAW,EAAE,KAAK;QAClB,WAAW,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG;KAC5B,CAAC,CAAC;IACH,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QAChB,OAAO,OAAO,CACZ,eAAe,EACf,mCAAmC,IAAI,CAAC,MAAM,EAAE,EAChD,GAAG,CACJ,CAAC;IACJ,CAAC;IAED,6EAA6E;IAC7E,2EAA2E;IAC3E,IAAI,YAAY,IAAI,MAAM,CAAC,eAAe,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;QACvD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;QAC1C,MAAM,KAAK,GAAG,MAAM,qBAAqB,CAAC,GAAG,CAAC,CAAC,WAAW,CACxD,IAAI,CAAC,GAAG,EACR,GAAG,GAAG,CAAC,GAAG,uBAAuB,EACjC,GAAG,CACJ,CAAC;QACF,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,OAAO,CACZ,eAAe,EACf,oDAAoD,EACpD,GAAG,CACJ,CAAC;QACJ,CAAC;IACH,CAAC;IAED,+EAA+E;IAC/E,4CAA4C;IAC5C,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,oBAAoB,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;QAC1C,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC;YAClD,OAAO,OAAO,CAAC,eAAe,EAAE,+BAA+B,EAAE,GAAG,CAAC,CAAC;QACxE,CAAC;IACH,CAAC;IAED,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,cAAc,CAAC,EAAE,CAAC;QACzE,OAAO,OAAO,CACZ,oBAAoB,EACpB,2CAA2C,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EACtE,GAAG,CACJ,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;AAC9B,CAAC"}
|
package/dist/config.d.ts
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@dwk/microsub` — injected configuration and the Cloudflare `Env` fragment.
|
|
3
|
+
*
|
|
4
|
+
* Per the composition contract a package never reads the global environment
|
|
5
|
+
* directly: all config (base URL, owner `me`, endpoint URL, page size, poll
|
|
6
|
+
* cadence) is passed into {@link createMicrosub}, so the server can be
|
|
7
|
+
* instantiated multiple times and unit-tested in isolation. The Cloudflare
|
|
8
|
+
* bindings it needs are declared as a TypeScript `Env` fragment that the
|
|
9
|
+
* composed Worker's `Env` is a superset of. See `spec/composition-contract.md`
|
|
10
|
+
* and `spec/packages/microsub.md`.
|
|
11
|
+
*
|
|
12
|
+
* @packageDocumentation
|
|
13
|
+
*/
|
|
14
|
+
import { type Logger, type Metrics } from "@dwk/log";
|
|
15
|
+
import type { D1Database, Queue } from "@cloudflare/workers-types";
|
|
16
|
+
import type { FetchLike } from "./fetch";
|
|
17
|
+
import type { MicrosubJob } from "./queue";
|
|
18
|
+
/**
|
|
19
|
+
* Cloudflare bindings required by the Microsub handler, poller, and queue
|
|
20
|
+
* consumer.
|
|
21
|
+
*
|
|
22
|
+
* The subscription + timeline store **MUST** be strongly consistent — D1
|
|
23
|
+
* (session consistency), never KV: a lost subscription or a dropped read-state
|
|
24
|
+
* flag is a correctness bug, not a safe-to-be-stale cache
|
|
25
|
+
* (`spec/non-functional-requirements.md`).
|
|
26
|
+
*/
|
|
27
|
+
export interface MicrosubEnv {
|
|
28
|
+
/** D1 database for channels, follows, timeline items, and the poll cache. */
|
|
29
|
+
readonly MICROSUB_DB: D1Database;
|
|
30
|
+
/** Queue for feed-poll fan-out and retries. */
|
|
31
|
+
readonly MICROSUB_QUEUE: Queue<MicrosubJob>;
|
|
32
|
+
/** The `@dwk/indieauth` issued-token store (D1), consulted for revocation. */
|
|
33
|
+
readonly AUTH_DB: D1Database;
|
|
34
|
+
/** HMAC key the IndieAuth token endpoint signed access tokens with. */
|
|
35
|
+
readonly TOKEN_SIGNING_KEY: string;
|
|
36
|
+
}
|
|
37
|
+
/** Configuration passed to {@link createMicrosub} and its poller/consumer. */
|
|
38
|
+
export interface MicrosubConfig {
|
|
39
|
+
/** The identity root / base URL (e.g. `https://example.com`). */
|
|
40
|
+
readonly baseUrl: string;
|
|
41
|
+
/**
|
|
42
|
+
* The site owner's IndieAuth profile URL (`me`). A token only authorizes a
|
|
43
|
+
* request when its subject (`sub`) equals this, after canonicalization — so a
|
|
44
|
+
* token minted by the same issuer for a *different* `me` cannot read here.
|
|
45
|
+
* Required: a Microsub endpoint serves exactly one user.
|
|
46
|
+
*/
|
|
47
|
+
readonly me: string;
|
|
48
|
+
/** Absolute Microsub endpoint URL. Defaults to `${origin}/microsub`. */
|
|
49
|
+
readonly microsubEndpoint?: string;
|
|
50
|
+
/**
|
|
51
|
+
* Expected access-token issuer (`iss`). Defaults to `baseUrl`, matching the
|
|
52
|
+
* `@dwk/indieauth` issuer default.
|
|
53
|
+
*/
|
|
54
|
+
readonly tokenIssuer?: string;
|
|
55
|
+
/** Default timeline page size. Defaults to 20. */
|
|
56
|
+
readonly pageSize?: number;
|
|
57
|
+
/**
|
|
58
|
+
* Per-channel retention ceiling: when a poll pushes a channel above this, the
|
|
59
|
+
* oldest items are reaped. Defaults to 5000. Keeps a runaway feed from filling
|
|
60
|
+
* the D1 store unbounded.
|
|
61
|
+
*/
|
|
62
|
+
readonly maxItemsPerChannel?: number;
|
|
63
|
+
/**
|
|
64
|
+
* Whether to check each token against the issued-token store (revocation).
|
|
65
|
+
* Defaults to `true` — staleness here is a security bug, so the check hits the
|
|
66
|
+
* strongly-consistent `AUTH_DB` rather than any cache.
|
|
67
|
+
*/
|
|
68
|
+
readonly checkRevocation?: boolean;
|
|
69
|
+
/**
|
|
70
|
+
* Whether to reject replayed DPoP proofs on state-changing (`POST`) requests
|
|
71
|
+
* by tracking each accepted proof's `jti` in the strongly-consistent
|
|
72
|
+
* `MICROSUB_DB`. Defaults to `true`.
|
|
73
|
+
*/
|
|
74
|
+
readonly checkDpopReplay?: boolean;
|
|
75
|
+
/** `fetch` implementation for discovery/polling/preview; defaults to global `fetch`. */
|
|
76
|
+
readonly fetch?: FetchLike;
|
|
77
|
+
/** Logger; defaults to a no-op (see `@dwk/log`). */
|
|
78
|
+
readonly logger?: Logger;
|
|
79
|
+
/** Metrics sink; defaults to a no-op (see `@dwk/log`). */
|
|
80
|
+
readonly metrics?: Metrics;
|
|
81
|
+
}
|
|
82
|
+
/** Fully resolved configuration with defaults applied and the path parsed. */
|
|
83
|
+
export interface ResolvedConfig {
|
|
84
|
+
readonly me: string;
|
|
85
|
+
readonly microsubEndpoint: string;
|
|
86
|
+
readonly microsubPath: string;
|
|
87
|
+
readonly tokenIssuer: string;
|
|
88
|
+
readonly pageSize: number;
|
|
89
|
+
readonly maxItemsPerChannel: number;
|
|
90
|
+
readonly checkRevocation: boolean;
|
|
91
|
+
readonly checkDpopReplay: boolean;
|
|
92
|
+
readonly fetch: FetchLike;
|
|
93
|
+
readonly logger: Logger;
|
|
94
|
+
readonly metrics: Metrics;
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* Resolve user config into a {@link ResolvedConfig}, applying defaults and
|
|
98
|
+
* pre-computing the endpoint pathname for request routing. Throws if `baseUrl`
|
|
99
|
+
* or `me` (or an explicitly supplied endpoint URL) is invalid.
|
|
100
|
+
*/
|
|
101
|
+
export declare function resolveConfig(config: MicrosubConfig): ResolvedConfig;
|
|
102
|
+
//# sourceMappingURL=config.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAGH,OAAO,EAA2B,KAAK,MAAM,EAAE,KAAK,OAAO,EAAE,MAAM,UAAU,CAAC;AAC9E,OAAO,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,2BAA2B,CAAC;AAEnE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACzC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAE3C;;;;;;;;GAQG;AACH,MAAM,WAAW,WAAW;IAC1B,6EAA6E;IAC7E,QAAQ,CAAC,WAAW,EAAE,UAAU,CAAC;IACjC,+CAA+C;IAC/C,QAAQ,CAAC,cAAc,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC;IAC5C,8EAA8E;IAC9E,QAAQ,CAAC,OAAO,EAAE,UAAU,CAAC;IAC7B,uEAAuE;IACvE,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;CACpC;AAED,8EAA8E;AAC9E,MAAM,WAAW,cAAc;IAC7B,iEAAiE;IACjE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,wEAAwE;IACxE,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACnC;;;OAGG;IACH,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,kDAAkD;IAClD,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B;;;;OAIG;IACH,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IACrC;;;;OAIG;IACH,QAAQ,CAAC,eAAe,CAAC,EAAE,OAAO,CAAC;IACnC;;;;OAIG;IACH,QAAQ,CAAC,eAAe,CAAC,EAAE,OAAO,CAAC;IACnC,wFAAwF;IACxF,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,CAAC;IAC3B,oDAAoD;IACpD,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,0DAA0D;IAC1D,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,8EAA8E;AAC9E,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,eAAe,EAAE,OAAO,CAAC;IAClC,QAAQ,CAAC,eAAe,EAAE,OAAO,CAAC;IAClC,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC;IAC1B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;CAC3B;AAaD;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,cAAc,GAAG,cAAc,CAqCpE"}
|
package/dist/config.js
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@dwk/microsub` — injected configuration and the Cloudflare `Env` fragment.
|
|
3
|
+
*
|
|
4
|
+
* Per the composition contract a package never reads the global environment
|
|
5
|
+
* directly: all config (base URL, owner `me`, endpoint URL, page size, poll
|
|
6
|
+
* cadence) is passed into {@link createMicrosub}, so the server can be
|
|
7
|
+
* instantiated multiple times and unit-tested in isolation. The Cloudflare
|
|
8
|
+
* bindings it needs are declared as a TypeScript `Env` fragment that the
|
|
9
|
+
* composed Worker's `Env` is a superset of. See `spec/composition-contract.md`
|
|
10
|
+
* and `spec/packages/microsub.md`.
|
|
11
|
+
*
|
|
12
|
+
* @packageDocumentation
|
|
13
|
+
*/
|
|
14
|
+
import { canonicalizeProfileUrl } from "@dwk/indieauth";
|
|
15
|
+
import { noopLogger, noopMetrics } from "@dwk/log";
|
|
16
|
+
const DEFAULT_PAGE_SIZE = 20;
|
|
17
|
+
const DEFAULT_MAX_ITEMS = 5000;
|
|
18
|
+
function pathOf(absoluteUrl, label) {
|
|
19
|
+
try {
|
|
20
|
+
return new URL(absoluteUrl).pathname;
|
|
21
|
+
}
|
|
22
|
+
catch {
|
|
23
|
+
throw new Error(`@dwk/microsub: ${label} is not a valid URL`);
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Resolve user config into a {@link ResolvedConfig}, applying defaults and
|
|
28
|
+
* pre-computing the endpoint pathname for request routing. Throws if `baseUrl`
|
|
29
|
+
* or `me` (or an explicitly supplied endpoint URL) is invalid.
|
|
30
|
+
*/
|
|
31
|
+
export function resolveConfig(config) {
|
|
32
|
+
let base;
|
|
33
|
+
try {
|
|
34
|
+
base = new URL(config.baseUrl);
|
|
35
|
+
}
|
|
36
|
+
catch {
|
|
37
|
+
throw new Error("@dwk/microsub: `baseUrl` is not a valid URL");
|
|
38
|
+
}
|
|
39
|
+
const me = canonicalizeProfileUrl(config.me);
|
|
40
|
+
if (me === null) {
|
|
41
|
+
throw new Error("@dwk/microsub: `me` is not a valid profile URL");
|
|
42
|
+
}
|
|
43
|
+
const microsubEndpoint = config.microsubEndpoint ?? `${base.origin}/microsub`;
|
|
44
|
+
const pageSize = config.pageSize !== undefined && config.pageSize > 0
|
|
45
|
+
? Math.floor(config.pageSize)
|
|
46
|
+
: DEFAULT_PAGE_SIZE;
|
|
47
|
+
const maxItemsPerChannel = config.maxItemsPerChannel !== undefined && config.maxItemsPerChannel > 0
|
|
48
|
+
? Math.floor(config.maxItemsPerChannel)
|
|
49
|
+
: DEFAULT_MAX_ITEMS;
|
|
50
|
+
return {
|
|
51
|
+
me,
|
|
52
|
+
microsubEndpoint,
|
|
53
|
+
microsubPath: pathOf(microsubEndpoint, "microsubEndpoint"),
|
|
54
|
+
tokenIssuer: config.tokenIssuer ?? config.baseUrl,
|
|
55
|
+
pageSize,
|
|
56
|
+
maxItemsPerChannel,
|
|
57
|
+
checkRevocation: config.checkRevocation ?? true,
|
|
58
|
+
checkDpopReplay: config.checkDpopReplay ?? true,
|
|
59
|
+
fetch: config.fetch ?? ((input, init) => fetch(input, init)),
|
|
60
|
+
logger: config.logger ?? noopLogger,
|
|
61
|
+
metrics: config.metrics ?? noopMetrics,
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
//# sourceMappingURL=config.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,WAAW,EAA6B,MAAM,UAAU,CAAC;AAuF9E,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAC7B,MAAM,iBAAiB,GAAG,IAAI,CAAC;AAE/B,SAAS,MAAM,CAAC,WAAmB,EAAE,KAAa;IAChD,IAAI,CAAC;QACH,OAAO,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC,QAAQ,CAAC;IACvC,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,kBAAkB,KAAK,qBAAqB,CAAC,CAAC;IAChE,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAAC,MAAsB;IAClD,IAAI,IAAS,CAAC;IACd,IAAI,CAAC;QACH,IAAI,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACjC,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IACjE,CAAC;IAED,MAAM,EAAE,GAAG,sBAAsB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC7C,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;IACpE,CAAC;IAED,MAAM,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,IAAI,GAAG,IAAI,CAAC,MAAM,WAAW,CAAC;IAE9E,MAAM,QAAQ,GACZ,MAAM,CAAC,QAAQ,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,GAAG,CAAC;QAClD,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC;QAC7B,CAAC,CAAC,iBAAiB,CAAC;IACxB,MAAM,kBAAkB,GACtB,MAAM,CAAC,kBAAkB,KAAK,SAAS,IAAI,MAAM,CAAC,kBAAkB,GAAG,CAAC;QACtE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,kBAAkB,CAAC;QACvC,CAAC,CAAC,iBAAiB,CAAC;IAExB,OAAO;QACL,EAAE;QACF,gBAAgB;QAChB,YAAY,EAAE,MAAM,CAAC,gBAAgB,EAAE,kBAAkB,CAAC;QAC1D,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,OAAO;QACjD,QAAQ;QACR,kBAAkB;QAClB,eAAe,EAAE,MAAM,CAAC,eAAe,IAAI,IAAI;QAC/C,eAAe,EAAE,MAAM,CAAC,eAAe,IAAI,IAAI;QAC/C,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAC5D,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,UAAU;QACnC,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,WAAW;KACvC,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@dwk/microsub` — the poll-queue consumer.
|
|
3
|
+
*
|
|
4
|
+
* The slow work the scheduled poller deferred runs here, off the request path,
|
|
5
|
+
* with the queue providing retries and backoff. For each `poll` job it:
|
|
6
|
+
*
|
|
7
|
+
* 1. fetches the feed conditionally (sending the cached `ETag` /
|
|
8
|
+
* `Last-Modified`); a `304` is acked with nothing to do;
|
|
9
|
+
* 2. parses the body to JF2 (Atom / RSS / JSON Feed / `h-feed`);
|
|
10
|
+
* 3. appends new entries — deduped by entry id — to every channel following
|
|
11
|
+
* that feed, reaping past the per-channel retention ceiling; and
|
|
12
|
+
* 4. records the returned validators for the next poll.
|
|
13
|
+
*
|
|
14
|
+
* A job whose fetch fails (unreachable / blocked / non-2xx) or whose store work
|
|
15
|
+
* throws is retried; everything else is acked. Feeds are assumed newest-first,
|
|
16
|
+
* so entries are inserted oldest-first and the newest gets the highest paging
|
|
17
|
+
* `seq`. See `spec/packages/microsub.md`.
|
|
18
|
+
*
|
|
19
|
+
* @packageDocumentation
|
|
20
|
+
*/
|
|
21
|
+
import type { ExecutionContext, MessageBatch } from "@cloudflare/workers-types";
|
|
22
|
+
import { type MicrosubConfig, type MicrosubEnv } from "./config";
|
|
23
|
+
import type { MicrosubJob } from "./queue";
|
|
24
|
+
import { type MicrosubStore } from "./store";
|
|
25
|
+
/** A Queue consumer for Microsub poll jobs. */
|
|
26
|
+
export type MicrosubQueueConsumer = (batch: MessageBatch<MicrosubJob>, env: MicrosubEnv, ctx: ExecutionContext) => Promise<void>;
|
|
27
|
+
/** Extra wiring for the consumer, primarily to inject a store/clock in tests. */
|
|
28
|
+
export interface ConsumerOptions {
|
|
29
|
+
/** Override the store; defaults to a D1 store over `MICROSUB_DB`. */
|
|
30
|
+
readonly store?: MicrosubStore;
|
|
31
|
+
/** Clock injection for deterministic tests; defaults to `Date.now`. */
|
|
32
|
+
readonly now?: () => number;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Build the Queue consumer that polls feeds and appends to channel timelines.
|
|
36
|
+
* Fails loudly if no store is configured (neither `options.store` nor the
|
|
37
|
+
* `MICROSUB_DB` binding).
|
|
38
|
+
*/
|
|
39
|
+
export declare function createMicrosubQueueConsumer(config: MicrosubConfig, options?: ConsumerOptions): MicrosubQueueConsumer;
|
|
40
|
+
//# sourceMappingURL=consumer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"consumer.d.ts","sourceRoot":"","sources":["../src/consumer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAGH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAEhF,OAAO,EAEL,KAAK,cAAc,EACnB,KAAK,WAAW,EAEjB,MAAM,UAAU,CAAC;AAIlB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,EAAuB,KAAK,aAAa,EAAE,MAAM,SAAS,CAAC;AAElE,+CAA+C;AAC/C,MAAM,MAAM,qBAAqB,GAAG,CAClC,KAAK,EAAE,YAAY,CAAC,WAAW,CAAC,EAChC,GAAG,EAAE,WAAW,EAChB,GAAG,EAAE,gBAAgB,KAClB,OAAO,CAAC,IAAI,CAAC,CAAC;AAEnB,iFAAiF;AACjF,MAAM,WAAW,eAAe;IAC9B,qEAAqE;IACrE,QAAQ,CAAC,KAAK,CAAC,EAAE,aAAa,CAAC;IAC/B,uEAAuE;IACvE,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CAC7B;AAWD;;;;GAIG;AACH,wBAAgB,2BAA2B,CACzC,MAAM,EAAE,cAAc,EACtB,OAAO,CAAC,EAAE,eAAe,GACxB,qBAAqB,CAuEvB"}
|
package/dist/consumer.js
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@dwk/microsub` — the poll-queue consumer.
|
|
3
|
+
*
|
|
4
|
+
* The slow work the scheduled poller deferred runs here, off the request path,
|
|
5
|
+
* with the queue providing retries and backoff. For each `poll` job it:
|
|
6
|
+
*
|
|
7
|
+
* 1. fetches the feed conditionally (sending the cached `ETag` /
|
|
8
|
+
* `Last-Modified`); a `304` is acked with nothing to do;
|
|
9
|
+
* 2. parses the body to JF2 (Atom / RSS / JSON Feed / `h-feed`);
|
|
10
|
+
* 3. appends new entries — deduped by entry id — to every channel following
|
|
11
|
+
* that feed, reaping past the per-channel retention ceiling; and
|
|
12
|
+
* 4. records the returned validators for the next poll.
|
|
13
|
+
*
|
|
14
|
+
* A job whose fetch fails (unreachable / blocked / non-2xx) or whose store work
|
|
15
|
+
* throws is retried; everything else is acked. Feeds are assumed newest-first,
|
|
16
|
+
* so entries are inserted oldest-first and the newest gets the highest paging
|
|
17
|
+
* `seq`. See `spec/packages/microsub.md`.
|
|
18
|
+
*
|
|
19
|
+
* @packageDocumentation
|
|
20
|
+
*/
|
|
21
|
+
import { hostFromUrl } from "@dwk/log";
|
|
22
|
+
import { resolveConfig, } from "./config";
|
|
23
|
+
import { fetchFeed } from "./discovery";
|
|
24
|
+
import { orderEntriesForInsert } from "./jf2";
|
|
25
|
+
import { MicrosubLogEvent } from "./log";
|
|
26
|
+
import { createMicrosubStore } from "./store";
|
|
27
|
+
function emit(config, event, fields) {
|
|
28
|
+
config.logger.info(event, fields);
|
|
29
|
+
config.metrics.count(event, fields);
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Build the Queue consumer that polls feeds and appends to channel timelines.
|
|
33
|
+
* Fails loudly if no store is configured (neither `options.store` nor the
|
|
34
|
+
* `MICROSUB_DB` binding).
|
|
35
|
+
*/
|
|
36
|
+
export function createMicrosubQueueConsumer(config, options) {
|
|
37
|
+
const resolved = resolveConfig(config);
|
|
38
|
+
const clock = options?.now ?? (() => Math.floor(Date.now() / 1000));
|
|
39
|
+
return async (batch, env, _ctx) => {
|
|
40
|
+
const store = options?.store ??
|
|
41
|
+
(env.MICROSUB_DB
|
|
42
|
+
? createMicrosubStore(env)
|
|
43
|
+
: (() => {
|
|
44
|
+
throw new Error("@dwk/microsub: missing required D1 binding `MICROSUB_DB`");
|
|
45
|
+
})());
|
|
46
|
+
for (const message of batch.messages) {
|
|
47
|
+
const job = message.body;
|
|
48
|
+
try {
|
|
49
|
+
const cache = await store.getFeedCache(job.feedUrl);
|
|
50
|
+
const fetched = await fetchFeed(job.feedUrl, {
|
|
51
|
+
fetch: resolved.fetch,
|
|
52
|
+
logger: resolved.logger,
|
|
53
|
+
metrics: resolved.metrics,
|
|
54
|
+
}, cache ?? undefined);
|
|
55
|
+
if (fetched === null) {
|
|
56
|
+
// Unreachable / blocked / non-2xx — retry the whole job later.
|
|
57
|
+
message.retry();
|
|
58
|
+
continue;
|
|
59
|
+
}
|
|
60
|
+
const now = clock();
|
|
61
|
+
await store.setFeedCache(job.feedUrl, fetched.etag, fetched.lastModified, now);
|
|
62
|
+
let added = 0;
|
|
63
|
+
if (!fetched.notModified && fetched.entries.length > 0) {
|
|
64
|
+
// Order oldest-first so the newest entry gets the highest paging `seq`.
|
|
65
|
+
const ordered = orderEntriesForInsert(fetched.entries);
|
|
66
|
+
const channels = await store.channelsForFeed(job.feedUrl);
|
|
67
|
+
for (const channel of channels) {
|
|
68
|
+
added += await store.insertItems(channel, job.feedUrl, ordered, now, resolved.maxItemsPerChannel);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
emit(resolved, MicrosubLogEvent.FeedPolled, {
|
|
72
|
+
added,
|
|
73
|
+
host: hostFromUrl(job.feedUrl),
|
|
74
|
+
});
|
|
75
|
+
message.ack();
|
|
76
|
+
}
|
|
77
|
+
catch (err) {
|
|
78
|
+
emit(resolved, MicrosubLogEvent.PollRetry, {
|
|
79
|
+
error: err instanceof Error ? err.name : "unknown",
|
|
80
|
+
host: hostFromUrl(job.feedUrl),
|
|
81
|
+
});
|
|
82
|
+
message.retry();
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
//# sourceMappingURL=consumer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"consumer.js","sourceRoot":"","sources":["../src/consumer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAGvC,OAAO,EACL,aAAa,GAId,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,qBAAqB,EAAE,MAAM,OAAO,CAAC;AAC9C,OAAO,EAAE,gBAAgB,EAAE,MAAM,OAAO,CAAC;AAEzC,OAAO,EAAE,mBAAmB,EAAsB,MAAM,SAAS,CAAC;AAiBlE,SAAS,IAAI,CACX,MAAsB,EACtB,KAAa,EACb,MAAgC;IAEhC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAClC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;AACtC,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,2BAA2B,CACzC,MAAsB,EACtB,OAAyB;IAEzB,MAAM,QAAQ,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IACvC,MAAM,KAAK,GAAG,OAAO,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC;IAEpE,OAAO,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE;QAChC,MAAM,KAAK,GACT,OAAO,EAAE,KAAK;YACd,CAAC,GAAG,CAAC,WAAW;gBACd,CAAC,CAAC,mBAAmB,CAAC,GAAG,CAAC;gBAC1B,CAAC,CAAC,CAAC,GAAG,EAAE;oBACJ,MAAM,IAAI,KAAK,CACb,0DAA0D,CAC3D,CAAC;gBACJ,CAAC,CAAC,EAAE,CAAC,CAAC;QAEZ,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YACrC,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;YACzB,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;gBACpD,MAAM,OAAO,GAAG,MAAM,SAAS,CAC7B,GAAG,CAAC,OAAO,EACX;oBACE,KAAK,EAAE,QAAQ,CAAC,KAAK;oBACrB,MAAM,EAAE,QAAQ,CAAC,MAAM;oBACvB,OAAO,EAAE,QAAQ,CAAC,OAAO;iBAC1B,EACD,KAAK,IAAI,SAAS,CACnB,CAAC;gBACF,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;oBACrB,+DAA+D;oBAC/D,OAAO,CAAC,KAAK,EAAE,CAAC;oBAChB,SAAS;gBACX,CAAC;gBAED,MAAM,GAAG,GAAG,KAAK,EAAE,CAAC;gBACpB,MAAM,KAAK,CAAC,YAAY,CACtB,GAAG,CAAC,OAAO,EACX,OAAO,CAAC,IAAI,EACZ,OAAO,CAAC,YAAY,EACpB,GAAG,CACJ,CAAC;gBAEF,IAAI,KAAK,GAAG,CAAC,CAAC;gBACd,IAAI,CAAC,OAAO,CAAC,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACvD,wEAAwE;oBACxE,MAAM,OAAO,GAAG,qBAAqB,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;oBACvD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,eAAe,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;oBAC1D,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;wBAC/B,KAAK,IAAI,MAAM,KAAK,CAAC,WAAW,CAC9B,OAAO,EACP,GAAG,CAAC,OAAO,EACX,OAAO,EACP,GAAG,EACH,QAAQ,CAAC,kBAAkB,CAC5B,CAAC;oBACJ,CAAC;gBACH,CAAC;gBACD,IAAI,CAAC,QAAQ,EAAE,gBAAgB,CAAC,UAAU,EAAE;oBAC1C,KAAK;oBACL,IAAI,EAAE,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC;iBAC/B,CAAC,CAAC;gBACH,OAAO,CAAC,GAAG,EAAE,CAAC;YAChB,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,QAAQ,EAAE,gBAAgB,CAAC,SAAS,EAAE;oBACzC,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;oBAClD,IAAI,EAAE,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC;iBAC/B,CAAC,CAAC;gBACH,OAAO,CAAC,KAAK,EAAE,CAAC;YAClB,CAAC;QACH,CAAC;IACH,CAAC,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@dwk/microsub` — feed discovery and fetching.
|
|
3
|
+
*
|
|
4
|
+
* `follow` and `preview` take a URL the user typed; the server must work out
|
|
5
|
+
* what to actually poll. {@link discoverFeed} fetches that URL (through the
|
|
6
|
+
* SSRF-safe wrapper), and:
|
|
7
|
+
*
|
|
8
|
+
* - if it is already a syndication feed (Atom / RSS / JSON Feed), parses it;
|
|
9
|
+
* - if it is HTML, looks for a `<link rel="alternate">` feed and follows the
|
|
10
|
+
* first one; failing that, parses the page's own `h-feed` microformats.
|
|
11
|
+
*
|
|
12
|
+
* {@link fetchFeed} re-fetches an already-resolved feed URL on each poll,
|
|
13
|
+
* sending the cached `ETag` / `Last-Modified` so an unchanged feed returns `304`
|
|
14
|
+
* and is skipped. Every outbound request goes through {@link safeFetch}, and the
|
|
15
|
+
* body is read with a hard size cap.
|
|
16
|
+
*
|
|
17
|
+
* @packageDocumentation
|
|
18
|
+
*/
|
|
19
|
+
import { type Logger, type Metrics } from "@dwk/log";
|
|
20
|
+
import { type FetchLike } from "./fetch";
|
|
21
|
+
import { type Jf2Entry } from "./jf2";
|
|
22
|
+
/** Shared options for the discovery/fetch helpers. */
|
|
23
|
+
export interface DiscoveryOptions {
|
|
24
|
+
readonly fetch?: FetchLike;
|
|
25
|
+
readonly logger?: Logger;
|
|
26
|
+
readonly metrics?: Metrics;
|
|
27
|
+
}
|
|
28
|
+
/** A resolved feed: the URL to poll and the entries parsed from its first fetch. */
|
|
29
|
+
export interface DiscoveredFeed {
|
|
30
|
+
/** The URL the poller should re-fetch (a feed, or an HTML `h-feed` page). */
|
|
31
|
+
readonly feedUrl: string;
|
|
32
|
+
/** The entries parsed from the discovery fetch. */
|
|
33
|
+
readonly entries: readonly Jf2Entry[];
|
|
34
|
+
}
|
|
35
|
+
/** A completed feed fetch. */
|
|
36
|
+
export interface FetchedFeed {
|
|
37
|
+
/** The parsed entries (empty when not modified or unparseable). */
|
|
38
|
+
readonly entries: readonly Jf2Entry[];
|
|
39
|
+
/** `true` when the server answered `304 Not Modified`. */
|
|
40
|
+
readonly notModified: boolean;
|
|
41
|
+
readonly etag: string | null;
|
|
42
|
+
readonly lastModified: string | null;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Discover the feed at `target`. Returns the URL to poll plus the entries from
|
|
46
|
+
* this first fetch, or `null` when the target is unreachable, blocked, or has no
|
|
47
|
+
* feed and no `h-entry` content.
|
|
48
|
+
*/
|
|
49
|
+
export declare function discoverFeed(target: string, options?: DiscoveryOptions): Promise<DiscoveredFeed | null>;
|
|
50
|
+
/**
|
|
51
|
+
* Fetch and parse an already-resolved feed URL, sending conditional-request
|
|
52
|
+
* validators when supplied. Returns `notModified` on a `304`, the parsed
|
|
53
|
+
* entries otherwise, or `null` when the fetch fails or is blocked.
|
|
54
|
+
*/
|
|
55
|
+
export declare function fetchFeed(feedUrl: string, options?: DiscoveryOptions, cache?: {
|
|
56
|
+
etag: string | null;
|
|
57
|
+
lastModified: string | null;
|
|
58
|
+
}): Promise<FetchedFeed | null>;
|
|
59
|
+
//# sourceMappingURL=discovery.d.ts.map
|