@dwk/microsub 0.1.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +15 -0
- package/README.md +92 -0
- package/dist/auth.d.ts +53 -0
- package/dist/auth.d.ts.map +1 -0
- package/dist/auth.js +102 -0
- package/dist/auth.js.map +1 -0
- package/dist/config.d.ts +102 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +64 -0
- package/dist/config.js.map +1 -0
- package/dist/consumer.d.ts +40 -0
- package/dist/consumer.d.ts.map +1 -0
- package/dist/consumer.js +87 -0
- package/dist/consumer.js.map +1 -0
- package/dist/discovery.d.ts +59 -0
- package/dist/discovery.d.ts.map +1 -0
- package/dist/discovery.js +190 -0
- package/dist/discovery.js.map +1 -0
- package/dist/fetch.d.ts +28 -0
- package/dist/fetch.d.ts.map +1 -0
- package/dist/fetch.js +72 -0
- package/dist/fetch.js.map +1 -0
- package/dist/handler.d.ts +24 -0
- package/dist/handler.d.ts.map +1 -0
- package/dist/handler.js +434 -0
- package/dist/handler.js.map +1 -0
- package/dist/hfeed.d.ts +25 -0
- package/dist/hfeed.d.ts.map +1 -0
- package/dist/hfeed.js +252 -0
- package/dist/hfeed.js.map +1 -0
- package/dist/index.d.ts +39 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +32 -0
- package/dist/index.js.map +1 -0
- package/dist/jf2.d.ts +69 -0
- package/dist/jf2.d.ts.map +1 -0
- package/dist/jf2.js +295 -0
- package/dist/jf2.js.map +1 -0
- package/dist/log.d.ts +44 -0
- package/dist/log.d.ts.map +1 -0
- package/dist/log.js +42 -0
- package/dist/log.js.map +1 -0
- package/dist/poll.d.ts +22 -0
- package/dist/poll.d.ts.map +1 -0
- package/dist/poll.js +39 -0
- package/dist/poll.js.map +1 -0
- package/dist/queue.d.ts +25 -0
- package/dist/queue.d.ts.map +1 -0
- package/dist/queue.js +13 -0
- package/dist/queue.js.map +1 -0
- package/dist/replay.d.ts +34 -0
- package/dist/replay.d.ts.map +1 -0
- package/dist/replay.js +49 -0
- package/dist/replay.js.map +1 -0
- package/dist/safe-fetch.d.ts +86 -0
- package/dist/safe-fetch.d.ts.map +1 -0
- package/dist/safe-fetch.js +311 -0
- package/dist/safe-fetch.js.map +1 -0
- package/dist/store.d.ts +131 -0
- package/dist/store.d.ts.map +1 -0
- package/dist/store.js +393 -0
- package/dist/store.js.map +1 -0
- package/dist/xml.d.ts +51 -0
- package/dist/xml.d.ts.map +1 -0
- package/dist/xml.js +196 -0
- package/dist/xml.js.map +1 -0
- package/package.json +49 -0
- package/src/auth.ts +184 -0
- package/src/config.ts +156 -0
- package/src/consumer.ts +140 -0
- package/src/discovery.ts +270 -0
- package/src/fetch.ts +82 -0
- package/src/handler.ts +594 -0
- package/src/hfeed.ts +287 -0
- package/src/index.ts +86 -0
- package/src/jf2.ts +394 -0
- package/src/log.ts +46 -0
- package/src/poll.ts +72 -0
- package/src/queue.ts +26 -0
- package/src/replay.ts +68 -0
- package/src/safe-fetch.ts +346 -0
- package/src/store.ts +644 -0
- package/src/xml.ts +229 -0
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@dwk/microsub` — SSRF-safe outbound fetch.
|
|
3
|
+
*
|
|
4
|
+
* A Microsub server fetches user- and feed-supplied URLs: a `follow` discovers
|
|
5
|
+
* a feed at a URL the user typed, polling re-fetches it, and `preview`/`search`
|
|
6
|
+
* fetch an arbitrary URL. Without guardrails any of these could be pointed at
|
|
7
|
+
* the Worker's own network — loopback, the link-local cloud metadata IP
|
|
8
|
+
* (`169.254.169.254`), or RFC 1918 ranges — to exfiltrate credentials or probe
|
|
9
|
+
* internal services. This module is the single choke point every outbound fetch
|
|
10
|
+
* in the package goes through. It:
|
|
11
|
+
*
|
|
12
|
+
* 1. rejects URLs whose host is a private, loopback, link-local, or otherwise
|
|
13
|
+
* non-public address (or a name like `localhost` / `*.internal`),
|
|
14
|
+
* 2. follows redirects manually, re-validating the host on every `Location`
|
|
15
|
+
* hop so a public-looking host cannot 302 to an internal one, and capping
|
|
16
|
+
* the hop count, and
|
|
17
|
+
* 3. bounds the whole operation with a single timeout, so a slow-loris source
|
|
18
|
+
* cannot pin a poll-queue invocation.
|
|
19
|
+
*
|
|
20
|
+
* Host validation is purely syntactic on the URL host — DNS rebinding (a name
|
|
21
|
+
* that resolves to a private IP) is out of scope here, as the Workers runtime
|
|
22
|
+
* does not expose name resolution to user code. See `spec/packages/microsub.md`
|
|
23
|
+
* and `spec/non-functional-requirements.md`.
|
|
24
|
+
*
|
|
25
|
+
* @packageDocumentation
|
|
26
|
+
*/
|
|
27
|
+
import { type Logger, type Metrics } from "@dwk/log";
|
|
28
|
+
import type { FetchLike } from "./fetch";
|
|
29
|
+
/** Default cap on redirect hops before a fetch is abandoned. */
|
|
30
|
+
export declare const DEFAULT_MAX_REDIRECTS = 5;
|
|
31
|
+
/** Default overall timeout (ms) bounding a fetch, redirects included. */
|
|
32
|
+
export declare const DEFAULT_TIMEOUT_MS = 10000;
|
|
33
|
+
/** Machine-readable cause of an {@link SsrfError}. */
|
|
34
|
+
export type SsrfReason = "invalid_url" | "disallowed_scheme" | "blocked_host" | "too_many_redirects";
|
|
35
|
+
/**
|
|
36
|
+
* Raised when a request is refused on SSRF grounds (blocked host, disallowed
|
|
37
|
+
* scheme, or too many redirects). Callers catch this exactly like a network
|
|
38
|
+
* failure — a blocked attempt looks the same as an unreachable host — but
|
|
39
|
+
* {@link safeFetch} logs it first (event `microsub.ssrf.blocked`) so the single
|
|
40
|
+
* most security-relevant event in the package still produces a signal.
|
|
41
|
+
*/
|
|
42
|
+
export declare class SsrfError extends Error {
|
|
43
|
+
readonly reason: SsrfReason;
|
|
44
|
+
readonly host?: string;
|
|
45
|
+
constructor(message: string, reason: SsrfReason, host?: string);
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Decide whether a URL host is private, loopback, link-local, or otherwise
|
|
49
|
+
* not safe to fetch from inside the Worker's network. Accepts the raw
|
|
50
|
+
* `URL.hostname` form (IPv6 hosts may arrive wrapped in `[...]`).
|
|
51
|
+
*/
|
|
52
|
+
export declare function isPrivateOrReservedHost(hostname: string): boolean;
|
|
53
|
+
/**
|
|
54
|
+
* Validate that `rawUrl` is a fetchable public `http(s)` URL, returning the
|
|
55
|
+
* parsed {@link URL}. Throws {@link SsrfError} for an unparseable URL, a
|
|
56
|
+
* non-`http(s)` scheme (e.g. `file:`, `javascript:`), or a private/reserved
|
|
57
|
+
* host.
|
|
58
|
+
*/
|
|
59
|
+
export declare function assertPublicUrl(rawUrl: string): URL;
|
|
60
|
+
/** Tunables for {@link safeFetch}. */
|
|
61
|
+
export interface SafeFetchOptions {
|
|
62
|
+
readonly maxRedirects?: number;
|
|
63
|
+
readonly timeoutMs?: number;
|
|
64
|
+
readonly logger?: Logger;
|
|
65
|
+
readonly metrics?: Metrics;
|
|
66
|
+
}
|
|
67
|
+
/** A completed {@link safeFetch}: the final response and the URL it came from. */
|
|
68
|
+
export interface SafeFetchResult {
|
|
69
|
+
readonly response: Response;
|
|
70
|
+
readonly url: string;
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Fetch `rawUrl` through `doFetch` with SSRF guardrails.
|
|
74
|
+
*
|
|
75
|
+
* The initial host and every redirect target are validated with
|
|
76
|
+
* {@link assertPublicUrl}; redirects are followed manually (`redirect:
|
|
77
|
+
* "manual"`) up to `maxRedirects` hops; and a single {@link AbortSignal.timeout}
|
|
78
|
+
* bounds the whole chain. Credential-bearing headers are stripped on a
|
|
79
|
+
* cross-origin redirect, matching what a browser's `fetch` does.
|
|
80
|
+
*
|
|
81
|
+
* @throws {SsrfError} when a host is blocked, a scheme is disallowed, or the
|
|
82
|
+
* redirect cap is exceeded. Other failures (network, timeout) propagate as the
|
|
83
|
+
* underlying fetch rejection. Callers treat any throw as "fetch failed".
|
|
84
|
+
*/
|
|
85
|
+
export declare function safeFetch(doFetch: FetchLike, rawUrl: string, init: RequestInit, options?: SafeFetchOptions): Promise<SafeFetchResult>;
|
|
86
|
+
//# sourceMappingURL=safe-fetch.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"safe-fetch.d.ts","sourceRoot":"","sources":["../src/safe-fetch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,EAA2B,KAAK,MAAM,EAAE,KAAK,OAAO,EAAE,MAAM,UAAU,CAAC;AAC9E,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAGzC,gEAAgE;AAChE,eAAO,MAAM,qBAAqB,IAAI,CAAC;AACvC,yEAAyE;AACzE,eAAO,MAAM,kBAAkB,QAAS,CAAC;AAKzC,sDAAsD;AACtD,MAAM,MAAM,UAAU,GAClB,aAAa,GACb,mBAAmB,GACnB,cAAc,GACd,oBAAoB,CAAC;AAEzB;;;;;;GAMG;AACH,qBAAa,SAAU,SAAQ,KAAK;IAClC,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC;IAC5B,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;gBACX,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,CAAC,EAAE,MAAM;CAM/D;AA0ID;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAajE;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,GAAG,CAsBnD;AAED,sCAAsC;AACtC,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,kFAAkF;AAClF,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC;IAC5B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;CACtB;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,SAAS,CAC7B,OAAO,EAAE,SAAS,EAClB,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,WAAW,EACjB,OAAO,CAAC,EAAE,gBAAgB,GACzB,OAAO,CAAC,eAAe,CAAC,CA+D1B"}
|
|
@@ -0,0 +1,311 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@dwk/microsub` — SSRF-safe outbound fetch.
|
|
3
|
+
*
|
|
4
|
+
* A Microsub server fetches user- and feed-supplied URLs: a `follow` discovers
|
|
5
|
+
* a feed at a URL the user typed, polling re-fetches it, and `preview`/`search`
|
|
6
|
+
* fetch an arbitrary URL. Without guardrails any of these could be pointed at
|
|
7
|
+
* the Worker's own network — loopback, the link-local cloud metadata IP
|
|
8
|
+
* (`169.254.169.254`), or RFC 1918 ranges — to exfiltrate credentials or probe
|
|
9
|
+
* internal services. This module is the single choke point every outbound fetch
|
|
10
|
+
* in the package goes through. It:
|
|
11
|
+
*
|
|
12
|
+
* 1. rejects URLs whose host is a private, loopback, link-local, or otherwise
|
|
13
|
+
* non-public address (or a name like `localhost` / `*.internal`),
|
|
14
|
+
* 2. follows redirects manually, re-validating the host on every `Location`
|
|
15
|
+
* hop so a public-looking host cannot 302 to an internal one, and capping
|
|
16
|
+
* the hop count, and
|
|
17
|
+
* 3. bounds the whole operation with a single timeout, so a slow-loris source
|
|
18
|
+
* cannot pin a poll-queue invocation.
|
|
19
|
+
*
|
|
20
|
+
* Host validation is purely syntactic on the URL host — DNS rebinding (a name
|
|
21
|
+
* that resolves to a private IP) is out of scope here, as the Workers runtime
|
|
22
|
+
* does not expose name resolution to user code. See `spec/packages/microsub.md`
|
|
23
|
+
* and `spec/non-functional-requirements.md`.
|
|
24
|
+
*
|
|
25
|
+
* @packageDocumentation
|
|
26
|
+
*/
|
|
27
|
+
import { noopLogger, noopMetrics } from "@dwk/log";
|
|
28
|
+
import { MicrosubLogEvent } from "./log";
|
|
29
|
+
/** Default cap on redirect hops before a fetch is abandoned. */
|
|
30
|
+
export const DEFAULT_MAX_REDIRECTS = 5;
|
|
31
|
+
/** Default overall timeout (ms) bounding a fetch, redirects included. */
|
|
32
|
+
export const DEFAULT_TIMEOUT_MS = 10_000;
|
|
33
|
+
/** HTTP status codes that carry a `Location` we may follow. */
|
|
34
|
+
const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
|
|
35
|
+
/**
|
|
36
|
+
* Raised when a request is refused on SSRF grounds (blocked host, disallowed
|
|
37
|
+
* scheme, or too many redirects). Callers catch this exactly like a network
|
|
38
|
+
* failure — a blocked attempt looks the same as an unreachable host — but
|
|
39
|
+
* {@link safeFetch} logs it first (event `microsub.ssrf.blocked`) so the single
|
|
40
|
+
* most security-relevant event in the package still produces a signal.
|
|
41
|
+
*/
|
|
42
|
+
export class SsrfError extends Error {
|
|
43
|
+
reason;
|
|
44
|
+
host;
|
|
45
|
+
constructor(message, reason, host) {
|
|
46
|
+
super(message);
|
|
47
|
+
this.name = "SsrfError";
|
|
48
|
+
this.reason = reason;
|
|
49
|
+
this.host = host;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
/** Parse a canonical dotted-decimal IPv4 host into its four octets. */
|
|
53
|
+
function parseIPv4(host) {
|
|
54
|
+
const match = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/.exec(host);
|
|
55
|
+
if (match === null)
|
|
56
|
+
return null;
|
|
57
|
+
const octets = [];
|
|
58
|
+
for (let group = 1; group <= 4; group++) {
|
|
59
|
+
const part = match[group];
|
|
60
|
+
if (part === undefined)
|
|
61
|
+
return null;
|
|
62
|
+
const octet = Number.parseInt(part, 10);
|
|
63
|
+
if (octet > 255)
|
|
64
|
+
return null;
|
|
65
|
+
octets.push(octet);
|
|
66
|
+
}
|
|
67
|
+
return octets;
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* True when `octets` falls in a range that must never be fetched from inside
|
|
71
|
+
* the Worker's network: this-network, loopback, link-local (incl. the cloud
|
|
72
|
+
* metadata IP), the RFC 1918 private blocks, CGNAT, IETF protocol/benchmark
|
|
73
|
+
* assignments, and the multicast/reserved/broadcast space.
|
|
74
|
+
*/
|
|
75
|
+
function isPrivateIPv4(octets) {
|
|
76
|
+
const [a, b, c] = octets;
|
|
77
|
+
if (a === 0)
|
|
78
|
+
return true; // 0.0.0.0/8 ("this network", incl. 0.0.0.0)
|
|
79
|
+
if (a === 10)
|
|
80
|
+
return true; // 10.0.0.0/8 private
|
|
81
|
+
if (a === 127)
|
|
82
|
+
return true; // 127.0.0.0/8 loopback
|
|
83
|
+
if (a === 100 && b >= 64 && b <= 127)
|
|
84
|
+
return true; // 100.64.0.0/10 CGNAT
|
|
85
|
+
if (a === 169 && b === 254)
|
|
86
|
+
return true; // 169.254.0.0/16 link-local (metadata)
|
|
87
|
+
if (a === 172 && b >= 16 && b <= 31)
|
|
88
|
+
return true; // 172.16.0.0/12 private
|
|
89
|
+
if (a === 192 && b === 0 && c === 0)
|
|
90
|
+
return true; // 192.0.0.0/24 IETF protocol
|
|
91
|
+
if (a === 192 && b === 0 && c === 2)
|
|
92
|
+
return true; // 192.0.2.0/24 TEST-NET-1
|
|
93
|
+
if (a === 192 && b === 168)
|
|
94
|
+
return true; // 192.168.0.0/16 private
|
|
95
|
+
if (a === 198 && b === 51 && c === 100)
|
|
96
|
+
return true; // 198.51.100.0/24 TEST-NET-2
|
|
97
|
+
if (a === 198 && (b === 18 || b === 19))
|
|
98
|
+
return true; // 198.18.0.0/15 benchmark
|
|
99
|
+
if (a === 203 && b === 0 && c === 113)
|
|
100
|
+
return true; // 203.0.113.0/24 TEST-NET-3
|
|
101
|
+
if (a >= 224)
|
|
102
|
+
return true; // 224.0.0.0/4 multicast + 240.0.0.0/4 reserved + broadcast
|
|
103
|
+
return false;
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Parse an IPv6 host (brackets already stripped) into its eight 16-bit groups,
|
|
107
|
+
* expanding `::` compression and any trailing embedded IPv4 literal. Returns
|
|
108
|
+
* `null` when `host` is not a valid IPv6 address.
|
|
109
|
+
*/
|
|
110
|
+
function parseIPv6(host) {
|
|
111
|
+
if (!host.includes(":"))
|
|
112
|
+
return null;
|
|
113
|
+
let str = host;
|
|
114
|
+
const v4Match = /(?:^|:)((?:\d{1,3}\.){3}\d{1,3})$/.exec(str);
|
|
115
|
+
const v4Str = v4Match?.[1];
|
|
116
|
+
if (v4Str !== undefined) {
|
|
117
|
+
const v4 = parseIPv4(v4Str);
|
|
118
|
+
if (v4 === null)
|
|
119
|
+
return null;
|
|
120
|
+
const hi = ((v4[0] << 8) | v4[1]).toString(16);
|
|
121
|
+
const lo = ((v4[2] << 8) | v4[3]).toString(16);
|
|
122
|
+
str = `${str.slice(0, str.length - v4Str.length)}${hi}:${lo}`;
|
|
123
|
+
}
|
|
124
|
+
if (str.indexOf("::") !== str.lastIndexOf("::"))
|
|
125
|
+
return null;
|
|
126
|
+
const toGroups = (part) => {
|
|
127
|
+
if (part === "")
|
|
128
|
+
return [];
|
|
129
|
+
const groups = [];
|
|
130
|
+
for (const token of part.split(":")) {
|
|
131
|
+
if (!/^[0-9a-fA-F]{1,4}$/.test(token))
|
|
132
|
+
return null;
|
|
133
|
+
groups.push(Number.parseInt(token, 16));
|
|
134
|
+
}
|
|
135
|
+
return groups;
|
|
136
|
+
};
|
|
137
|
+
if (str.includes("::")) {
|
|
138
|
+
const parts = str.split("::");
|
|
139
|
+
const left = toGroups(parts[0] ?? "");
|
|
140
|
+
const right = toGroups(parts[1] ?? "");
|
|
141
|
+
if (left === null || right === null)
|
|
142
|
+
return null;
|
|
143
|
+
const missing = 8 - left.length - right.length;
|
|
144
|
+
if (missing < 1)
|
|
145
|
+
return null;
|
|
146
|
+
return [...left, ...new Array(missing).fill(0), ...right];
|
|
147
|
+
}
|
|
148
|
+
const all = toGroups(str);
|
|
149
|
+
if (all === null || all.length !== 8)
|
|
150
|
+
return null;
|
|
151
|
+
return all;
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* True when `groups` (eight 16-bit values) is an IPv6 address that must never
|
|
155
|
+
* be fetched: unspecified, loopback, link-local, site-local, unique-local,
|
|
156
|
+
* multicast, the documentation prefix, or an address that embeds a private
|
|
157
|
+
* IPv4.
|
|
158
|
+
*/
|
|
159
|
+
function isPrivateIPv6(groups) {
|
|
160
|
+
const first = groups[0] ?? 0;
|
|
161
|
+
const g6 = groups[6] ?? 0;
|
|
162
|
+
const g7 = groups[7] ?? 0;
|
|
163
|
+
if (groups.every((group) => group === 0))
|
|
164
|
+
return true; // :: unspecified
|
|
165
|
+
if (groups.slice(0, 7).every((group) => group === 0) && g7 === 1)
|
|
166
|
+
return true; // ::1 loopback
|
|
167
|
+
if ((first & 0xffc0) === 0xfe80)
|
|
168
|
+
return true; // fe80::/10 link-local
|
|
169
|
+
if ((first & 0xffc0) === 0xfec0)
|
|
170
|
+
return true; // fec0::/10 site-local (deprecated)
|
|
171
|
+
if ((first & 0xfe00) === 0xfc00)
|
|
172
|
+
return true; // fc00::/7 unique local
|
|
173
|
+
if ((first & 0xff00) === 0xff00)
|
|
174
|
+
return true; // ff00::/8 multicast
|
|
175
|
+
if (first === 0x2001 && groups[1] === 0x0db8)
|
|
176
|
+
return true; // 2001:db8::/32 documentation
|
|
177
|
+
const embeddedV4 = [
|
|
178
|
+
g6 >> 8,
|
|
179
|
+
g6 & 0xff,
|
|
180
|
+
g7 >> 8,
|
|
181
|
+
g7 & 0xff,
|
|
182
|
+
];
|
|
183
|
+
if (groups.slice(0, 5).every((group) => group === 0) &&
|
|
184
|
+
(groups[5] === 0xffff || groups[5] === 0x0000)) {
|
|
185
|
+
return isPrivateIPv4(embeddedV4);
|
|
186
|
+
}
|
|
187
|
+
if (first === 0x0064 &&
|
|
188
|
+
groups[1] === 0xff9b &&
|
|
189
|
+
groups.slice(2, 6).every((group) => group === 0)) {
|
|
190
|
+
return isPrivateIPv4(embeddedV4);
|
|
191
|
+
}
|
|
192
|
+
return false;
|
|
193
|
+
}
|
|
194
|
+
/** Hostnames (non-IP) that are never public and must never be fetched. */
|
|
195
|
+
function isBlockedHostname(host) {
|
|
196
|
+
const lower = host.toLowerCase();
|
|
197
|
+
return (lower === "localhost" ||
|
|
198
|
+
lower.endsWith(".localhost") ||
|
|
199
|
+
lower.endsWith(".local") ||
|
|
200
|
+
lower.endsWith(".internal"));
|
|
201
|
+
}
|
|
202
|
+
/**
|
|
203
|
+
* Decide whether a URL host is private, loopback, link-local, or otherwise
|
|
204
|
+
* not safe to fetch from inside the Worker's network. Accepts the raw
|
|
205
|
+
* `URL.hostname` form (IPv6 hosts may arrive wrapped in `[...]`).
|
|
206
|
+
*/
|
|
207
|
+
export function isPrivateOrReservedHost(hostname) {
|
|
208
|
+
if (hostname === "")
|
|
209
|
+
return true;
|
|
210
|
+
const host = (hostname.startsWith("[") && hostname.endsWith("]")
|
|
211
|
+
? hostname.slice(1, -1)
|
|
212
|
+
: hostname).replace(/\.$/, "");
|
|
213
|
+
const v4 = parseIPv4(host);
|
|
214
|
+
if (v4 !== null)
|
|
215
|
+
return isPrivateIPv4(v4);
|
|
216
|
+
const v6 = parseIPv6(host);
|
|
217
|
+
if (v6 !== null)
|
|
218
|
+
return isPrivateIPv6(v6);
|
|
219
|
+
return isBlockedHostname(host);
|
|
220
|
+
}
|
|
221
|
+
/**
|
|
222
|
+
* Validate that `rawUrl` is a fetchable public `http(s)` URL, returning the
|
|
223
|
+
* parsed {@link URL}. Throws {@link SsrfError} for an unparseable URL, a
|
|
224
|
+
* non-`http(s)` scheme (e.g. `file:`, `javascript:`), or a private/reserved
|
|
225
|
+
* host.
|
|
226
|
+
*/
|
|
227
|
+
export function assertPublicUrl(rawUrl) {
|
|
228
|
+
let url;
|
|
229
|
+
try {
|
|
230
|
+
url = new URL(rawUrl);
|
|
231
|
+
}
|
|
232
|
+
catch {
|
|
233
|
+
throw new SsrfError(`invalid URL: ${rawUrl}`, "invalid_url");
|
|
234
|
+
}
|
|
235
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
|
236
|
+
throw new SsrfError(`disallowed scheme: ${url.protocol}`, "disallowed_scheme", url.hostname);
|
|
237
|
+
}
|
|
238
|
+
if (isPrivateOrReservedHost(url.hostname)) {
|
|
239
|
+
throw new SsrfError(`blocked host: ${url.hostname}`, "blocked_host", url.hostname);
|
|
240
|
+
}
|
|
241
|
+
return url;
|
|
242
|
+
}
|
|
243
|
+
/**
|
|
244
|
+
* Fetch `rawUrl` through `doFetch` with SSRF guardrails.
|
|
245
|
+
*
|
|
246
|
+
* The initial host and every redirect target are validated with
|
|
247
|
+
* {@link assertPublicUrl}; redirects are followed manually (`redirect:
|
|
248
|
+
* "manual"`) up to `maxRedirects` hops; and a single {@link AbortSignal.timeout}
|
|
249
|
+
* bounds the whole chain. Credential-bearing headers are stripped on a
|
|
250
|
+
* cross-origin redirect, matching what a browser's `fetch` does.
|
|
251
|
+
*
|
|
252
|
+
* @throws {SsrfError} when a host is blocked, a scheme is disallowed, or the
|
|
253
|
+
* redirect cap is exceeded. Other failures (network, timeout) propagate as the
|
|
254
|
+
* underlying fetch rejection. Callers treat any throw as "fetch failed".
|
|
255
|
+
*/
|
|
256
|
+
export async function safeFetch(doFetch, rawUrl, init, options) {
|
|
257
|
+
const maxRedirects = options?.maxRedirects ?? DEFAULT_MAX_REDIRECTS;
|
|
258
|
+
const timeoutMs = options?.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
259
|
+
const logger = options?.logger ?? noopLogger;
|
|
260
|
+
const metrics = options?.metrics ?? noopMetrics;
|
|
261
|
+
const timeoutSignal = AbortSignal.timeout(timeoutMs);
|
|
262
|
+
const signal = init.signal != null
|
|
263
|
+
? AbortSignal.any([init.signal, timeoutSignal])
|
|
264
|
+
: timeoutSignal;
|
|
265
|
+
try {
|
|
266
|
+
let currentUrl = assertPublicUrl(rawUrl).toString();
|
|
267
|
+
let currentInit = { ...init };
|
|
268
|
+
for (let hop = 0;; hop++) {
|
|
269
|
+
const response = await doFetch(currentUrl, {
|
|
270
|
+
...currentInit,
|
|
271
|
+
redirect: "manual",
|
|
272
|
+
signal,
|
|
273
|
+
});
|
|
274
|
+
if (!REDIRECT_STATUSES.has(response.status)) {
|
|
275
|
+
return { response, url: currentUrl };
|
|
276
|
+
}
|
|
277
|
+
const location = response.headers.get("location");
|
|
278
|
+
if (location === null || location === "") {
|
|
279
|
+
return { response, url: currentUrl };
|
|
280
|
+
}
|
|
281
|
+
if (hop >= maxRedirects) {
|
|
282
|
+
throw new SsrfError(`too many redirects (> ${maxRedirects})`, "too_many_redirects", new URL(currentUrl).host);
|
|
283
|
+
}
|
|
284
|
+
const next = assertPublicUrl(new URL(location, currentUrl).toString());
|
|
285
|
+
await response.body?.cancel().catch(() => undefined);
|
|
286
|
+
if (currentInit.headers && new URL(currentUrl).origin !== next.origin) {
|
|
287
|
+
const headers = new Headers(currentInit.headers);
|
|
288
|
+
for (const name of [
|
|
289
|
+
"authorization",
|
|
290
|
+
"cookie",
|
|
291
|
+
"cookie2",
|
|
292
|
+
"proxy-authorization",
|
|
293
|
+
"set-cookie",
|
|
294
|
+
]) {
|
|
295
|
+
headers.delete(name);
|
|
296
|
+
}
|
|
297
|
+
currentInit = { ...currentInit, headers };
|
|
298
|
+
}
|
|
299
|
+
currentUrl = next.toString();
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
catch (err) {
|
|
303
|
+
if (err instanceof SsrfError) {
|
|
304
|
+
const fields = { reason: err.reason, host: err.host };
|
|
305
|
+
logger.warn(MicrosubLogEvent.SsrfBlocked, fields);
|
|
306
|
+
metrics.count(MicrosubLogEvent.SsrfBlocked, fields);
|
|
307
|
+
}
|
|
308
|
+
throw err;
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
//# sourceMappingURL=safe-fetch.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"safe-fetch.js","sourceRoot":"","sources":["../src/safe-fetch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,EAAE,UAAU,EAAE,WAAW,EAA6B,MAAM,UAAU,CAAC;AAE9E,OAAO,EAAE,gBAAgB,EAAE,MAAM,OAAO,CAAC;AAEzC,gEAAgE;AAChE,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC;AACvC,yEAAyE;AACzE,MAAM,CAAC,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAEzC,+DAA+D;AAC/D,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAS7D;;;;;;GAMG;AACH,MAAM,OAAO,SAAU,SAAQ,KAAK;IACzB,MAAM,CAAa;IACnB,IAAI,CAAU;IACvB,YAAY,OAAe,EAAE,MAAkB,EAAE,IAAa;QAC5D,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,WAAW,CAAC;QACxB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;CACF;AAED,uEAAuE;AACvE,SAAS,SAAS,CAAC,IAAY;IAC7B,MAAM,KAAK,GAAG,8CAA8C,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACxE,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IAChC,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC;QACxC,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;QAC1B,IAAI,IAAI,KAAK,SAAS;YAAE,OAAO,IAAI,CAAC;QACpC,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACxC,IAAI,KAAK,GAAG,GAAG;YAAE,OAAO,IAAI,CAAC;QAC7B,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACrB,CAAC;IACD,OAAO,MAA0C,CAAC;AACpD,CAAC;AAED;;;;;GAKG;AACH,SAAS,aAAa,CAAC,MAAwC;IAC7D,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC;IACzB,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC,CAAC,4CAA4C;IACtE,IAAI,CAAC,KAAK,EAAE;QAAE,OAAO,IAAI,CAAC,CAAC,qBAAqB;IAChD,IAAI,CAAC,KAAK,GAAG;QAAE,OAAO,IAAI,CAAC,CAAC,uBAAuB;IACnD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,GAAG;QAAE,OAAO,IAAI,CAAC,CAAC,sBAAsB;IACzE,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG;QAAE,OAAO,IAAI,CAAC,CAAC,uCAAuC;IAChF,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE;QAAE,OAAO,IAAI,CAAC,CAAC,wBAAwB;IAC1E,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC,CAAC,6BAA6B;IAC/E,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC,CAAC,0BAA0B;IAC5E,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG;QAAE,OAAO,IAAI,CAAC,CAAC,yBAAyB;IAClE,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,GAAG;QAAE,OAAO,IAAI,CAAC,CAAC,6BAA6B;IAClF,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;QAAE,OAAO,IAAI,CAAC,CAAC,0BAA0B;IAChF,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG;QAAE,OAAO,IAAI,CAAC,CAAC,4BAA4B;IAChF,IAAI,CAAC,IAAI,GAAG;QAAE,OAAO,IAAI,CAAC,CAAC,2DAA2D;IACtF,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;GAIG;AACH,SAAS,SAAS,CAAC,IAAY;IAC7B,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IACrC,IAAI,GAAG,GAAG,IAAI,CAAC;IAEf,MAAM,OAAO,GAAG,mCAAmC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC9D,MAAM,KAAK,GAAG,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;IAC3B,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,MAAM,EAAE,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;QAC5B,IAAI,EAAE,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QAC7B,MAAM,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QAC/C,MAAM,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QAC/C,GAAG,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC;IAChE,CAAC;IAED,IAAI,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAE7D,MAAM,QAAQ,GAAG,CAAC,IAAY,EAAmB,EAAE;QACjD,IAAI,IAAI,KAAK,EAAE;YAAE,OAAO,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;YACpC,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,KAAK,CAAC;gBAAE,OAAO,IAAI,CAAC;YACnD,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC;QAC1C,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC,CAAC;IAEF,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC9B,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACtC,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACvC,IAAI,IAAI,KAAK,IAAI,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QACjD,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;QAC/C,IAAI,OAAO,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC;QAC7B,OAAO,CAAC,GAAG,IAAI,EAAE,GAAG,IAAI,KAAK,CAAS,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,CAAC;IACpE,CAAC;IAED,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;IAC1B,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAClD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;GAKG;AACH,SAAS,aAAa,CAAC,MAAgB;IACrC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC7B,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC1B,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC1B,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC,CAAC,iBAAiB;IACxE,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC,CAAC,eAAe;IAC9F,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC,CAAC,uBAAuB;IACrE,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC,CAAC,oCAAoC;IAClF,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC,CAAC,wBAAwB;IACtE,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC,CAAC,qBAAqB;IACnE,IAAI,KAAK,KAAK,MAAM,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC,CAAC,8BAA8B;IAEzF,MAAM,UAAU,GAAqC;QACnD,EAAE,IAAI,CAAC;QACP,EAAE,GAAG,IAAI;QACT,EAAE,IAAI,CAAC;QACP,EAAE,GAAG,IAAI;KACV,CAAC;IACF,IACE,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,CAAC,CAAC;QAChD,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,EAC9C,CAAC;QACD,OAAO,aAAa,CAAC,UAAU,CAAC,CAAC;IACnC,CAAC;IACD,IACE,KAAK,KAAK,MAAM;QAChB,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM;QACpB,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,CAAC,CAAC,EAChD,CAAC;QACD,OAAO,aAAa,CAAC,UAAU,CAAC,CAAC;IACnC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,0EAA0E;AAC1E,SAAS,iBAAiB,CAAC,IAAY;IACrC,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;IACjC,OAAO,CACL,KAAK,KAAK,WAAW;QACrB,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC;QAC5B,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC;QACxB,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,CAC5B,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,uBAAuB,CAAC,QAAgB;IACtD,IAAI,QAAQ,KAAK,EAAE;QAAE,OAAO,IAAI,CAAC;IACjC,MAAM,IAAI,GAAG,CACX,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC;QAChD,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACvB,CAAC,CAAC,QAAQ,CACb,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAErB,MAAM,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAC3B,IAAI,EAAE,KAAK,IAAI;QAAE,OAAO,aAAa,CAAC,EAAE,CAAC,CAAC;IAC1C,MAAM,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAC3B,IAAI,EAAE,KAAK,IAAI;QAAE,OAAO,aAAa,CAAC,EAAE,CAAC,CAAC;IAC1C,OAAO,iBAAiB,CAAC,IAAI,CAAC,CAAC;AACjC,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,eAAe,CAAC,MAAc;IAC5C,IAAI,GAAQ,CAAC;IACb,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IACxB,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,SAAS,CAAC,gBAAgB,MAAM,EAAE,EAAE,aAAa,CAAC,CAAC;IAC/D,CAAC;IACD,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1D,MAAM,IAAI,SAAS,CACjB,sBAAsB,GAAG,CAAC,QAAQ,EAAE,EACpC,mBAAmB,EACnB,GAAG,CAAC,QAAQ,CACb,CAAC;IACJ,CAAC;IACD,IAAI,uBAAuB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1C,MAAM,IAAI,SAAS,CACjB,iBAAiB,GAAG,CAAC,QAAQ,EAAE,EAC/B,cAAc,EACd,GAAG,CAAC,QAAQ,CACb,CAAC;IACJ,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAgBD;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,OAAkB,EAClB,MAAc,EACd,IAAiB,EACjB,OAA0B;IAE1B,MAAM,YAAY,GAAG,OAAO,EAAE,YAAY,IAAI,qBAAqB,CAAC;IACpE,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,kBAAkB,CAAC;IAC3D,MAAM,MAAM,GAAG,OAAO,EAAE,MAAM,IAAI,UAAU,CAAC;IAC7C,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,IAAI,WAAW,CAAC;IAChD,MAAM,aAAa,GAAG,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACrD,MAAM,MAAM,GACV,IAAI,CAAC,MAAM,IAAI,IAAI;QACjB,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;QAC/C,CAAC,CAAC,aAAa,CAAC;IAEpB,IAAI,CAAC;QACH,IAAI,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC;QACpD,IAAI,WAAW,GAAgB,EAAE,GAAG,IAAI,EAAE,CAAC;QAC3C,KAAK,IAAI,GAAG,GAAG,CAAC,GAAI,GAAG,EAAE,EAAE,CAAC;YAC1B,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,UAAU,EAAE;gBACzC,GAAG,WAAW;gBACd,QAAQ,EAAE,QAAQ;gBAClB,MAAM;aACP,CAAC,CAAC;YAEH,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC5C,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC;YACvC,CAAC;YAED,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAClD,IAAI,QAAQ,KAAK,IAAI,IAAI,QAAQ,KAAK,EAAE,EAAE,CAAC;gBACzC,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC;YACvC,CAAC;YACD,IAAI,GAAG,IAAI,YAAY,EAAE,CAAC;gBACxB,MAAM,IAAI,SAAS,CACjB,yBAAyB,YAAY,GAAG,EACxC,oBAAoB,EACpB,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,IAAI,CACzB,CAAC;YACJ,CAAC;YAED,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;YACvE,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YAErD,IAAI,WAAW,CAAC,OAAO,IAAI,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;gBACtE,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,WAAW,CAAC,OAAsB,CAAC,CAAC;gBAChE,KAAK,MAAM,IAAI,IAAI;oBACjB,eAAe;oBACf,QAAQ;oBACR,SAAS;oBACT,qBAAqB;oBACrB,YAAY;iBACb,EAAE,CAAC;oBACF,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBACvB,CAAC;gBACD,WAAW,GAAG,EAAE,GAAG,WAAW,EAAE,OAAO,EAAE,CAAC;YAC5C,CAAC;YACD,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC/B,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,GAAG,YAAY,SAAS,EAAE,CAAC;YAC7B,MAAM,MAAM,GAAG,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC;YACtD,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;YAClD,OAAO,CAAC,KAAK,CAAC,gBAAgB,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;QACtD,CAAC;QACD,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC"}
|
package/dist/store.d.ts
ADDED
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@dwk/microsub` — D1-backed authoritative state.
|
|
3
|
+
*
|
|
4
|
+
* Channels, the feeds each channel follows, the normalised JF2 timeline items,
|
|
5
|
+
* their per-item read flags, and a small per-feed poll cache (`ETag` /
|
|
6
|
+
* `Last-Modified`) all live here. This is correctness-sensitive state — a lost
|
|
7
|
+
* subscription or a dropped read flag is a bug, not a stale cache — so it lives
|
|
8
|
+
* in D1, a strongly-consistent store, and **never** KV (see
|
|
9
|
+
* `spec/non-functional-requirements.md`). The schema is created lazily on first
|
|
10
|
+
* use.
|
|
11
|
+
*
|
|
12
|
+
* Timeline paging uses a monotonic `seq` (an autoincrement rowid): the opaque
|
|
13
|
+
* `before` / `after` cursors a Microsub client round-trips are just `seq`
|
|
14
|
+
* values, so paging is stable under concurrent inserts and deletes.
|
|
15
|
+
*
|
|
16
|
+
* @packageDocumentation
|
|
17
|
+
*/
|
|
18
|
+
import type { D1Database } from "@cloudflare/workers-types";
|
|
19
|
+
import type { Jf2Entry } from "./jf2";
|
|
20
|
+
/** The reserved channel that always exists and cannot be deleted or renamed. */
|
|
21
|
+
export declare const NOTIFICATIONS_CHANNEL = "notifications";
|
|
22
|
+
/** Cloudflare binding required by the Microsub store. */
|
|
23
|
+
export interface MicrosubStoreEnv {
|
|
24
|
+
/** D1 database holding channels, follows, items, and the poll cache. */
|
|
25
|
+
readonly MICROSUB_DB: D1Database;
|
|
26
|
+
}
|
|
27
|
+
/** A channel: its stable uid, display name, and sort position. */
|
|
28
|
+
export interface ChannelRecord {
|
|
29
|
+
readonly uid: string;
|
|
30
|
+
readonly name: string;
|
|
31
|
+
readonly position: number;
|
|
32
|
+
readonly createdAt: number;
|
|
33
|
+
}
|
|
34
|
+
/** A subscription within a channel. */
|
|
35
|
+
export interface FollowRecord {
|
|
36
|
+
/** The resolved feed URL polled by the server. */
|
|
37
|
+
readonly feedUrl: string;
|
|
38
|
+
/** The page URL the user originally followed (feed discovered from it). */
|
|
39
|
+
readonly pageUrl: string;
|
|
40
|
+
readonly createdAt: number;
|
|
41
|
+
}
|
|
42
|
+
/** A stored timeline item: the JF2 entry plus its bookkeeping. */
|
|
43
|
+
export interface StoredItem {
|
|
44
|
+
readonly seq: number;
|
|
45
|
+
readonly channel: string;
|
|
46
|
+
readonly entryId: string;
|
|
47
|
+
readonly feedUrl: string;
|
|
48
|
+
readonly isRead: boolean;
|
|
49
|
+
readonly entry: Jf2Entry;
|
|
50
|
+
}
|
|
51
|
+
/** A page of timeline items with the opaque cursors to fetch adjacent pages. */
|
|
52
|
+
export interface ItemPage {
|
|
53
|
+
readonly items: readonly StoredItem[];
|
|
54
|
+
readonly before?: string;
|
|
55
|
+
readonly after?: string;
|
|
56
|
+
}
|
|
57
|
+
/** A cached conditional-request validator for a feed. */
|
|
58
|
+
export interface FeedCache {
|
|
59
|
+
readonly etag: string | null;
|
|
60
|
+
readonly lastModified: string | null;
|
|
61
|
+
}
|
|
62
|
+
/** Options for {@link MicrosubStore.listItems}. */
|
|
63
|
+
export interface ListOptions {
|
|
64
|
+
readonly before?: string;
|
|
65
|
+
readonly after?: string;
|
|
66
|
+
readonly limit: number;
|
|
67
|
+
}
|
|
68
|
+
/** Storage interface over the Microsub state. */
|
|
69
|
+
export interface MicrosubStore {
|
|
70
|
+
/** Create the schema if absent. Idempotent. */
|
|
71
|
+
init(): Promise<void>;
|
|
72
|
+
/** List channels in sort order, the reserved `notifications` channel first. */
|
|
73
|
+
listChannels(): Promise<ChannelRecord[]>;
|
|
74
|
+
/** Whether a channel with this uid exists. */
|
|
75
|
+
channelExists(uid: string): Promise<boolean>;
|
|
76
|
+
/** Create a channel with a freshly generated uid. */
|
|
77
|
+
createChannel(name: string, now: number): Promise<ChannelRecord>;
|
|
78
|
+
/** Rename a channel; returns the updated record, or `null` if unknown. */
|
|
79
|
+
renameChannel(uid: string, name: string): Promise<ChannelRecord | null>;
|
|
80
|
+
/**
|
|
81
|
+
* Delete a channel and everything it owns (follows + items). Returns `false`
|
|
82
|
+
* if the uid is unknown or names the reserved channel.
|
|
83
|
+
*/
|
|
84
|
+
deleteChannel(uid: string): Promise<boolean>;
|
|
85
|
+
/** Apply a new sort order; uids absent from `order` keep their relative order after. */
|
|
86
|
+
reorderChannels(order: readonly string[]): Promise<void>;
|
|
87
|
+
/** List a channel's subscriptions. */
|
|
88
|
+
listFollows(channel: string): Promise<FollowRecord[]>;
|
|
89
|
+
/** Add (or refresh) a subscription. */
|
|
90
|
+
addFollow(channel: string, feedUrl: string, pageUrl: string, now: number): Promise<void>;
|
|
91
|
+
/**
|
|
92
|
+
* Remove a subscription, matching `url` against either the resolved feed URL
|
|
93
|
+
* or the page URL the user originally followed (a client unfollows with the
|
|
94
|
+
* URL it was given back, which is the page URL). Returns `false` if it was not
|
|
95
|
+
* present.
|
|
96
|
+
*/
|
|
97
|
+
removeFollow(channel: string, url: string): Promise<boolean>;
|
|
98
|
+
/** Every distinct feed URL across all channels (for the poller). */
|
|
99
|
+
distinctFeedUrls(): Promise<string[]>;
|
|
100
|
+
/** Channels that follow a given feed URL (for the consumer). */
|
|
101
|
+
channelsForFeed(feedUrl: string): Promise<string[]>;
|
|
102
|
+
/**
|
|
103
|
+
* Append entries to a channel, deduped by entry id. Returns the count newly
|
|
104
|
+
* inserted. Entries should be passed newest-last so the newest gets the
|
|
105
|
+
* highest `seq`. Reaps beyond `maxItems` oldest items afterwards.
|
|
106
|
+
*/
|
|
107
|
+
insertItems(channel: string, feedUrl: string, entries: readonly Jf2Entry[], now: number, maxItems: number): Promise<number>;
|
|
108
|
+
/** A page of a channel's timeline. */
|
|
109
|
+
listItems(channel: string, options: ListOptions): Promise<ItemPage>;
|
|
110
|
+
/** Count of unread items in a channel. */
|
|
111
|
+
unreadCount(channel: string): Promise<number>;
|
|
112
|
+
/** Mark specific items read/unread. */
|
|
113
|
+
markRead(channel: string, entryIds: readonly string[], read: boolean): Promise<void>;
|
|
114
|
+
/** Mark every item up to and including `entryId` (by `seq`) as read. */
|
|
115
|
+
markReadThrough(channel: string, entryId: string): Promise<void>;
|
|
116
|
+
/** Remove an item from a channel. Returns `false` if it was not present. */
|
|
117
|
+
removeItem(channel: string, entryId: string): Promise<boolean>;
|
|
118
|
+
/** Substring search over a channel's items (name + content). */
|
|
119
|
+
searchItems(channel: string, query: string, limit: number): Promise<StoredItem[]>;
|
|
120
|
+
/** The cached validators for a feed, or `null`. */
|
|
121
|
+
getFeedCache(feedUrl: string): Promise<FeedCache | null>;
|
|
122
|
+
/** Record the validators returned by a feed fetch. */
|
|
123
|
+
setFeedCache(feedUrl: string, etag: string | null, lastModified: string | null, now: number): Promise<void>;
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* Create the D1-backed {@link MicrosubStore}. Fails loudly if the required
|
|
127
|
+
* `MICROSUB_DB` binding is missing — no silent degradation (composition
|
|
128
|
+
* contract).
|
|
129
|
+
*/
|
|
130
|
+
export declare function createMicrosubStore(env: MicrosubStoreEnv): MicrosubStore;
|
|
131
|
+
//# sourceMappingURL=store.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../src/store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAE5D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,OAAO,CAAC;AAEtC,gFAAgF;AAChF,eAAO,MAAM,qBAAqB,kBAAkB,CAAC;AAErD,yDAAyD;AACzD,MAAM,WAAW,gBAAgB;IAC/B,wEAAwE;IACxE,QAAQ,CAAC,WAAW,EAAE,UAAU,CAAC;CAClC;AAED,kEAAkE;AAClE,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAED,uCAAuC;AACvC,MAAM,WAAW,YAAY;IAC3B,kDAAkD;IAClD,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,2EAA2E;IAC3E,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAED,kEAAkE;AAClE,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC;CAC1B;AAED,gFAAgF;AAChF,MAAM,WAAW,QAAQ;IACvB,QAAQ,CAAC,KAAK,EAAE,SAAS,UAAU,EAAE,CAAC;IACtC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,yDAAyD;AACzD,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CACtC;AAED,mDAAmD;AACnD,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED,iDAAiD;AACjD,MAAM,WAAW,aAAa;IAC5B,+CAA+C;IAC/C,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAGtB,+EAA+E;IAC/E,YAAY,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC;IACzC,8CAA8C;IAC9C,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC7C,qDAAqD;IACrD,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IACjE,0EAA0E;IAC1E,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CAAC;IACxE;;;OAGG;IACH,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC7C,wFAAwF;IACxF,eAAe,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAGzD,sCAAsC;IACtC,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;IACtD,uCAAuC;IACvC,SAAS,CACP,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,MAAM,EACf,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,IAAI,CAAC,CAAC;IACjB;;;;;OAKG;IACH,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC7D,oEAAoE;IACpE,gBAAgB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACtC,gEAAgE;IAChE,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAGpD;;;;OAIG;IACH,WAAW,CACT,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,SAAS,QAAQ,EAAE,EAC5B,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,MAAM,CAAC,CAAC;IACnB,sCAAsC;IACtC,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IACpE,0CAA0C;IAC1C,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAC9C,uCAAuC;IACvC,QAAQ,CACN,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,SAAS,MAAM,EAAE,EAC3B,IAAI,EAAE,OAAO,GACZ,OAAO,CAAC,IAAI,CAAC,CAAC;IACjB,wEAAwE;IACxE,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACjE,4EAA4E;IAC5E,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC/D,gEAAgE;IAChE,WAAW,CACT,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;IAGzB,mDAAmD;IACnD,YAAY,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC;IACzD,sDAAsD;IACtD,YAAY,CACV,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,MAAM,GAAG,IAAI,EACnB,YAAY,EAAE,MAAM,GAAG,IAAI,EAC3B,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,IAAI,CAAC,CAAC;CAClB;AAgGD;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,gBAAgB,GAAG,aAAa,CAwXxE"}
|