@wave-rf/cloudflare-md-router 0.0.0-dev.hd43e9a23e496 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -0
- package/package.json +1 -1
- package/src/worker.ts +36 -3
package/README.md
CHANGED
|
@@ -22,6 +22,8 @@ Link: </foo/bar.md>; rel="alternate"; type="text/markdown"
|
|
|
22
22
|
|
|
23
23
|
This is on by default (only for a `200 text/html` reply to an extension-less GET); disable it with `advertiseTwin: false`.
|
|
24
24
|
|
|
25
|
+
Because the worker negotiates on the `Accept` header, two clients can get different representations of the same URL. If you put a shared cache (a CDN, Cloudflare's own cache) in front of it, set `vary: true` to add a `Vary: Accept` header to the negotiated responses so the cache keys on `Accept` and doesn't serve the HTML page to a client that asked for markdown — or vice-versa. It's off by default (verbatim pass-through stays byte-for-byte; `Vary: User-Agent` is intentionally not added, as it would defeat shared caching).
|
|
26
|
+
|
|
25
27
|
The included bot list covers the common ones: GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, Claude-Web, anthropic-ai, PerplexityBot, CCBot, Applebot-Extended, Google-Extended, cohere-ai, Bytespider, Diffbot. See `src/bots.ts`.
|
|
26
28
|
|
|
27
29
|
## Install
|
|
@@ -80,6 +82,10 @@ export default createMdRouter({
|
|
|
80
82
|
|
|
81
83
|
// Don't advertise the `.md` twin via a `Link` header (default: true).
|
|
82
84
|
advertiseTwin: false,
|
|
85
|
+
|
|
86
|
+
// Add `Vary: Accept` to negotiated responses so a shared cache doesn't
|
|
87
|
+
// cross-serve the HTML and markdown representations (default: false).
|
|
88
|
+
vary: true,
|
|
83
89
|
});
|
|
84
90
|
```
|
|
85
91
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@wave-rf/cloudflare-md-router",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.1",
|
|
4
4
|
"description": "Cloudflare Worker that serves a `.md` twin of any static page when the request is from a known LLM crawler or explicitly asks for `text/markdown`. Falls back to the HTML response otherwise.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"cloudflare",
|
package/src/worker.ts
CHANGED
|
@@ -21,6 +21,17 @@ export interface MdRouterOptions {
|
|
|
21
21
|
* `text/html` reply to an extension-less GET); the `.md` and pass-through
|
|
22
22
|
* responses are untouched. Default: `true`. */
|
|
23
23
|
advertiseTwin?: boolean;
|
|
24
|
+
/** Append `Vary: Accept` to the responses this worker negotiates on the
|
|
25
|
+
* `Accept` header (the `.md` twin, the HTML fallback when the twin is
|
|
26
|
+
* missing, and the normal HTML page), so a shared cache keyed only on the
|
|
27
|
+
* URL doesn't serve the HTML representation to a client that asked for
|
|
28
|
+
* markdown — or vice-versa (RFC 9110 §12.5.5). Merges with any existing
|
|
29
|
+
* `Vary` rather than replacing it; verbatim pass-through responses (non-GET,
|
|
30
|
+
* extension-bearing paths) are left untouched. Only `Accept` is listed:
|
|
31
|
+
* `User-Agent` is also a routing input, but `Vary: User-Agent` defeats
|
|
32
|
+
* shared caching (huge UA cardinality), so it is intentionally omitted.
|
|
33
|
+
* Default: `false`. */
|
|
34
|
+
vary?: boolean;
|
|
24
35
|
}
|
|
25
36
|
|
|
26
37
|
const defaultMdPathFor = (pathname: string): string => {
|
|
@@ -36,7 +47,10 @@ const defaultMdPathFor = (pathname: string): string => {
|
|
|
36
47
|
* Pass through unchanged for non-GET, requests with a file extension,
|
|
37
48
|
* and "normal" browser requests — though the normal HTML page response also
|
|
38
49
|
* gets a `Link` header advertising its `.md` twin unless `advertiseTwin` is
|
|
39
|
-
* disabled (see {@link MdRouterOptions.advertiseTwin}).
|
|
50
|
+
* disabled (see {@link MdRouterOptions.advertiseTwin}). Opt into a
|
|
51
|
+
* `Vary: Accept` header on the `Accept`-negotiated responses with
|
|
52
|
+
* {@link MdRouterOptions.vary} so shared caches don't cross-serve the two
|
|
53
|
+
* representations.
|
|
40
54
|
*
|
|
41
55
|
* Requires an `ASSETS` Fetcher binding in your `wrangler.jsonc`:
|
|
42
56
|
* ```jsonc
|
|
@@ -59,6 +73,19 @@ export function createMdRouter<Env extends MdRouterEnv = MdRouterEnv>(
|
|
|
59
73
|
const mdPathFor = options.mdPathFor ?? defaultMdPathFor;
|
|
60
74
|
const acceptTokens = ["text/markdown", ...(options.acceptMarkdown ?? [])];
|
|
61
75
|
const advertiseTwin = options.advertiseTwin ?? true;
|
|
76
|
+
const vary = options.vary ?? false;
|
|
77
|
+
|
|
78
|
+
// Append `Vary: Accept` to a negotiated response when `vary` is enabled.
|
|
79
|
+
// Wraps via `new Response(body, response)` because an asset response's
|
|
80
|
+
// headers are immutable; `append` (not `set`) preserves any existing `Vary`.
|
|
81
|
+
const withVary = (response: Response): Response => {
|
|
82
|
+
if (!vary) {
|
|
83
|
+
return response;
|
|
84
|
+
}
|
|
85
|
+
const varied = new Response(response.body, response);
|
|
86
|
+
varied.headers.append("Vary", "Accept");
|
|
87
|
+
return varied;
|
|
88
|
+
};
|
|
62
89
|
|
|
63
90
|
return {
|
|
64
91
|
async fetch(request, env): Promise<Response> {
|
|
@@ -80,7 +107,7 @@ export function createMdRouter<Env extends MdRouterEnv = MdRouterEnv>(
|
|
|
80
107
|
.trim();
|
|
81
108
|
const isHtml = contentType === "text/html";
|
|
82
109
|
if (!advertiseTwin || response.status !== 200 || !isHtml) {
|
|
83
|
-
return response;
|
|
110
|
+
return withVary(response);
|
|
84
111
|
}
|
|
85
112
|
// `new Response(body, response)` is the only way to add a header to an
|
|
86
113
|
// otherwise-immutable asset response. A relative URI-Reference target is
|
|
@@ -91,13 +118,19 @@ export function createMdRouter<Env extends MdRouterEnv = MdRouterEnv>(
|
|
|
91
118
|
.map((segment) => encodeURIComponent(segment))
|
|
92
119
|
.join("/");
|
|
93
120
|
withTwin.headers.append("Link", `<${encodedPath}>; rel="alternate"; type="text/markdown"`);
|
|
121
|
+
// Already a mutable wrapper — append `Vary` here rather than re-wrapping.
|
|
122
|
+
if (vary) {
|
|
123
|
+
withTwin.headers.append("Vary", "Accept");
|
|
124
|
+
}
|
|
94
125
|
return withTwin;
|
|
95
126
|
}
|
|
96
127
|
|
|
97
128
|
const mdPath = mdPathFor(url.pathname);
|
|
98
129
|
const mdResponse = await env.ASSETS.fetch(new Request(new URL(mdPath, url.origin), request));
|
|
99
130
|
|
|
100
|
-
return mdResponse.status === 404
|
|
131
|
+
return mdResponse.status === 404
|
|
132
|
+
? withVary(await env.ASSETS.fetch(request))
|
|
133
|
+
: withVary(mdResponse);
|
|
101
134
|
},
|
|
102
135
|
};
|
|
103
136
|
}
|