@wave-rf/cloudflare-md-router 0.2.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -22,6 +22,8 @@ Link: </foo/bar.md>; rel="alternate"; type="text/markdown"
22
22
 
23
23
  This is on by default (only for a `200 text/html` reply to an extension-less GET); disable it with `advertiseTwin: false`.
24
24
 
25
+ Because the worker negotiates on the `Accept` header, two clients can get different representations of the same URL. If you put a shared cache (a CDN, Cloudflare's own cache) in front of it, set `vary: true` to add a `Vary: Accept` header to the negotiated responses so the cache keys on `Accept` and doesn't serve the HTML page to a client that asked for markdown — or vice-versa. It's off by default (verbatim pass-through stays byte-for-byte; `Vary: User-Agent` is intentionally not added, as it would defeat shared caching).
26
+
25
27
  The included bot list covers the common ones: GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, Claude-Web, anthropic-ai, PerplexityBot, CCBot, Applebot-Extended, Google-Extended, cohere-ai, Bytespider, Diffbot. See `src/bots.ts`.
26
28
 
27
29
  ## Install
@@ -80,6 +82,10 @@ export default createMdRouter({
80
82
 
81
83
  // Don't advertise the `.md` twin via a `Link` header (default: true).
82
84
  advertiseTwin: false,
85
+
86
+ // Add `Vary: Accept` to negotiated responses so a shared cache doesn't
87
+ // cross-serve the HTML and markdown representations (default: false).
88
+ vary: true,
83
89
  });
84
90
  ```
85
91
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@wave-rf/cloudflare-md-router",
3
- "version": "0.2.0",
3
+ "version": "0.2.1",
4
4
  "description": "Cloudflare Worker that serves a `.md` twin of any static page when the request is from a known LLM crawler or explicitly asks for `text/markdown`. Falls back to the HTML response otherwise.",
5
5
  "keywords": [
6
6
  "cloudflare",
@@ -62,6 +62,6 @@
62
62
  "devDependencies": {
63
63
  "@biomejs/biome": "^2.4.15",
64
64
  "@cloudflare/workers-types": "^4.20240208.0",
65
- "typescript": "^5.6.0"
65
+ "typescript": "^6.0.3"
66
66
  }
67
67
  }
package/src/worker.ts CHANGED
@@ -21,6 +21,17 @@ export interface MdRouterOptions {
21
21
  * `text/html` reply to an extension-less GET); the `.md` and pass-through
22
22
  * responses are untouched. Default: `true`. */
23
23
  advertiseTwin?: boolean;
24
+ /** Append `Vary: Accept` to the responses this worker negotiates on the
25
+ * `Accept` header (the `.md` twin, the HTML fallback when the twin is
26
+ * missing, and the normal HTML page), so a shared cache keyed only on the
27
+ * URL doesn't serve the HTML representation to a client that asked for
28
+ * markdown — or vice-versa (RFC 9110 §12.5.5). Merges with any existing
29
+ * `Vary` rather than replacing it; verbatim pass-through responses (non-GET,
30
+ * extension-bearing paths) are left untouched. Only `Accept` is listed:
31
+ * `User-Agent` is also a routing input, but `Vary: User-Agent` defeats
32
+ * shared caching (huge UA cardinality), so it is intentionally omitted.
33
+ * Default: `false`. */
34
+ vary?: boolean;
24
35
  }
25
36
 
26
37
  const defaultMdPathFor = (pathname: string): string => {
@@ -36,7 +47,10 @@ const defaultMdPathFor = (pathname: string): string => {
36
47
  * Pass through unchanged for non-GET, requests with a file extension,
37
48
  * and "normal" browser requests — though the normal HTML page response also
38
49
  * gets a `Link` header advertising its `.md` twin unless `advertiseTwin` is
39
- * disabled (see {@link MdRouterOptions.advertiseTwin}).
50
+ * disabled (see {@link MdRouterOptions.advertiseTwin}). Opt into a
51
+ * `Vary: Accept` header on the `Accept`-negotiated responses with
52
+ * {@link MdRouterOptions.vary} so shared caches don't cross-serve the two
53
+ * representations.
40
54
  *
41
55
  * Requires an `ASSETS` Fetcher binding in your `wrangler.jsonc`:
42
56
  * ```jsonc
@@ -59,6 +73,19 @@ export function createMdRouter<Env extends MdRouterEnv = MdRouterEnv>(
59
73
  const mdPathFor = options.mdPathFor ?? defaultMdPathFor;
60
74
  const acceptTokens = ["text/markdown", ...(options.acceptMarkdown ?? [])];
61
75
  const advertiseTwin = options.advertiseTwin ?? true;
76
+ const vary = options.vary ?? false;
77
+
78
+ // Append `Vary: Accept` to a negotiated response when `vary` is enabled.
79
+ // Wraps via `new Response(body, response)` because an asset response's
80
+ // headers are immutable; `append` (not `set`) preserves any existing `Vary`.
81
+ const withVary = (response: Response): Response => {
82
+ if (!vary) {
83
+ return response;
84
+ }
85
+ const varied = new Response(response.body, response);
86
+ varied.headers.append("Vary", "Accept");
87
+ return varied;
88
+ };
62
89
 
63
90
  return {
64
91
  async fetch(request, env): Promise<Response> {
@@ -80,7 +107,7 @@ export function createMdRouter<Env extends MdRouterEnv = MdRouterEnv>(
80
107
  .trim();
81
108
  const isHtml = contentType === "text/html";
82
109
  if (!advertiseTwin || response.status !== 200 || !isHtml) {
83
- return response;
110
+ return withVary(response);
84
111
  }
85
112
  // `new Response(body, response)` is the only way to add a header to an
86
113
  // otherwise-immutable asset response. A relative URI-Reference target is
@@ -91,13 +118,19 @@ export function createMdRouter<Env extends MdRouterEnv = MdRouterEnv>(
91
118
  .map((segment) => encodeURIComponent(segment))
92
119
  .join("/");
93
120
  withTwin.headers.append("Link", `<${encodedPath}>; rel="alternate"; type="text/markdown"`);
121
+ // Already a mutable wrapper — append `Vary` here rather than re-wrapping.
122
+ if (vary) {
123
+ withTwin.headers.append("Vary", "Accept");
124
+ }
94
125
  return withTwin;
95
126
  }
96
127
 
97
128
  const mdPath = mdPathFor(url.pathname);
98
129
  const mdResponse = await env.ASSETS.fetch(new Request(new URL(mdPath, url.origin), request));
99
130
 
100
- return mdResponse.status === 404 ? env.ASSETS.fetch(request) : mdResponse;
131
+ return mdResponse.status === 404
132
+ ? withVary(await env.ASSETS.fetch(request))
133
+ : withVary(mdResponse);
101
134
  },
102
135
  };
103
136
  }