@wave-rf/cloudflare-md-router 0.0.0-dev.h3bcd1aebd5d9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Wave RF
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md ADDED
@@ -0,0 +1,102 @@
1
+ # cloudflare-md-router
2
+
3
+ A tiny Cloudflare Worker that serves the `.md` twin of any static page when the request is from a known LLM crawler or explicitly asks for `text/markdown`. Falls back to the HTML response when the `.md` twin doesn't exist.
4
+
5
+ If you're building a docs site that already emits a per-page raw-markdown twin (e.g. `/foo/bar` and `/foo/bar.md`), this lets every page do content negotiation transparently — Claude, ChatGPT, Perplexity, etc. fetch the model-friendly version automatically; humans keep getting the styled HTML page.
6
+
7
+ ## Behavior
8
+
9
+ | Request | Worker serves |
10
+ | ---------------------------------------------- | ------------------- |
11
+ | Anything with a file extension (`.css`, `.png`, `.md`, …) | Pass-through to ASSETS |
12
+ | Non-GET | Pass-through to ASSETS |
13
+ | `Accept: text/markdown` | `<path>.md` (HTML fallback on 404) |
14
+ | `User-Agent` matches a known LLM bot | `<path>.md` (HTML fallback on 404) |
15
+ | Everything else | HTML page — plus a `Link` header advertising its `.md` twin |
16
+
17
+ On the normal HTML page response, the worker also adds an [RFC 8288](https://www.rfc-editor.org/rfc/rfc8288) `Link` header so an agent can discover the markdown twin from a plain GET, without sniffing the User-Agent or guessing the right `Accept`:
18
+
19
+ ```http
20
+ Link: </foo/bar.md>; rel="alternate"; type="text/markdown"
21
+ ```
22
+
23
+ This is on by default (only for a `200 text/html` reply to an extension-less GET); disable it with `advertiseTwin: false`.
24
+
25
+ Because the worker negotiates on the `Accept` header, two clients can get different representations of the same URL. If you put a shared cache (a CDN, Cloudflare's own cache) in front of it, set `vary: true` to add a `Vary: Accept` header to the negotiated responses so the cache keys on `Accept` and doesn't serve the HTML page to a client that asked for markdown — or vice-versa. It's off by default (verbatim pass-through stays byte-for-byte; `Vary: User-Agent` is intentionally not added, as it would defeat shared caching).
26
+
27
+ The included bot list covers the common ones: GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, Claude-Web, anthropic-ai, PerplexityBot, CCBot, Applebot-Extended, Google-Extended, cohere-ai, Bytespider, Diffbot. See `src/bots.ts`.
28
+
29
+ ## Install
30
+
31
+ ```sh
32
+ pnpm add @wave-rf/cloudflare-md-router
33
+ ```
34
+
35
+ This package ships raw TypeScript with no build step — bundle it with Wrangler/esbuild (the Cloudflare Workers default), which resolve the `.ts` entry points directly.
36
+
37
+ ## Use
38
+
39
+ The simplest setup — re-export the default handler from your worker entrypoint:
40
+
41
+ ```ts
42
+ // worker/index.ts
43
+ export { default } from "@wave-rf/cloudflare-md-router/worker";
44
+ ```
45
+
46
+ Configure your `wrangler.jsonc` with an ASSETS binding pointing at your built static site:
47
+
48
+ ```jsonc
49
+ {
50
+ "name": "my-docs",
51
+ "main": "worker/index.ts",
52
+ "compatibility_date": "2025-01-01",
53
+ "assets": {
54
+ "directory": "./dist",
55
+ "binding": "ASSETS",
56
+ "not_found_handling": "404-page",
57
+ "html_handling": "drop-trailing-slash",
58
+ "run_worker_first": true
59
+ }
60
+ }
61
+ ```
62
+
63
+ `run_worker_first` is required so the worker sees the request before Cloudflare's static-asset matcher does — otherwise the worker only ever runs on 404s.
64
+
65
+ ## Customizing
66
+
67
+ Use `createMdRouter()` if you need to extend the bot list, change the `.md` path mapping, or add other Accept tokens:
68
+
69
+ ```ts
70
+ // worker/index.ts
71
+ import { createMdRouter, LLM_BOT_UA } from "@wave-rf/cloudflare-md-router";
72
+
73
+ export default createMdRouter({
74
+ // Add your own bots:
75
+ botUserAgents: new RegExp(LLM_BOT_UA.source + "|mybot", "i"),
76
+
77
+ // Treat `Accept: text/x-markdown` as markdown too:
78
+ acceptMarkdown: ["text/x-markdown"],
79
+
80
+ // Custom .md path strategy. Default: `/foo/` → `/foo.md`, `/` → `/index.md`.
81
+ mdPathFor: (pathname) => `/markdown${pathname.replace(/\/$/, "")}.md`,
82
+
83
+ // Don't advertise the `.md` twin via a `Link` header (default: true).
84
+ advertiseTwin: false,
85
+
86
+ // Add `Vary: Accept` to negotiated responses so a shared cache doesn't
87
+ // cross-serve the HTML and markdown representations (default: false).
88
+ vary: true,
89
+ });
90
+ ```
91
+
92
+ ## Why content-negotiate?
93
+
94
+ Most LLMs do better with raw markdown than with rendered HTML — less DOM noise, no Starlight nav chrome, no script tags. Serving the same content at one URL with two representations means:
95
+
96
+ - One canonical URL per page (good for citations and link-sharing).
97
+ - Crawlers and human readers stay aligned automatically.
98
+ - Your `llms.txt` can advertise `<page>.md` for explicit fetches; the worker covers the case where the LLM hits the HTML URL anyway.
99
+
100
+ ## License
101
+
102
+ MIT — see [LICENSE](./LICENSE).
package/package.json ADDED
@@ -0,0 +1,67 @@
1
+ {
2
+ "name": "@wave-rf/cloudflare-md-router",
3
+ "version": "0.0.0-dev.h3bcd1aebd5d9",
4
+ "description": "Cloudflare Worker that serves a `.md` twin of any static page when the request is from a known LLM crawler or explicitly asks for `text/markdown`. Falls back to the HTML response otherwise.",
5
+ "keywords": [
6
+ "cloudflare",
7
+ "cloudflare-workers",
8
+ "llm",
9
+ "llms-txt",
10
+ "markdown",
11
+ "content-negotiation",
12
+ "static-site"
13
+ ],
14
+ "license": "MIT",
15
+ "author": {
16
+ "name": "Wave RF",
17
+ "email": "github@wave-rf.com",
18
+ "url": "https://wave-rf.com"
19
+ },
20
+ "contributors": [
21
+ "Eric Andrechek <eric@wave-rf.com>"
22
+ ],
23
+ "repository": {
24
+ "type": "git",
25
+ "url": "git+https://github.com/Wave-RF/cloudflare-md-router.git"
26
+ },
27
+ "bugs": {
28
+ "url": "https://github.com/Wave-RF/cloudflare-md-router/issues"
29
+ },
30
+ "homepage": "https://github.com/Wave-RF/cloudflare-md-router#readme",
31
+ "type": "module",
32
+ "main": "./src/index.ts",
33
+ "types": "./src/index.ts",
34
+ "exports": {
35
+ ".": "./src/index.ts",
36
+ "./worker": "./src/worker.ts",
37
+ "./bots": "./src/bots.ts"
38
+ },
39
+ "files": [
40
+ "src",
41
+ "README.md",
42
+ "LICENSE"
43
+ ],
44
+ "publishConfig": {
45
+ "access": "public"
46
+ },
47
+ "packageManager": "pnpm@11.1.3",
48
+ "engines": {
49
+ "node": ">=18.0.0"
50
+ },
51
+ "scripts": {
52
+ "typecheck": "tsc --noEmit",
53
+ "lint": "biome lint .",
54
+ "format": "biome format --write .",
55
+ "check": "biome check .",
56
+ "verify": "biome check . && pnpm run typecheck && bash scripts/ci-marker.sh write",
57
+ "setup": "git config core.hooksPath .githooks"
58
+ },
59
+ "peerDependencies": {
60
+ "@cloudflare/workers-types": ">=4.0.0"
61
+ },
62
+ "devDependencies": {
63
+ "@biomejs/biome": "^2.4.15",
64
+ "@cloudflare/workers-types": "^5.20260706.1",
65
+ "typescript": "^6.0.3"
66
+ }
67
+ }
package/src/bots.ts ADDED
@@ -0,0 +1,13 @@
1
+ // Default User-Agent regex for known LLM crawlers and AI fetchers. Extend
2
+ // this in your own worker by composing a new RegExp with the source:
3
+ // new RegExp(LLM_BOT_UA.source + "|mybot", "i")
4
+ //
5
+ // Sources / references:
6
+ // - https://platform.openai.com/docs/bots
7
+ // - https://docs.anthropic.com/en/docs/agents-and-tools/claude-bot
8
+ // - https://docs.perplexity.ai/guides/bots
9
+ // - https://support.apple.com/en-us/119829 (Applebot-Extended)
10
+ // - https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers (Google-Extended)
11
+
12
+ export const LLM_BOT_UA =
13
+ /gptbot|chatgpt-user|oai-searchbot|claudebot|claude-web|anthropic-ai|perplexitybot|ccbot|applebot-extended|google-extended|cohere-ai|bytespider|diffbot/i;
package/src/index.ts ADDED
@@ -0,0 +1,9 @@
1
+ // biome-ignore assist/source/organizeImports: keep the hand-authored barrel order — the named API exports grouped, then the default handler on its own readable line (Biome would merge `default` into the named-export braces).
2
+ export { LLM_BOT_UA } from "./bots.ts";
3
+ export {
4
+ createMdRouter,
5
+ mdRouter,
6
+ type MdRouterEnv,
7
+ type MdRouterOptions,
8
+ } from "./worker.ts";
9
+ export { default } from "./worker.ts";
package/src/worker.ts ADDED
@@ -0,0 +1,148 @@
1
+ import { LLM_BOT_UA } from "./bots.ts";
2
+
3
+ export interface MdRouterEnv {
4
+ ASSETS: Fetcher;
5
+ }
6
+
7
+ export interface MdRouterOptions {
8
+ /** Override the default LLM-bot User-Agent matcher. Default: {@link LLM_BOT_UA}. */
9
+ botUserAgents?: RegExp;
10
+ /** Map a request URL pathname (e.g. `/foo/bar/`) to its `.md` twin path
11
+ * (e.g. `/foo/bar.md`). Default strips a trailing slash and appends
12
+ * `.md`; the special case for `/` returns `/index.md`. */
13
+ mdPathFor?: (pathname: string) => string;
14
+ /** Extra Accept-header tokens that should also trigger markdown
15
+ * routing. `text/markdown` is always matched. */
16
+ acceptMarkdown?: string[];
17
+ /** Advertise each HTML page's `.md` twin via an RFC 8288 `Link` response
18
+ * header (`Link: <…/foo.md>; rel="alternate"; type="text/markdown"`), so an
19
+ * agent can discover the twin from a plain GET — no UA match or `Accept`
20
+ * guess needed. Applies only to the normal HTML page response (a `200`
21
+ * `text/html` reply to an extension-less GET); the `.md` and pass-through
22
+ * responses are untouched. Default: `true`. */
23
+ advertiseTwin?: boolean;
24
+ /** Append `Vary: Accept` to the responses this worker negotiates on the
25
+ * `Accept` header (the `.md` twin, the HTML fallback when the twin is
26
+ * missing, and the normal HTML page), so a shared cache keyed only on the
27
+ * URL doesn't serve the HTML representation to a client that asked for
28
+ * markdown — or vice-versa (RFC 9110 §12.5.5). Merges with any existing
29
+ * `Vary` rather than replacing it; verbatim pass-through responses (non-GET,
30
+ * extension-bearing paths) are left untouched. Only `Accept` is listed:
31
+ * `User-Agent` is also a routing input, but `Vary: User-Agent` defeats
32
+ * shared caching (huge UA cardinality), so it is intentionally omitted.
33
+ * Default: `false`. */
34
+ vary?: boolean;
35
+ }
36
+
37
+ const defaultMdPathFor = (pathname: string): string => {
38
+ const trimmed = pathname.replace(/\/$/, "");
39
+ return `${trimmed || "/index"}.md`;
40
+ };
41
+
42
+ /** Build a Cloudflare Workers fetch handler that serves the `.md` twin of
43
+ * a static page when the request looks like an LLM fetcher (known bot UA
44
+ * or `Accept: text/markdown`), falling back to the HTML response if the
45
+ * `.md` twin doesn't exist (404).
46
+ *
47
+ * Pass through unchanged for non-GET, requests with a file extension,
48
+ * and "normal" browser requests — though the normal HTML page response also
49
+ * gets a `Link` header advertising its `.md` twin unless `advertiseTwin` is
50
+ * disabled (see {@link MdRouterOptions.advertiseTwin}). Opt into a
51
+ * `Vary: Accept` header on the `Accept`-negotiated responses with
52
+ * {@link MdRouterOptions.vary} so shared caches don't cross-serve the two
53
+ * representations.
54
+ *
55
+ * Requires an `ASSETS` Fetcher binding in your `wrangler.jsonc`:
56
+ * ```jsonc
57
+ * {
58
+ * "main": "worker/index.ts",
59
+ * "assets": {
60
+ * "directory": "./dist",
61
+ * "binding": "ASSETS",
62
+ * "not_found_handling": "404-page",
63
+ * "html_handling": "drop-trailing-slash",
64
+ * "run_worker_first": true
65
+ * }
66
+ * }
67
+ * ```
68
+ */
69
+ export function createMdRouter<Env extends MdRouterEnv = MdRouterEnv>(
70
+ options: MdRouterOptions = {}
71
+ ): ExportedHandler<Env> {
72
+ const botUa = options.botUserAgents ?? LLM_BOT_UA;
73
+ const mdPathFor = options.mdPathFor ?? defaultMdPathFor;
74
+ const acceptTokens = ["text/markdown", ...(options.acceptMarkdown ?? [])];
75
+ const advertiseTwin = options.advertiseTwin ?? true;
76
+ const vary = options.vary ?? false;
77
+
78
+ // Append `Vary: Accept` to a negotiated response when `vary` is enabled.
79
+ // Wraps via `new Response(body, response)` because an asset response's
80
+ // headers are immutable; `append` (not `set`) preserves any existing `Vary`.
81
+ const withVary = (response: Response): Response => {
82
+ if (!vary) {
83
+ return response;
84
+ }
85
+ const varied = new Response(response.body, response);
86
+ varied.headers.append("Vary", "Accept");
87
+ return varied;
88
+ };
89
+
90
+ return {
91
+ async fetch(request, env): Promise<Response> {
92
+ const url = new URL(request.url);
93
+
94
+ if (request.method !== "GET" || /\.[a-zA-Z0-9]+$/.test(url.pathname)) {
95
+ return env.ASSETS.fetch(request);
96
+ }
97
+
98
+ const accept = request.headers.get("Accept") ?? "";
99
+ const ua = request.headers.get("User-Agent") ?? "";
100
+ const wantsMarkdown = acceptTokens.some((tok) => accept.includes(tok)) || botUa.test(ua);
101
+
102
+ if (!wantsMarkdown) {
103
+ const response = await env.ASSETS.fetch(request);
104
+ const contentType = (response.headers.get("Content-Type") ?? "")
105
+ .toLowerCase()
106
+ .split(";")[0]
107
+ .trim();
108
+ const isHtml = contentType === "text/html";
109
+ if (!advertiseTwin || response.status !== 200 || !isHtml) {
110
+ return withVary(response);
111
+ }
112
+ // `new Response(body, response)` is the only way to add a header to an
113
+ // otherwise-immutable asset response. A relative URI-Reference target is
114
+ // valid per RFC 8288 (resolved against the request URL).
115
+ const withTwin = new Response(response.body, response);
116
+ const encodedPath = mdPathFor(url.pathname)
117
+ .split("/")
118
+ .map((segment) => encodeURIComponent(segment))
119
+ .join("/");
120
+ withTwin.headers.append("Link", `<${encodedPath}>; rel="alternate"; type="text/markdown"`);
121
+ // Already a mutable wrapper — append `Vary` here rather than re-wrapping.
122
+ if (vary) {
123
+ withTwin.headers.append("Vary", "Accept");
124
+ }
125
+ return withTwin;
126
+ }
127
+
128
+ const mdPath = mdPathFor(url.pathname);
129
+ const mdResponse = await env.ASSETS.fetch(new Request(new URL(mdPath, url.origin), request));
130
+
131
+ return mdResponse.status === 404
132
+ ? withVary(await env.ASSETS.fetch(request))
133
+ : withVary(mdResponse);
134
+ },
135
+ };
136
+ }
137
+
138
+ /** Default-configured handler. Equivalent to `createMdRouter()`. Re-export
139
+ * this as your worker's default to use it without any customization:
140
+ *
141
+ * ```ts
142
+ * // worker/index.ts
143
+ * export { default } from "cloudflare-md-router/worker";
144
+ * ```
145
+ */
146
+ export const mdRouter: ExportedHandler<MdRouterEnv> = createMdRouter();
147
+
148
+ export default mdRouter;