@wave-rf/cloudflare-md-router 0.0.0-dev.h55da3453dfaa
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +102 -0
- package/package.json +67 -0
- package/src/bots.ts +13 -0
- package/src/index.ts +9 -0
- package/src/worker.ts +148 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Wave RF
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
# cloudflare-md-router
|
|
2
|
+
|
|
3
|
+
A tiny Cloudflare Worker that serves the `.md` twin of any static page when the request is from a known LLM crawler or explicitly asks for `text/markdown`. Falls back to the HTML response when the `.md` twin doesn't exist.
|
|
4
|
+
|
|
5
|
+
If you're building a docs site that already emits a per-page raw-markdown twin (e.g. `/foo/bar` and `/foo/bar.md`), this lets every page do content negotiation transparently — Claude, ChatGPT, Perplexity, etc. fetch the model-friendly version automatically; humans keep getting the styled HTML page.
|
|
6
|
+
|
|
7
|
+
## Behavior
|
|
8
|
+
|
|
9
|
+
| Request | Worker serves |
|
|
10
|
+
| ---------------------------------------------- | ------------------- |
|
|
11
|
+
| Anything with a file extension (`.css`, `.png`, `.md`, …) | Pass-through to ASSETS |
|
|
12
|
+
| Non-GET | Pass-through to ASSETS |
|
|
13
|
+
| `Accept: text/markdown` | `<path>.md` (HTML fallback on 404) |
|
|
14
|
+
| `User-Agent` matches a known LLM bot | `<path>.md` (HTML fallback on 404) |
|
|
15
|
+
| Everything else | HTML page — plus a `Link` header advertising its `.md` twin |
|
|
16
|
+
|
|
17
|
+
On the normal HTML page response, the worker also adds an [RFC 8288](https://www.rfc-editor.org/rfc/rfc8288) `Link` header so an agent can discover the markdown twin from a plain GET, without sniffing the User-Agent or guessing the right `Accept`:
|
|
18
|
+
|
|
19
|
+
```http
|
|
20
|
+
Link: </foo/bar.md>; rel="alternate"; type="text/markdown"
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
This is on by default (only for a `200 text/html` reply to an extension-less GET); disable it with `advertiseTwin: false`.
|
|
24
|
+
|
|
25
|
+
Because the worker negotiates on the `Accept` header, two clients can get different representations of the same URL. If you put a shared cache (a CDN, Cloudflare's own cache) in front of it, set `vary: true` to add a `Vary: Accept` header to the negotiated responses so the cache keys on `Accept` and doesn't serve the HTML page to a client that asked for markdown — or vice-versa. It's off by default (verbatim pass-through stays byte-for-byte; `Vary: User-Agent` is intentionally not added, as it would defeat shared caching).
|
|
26
|
+
|
|
27
|
+
The included bot list covers the common ones: GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, Claude-Web, anthropic-ai, PerplexityBot, CCBot, Applebot-Extended, Google-Extended, cohere-ai, Bytespider, Diffbot. See `src/bots.ts`.
|
|
28
|
+
|
|
29
|
+
## Install
|
|
30
|
+
|
|
31
|
+
```sh
|
|
32
|
+
pnpm add @wave-rf/cloudflare-md-router
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
This package ships raw TypeScript with no build step — bundle it with Wrangler/esbuild (the Cloudflare Workers default), which resolve the `.ts` entry points directly.
|
|
36
|
+
|
|
37
|
+
## Use
|
|
38
|
+
|
|
39
|
+
The simplest setup — re-export the default handler from your worker entrypoint:
|
|
40
|
+
|
|
41
|
+
```ts
|
|
42
|
+
// worker/index.ts
|
|
43
|
+
export { default } from "@wave-rf/cloudflare-md-router/worker";
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Configure your `wrangler.jsonc` with an ASSETS binding pointing at your built static site:
|
|
47
|
+
|
|
48
|
+
```jsonc
|
|
49
|
+
{
|
|
50
|
+
"name": "my-docs",
|
|
51
|
+
"main": "worker/index.ts",
|
|
52
|
+
"compatibility_date": "2025-01-01",
|
|
53
|
+
"assets": {
|
|
54
|
+
"directory": "./dist",
|
|
55
|
+
"binding": "ASSETS",
|
|
56
|
+
"not_found_handling": "404-page",
|
|
57
|
+
"html_handling": "drop-trailing-slash",
|
|
58
|
+
"run_worker_first": true
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
`run_worker_first` is required so the worker sees the request before Cloudflare's static-asset matcher does — otherwise the worker only ever runs on 404s.
|
|
64
|
+
|
|
65
|
+
## Customizing
|
|
66
|
+
|
|
67
|
+
Use `createMdRouter()` if you need to extend the bot list, change the `.md` path mapping, or add other Accept tokens:
|
|
68
|
+
|
|
69
|
+
```ts
|
|
70
|
+
// worker/index.ts
|
|
71
|
+
import { createMdRouter, LLM_BOT_UA } from "@wave-rf/cloudflare-md-router";
|
|
72
|
+
|
|
73
|
+
export default createMdRouter({
|
|
74
|
+
// Add your own bots:
|
|
75
|
+
botUserAgents: new RegExp(LLM_BOT_UA.source + "|mybot", "i"),
|
|
76
|
+
|
|
77
|
+
// Treat `Accept: text/x-markdown` as markdown too:
|
|
78
|
+
acceptMarkdown: ["text/x-markdown"],
|
|
79
|
+
|
|
80
|
+
// Custom .md path strategy. Default: `/foo/` → `/foo.md`, `/` → `/index.md`.
|
|
81
|
+
mdPathFor: (pathname) => `/markdown${pathname.replace(/\/$/, "")}.md`,
|
|
82
|
+
|
|
83
|
+
// Don't advertise the `.md` twin via a `Link` header (default: true).
|
|
84
|
+
advertiseTwin: false,
|
|
85
|
+
|
|
86
|
+
// Add `Vary: Accept` to negotiated responses so a shared cache doesn't
|
|
87
|
+
// cross-serve the HTML and markdown representations (default: false).
|
|
88
|
+
vary: true,
|
|
89
|
+
});
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
## Why content-negotiate?
|
|
93
|
+
|
|
94
|
+
Most LLMs do better with raw markdown than with rendered HTML — less DOM noise, no Starlight nav chrome, no script tags. Serving the same content at one URL with two representations means:
|
|
95
|
+
|
|
96
|
+
- One canonical URL per page (good for citations and link-sharing).
|
|
97
|
+
- Crawlers and human readers stay aligned automatically.
|
|
98
|
+
- Your `llms.txt` can advertise `<page>.md` for explicit fetches; the worker covers the case where the LLM hits the HTML URL anyway.
|
|
99
|
+
|
|
100
|
+
## License
|
|
101
|
+
|
|
102
|
+
MIT — see [LICENSE](./LICENSE).
|
package/package.json
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@wave-rf/cloudflare-md-router",
|
|
3
|
+
"version": "0.0.0-dev.h55da3453dfaa",
|
|
4
|
+
"description": "Cloudflare Worker that serves a `.md` twin of any static page when the request is from a known LLM crawler or explicitly asks for `text/markdown`. Falls back to the HTML response otherwise.",
|
|
5
|
+
"keywords": [
|
|
6
|
+
"cloudflare",
|
|
7
|
+
"cloudflare-workers",
|
|
8
|
+
"llm",
|
|
9
|
+
"llms-txt",
|
|
10
|
+
"markdown",
|
|
11
|
+
"content-negotiation",
|
|
12
|
+
"static-site"
|
|
13
|
+
],
|
|
14
|
+
"license": "MIT",
|
|
15
|
+
"author": {
|
|
16
|
+
"name": "Wave RF",
|
|
17
|
+
"email": "github@wave-rf.com",
|
|
18
|
+
"url": "https://wave-rf.com"
|
|
19
|
+
},
|
|
20
|
+
"contributors": [
|
|
21
|
+
"Eric Andrechek <eric@wave-rf.com>"
|
|
22
|
+
],
|
|
23
|
+
"repository": {
|
|
24
|
+
"type": "git",
|
|
25
|
+
"url": "git+https://github.com/Wave-RF/cloudflare-md-router.git"
|
|
26
|
+
},
|
|
27
|
+
"bugs": {
|
|
28
|
+
"url": "https://github.com/Wave-RF/cloudflare-md-router/issues"
|
|
29
|
+
},
|
|
30
|
+
"homepage": "https://github.com/Wave-RF/cloudflare-md-router#readme",
|
|
31
|
+
"type": "module",
|
|
32
|
+
"main": "./src/index.ts",
|
|
33
|
+
"types": "./src/index.ts",
|
|
34
|
+
"exports": {
|
|
35
|
+
".": "./src/index.ts",
|
|
36
|
+
"./worker": "./src/worker.ts",
|
|
37
|
+
"./bots": "./src/bots.ts"
|
|
38
|
+
},
|
|
39
|
+
"files": [
|
|
40
|
+
"src",
|
|
41
|
+
"README.md",
|
|
42
|
+
"LICENSE"
|
|
43
|
+
],
|
|
44
|
+
"publishConfig": {
|
|
45
|
+
"access": "public"
|
|
46
|
+
},
|
|
47
|
+
"packageManager": "pnpm@11.1.3",
|
|
48
|
+
"engines": {
|
|
49
|
+
"node": ">=18.0.0"
|
|
50
|
+
},
|
|
51
|
+
"scripts": {
|
|
52
|
+
"typecheck": "tsc --noEmit",
|
|
53
|
+
"lint": "biome lint .",
|
|
54
|
+
"format": "biome format --write .",
|
|
55
|
+
"check": "biome check .",
|
|
56
|
+
"verify": "biome check . && pnpm run typecheck && bash scripts/ci-marker.sh write",
|
|
57
|
+
"setup": "git config core.hooksPath .githooks"
|
|
58
|
+
},
|
|
59
|
+
"peerDependencies": {
|
|
60
|
+
"@cloudflare/workers-types": ">=4.0.0"
|
|
61
|
+
},
|
|
62
|
+
"devDependencies": {
|
|
63
|
+
"@biomejs/biome": "^2.4.15",
|
|
64
|
+
"@cloudflare/workers-types": "^4.20240208.0",
|
|
65
|
+
"typescript": "^6.0.3"
|
|
66
|
+
}
|
|
67
|
+
}
|
package/src/bots.ts
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
// Default User-Agent regex for known LLM crawlers and AI fetchers. Extend
|
|
2
|
+
// this in your own worker by composing a new RegExp with the source:
|
|
3
|
+
// new RegExp(LLM_BOT_UA.source + "|mybot", "i")
|
|
4
|
+
//
|
|
5
|
+
// Sources / references:
|
|
6
|
+
// - https://platform.openai.com/docs/bots
|
|
7
|
+
// - https://docs.anthropic.com/en/docs/agents-and-tools/claude-bot
|
|
8
|
+
// - https://docs.perplexity.ai/guides/bots
|
|
9
|
+
// - https://support.apple.com/en-us/119829 (Applebot-Extended)
|
|
10
|
+
// - https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers (Google-Extended)
|
|
11
|
+
|
|
12
|
+
export const LLM_BOT_UA =
|
|
13
|
+
/gptbot|chatgpt-user|oai-searchbot|claudebot|claude-web|anthropic-ai|perplexitybot|ccbot|applebot-extended|google-extended|cohere-ai|bytespider|diffbot/i;
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
// biome-ignore assist/source/organizeImports: keep the hand-authored barrel order — the named API exports grouped, then the default handler on its own readable line (Biome would merge `default` into the named-export braces).
|
|
2
|
+
export { LLM_BOT_UA } from "./bots.ts";
|
|
3
|
+
export {
|
|
4
|
+
createMdRouter,
|
|
5
|
+
mdRouter,
|
|
6
|
+
type MdRouterEnv,
|
|
7
|
+
type MdRouterOptions,
|
|
8
|
+
} from "./worker.ts";
|
|
9
|
+
export { default } from "./worker.ts";
|
package/src/worker.ts
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
import { LLM_BOT_UA } from "./bots.ts";
|
|
2
|
+
|
|
3
|
+
export interface MdRouterEnv {
|
|
4
|
+
ASSETS: Fetcher;
|
|
5
|
+
}
|
|
6
|
+
|
|
7
|
+
export interface MdRouterOptions {
|
|
8
|
+
/** Override the default LLM-bot User-Agent matcher. Default: {@link LLM_BOT_UA}. */
|
|
9
|
+
botUserAgents?: RegExp;
|
|
10
|
+
/** Map a request URL pathname (e.g. `/foo/bar/`) to its `.md` twin path
|
|
11
|
+
* (e.g. `/foo/bar.md`). Default strips a trailing slash and appends
|
|
12
|
+
* `.md`; the special case for `/` returns `/index.md`. */
|
|
13
|
+
mdPathFor?: (pathname: string) => string;
|
|
14
|
+
/** Extra Accept-header tokens that should also trigger markdown
|
|
15
|
+
* routing. `text/markdown` is always matched. */
|
|
16
|
+
acceptMarkdown?: string[];
|
|
17
|
+
/** Advertise each HTML page's `.md` twin via an RFC 8288 `Link` response
|
|
18
|
+
* header (`Link: <…/foo.md>; rel="alternate"; type="text/markdown"`), so an
|
|
19
|
+
* agent can discover the twin from a plain GET — no UA match or `Accept`
|
|
20
|
+
* guess needed. Applies only to the normal HTML page response (a `200`
|
|
21
|
+
* `text/html` reply to an extension-less GET); the `.md` and pass-through
|
|
22
|
+
* responses are untouched. Default: `true`. */
|
|
23
|
+
advertiseTwin?: boolean;
|
|
24
|
+
/** Append `Vary: Accept` to the responses this worker negotiates on the
|
|
25
|
+
* `Accept` header (the `.md` twin, the HTML fallback when the twin is
|
|
26
|
+
* missing, and the normal HTML page), so a shared cache keyed only on the
|
|
27
|
+
* URL doesn't serve the HTML representation to a client that asked for
|
|
28
|
+
* markdown — or vice-versa (RFC 9110 §12.5.5). Merges with any existing
|
|
29
|
+
* `Vary` rather than replacing it; verbatim pass-through responses (non-GET,
|
|
30
|
+
* extension-bearing paths) are left untouched. Only `Accept` is listed:
|
|
31
|
+
* `User-Agent` is also a routing input, but `Vary: User-Agent` defeats
|
|
32
|
+
* shared caching (huge UA cardinality), so it is intentionally omitted.
|
|
33
|
+
* Default: `false`. */
|
|
34
|
+
vary?: boolean;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const defaultMdPathFor = (pathname: string): string => {
|
|
38
|
+
const trimmed = pathname.replace(/\/$/, "");
|
|
39
|
+
return `${trimmed || "/index"}.md`;
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
/** Build a Cloudflare Workers fetch handler that serves the `.md` twin of
|
|
43
|
+
* a static page when the request looks like an LLM fetcher (known bot UA
|
|
44
|
+
* or `Accept: text/markdown`), falling back to the HTML response if the
|
|
45
|
+
* `.md` twin doesn't exist (404).
|
|
46
|
+
*
|
|
47
|
+
* Pass through unchanged for non-GET, requests with a file extension,
|
|
48
|
+
* and "normal" browser requests — though the normal HTML page response also
|
|
49
|
+
* gets a `Link` header advertising its `.md` twin unless `advertiseTwin` is
|
|
50
|
+
* disabled (see {@link MdRouterOptions.advertiseTwin}). Opt into a
|
|
51
|
+
* `Vary: Accept` header on the `Accept`-negotiated responses with
|
|
52
|
+
* {@link MdRouterOptions.vary} so shared caches don't cross-serve the two
|
|
53
|
+
* representations.
|
|
54
|
+
*
|
|
55
|
+
* Requires an `ASSETS` Fetcher binding in your `wrangler.jsonc`:
|
|
56
|
+
* ```jsonc
|
|
57
|
+
* {
|
|
58
|
+
* "main": "worker/index.ts",
|
|
59
|
+
* "assets": {
|
|
60
|
+
* "directory": "./dist",
|
|
61
|
+
* "binding": "ASSETS",
|
|
62
|
+
* "not_found_handling": "404-page",
|
|
63
|
+
* "html_handling": "drop-trailing-slash",
|
|
64
|
+
* "run_worker_first": true
|
|
65
|
+
* }
|
|
66
|
+
* }
|
|
67
|
+
* ```
|
|
68
|
+
*/
|
|
69
|
+
export function createMdRouter<Env extends MdRouterEnv = MdRouterEnv>(
|
|
70
|
+
options: MdRouterOptions = {}
|
|
71
|
+
): ExportedHandler<Env> {
|
|
72
|
+
const botUa = options.botUserAgents ?? LLM_BOT_UA;
|
|
73
|
+
const mdPathFor = options.mdPathFor ?? defaultMdPathFor;
|
|
74
|
+
const acceptTokens = ["text/markdown", ...(options.acceptMarkdown ?? [])];
|
|
75
|
+
const advertiseTwin = options.advertiseTwin ?? true;
|
|
76
|
+
const vary = options.vary ?? false;
|
|
77
|
+
|
|
78
|
+
// Append `Vary: Accept` to a negotiated response when `vary` is enabled.
|
|
79
|
+
// Wraps via `new Response(body, response)` because an asset response's
|
|
80
|
+
// headers are immutable; `append` (not `set`) preserves any existing `Vary`.
|
|
81
|
+
const withVary = (response: Response): Response => {
|
|
82
|
+
if (!vary) {
|
|
83
|
+
return response;
|
|
84
|
+
}
|
|
85
|
+
const varied = new Response(response.body, response);
|
|
86
|
+
varied.headers.append("Vary", "Accept");
|
|
87
|
+
return varied;
|
|
88
|
+
};
|
|
89
|
+
|
|
90
|
+
return {
|
|
91
|
+
async fetch(request, env): Promise<Response> {
|
|
92
|
+
const url = new URL(request.url);
|
|
93
|
+
|
|
94
|
+
if (request.method !== "GET" || /\.[a-zA-Z0-9]+$/.test(url.pathname)) {
|
|
95
|
+
return env.ASSETS.fetch(request);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
const accept = request.headers.get("Accept") ?? "";
|
|
99
|
+
const ua = request.headers.get("User-Agent") ?? "";
|
|
100
|
+
const wantsMarkdown = acceptTokens.some((tok) => accept.includes(tok)) || botUa.test(ua);
|
|
101
|
+
|
|
102
|
+
if (!wantsMarkdown) {
|
|
103
|
+
const response = await env.ASSETS.fetch(request);
|
|
104
|
+
const contentType = (response.headers.get("Content-Type") ?? "")
|
|
105
|
+
.toLowerCase()
|
|
106
|
+
.split(";")[0]
|
|
107
|
+
.trim();
|
|
108
|
+
const isHtml = contentType === "text/html";
|
|
109
|
+
if (!advertiseTwin || response.status !== 200 || !isHtml) {
|
|
110
|
+
return withVary(response);
|
|
111
|
+
}
|
|
112
|
+
// `new Response(body, response)` is the only way to add a header to an
|
|
113
|
+
// otherwise-immutable asset response. A relative URI-Reference target is
|
|
114
|
+
// valid per RFC 8288 (resolved against the request URL).
|
|
115
|
+
const withTwin = new Response(response.body, response);
|
|
116
|
+
const encodedPath = mdPathFor(url.pathname)
|
|
117
|
+
.split("/")
|
|
118
|
+
.map((segment) => encodeURIComponent(segment))
|
|
119
|
+
.join("/");
|
|
120
|
+
withTwin.headers.append("Link", `<${encodedPath}>; rel="alternate"; type="text/markdown"`);
|
|
121
|
+
// Already a mutable wrapper — append `Vary` here rather than re-wrapping.
|
|
122
|
+
if (vary) {
|
|
123
|
+
withTwin.headers.append("Vary", "Accept");
|
|
124
|
+
}
|
|
125
|
+
return withTwin;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
const mdPath = mdPathFor(url.pathname);
|
|
129
|
+
const mdResponse = await env.ASSETS.fetch(new Request(new URL(mdPath, url.origin), request));
|
|
130
|
+
|
|
131
|
+
return mdResponse.status === 404
|
|
132
|
+
? withVary(await env.ASSETS.fetch(request))
|
|
133
|
+
: withVary(mdResponse);
|
|
134
|
+
},
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/** Default-configured handler. Equivalent to `createMdRouter()`. Re-export
|
|
139
|
+
* this as your worker's default to use it without any customization:
|
|
140
|
+
*
|
|
141
|
+
* ```ts
|
|
142
|
+
* // worker/index.ts
|
|
143
|
+
* export { default } from "cloudflare-md-router/worker";
|
|
144
|
+
* ```
|
|
145
|
+
*/
|
|
146
|
+
export const mdRouter: ExportedHandler<MdRouterEnv> = createMdRouter();
|
|
147
|
+
|
|
148
|
+
export default mdRouter;
|