@wave-rf/cloudflare-md-router 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -1
- package/package.json +2 -2
- package/src/bots.ts +1 -1
- package/src/index.ts +2 -0
- package/src/worker.ts +134 -41
package/README.md
CHANGED
|
@@ -26,7 +26,13 @@ This is on by default (only for a `200 text/html` reply to an extension-less GET
|
|
|
26
26
|
|
|
27
27
|
Because the worker negotiates on the `Accept` header, two clients can get different representations of the same URL. If you put a shared cache (a CDN, Cloudflare's own cache) in front of it, set `vary: true` to add a `Vary: Accept` header to the negotiated responses so the cache keys on `Accept` and doesn't serve the HTML page to a client that asked for markdown — or vice-versa. It's off by default (verbatim pass-through stays byte-for-byte; `Vary: User-Agent` is intentionally not added, as it would defeat shared caching).
|
|
28
28
|
|
|
29
|
-
The included bot list covers the common ones: GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, Claude-Web, anthropic-ai, PerplexityBot, CCBot, Applebot-Extended, Google-Extended, cohere-ai, Bytespider, Diffbot. See `src/bots.ts`.
|
|
29
|
+
The included bot list covers the common ones: GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, Claude-Web, Claude-User, Claude-SearchBot, Amazonbot, DuckAssistBot, anthropic-ai, PerplexityBot, CCBot, Applebot-Extended, Google-Extended, cohere-ai, Bytespider, Diffbot. See `src/bots.ts`.
|
|
30
|
+
|
|
31
|
+
Three more opt-in features, all off by default:
|
|
32
|
+
|
|
33
|
+
- `checkTwin: true` fetches the `.md` twin from `ASSETS` before advertising it, so a route with no twin (an error page, say) gets no dead `Link`. It costs one extra local asset lookup per HTML page.
|
|
34
|
+
- `advertiseLlmsTxt: true` (or a path) adds `Link: </llms.txt>; rel="alternate"; type="text/plain"` to HTML pages.
|
|
35
|
+
- `canonicalOrigin: "https://example.com"` rewrites absolute `og:image` / `twitter:image` URLs under that origin to the origin being browsed when the request host differs (a preview deploy), via `HTMLRewriter`. `canonical` and `og:url` are left alone.
|
|
30
36
|
|
|
31
37
|
## Install
|
|
32
38
|
|
|
@@ -88,9 +94,27 @@ export default createMdRouter({
|
|
|
88
94
|
// Add `Vary: Accept` to negotiated responses so a shared cache doesn't
|
|
89
95
|
// cross-serve the HTML and markdown representations (default: false).
|
|
90
96
|
vary: true,
|
|
97
|
+
|
|
98
|
+
// Only advertise a twin that exists (default: false).
|
|
99
|
+
checkTwin: true,
|
|
100
|
+
|
|
101
|
+
// Also advertise /llms.txt via a Link header (default: false).
|
|
102
|
+
advertiseLlmsTxt: true,
|
|
103
|
+
|
|
104
|
+
// Point og:image / twitter:image at the browsed origin on preview hosts.
|
|
105
|
+
canonicalOrigin: "https://example.com",
|
|
91
106
|
});
|
|
92
107
|
```
|
|
93
108
|
|
|
109
|
+
To run the negotiation inside a larger Worker, use `createMdResponder(options)`, which returns a plain `(request, env) => Promise<Response>`:
|
|
110
|
+
|
|
111
|
+
```ts
|
|
112
|
+
const respond = createMdResponder({ vary: true });
|
|
113
|
+
export default {
|
|
114
|
+
fetch: (request, env) => (isApi(request) ? api(request, env) : respond(request, env)),
|
|
115
|
+
};
|
|
116
|
+
```
|
|
117
|
+
|
|
94
118
|
## Dev loop (`cloudflare-md-router dev`)
|
|
95
119
|
|
|
96
120
|
`astro dev` (or any framework dev server) skips everything the Worker adds in production: the `.md` twins, the `Link` header, and any outputs that only exist in a real build. If your Worker serves something the dev server can't, use this instead of it. If the Worker adds nothing you are iterating on, keep using the dev server — it has HMR and an error overlay; this does not.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@wave-rf/cloudflare-md-router",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.0",
|
|
4
4
|
"description": "Cloudflare Worker that serves a `.md` twin of any static page when the request is from a known LLM crawler or explicitly asks for `text/markdown`. Falls back to the HTML response otherwise.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"cloudflare",
|
|
@@ -72,6 +72,6 @@
|
|
|
72
72
|
"@semantic-release/release-notes-generator": "14.1.1",
|
|
73
73
|
"conventional-changelog-conventionalcommits": "9.3.1",
|
|
74
74
|
"semantic-release": "25.0.9",
|
|
75
|
-
"typescript": "^
|
|
75
|
+
"typescript": "^7.0.2"
|
|
76
76
|
}
|
|
77
77
|
}
|
package/src/bots.ts
CHANGED
|
@@ -10,4 +10,4 @@
|
|
|
10
10
|
// - https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers (Google-Extended)
|
|
11
11
|
|
|
12
12
|
export const LLM_BOT_UA =
|
|
13
|
-
/gptbot|chatgpt-user|oai-searchbot|claudebot|claude-web|anthropic-ai|perplexitybot|ccbot|applebot-extended|google-extended|cohere-ai|bytespider|diffbot/i;
|
|
13
|
+
/gptbot|chatgpt-user|oai-searchbot|claudebot|claude-web|anthropic-ai|perplexitybot|ccbot|applebot-extended|google-extended|cohere-ai|bytespider|diffbot|claude-user|claude-searchbot|amazonbot|duckassistbot/i;
|
package/src/index.ts
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
// biome-ignore assist/source/organizeImports: keep the hand-authored barrel order — the named API exports grouped, then the default handler on its own readable line (Biome would merge `default` into the named-export braces).
|
|
2
2
|
export { LLM_BOT_UA } from "./bots.ts";
|
|
3
3
|
export {
|
|
4
|
+
createMdResponder,
|
|
4
5
|
createMdRouter,
|
|
5
6
|
mdRouter,
|
|
7
|
+
type MdResponder,
|
|
6
8
|
type MdRouterEnv,
|
|
7
9
|
type MdRouterOptions,
|
|
8
10
|
} from "./worker.ts";
|
package/src/worker.ts
CHANGED
|
@@ -32,8 +32,36 @@ export interface MdRouterOptions {
|
|
|
32
32
|
* shared caching (huge UA cardinality), so it is intentionally omitted.
|
|
33
33
|
* Default: `false`. */
|
|
34
34
|
vary?: boolean;
|
|
35
|
+
/** Before advertising a page's `.md` twin in the `Link` header, fetch it from
|
|
36
|
+
* `ASSETS` and advertise it only when it exists (a `2xx` reply). Costs one
|
|
37
|
+
* extra local asset lookup per HTML page; worth it when some routes (error
|
|
38
|
+
* pages, dynamic outcomes) have no twin and an agent following the link
|
|
39
|
+
* would land on an error. Has no effect when `advertiseTwin` is `false`.
|
|
40
|
+
* Default: `false`. */
|
|
41
|
+
checkTwin?: boolean;
|
|
42
|
+
/** Also advertise an `llms.txt` index on each HTML page via
|
|
43
|
+
* `Link: </llms.txt>; rel="alternate"; type="text/plain"`. Pass `true` for
|
|
44
|
+
* `/llms.txt` or a path string for another location. Independent of
|
|
45
|
+
* `advertiseTwin`. Default: `false`. */
|
|
46
|
+
advertiseLlmsTxt?: boolean | string;
|
|
47
|
+
/** The origin your build bakes into absolute URLs (e.g. `https://example.com`,
|
|
48
|
+
* your Astro/Next `site`). On a request whose hostname differs (a preview
|
|
49
|
+
* deployment, a staging host), `og:image` and `twitter:image` URLs under this
|
|
50
|
+
* origin are rewritten to the origin being browsed, so images added in a
|
|
51
|
+
* preview resolve there. Nothing else is rewritten (`canonical` and `og:url`
|
|
52
|
+
* keep pointing at production), and it is a no-op on the canonical host.
|
|
53
|
+
* Uses `HTMLRewriter`; applies to HTML responses this worker negotiates.
|
|
54
|
+
* Default: unset (no rewriting). */
|
|
55
|
+
canonicalOrigin?: string;
|
|
35
56
|
}
|
|
36
57
|
|
|
58
|
+
/** The function behind {@link createMdRouter}: call it from your own `fetch`
|
|
59
|
+
* when markdown negotiation is one step of a larger Worker. */
|
|
60
|
+
export type MdResponder<Env extends MdRouterEnv = MdRouterEnv> = (
|
|
61
|
+
request: Request,
|
|
62
|
+
env: Env
|
|
63
|
+
) => Promise<Response>;
|
|
64
|
+
|
|
37
65
|
const defaultMdPathFor = (pathname: string): string => {
|
|
38
66
|
const trimmed = pathname.replace(/\/$/, "");
|
|
39
67
|
return `${trimmed || "/index"}.md`;
|
|
@@ -69,11 +97,38 @@ const defaultMdPathFor = (pathname: string): string => {
|
|
|
69
97
|
export function createMdRouter<Env extends MdRouterEnv = MdRouterEnv>(
|
|
70
98
|
options: MdRouterOptions = {}
|
|
71
99
|
): ExportedHandler<Env> {
|
|
100
|
+
const respond = createMdResponder<Env>(options);
|
|
101
|
+
return {
|
|
102
|
+
fetch: (request, env) => respond(request, env),
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** Like {@link createMdRouter}, but returns a plain `(request, env) => Response`
|
|
107
|
+
* function so it can be called from inside a larger Worker, for example as the
|
|
108
|
+
* fallthrough after your own routes:
|
|
109
|
+
*
|
|
110
|
+
* ```ts
|
|
111
|
+
* const respond = createMdResponder({ vary: true });
|
|
112
|
+
* export default {
|
|
113
|
+
* fetch: (request, env) =>
|
|
114
|
+
* new URL(request.url).pathname.startsWith("/api/")
|
|
115
|
+
* ? handleApi(request, env)
|
|
116
|
+
* : respond(request, env),
|
|
117
|
+
* };
|
|
118
|
+
* ```
|
|
119
|
+
*/
|
|
120
|
+
export function createMdResponder<Env extends MdRouterEnv = MdRouterEnv>(
|
|
121
|
+
options: MdRouterOptions = {}
|
|
122
|
+
): MdResponder<Env> {
|
|
72
123
|
const botUa = options.botUserAgents ?? LLM_BOT_UA;
|
|
73
124
|
const mdPathFor = options.mdPathFor ?? defaultMdPathFor;
|
|
74
125
|
const acceptTokens = ["text/markdown", ...(options.acceptMarkdown ?? [])];
|
|
75
126
|
const advertiseTwin = options.advertiseTwin ?? true;
|
|
76
127
|
const vary = options.vary ?? false;
|
|
128
|
+
const checkTwin = options.checkTwin ?? false;
|
|
129
|
+
const llmsTxtPath =
|
|
130
|
+
options.advertiseLlmsTxt === true ? "/llms.txt" : options.advertiseLlmsTxt || undefined;
|
|
131
|
+
const canonical = options.canonicalOrigin ? new URL(options.canonicalOrigin) : undefined;
|
|
77
132
|
|
|
78
133
|
// Append `Vary: Accept` to a negotiated response when `vary` is enabled.
|
|
79
134
|
// Wraps via `new Response(body, response)` because an asset response's
|
|
@@ -87,51 +142,89 @@ export function createMdRouter<Env extends MdRouterEnv = MdRouterEnv>(
|
|
|
87
142
|
return varied;
|
|
88
143
|
};
|
|
89
144
|
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
145
|
+
// Point absolute og:image / twitter:image URLs at the browsed origin. Matched
|
|
146
|
+
// on hostname, not origin: `wrangler dev` serves a configured route's host
|
|
147
|
+
// over plain http, and an origin comparison would turn `https://host/...`
|
|
148
|
+
// into a `http://` URL that resolves nowhere.
|
|
149
|
+
const withSocialImages = (response: Response, url: URL): Response => {
|
|
150
|
+
if (!canonical || !response.body || url.hostname === canonical.hostname) {
|
|
151
|
+
return response;
|
|
152
|
+
}
|
|
153
|
+
const type = (response.headers.get("Content-Type") ?? "").toLowerCase();
|
|
154
|
+
if (!type.startsWith("text/html")) {
|
|
155
|
+
return response;
|
|
156
|
+
}
|
|
157
|
+
const prefix = `${canonical.origin}/`;
|
|
158
|
+
return new HTMLRewriter()
|
|
159
|
+
.on('meta[property="og:image"], meta[name="twitter:image"]', {
|
|
160
|
+
element(el) {
|
|
161
|
+
const content = el.getAttribute("content");
|
|
162
|
+
if (content?.startsWith(prefix)) {
|
|
163
|
+
el.setAttribute("content", url.origin + content.slice(canonical.origin.length));
|
|
164
|
+
}
|
|
165
|
+
},
|
|
166
|
+
})
|
|
167
|
+
.transform(response);
|
|
168
|
+
};
|
|
93
169
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
170
|
+
const encodePath = (path: string): string =>
|
|
171
|
+
path
|
|
172
|
+
.split("/")
|
|
173
|
+
.map((segment) => encodeURIComponent(segment))
|
|
174
|
+
.join("/");
|
|
175
|
+
|
|
176
|
+
return async (request, env) => {
|
|
177
|
+
const url = new URL(request.url);
|
|
97
178
|
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
179
|
+
if (request.method !== "GET" || /\.[a-zA-Z0-9]+$/.test(url.pathname)) {
|
|
180
|
+
return env.ASSETS.fetch(request);
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
const accept = request.headers.get("Accept") ?? "";
|
|
184
|
+
const ua = request.headers.get("User-Agent") ?? "";
|
|
185
|
+
const wantsMarkdown = acceptTokens.some((tok) => accept.includes(tok)) || botUa.test(ua);
|
|
186
|
+
const mdPath = mdPathFor(url.pathname);
|
|
187
|
+
const fetchTwin = () => env.ASSETS.fetch(new Request(new URL(mdPath, url.origin), request));
|
|
188
|
+
|
|
189
|
+
if (!wantsMarkdown) {
|
|
190
|
+
const response = await env.ASSETS.fetch(request);
|
|
191
|
+
const contentType = (response.headers.get("Content-Type") ?? "")
|
|
192
|
+
.toLowerCase()
|
|
193
|
+
.split(";")[0]
|
|
194
|
+
.trim();
|
|
195
|
+
if (contentType !== "text/html") {
|
|
196
|
+
return withVary(response);
|
|
197
|
+
}
|
|
198
|
+
const advertiseMd =
|
|
199
|
+
advertiseTwin && response.status === 200 && (!checkTwin || (await fetchTwin()).ok);
|
|
200
|
+
const advertiseLlms = llmsTxtPath !== undefined && response.status === 200;
|
|
201
|
+
// `new Response(body, response)` is the only way to add a header to an
|
|
202
|
+
// otherwise-immutable asset response. A relative URI-Reference target is
|
|
203
|
+
// valid per RFC 8288 (resolved against the request URL).
|
|
204
|
+
const out = new Response(response.body, response);
|
|
205
|
+
if (advertiseMd) {
|
|
206
|
+
out.headers.append(
|
|
207
|
+
"Link",
|
|
208
|
+
`<${encodePath(mdPath)}>; rel="alternate"; type="text/markdown"`
|
|
209
|
+
);
|
|
126
210
|
}
|
|
211
|
+
if (advertiseLlms) {
|
|
212
|
+
out.headers.append(
|
|
213
|
+
"Link",
|
|
214
|
+
`<${encodePath(llmsTxtPath)}>; rel="alternate"; type="text/plain"`
|
|
215
|
+
);
|
|
216
|
+
}
|
|
217
|
+
if (vary) {
|
|
218
|
+
out.headers.append("Vary", "Accept");
|
|
219
|
+
}
|
|
220
|
+
return withSocialImages(out, url);
|
|
221
|
+
}
|
|
127
222
|
|
|
128
|
-
|
|
129
|
-
const mdResponse = await env.ASSETS.fetch(new Request(new URL(mdPath, url.origin), request));
|
|
223
|
+
const mdResponse = await fetchTwin();
|
|
130
224
|
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
},
|
|
225
|
+
return mdResponse.status === 404
|
|
226
|
+
? withSocialImages(withVary(await env.ASSETS.fetch(request)), url)
|
|
227
|
+
: withVary(mdResponse);
|
|
135
228
|
};
|
|
136
229
|
}
|
|
137
230
|
|
|
@@ -140,7 +233,7 @@ export function createMdRouter<Env extends MdRouterEnv = MdRouterEnv>(
|
|
|
140
233
|
*
|
|
141
234
|
* ```ts
|
|
142
235
|
* // worker/index.ts
|
|
143
|
-
* export { default } from "cloudflare-md-router/worker";
|
|
236
|
+
* export { default } from "@wave-rf/cloudflare-md-router/worker";
|
|
144
237
|
* ```
|
|
145
238
|
*/
|
|
146
239
|
export const mdRouter: ExportedHandler<MdRouterEnv> = createMdRouter();
|