@wave-rf/cloudflare-md-router 0.2.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +78 -1
- package/bin/cli.mjs +34 -0
- package/bin/dev.mjs +575 -0
- package/bin/lib.mjs +682 -0
- package/package.json +15 -5
- package/src/bots.ts +1 -1
- package/src/index.ts +2 -0
- package/src/worker.ts +134 -41
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@wave-rf/cloudflare-md-router",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.0",
|
|
4
4
|
"description": "Cloudflare Worker that serves a `.md` twin of any static page when the request is from a known LLM crawler or explicitly asks for `text/markdown`. Falls back to the HTML response otherwise.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"cloudflare",
|
|
@@ -31,6 +31,9 @@
|
|
|
31
31
|
"type": "module",
|
|
32
32
|
"main": "./src/index.ts",
|
|
33
33
|
"types": "./src/index.ts",
|
|
34
|
+
"bin": {
|
|
35
|
+
"cloudflare-md-router": "./bin/cli.mjs"
|
|
36
|
+
},
|
|
34
37
|
"exports": {
|
|
35
38
|
".": "./src/index.ts",
|
|
36
39
|
"./worker": "./src/worker.ts",
|
|
@@ -38,6 +41,7 @@
|
|
|
38
41
|
},
|
|
39
42
|
"files": [
|
|
40
43
|
"src",
|
|
44
|
+
"bin",
|
|
41
45
|
"README.md",
|
|
42
46
|
"LICENSE"
|
|
43
47
|
],
|
|
@@ -53,15 +57,21 @@
|
|
|
53
57
|
"lint": "biome lint .",
|
|
54
58
|
"format": "biome format --write .",
|
|
55
59
|
"check": "biome check .",
|
|
56
|
-
"
|
|
57
|
-
"
|
|
60
|
+
"check:release": "node scripts/check-release-config.mjs",
|
|
61
|
+
"verify": "biome check . && pnpm run typecheck && pnpm run test && pnpm run check:release && bash scripts/ci-marker.sh write",
|
|
62
|
+
"setup": "git config core.hooksPath .githooks",
|
|
63
|
+
"test": "node --test"
|
|
58
64
|
},
|
|
59
65
|
"peerDependencies": {
|
|
60
66
|
"@cloudflare/workers-types": ">=4.0.0"
|
|
61
67
|
},
|
|
62
68
|
"devDependencies": {
|
|
63
69
|
"@biomejs/biome": "^2.4.15",
|
|
64
|
-
"@cloudflare/workers-types": "^
|
|
65
|
-
"
|
|
70
|
+
"@cloudflare/workers-types": "^5.20260706.1",
|
|
71
|
+
"@semantic-release/commit-analyzer": "13.0.1",
|
|
72
|
+
"@semantic-release/release-notes-generator": "14.1.1",
|
|
73
|
+
"conventional-changelog-conventionalcommits": "9.3.1",
|
|
74
|
+
"semantic-release": "25.0.9",
|
|
75
|
+
"typescript": "^7.0.2"
|
|
66
76
|
}
|
|
67
77
|
}
|
package/src/bots.ts
CHANGED
|
@@ -10,4 +10,4 @@
|
|
|
10
10
|
// - https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers (Google-Extended)
|
|
11
11
|
|
|
12
12
|
export const LLM_BOT_UA =
|
|
13
|
-
/gptbot|chatgpt-user|oai-searchbot|claudebot|claude-web|anthropic-ai|perplexitybot|ccbot|applebot-extended|google-extended|cohere-ai|bytespider|diffbot/i;
|
|
13
|
+
/gptbot|chatgpt-user|oai-searchbot|claudebot|claude-web|anthropic-ai|perplexitybot|ccbot|applebot-extended|google-extended|cohere-ai|bytespider|diffbot|claude-user|claude-searchbot|amazonbot|duckassistbot/i;
|
package/src/index.ts
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
// biome-ignore assist/source/organizeImports: keep the hand-authored barrel order — the named API exports grouped, then the default handler on its own readable line (Biome would merge `default` into the named-export braces).
|
|
2
2
|
export { LLM_BOT_UA } from "./bots.ts";
|
|
3
3
|
export {
|
|
4
|
+
createMdResponder,
|
|
4
5
|
createMdRouter,
|
|
5
6
|
mdRouter,
|
|
7
|
+
type MdResponder,
|
|
6
8
|
type MdRouterEnv,
|
|
7
9
|
type MdRouterOptions,
|
|
8
10
|
} from "./worker.ts";
|
package/src/worker.ts
CHANGED
|
@@ -32,8 +32,36 @@ export interface MdRouterOptions {
|
|
|
32
32
|
* shared caching (huge UA cardinality), so it is intentionally omitted.
|
|
33
33
|
* Default: `false`. */
|
|
34
34
|
vary?: boolean;
|
|
35
|
+
/** Before advertising a page's `.md` twin in the `Link` header, fetch it from
|
|
36
|
+
* `ASSETS` and advertise it only when it exists (a `2xx` reply). Costs one
|
|
37
|
+
* extra local asset lookup per HTML page; worth it when some routes (error
|
|
38
|
+
* pages, dynamic outcomes) have no twin and an agent following the link
|
|
39
|
+
* would land on an error. Has no effect when `advertiseTwin` is `false`.
|
|
40
|
+
* Default: `false`. */
|
|
41
|
+
checkTwin?: boolean;
|
|
42
|
+
/** Also advertise an `llms.txt` index on each HTML page via
|
|
43
|
+
* `Link: </llms.txt>; rel="alternate"; type="text/plain"`. Pass `true` for
|
|
44
|
+
* `/llms.txt` or a path string for another location. Independent of
|
|
45
|
+
* `advertiseTwin`. Default: `false`. */
|
|
46
|
+
advertiseLlmsTxt?: boolean | string;
|
|
47
|
+
/** The origin your build bakes into absolute URLs (e.g. `https://example.com`,
|
|
48
|
+
* your Astro/Next `site`). On a request whose hostname differs (a preview
|
|
49
|
+
* deployment, a staging host), `og:image` and `twitter:image` URLs under this
|
|
50
|
+
* origin are rewritten to the origin being browsed, so images added in a
|
|
51
|
+
* preview resolve there. Nothing else is rewritten (`canonical` and `og:url`
|
|
52
|
+
* keep pointing at production), and it is a no-op on the canonical host.
|
|
53
|
+
* Uses `HTMLRewriter`; applies to HTML responses this worker negotiates.
|
|
54
|
+
* Default: unset (no rewriting). */
|
|
55
|
+
canonicalOrigin?: string;
|
|
35
56
|
}
|
|
36
57
|
|
|
58
|
+
/** The function behind {@link createMdRouter}: call it from your own `fetch`
|
|
59
|
+
* when markdown negotiation is one step of a larger Worker. */
|
|
60
|
+
export type MdResponder<Env extends MdRouterEnv = MdRouterEnv> = (
|
|
61
|
+
request: Request,
|
|
62
|
+
env: Env
|
|
63
|
+
) => Promise<Response>;
|
|
64
|
+
|
|
37
65
|
const defaultMdPathFor = (pathname: string): string => {
|
|
38
66
|
const trimmed = pathname.replace(/\/$/, "");
|
|
39
67
|
return `${trimmed || "/index"}.md`;
|
|
@@ -69,11 +97,38 @@ const defaultMdPathFor = (pathname: string): string => {
|
|
|
69
97
|
export function createMdRouter<Env extends MdRouterEnv = MdRouterEnv>(
|
|
70
98
|
options: MdRouterOptions = {}
|
|
71
99
|
): ExportedHandler<Env> {
|
|
100
|
+
const respond = createMdResponder<Env>(options);
|
|
101
|
+
return {
|
|
102
|
+
fetch: (request, env) => respond(request, env),
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** Like {@link createMdRouter}, but returns a plain `(request, env) => Response`
|
|
107
|
+
* function so it can be called from inside a larger Worker, for example as the
|
|
108
|
+
* fallthrough after your own routes:
|
|
109
|
+
*
|
|
110
|
+
* ```ts
|
|
111
|
+
* const respond = createMdResponder({ vary: true });
|
|
112
|
+
* export default {
|
|
113
|
+
* fetch: (request, env) =>
|
|
114
|
+
* new URL(request.url).pathname.startsWith("/api/")
|
|
115
|
+
* ? handleApi(request, env)
|
|
116
|
+
* : respond(request, env),
|
|
117
|
+
* };
|
|
118
|
+
* ```
|
|
119
|
+
*/
|
|
120
|
+
export function createMdResponder<Env extends MdRouterEnv = MdRouterEnv>(
|
|
121
|
+
options: MdRouterOptions = {}
|
|
122
|
+
): MdResponder<Env> {
|
|
72
123
|
const botUa = options.botUserAgents ?? LLM_BOT_UA;
|
|
73
124
|
const mdPathFor = options.mdPathFor ?? defaultMdPathFor;
|
|
74
125
|
const acceptTokens = ["text/markdown", ...(options.acceptMarkdown ?? [])];
|
|
75
126
|
const advertiseTwin = options.advertiseTwin ?? true;
|
|
76
127
|
const vary = options.vary ?? false;
|
|
128
|
+
const checkTwin = options.checkTwin ?? false;
|
|
129
|
+
const llmsTxtPath =
|
|
130
|
+
options.advertiseLlmsTxt === true ? "/llms.txt" : options.advertiseLlmsTxt || undefined;
|
|
131
|
+
const canonical = options.canonicalOrigin ? new URL(options.canonicalOrigin) : undefined;
|
|
77
132
|
|
|
78
133
|
// Append `Vary: Accept` to a negotiated response when `vary` is enabled.
|
|
79
134
|
// Wraps via `new Response(body, response)` because an asset response's
|
|
@@ -87,51 +142,89 @@ export function createMdRouter<Env extends MdRouterEnv = MdRouterEnv>(
|
|
|
87
142
|
return varied;
|
|
88
143
|
};
|
|
89
144
|
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
145
|
+
// Point absolute og:image / twitter:image URLs at the browsed origin. Matched
|
|
146
|
+
// on hostname, not origin: `wrangler dev` serves a configured route's host
|
|
147
|
+
// over plain http, and an origin comparison would turn `https://host/...`
|
|
148
|
+
// into a `http://` URL that resolves nowhere.
|
|
149
|
+
const withSocialImages = (response: Response, url: URL): Response => {
|
|
150
|
+
if (!canonical || !response.body || url.hostname === canonical.hostname) {
|
|
151
|
+
return response;
|
|
152
|
+
}
|
|
153
|
+
const type = (response.headers.get("Content-Type") ?? "").toLowerCase();
|
|
154
|
+
if (!type.startsWith("text/html")) {
|
|
155
|
+
return response;
|
|
156
|
+
}
|
|
157
|
+
const prefix = `${canonical.origin}/`;
|
|
158
|
+
return new HTMLRewriter()
|
|
159
|
+
.on('meta[property="og:image"], meta[name="twitter:image"]', {
|
|
160
|
+
element(el) {
|
|
161
|
+
const content = el.getAttribute("content");
|
|
162
|
+
if (content?.startsWith(prefix)) {
|
|
163
|
+
el.setAttribute("content", url.origin + content.slice(canonical.origin.length));
|
|
164
|
+
}
|
|
165
|
+
},
|
|
166
|
+
})
|
|
167
|
+
.transform(response);
|
|
168
|
+
};
|
|
93
169
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
170
|
+
const encodePath = (path: string): string =>
|
|
171
|
+
path
|
|
172
|
+
.split("/")
|
|
173
|
+
.map((segment) => encodeURIComponent(segment))
|
|
174
|
+
.join("/");
|
|
175
|
+
|
|
176
|
+
return async (request, env) => {
|
|
177
|
+
const url = new URL(request.url);
|
|
97
178
|
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
179
|
+
if (request.method !== "GET" || /\.[a-zA-Z0-9]+$/.test(url.pathname)) {
|
|
180
|
+
return env.ASSETS.fetch(request);
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
const accept = request.headers.get("Accept") ?? "";
|
|
184
|
+
const ua = request.headers.get("User-Agent") ?? "";
|
|
185
|
+
const wantsMarkdown = acceptTokens.some((tok) => accept.includes(tok)) || botUa.test(ua);
|
|
186
|
+
const mdPath = mdPathFor(url.pathname);
|
|
187
|
+
const fetchTwin = () => env.ASSETS.fetch(new Request(new URL(mdPath, url.origin), request));
|
|
188
|
+
|
|
189
|
+
if (!wantsMarkdown) {
|
|
190
|
+
const response = await env.ASSETS.fetch(request);
|
|
191
|
+
const contentType = (response.headers.get("Content-Type") ?? "")
|
|
192
|
+
.toLowerCase()
|
|
193
|
+
.split(";")[0]
|
|
194
|
+
.trim();
|
|
195
|
+
if (contentType !== "text/html") {
|
|
196
|
+
return withVary(response);
|
|
197
|
+
}
|
|
198
|
+
const advertiseMd =
|
|
199
|
+
advertiseTwin && response.status === 200 && (!checkTwin || (await fetchTwin()).ok);
|
|
200
|
+
const advertiseLlms = llmsTxtPath !== undefined && response.status === 200;
|
|
201
|
+
// `new Response(body, response)` is the only way to add a header to an
|
|
202
|
+
// otherwise-immutable asset response. A relative URI-Reference target is
|
|
203
|
+
// valid per RFC 8288 (resolved against the request URL).
|
|
204
|
+
const out = new Response(response.body, response);
|
|
205
|
+
if (advertiseMd) {
|
|
206
|
+
out.headers.append(
|
|
207
|
+
"Link",
|
|
208
|
+
`<${encodePath(mdPath)}>; rel="alternate"; type="text/markdown"`
|
|
209
|
+
);
|
|
126
210
|
}
|
|
211
|
+
if (advertiseLlms) {
|
|
212
|
+
out.headers.append(
|
|
213
|
+
"Link",
|
|
214
|
+
`<${encodePath(llmsTxtPath)}>; rel="alternate"; type="text/plain"`
|
|
215
|
+
);
|
|
216
|
+
}
|
|
217
|
+
if (vary) {
|
|
218
|
+
out.headers.append("Vary", "Accept");
|
|
219
|
+
}
|
|
220
|
+
return withSocialImages(out, url);
|
|
221
|
+
}
|
|
127
222
|
|
|
128
|
-
|
|
129
|
-
const mdResponse = await env.ASSETS.fetch(new Request(new URL(mdPath, url.origin), request));
|
|
223
|
+
const mdResponse = await fetchTwin();
|
|
130
224
|
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
},
|
|
225
|
+
return mdResponse.status === 404
|
|
226
|
+
? withSocialImages(withVary(await env.ASSETS.fetch(request)), url)
|
|
227
|
+
: withVary(mdResponse);
|
|
135
228
|
};
|
|
136
229
|
}
|
|
137
230
|
|
|
@@ -140,7 +233,7 @@ export function createMdRouter<Env extends MdRouterEnv = MdRouterEnv>(
|
|
|
140
233
|
*
|
|
141
234
|
* ```ts
|
|
142
235
|
* // worker/index.ts
|
|
143
|
-
* export { default } from "cloudflare-md-router/worker";
|
|
236
|
+
* export { default } from "@wave-rf/cloudflare-md-router/worker";
|
|
144
237
|
* ```
|
|
145
238
|
*/
|
|
146
239
|
export const mdRouter: ExportedHandler<MdRouterEnv> = createMdRouter();
|