@wave-rf/cloudflare-md-router 0.2.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@wave-rf/cloudflare-md-router",
3
- "version": "0.2.1",
3
+ "version": "0.4.0",
4
4
  "description": "Cloudflare Worker that serves a `.md` twin of any static page when the request is from a known LLM crawler or explicitly asks for `text/markdown`. Falls back to the HTML response otherwise.",
5
5
  "keywords": [
6
6
  "cloudflare",
@@ -31,6 +31,9 @@
31
31
  "type": "module",
32
32
  "main": "./src/index.ts",
33
33
  "types": "./src/index.ts",
34
+ "bin": {
35
+ "cloudflare-md-router": "./bin/cli.mjs"
36
+ },
34
37
  "exports": {
35
38
  ".": "./src/index.ts",
36
39
  "./worker": "./src/worker.ts",
@@ -38,6 +41,7 @@
38
41
  },
39
42
  "files": [
40
43
  "src",
44
+ "bin",
41
45
  "README.md",
42
46
  "LICENSE"
43
47
  ],
@@ -53,15 +57,21 @@
53
57
  "lint": "biome lint .",
54
58
  "format": "biome format --write .",
55
59
  "check": "biome check .",
56
- "verify": "biome check . && pnpm run typecheck && bash scripts/ci-marker.sh write",
57
- "setup": "git config core.hooksPath .githooks"
60
+ "check:release": "node scripts/check-release-config.mjs",
61
+ "verify": "biome check . && pnpm run typecheck && pnpm run test && pnpm run check:release && bash scripts/ci-marker.sh write",
62
+ "setup": "git config core.hooksPath .githooks",
63
+ "test": "node --test"
58
64
  },
59
65
  "peerDependencies": {
60
66
  "@cloudflare/workers-types": ">=4.0.0"
61
67
  },
62
68
  "devDependencies": {
63
69
  "@biomejs/biome": "^2.4.15",
64
- "@cloudflare/workers-types": "^4.20240208.0",
65
- "typescript": "^6.0.3"
70
+ "@cloudflare/workers-types": "^5.20260706.1",
71
+ "@semantic-release/commit-analyzer": "13.0.1",
72
+ "@semantic-release/release-notes-generator": "14.1.1",
73
+ "conventional-changelog-conventionalcommits": "9.3.1",
74
+ "semantic-release": "25.0.9",
75
+ "typescript": "^7.0.2"
66
76
  }
67
77
  }
package/src/bots.ts CHANGED
@@ -10,4 +10,4 @@
10
10
  // - https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers (Google-Extended)
11
11
 
12
12
  export const LLM_BOT_UA =
13
- /gptbot|chatgpt-user|oai-searchbot|claudebot|claude-web|anthropic-ai|perplexitybot|ccbot|applebot-extended|google-extended|cohere-ai|bytespider|diffbot/i;
13
+ /gptbot|chatgpt-user|oai-searchbot|claudebot|claude-web|anthropic-ai|perplexitybot|ccbot|applebot-extended|google-extended|cohere-ai|bytespider|diffbot|claude-user|claude-searchbot|amazonbot|duckassistbot/i;
package/src/index.ts CHANGED
@@ -1,8 +1,10 @@
1
1
  // biome-ignore assist/source/organizeImports: keep the hand-authored barrel order — the named API exports grouped, then the default handler on its own readable line (Biome would merge `default` into the named-export braces).
2
2
  export { LLM_BOT_UA } from "./bots.ts";
3
3
  export {
4
+ createMdResponder,
4
5
  createMdRouter,
5
6
  mdRouter,
7
+ type MdResponder,
6
8
  type MdRouterEnv,
7
9
  type MdRouterOptions,
8
10
  } from "./worker.ts";
package/src/worker.ts CHANGED
@@ -32,8 +32,36 @@ export interface MdRouterOptions {
32
32
  * shared caching (huge UA cardinality), so it is intentionally omitted.
33
33
  * Default: `false`. */
34
34
  vary?: boolean;
35
+ /** Before advertising a page's `.md` twin in the `Link` header, fetch it from
36
+ * `ASSETS` and advertise it only when it exists (a `2xx` reply). Costs one
37
+ * extra local asset lookup per HTML page; worth it when some routes (error
38
+ * pages, dynamic outcomes) have no twin and an agent following the link
39
+ * would land on an error. Has no effect when `advertiseTwin` is `false`.
40
+ * Default: `false`. */
41
+ checkTwin?: boolean;
42
+ /** Also advertise an `llms.txt` index on each HTML page via
43
+ * `Link: </llms.txt>; rel="alternate"; type="text/plain"`. Pass `true` for
44
+ * `/llms.txt` or a path string for another location. Independent of
45
+ * `advertiseTwin`. Default: `false`. */
46
+ advertiseLlmsTxt?: boolean | string;
47
+ /** The origin your build bakes into absolute URLs (e.g. `https://example.com`,
48
+ * your Astro/Next `site`). On a request whose hostname differs (a preview
49
+ * deployment, a staging host), `og:image` and `twitter:image` URLs under this
50
+ * origin are rewritten to the origin being browsed, so images added in a
51
+ * preview resolve there. Nothing else is rewritten (`canonical` and `og:url`
52
+ * keep pointing at production), and it is a no-op on the canonical host.
53
+ * Uses `HTMLRewriter`; applies to HTML responses this worker negotiates.
54
+ * Default: unset (no rewriting). */
55
+ canonicalOrigin?: string;
35
56
  }
36
57
 
58
+ /** The function behind {@link createMdRouter}: call it from your own `fetch`
59
+ * when markdown negotiation is one step of a larger Worker. */
60
+ export type MdResponder<Env extends MdRouterEnv = MdRouterEnv> = (
61
+ request: Request,
62
+ env: Env
63
+ ) => Promise<Response>;
64
+
37
65
  const defaultMdPathFor = (pathname: string): string => {
38
66
  const trimmed = pathname.replace(/\/$/, "");
39
67
  return `${trimmed || "/index"}.md`;
@@ -69,11 +97,38 @@ const defaultMdPathFor = (pathname: string): string => {
69
97
  export function createMdRouter<Env extends MdRouterEnv = MdRouterEnv>(
70
98
  options: MdRouterOptions = {}
71
99
  ): ExportedHandler<Env> {
100
+ const respond = createMdResponder<Env>(options);
101
+ return {
102
+ fetch: (request, env) => respond(request, env),
103
+ };
104
+ }
105
+
106
+ /** Like {@link createMdRouter}, but returns a plain `(request, env) => Response`
107
+ * function so it can be called from inside a larger Worker, for example as the
108
+ * fallthrough after your own routes:
109
+ *
110
+ * ```ts
111
+ * const respond = createMdResponder({ vary: true });
112
+ * export default {
113
+ * fetch: (request, env) =>
114
+ * new URL(request.url).pathname.startsWith("/api/")
115
+ * ? handleApi(request, env)
116
+ * : respond(request, env),
117
+ * };
118
+ * ```
119
+ */
120
+ export function createMdResponder<Env extends MdRouterEnv = MdRouterEnv>(
121
+ options: MdRouterOptions = {}
122
+ ): MdResponder<Env> {
72
123
  const botUa = options.botUserAgents ?? LLM_BOT_UA;
73
124
  const mdPathFor = options.mdPathFor ?? defaultMdPathFor;
74
125
  const acceptTokens = ["text/markdown", ...(options.acceptMarkdown ?? [])];
75
126
  const advertiseTwin = options.advertiseTwin ?? true;
76
127
  const vary = options.vary ?? false;
128
+ const checkTwin = options.checkTwin ?? false;
129
+ const llmsTxtPath =
130
+ options.advertiseLlmsTxt === true ? "/llms.txt" : options.advertiseLlmsTxt || undefined;
131
+ const canonical = options.canonicalOrigin ? new URL(options.canonicalOrigin) : undefined;
77
132
 
78
133
  // Append `Vary: Accept` to a negotiated response when `vary` is enabled.
79
134
  // Wraps via `new Response(body, response)` because an asset response's
@@ -87,51 +142,89 @@ export function createMdRouter<Env extends MdRouterEnv = MdRouterEnv>(
87
142
  return varied;
88
143
  };
89
144
 
90
- return {
91
- async fetch(request, env): Promise<Response> {
92
- const url = new URL(request.url);
145
+ // Point absolute og:image / twitter:image URLs at the browsed origin. Matched
146
+ // on hostname, not origin: `wrangler dev` serves a configured route's host
147
+ // over plain http, and an origin comparison would turn `https://host/...`
148
+ // into a `http://` URL that resolves nowhere.
149
+ const withSocialImages = (response: Response, url: URL): Response => {
150
+ if (!canonical || !response.body || url.hostname === canonical.hostname) {
151
+ return response;
152
+ }
153
+ const type = (response.headers.get("Content-Type") ?? "").toLowerCase();
154
+ if (!type.startsWith("text/html")) {
155
+ return response;
156
+ }
157
+ const prefix = `${canonical.origin}/`;
158
+ return new HTMLRewriter()
159
+ .on('meta[property="og:image"], meta[name="twitter:image"]', {
160
+ element(el) {
161
+ const content = el.getAttribute("content");
162
+ if (content?.startsWith(prefix)) {
163
+ el.setAttribute("content", url.origin + content.slice(canonical.origin.length));
164
+ }
165
+ },
166
+ })
167
+ .transform(response);
168
+ };
93
169
 
94
- if (request.method !== "GET" || /\.[a-zA-Z0-9]+$/.test(url.pathname)) {
95
- return env.ASSETS.fetch(request);
96
- }
170
+ const encodePath = (path: string): string =>
171
+ path
172
+ .split("/")
173
+ .map((segment) => encodeURIComponent(segment))
174
+ .join("/");
175
+
176
+ return async (request, env) => {
177
+ const url = new URL(request.url);
97
178
 
98
- const accept = request.headers.get("Accept") ?? "";
99
- const ua = request.headers.get("User-Agent") ?? "";
100
- const wantsMarkdown = acceptTokens.some((tok) => accept.includes(tok)) || botUa.test(ua);
101
-
102
- if (!wantsMarkdown) {
103
- const response = await env.ASSETS.fetch(request);
104
- const contentType = (response.headers.get("Content-Type") ?? "")
105
- .toLowerCase()
106
- .split(";")[0]
107
- .trim();
108
- const isHtml = contentType === "text/html";
109
- if (!advertiseTwin || response.status !== 200 || !isHtml) {
110
- return withVary(response);
111
- }
112
- // `new Response(body, response)` is the only way to add a header to an
113
- // otherwise-immutable asset response. A relative URI-Reference target is
114
- // valid per RFC 8288 (resolved against the request URL).
115
- const withTwin = new Response(response.body, response);
116
- const encodedPath = mdPathFor(url.pathname)
117
- .split("/")
118
- .map((segment) => encodeURIComponent(segment))
119
- .join("/");
120
- withTwin.headers.append("Link", `<${encodedPath}>; rel="alternate"; type="text/markdown"`);
121
- // Already a mutable wrapper — append `Vary` here rather than re-wrapping.
122
- if (vary) {
123
- withTwin.headers.append("Vary", "Accept");
124
- }
125
- return withTwin;
179
+ if (request.method !== "GET" || /\.[a-zA-Z0-9]+$/.test(url.pathname)) {
180
+ return env.ASSETS.fetch(request);
181
+ }
182
+
183
+ const accept = request.headers.get("Accept") ?? "";
184
+ const ua = request.headers.get("User-Agent") ?? "";
185
+ const wantsMarkdown = acceptTokens.some((tok) => accept.includes(tok)) || botUa.test(ua);
186
+ const mdPath = mdPathFor(url.pathname);
187
+ const fetchTwin = () => env.ASSETS.fetch(new Request(new URL(mdPath, url.origin), request));
188
+
189
+ if (!wantsMarkdown) {
190
+ const response = await env.ASSETS.fetch(request);
191
+ const contentType = (response.headers.get("Content-Type") ?? "")
192
+ .toLowerCase()
193
+ .split(";")[0]
194
+ .trim();
195
+ if (contentType !== "text/html") {
196
+ return withVary(response);
197
+ }
198
+ const advertiseMd =
199
+ advertiseTwin && response.status === 200 && (!checkTwin || (await fetchTwin()).ok);
200
+ const advertiseLlms = llmsTxtPath !== undefined && response.status === 200;
201
+ // `new Response(body, response)` is the only way to add a header to an
202
+ // otherwise-immutable asset response. A relative URI-Reference target is
203
+ // valid per RFC 8288 (resolved against the request URL).
204
+ const out = new Response(response.body, response);
205
+ if (advertiseMd) {
206
+ out.headers.append(
207
+ "Link",
208
+ `<${encodePath(mdPath)}>; rel="alternate"; type="text/markdown"`
209
+ );
126
210
  }
211
+ if (advertiseLlms) {
212
+ out.headers.append(
213
+ "Link",
214
+ `<${encodePath(llmsTxtPath)}>; rel="alternate"; type="text/plain"`
215
+ );
216
+ }
217
+ if (vary) {
218
+ out.headers.append("Vary", "Accept");
219
+ }
220
+ return withSocialImages(out, url);
221
+ }
127
222
 
128
- const mdPath = mdPathFor(url.pathname);
129
- const mdResponse = await env.ASSETS.fetch(new Request(new URL(mdPath, url.origin), request));
223
+ const mdResponse = await fetchTwin();
130
224
 
131
- return mdResponse.status === 404
132
- ? withVary(await env.ASSETS.fetch(request))
133
- : withVary(mdResponse);
134
- },
225
+ return mdResponse.status === 404
226
+ ? withSocialImages(withVary(await env.ASSETS.fetch(request)), url)
227
+ : withVary(mdResponse);
135
228
  };
136
229
  }
137
230
 
@@ -140,7 +233,7 @@ export function createMdRouter<Env extends MdRouterEnv = MdRouterEnv>(
140
233
  *
141
234
  * ```ts
142
235
  * // worker/index.ts
143
- * export { default } from "cloudflare-md-router/worker";
236
+ * export { default } from "@wave-rf/cloudflare-md-router/worker";
144
237
  * ```
145
238
  */
146
239
  export const mdRouter: ExportedHandler<MdRouterEnv> = createMdRouter();