@markdown-for-agents/web 0.1.1 → 0.1.3

package/README.md

@@ -0,0 +1,116 @@
+# @markdown-for-agents/web
+
+Web Standard (Request/Response) middleware for [markdown-for-agents](https://www.npmjs.com/package/markdown-for-agents) — a runtime-agnostic HTML to Markdown converter built for AI agents.
+
+Works anywhere the Web Standard Request/Response API is available: Cloudflare Workers, Deno, Bun, and Node.js. AI agents get clean, token-efficient Markdown instead of HTML. Normal browser requests pass through untouched.
+
+## How it works
+
+The middleware uses content negotiation. When a client sends `Accept: text/markdown`, HTML responses are automatically converted to Markdown. The response includes:
+
+- `Content-Type: text/markdown; charset=utf-8`
+- `x-markdown-tokens` header with the token count
+- `ETag` header with a content hash for cache validation
+- `Vary: Accept` header so CDNs cache HTML and Markdown separately
+
+## Install
+
+```bash
+npm install @markdown-for-agents/web markdown-for-agents
+```
+
+## Usage
+
+### Cloudflare Workers
+
+```ts
+import { markdownMiddleware } from '@markdown-for-agents/web';
+
+const mw = markdownMiddleware({ extract: true });
+
+export default {
+    async fetch(request: Request): Promise<Response> {
+        return mw(request, async () => {
+            const html = await renderPage();
+            return new Response(html, {
+                headers: { 'content-type': 'text/html' }
+            });
+        });
+    }
+};
+```
+
+### Deno
+
+```ts
+import { markdownMiddleware } from '@markdown-for-agents/web';
+
+const mw = markdownMiddleware({ extract: true });
+
+Deno.serve(async request => {
+    return mw(request, async () => {
+        return new Response('<h1>Hello</h1>', {
+            headers: { 'content-type': 'text/html' }
+        });
+    });
+});
+```
+
+### Bun
+
+```ts
+import { markdownMiddleware } from '@markdown-for-agents/web';
+
+const mw = markdownMiddleware({ extract: true });
+
+Bun.serve({
+    async fetch(request) {
+        return mw(request, async () => {
+            return new Response('<h1>Hello</h1>', {
+                headers: { 'content-type': 'text/html' }
+            });
+        });
+    }
+});
+```
+
+```bash
+# Normal HTML response
+curl http://localhost:3000
+
+# Markdown response for AI agents
+curl -H "Accept: text/markdown" http://localhost:3000
+```
+
+## Options
+
+Accepts all [`markdown-for-agents` options](https://www.npmjs.com/package/markdown-for-agents#options):
+
+```ts
+const mw = markdownMiddleware({
+    // Strip nav, ads, sidebars, cookie banners
+    extract: true,
+
+    // Resolve relative URLs
+    baseUrl: 'https://example.com',
+
+    // Remove duplicate content blocks
+    deduplicate: true,
+
+    // Custom token counter (e.g. tiktoken)
+    tokenCounter: text => ({ tokens: enc.encode(text).length, characters: text.length, words: text.split(/\s+/).filter(Boolean).length })
+});
+```
+
+## Other frameworks
+
+| Package                                                                                      | Framework |
+| -------------------------------------------------------------------------------------------- | --------- |
+| [`@markdown-for-agents/express`](https://www.npmjs.com/package/@markdown-for-agents/express) | Express   |
+| [`@markdown-for-agents/fastify`](https://www.npmjs.com/package/@markdown-for-agents/fastify) | Fastify   |
+| [`@markdown-for-agents/hono`](https://www.npmjs.com/package/@markdown-for-agents/hono)       | Hono      |
+| [`@markdown-for-agents/nextjs`](https://www.npmjs.com/package/@markdown-for-agents/nextjs)   | Next.js   |
+
+## License
+
+MIT

package/dist/index.mjs

@@ -25,13 +25,15 @@ function wantsMarkdown(request) {
 function markdownMiddleware(options) {
 	const tokenHeader = options?.tokenHeader ?? "x-markdown-tokens";
 	return async (request, next) => {
-		if (!wantsMarkdown(request)) return next(request);
 		const response = await next(request);
+		response.headers.append("vary", "Accept");
+		if (!wantsMarkdown(request)) return response;
 		if (!(response.headers.get("content-type") ?? "").includes("text/html")) return response;
-		const { markdown, tokenEstimate } = convert(await response.text(), options);
+		const { markdown, tokenEstimate, contentHash } = convert(await response.text(), options);
 		const headers = new Headers(response.headers);
 		headers.set("content-type", "text/markdown; charset=utf-8");
 		headers.set(tokenHeader, String(tokenEstimate.tokens));
+		headers.set("etag", `"${contentHash}"`);
 		return new Response(markdown, {
 			status: response.status,
 			statusText: response.statusText,

package/dist/index.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"index.mjs","names":[],"sources":["../src/index.ts"],"sourcesContent":["import { convert } from 'markdown-for-agents';\nimport type { MiddlewareOptions } from 'markdown-for-agents';\n\nexport type { MiddlewareOptions } from 'markdown-for-agents';\n\ntype Handler = (request: Request) => Response | Promise<Response>;\ntype Middleware = (request: Request, next: Handler) => Response | Promise<Response>;\n\nfunction wantsMarkdown(request: Request): boolean {\n    const accept = request.headers.get('accept') ?? '';\n    return accept.includes('text/markdown');\n}\n\n/**\n * Web-standard middleware that converts HTML responses to markdown\n * when the client sends an `Accept: text/markdown` header.\n *\n * Compatible with any runtime that supports the Fetch API\n * (Cloudflare Workers, Deno, Bun, etc.).\n *\n * @param options - Conversion and middleware options.\n * @returns A middleware function that wraps a {@link Handler}.\n *\n * @example\n * ```ts\n * import { markdownMiddleware } from \"@markdown-for-agents/web\";\n *\n * const md = markdownMiddleware({ extract: true });\n * const response = await md(request, handler);\n * ```\n */\nexport function markdownMiddleware(options?: MiddlewareOptions): Middleware {\n    const tokenHeader = options?.tokenHeader ?? 'x-markdown-tokens';\n\n    return async (request: Request, next: Handler): Promise<Response> => {\n        if (!wantsMarkdown(request)) {\n            return next(request);\n        }\n\n        const response = await next(request);\n        const contentType = response.headers.get('content-type') ?? '';\n\n        if (!contentType.includes('text/html')) {\n            return response;\n        }\n\n        const html = await response.text();\n        const { markdown, tokenEstimate } = convert(html, options);\n\n        const headers = new Headers(response.headers);\n        headers.set('content-type', 'text/markdown; charset=utf-8');\n        headers.set(tokenHeader, String(tokenEstimate.tokens));\n\n        return new Response(markdown, {\n            status: response.status,\n            statusText: response.statusText,\n            headers\n        });\n    };\n}\n"],"mappings":";;;AAQA,SAAS,cAAc,SAA2B;AAE9C,SADe,QAAQ,QAAQ,IAAI,SAAS,IAAI,IAClC,SAAS,gBAAgB;;;;;;;;;;;;;;;;;;;;AAqB3C,SAAgB,mBAAmB,SAAyC;CACxE,MAAM,cAAc,SAAS,eAAe;AAE5C,QAAO,OAAO,SAAkB,SAAqC;AACjE,MAAI,CAAC,cAAc,QAAQ,CACvB,QAAO,KAAK,QAAQ;EAGxB,MAAM,WAAW,MAAM,KAAK,QAAQ;AAGpC,MAAI,EAFgB,SAAS,QAAQ,IAAI,eAAe,IAAI,IAE3C,SAAS,YAAY,CAClC,QAAO;EAIX,MAAM,EAAE,UAAU,kBAAkB,QADvB,MAAM,SAAS,MAAM,EACgB,QAAQ;EAE1D,MAAM,UAAU,IAAI,QAAQ,SAAS,QAAQ;AAC7C,UAAQ,IAAI,gBAAgB,+BAA+B;AAC3D,UAAQ,IAAI,aAAa,OAAO,cAAc,OAAO,CAAC;AAEtD,SAAO,IAAI,SAAS,UAAU;GAC1B,QAAQ,SAAS;GACjB,YAAY,SAAS;GACrB;GACH,CAAC"}
+{"version":3,"file":"index.mjs","names":[],"sources":["../src/index.ts"],"sourcesContent":["import { convert } from 'markdown-for-agents';\nimport type { MiddlewareOptions } from 'markdown-for-agents';\n\nexport type { MiddlewareOptions } from 'markdown-for-agents';\n\ntype Handler = (request: Request) => Response | Promise<Response>;\ntype Middleware = (request: Request, next: Handler) => Response | Promise<Response>;\n\nfunction wantsMarkdown(request: Request): boolean {\n    const accept = request.headers.get('accept') ?? '';\n    return accept.includes('text/markdown');\n}\n\n/**\n * Web-standard middleware that converts HTML responses to markdown\n * when the client sends an `Accept: text/markdown` header.\n *\n * Compatible with any runtime that supports the Fetch API\n * (Cloudflare Workers, Deno, Bun, etc.).\n *\n * @param options - Conversion and middleware options.\n * @returns A middleware function that wraps a {@link Handler}.\n *\n * @example\n * ```ts\n * import { markdownMiddleware } from \"@markdown-for-agents/web\";\n *\n * const md = markdownMiddleware({ extract: true });\n * const response = await md(request, handler);\n * ```\n */\nexport function markdownMiddleware(options?: MiddlewareOptions): Middleware {\n    const tokenHeader = options?.tokenHeader ?? 'x-markdown-tokens';\n\n    return async (request: Request, next: Handler): Promise<Response> => {\n        const response = await next(request);\n\n        // Always signal that responses vary by Accept so caches store\n        // separate entries for HTML and Markdown representations.\n        response.headers.append('vary', 'Accept');\n\n        if (!wantsMarkdown(request)) {\n            return response;\n        }\n\n        const contentType = response.headers.get('content-type') ?? '';\n\n        if (!contentType.includes('text/html')) {\n            return response;\n        }\n\n        const html = await response.text();\n        const { markdown, tokenEstimate, contentHash } = convert(html, options);\n\n        const headers = new Headers(response.headers);\n        headers.set('content-type', 'text/markdown; charset=utf-8');\n        headers.set(tokenHeader, String(tokenEstimate.tokens));\n        headers.set('etag', `\"${contentHash}\"`);\n\n        return new Response(markdown, {\n            status: response.status,\n            statusText: response.statusText,\n            headers\n        });\n    };\n}\n"],"mappings":";;;AAQA,SAAS,cAAc,SAA2B;AAE9C,SADe,QAAQ,QAAQ,IAAI,SAAS,IAAI,IAClC,SAAS,gBAAgB;;;;;;;;;;;;;;;;;;;;AAqB3C,SAAgB,mBAAmB,SAAyC;CACxE,MAAM,cAAc,SAAS,eAAe;AAE5C,QAAO,OAAO,SAAkB,SAAqC;EACjE,MAAM,WAAW,MAAM,KAAK,QAAQ;AAIpC,WAAS,QAAQ,OAAO,QAAQ,SAAS;AAEzC,MAAI,CAAC,cAAc,QAAQ,CACvB,QAAO;AAKX,MAAI,EAFgB,SAAS,QAAQ,IAAI,eAAe,IAAI,IAE3C,SAAS,YAAY,CAClC,QAAO;EAIX,MAAM,EAAE,UAAU,eAAe,gBAAgB,QADpC,MAAM,SAAS,MAAM,EAC6B,QAAQ;EAEvE,MAAM,UAAU,IAAI,QAAQ,SAAS,QAAQ;AAC7C,UAAQ,IAAI,gBAAgB,+BAA+B;AAC3D,UAAQ,IAAI,aAAa,OAAO,cAAc,OAAO,CAAC;AACtD,UAAQ,IAAI,QAAQ,IAAI,YAAY,GAAG;AAEvC,SAAO,IAAI,SAAS,UAAU;GAC1B,QAAQ,SAAS;GACjB,YAAY,SAAS;GACrB;GACH,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@markdown-for-agents/web",
-  "version": "0.1.1",
+  "version": "0.1.3",
   "description": "Web Standard (Request/Response) middleware for markdown-for-agents",
   "type": "module",
   "sideEffects": false,
@@ -20,7 +20,7 @@
   "main": "./dist/index.mjs",
   "types": "./dist/index.d.mts",
   "dependencies": {
-    "markdown-for-agents": "0.1.1"
+    "markdown-for-agents": "0.1.3"
   },
   "keywords": [
     "html",