@markdown-for-agents/web 0.1.2 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. package/README.md +7 -1
  2. package/dist/index.mjs +9 -3
  3. package/dist/index.mjs.map +1 -1
  4. package/package.json +2 -2
package/README.md CHANGED
@@ -10,6 +10,9 @@ The middleware uses content negotiation. When a client sends `Accept: text/markd
10
10
 
11
11
  - `Content-Type: text/markdown; charset=utf-8`
12
12
  - `x-markdown-tokens` header with the token count
13
+ - `ETag` header with a content hash for cache validation
14
+ - `Vary: Accept` header so CDNs cache HTML and Markdown separately
15
+ - `content-signal` header with publisher consent signals (when configured)
13
16
 
14
17
  ## Install
15
18
 
@@ -96,7 +99,10 @@ const mw = markdownMiddleware({
96
99
  deduplicate: true,
97
100
 
98
101
  // Custom token counter (e.g. tiktoken)
99
- tokenCounter: text => ({ tokens: enc.encode(text).length, characters: text.length, words: text.split(/\s+/).filter(Boolean).length })
102
+ tokenCounter: text => ({ tokens: enc.encode(text).length, characters: text.length, words: text.split(/\s+/).filter(Boolean).length }),
103
+
104
+ // Publisher consent signal header
105
+ contentSignal: { aiTrain: true, search: true, aiInput: true }
100
106
  });
101
107
  ```
102
108
 
package/dist/index.mjs CHANGED
@@ -1,4 +1,4 @@
1
- import { convert } from "markdown-for-agents";
1
+ import { buildContentSignalHeader, convert } from "markdown-for-agents";
2
2
 
3
3
  //#region src/index.ts
4
4
  function wantsMarkdown(request) {
@@ -25,13 +25,19 @@ function wantsMarkdown(request) {
25
25
  function markdownMiddleware(options) {
26
26
  const tokenHeader = options?.tokenHeader ?? "x-markdown-tokens";
27
27
  return async (request, next) => {
28
- if (!wantsMarkdown(request)) return next(request);
29
28
  const response = await next(request);
29
+ response.headers.append("vary", "Accept");
30
+ if (!wantsMarkdown(request)) return response;
30
31
  if (!(response.headers.get("content-type") ?? "").includes("text/html")) return response;
31
- const { markdown, tokenEstimate } = convert(await response.text(), options);
32
+ const { markdown, tokenEstimate, contentHash } = convert(await response.text(), options);
32
33
  const headers = new Headers(response.headers);
33
34
  headers.set("content-type", "text/markdown; charset=utf-8");
34
35
  headers.set(tokenHeader, String(tokenEstimate.tokens));
36
+ headers.set("etag", `"${contentHash}"`);
37
+ if (options?.contentSignal) {
38
+ const signalValue = buildContentSignalHeader(options.contentSignal);
39
+ if (signalValue) headers.set("content-signal", signalValue);
40
+ }
35
41
  return new Response(markdown, {
36
42
  status: response.status,
37
43
  statusText: response.statusText,
package/dist/index.mjs.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.mjs","names":[],"sources":["../src/index.ts"],"sourcesContent":["import { convert } from 'markdown-for-agents';\nimport type { MiddlewareOptions } from 'markdown-for-agents';\n\nexport type { MiddlewareOptions } from 'markdown-for-agents';\n\ntype Handler = (request: Request) => Response | Promise<Response>;\ntype Middleware = (request: Request, next: Handler) => Response | Promise<Response>;\n\nfunction wantsMarkdown(request: Request): boolean {\n const accept = request.headers.get('accept') ?? '';\n return accept.includes('text/markdown');\n}\n\n/**\n * Web-standard middleware that converts HTML responses to markdown\n * when the client sends an `Accept: text/markdown` header.\n *\n * Compatible with any runtime that supports the Fetch API\n * (Cloudflare Workers, Deno, Bun, etc.).\n *\n * @param options - Conversion and middleware options.\n * @returns A middleware function that wraps a {@link Handler}.\n *\n * @example\n * ```ts\n * import { markdownMiddleware } from \"@markdown-for-agents/web\";\n *\n * const md = markdownMiddleware({ extract: true });\n * const response = await md(request, handler);\n * ```\n */\nexport function markdownMiddleware(options?: MiddlewareOptions): Middleware {\n const tokenHeader = options?.tokenHeader ?? 'x-markdown-tokens';\n\n return async (request: Request, next: Handler): Promise<Response> => {\n if (!wantsMarkdown(request)) {\n return next(request);\n }\n\n const response = await next(request);\n const contentType = response.headers.get('content-type') ?? '';\n\n if (!contentType.includes('text/html')) {\n return response;\n }\n\n const html = await response.text();\n const { markdown, tokenEstimate } = convert(html, options);\n\n const headers = new Headers(response.headers);\n headers.set('content-type', 'text/markdown; charset=utf-8');\n headers.set(tokenHeader, String(tokenEstimate.tokens));\n\n return new Response(markdown, {\n status: response.status,\n statusText: response.statusText,\n headers\n });\n };\n}\n"],"mappings":";;;AAQA,SAAS,cAAc,SAA2B;AAE9C,SADe,QAAQ,QAAQ,IAAI,SAAS,IAAI,IAClC,SAAS,gBAAgB;;;;;;;;;;;;;;;;;;;;AAqB3C,SAAgB,mBAAmB,SAAyC;CACxE,MAAM,cAAc,SAAS,eAAe;AAE5C,QAAO,OAAO,SAAkB,SAAqC;AACjE,MAAI,CAAC,cAAc,QAAQ,CACvB,QAAO,KAAK,QAAQ;EAGxB,MAAM,WAAW,MAAM,KAAK,QAAQ;AAGpC,MAAI,EAFgB,SAAS,QAAQ,IAAI,eAAe,IAAI,IAE3C,SAAS,YAAY,CAClC,QAAO;EAIX,MAAM,EAAE,UAAU,kBAAkB,QADvB,MAAM,SAAS,MAAM,EACgB,QAAQ;EAE1D,MAAM,UAAU,IAAI,QAAQ,SAAS,QAAQ;AAC7C,UAAQ,IAAI,gBAAgB,+BAA+B;AAC3D,UAAQ,IAAI,aAAa,OAAO,cAAc,OAAO,CAAC;AAEtD,SAAO,IAAI,SAAS,UAAU;GAC1B,QAAQ,SAAS;GACjB,YAAY,SAAS;GACrB;GACH,CAAC"}
1
+ {"version":3,"file":"index.mjs","names":[],"sources":["../src/index.ts"],"sourcesContent":["import { convert, buildContentSignalHeader } from 'markdown-for-agents';\nimport type { MiddlewareOptions } from 'markdown-for-agents';\n\nexport type { MiddlewareOptions } from 'markdown-for-agents';\n\ntype Handler = (request: Request) => Response | Promise<Response>;\ntype Middleware = (request: Request, next: Handler) => Response | Promise<Response>;\n\nfunction wantsMarkdown(request: Request): boolean {\n const accept = request.headers.get('accept') ?? '';\n return accept.includes('text/markdown');\n}\n\n/**\n * Web-standard middleware that converts HTML responses to markdown\n * when the client sends an `Accept: text/markdown` header.\n *\n * Compatible with any runtime that supports the Fetch API\n * (Cloudflare Workers, Deno, Bun, etc.).\n *\n * @param options - Conversion and middleware options.\n * @returns A middleware function that wraps a {@link Handler}.\n *\n * @example\n * ```ts\n * import { markdownMiddleware } from \"@markdown-for-agents/web\";\n *\n * const md = markdownMiddleware({ extract: true });\n * const response = await md(request, handler);\n * ```\n */\nexport function markdownMiddleware(options?: MiddlewareOptions): Middleware {\n const tokenHeader = options?.tokenHeader ?? 'x-markdown-tokens';\n\n return async (request: Request, next: Handler): Promise<Response> => {\n const response = await next(request);\n\n // Always signal that responses vary by Accept so caches store\n // separate entries for HTML and Markdown representations.\n response.headers.append('vary', 'Accept');\n\n if (!wantsMarkdown(request)) {\n return response;\n }\n\n const contentType = response.headers.get('content-type') ?? '';\n\n if (!contentType.includes('text/html')) {\n return response;\n }\n\n const html = await response.text();\n const { markdown, tokenEstimate, contentHash } = convert(html, options);\n\n const headers = new Headers(response.headers);\n headers.set('content-type', 'text/markdown; charset=utf-8');\n headers.set(tokenHeader, String(tokenEstimate.tokens));\n headers.set('etag', `\"${contentHash}\"`);\n if (options?.contentSignal) {\n const signalValue = buildContentSignalHeader(options.contentSignal);\n if (signalValue) headers.set('content-signal', signalValue);\n }\n\n return new Response(markdown, {\n status: response.status,\n statusText: response.statusText,\n headers\n });\n };\n}\n"],"mappings":";;;AAQA,SAAS,cAAc,SAA2B;AAE9C,SADe,QAAQ,QAAQ,IAAI,SAAS,IAAI,IAClC,SAAS,gBAAgB;;;;;;;;;;;;;;;;;;;;AAqB3C,SAAgB,mBAAmB,SAAyC;CACxE,MAAM,cAAc,SAAS,eAAe;AAE5C,QAAO,OAAO,SAAkB,SAAqC;EACjE,MAAM,WAAW,MAAM,KAAK,QAAQ;AAIpC,WAAS,QAAQ,OAAO,QAAQ,SAAS;AAEzC,MAAI,CAAC,cAAc,QAAQ,CACvB,QAAO;AAKX,MAAI,EAFgB,SAAS,QAAQ,IAAI,eAAe,IAAI,IAE3C,SAAS,YAAY,CAClC,QAAO;EAIX,MAAM,EAAE,UAAU,eAAe,gBAAgB,QADpC,MAAM,SAAS,MAAM,EAC6B,QAAQ;EAEvE,MAAM,UAAU,IAAI,QAAQ,SAAS,QAAQ;AAC7C,UAAQ,IAAI,gBAAgB,+BAA+B;AAC3D,UAAQ,IAAI,aAAa,OAAO,cAAc,OAAO,CAAC;AACtD,UAAQ,IAAI,QAAQ,IAAI,YAAY,GAAG;AACvC,MAAI,SAAS,eAAe;GACxB,MAAM,cAAc,yBAAyB,QAAQ,cAAc;AACnE,OAAI,YAAa,SAAQ,IAAI,kBAAkB,YAAY;;AAG/D,SAAO,IAAI,SAAS,UAAU;GAC1B,QAAQ,SAAS;GACjB,YAAY,SAAS;GACrB;GACH,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@markdown-for-agents/web",
3
- "version": "0.1.2",
3
+ "version": "0.2.0",
4
4
  "description": "Web Standard (Request/Response) middleware for markdown-for-agents",
5
5
  "type": "module",
6
6
  "sideEffects": false,
@@ -20,7 +20,7 @@
20
20
  "main": "./dist/index.mjs",
21
21
  "types": "./dist/index.d.mts",
22
22
  "dependencies": {
23
- "markdown-for-agents": "0.1.2"
23
+ "markdown-for-agents": "0.2.0"
24
24
  },
25
25
  "keywords": [
26
26
  "html",