@docsector/docsector-reader 1.2.1 → 1.2.3

package/README.md

@@ -23,6 +23,7 @@ Transform Markdown content into beautiful, navigable documentation sites — wit
 - 📋 **Copy Page** — One-click button copies the current page as raw Markdown, ready to paste into LLMs
 - 📄 **View as Markdown** — Open any page as plain text by appending `.md` to the URL, with locale support (`?lang=`)
 - 🧠 **Markdown Negotiation** — Requests with `Accept: text/markdown` receive markdown responses, while browsers keep HTML by default
+- 🔐 **Web Bot Auth Directory** — Optional signed JWKS directory at `/.well-known/http-message-signatures-directory` for bot identity verification
 - 🤖 **Open in ChatGPT / Claude** — One-click links to open the current page directly in ChatGPT or Claude for Q&A
 - 🤖 **LLM Bot Detection** — Automatically serves raw Markdown to known AI crawlers (GPTBot, ClaudeBot, PerplexityBot, GrokBot, and others)
 - 🗺️ **Sitemap Generation** — Automatic `sitemap.xml` generation at build time with all page URLs (requires `siteUrl` in config)
@@ -48,6 +49,7 @@ Transform Markdown content into beautiful, navigable documentation sites — wit
 - 📅 **Last Updated Date** — Automatic per-page "last updated" date from git commit history, locale-formatted
 - 📊 **Translation Progress** — Automatic translation percentage based on header coverage
 - 🧠 **Markdown Negotiation** — Responds with Markdown when clients send `Accept: text/markdown`, while keeping HTML as browser default
+- 🔐 **Web Bot Auth** — Can publish a signed HTTP message signatures directory and includes helpers to sign outbound bot requests
 - 🏠 **Markdown Home at Root** — Homepage is rendered from `src/pages/Homepage.{lang}.md` directly at `/`
 - 🧭 **Quick Links Custom Element** — Use `<d-quick-links>` and `<d-quick-link>` in Markdown to render rich home navigation cards
 - 🗂️ **API Catalog Well-Known** — Auto-generates `/.well-known/api-catalog` as Linkset JSON for machine-readable API discovery
@@ -202,6 +204,62 @@ export default {
 
 Set any target to `null` or `false` to disable that relation.
 
+---
+
+## 🔐 Web Bot Auth
+
+Docsector Reader can publish a signed Web Bot Auth directory at:
+
+- `/.well-known/http-message-signatures-directory`
+
+This response is served by Cloudflare Pages runtime middleware and includes:
+
+- `Content-Type: application/http-message-signatures-directory+json`
+- `Signature`
+- `Signature-Input`
+
+### Configure directory publishing
+
+```javascript
+export default {
+  // ...other config
+
+  webBotAuth: {
+    enabled: true,
+    directoryPath: '/.well-known/http-message-signatures-directory',
+    jwksEnv: 'WEB_BOT_AUTH_JWKS',
+    privateJwkEnv: 'WEB_BOT_AUTH_PRIVATE_JWK',
+    keyIdEnv: 'WEB_BOT_AUTH_KEY_ID',
+    keyId: null,
+    signatureMaxAge: 300,
+    signatureLabel: 'sig1'
+  }
+}
+```
+
+Required runtime variables (Cloudflare Pages / Workers environment):
+
+- `WEB_BOT_AUTH_JWKS`: JSON string with a valid JWKS payload (`{ "keys": [...] }`)
+- `WEB_BOT_AUTH_PRIVATE_JWK`: JSON string for an Ed25519 private JWK used to sign directory responses
+- `WEB_BOT_AUTH_KEY_ID`: optional key id override (thumbprint or `kid`)
+
+### Sign outbound bot requests
+
+Use the helper export:
+
+```javascript
+import { createWebBotAuthHeaders } from '@docsector/docsector-reader/web-bot-auth'
+
+const signed = await createWebBotAuthHeaders({
+  url: 'https://crawltest.com/cdn-cgi/web-bot-auth',
+  privateJwk,
+  keyId: 'your-jwk-thumbprint',
+  signatureAgent: 'https://docs.example.com/.well-known/http-message-signatures-directory'
+})
+```
+
+Attach returned headers to your outbound request (`Signature-Agent`, `Signature-Input`, `Signature`).
+
 ### Validate
 
 ```bash

package/bin/docsector.js

@@ -23,7 +23,7 @@ const packageRoot = resolve(__dirname, '..')
 const args = process.argv.slice(2)
 const command = args[0]
 
-const VERSION = '1.2.1'
+const VERSION = '1.2.3'
 
 const HELP = `
   Docsector Reader v${VERSION}
@@ -155,6 +155,21 @@ export default {
   //   agentFallback: true
   // },
 
+  // @ Web Bot Auth (optional)
+  // Publishes a signed JWKS directory at
+  // /.well-known/http-message-signatures-directory
+  // using runtime environment variables.
+  // webBotAuth: {
+  //   enabled: true,
+  //   directoryPath: '/.well-known/http-message-signatures-directory',
+  //   jwksEnv: 'WEB_BOT_AUTH_JWKS',
+  //   privateJwkEnv: 'WEB_BOT_AUTH_PRIVATE_JWK',
+  //   keyIdEnv: 'WEB_BOT_AUTH_KEY_ID',
+  //   keyId: null,
+  //   signatureMaxAge: 300,
+  //   signatureLabel: 'sig1'
+  // },
+
   // @ Languages
   languages: [
     {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@docsector/docsector-reader",
-  "version": "1.2.1",
+  "version": "1.2.3",
   "description": "A documentation rendering engine built with Vue 3, Quasar v2 and Vite. Transform Markdown into beautiful, navigable documentation sites.",
   "productName": "Docsector Reader",
   "author": "Rodrigo de Araujo Vieira",
@@ -13,6 +13,7 @@
     ".": "./src/index.js",
     "./config": "./docsector.config.js",
     "./quasar-factory": "./src/quasar.factory.js",
+    "./web-bot-auth": "./src/web-bot-auth/index.js",
     "./i18n": "./src/i18n/helpers.js",
     "./boot/store": "./src/boot/store.js",
     "./boot/QZoom": "./src/boot/QZoom.js",

package/src/index.js

@@ -57,6 +57,15 @@
  * @param {Object} [config.markdownNegotiation] - Markdown content negotiation settings for agents
  * @param {boolean} [config.markdownNegotiation.enabled=true] - Enables markdown negotiation by Accept header in production runtime
  * @param {boolean} [config.markdownNegotiation.agentFallback=true] - Enables markdown fallback for known AI bot user agents when Accept is absent
+ * @param {Object} [config.webBotAuth] - Web Bot Auth settings for signed bot identity
+ * @param {boolean} [config.webBotAuth.enabled=false] - Enables Web Bot Auth directory publishing and response signature headers
+ * @param {string} [config.webBotAuth.directoryPath='/.well-known/http-message-signatures-directory'] - Well-known URI where JWKS directory is exposed
+ * @param {string} [config.webBotAuth.jwksEnv='WEB_BOT_AUTH_JWKS'] - Environment variable containing JWKS JSON
+ * @param {string} [config.webBotAuth.privateJwkEnv='WEB_BOT_AUTH_PRIVATE_JWK'] - Environment variable containing private Ed25519 JWK JSON used to sign the directory response
+ * @param {string} [config.webBotAuth.keyIdEnv='WEB_BOT_AUTH_KEY_ID'] - Environment variable containing signature key identifier (thumbprint or kid)
+ * @param {string|null} [config.webBotAuth.keyId=null] - Optional static fallback key identifier when env var is absent
+ * @param {number} [config.webBotAuth.signatureMaxAge=300] - Signature validity window in seconds for directory responses
+ * @param {string} [config.webBotAuth.signatureLabel='sig1'] - Signature label used in Signature and Signature-Input headers
  * @returns {Object} Resolved Docsector configuration
  */
 export function createDocsector (config = {}) {
@@ -118,6 +127,18 @@ export function createDocsector (config = {}) {
       enabled: true,
       agentFallback: true,
       ...config.markdownNegotiation
+    },
+
+    webBotAuth: {
+      enabled: false,
+      directoryPath: '/.well-known/http-message-signatures-directory',
+      jwksEnv: 'WEB_BOT_AUTH_JWKS',
+      privateJwkEnv: 'WEB_BOT_AUTH_PRIVATE_JWK',
+      keyIdEnv: 'WEB_BOT_AUTH_KEY_ID',
+      keyId: null,
+      signatureMaxAge: 300,
+      signatureLabel: 'sig1',
+      ...config.webBotAuth
     }
   }
 }

package/src/quasar.factory.js

@@ -742,15 +742,172 @@ function createMarkdownBuildPlugin (projectRoot) {
       const markdownNegotiationConfig = config.markdownNegotiation || {}
       const markdownNegotiationEnabled = markdownNegotiationConfig.enabled !== false
       const markdownAgentFallback = markdownNegotiationConfig.agentFallback !== false
-
-      if (markdownNegotiationEnabled) {
+      const webBotAuthConfig = config.webBotAuth || {}
+      const webBotAuthEnabled = webBotAuthConfig.enabled === true
+      const webBotAuthDirectoryPath = webBotAuthConfig.directoryPath || '/.well-known/http-message-signatures-directory'
+      const webBotAuthJwksEnv = webBotAuthConfig.jwksEnv || 'WEB_BOT_AUTH_JWKS'
+      const webBotAuthPrivateJwkEnv = webBotAuthConfig.privateJwkEnv || 'WEB_BOT_AUTH_PRIVATE_JWK'
+      const webBotAuthKeyIdEnv = webBotAuthConfig.keyIdEnv || 'WEB_BOT_AUTH_KEY_ID'
+      const webBotAuthStaticKeyId = webBotAuthConfig.keyId || null
+      const webBotAuthSignatureMaxAge = Number.isFinite(webBotAuthConfig.signatureMaxAge)
+        ? Math.max(30, Number(webBotAuthConfig.signatureMaxAge))
+        : 300
+      const webBotAuthSignatureLabel = webBotAuthConfig.signatureLabel || 'sig1'
+
+      if (markdownNegotiationEnabled || webBotAuthEnabled) {
         const functionsDir = resolve(projectRoot, 'functions')
         mkdirSync(functionsDir, { recursive: true })
 
         const middlewareCode = `const LLM_BOT_PATTERN = /GPTBot|ChatGPT-User|OAI-SearchBot|ClaudeBot|Claude-User|Claude-SearchBot|anthropic-ai|Google-Extended|Gemini-Deep-Research|PerplexityBot|Perplexity-User|Bytespider|CCBot|Meta-ExternalAgent|FacebookBot|Amazonbot|Applebot-Extended|cohere-ai|DuckAssistBot|GrokBot|AI2Bot|YouBot|PetalBot/i
 
 const DEFAULT_LANG = ${JSON.stringify(defaultLang)}
+const MARKDOWN_ENABLED = ${markdownNegotiationEnabled ? 'true' : 'false'}
 const AGENT_FALLBACK = ${markdownAgentFallback ? 'true' : 'false'}
+const WEB_BOT_AUTH_ENABLED = ${webBotAuthEnabled ? 'true' : 'false'}
+const WEB_BOT_AUTH_DIRECTORY_PATH = ${JSON.stringify(webBotAuthDirectoryPath)}
+const WEB_BOT_AUTH_JWKS_ENV = ${JSON.stringify(webBotAuthJwksEnv)}
+const WEB_BOT_AUTH_PRIVATE_JWK_ENV = ${JSON.stringify(webBotAuthPrivateJwkEnv)}
+const WEB_BOT_AUTH_KEY_ID_ENV = ${JSON.stringify(webBotAuthKeyIdEnv)}
+const WEB_BOT_AUTH_STATIC_KEY_ID = ${JSON.stringify(webBotAuthStaticKeyId)}
+const WEB_BOT_AUTH_SIGNATURE_MAX_AGE = ${webBotAuthSignatureMaxAge}
+const WEB_BOT_AUTH_SIGNATURE_LABEL = ${JSON.stringify(webBotAuthSignatureLabel)}
+
+const textEncoder = new TextEncoder()
+
+function bytesToBase64 (bytes) {
+  let binary = ''
+  for (let i = 0; i < bytes.length; i++) {
+    binary += String.fromCharCode(bytes[i])
+  }
+  return btoa(binary)
+}
+
+function bytesToBase64Url (bytes) {
+  return bytesToBase64(bytes).replace(/\\+/g, '-').replace(/\\//g, '_').replace(/=+$/g, '')
+}
+
+async function sha256 (input) {
+  const digest = await crypto.subtle.digest('SHA-256', textEncoder.encode(input))
+  return new Uint8Array(digest)
+}
+
+async function computeJwkThumbprint (jwk) {
+  if (!jwk || jwk.kty !== 'OKP' || jwk.crv !== 'Ed25519' || !jwk.x) {
+    return null
+  }
+
+  const canonical = JSON.stringify({ crv: jwk.crv, kty: jwk.kty, x: jwk.x })
+  const digest = await sha256(canonical)
+  return bytesToBase64Url(digest)
+}
+
+function parseJsonEnv (env, key) {
+  const raw = env?.[key]
+  if (!raw || typeof raw !== 'string') {
+    return null
+  }
+
+  try {
+    return JSON.parse(raw)
+  } catch {
+    return null
+  }
+}
+
+function badWebBotAuthResponse (message, status = 500) {
+  return new Response(JSON.stringify({ error: message }), {
+    status,
+    headers: {
+      'Content-Type': 'application/json; charset=utf-8',
+      'Cache-Control': 'no-store'
+    }
+  })
+}
+
+async function importEd25519PrivateKey (privateJwk) {
+  if (!privateJwk || privateJwk.kty !== 'OKP' || privateJwk.crv !== 'Ed25519' || !privateJwk.d || !privateJwk.x) {
+    return null
+  }
+
+  return crypto.subtle.importKey(
+    'jwk',
+    privateJwk,
+    { name: 'Ed25519' },
+    false,
+    ['sign']
+  )
+}
+
+async function signDirectoryResponse ({ authority, keyId, created, expires, privateKey }) {
+  const params = '("@authority";req);created=' + created + ';expires=' + expires + ';keyid="' + keyId + '";alg="ed25519";tag="http-message-signatures-directory"'
+  const signatureInput = WEB_BOT_AUTH_SIGNATURE_LABEL + '=' + params
+  const base = '"@authority";req: ' + authority + '\\n"@signature-params": ' + params
+  const signature = await crypto.subtle.sign('Ed25519', privateKey, textEncoder.encode(base))
+
+  return {
+    signatureInput,
+    signature: WEB_BOT_AUTH_SIGNATURE_LABEL + '=:' + bytesToBase64(new Uint8Array(signature)) + ':'
+  }
+}
+
+async function handleWebBotAuthDirectory (request, env, pathname) {
+  if (!WEB_BOT_AUTH_ENABLED || pathname !== WEB_BOT_AUTH_DIRECTORY_PATH) {
+    return null
+  }
+
+  const jwks = parseJsonEnv(env, WEB_BOT_AUTH_JWKS_ENV)
+  if (!jwks || !Array.isArray(jwks.keys) || jwks.keys.length === 0) {
+    return badWebBotAuthResponse('Missing or invalid JWKS in env var ' + WEB_BOT_AUTH_JWKS_ENV)
+  }
+
+  const privateJwk = parseJsonEnv(env, WEB_BOT_AUTH_PRIVATE_JWK_ENV)
+  if (!privateJwk) {
+    return badWebBotAuthResponse('Missing or invalid private JWK in env var ' + WEB_BOT_AUTH_PRIVATE_JWK_ENV)
+  }
+
+  const privateKey = await importEd25519PrivateKey(privateJwk)
+  if (!privateKey) {
+    return badWebBotAuthResponse('Private JWK must be an Ed25519 OKP key with d and x')
+  }
+
+  const selectedPublicJwk = jwks.keys.find((key) => key && key.kty === 'OKP' && key.crv === 'Ed25519' && typeof key.x === 'string')
+  if (!selectedPublicJwk) {
+    return badWebBotAuthResponse('JWKS must include at least one Ed25519 public JWK')
+  }
+
+  const envKeyId = env?.[WEB_BOT_AUTH_KEY_ID_ENV]
+  const computedKeyId = await computeJwkThumbprint(selectedPublicJwk)
+  const keyId = envKeyId || WEB_BOT_AUTH_STATIC_KEY_ID || selectedPublicJwk.kid || computedKeyId
+  if (!keyId) {
+    return badWebBotAuthResponse('Unable to resolve keyid for directory signature')
+  }
+
+  const created = Math.floor(Date.now() / 1000)
+  const expires = created + WEB_BOT_AUTH_SIGNATURE_MAX_AGE
+  const authority = new URL(request.url).host
+
+  const signedHeaders = await signDirectoryResponse({
+    authority,
+    keyId,
+    created,
+    expires,
+    privateKey
+  })
+
+  const body = JSON.stringify(jwks, null, 2) + '\n'
+  const headers = new Headers({
+    'Content-Type': 'application/http-message-signatures-directory+json',
+    'Cache-Control': 'public, max-age=60',
+    Signature: signedHeaders.signature,
+    'Signature-Input': signedHeaders.signatureInput
+  })
+
+  if (request.method === 'HEAD') {
+    return new Response(null, { status: 200, headers })
+  }
+
+  return new Response(body, { status: 200, headers })
+}
 
 function wantsMarkdown (request) {
   const accept = (request.headers.get('accept') || '').toLowerCase()
@@ -792,6 +949,15 @@ export async function onRequest (context) {
   }
 
   const url = new URL(request.url)
+  const webBotAuthResponse = await handleWebBotAuthDirectory(request, env, url.pathname)
+  if (webBotAuthResponse) {
+    return webBotAuthResponse
+  }
+
+  if (!MARKDOWN_ENABLED) {
+    return next()
+  }
+
   if (shouldBypass(url.pathname)) {
     return next()
   }
@@ -833,7 +999,7 @@ export async function onRequest (context) {
 `
 
         writeFileSync(resolve(functionsDir, '_middleware.js'), middlewareCode)
-        console.log(`\x1b[36m[docsector]\x1b[0m Generated markdown negotiation middleware at functions/_middleware.js`)
+        console.log(`\x1b[36m[docsector]\x1b[0m Generated runtime middleware at functions/_middleware.js`)
       }
       console.log(`\x1b[36m[docsector]\x1b[0m Added _headers rule for .md files`)
 
@@ -1049,7 +1215,7 @@ export async function onRequest (context) {
       }
 
       // Generate or merge _routes.json for Cloudflare Pages functions
-      if (config.mcp || markdownNegotiationEnabled) {
+      if (config.mcp || markdownNegotiationEnabled || webBotAuthEnabled) {
         const routesPath = resolve(distDir, '_routes.json')
         let routes = { version: 1, include: [], exclude: [] }
         if (existsSync(routesPath)) {
@@ -1068,10 +1234,14 @@ export async function onRequest (context) {
           routes.include.push('/mcp')
         }
 
+        if (webBotAuthEnabled && !markdownNegotiationEnabled && !routes.include.includes(webBotAuthDirectoryPath)) {
+          routes.include.push(webBotAuthDirectoryPath)
+        }
+
         // Cloudflare Pages rejects overlapping include rules (e.g. "/mcp" with "/*").
         // Keep only the catch-all when markdown negotiation is enabled.
         if (routes.include.includes('/*')) {
-          routes.include = routes.include.filter((route) => route !== '/mcp')
+          routes.include = ['/*']
         }
 
         const markdownExcludes = [

package/src/web-bot-auth/index.js

@@ -0,0 +1,121 @@
+/**
+ * Web Bot Auth request signing helper.
+ *
+ * This utility prepares Signature-Agent, Signature-Input and Signature headers
+ * for outbound bot/agent requests following the Web Bot Auth profile.
+ */
+
+const encoder = new TextEncoder()
+
+function bytesToBase64 (bytes) {
+  let binary = ''
+  for (let i = 0; i < bytes.length; i++) {
+    binary += String.fromCharCode(bytes[i])
+  }
+
+  if (typeof btoa === 'function') {
+    return btoa(binary)
+  }
+
+  return Buffer.from(bytes).toString('base64')
+}
+
+function assertHttpsUrl (value, fieldName) {
+  let url
+  try {
+    url = new URL(value)
+  } catch {
+    throw new Error(`${fieldName} must be a valid URL`)
+  }
+
+  if (url.protocol !== 'https:') {
+    throw new Error(`${fieldName} must use https://`)
+  }
+
+  return url
+}
+
+async function importPrivateKey (privateJwk) {
+  if (!privateJwk || privateJwk.kty !== 'OKP' || privateJwk.crv !== 'Ed25519' || !privateJwk.d || !privateJwk.x) {
+    throw new Error('privateJwk must be an Ed25519 private JWK with kty=OKP, crv=Ed25519, d and x')
+  }
+
+  return crypto.subtle.importKey('jwk', privateJwk, { name: 'Ed25519' }, false, ['sign'])
+}
+
+function formatSignatureParams ({ label, keyId, created, expires }) {
+  return `${label}=("@authority" "signature-agent");created=${created};expires=${expires};keyid="${keyId}";alg="ed25519";tag="web-bot-auth"`
+}
+
+function formatSignatureBase ({ authority, signatureAgentHeader, params }) {
+  return `"@authority": ${authority}\n"signature-agent": ${signatureAgentHeader}\n"@signature-params": ${params}`
+}
+
+/**
+ * Build Web Bot Auth headers for an outbound request.
+ *
+ * @param {Object} options
+ * @param {string} options.url - Target request URL.
+ * @param {Object} options.privateJwk - Private Ed25519 JWK.
+ * @param {string} options.keyId - JWK thumbprint or configured key id.
+ * @param {string} options.signatureAgent - HTTPS URL of your signatures directory.
+ * @param {number} [options.expiresIn=60] - Signature validity window in seconds.
+ * @param {number} [options.createdAt] - Optional created timestamp (unix seconds).
+ * @param {string} [options.label='sig1'] - Signature label.
+ * @returns {Promise<Object>} Headers object with Signature-Agent, Signature-Input and Signature.
+ */
+export async function createWebBotAuthHeaders ({
+  url,
+  privateJwk,
+  keyId,
+  signatureAgent,
+  expiresIn = 60,
+  createdAt,
+  label = 'sig1'
+}) {
+  if (!keyId || typeof keyId !== 'string') {
+    throw new Error('keyId is required')
+  }
+
+  const target = assertHttpsUrl(url, 'url')
+  const signatureAgentUrl = assertHttpsUrl(signatureAgent, 'signatureAgent')
+  const signatureAgentHeader = `"${signatureAgentUrl.toString()}"`
+
+  const created = Number.isFinite(createdAt) ? Math.floor(createdAt) : Math.floor(Date.now() / 1000)
+  const ttl = Number.isFinite(expiresIn) ? Math.max(30, Math.floor(expiresIn)) : 60
+  const expires = created + ttl
+
+  const params = formatSignatureParams({ label, keyId, created, expires })
+  const paramsValue = params.slice(label.length + 1)
+  const base = formatSignatureBase({
+    authority: target.host,
+    signatureAgentHeader,
+    params: paramsValue
+  })
+
+  const privateKey = await importPrivateKey(privateJwk)
+  const signatureBytes = await crypto.subtle.sign('Ed25519', privateKey, encoder.encode(base))
+  const signature = `${label}=:${bytesToBase64(new Uint8Array(signatureBytes))}:`
+
+  return {
+    'Signature-Agent': signatureAgentHeader,
+    'Signature-Input': params,
+    Signature: signature
+  }
+}
+
+/**
+ * Mutates a Headers object by adding Web Bot Auth headers.
+ *
+ * @param {Headers} headers - Headers instance to mutate.
+ * @param {Object} options - Same options accepted by createWebBotAuthHeaders.
+ * @returns {Promise<Headers>} The same headers instance with Web Bot Auth headers.
+ */
+export async function applyWebBotAuthHeaders (headers, options) {
+  const signedHeaders = await createWebBotAuthHeaders(options)
+  for (const [name, value] of Object.entries(signedHeaders)) {
+    headers.set(name, value)
+  }
+
+  return headers
+}