@docsector/docsector-reader 1.1.0 → 1.2.1

package/README.md

@@ -22,6 +22,7 @@ Transform Markdown content into beautiful, navigable documentation sites — wit
 
 - 📋 **Copy Page** — One-click button copies the current page as raw Markdown, ready to paste into LLMs
 - 📄 **View as Markdown** — Open any page as plain text by appending `.md` to the URL, with locale support (`?lang=`)
+- 🧠 **Markdown Negotiation** — Requests with `Accept: text/markdown` receive markdown responses, while browsers keep HTML by default
 - 🤖 **Open in ChatGPT / Claude** — One-click links to open the current page directly in ChatGPT or Claude for Q&A
 - 🤖 **LLM Bot Detection** — Automatically serves raw Markdown to known AI crawlers (GPTBot, ClaudeBot, PerplexityBot, GrokBot, and others)
 - 🗺️ **Sitemap Generation** — Automatic `sitemap.xml` generation at build time with all page URLs (requires `siteUrl` in config)
@@ -46,6 +47,7 @@ Transform Markdown content into beautiful, navigable documentation sites — wit
 - ✏️ **Edit on GitHub** — Direct links to edit pages on your repository
 - 📅 **Last Updated Date** — Automatic per-page "last updated" date from git commit history, locale-formatted
 - 📊 **Translation Progress** — Automatic translation percentage based on header coverage
+- 🧠 **Markdown Negotiation** — Responds with Markdown when clients send `Accept: text/markdown`, while keeping HTML as browser default
 - 🏠 **Markdown Home at Root** — Homepage is rendered from `src/pages/Homepage.{lang}.md` directly at `/`
 - 🧭 **Quick Links Custom Element** — Use `<d-quick-links>` and `<d-quick-link>` in Markdown to render rich home navigation cards
 - 🗂️ **API Catalog Well-Known** — Auto-generates `/.well-known/api-catalog` as Linkset JSON for machine-readable API discovery
@@ -369,6 +371,11 @@ export default {
     items: []
   },
 
+  markdownNegotiation: {
+    enabled: true,
+    agentFallback: true
+  },
+
   languages: [
     { image: '/images/flags/united-states-of-america.png', label: 'English (US)', value: 'en-US' },
     { image: '/images/flags/brazil.png', label: 'Português (BR)', value: 'pt-BR' }

package/bin/docsector.js

@@ -23,7 +23,7 @@ const packageRoot = resolve(__dirname, '..')
 const args = process.argv.slice(2)
 const command = args[0]
 
-const VERSION = '1.1.0'
+const VERSION = '1.2.1'
 
 const HELP = `
   Docsector Reader v${VERSION}
@@ -149,6 +149,12 @@ export default {
   //   items: ['/mcp']
   // },
 
+  // @ Markdown negotiation for agents (optional)
+  // markdownNegotiation: {
+  //   enabled: true,
+  //   agentFallback: true
+  // },
+
   // @ Languages
   languages: [
     {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@docsector/docsector-reader",
-  "version": "1.1.0",
+  "version": "1.2.1",
   "description": "A documentation rendering engine built with Vue 3, Quasar v2 and Vite. Transform Markdown into beautiful, navigable documentation sites.",
   "productName": "Docsector Reader",
   "author": "Rodrigo de Araujo Vieira",

package/src/index.js

@@ -54,6 +54,9 @@
  * @param {boolean} [config.apiCatalog.enabled=true] - Enables generation of API catalog artifact
  * @param {string} [config.apiCatalog.path='/.well-known/api-catalog'] - Output URI path for API catalog artifact
  * @param {Array<string|{href: string}>} [config.apiCatalog.items=[]] - Additional API endpoint links to include as item relations
+ * @param {Object} [config.markdownNegotiation] - Markdown content negotiation settings for agents
+ * @param {boolean} [config.markdownNegotiation.enabled=true] - Enables markdown negotiation by Accept header in production runtime
+ * @param {boolean} [config.markdownNegotiation.agentFallback=true] - Enables markdown fallback for known AI bot user agents when Accept is absent
  * @returns {Object} Resolved Docsector configuration
  */
 export function createDocsector (config = {}) {
@@ -109,6 +112,12 @@ export function createDocsector (config = {}) {
       path: '/.well-known/api-catalog',
       items: [],
       ...config.apiCatalog
+    },
+
+    markdownNegotiation: {
+      enabled: true,
+      agentFallback: true,
+      ...config.markdownNegotiation
     }
   }
 }

package/src/quasar.factory.js

@@ -457,6 +457,38 @@ function createMarkdownEndpointPlugin (projectRoot) {
     return null
   }
 
+  function resolveNegotiatedFile (urlPath, lang) {
+    const pathname = (urlPath || '').split('?')[0]
+
+    if (pathname === '/' || pathname === '/index.html') {
+      const homepage = resolve(pagesDir, `Homepage.${lang}.md`)
+      return existsSync(homepage) ? homepage : null
+    }
+
+    if (pathname.endsWith('.md')) {
+      return resolveMarkdownFile(pathname, lang)
+    }
+
+    let clean = pathname
+    if (clean.endsWith('/index.html')) clean = clean.slice(0, -11)
+    if (clean.endsWith('/')) clean = clean.slice(0, -1)
+
+    if (!clean) {
+      const homepage = resolve(pagesDir, `Homepage.${lang}.md`)
+      return existsSync(homepage) ? homepage : null
+    }
+
+    if (clean.endsWith('/overview') || clean.endsWith('/showcase') || clean.endsWith('/vs')) {
+      return resolveMarkdownFile(`${clean}.md`, lang)
+    }
+
+    return resolveMarkdownFile(`${clean}/overview.md`, lang)
+  }
+
+  function estimateMarkdownTokens (markdown = '') {
+    return Math.max(1, Math.ceil(markdown.length / 4))
+  }
+
   // LLM bot user-agent patterns
   const LLM_BOT_PATTERN = /GPTBot|ChatGPT-User|OAI-SearchBot|ClaudeBot|Claude-User|Claude-SearchBot|anthropic-ai|Google-Extended|Gemini-Deep-Research|PerplexityBot|Perplexity-User|Bytespider|CCBot|Meta-ExternalAgent|FacebookBot|Amazonbot|Applebot-Extended|cohere-ai|DuckAssistBot|GrokBot|AI2Bot|YouBot|PetalBot/i
 
@@ -466,6 +498,8 @@ function createMarkdownEndpointPlugin (projectRoot) {
     configureServer (server) {
       // Read default language from config
       let defaultLang = 'en-US'
+      let markdownNegotiationEnabled = true
+      let markdownAgentFallback = true
       try {
         const configPath = resolve(projectRoot, 'docsector.config.js')
         if (existsSync(configPath)) {
@@ -473,34 +507,56 @@ function createMarkdownEndpointPlugin (projectRoot) {
           const configContent = readFileSync(configPath, 'utf-8')
           const match = configContent.match(/defaultLanguage\s*:\s*['"]([^'"]+)['"]/)
           if (match) defaultLang = match[1]
+
+          const enabledMatch = configContent.match(/markdownNegotiation\s*:\s*\{[\s\S]*?enabled\s*:\s*(true|false)/)
+          if (enabledMatch) markdownNegotiationEnabled = enabledMatch[1] === 'true'
+
+          const fallbackMatch = configContent.match(/markdownNegotiation\s*:\s*\{[\s\S]*?agentFallback\s*:\s*(true|false)/)
+          if (fallbackMatch) markdownAgentFallback = fallbackMatch[1] === 'true'
         }
       } catch { /* use fallback */ }
 
       server.middlewares.use((req, res, next) => {
         const url = new URL(req.url, 'http://localhost')
+        const accept = (req.headers.accept || '').toLowerCase()
+        const wantsMarkdown = accept.includes('text/markdown')
+        const lang = url.searchParams.get('lang') || defaultLang
 
         // Explicit .md request
         if (url.pathname.endsWith('.md')) {
-          const lang = url.searchParams.get('lang') || defaultLang
           const file = resolveMarkdownFile(url.pathname, lang)
           if (!file) return next()
 
           const content = readFileSync(file, 'utf-8')
           res.setHeader('Content-Type', 'text/markdown; charset=utf-8')
+          res.setHeader('Vary', 'Accept')
+          res.setHeader('x-markdown-tokens', String(estimateMarkdownTokens(content)))
           res.end(content)
           return
         }
 
+        // Content negotiation for agents requesting markdown explicitly
+        if (markdownNegotiationEnabled && wantsMarkdown) {
+          const file = resolveNegotiatedFile(url.pathname, lang)
+          if (file) {
+            const content = readFileSync(file, 'utf-8')
+            res.setHeader('Content-Type', 'text/markdown; charset=utf-8')
+            res.setHeader('Vary', 'Accept')
+            res.setHeader('x-markdown-tokens', String(estimateMarkdownTokens(content)))
+            res.end(content)
+            return
+          }
+        }
+
         // Auto-serve markdown to LLM bot crawlers
         const ua = req.headers['user-agent'] || ''
-        if (LLM_BOT_PATTERN.test(ua)) {
-          const lang = url.searchParams.get('lang') || defaultLang
-          // Try appending /overview as the default subpage
-          const mdPath = url.pathname.replace(/\/$/, '') + '/overview.md'
-          const file = resolveMarkdownFile(mdPath, lang)
+        if (markdownAgentFallback && LLM_BOT_PATTERN.test(ua)) {
+          const file = resolveNegotiatedFile(url.pathname, lang)
           if (file) {
             const content = readFileSync(file, 'utf-8')
             res.setHeader('Content-Type', 'text/markdown; charset=utf-8')
+            res.setHeader('Vary', 'Accept')
+            res.setHeader('x-markdown-tokens', String(estimateMarkdownTokens(content)))
             res.end(content)
             return
           }
@@ -563,6 +619,25 @@ function createMarkdownBuildPlugin (projectRoot) {
         }
       }
 
+      // Generate homepage markdown files so root content can be negotiated in production
+      const languageValues = config.languages?.map(language => language.value).filter(Boolean) || []
+      const allLangs = [...new Set([defaultLang, ...languageValues])]
+      let homepageCount = 0
+      for (const lang of allLangs) {
+        const homepageSrc = resolve(pagesDir, `Homepage.${lang}.md`)
+        if (!existsSync(homepageSrc)) continue
+
+        const homepageContent = readFileSync(homepageSrc, 'utf-8')
+        writeFileSync(resolve(distDir, `homepage.${lang}.md`), homepageContent)
+        if (lang === defaultLang) {
+          writeFileSync(resolve(distDir, 'homepage.md'), homepageContent)
+        }
+        homepageCount++
+      }
+      if (homepageCount > 0) {
+        console.log(`\x1b[36m[docsector]\x1b[0m Generated ${homepageCount} homepage markdown file(s)`)
+      }
+
       console.log(`\x1b[36m[docsector]\x1b[0m Generated ${count} static .md files`)
 
       // Generate sitemap.xml if siteUrl is configured
@@ -663,6 +738,103 @@ function createMarkdownBuildPlugin (projectRoot) {
       } else {
         writeFileSync(headersPath, headersRule)
       }
+
+      const markdownNegotiationConfig = config.markdownNegotiation || {}
+      const markdownNegotiationEnabled = markdownNegotiationConfig.enabled !== false
+      const markdownAgentFallback = markdownNegotiationConfig.agentFallback !== false
+
+      if (markdownNegotiationEnabled) {
+        const functionsDir = resolve(projectRoot, 'functions')
+        mkdirSync(functionsDir, { recursive: true })
+
+        const middlewareCode = `const LLM_BOT_PATTERN = /GPTBot|ChatGPT-User|OAI-SearchBot|ClaudeBot|Claude-User|Claude-SearchBot|anthropic-ai|Google-Extended|Gemini-Deep-Research|PerplexityBot|Perplexity-User|Bytespider|CCBot|Meta-ExternalAgent|FacebookBot|Amazonbot|Applebot-Extended|cohere-ai|DuckAssistBot|GrokBot|AI2Bot|YouBot|PetalBot/i
+
+const DEFAULT_LANG = ${JSON.stringify(defaultLang)}
+const AGENT_FALLBACK = ${markdownAgentFallback ? 'true' : 'false'}
+
+function wantsMarkdown (request) {
+  const accept = (request.headers.get('accept') || '').toLowerCase()
+  return accept.includes('text/markdown')
+}
+
+function estimateTokens (markdown = '') {
+  return Math.max(1, Math.ceil(markdown.length / 4))
+}
+
+function shouldBypass (pathname) {
+  if (pathname === '/mcp' || pathname.startsWith('/mcp/')) return true
+  if (pathname.startsWith('/.well-known/')) return true
+  return /\\.(js|css|map|png|jpg|jpeg|gif|webp|svg|ico|woff2?|ttf|eot|xml|json|txt)$/i.test(pathname)
+}
+
+function resolveMarkdownPath (pathname, lang) {
+  if (!pathname) return null
+  if (pathname.endsWith('.md')) return pathname
+  if (pathname === '/' || pathname === '/index.html') return '/homepage.' + lang + '.md'
+
+  let clean = pathname
+  if (clean.endsWith('/index.html')) clean = clean.slice(0, -11)
+  if (clean.endsWith('/')) clean = clean.slice(0, -1)
+
+  if (!clean) return '/homepage.' + lang + '.md'
+  if (clean.endsWith('/overview') || clean.endsWith('/showcase') || clean.endsWith('/vs')) {
+    return clean + '.md'
+  }
+
+  return clean + '/overview.md'
+}
+
+export async function onRequest (context) {
+  const { request, env, next } = context
+
+  if (request.method !== 'GET' && request.method !== 'HEAD') {
+    return next()
+  }
+
+  const url = new URL(request.url)
+  if (shouldBypass(url.pathname)) {
+    return next()
+  }
+
+  const ua = request.headers.get('user-agent') || ''
+  const acceptMarkdown = wantsMarkdown(request)
+  const fallbackMarkdown = AGENT_FALLBACK && LLM_BOT_PATTERN.test(ua)
+  if (!acceptMarkdown && !fallbackMarkdown) {
+    return next()
+  }
+
+  const lang = url.searchParams.get('lang') || DEFAULT_LANG
+  const markdownPath = resolveMarkdownPath(url.pathname, lang)
+  if (!markdownPath) {
+    return next()
+  }
+
+  const markdownUrl = new URL(url.toString())
+  markdownUrl.pathname = markdownPath
+  const markdownRequest = new Request(markdownUrl.toString(), request)
+  const markdownResponse = await env.ASSETS.fetch(markdownRequest)
+  if (!markdownResponse.ok) {
+    return next()
+  }
+
+  const markdown = await markdownResponse.text()
+  const headers = new Headers(markdownResponse.headers)
+  headers.set('Content-Type', 'text/markdown; charset=utf-8')
+  const vary = headers.get('Vary')
+  headers.set('Vary', vary ? vary + ', Accept' : 'Accept')
+  headers.set('x-markdown-tokens', String(estimateTokens(markdown)))
+
+  if (request.method === 'HEAD') {
+    return new Response(null, { status: markdownResponse.status, headers })
+  }
+
+  return new Response(markdown, { status: markdownResponse.status, headers })
+}
+`
+
+        writeFileSync(resolve(functionsDir, '_middleware.js'), middlewareCode)
+        console.log(`\x1b[36m[docsector]\x1b[0m Generated markdown negotiation middleware at functions/_middleware.js`)
+      }
       console.log(`\x1b[36m[docsector]\x1b[0m Added _headers rule for .md files`)
 
       // Add homepage Link headers for agent discovery (RFC 8288 / RFC 9727)
@@ -795,12 +967,12 @@ function createMarkdownBuildPlugin (projectRoot) {
 
             const headersWithLinks = readFileSync(headersPath, 'utf-8')
             if (!headersWithLinks.includes(catalogHref)) {
-              const apiCatalogHeaders = `${catalogHref}\n  Content-Type: application/linkset+json; profile=\"https://www.rfc-editor.org/info/rfc9727\"\n`
+              const apiCatalogHeaders = `${catalogHref}\n  Content-Type: application/linkset+json; profile="https://www.rfc-editor.org/info/rfc9727"\n`
               writeFileSync(headersPath, headersWithLinks.trimEnd() + '\n\n' + apiCatalogHeaders)
               console.log(`\x1b[36m[docsector]\x1b[0m Added _headers rule for ${catalogHref}`)
             }
           } else {
-            console.warn(`\x1b[33m[docsector]\x1b[0m Skipped API catalog generation: path must be a local URI path, got \"${apiCatalogPath}\"`)
+            console.warn(`\x1b[33m[docsector]\x1b[0m Skipped API catalog generation: path must be a local URI path, got "${apiCatalogPath}"`)
           }
         }
       }
@@ -874,7 +1046,10 @@ function createMarkdownBuildPlugin (projectRoot) {
           writeFileSync(headersPath, currentHeaders.trimEnd() + '\n\n' + mcpHeaders)
         }
 
-        // Generate or merge _routes.json for Cloudflare Pages
+      }
+
+      // Generate or merge _routes.json for Cloudflare Pages functions
+      if (config.mcp || markdownNegotiationEnabled) {
         const routesPath = resolve(distDir, '_routes.json')
         let routes = { version: 1, include: [], exclude: [] }
         if (existsSync(routesPath)) {
@@ -884,11 +1059,46 @@ function createMarkdownBuildPlugin (projectRoot) {
             // empty
           }
         }
-        if (!routes.include.includes('/mcp')) {
+
+        if (markdownNegotiationEnabled && !routes.include.includes('/*')) {
+          routes.include.push('/*')
+        }
+
+        if (config.mcp && !markdownNegotiationEnabled && !routes.include.includes('/mcp')) {
           routes.include.push('/mcp')
         }
+
+        // Cloudflare Pages rejects overlapping include rules (e.g. "/mcp" with "/*").
+        // Keep only the catch-all when markdown negotiation is enabled.
+        if (routes.include.includes('/*')) {
+          routes.include = routes.include.filter((route) => route !== '/mcp')
+        }
+
+        const markdownExcludes = [
+          '/assets/*',
+          '/*.js',
+          '/*.css',
+          '/*.png',
+          '/*.jpg',
+          '/*.jpeg',
+          '/*.gif',
+          '/*.webp',
+          '/*.svg',
+          '/*.ico',
+          '/*.woff',
+          '/*.woff2',
+          '/*.ttf',
+          '/*.map'
+        ]
+
+        for (const excludePath of markdownExcludes) {
+          if (!routes.exclude.includes(excludePath)) {
+            routes.exclude.push(excludePath)
+          }
+        }
+
         writeFileSync(routesPath, JSON.stringify(routes, null, 2))
-        console.log(`\x1b[36m[docsector]\x1b[0m Added /mcp to _routes.json`)
+        console.log(`\x1b[36m[docsector]\x1b[0m Updated _routes.json for functions runtime`)
       }
     }
   }