@docsector/docsector-reader 0.8.1 → 0.8.4

package/bin/docsector.js

@@ -23,7 +23,7 @@ const packageRoot = resolve(__dirname, '..')
 const args = process.argv.slice(2)
 const command = args[0]
 
-const VERSION = '0.8.1'
+const VERSION = '0.8.4'
 
 const HELP = `
   Docsector Reader v${VERSION}
@@ -686,6 +686,96 @@ npm-debug.log*
 .thumbs.db
 `
 
+const TEMPLATE_ROBOTS_TXT = `\
+User-agent: *
+Allow: /
+
+# Explicitly allow AI crawlers
+# OpenAI
+User-agent: GPTBot
+Allow: /
+
+User-agent: ChatGPT-User
+Allow: /
+
+User-agent: OAI-SearchBot
+Allow: /
+
+# Anthropic
+User-agent: ClaudeBot
+Allow: /
+
+User-agent: Claude-User
+Allow: /
+
+User-agent: Claude-SearchBot
+Allow: /
+
+User-agent: anthropic-ai
+Allow: /
+
+# Google
+User-agent: Google-Extended
+Allow: /
+
+User-agent: Gemini-Deep-Research
+Allow: /
+
+# Perplexity
+User-agent: PerplexityBot
+Allow: /
+
+User-agent: Perplexity-User
+Allow: /
+
+# ByteDance
+User-agent: Bytespider
+Allow: /
+
+# Common Crawl
+User-agent: CCBot
+Allow: /
+
+# Meta
+User-agent: Meta-ExternalAgent
+Allow: /
+
+User-agent: FacebookBot
+Allow: /
+
+# Amazon
+User-agent: Amazonbot
+Allow: /
+
+# Apple
+User-agent: Applebot-Extended
+Allow: /
+
+# Cohere
+User-agent: cohere-ai
+Allow: /
+
+# DuckDuckGo
+User-agent: DuckAssistBot
+Allow: /
+
+# xAI
+User-agent: GrokBot
+Allow: /
+
+# Allen AI
+User-agent: AI2Bot
+Allow: /
+
+# You.com
+User-agent: YouBot
+Allow: /
+
+# Huawei
+User-agent: PetalBot
+Allow: /
+`
+
 const TEMPLATE_GETTING_STARTED_MD = `\
 ## Installation
 
@@ -840,6 +930,7 @@ function initProject (name) {
     ['index.html', TEMPLATE_INDEX_HTML],
     ['postcss.config.cjs', TEMPLATE_POSTCSS],
     ['.gitignore', TEMPLATE_GITIGNORE],
+    ['public/robots.txt', TEMPLATE_ROBOTS_TXT],
     ['src/css/app.sass', TEMPLATE_CSS_STUB],
     ['src/i18n/index.js', TEMPLATE_I18N_INDEX],
     ['src/i18n/languages/en-US.hjson', TEMPLATE_I18N_HJSON],
@@ -869,6 +960,7 @@ function initProject (name) {
   console.log('    ├── postcss.config.cjs')
   console.log('    ├── .gitignore')
   console.log('    ├── public/')
+  console.log('    │   ├── robots.txt')
   console.log('    │   └── images/')
   console.log('    │       └── logo.png')
   console.log('    └── src/')

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@docsector/docsector-reader",
-  "version": "0.8.1",
+  "version": "0.8.4",
   "description": "A documentation rendering engine built with Vue 3, Quasar v2 and Vite. Transform Markdown into beautiful, navigable documentation sites.",
   "productName": "Docsector Reader",
   "author": "Rodrigo de Araujo Vieira",

package/src/quasar.factory.js

@@ -360,7 +360,7 @@ function createMarkdownEndpointPlugin (projectRoot) {
           if (!file) return next()
 
           const content = readFileSync(file, 'utf-8')
-          res.setHeader('Content-Type', 'text/plain; charset=utf-8')
+          res.setHeader('Content-Type', 'text/markdown; charset=utf-8')
           res.end(content)
           return
         }
@@ -374,7 +374,7 @@ function createMarkdownEndpointPlugin (projectRoot) {
           const file = resolveMarkdownFile(mdPath, lang)
           if (file) {
             const content = readFileSync(file, 'utf-8')
-            res.setHeader('Content-Type', 'text/plain; charset=utf-8')
+            res.setHeader('Content-Type', 'text/markdown; charset=utf-8')
             res.end(content)
             return
           }
@@ -438,6 +438,49 @@ function createMarkdownBuildPlugin (projectRoot) {
       }
 
       console.log(`\x1b[36m[docsector]\x1b[0m Generated ${count} static .md files`)
+
+      // Generate sitemap.xml if siteUrl is configured
+      const siteUrl = (config.siteUrl || '').replace(/\/+$/, '')
+      if (siteUrl) {
+        const today = new Date().toISOString().split('T')[0]
+        let urls = ''
+
+        for (const [pagePath, page] of Object.entries(pages)) {
+          if (page.config === null) continue
+          if (page.config.status === 'empty') continue
+
+          const type = page.config.type ?? 'manual'
+
+          const subpages = ['overview']
+          if (page.config.subpages?.showcase) subpages.push('showcase')
+          if (page.config.subpages?.vs) subpages.push('vs')
+
+          for (const subpage of subpages) {
+            const srcFile = resolve(pagesDir, `${type}${pagePath}.${subpage}.${defaultLang}.md`)
+            if (!existsSync(srcFile)) continue
+
+            const routePath = `/${type}${pagePath}/${subpage}`
+            urls += `  <url>\n    <loc>${siteUrl}${routePath}</loc>\n    <lastmod>${today}</lastmod>\n  </url>\n`
+          }
+        }
+
+        const sitemap = `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${urls}</urlset>\n`
+        writeFileSync(resolve(distDir, 'sitemap.xml'), sitemap)
+        console.log(`\x1b[36m[docsector]\x1b[0m Generated sitemap.xml`)
+      }
+
+      // Generate _headers file for Cloudflare Pages (append if exists)
+      const headersPath = resolve(distDir, '_headers')
+      const headersRule = '/*.md\n  Content-Type: text/markdown; charset=utf-8\n'
+      if (existsSync(headersPath)) {
+        const existing = readFileSync(headersPath, 'utf-8')
+        if (!existing.includes('*.md')) {
+          writeFileSync(headersPath, existing.trimEnd() + '\n\n' + headersRule)
+        }
+      } else {
+        writeFileSync(headersPath, headersRule)
+      }
+      console.log(`\x1b[36m[docsector]\x1b[0m Added _headers rule for .md files`)
     }
   }
 }