@docsector/docsector-reader 1.2.4 → 1.3.1

package/README.md

@@ -28,6 +28,7 @@ Transform Markdown content into beautiful, navigable documentation sites — wit
 - 🤖 **LLM Bot Detection** — Automatically serves raw Markdown to known AI crawlers (GPTBot, ClaudeBot, PerplexityBot, GrokBot, and others)
 - 🗺️ **Sitemap Generation** — Automatic `sitemap.xml` generation at build time with all page URLs (requires `siteUrl` in config)
 - 🤖 **AI-Friendly robots.txt** — Scaffold includes a `robots.txt` explicitly allowing 23 AI crawlers (GPTBot, ClaudeBot, PerplexityBot, GrokBot, etc.)
+- 🧭 **Content Signals** — Optional `Content-Signal` directive for declaring AI usage policy (`ai-train`, `search`, `ai-input`) in `robots.txt`
 - 🔗 **Homepage Link Headers** — Auto-generated `Link` response headers for agent discovery (`api-catalog`, `service-doc`, `service-desc`, `describedby`) per RFC 8288 / RFC 9727
 - 🔌 **MCP Server** — Auto-generated [MCP](https://modelcontextprotocol.io) server at `/mcp` for AI assistant integration (Claude Desktop, VS Code, etc.)
 - 📄 **llms.txt / llms-full.txt** — Auto-generated [llms.txt](https://llmstxt.org) index and full-content file for LLM discovery (requires `siteUrl` in config)
@@ -46,10 +47,12 @@ Transform Markdown content into beautiful, navigable documentation sites — wit
 - 📱 **Responsive** — Mobile-friendly with collapsible sidebar and drawers
 - 🏷️ **Status Badges** — Mark pages as `done`, `draft`, or `empty` with visual indicators
 - ✏️ **Edit on GitHub** — Direct links to edit pages on your repository
+- 🧭 **Robust Edit Link Mapping** — Normalizes route paths (including trailing slashes) into `page.subpage.locale.md` source files for reliable GitHub edit URLs
 - 📅 **Last Updated Date** — Automatic per-page "last updated" date from git commit history, locale-formatted
 - 📊 **Translation Progress** — Automatic translation percentage based on header coverage
 - 🧠 **Markdown Negotiation** — Responds with Markdown when clients send `Accept: text/markdown`, while keeping HTML as browser default
 - 🔐 **Web Bot Auth** — Can publish a signed HTTP message signatures directory and includes helpers to sign outbound bot requests
+- 🧭 **Content Signals** — Injects `Content-Signal` policy in `robots.txt` with deterministic, idempotent build output
 - 🏠 **Markdown Home at Root** — Homepage is rendered from `src/pages/Homepage.{lang}.md` directly at `/`
 - 🧭 **Quick Links Custom Element** — Use `<d-quick-links>` and `<d-quick-link>` in Markdown to render rich home navigation cards
 - 🗂️ **API Catalog Well-Known** — Auto-generates `/.well-known/api-catalog` as Linkset JSON for machine-readable API discovery
@@ -280,6 +283,56 @@ Check `checks.discoverability.linkHeaders.status` equals `"pass"`.
 
 ---
 
+## 🧭 Content Signals
+
+Docsector Reader can declare AI usage preferences in `robots.txt` via `Content-Signal`.
+
+When enabled, build output ensures a deterministic directive format:
+
+- `Content-Signal: ai-train=..., search=..., ai-input=...`
+
+### Configure
+
+```javascript
+export default {
+  // ...other config
+
+  contentSignals: {
+    enabled: true,
+    aiTrain: 'yes',
+    search: 'yes',
+    aiInput: 'yes',
+    userAgent: '*',
+    applyToAllBlocks: false
+  }
+}
+```
+
+Notes:
+
+- `aiTrain`, `search`, and `aiInput` accept `yes` / `no` (or booleans).
+- Default scope is only `User-agent: *`.
+- Build patch is idempotent: repeated builds do not duplicate `Content-Signal` lines.
+
+### Validate
+
+```bash
+npx docsector build
+cat dist/spa/robots.txt
+```
+
+Optional external validation:
+
+```bash
+curl -X POST https://isitagentready.com/api/scan \
+  -H 'Content-Type: application/json' \
+  -d '{"url":"https://YOUR-SITE.com"}'
+```
+
+Check `checks.botAccessControl.contentSignals.status` equals `"pass"`.
+
+---
+
 ## �🚀 Quick Start
 
 ### 📦 Install
@@ -434,6 +487,15 @@ export default {
     agentFallback: true
   },
 
+  contentSignals: {
+    enabled: true,
+    aiTrain: 'yes',
+    search: 'yes',
+    aiInput: 'yes',
+    userAgent: '*',
+    applyToAllBlocks: false
+  },
+
   languages: [
     { image: '/images/flags/united-states-of-america.png', label: 'English (US)', value: 'en-US' },
     { image: '/images/flags/brazil.png', label: 'Português (BR)', value: 'pt-BR' }

package/bin/docsector.js

@@ -23,7 +23,7 @@ const packageRoot = resolve(__dirname, '..')
 const args = process.argv.slice(2)
 const command = args[0]
 
-const VERSION = '1.2.4'
+const VERSION = '1.3.1'
 
 const HELP = `
   Docsector Reader v${VERSION}
@@ -170,6 +170,17 @@ export default {
   //   signatureLabel: 'sig1'
   // },
 
+  // @ Content Signals (optional)
+  // Declares AI usage preferences in robots.txt.
+  // contentSignals: {
+  //   enabled: true,
+  //   aiTrain: 'yes',
+  //   search: 'yes',
+  //   aiInput: 'yes',
+  //   userAgent: '*',
+  //   applyToAllBlocks: false
+  // },
+
   // @ Languages
   languages: [
     {
@@ -763,6 +774,7 @@ const TEMPLATE_MARKDOWNLINT = `\
 const TEMPLATE_ROBOTS_TXT = `\
 User-agent: *
 Allow: /
+Content-Signal: ai-train=yes, search=yes, ai-input=yes
 
 # Explicitly allow AI crawlers
 # OpenAI

package/docsector.config.js

@@ -24,7 +24,7 @@ export default {
     discussions: 'https://github.com/docsector/docsector-reader/discussions',
     chat: null, // e.g., Discord/Slack invite URL
     email: null, // e.g., 'mailto:contact@example.com'
-    changelog: '/changelog',
+    changelog: 'https://github.com/docsector/docsector-reader/releases',
     roadmap: null, // e.g., external roadmap URL
     sponsor: null, // e.g., GitHub Sponsors URL
     explore: null // e.g., URL to explore related repos
@@ -58,5 +58,8 @@ export default {
   ],
 
   // @ Default language
-  defaultLanguage: 'en-US'
+  defaultLanguage: 'en-US',
+  contentSignals: {
+    enabled: true
+  }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@docsector/docsector-reader",
-  "version": "1.2.4",
+  "version": "1.3.1",
   "description": "A documentation rendering engine built with Vue 3, Quasar v2 and Vite. Transform Markdown into beautiful, navigable documentation sites.",
   "productName": "Docsector Reader",
   "author": "Rodrigo de Araujo Vieira",

package/src/components/DPageMeta.vue

@@ -12,11 +12,34 @@ const route = useRoute()
 const router = useRouter()
 const { t, locale, availableLocales, te, tm } = useI18n()
 
-const base = docsectorConfig.github?.editBaseUrl || ''
+function normalizeEditBaseUrl (url = '') {
+  const normalized = String(url).trim().replace(/\/+$/, '')
+  return normalized.replace(/(github\.com\/[^/]+\/[^/]+)\/(blob|tree)\//, '$1/edit/')
+}
+
+const base = normalizeEditBaseUrl(docsectorConfig.github?.editBaseUrl || '')
+
+function routePathToSourcePath (path = '') {
+  const cleanPath = String(path)
+    .replace(/\/index\.html$/, '')
+    .replace(/\/+$/, '')
+
+  if (cleanPath === '' || cleanPath === '/') {
+    return '/Homepage'
+  }
+
+  const segments = cleanPath.replace(/^\//, '').split('/').filter(Boolean)
+  if (segments.length < 2) {
+    return `/${segments.join('/')}`
+  }
+
+  const subpage = segments.pop()
+  return `/${segments.join('/')}.${subpage}`
+}
 
 const status = computed(() => route.meta.status)
 const URL = computed(() => {
-  const path = route.path.replace(/\/([^/]*)$/, '.$1')
+  const path = routePathToSourcePath(route.path)
   return `${base}${path}.${locale.value}.md`
 })
 const color = computed(() => {

package/src/index.js

@@ -66,6 +66,13 @@
  * @param {string|null} [config.webBotAuth.keyId=null] - Optional static fallback key identifier when env var is absent
  * @param {number} [config.webBotAuth.signatureMaxAge=300] - Signature validity window in seconds for directory responses
  * @param {string} [config.webBotAuth.signatureLabel='sig1'] - Signature label used in Signature and Signature-Input headers
+ * @param {Object} [config.contentSignals] - Content Signals policy for robots.txt
+ * @param {boolean} [config.contentSignals.enabled=false] - Enables Content-Signal injection in robots.txt during build
+ * @param {'yes'|'no'|boolean} [config.contentSignals.aiTrain='yes'] - Permission for AI model training consumption
+ * @param {'yes'|'no'|boolean} [config.contentSignals.search='yes'] - Permission for AI search indexing/discovery consumption
+ * @param {'yes'|'no'|boolean} [config.contentSignals.aiInput='yes'] - Permission for AI input/inference-time consumption
+ * @param {string} [config.contentSignals.userAgent='*'] - Target User-agent block for directive injection
+ * @param {boolean} [config.contentSignals.applyToAllBlocks=false] - When true, applies directive to every User-agent block
  * @returns {Object} Resolved Docsector configuration
  */
 export function createDocsector (config = {}) {
@@ -139,6 +146,16 @@ export function createDocsector (config = {}) {
       signatureMaxAge: 300,
       signatureLabel: 'sig1',
       ...config.webBotAuth
+    },
+
+    contentSignals: {
+      enabled: false,
+      aiTrain: 'yes',
+      search: 'yes',
+      aiInput: 'yes',
+      userAgent: '*',
+      applyToAllBlocks: false,
+      ...config.contentSignals
     }
   }
 }

package/src/quasar.factory.js

@@ -753,6 +753,8 @@ function createMarkdownBuildPlugin (projectRoot) {
         ? Math.max(30, Number(webBotAuthConfig.signatureMaxAge))
         : 300
       const webBotAuthSignatureLabel = webBotAuthConfig.signatureLabel || 'sig1'
+      const contentSignalsConfig = config.contentSignals || {}
+      const contentSignalsEnabled = contentSignalsConfig.enabled === true
 
       if (markdownNegotiationEnabled || webBotAuthEnabled) {
         const functionsDir = resolve(projectRoot, 'functions')
@@ -1001,6 +1003,25 @@ export async function onRequest (context) {
         writeFileSync(resolve(functionsDir, '_middleware.js'), middlewareCode)
         console.log(`\x1b[36m[docsector]\x1b[0m Generated runtime middleware at functions/_middleware.js`)
       }
+
+      if (contentSignalsEnabled) {
+        const robotsPath = resolve(distDir, 'robots.txt')
+        const existingRobots = existsSync(robotsPath)
+          ? readFileSync(robotsPath, 'utf-8')
+          : 'User-agent: *\nAllow: /\n'
+        const contentSignalLine = buildContentSignalLine(contentSignalsConfig)
+        const patchedRobots = applyContentSignalsToRobots(existingRobots, {
+          contentSignalLine,
+          userAgent: contentSignalsConfig.userAgent || '*',
+          applyToAllBlocks: contentSignalsConfig.applyToAllBlocks === true
+        })
+
+        if (patchedRobots !== existingRobots) {
+          writeFileSync(robotsPath, patchedRobots)
+          console.log(`\x1b[36m[docsector]\x1b[0m Updated robots.txt with Content-Signal policy`)
+        }
+      }
+
       console.log(`\x1b[36m[docsector]\x1b[0m Added _headers rule for .md files`)
 
       // Add homepage Link headers for agent discovery (RFC 8288 / RFC 9727)
@@ -1274,6 +1295,87 @@ export async function onRequest (context) {
   }
 }
 
+function normalizeContentSignalValue (value, fallback = 'yes') {
+  if (typeof value === 'boolean') return value ? 'yes' : 'no'
+  if (typeof value === 'string') {
+    const normalized = value.trim().toLowerCase()
+    if (normalized === 'yes' || normalized === 'no') return normalized
+  }
+  return fallback
+}
+
+function buildContentSignalLine (contentSignalsConfig = {}) {
+  const aiTrain = normalizeContentSignalValue(contentSignalsConfig.aiTrain, 'yes')
+  const search = normalizeContentSignalValue(contentSignalsConfig.search, 'yes')
+  const aiInput = normalizeContentSignalValue(contentSignalsConfig.aiInput, 'yes')
+  return `Content-Signal: ai-train=${aiTrain}, search=${search}, ai-input=${aiInput}`
+}
+
+function ensureContentSignalInBlock (blockLines, contentSignalLine) {
+  const cleanedBlock = blockLines.filter((line, index) => {
+    if (index === 0) return true
+    return !/^\s*Content-Signal\s*:/i.test(line)
+  })
+
+  let insertAt = cleanedBlock.findIndex(line => /^\s*Allow\s*:/i.test(line))
+  if (insertAt === -1) {
+    insertAt = 0
+  }
+
+  cleanedBlock.splice(insertAt + 1, 0, contentSignalLine)
+  return cleanedBlock
+}
+
+function applyContentSignalsToRobots (robotsContent, { contentSignalLine, userAgent = '*', applyToAllBlocks = false }) {
+  const input = typeof robotsContent === 'string' ? robotsContent : ''
+  const lines = input.replace(/\r\n/g, '\n').split('\n')
+  const userAgentIndexes = []
+
+  for (let i = 0; i < lines.length; i++) {
+    if (/^\s*User-agent\s*:/i.test(lines[i])) {
+      userAgentIndexes.push(i)
+    }
+  }
+
+  if (userAgentIndexes.length === 0) {
+    const scaffold = [
+      `User-agent: ${userAgent}`,
+      'Allow: /',
+      contentSignalLine,
+      ''
+    ].join('\n')
+    return scaffold.endsWith('\n') ? scaffold : scaffold + '\n'
+  }
+
+  const targetIndexes = []
+  for (const startIndex of userAgentIndexes) {
+    const uaValue = lines[startIndex].split(':').slice(1).join(':').trim()
+    if (applyToAllBlocks || uaValue.toLowerCase() === String(userAgent).toLowerCase()) {
+      targetIndexes.push(startIndex)
+    }
+  }
+
+  if (targetIndexes.length === 0) {
+    targetIndexes.push(userAgentIndexes[0])
+  }
+
+  let updated = []
+  let cursor = 0
+
+  for (const startIndex of targetIndexes) {
+    const nextUserAgent = userAgentIndexes.find(index => index > startIndex)
+    const endIndex = nextUserAgent === undefined ? lines.length : nextUserAgent
+
+    updated = updated.concat(lines.slice(cursor, startIndex))
+    const currentBlock = lines.slice(startIndex, endIndex)
+    updated = updated.concat(ensureContentSignalInBlock(currentBlock, contentSignalLine))
+    cursor = endIndex
+  }
+
+  updated = updated.concat(lines.slice(cursor))
+  return updated.join('\n').replace(/\n+$/g, '') + '\n'
+}
+
 /**
  * Create a complete Quasar configuration for a docsector-reader consumer project.
  *