@docsector/docsector-reader 4.3.3 → 4.4.0

package/src/pages/manual/basic/ai-assistant.overview.pt-BR.md

@@ -0,0 +1,69 @@
+## Visão Geral
+
+O AI Assistant adiciona um painel opcional de chat às páginas do Docsector. Ele foi desenhado para fluxos de RAG semântico na Cloudflare, mantendo a integração do navegador simples: usuários abrem um drawer pelo header global, e a Cloudflare Pages Function conversa com o provedor de IA configurado.
+
+O primeiro provedor é o Cloudflare AI Search. Ele pode rastrear o sitemap Markdown gerado pelo Docsector, recuperar trechos relevantes com busca híbrida e transmitir uma resposta com trechos de origem que o painel mostra como citações.
+
+## O Que Ele Adiciona
+
+- Um drawer lateral direito no desktop.
+- Um diálogo em tela cheia no mobile.
+- Prompts sugeridos, contexto da página atual, respostas em streaming e links de fontes.
+- Artefatos de build para AI Search quando `siteUrl` está configurado.
+- Um endpoint interno same-origin para manter credenciais no servidor.
+
+## Habilitar
+
+```javascript
+export default {
+  siteUrl: 'https://docs.example.com',
+
+  aiAssistant: {
+    enabled: true,
+    provider: 'aiSearch',
+    endpoint: '/assistant',
+    aiSearch: {
+      binding: 'AI_SEARCH',
+      instanceNameEnv: 'AI_SEARCH_INSTANCE_NAME',
+      accountIdEnv: 'CLOUDFLARE_ACCOUNT_ID',
+      apiTokenEnv: 'CLOUDFLARE_API_TOKEN',
+      retrievalType: 'hybrid',
+      maxResults: 6,
+      stream: true
+    }
+  }
+}
+```
+
+Defina `AI_SEARCH_INSTANCE_NAME` nas variáveis de ambiente do Cloudflare Pages em deploy, ou em `.dev.vars` quando usar `wrangler pages dev` localmente.
+
+## Cloudflare AI Search
+
+Crie uma instância do AI Search e configure uma fonte de dados Website. O Docsector sempre publica `/sitemap.xml` no build e anuncia esse arquivo em `robots.txt`, então o crawler da Cloudflare consegue descobrir o site automaticamente.
+
+Para uma recuperação mais limpa, aponte a configuração de sitemap específico para:
+
+```text
+https://docs.example.com/ai-search-sitemap.xml
+```
+
+O sitemap do AI Search aponta para URLs Markdown, que são mais limpas para recuperação do que HTML renderizado pela SPA. O manifest em `/.well-known/ai-search/manifest.json` lista títulos, rotas, locales, books, versões e subpáginas do mesmo conjunto de fontes.
+
+## Endpoint Runtime
+
+A Pages Function gerada aceita mensagens de chat, metadados da rota atual, locale e texto selecionado opcional. Ela encaminha a solicitação ao AI Search por binding quando disponível, ou por REST usando variáveis de ambiente criptografadas da Cloudflare. O endpoint é uma API interna do drawer, não uma página para o usuário navegar.
+
+O navegador nunca precisa de um token da API Cloudflare.
+
+## Validar
+
+```bash
+npx docsector build
+cat dist/spa/sitemap.xml
+cat dist/spa/robots.txt
+cat dist/spa/ai-search-sitemap.xml
+cat dist/spa/.well-known/ai-search/manifest.json
+npx wrangler pages dev dist/spa
+```
+
+Bindings remotos de AI Search e Workers AI podem gerar uso cobrado pela Cloudflare durante o desenvolvimento local.

package/src/pages/manual.index.js

@@ -265,6 +265,35 @@ export default {
     }
   },
 
+  '/basic/ai-assistant': {
+    config: {
+      icon: 'auto_awesome',
+      status: 'new',
+      version: 'v4.4.0',
+      meta: {
+        description: {
+          'en-US': 'AI Assistant — Documentation of Docsector Reader',
+          'pt-BR': 'Assistente de IA — Documentacao do Docsector Reader'
+        }
+      },
+      book: 'manual',
+      menu: {},
+      subpages: {
+        showcase: false
+      }
+    },
+    data: {
+      'en-US': { title: 'AI Assistant' },
+      'pt-BR': { title: 'Assistente de IA' }
+    },
+    metadata: {
+      tags: {
+        'en-US': 'ai assistant cloudflare ai search rag drawer semantic search citations streaming',
+        'pt-BR': 'ia assistente cloudflare ai search rag drawer busca semântica citações streaming'
+      }
+    }
+  },
+
   '/basic/agent-skills': {
     config: {
       icon: 'psychology',

package/src/quasar.factory.js

@@ -30,6 +30,11 @@ import { resolve } from 'path'
 import { pathToFileURL } from 'url'
 import HJSON from 'hjson'
 
+import { normalizeAiAssistantConfig } from './ai-assistant/config.js'
+import { createAiSearchIndexArtifacts } from './ai-assistant/indexing.js'
+import { MARKDOWN_AGENT_USER_AGENT_SOURCE, matchesMarkdownAgentUserAgent } from './markdown-agent.js'
+import { appendSitemapsToRobots, createSitemap } from './sitemap.js'
+
 /**
  * No-op configure wrapper.
  * Quasar's `configure` from `quasar/wrappers` is a TypeScript identity function.
@@ -1751,9 +1756,6 @@ function createMarkdownEndpointPlugin (projectRoot) {
     return Math.max(1, Math.ceil(markdown.length / 4))
   }
 
-  // LLM bot user-agent patterns
-  const LLM_BOT_PATTERN = /GPTBot|ChatGPT-User|OAI-SearchBot|ClaudeBot|Claude-User|Claude-SearchBot|anthropic-ai|Google-Extended|Gemini-Deep-Research|PerplexityBot|Perplexity-User|Bytespider|CCBot|Meta-ExternalAgent|FacebookBot|Amazonbot|Applebot-Extended|cohere-ai|DuckAssistBot|GrokBot|AI2Bot|YouBot|PetalBot/i
-
   return {
     name: 'docsector-markdown-endpoint',
 
@@ -1806,7 +1808,7 @@ function createMarkdownEndpointPlugin (projectRoot) {
         }
 
         if (homepagePath && typeof remoteHomepage === 'string' && remoteHomepage.length > 0) {
-          if ((markdownNegotiationEnabled && wantsMarkdown) || (markdownAgentFallback && LLM_BOT_PATTERN.test(req.headers['user-agent'] || ''))) {
+          if ((markdownNegotiationEnabled && wantsMarkdown) || (markdownAgentFallback && matchesMarkdownAgentUserAgent(req.headers['user-agent'] || ''))) {
             res.setHeader('Content-Type', 'text/markdown; charset=utf-8')
             res.setHeader('Vary', 'Accept')
             res.setHeader('x-markdown-tokens', String(estimateMarkdownTokens(remoteHomepage)))
@@ -1843,7 +1845,7 @@ function createMarkdownEndpointPlugin (projectRoot) {
 
         // Auto-serve markdown to LLM bot crawlers
         const ua = req.headers['user-agent'] || ''
-        if (markdownAgentFallback && LLM_BOT_PATTERN.test(ua)) {
+        if (markdownAgentFallback && matchesMarkdownAgentUserAgent(ua)) {
           const file = resolveNegotiatedFile(url.pathname, lang)
           if (file) {
             const content = readFileSync(file, 'utf-8')
@@ -1863,6 +1865,74 @@ function createMarkdownEndpointPlugin (projectRoot) {
   }
 }
 
+function collectAiSearchIndexEntries ({ pagesDir, pageEntries = [], defaultLang = 'en-US' } = {}) {
+  const entries = []
+
+  for (const entry of pageEntries) {
+    const { book, pagePath, page } = entry
+    if (page?.config === null) continue
+    if (page?.config?.status === 'empty') continue
+
+    const title = page?.data?.['*']?.title
+      || page?.data?.[defaultLang]?.title
+      || page?.data?.['en-US']?.title
+      || pagePath?.split('/').pop()
+      || pagePath
+
+    const subpages = ['overview']
+    if (page?.config?.subpages?.showcase) subpages.push('showcase')
+    if (page?.config?.subpages?.vs) subpages.push('vs')
+
+    for (const subpage of subpages) {
+      const srcFile = resolveMarkdownSourceFile(pagesDir, entry, subpage, defaultLang)
+      if (!existsSync(srcFile)) continue
+
+      const routePath = buildPageRoutePath(entry, subpage)
+      entries.push({
+        title,
+        path: routePath,
+        markdownPath: `${routePath}.md`,
+        locale: defaultLang,
+        book,
+        version: entry.version,
+        subpage
+      })
+    }
+  }
+
+  return entries
+}
+
+function collectStandardSitemapEntries ({ pagesDir, pageEntries = [], defaultLang = 'en-US' } = {}) {
+  const entries = [
+    { path: '/', priority: '1.0' }
+  ]
+  const seenPaths = new Set(['/'])
+
+  for (const entry of pageEntries) {
+    const { page } = entry
+    if (page?.config === null) continue
+    if (page?.config?.status === 'empty') continue
+
+    const subpages = ['overview']
+    if (page?.config?.subpages?.showcase) subpages.push('showcase')
+    if (page?.config?.subpages?.vs) subpages.push('vs')
+
+    for (const subpage of subpages) {
+      const srcFile = resolveMarkdownSourceFile(pagesDir, entry, subpage, defaultLang)
+      if (!existsSync(srcFile)) continue
+
+      const routePath = buildPageRoutePath(entry, subpage, { leadingSlash: true })
+      if (seenPaths.has(routePath)) continue
+
+      seenPaths.add(routePath)
+      entries.push({ path: routePath })
+    }
+  }
+
+  return entries
+}
+
 /**
  * Create a Vite plugin that generates static `.md` files at build time.
  *
@@ -1883,6 +1953,7 @@ function createMarkdownBuildPlugin (projectRoot) {
 
       const { default: config } = await import(configUrl)
       const { pageEntries } = await loadBooksRegistry(projectRoot)
+      const assistantConfig = normalizeAiAssistantConfig(config)
 
       const defaultLang = config.defaultLanguage || config.languages?.[0]?.value || 'en-US'
       let count = 0
@@ -1929,34 +2000,23 @@ function createMarkdownBuildPlugin (projectRoot) {
 
       console.log(`\x1b[36m[docsector]\x1b[0m Generated ${count} static .md files`)
 
-      // Generate sitemap.xml if siteUrl is configured
       const siteUrl = (config.siteUrl || '').replace(/\/+$/, '')
-      if (siteUrl) {
-        const today = new Date().toISOString().split('T')[0]
-        let urls = ''
-
-        for (const entry of pageEntries) {
-          const { page } = entry
-          if (page.config === null) continue
-          if (page.config.status === 'empty') continue
-
-          const subpages = ['overview']
-          if (page.config.subpages?.showcase) subpages.push('showcase')
-          if (page.config.subpages?.vs) subpages.push('vs')
-
-          for (const subpage of subpages) {
-            const srcFile = resolveMarkdownSourceFile(pagesDir, entry, subpage, defaultLang)
-            if (!existsSync(srcFile)) continue
-
-            const routePath = buildPageRoutePath(entry, subpage, { leadingSlash: true })
-            urls += `  <url>\n    <loc>${siteUrl}${routePath}</loc>\n    <lastmod>${today}</lastmod>\n  </url>\n`
-          }
-        }
-
-        const sitemap = `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${urls}</urlset>\n`
+      const generatedAt = new Date().toISOString()
+      const sitemapConfig = config.sitemap || {}
+      const sitemapEnabled = sitemapConfig.enabled !== false
+
+      if (sitemapEnabled) {
+        const sitemapEntries = collectStandardSitemapEntries({ pagesDir, pageEntries, defaultLang })
+        const sitemap = createSitemap({
+          entries: sitemapEntries,
+          generatedAt,
+          siteUrl
+        })
         writeFileSync(resolve(distDir, 'sitemap.xml'), sitemap)
-        console.log(`\x1b[36m[docsector]\x1b[0m Generated sitemap.xml`)
+        console.log(`\x1b[36m[docsector]\x1b[0m Generated sitemap.xml (${sitemapEntries.length} URLs)`)
+      }
 
+      if (siteUrl) {
         // Generate llms.txt and llms-full.txt (LLM-friendly page index and full content)
         const brandingName = config.branding?.name || 'Documentation'
         const brandingVersion = config.branding?.version || ''
@@ -2047,6 +2107,8 @@ function createMarkdownBuildPlugin (projectRoot) {
       const agentSkillsEnabled = agentSkillsConfig.enabled === true
       const mcpServerCardConfig = config.mcpServerCard || {}
       const mcpServerCardEnabled = mcpServerCardConfig.enabled === true
+      const aiAssistantEnabled = assistantConfig.enabled === true
+      let aiSearchSitemapGenerated = false
 
       const toUrl = (href) => {
         if (!href) return null
@@ -2061,11 +2123,57 @@ function createMarkdownBuildPlugin (projectRoot) {
         return path || null
       }
 
+      if (aiAssistantEnabled && assistantConfig.provider === 'aiSearch') {
+        const aiSearchEntries = collectAiSearchIndexEntries({
+          pagesDir,
+          pageEntries,
+          defaultLang
+        })
+        const artifacts = createAiSearchIndexArtifacts({
+          siteUrl,
+          entries: aiSearchEntries
+        })
+        const manifestPath = '.well-known/ai-search/manifest.json'
+        const sitemapPath = 'ai-search-sitemap.xml'
+
+        mkdirSync(resolve(distDir, '.well-known', 'ai-search'), { recursive: true })
+        writeFileSync(resolve(distDir, manifestPath), JSON.stringify(artifacts.manifest, null, 2) + '\n')
+        if (artifacts.sitemap) {
+          writeFileSync(resolve(distDir, sitemapPath), artifacts.sitemap)
+          aiSearchSitemapGenerated = true
+        }
+        console.log(`\x1b[36m[docsector]\x1b[0m Generated AI Search index artifacts (${aiSearchEntries.length} pages)`)
+
+        const headersWithAiSearch = readFileSync(headersPath, 'utf-8')
+        const aiSearchHeaders = [
+          '/.well-known/ai-search/manifest.json\n  Content-Type: application/json; charset=utf-8',
+          '/ai-search-sitemap.xml\n  Content-Type: application/xml; charset=utf-8'
+        ].filter(rule => !headersWithAiSearch.includes(rule.split('\n')[0])).join('\n\n')
+        if (aiSearchHeaders) {
+          writeFileSync(headersPath, headersWithAiSearch.trimEnd() + '\n\n' + aiSearchHeaders + '\n')
+        }
+      }
+
+      if (aiAssistantEnabled) {
+        const functionsDir = resolve(projectRoot, 'functions')
+        const packageRoot = getPackageRoot(projectRoot)
+        const templatePath = resolve(packageRoot, 'src', 'ai-assistant', 'server.js')
+        mkdirSync(functionsDir, { recursive: true })
+
+        if (existsSync(templatePath)) {
+          const serverCode = readFileSync(templatePath, 'utf-8')
+            .replaceAll('__AI_ASSISTANT_CONFIG__', JSON.stringify(assistantConfig, null, 2))
+
+          writeFileSync(resolve(functionsDir, 'assistant.js'), serverCode)
+          console.log(`\x1b[36m[docsector]\x1b[0m Generated AI Assistant endpoint at functions/assistant.js`)
+        }
+      }
+
       if (markdownNegotiationEnabled || webBotAuthEnabled) {
         const functionsDir = resolve(projectRoot, 'functions')
         mkdirSync(functionsDir, { recursive: true })
 
-        const middlewareCode = `const LLM_BOT_PATTERN = /GPTBot|ChatGPT-User|OAI-SearchBot|ClaudeBot|Claude-User|Claude-SearchBot|anthropic-ai|Google-Extended|Gemini-Deep-Research|PerplexityBot|Perplexity-User|Bytespider|CCBot|Meta-ExternalAgent|FacebookBot|Amazonbot|Applebot-Extended|cohere-ai|DuckAssistBot|GrokBot|AI2Bot|YouBot|PetalBot/i
+        const middlewareCode = `const LLM_BOT_PATTERN = new RegExp(${JSON.stringify(MARKDOWN_AGENT_USER_AGENT_SOURCE)}, 'i')
 
 const DEFAULT_LANG = ${JSON.stringify(defaultLang)}
 const MARKDOWN_ENABLED = ${markdownNegotiationEnabled ? 'true' : 'false'}
@@ -2227,6 +2335,7 @@ function estimateTokens (markdown = '') {
 
 function shouldBypass (pathname) {
   if (pathname === '/mcp' || pathname.startsWith('/mcp/')) return true
+  if (pathname === '/assistant' || pathname.startsWith('/assistant/')) return true
   if (pathname.startsWith('/.well-known/')) return true
   return /\\.(js|css|map|png|jpg|jpeg|gif|webp|svg|ico|woff2?|ttf|eot|xml|json|txt)$/i.test(pathname)
 }
@@ -2327,6 +2436,26 @@ export async function onRequest (context) {
         }
       }
 
+      const robotsSitemapPaths = []
+      if (sitemapEnabled) robotsSitemapPaths.push('/sitemap.xml')
+      if (aiSearchSitemapGenerated) robotsSitemapPaths.push('/ai-search-sitemap.xml')
+
+      if (robotsSitemapPaths.length > 0) {
+        const robotsPath = resolve(distDir, 'robots.txt')
+        const existingRobots = existsSync(robotsPath)
+          ? readFileSync(robotsPath, 'utf-8')
+          : 'User-agent: *\nAllow: /\n'
+        const patchedRobots = appendSitemapsToRobots(existingRobots, {
+          sitemaps: robotsSitemapPaths,
+          siteUrl
+        })
+
+        if (patchedRobots !== existingRobots) {
+          writeFileSync(robotsPath, patchedRobots)
+          console.log(`\x1b[36m[docsector]\x1b[0m Updated robots.txt with sitemap discovery`)
+        }
+      }
+
       if (agentSkillsEnabled) {
         const agentSkillsPath = agentSkillsConfig.path || '/.well-known/agent-skills/index.json'
         const agentSkillsSchema = agentSkillsConfig.schema || 'https://schemas.agentskills.io/discovery/0.2.0/schema.json'
@@ -2653,7 +2782,7 @@ export async function onRequest (context) {
       }
 
       // Generate or merge _routes.json for Cloudflare Pages functions
-      if (config.mcp || markdownNegotiationEnabled || webBotAuthEnabled) {
+      if (config.mcp || aiAssistantEnabled || markdownNegotiationEnabled || webBotAuthEnabled) {
         const routesPath = resolve(distDir, '_routes.json')
         let routes = { version: 1, include: [], exclude: [] }
         if (existsSync(routesPath)) {
@@ -2672,6 +2801,10 @@ export async function onRequest (context) {
           routes.include.push('/mcp')
         }
 
+        if (aiAssistantEnabled && !markdownNegotiationEnabled && !routes.include.includes('/assistant')) {
+          routes.include.push('/assistant')
+        }
+
         if (webBotAuthEnabled && !markdownNegotiationEnabled && !routes.include.includes(webBotAuthDirectoryPath)) {
           routes.include.push(webBotAuthDirectoryPath)
         }
@@ -3144,7 +3277,7 @@ export function createQuasarConfig (options = {}) {
       config: {},
       lang: 'en-US',
       plugins: [
-        'Meta', 'LocalStorage', 'SessionStorage'
+        'Meta', 'Notify', 'LocalStorage', 'SessionStorage'
       ]
     },
 

package/src/sitemap.js

@@ -0,0 +1,103 @@
+function escapeXml (value) {
+  return String(value || '')
+    .replace(/&/g, '&')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;')
+    .replace(/'/g, '&apos;')
+}
+
+function normalizeBaseUrl (siteUrl) {
+  return String(siteUrl || '').replace(/\/+$/g, '')
+}
+
+function normalizeLocalUrl (value) {
+  const path = String(value || '').trim()
+  if (!path || path === '/') return '/'
+  if (/^https?:\/\//i.test(path)) return path
+  return `/${path.replace(/^\/+/, '')}`
+}
+
+function resolveSitemapUrl (path, siteUrl) {
+  const localUrl = normalizeLocalUrl(path)
+  if (/^https?:\/\//i.test(localUrl)) return localUrl
+
+  const baseUrl = normalizeBaseUrl(siteUrl)
+  return baseUrl ? `${baseUrl}${localUrl}` : localUrl
+}
+
+function normalizeSitemapIdentity (value) {
+  const normalized = normalizeLocalUrl(String(value || '').trim())
+  if (!/^https?:\/\//i.test(normalized)) return normalized.toLowerCase()
+
+  try {
+    const url = new URL(normalized)
+    return `${url.pathname}${url.search}`.toLowerCase()
+  } catch {
+    return normalized.toLowerCase()
+  }
+}
+
+function normalizeSitemapEntry (entry) {
+  if (typeof entry === 'string') return { path: entry }
+  return entry && typeof entry === 'object' ? entry : null
+}
+
+export function createSitemap ({ entries = [], siteUrl = '', generatedAt = new Date().toISOString() } = {}) {
+  const lastmod = String(generatedAt || new Date().toISOString()).slice(0, 10)
+  const urls = (Array.isArray(entries) ? entries : [])
+    .map(normalizeSitemapEntry)
+    .filter(entry => entry?.path)
+    .map(entry => {
+      const lines = [
+        '  <url>',
+        `    <loc>${escapeXml(resolveSitemapUrl(entry.path, siteUrl))}</loc>`,
+        `    <lastmod>${escapeXml(entry.lastmod || lastmod)}</lastmod>`
+      ]
+
+      if (entry.changefreq) {
+        lines.push(`    <changefreq>${escapeXml(entry.changefreq)}</changefreq>`)
+      }
+
+      if (entry.priority !== undefined && entry.priority !== null) {
+        lines.push(`    <priority>${escapeXml(entry.priority)}</priority>`)
+      }
+
+      lines.push('  </url>')
+      return lines.join('\n')
+    })
+    .join('\n')
+
+  return `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${urls ? `${urls}\n` : ''}</urlset>\n`
+}
+
+export function appendSitemapsToRobots (robotsContent, { sitemaps = [], siteUrl = '' } = {}) {
+  const input = typeof robotsContent === 'string' && robotsContent.trim()
+    ? robotsContent
+    : 'User-agent: *\nAllow: /\n'
+
+  const existingIdentities = new Set(
+    input
+      .replace(/\r\n/g, '\n')
+      .split('\n')
+      .map(line => line.match(/^\s*Sitemap\s*:\s*(.+?)\s*$/i)?.[1])
+      .filter(Boolean)
+      .map(normalizeSitemapIdentity)
+  )
+
+  const addedIdentities = new Set()
+  const sitemapLines = (Array.isArray(sitemaps) ? sitemaps : [sitemaps])
+    .filter(Boolean)
+    .map(sitemap => resolveSitemapUrl(sitemap, siteUrl))
+    .filter(sitemap => {
+      const identity = normalizeSitemapIdentity(sitemap)
+      if (existingIdentities.has(identity) || addedIdentities.has(identity)) return false
+      addedIdentities.add(identity)
+      return true
+    })
+    .map(sitemap => `Sitemap: ${sitemap}`)
+
+  if (sitemapLines.length === 0) return input
+
+  return `${input.replace(/\s+$/g, '')}\n${sitemapLines.join('\n')}\n`
+}

package/src/store/Layout.js

@@ -29,7 +29,9 @@ export default {
     // Main
     scrolling: true,
     meta: true,
-    metaToggle: false
+    metaToggle: false,
+    assistant: false,
+    assistantWidth: 380
   },
   getters: {
     view (state) {
@@ -118,6 +120,12 @@ export default {
     },
     setMetaToggle (state, val) {
       state.metaToggle = val
+    },
+    setAssistant (state, val) {
+      state.assistant = val
+    },
+    setAssistantWidth (state, val) {
+      state.assistantWidth = val
     }
   },
   actions: {}