@docsector/docsector-reader 2.3.0 → 3.0.0

package/src/components/page-section-tokens.js

@@ -0,0 +1,613 @@
+import MarkdownIt from 'markdown-it'
+import attrs from 'markdown-it-attrs'
+
+const ALERT_MESSAGE_TYPES = new Set([
+  'note',
+  'tip',
+  'important',
+  'warning',
+  'caution'
+])
+
+const QUICK_LINKS_MARKER_PREFIX = '@@DOCSECTOR_QUICK_LINKS_'
+const EXPANDABLE_MARKER_PREFIX = '@@DOCSECTOR_EXPANDABLE_'
+const CODE_SEGMENT_MARKER_PREFIX = '@@DOCSECTOR_CODE_SEGMENT_'
+
+const parseAlertMarker = (rawContent = '') => {
+  const match = String(rawContent).trim().match(/^\[!\s*([A-Za-z]+)\s*\]\s*(.*)$/s)
+  if (!match) {
+    return null
+  }
+
+  const type = match[1].toLowerCase()
+  if (!ALERT_MESSAGE_TYPES.has(type)) {
+    return null
+  }
+
+  return {
+    type,
+    content: (match[2] || '').trim()
+  }
+}
+
+export const parseCustomTagAttributes = (raw = '') => {
+  const parsed = {}
+  const pattern = /([\w-]+)\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+))/g
+
+  let match = pattern.exec(String(raw))
+  while (match !== null) {
+    const key = match[1]
+    const value = match[2] || match[3] || match[4] || ''
+    parsed[key] = value
+    match = pattern.exec(String(raw))
+  }
+
+  return parsed
+}
+
+const shieldInlineCodeSegments = (line = '', createMarker, codeSegmentsMap) => {
+  return String(line).replace(/(`+)([^`\n]*?)\1/g, (match) => {
+    const marker = createMarker()
+    codeSegmentsMap.set(marker, match)
+    return marker
+  })
+}
+
+const shieldMarkdownCodeSegments = (source = '') => {
+  const codeSegmentsMap = new Map()
+  const output = []
+  const lines = String(source).split(/\r?\n/)
+  let index = 0
+
+  const createMarker = () => `${CODE_SEGMENT_MARKER_PREFIX}${index++}@@`
+
+  for (let lineIndex = 0; lineIndex < lines.length; lineIndex++) {
+    const line = lines[lineIndex]
+    const fenceMatch = line.match(/^( {0,3})(`{3,}|~{3,})(.*)$/)
+
+    if (fenceMatch) {
+      const fenceToken = fenceMatch[2]
+      const fenceChar = fenceToken[0]
+      const closingPattern = new RegExp(`^( {0,3})${fenceChar}{${fenceToken.length},}\\s*$`)
+      const blockLines = [line]
+
+      while (lineIndex + 1 < lines.length) {
+        lineIndex++
+        const nextLine = lines[lineIndex]
+        blockLines.push(nextLine)
+
+        if (closingPattern.test(nextLine)) {
+          break
+        }
+      }
+
+      const marker = createMarker()
+      codeSegmentsMap.set(marker, blockLines.join('\n'))
+      output.push(marker)
+      continue
+    }
+
+    output.push(shieldInlineCodeSegments(line, createMarker, codeSegmentsMap))
+  }
+
+  return {
+    source: output.join('\n'),
+    codeSegmentsMap
+  }
+}
+
+const restoreShieldedCodeSegments = (source = '', codeSegmentsMap = new Map()) => {
+  let restored = String(source)
+
+  codeSegmentsMap.forEach((content, marker) => {
+    restored = restored.replaceAll(marker, content)
+  })
+
+  return restored
+}
+
+const extractQuickLinksBlocks = (source = '') => {
+  const map = new Map()
+  let index = 0
+
+  const blockPattern = /<d-quick-links\b([^>]*)>([\s\S]*?)<\/d-quick-links>/gi
+  const replaced = String(source).replace(blockPattern, (_, blockAttrsRaw, inner) => {
+    const blockAttrs = parseCustomTagAttributes(blockAttrsRaw)
+    const items = []
+    const itemPattern = /<d-quick-link\b([^>]*)\/?\s*>/gi
+
+    let itemMatch = itemPattern.exec(inner)
+    while (itemMatch !== null) {
+      const itemAttrs = parseCustomTagAttributes(itemMatch[1])
+      const title = itemAttrs.title || ''
+      const description = itemAttrs.description || ''
+      const to = itemAttrs.to || ''
+      const href = itemAttrs.href || ''
+
+      if (title && description && (to || href)) {
+        items.push({
+          icon: itemAttrs.icon || 'link',
+          title,
+          description,
+          to,
+          href
+        })
+      }
+
+      itemMatch = itemPattern.exec(inner)
+    }
+
+    const marker = `${QUICK_LINKS_MARKER_PREFIX}${index}@@`
+    index++
+
+    map.set(marker, {
+      title: blockAttrs.title || '',
+      items
+    })
+
+    return `\n${marker}\n`
+  })
+
+  return {
+    source: replaced,
+    quickLinksMap: map
+  }
+}
+
+const parseExpandableOpenState = (raw = '') => {
+  return ['1', 'true', 'yes', 'on'].includes(String(raw).trim().toLowerCase())
+}
+
+const extractExpandableBlocks = (source = '') => {
+  const map = new Map()
+  let index = 0
+
+  const blockPattern = /<d-expandable\b([^>]*)>([\s\S]*?)<\/d-expandable>/gi
+  const replaced = String(source).replace(blockPattern, (_, rawAttrs, inner) => {
+    const attrs = parseCustomTagAttributes(rawAttrs)
+    const marker = `${EXPANDABLE_MARKER_PREFIX}${index}@@`
+    index++
+
+    map.set(marker, {
+      title: attrs.title || '',
+      open: parseExpandableOpenState(attrs.open),
+      content: inner
+    })
+
+    return `\n${marker}\n`
+  })
+
+  return {
+    source: replaced,
+    expandableMap: map
+  }
+}
+
+const parseFenceAttributes = (raw = '') => {
+  const parsed = {}
+  const pattern = /([\w-]+)\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s;]+))/g
+
+  let match = pattern.exec(String(raw))
+  while (match !== null) {
+    const key = match[1]
+    const value = match[2] || match[3] || match[4] || ''
+    parsed[key] = value
+    match = pattern.exec(String(raw))
+  }
+
+  return parsed
+}
+
+const parseTokenAttributes = (element) => {
+  const parsed = {}
+
+  ;(element.attrs || []).forEach(([key, value]) => {
+    parsed[key] = value || ''
+  })
+
+  return parsed
+}
+
+const parseFenceLanguage = (raw = '') => {
+  const cleaned = String(raw)
+    .replace(/:[^;]*;/g, ' ')
+    .replace(/[\w-]+\s*=\s*(?:"[^"]*"|'[^']*'|[^\s;]+)/g, ' ')
+    .replace(/[;:]/g, ' ')
+    .trim()
+
+  return cleaned.split(/\s+/)[0] || ''
+}
+
+const parseFenceMeta = (element) => {
+  const rawInfo = String(element.info || '')
+  const meta = {
+    ...parseFenceAttributes(rawInfo),
+    ...parseTokenAttributes(element)
+  }
+
+  return {
+    ...meta,
+    language: parseFenceLanguage(rawInfo) || meta.language || 'html'
+  }
+}
+
+const parseBreadcrumb = (raw = '') => {
+  const value = String(raw).trim()
+
+  if (!value) {
+    return []
+  }
+
+  const separator = value.includes('>') ? '>' : '/'
+
+  return value
+    .split(separator)
+    .map((segment) => segment.trim())
+    .filter(Boolean)
+}
+
+const createSourceCodeTab = (element, meta) => ({
+  label: meta.tab || meta.filename || meta.language || 'Code',
+  language: meta.language || 'html',
+  text: element.content,
+  filename: meta.filename || '',
+  breadcrumbs: parseBreadcrumb(meta.breadcrumb)
+})
+
+const pushSourceCodeToken = (tokens, element, parserState) => {
+  const meta = parseFenceMeta(element)
+
+  if (meta.language === 'mermaid') {
+    tokens.push({
+      tag: 'mermaid',
+      content: element.content
+    })
+    return
+  }
+
+  const tab = createSourceCodeTab(element, meta)
+
+  if (meta.group) {
+    const previous = tokens[tokens.length - 1]
+
+    if (previous?.tag === 'code' && previous.group === meta.group && Array.isArray(previous.tabs)) {
+      previous.tabs.push(tab)
+      return
+    }
+
+    tokens.push({
+      tag: 'code',
+      codeIndex: parserState.codeIndex++,
+      group: meta.group,
+      content: tab.text,
+      info: tab.language,
+      filename: tab.filename,
+      breadcrumbs: tab.breadcrumbs,
+      tabs: [tab]
+    })
+    return
+  }
+
+  tokens.push({
+    tag: 'code',
+    codeIndex: parserState.codeIndex++,
+    content: tab.text,
+    info: tab.language,
+    filename: tab.filename,
+    breadcrumbs: tab.breadcrumbs,
+    tabs: []
+  })
+}
+
+const createMarkdownBlockParser = () => {
+  const markdown = new MarkdownIt({
+    html: true
+  })
+
+  markdown.use(attrs, {
+    leftDelimiter: ':',
+    rightDelimiter: ';',
+    allowedAttributes: ['filename', 'group', 'tab', 'breadcrumb']
+  })
+
+  return markdown
+}
+
+const createMarkdownInlineParser = () => {
+  return new MarkdownIt({
+    html: true
+  })
+}
+
+const normalizePageSectionSource = (source = '') => {
+  return String(source)
+    .replace(/&#123;/g, '{')
+    .replace(/&#125;/g, '}')
+    .replace(/&amp;/g, '&')
+}
+
+export const tokenizePageSectionSource = (source = '', options = {}) => {
+  const {
+    allowHeadingTokens = true,
+    parserState = { codeIndex: 0 }
+  } = options
+  const normalizedSource = normalizePageSectionSource(source)
+  const { source: sourceWithShieldedCode, codeSegmentsMap } = shieldMarkdownCodeSegments(normalizedSource)
+  const { source: sourceWithExpandables, expandableMap } = extractExpandableBlocks(sourceWithShieldedCode)
+
+  expandableMap.forEach((data, marker) => {
+    expandableMap.set(marker, {
+      ...data,
+      content: restoreShieldedCodeSegments(data.content, codeSegmentsMap)
+    })
+  })
+
+  const { source: sourceWithQuickLinks, quickLinksMap } = extractQuickLinksBlocks(sourceWithExpandables)
+  const markdown = createMarkdownBlockParser()
+  const markdownInline = createMarkdownInlineParser()
+  const markdownEnv = {}
+  const parsed = markdown.parse(restoreShieldedCodeSegments(sourceWithQuickLinks, codeSegmentsMap), markdownEnv)
+  const tokens = []
+
+  let level = 0
+  let tag = ''
+
+  const blockquote = {
+    depth: 0,
+    content: '',
+    alertType: '',
+    firstInline: true
+  }
+
+  const resetBlockquote = () => {
+    blockquote.content = ''
+    blockquote.alertType = ''
+    blockquote.firstInline = true
+  }
+
+  const flushBlockquote = () => {
+    const content = blockquote.content
+      .replace(/<p>\s*<\/p>/g, '')
+      .trim()
+
+    tokens.push({
+      tag: 'blockquote',
+      content,
+      alertType: blockquote.alertType
+    })
+
+    resetBlockquote()
+  }
+
+  const appendBlockquoteTag = (element, open) => {
+    if (!element.tag || element.tag === 'blockquote') {
+      return
+    }
+
+    blockquote.content += open
+      ? `<${element.tag}>`
+      : `</${element.tag}>`
+  }
+
+  parsed.forEach((element) => {
+    if (element.type === 'blockquote_open') {
+      if (blockquote.depth === 0) {
+        resetBlockquote()
+      }
+
+      blockquote.depth++
+      return
+    }
+
+    if (element.type === 'blockquote_close' && blockquote.depth > 0) {
+      blockquote.depth--
+
+      if (blockquote.depth === 0) {
+        flushBlockquote()
+      }
+
+      return
+    }
+
+    if (blockquote.depth > 0) {
+      if (element.type === 'inline') {
+        const rawInline = element.content
+
+        if (blockquote.firstInline) {
+          blockquote.firstInline = false
+
+          const alert = parseAlertMarker(rawInline)
+          if (alert !== null) {
+            blockquote.alertType = alert.type
+
+            if (alert.content !== '') {
+              blockquote.content += markdownInline.renderInline(alert.content, markdownEnv)
+            }
+
+            return
+          }
+        }
+
+        blockquote.content += markdownInline.renderInline(rawInline, markdownEnv)
+        return
+      }
+
+      if (element.type === 'fence') {
+        const language = parseFenceLanguage(element.info)
+        const escaped = String(element.content)
+          .replace(/&/g, '&amp;')
+          .replace(/</g, '&lt;')
+          .replace(/>/g, '&gt;')
+
+        const languageClass = language ? ` class="language-${language}"` : ''
+        blockquote.content += `<pre><code${languageClass}>${escaped}</code></pre>`
+        return
+      }
+
+      if (element.type.endsWith('_open')) {
+        appendBlockquoteTag(element, true)
+        return
+      }
+
+      if (element.type.endsWith('_close')) {
+        appendBlockquoteTag(element, false)
+        return
+      }
+
+      return
+    }
+
+    switch (element.type) {
+      case 'bullet_list_open':
+      case 'ordered_list_open':
+      case 'table_open':
+        level++
+    }
+
+    if (element.type === 'inline') {
+      element.content = markdownInline.renderInline(element.content, markdownEnv)
+    }
+
+    if (level === 0) {
+      switch (element.type) {
+        case 'heading_open':
+          tag = allowHeadingTokens ? element.tag : 'p'
+          break
+        case 'paragraph_open':
+        case 'list_item_open':
+          tag = element.tag
+          break
+      }
+
+      switch (element.type) {
+        case 'inline':
+          if (expandableMap.has(element.content.trim())) {
+            const data = expandableMap.get(element.content.trim())
+
+            tokens.push({
+              tag: 'expandable',
+              title: data.title,
+              open: data.open,
+              tokens: tokenizePageSectionSource(data.content, {
+                allowHeadingTokens: false,
+                parserState
+              })
+            })
+            break
+          }
+
+          if (quickLinksMap.has(element.content.trim())) {
+            const data = quickLinksMap.get(element.content.trim())
+
+            tokens.push({
+              tag: 'quick-links',
+              title: data.title,
+              items: data.items
+            })
+            break
+          }
+
+          tokens.push({
+            tag,
+            map: element.map,
+            content: element.content,
+            info: element.info
+          })
+          break
+
+        case 'fence':
+          pushSourceCodeToken(tokens, element, parserState)
+          break
+
+        case 'html_block':
+          tokens.push({
+            tag: 'html',
+            content: element.content
+          })
+          break
+      }
+    } else if (level === 1) {
+      const parent = tokens[tokens.length - 1]
+
+      switch (element.type) {
+        case 'bullet_list_open':
+          tokens.push({
+            tag: 'ul',
+            content: ''
+          })
+          break
+
+        case 'ordered_list_open':
+          tokens.push({
+            tag: 'ol',
+            content: ''
+          })
+          break
+
+        case 'table_open':
+          tokens.push({
+            tag: 'table',
+            content: ''
+          })
+          break
+
+        case 'list_item_open':
+          parent.content += '<li>'
+          break
+
+        case 'thead_open':
+          parent.content += '<thead>'
+          break
+        case 'tbody_open':
+          parent.content += '<tbody>'
+          break
+        case 'tr_open':
+          parent.content += '<tr>'
+          break
+        case 'th_open':
+          parent.content += '<th>'
+          break
+        case 'td_open':
+          parent.content += '<td>'
+          break
+
+        case 'inline':
+          parent.content += element.content
+          break
+        case 'html_inline':
+        case 'html_block':
+          parent.content += element.content
+          break
+
+        case 'list_item_close':
+          parent.content += '</li>'
+          break
+
+        case 'thead_close':
+          parent.content += '</thead>'
+          break
+        case 'tbody_close':
+          parent.content += '</tbody>'
+          break
+        case 'tr_close':
+          parent.content += '</tr>'
+          break
+        case 'th_close':
+          parent.content += '</th>'
+          break
+        case 'td_close':
+          parent.content += '</td>'
+          break
+      }
+    }
+
+    switch (element.type) {
+      case 'bullet_list_close':
+      case 'ordered_list_close':
+      case 'table_close':
+        level--
+    }
+  })
+
+  return tokens
+}

package/src/pages/guide/getting-started.overview.en-US.md

@@ -64,7 +64,7 @@ After `init`, your project will have this structure:
 - `src/pages/guide/` — Guide-type pages (Markdown files)
 - `src/pages/manual/` — Manual-type pages (Markdown files)
 - `src/i18n/index.js` — i18n loader using `buildMessages()` from the package
-- `src/i18n/tags.hjson` — Search keywords per route and locale
+- `src/pages/*.index.js` — Page registry plus per-page `metadata.tags` for menu search
 - `public/` — Static assets (logo, favicon, images)
 
 The rendering engine (components, layouts, router, store, composables) lives inside the `@docsector/docsector-reader` package — you only maintain your content and configuration.
@@ -74,5 +74,5 @@ The rendering engine (components, layouts, router, store, composables) lives ins
 - Configure your project branding in **docsector.config.js**
 - Define your pages in **src/pages/guide.index.js** (and other `*.index.js` registries)
 - Write your documentation in **Markdown**
-- Add search keywords in **src/i18n/tags.hjson**
+- Add search keywords via **metadata.tags** in each page entry of **src/pages/*.index.js**
 - Customize themes and appearance

package/src/pages/guide/getting-started.overview.pt-BR.md

@@ -64,7 +64,7 @@ Após o `init`, seu projeto terá esta estrutura:
 - `src/pages/guide/` — Páginas tipo guia (arquivos Markdown)
 - `src/pages/manual/` — Páginas tipo manual (arquivos Markdown)
 - `src/i18n/index.js` — Loader i18n usando `buildMessages()` do pacote
-- `src/i18n/tags.hjson` — Palavras-chave de busca por rota e idioma
+- `src/pages/*.index.js` — Registro de páginas com `metadata.tags` por página para busca no menu
 - `public/` — Assets estáticos (logo, favicon, imagens)
 
 O motor de renderização (componentes, layouts, router, store, composables) fica dentro do pacote `@docsector/docsector-reader` — você só mantém seu conteúdo e configuração.
@@ -74,5 +74,5 @@ O motor de renderização (componentes, layouts, router, store, composables) fic
 - Configure o branding do seu projeto em **docsector.config.js**
 - Defina suas páginas em **src/pages/guide.index.js** (e outros registros `*.index.js`)
 - Escreva sua documentação em **Markdown**
-- Adicione palavras-chave de busca em **src/i18n/tags.hjson**
+- Adicione palavras-chave de busca via **metadata.tags** em cada entrada de página em **src/pages/*.index.js**
 - Personalize temas e aparência

package/src/pages/guide/i18n-and-markdown.overview.en-US.md

@@ -94,6 +94,22 @@ echo "Example"
 
 `filename` is shown in the info bar for single blocks. Consecutive fences with the same `group` are rendered as tabs. `tab` sets the tab label, so filename-like labels such as `example.php` receive file icons in the tab. `breadcrumb` sets the breadcrumb segments above the active code block, and the final filename-like segment receives the same file icon.
 
+### Expandable Content
+
+Use `<d-expandable>` to hide secondary content without removing rich Markdown features from the page:
+
+```markdown
+<d-expandable title="More details">
+
+Optional explanations, operational notes, or longer examples.
+
+</d-expandable>
+```
+
+Set `open="true"` when the block should start expanded.
+
+The expandable body supports paragraphs, lists, alerts, code blocks, Mermaid diagrams, tables, raw HTML, and quick links. Keep headings outside the expandable block in this first version, because headings inside the body are flattened to regular paragraphs to preserve the page ToC.
+
 ## Adding a New Language
 
 1. Create `src/i18n/languages/xx-XX.hjson` with all UI translations

package/src/pages/guide/i18n-and-markdown.overview.pt-BR.md

@@ -94,6 +94,22 @@ echo "Exemplo"
 
 `filename` aparece na barra de info em blocos simples. Fences consecutivos com o mesmo `group` são renderizados como abas. `tab` define o rótulo da aba, então labels que parecem arquivo, como `exemplo.php`, recebem ícones na aba. `breadcrumb` define os segmentos acima do bloco ativo, e o segmento final que parece arquivo recebe o mesmo ícone.
 
+### Conteúdo Expansível
+
+Use `<d-expandable>` para esconder conteúdo secundário sem perder os recursos ricos de Markdown da página:
+
+```markdown
+<d-expandable title="Mais detalhes">
+
+Explicações opcionais, notas operacionais ou exemplos maiores.
+
+</d-expandable>
+```
+
+Defina `open="true"` quando o bloco precisar começar aberto.
+
+O corpo do expansível suporta parágrafos, listas, alertas, blocos de código, diagramas Mermaid, tabelas, HTML bruto e quick links. Nesta primeira versão, mantenha títulos fora do bloco expansível, porque títulos dentro do corpo viram parágrafos comuns para preservar o ToC da página.
+
 ## Adicionando um Novo Idioma
 
 1. Crie `src/i18n/languages/xx-XX.hjson` com todas as traduções de UI