@redpanda-data/docs-extensions-and-macros 4.15.2 → 4.15.3

package/extension-utils/url-utils.js

@@ -0,0 +1,39 @@
+'use strict'
+
+/**
+ * Shared URL utility functions for extensions
+ */
+
+/**
+ * Convert HTML URL to markdown URL
+ * @param {string} htmlUrl - The HTML URL (e.g., '/path/to/page/' or '/path/to/page/index.html')
+ * @returns {string} - The markdown URL (e.g., '/path/to/page.md')
+ */
+function toMarkdownUrl(htmlUrl) {
+  if (!htmlUrl) return ''
+
+  // Handle root path
+  if (htmlUrl === '/' || htmlUrl === '/index.html') {
+    return '/index.md'
+  }
+
+  // Remove trailing slash
+  let mdUrl = htmlUrl.replace(/\/$/, '')
+
+  // Replace /index.html with .md
+  mdUrl = mdUrl.replace(/\/index\.html$/, '.md')
+
+  // Replace .html with .md
+  mdUrl = mdUrl.replace(/\.html$/, '.md')
+
+  // If it doesn't end with .md yet, add it
+  if (!mdUrl.endsWith('.md')) {
+    mdUrl += '.md'
+  }
+
+  return mdUrl
+}
+
+module.exports = {
+  toMarkdownUrl,
+}

package/extensions/README.adoc

@@ -77,6 +77,10 @@ IMPORTANT: Extensions must be registered under the `antora.extensions` key in yo
 * **replace-attributes-in-attachments** - Replace AsciiDoc attributes in attached files
 * **collect-bloblang-samples** - Collect Bloblang code samples
 
+=== Git integration
+* **git-full-clone** - Ensure full git clones for accurate git dates
+* **add-git-dates** - Add git-based created and modified dates to pages
+
 === Content enhancement
 * **add-global-attributes** - Add attributes globally to all pages
 * **add-pages-to-root** - Add pages to the root navigation

package/extensions/REFERENCE.adoc

@@ -766,3 +766,427 @@ The processed attribute can be used in Handlebars templates:
   {{/each}}
 </div>
 ```
+
+== Git full clone
+
+This extension ensures Antora uses full git clones instead of shallow clones for remote repositories. Full git history is required for accurate git date extraction by the `add-git-dates` extension.
+
+The extension uses a two-phase approach:
+
+1. **Phase 1**: Sets `git.depth = 0` in playbook during registration (best effort)
+2. **Phase 2**: After content aggregation, detects repos with a `shallow` file and runs `git fetch --unshallow` to convert them to full clones
+
+This two-phase approach works around Antora's default shallow clone behavior, which can't always be overridden via playbook configuration alone.
+
+=== How it works
+
+. During extension registration, modifies playbook to request `depth=0` for all remote repos
+. After Antora aggregates content, checks each repo's git directory for a `shallow` file
+. For any shallow repos found, runs `git fetch --unshallow` with timeout protection
+. On subsequent builds with warm cache, repos are already full clones (no unshallow needed)
+
+=== Performance
+
+**First build** (cold cache):
+
+- Unshallow: ~1-5 seconds per remote repo
+- Total overhead: ~5-20 seconds depending on repo count
+
+**Subsequent builds** (warm cache):
+
+- Unshallow: 0 seconds (repos already full clones)
+- Total overhead: minimal
+
+**Scalability:**
+
+- 1k commits: ~1-2s unshallow time
+- 5k commits: ~3-5s unshallow time
+- 10k commits: ~5-10s unshallow time
+- 50k+ commits: Consider increasing timeout or pre-populating cache
+
+=== Environment variables
+
+This extension does not require any environment variables.
+
+=== Configuration options
+
+The extension accepts the following configuration options:
+
+skipUnshallow (optional, default: false):: Set to `true` to skip the unshallow phase entirely. Useful for air-gapped CI/CD environments or when git dates are not needed.
+
+unshallowTimeout (optional, default: 60000):: Timeout in milliseconds for the unshallow operation per repository. Increase this for very large repositories (50k+ commits).
+
+=== Registration
+
+Basic configuration (recommended):
+
+[,yaml]
+----
+antora:
+  extensions:
+  - require: '@redpanda-data/docs-extensions-and-macros/extensions/git-full-clone'
+----
+
+With custom timeout for large repos:
+
+[,yaml]
+----
+antora:
+  extensions:
+  - require: '@redpanda-data/docs-extensions-and-macros/extensions/git-full-clone'
+    unshallowTimeout: 120000  # 2 minutes for very large repos
+----
+
+For air-gapped environments:
+
+[,yaml]
+----
+antora:
+  extensions:
+  - require: '@redpanda-data/docs-extensions-and-macros/extensions/git-full-clone'
+    skipUnshallow: true  # Skip network-dependent unshallow
+----
+
+=== Production considerations
+
+**Netlify/CI builds:**
+
+- First build or cache clear: Pays unshallow cost (~5-20s)
+- Subsequent builds with warm cache: No unshallow needed
+- Consider pre-populating Antora cache in CI for faster cold builds
+
+**Timeout protection:**
+
+- Default 60-second timeout prevents hanging on slow networks
+- Graceful failure: Build continues even if unshallow times out
+- Git dates may be incomplete for timed-out repos (warning logged)
+
+**Error handling:**
+
+- Network failures don't break the build
+- Timeout failures are logged with guidance to increase timeout
+- Falls back to available git history if unshallow fails
+
+**Optimization for very large repos (50k+ commits):**
+
+1. Increase `unshallowTimeout` to 120000ms or higher
+2. Pre-populate Antora cache with full clones in CI/CD
+3. Use persistent cache in Netlify to avoid repeated unshallows
+4. Consider implementing Neon KV caching for git dates (future optimization)
+
+== Add Git dates
+
+This extension adds accurate git-based created and modified dates to documentation pages by analyzing commit history. It uses isomorphic-git (bundled with Antora) to walk the commit history of each repository and detects actual file modifications by comparing tree objects between commits.
+
+The extension adds the following attributes to each page:
+
+- `page-git-created-date`: The date when the file was first committed (YYYY-MM-DD format)
+- `page-git-modified-date`: The date when the file was last modified (YYYY-MM-DD format)
+
+These attributes are available for use in UI templates (for metadata, display banners, etc.) and are exported to markdown files for AI consumption.
+
+=== How it works
+
+. Groups pages by repository AND branch (same repo can have multiple branches: v/23.3, v/24.1, main)
+. Walks git log once per repo+branch using isomorphic-git
+. Compares tree objects between commits to detect actual file modifications (not just file existence)
+. Builds a filepath → {created, modified} map for each repo+branch
+. Applies dates to pages based on their source file path
+
+This approach is **O(commits)** instead of **O(files × commits)**, making it efficient for large repositories.
+
+=== Performance
+
+- **Full clone** (depth=0): ~15s for 4128 pages across 12 branches (14.5s git processing)
+- **Per-page overhead**: ~3-5ms per page
+
+=== Environment variables
+
+This extension does not require any environment variables.
+
+=== Configuration options
+
+This extension does not require configuration options. However, it works best with **full git clones** rather than shallow clones.
+
+To enable full clones for remote repositories, use the `git-full-clone` extension:
+
+```yaml
+antora:
+  extensions:
+  - require: '@redpanda-data/docs-extensions-and-macros/extensions/git-full-clone'
+  - require: '@redpanda-data/docs-extensions-and-macros/extensions/add-git-dates'
+```
+
+=== Registration
+
+```yaml
+antora:
+  extensions:
+  - require: '@redpanda-data/docs-extensions-and-macros/extensions/add-git-dates'
+```
+
+=== UI integration
+
+The git dates are available in Handlebars templates via `page.attributes`:
+
+```html
+{{#with page.attributes.git-created-date}}
+  <meta name="datePublished" content="{{this}}">
+{{/with}}
+{{#with page.attributes.git-modified-date}}
+  <meta name="dateModified" content="{{this}}">
+{{/with}}
+```
+
+They are also included in schema.org structured data for SEO.
+
+== FAQ structured data
+
+This extension generates schema.org FAQPage JSON-LD for better SEO and Google rich results. Writers define FAQs using page attributes, and the extension generates compliant structured data in the page `<head>`.
+
+=== Environment variables
+
+This extension does not require any environment variables.
+
+=== Configuration options
+
+This extension does not require configuration options.
+
+=== Registration
+
+```yaml
+antora:
+  extensions:
+  - require: '@redpanda-data/docs-extensions-and-macros/extensions/add-faq-structured-data'
+```
+
+=== Usage
+
+Add FAQ attributes to your AsciiDoc page header:
+
+```asciidoc
+= My Documentation Page
+:page-faq-1-question: How do I install Redpanda?
+:page-faq-1-answer: Download from redpanda.com and run the installer. See our installation guide for details.
+:page-faq-1-anchor: #installation
+
+:page-faq-2-question: What are the system requirements?
+:page-faq-2-answer: You need at least 2GB of RAM and 2 CPU cores for development.
+:page-faq-2-anchor: #requirements
+
+:page-faq-3-question: Does Redpanda support Kafka APIs?
+:page-faq-3-answer: Yes! Redpanda is fully compatible with Kafka APIs.
+```
+
+**Required attributes per FAQ:**
+
+- `page-faq-N-question` - The FAQ question
+- `page-faq-N-answer` - The FAQ answer
+
+**Optional:**
+
+- `page-faq-N-anchor` - Link to page section (e.g., `#installation`)
+
+**Tips:**
+
+- FAQs must be numbered sequentially (1, 2, 3...)
+- Answers can reference prose content: "See our installation guide for details"
+- Anchors create deep links to related content on the page
+- Keep answers concise (Google truncates after ~300 characters)
+
+=== Generated output
+
+The extension generates schema.org FAQPage JSON-LD in the page `<head>`:
+
+```json
+{
+  "@type": "FAQPage",
+  "mainEntity": [
+    {
+      "@type": "Question",
+      "name": "How do I install Redpanda?",
+      "acceptedAnswer": {
+        "@type": "Answer",
+        "text": "Download from redpanda.com and run the installer."
+      },
+      "url": "https://docs.redpanda.com/page#installation"
+    }
+  ]
+}
+```
+
+This structured data appears in Google search results as rich snippets, improving SEO and click-through rates.
+
+== Convert pages to markdown
+
+The `convert-to-markdown` extension converts all HTML pages to markdown format and exports them as `.md` files alongside the HTML. This creates AI-friendly markdown versions of the documentation with proper frontmatter.
+
+=== Features
+
+- Converts all HTML pages to GitHub-Flavored Markdown
+- Exports as `.md` files (e.g., `/page.html` → `/page.md`)
+- Adds YAML frontmatter with page metadata
+- Decodes HTML entities in titles
+- Supports content negotiation via `Accept: text/markdown` header
+- Stores markdown in `page.markdownContents` for other extensions
+
+=== Configuration
+
+Add to your Antora playbook:
+
+[,yaml]
+----
+antora:
+  extensions:
+  - require: '@redpanda-data/docs-extensions-and-macros/extensions/convert-to-markdown'
+----
+
+=== Frontmatter attributes
+
+The extension exports these attributes to YAML frontmatter:
+
+- `page-title`: Page title
+- `page-description`: Page description
+- `page-component-name`, `page-component-version`: Component info
+- `page-git-created-date`, `page-git-modified-date`: Git dates (if git-dates extension enabled)
+- `page-categories`: Categories
+- `page-topic-type`: Topic type (tutorial, how-to, concept, reference)
+- `page-personas`: Target audience
+- Custom attributes from allowlist
+
+=== Example output
+
+[,yaml]
+----
+---
+title: "Get started with Redpanda"
+description: "Quick start guide for Redpanda"
+component: ROOT
+version: current
+created: 2024-01-15
+modified: 2024-03-29
+categories:
+  - Getting Started
+topic-type: tutorial
+personas:
+  - developer
+---
+
+# Get started with Redpanda
+
+Your markdown content here...
+----
+
+== Generate llms.txt exports
+
+The `convert-llms-to-txt` extension generates AI-optimized documentation exports in plain text format following the llms.txt standard.
+
+=== Features
+
+- Generates `llms.txt` from a curated `llms.adoc` page in the home component
+- Creates `llms-full.txt` with all documentation pages
+- Generates component-specific exports (e.g., `ROOT-full.txt`, `cloud-full.txt`)
+- Unpublishes the HTML version of llms.adoc
+- Replaces site-url attribute dynamically for preview builds
+
+=== Configuration
+
+Add to your Antora playbook after `convert-to-markdown`:
+
+[,yaml]
+----
+antora:
+  extensions:
+  - require: '@redpanda-data/docs-extensions-and-macros/extensions/convert-to-markdown'
+  - require: '@redpanda-data/docs-extensions-and-macros/extensions/convert-llms-to-txt'
+----
+
+=== Setup
+
+Create `modules/home/pages/llms.adoc` in your docs repository with curated documentation overview content. The extension will:
+
+1. Convert it to markdown
+2. Place `llms.txt` at site root
+3. Generate `llms-full.txt` with all pages
+4. Generate component-specific full.txt files
+5. Unpublish the HTML version
+
+=== Generated files
+
+- `{site-url}/llms.txt` - Curated overview
+- `{site-url}/llms-full.txt` - Complete documentation
+- `{site-url}/ROOT-full.txt` - Self-Managed docs only
+- `{site-url}/cloud-full.txt` - Cloud docs only
+- `{site-url}/redpanda-connect-full.txt` - Connect docs only
+
+=== Attributes
+
+The extension replaces `{site-url}` attribute with:
+
+- `playbook.site.url` in production builds
+- `DEPLOY_PRIME_URL` environment variable in preview builds (when `PREVIEW=true`)
+
+== Convert sitemaps to markdown
+
+The `convert-sitemap-to-markdown` extension generates human-readable markdown versions of XML sitemap files.
+
+=== Features
+
+- Converts all sitemap XML files to markdown
+- Generates individual `.md` files for each sitemap
+- Creates master `sitemap-all.md` combining all pages
+- Organizes URLs by component/version
+- Includes page metadata (modified dates, priority)
+- Uses Antora content catalog (no filesystem operations)
+
+=== Configuration
+
+Add to your Antora playbook:
+
+[,yaml]
+----
+antora:
+  extensions:
+  - require: '@redpanda-data/docs-extensions-and-macros/extensions/convert-sitemap-to-markdown'
+----
+
+=== Generated files
+
+For a site with component-specific sitemaps:
+
+- `sitemap.md` - Sitemap index with links to sub-sitemaps
+- `sitemap-ROOT.md` - Self-Managed pages
+- `sitemap-home.md` - Home component pages
+- `sitemap-redpanda-cloud.md` - Cloud pages
+- `sitemap-redpanda-connect.md` - Connect pages
+- `sitemap-all.md` - Master combined sitemap with all pages
+
+=== Output format
+
+[,markdown]
+----
+# Complete documentation sitemap
+
+> Combined view of all 4,126 documentation pages from 8 sitemaps
+
+## Source sitemaps
+
+- [sitemap-ROOT.xml](sitemap-ROOT.md)
+- [sitemap-home.xml](sitemap-home.md)
+
+## Pages
+
+Total pages: 4,126
+
+### Current
+
+- [Get Started](https://docs.redpanda.com/current/get-started/) (modified: 2024-03-29)
+- [Deploy](https://docs.redpanda.com/current/deploy/) (modified: 2024-03-28)
+----
+
+=== Use cases
+
+- AI agents: Provide sitemap-all.md for quick site navigation
+- Documentation planning: Review complete site structure
+- Content audits: Identify gaps or outdated content by date
+- User discovery: Help users find content through browseable map

package/extensions/add-faq-structured-data.js

@@ -0,0 +1,153 @@
+'use strict'
+
+/**
+ * Generates FAQPage JSON-LD structured data for SEO.
+ *
+ * USAGE:
+ *   :page-faq-1-question: How do I install Redpanda?
+ *   :page-faq-1-answer: Download from redpanda.com and run the installer.
+ *   :page-faq-1-anchor: #installation (optional - links to section)
+ *
+ *   :page-faq-2-question: What are the system requirements?
+ *   :page-faq-2-answer: You need at least 2GB RAM and 2 CPU cores.
+ *   :page-faq-2-anchor: #requirements
+ *
+ * The extension:
+ * - Generates schema.org FAQPage JSON-LD in <head>
+ * - Supports multiple FAQs numbered sequentially (1, 2, 3...)
+ * - Anchor is optional and adds URL to the FAQ question
+ * - Writers can reference existing page content in answers
+ */
+
+/**
+ * Extract FAQ entries from page attributes
+ * @param {Object} attributes - Page attributes object
+ * @param {Object} logger - Logger instance
+ * @returns {Array<{question: string, answer: string, anchor?: string}>}
+ */
+function extractFaqs(attributes, logger) {
+  const faqs = []
+  const faqNumbers = new Set()
+
+  // Find all FAQ numbers by scanning for -question attributes
+  Object.keys(attributes).forEach(key => {
+    const match = key.match(/^page-faq-(\d+)-question$/)
+    if (match) {
+      faqNumbers.add(parseInt(match[1], 10))
+    }
+  })
+
+  if (faqNumbers.size === 0) return faqs
+
+  // Extract FAQs in numerical order
+  const sortedNumbers = Array.from(faqNumbers).sort((a, b) => a - b)
+
+  sortedNumbers.forEach(num => {
+    const question = attributes[`page-faq-${num}-question`]
+    const answer = attributes[`page-faq-${num}-answer`]
+    const anchor = attributes[`page-faq-${num}-anchor`]
+
+    // Both question and answer are required
+    if (!question) {
+      logger.warn(`FAQ ${num}: question missing`)
+      return
+    }
+
+    if (!answer) {
+      logger.warn(`FAQ ${num}: answer missing`)
+      return
+    }
+
+    const faq = {
+      question: question.trim(),
+      answer: answer.trim()
+    }
+
+    if (anchor) {
+      faq.anchor = anchor.trim()
+    }
+
+    faqs.push(faq)
+  })
+
+  return faqs
+}
+
+/**
+ * Generate FAQPage JSON-LD structure
+ * @param {Array} faqs - Array of FAQ objects
+ * @param {string} baseUrl - Base URL for the page
+ * @returns {Object} FAQPage JSON-LD object
+ */
+function generateFaqJsonLd(faqs, baseUrl) {
+  const mainEntity = faqs.map(faq => {
+    const question = {
+      '@type': 'Question',
+      'name': faq.question,
+      'acceptedAnswer': {
+        '@type': 'Answer',
+        'text': faq.answer
+      }
+    }
+
+    // Add URL with anchor if provided
+    if (faq.anchor) {
+      const anchor = faq.anchor.startsWith('#') ? faq.anchor : `#${faq.anchor}`
+      question.url = `${baseUrl}${anchor}`
+    }
+
+    return question
+  })
+
+  return {
+    '@type': 'FAQPage',
+    'mainEntity': mainEntity
+  }
+}
+
+module.exports.register = function () {
+  const logger = this.getLogger('add-faq-structured-data-extension')
+  let playbook
+
+  this.once('playbookBuilt', ({ playbook: pb }) => {
+    playbook = pb
+  })
+
+  this.on('documentsConverted', ({ contentCatalog }) => {
+    const pages = contentCatalog.getPages()
+    let processedCount = 0
+    let totalFaqs = 0
+    const siteUrl = playbook?.site?.url || 'https://docs.redpanda.com'
+
+    pages.forEach(page => {
+      const attributes = page.asciidoc?.attributes
+      if (!attributes) return
+
+      // Extract FAQs from attributes
+      const faqs = extractFaqs(attributes, logger)
+
+      if (faqs.length === 0) return
+
+      // Generate base URL for the page
+      let baseUrl = ''
+      if (page.pub?.url) {
+        baseUrl = `${siteUrl}${page.pub.url}`
+      }
+
+      // Generate FAQPage JSON-LD
+      const faqJsonLd = generateFaqJsonLd(faqs, baseUrl)
+
+      // Store as JSON string in page attribute for UI template
+      attributes['page-faq-json-ld'] = JSON.stringify(faqJsonLd, null, 2)
+
+      processedCount++
+      totalFaqs += faqs.length
+
+      logger.debug(`Added ${faqs.length} FAQs to ${page.src.relative}`)
+    })
+
+    if (processedCount > 0) {
+      logger.info(`Generated FAQ structured data for ${processedCount} pages (${totalFaqs} total FAQs)`)
+    }
+  })
+}