@redpanda-data/docs-extensions-and-macros 4.12.6 → 4.13.0

package/extensions/USER_GUIDE.adoc

@@ -0,0 +1,339 @@
+= Antora Extensions User Guide
+:toc:
+:toclevels: 3
+
+Complete guide to using Antora extensions in your documentation projects.
+
+== Getting started
+
+=== Installation
+
+Install the package via npm:
+
+[,bash]
+----
+npm i @redpanda-data/docs-extensions-and-macros
+----
+
+=== Basic configuration
+
+Extensions are configured in your Antora playbook under the `antora.extensions` key:
+
+[,yaml]
+----
+antora:
+  extensions:
+    - '@redpanda-data/docs-extensions-and-macros/extensions/extension-name'
+----
+
+IMPORTANT: Extensions must be under `antora.extensions`, not `asciidoc.extensions`. AsciiDoc macros go under `asciidoc.extensions`.
+
+=== Extension with options
+
+Pass configuration options to extensions:
+
+[,yaml]
+----
+antora:
+  extensions:
+    - require: '@redpanda-data/docs-extensions-and-macros/extensions/extension-name'
+      option1: value1
+      option2: value2
+----
+
+== Configuration patterns
+
+=== Simple registration
+
+For extensions without configuration options:
+
+[,yaml]
+----
+antora:
+  extensions:
+    - '@redpanda-data/docs-extensions-and-macros/extensions/unlisted-pages.js'
+----
+
+=== With environment variables
+
+Some extensions use environment variables for sensitive data:
+
+[,yaml]
+----
+antora:
+  extensions:
+    - require: '@redpanda-data/docs-extensions-and-macros/extensions/algolia-indexer'
+      algolia_app_id: $ALGOLIA_APP_ID
+      algolia_api_key: $ALGOLIA_API_KEY
+      algolia_index_name: docs
+----
+
+[,bash]
+----
+export ALGOLIA_APP_ID=your_app_id
+export ALGOLIA_API_KEY=your_api_key
+npx antora local-antora-playbook.yml
+----
+
+=== Multiple extensions
+
+Register multiple extensions in order:
+
+[,yaml]
+----
+antora:
+  extensions:
+    - '@redpanda-data/docs-extensions-and-macros/extensions/version-fetcher/set-latest-version'
+    - '@redpanda-data/docs-extensions-and-macros/extensions/generate-index-data.js'
+    - '@redpanda-data/docs-extensions-and-macros/extensions/produce-redirects.js'
+----
+
+Extensions run in the order they're listed.
+
+== Common workflows
+
+=== Version management
+
+Automatically fetch and set the latest Redpanda version:
+
+[,yaml]
+----
+antora:
+  extensions:
+    - '@redpanda-data/docs-extensions-and-macros/extensions/version-fetcher/set-latest-version'
+----
+
+This sets the `latest-redpanda-version` attribute for use in your documentation.
+
+=== Search integration
+
+Enable Algolia search indexing:
+
+[,yaml]
+----
+antora:
+  extensions:
+    - require: '@redpanda-data/docs-extensions-and-macros/extensions/algolia-indexer'
+      algolia_app_id: $ALGOLIA_APP_ID
+      algolia_api_key: $ALGOLIA_API_KEY
+      algolia_index_name: redpanda-docs
+      algolia_index_version: latest
+----
+
+=== Redirect management
+
+Generate redirects from old URLs to new ones:
+
+[,yaml]
+----
+antora:
+  extensions:
+    - '@redpanda-data/docs-extensions-and-macros/extensions/produce-redirects.js'
+----
+
+Define redirects in your `antora.yml`:
+
+[,yaml]
+----
+name: redpanda
+asciidoc:
+  attributes:
+    page-aliases: old-page.adoc,another-old-page.adoc
+----
+
+=== Content generation
+
+Generate component categories for Redpanda Connect:
+
+[,yaml]
+----
+antora:
+  extensions:
+    - '@redpanda-data/docs-extensions-and-macros/extensions/generate-rp-connect-categories.js'
+    - '@redpanda-data/docs-extensions-and-macros/extensions/generate-rp-connect-info.js'
+----
+
+=== Navigation customization
+
+Manage unlisted pages that appear in navigation:
+
+[,yaml]
+----
+antora:
+  extensions:
+    - '@redpanda-data/docs-extensions-and-macros/extensions/unlisted-pages.js'
+----
+
+Mark pages as unlisted in their AsciiDoc header:
+
+[,asciidoc]
+----
+= Page Title
+:page-unlisted:
+
+Content here...
+----
+
+== Extension order matters
+
+Extensions execute in registration order. Some extensions depend on others running first:
+
+[,yaml]
+----
+antora:
+  extensions:
+    # 1. Fetch versions first
+    - '@redpanda-data/docs-extensions-and-macros/extensions/version-fetcher/set-latest-version'
+    # 2. Then generate content that uses versions
+    - '@redpanda-data/docs-extensions-and-macros/extensions/generate-index-data.js'
+    # 3. Finally index for search
+    - require: '@redpanda-data/docs-extensions-and-macros/extensions/algolia-indexer'
+      algolia_app_id: $ALGOLIA_APP_ID
+      algolia_api_key: $ALGOLIA_API_KEY
+      algolia_index_name: docs
+----
+
+== Troubleshooting
+
+=== Extension not loading
+
+*Problem:* Extension doesn't seem to run
+
+*Solutions:*
+
+* Check the extension is under `antora.extensions`, not `asciidoc.extensions`
+* Verify the path to the extension is correct
+* Ensure the package is installed (`npm install`)
+* Check for typos in the extension name
+
+=== Environment variables not working
+
+*Problem:* Extensions using environment variables fail
+
+*Solutions:*
+
+* Set environment variables before running Antora:
++
+[,bash]
+----
+export VAR_NAME=value
+npx antora playbook.yml
+----
+
+* Check variable names match exactly (case-sensitive)
+* Use `$VAR_NAME` syntax in playbook
+* Verify variables are exported, not just set
+
+=== Extension execution order issues
+
+*Problem:* Extensions not working as expected
+
+*Solutions:*
+
+* Check extension order in playbook
+* Some extensions must run before others
+* Review extension dependencies in link:REFERENCE.adoc[Reference docs]
+
+=== Build performance
+
+*Problem:* Build is slow with extensions enabled
+
+*Solutions:*
+
+* Some extensions make network requests (for example, version fetcher, Algolia)
+* Use caching when possible
+* Disable non-essential extensions during development
+* Run only necessary extensions for your workflow
+
+=== Extension errors
+
+*Problem:* Extension throws errors during build
+
+*Solutions:*
+
+* Check extension configuration options are correct
+* Verify required environment variables are set
+* Review extension logs for specific error messages
+* Ensure your Antora version is compatible
+* Check link:REFERENCE.adoc[Reference docs] for requirements
+
+== Best practices
+
+=== Development vs production
+
+Use different playbooks for development and production:
+
+*Development playbook* (`local-antora-playbook.yml`):
+[,yaml]
+----
+antora:
+  extensions:
+    - '@redpanda-data/docs-extensions-and-macros/extensions/version-fetcher/set-latest-version'
+    # Skip search indexing during development
+----
+
+*Production playbook* (`antora-playbook.yml`):
+[,yaml]
+----
+antora:
+  extensions:
+    - '@redpanda-data/docs-extensions-and-macros/extensions/version-fetcher/set-latest-version'
+    - '@redpanda-data/docs-extensions-and-macros/extensions/generate-index-data.js'
+    - require: '@redpanda-data/docs-extensions-and-macros/extensions/algolia-indexer'
+      algolia_app_id: $ALGOLIA_APP_ID
+      algolia_api_key: $ALGOLIA_API_KEY
+      algolia_index_name: docs
+----
+
+=== Organizing playbooks
+
+Keep extension configuration organized:
+
+[,yaml]
+----
+antora:
+  extensions:
+    # Version management
+    - '@redpanda-data/docs-extensions-and-macros/extensions/version-fetcher/set-latest-version'
+
+    # Content generation
+    - '@redpanda-data/docs-extensions-and-macros/extensions/generate-rp-connect-categories.js'
+    - '@redpanda-data/docs-extensions-and-macros/extensions/generate-rp-connect-info.js'
+
+    # Search integration
+    - require: '@redpanda-data/docs-extensions-and-macros/extensions/algolia-indexer'
+      algolia_app_id: $ALGOLIA_APP_ID
+      algolia_api_key: $ALGOLIA_API_KEY
+      algolia_index_name: docs
+----
+
+=== Security
+
+* Never commit API keys or secrets to version control
+* Use environment variables for sensitive data
+* Set permissions appropriately in CI/CD
+* Rotate API keys regularly
+
+=== Testing
+
+Test your playbook locally before deploying:
+
+[,bash]
+----
+# Set test environment variables
+export ALGOLIA_APP_ID=test_app_id
+export ALGOLIA_API_KEY=test_api_key
+
+# Run Antora
+npx antora local-antora-playbook.yml
+
+# Check the output
+ls -la build/site/
+----
+
+== Related documentation
+
+* link:REFERENCE.adoc[Extensions reference] - Complete documentation for all extensions
+* link:DEVELOPMENT.adoc[Development guide] - How to create new extensions
+* https://docs.antora.org/antora/latest/playbook/[Antora Playbook Reference]
+* https://docs.antora.org/antora/latest/extend/extensions/[Antora Extensions]

package/extensions/generate-rp-connect-info.js

@@ -13,10 +13,9 @@ module.exports.register = function ({ config }) {
 
   async function loadOctokit() {
     const { Octokit } = await import('@octokit/rest');
-    if (!process.env.REDPANDA_GITHUB_TOKEN) return new Octokit()
-    return new Octokit({
-      auth: process.env.REDPANDA_GITHUB_TOKEN,
-    });
+    const { getGitHubToken } = require('../cli-utils/github-token')
+    const token = getGitHubToken()
+    return token ? new Octokit({ auth: token }) : new Octokit();
   }
 
   this.once('contentClassified', async ({ contentCatalog }) => {

package/extensions/version-fetcher/get-latest-console-version.js

@@ -1,35 +1,46 @@
+const { retryWithBackoff, isRetryableGitHubError } = require('./retry-util');
+
 module.exports = async (github, owner, repo) => {
   const semver = require('semver');
-  try {
-    // Fetch all the releases from the repository
-    const releases = await github.rest.repos.listReleases({
-      owner,
-      repo,
-      page: 1,
-      per_page: 50
-    });
 
-    // Filter valid semver tags and sort them to find the highest version
-    const sortedReleases = releases.data
-      .map(release => release.tag_name)
-      .filter(tag => semver.valid(tag.replace(/^v/, '')))
-      .sort((a, b) => semver.rcompare(a.replace(/^v/, ''), b.replace(/^v/, '')));
+  return retryWithBackoff(
+    async () => {
+      // Fetch all the releases from the repository
+      const releases = await github.rest.repos.listReleases({
+        owner,
+        repo,
+        page: 1,
+        per_page: 50
+      });
+
+      // Filter valid semver tags and sort them to find the highest version
+      const sortedReleases = releases.data
+        .map(release => release.tag_name)
+        .filter(tag => semver.valid(tag.replace(/^v/, '')))
+        .sort((a, b) => semver.rcompare(a.replace(/^v/, ''), b.replace(/^v/, '')));
 
-    if (sortedReleases.length > 0) {
-      // Find the highest versions for stable and beta releases
-      const latestStableReleaseVersion = sortedReleases.find(tag => !tag.includes('-beta'));
-      const latestBetaReleaseVersion = sortedReleases.find(tag => tag.includes('-beta'));
+      if (sortedReleases.length > 0) {
+        // Find the highest versions for stable and beta releases
+        const latestStableReleaseVersion = sortedReleases.find(tag => !tag.includes('-beta'));
+        const latestBetaReleaseVersion = sortedReleases.find(tag => tag.includes('-beta'));
 
-      return {
-        latestStableRelease: latestStableReleaseVersion || null,
-        latestBetaRelease: latestBetaReleaseVersion || null
-      };
-    } else {
-      console.log("No valid semver releases found.");
-      return { latestStableRelease: null, latestBetaRelease: null };
+        return {
+          latestStableRelease: latestStableReleaseVersion || null,
+          latestBetaRelease: latestBetaReleaseVersion || null
+        };
+      } else {
+        console.log("No valid semver releases found.");
+        return { latestStableRelease: null, latestBetaRelease: null };
+      }
+    },
+    {
+      maxRetries: 3,
+      initialDelay: 1000,
+      shouldRetry: isRetryableGitHubError,
+      operationName: `Fetch Console version from ${owner}/${repo}`
     }
-  } catch (error) {
-    console.error('Failed to fetch release information:', error);
+  ).catch(error => {
+    console.error('Failed to fetch release information after retries:', error);
     return { latestStableRelease: null, latestBetaRelease: null };
-  }
+  });
 };

package/extensions/version-fetcher/get-latest-redpanda-version.js

@@ -1,64 +1,75 @@
+const { retryWithBackoff, isRetryableGitHubError } = require('./retry-util');
+
 module.exports = async (github, owner, repo) => {
   const semver = require('semver');
-  try {
-    // Fetch all the releases from the repository
-    const releases = await github.rest.repos.listReleases({
-      owner,
-      repo,
-      page: 1,
-      per_page: 50
-    });
-
-    // Filter valid semver tags and sort them to find the highest version
-    const sortedReleases = releases.data
-      .filter(release => semver.valid(release.tag_name.replace(/^v/, '')))
-      .sort((a, b) => semver.rcompare(
-        a.tag_name.replace(/^v/, ''),
-        b.tag_name.replace(/^v/, '')
-      ));
-
-    // Find latest non-RC release that is NOT a draft
-    const latestRedpandaRelease = sortedReleases.find(
-      release => !release.tag_name.includes('-rc') && !release.draft
-    );
-
-    // Find latest RC release (can be draft or not, adjust if needed)
-    const latestRcRelease = sortedReleases.find(
-      release => release.tag_name.includes('-rc')
-    );
 
-    let latestRedpandaReleaseCommitHash = null;
-    if (latestRedpandaRelease) {
-      const commitData = await github.rest.git.getRef({
+  return retryWithBackoff(
+    async () => {
+      // Fetch all the releases from the repository
+      const releases = await github.rest.repos.listReleases({
         owner,
         repo,
-        ref: `tags/${latestRedpandaRelease.tag_name}`
+        page: 1,
+        per_page: 50
       });
-      latestRedpandaReleaseCommitHash = commitData.data.object.sha;
-    }
 
-    let latestRcReleaseCommitHash = null;
-    if (latestRcRelease) {
-      const rcCommitData = await github.rest.git.getRef({
-        owner,
-        repo,
-        ref: `tags/${latestRcRelease.tag_name}`
-      });
-      latestRcReleaseCommitHash = rcCommitData.data.object.sha;
-    }
+      // Filter valid semver tags and sort them to find the highest version
+      const sortedReleases = releases.data
+        .filter(release => semver.valid(release.tag_name.replace(/^v/, '')))
+        .sort((a, b) => semver.rcompare(
+          a.tag_name.replace(/^v/, ''),
+          b.tag_name.replace(/^v/, '')
+        ));
+
+      // Find latest non-RC release that is NOT a draft
+      const latestRedpandaRelease = sortedReleases.find(
+        release => !release.tag_name.includes('-rc') && !release.draft
+      );
+
+      // Find latest RC release (can be draft or not, adjust if needed)
+      const latestRcRelease = sortedReleases.find(
+        release => release.tag_name.includes('-rc')
+      );
 
-    return {
-      latestRedpandaRelease: latestRedpandaRelease ? {
-        version: latestRedpandaRelease.tag_name,
-        commitHash: latestRedpandaReleaseCommitHash.substring(0, 7)
-      } : null,
-      latestRcRelease: latestRcRelease ? {
-        version: latestRcRelease.tag_name,
-        commitHash: latestRcReleaseCommitHash.substring(0, 7)
-      } : null
-    };
-  } catch (error) {
-    console.error('Failed to fetch Redpanda release information:', error);
+      let latestRedpandaReleaseCommitHash = null;
+      if (latestRedpandaRelease) {
+        const commitData = await github.rest.git.getRef({
+          owner,
+          repo,
+          ref: `tags/${latestRedpandaRelease.tag_name}`
+        });
+        latestRedpandaReleaseCommitHash = commitData.data.object.sha;
+      }
+
+      let latestRcReleaseCommitHash = null;
+      if (latestRcRelease) {
+        const rcCommitData = await github.rest.git.getRef({
+          owner,
+          repo,
+          ref: `tags/${latestRcRelease.tag_name}`
+        });
+        latestRcReleaseCommitHash = rcCommitData.data.object.sha;
+      }
+
+      return {
+        latestRedpandaRelease: latestRedpandaRelease ? {
+          version: latestRedpandaRelease.tag_name,
+          commitHash: latestRedpandaReleaseCommitHash.substring(0, 7)
+        } : null,
+        latestRcRelease: latestRcRelease ? {
+          version: latestRcRelease.tag_name,
+          commitHash: latestRcReleaseCommitHash.substring(0, 7)
+        } : null
+      };
+    },
+    {
+      maxRetries: 3,
+      initialDelay: 1000,
+      shouldRetry: isRetryableGitHubError,
+      operationName: `Fetch Redpanda version from ${owner}/${repo}`
+    }
+  ).catch(error => {
+    console.error('Failed to fetch Redpanda release information after retries:', error);
     return { latestRedpandaRelease: null, latestRcRelease: null };
-  }
+  });
 };

package/extensions/version-fetcher/retry-util.js

@@ -0,0 +1,88 @@
+/**
+ * Retry utility with exponential backoff
+ *
+ * @param {Function} fn - Async function to retry
+ * @param {Object} options - Retry options
+ * @param {number} options.maxRetries - Maximum number of retries (default: 3)
+ * @param {number} options.initialDelay - Initial delay in ms (default: 1000)
+ * @param {number} options.maxDelay - Maximum delay in ms (default: 10000)
+ * @param {Function} options.shouldRetry - Function to determine if error is retryable (default: all errors)
+ * @param {string} options.operationName - Name for logging
+ * @returns {Promise<*>} Result of the function
+ */
+async function retryWithBackoff(fn, options = {}) {
+  const {
+    maxRetries = 3,
+    initialDelay = 1000,
+    maxDelay = 10000,
+    shouldRetry = () => true,
+    operationName = 'operation'
+  } = options;
+
+  let lastError;
+
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    try {
+      return await fn();
+    } catch (error) {
+      lastError = error;
+
+      // Don't retry if this is the last attempt or error is not retryable
+      if (attempt === maxRetries || !shouldRetry(error)) {
+        throw error;
+      }
+
+      // Calculate delay with exponential backoff and jitter
+      const baseDelay = Math.min(initialDelay * Math.pow(2, attempt), maxDelay);
+      const jitter = Math.random() * 0.3 * baseDelay; // Add up to 30% jitter
+      const delay = Math.floor(baseDelay + jitter);
+
+      console.error(`⚠️  ${operationName} failed (attempt ${attempt + 1}/${maxRetries + 1}): ${error.message}`);
+      console.error(`   Retrying in ${delay}ms...`);
+
+      // Wait before retrying
+      await new Promise(resolve => setTimeout(resolve, delay));
+    }
+  }
+
+  throw lastError;
+}
+
+/**
+ * Determine if a GitHub API error is retryable
+ * @param {Error} error - The error to check
+ * @returns {boolean} True if the error is retryable
+ */
+function isRetryableGitHubError(error) {
+  // Retry on network errors
+  if (error.code === 'ECONNRESET' ||
+      error.code === 'ETIMEDOUT' ||
+      error.code === 'ENOTFOUND' ||
+      error.code === 'EAI_AGAIN') {
+    return true;
+  }
+
+  // Retry on rate limit errors (though they should have a retry-after)
+  if (error.status === 403 && error.message?.includes('rate limit')) {
+    return true;
+  }
+
+  // Retry on 5xx server errors
+  if (error.status >= 500 && error.status < 600) {
+    return true;
+  }
+
+  // Retry on specific 4xx errors that might be transient
+  if (error.status === 408 || // Request Timeout
+      error.status === 429) {  // Too Many Requests
+    return true;
+  }
+
+  // Don't retry on other errors (4xx client errors, auth issues, etc.)
+  return false;
+}
+
+module.exports = {
+  retryWithBackoff,
+  isRetryableGitHubError
+};

package/extensions/version-fetcher/set-latest-version.js

@@ -8,8 +8,11 @@ module.exports.register = function ({ config }) {
   const GetLatestConnectVersion = require('./get-latest-connect');
   const logger = this.getLogger('set-latest-version-extension');
 
-  if (!process.env.REDPANDA_GITHUB_TOKEN) {
-    logger.warn('REDPANDA_GITHUB_TOKEN environment variable not set. Attempting unauthenticated request.');
+  const { getGitHubToken } = require('../../cli-utils/github-token');
+  const token = getGitHubToken();
+
+  if (!token) {
+    logger.warn('GitHub token not set (REDPANDA_GITHUB_TOKEN, GITHUB_TOKEN, or GH_TOKEN). Attempting unauthenticated request.');
   }
 
   this.on('contentClassified', async ({ contentCatalog }) => {
@@ -22,7 +25,7 @@ module.exports.register = function ({ config }) {
     const githubOptions = {
       userAgent: 'Redpanda Docs',
       baseUrl: 'https://api.github.com',
-      auth: process.env.REDPANDA_GITHUB_TOKEN || undefined,
+      auth: token || undefined,
     };
     const github = new OctokitWithRetries(githubOptions);
     const dockerNamespace = 'redpandadata'