@redpanda-data/docs-extensions-and-macros 3.6.6 → 3.7.1

package/README.adoc

@@ -226,7 +226,7 @@ antora:
 
 === Global attributes
 
-This extension collects Asciidoc attributes from the {url-playbook}[`shared` component] and makes them available to all component versions. Having global attributes is useful for consistent configuration of local and production builds.
+This extension collects Asciidoc attributes from the {url-playbook}[`shared` component] or a local YAML file and makes them available to all component versions. Having global attributes is useful for consistent configuration of local and production builds.
 
 ==== Environment variables
 
@@ -234,16 +234,21 @@ This extension does not require any environment variables.
 
 ==== Configuration options
 
-There are no configurable options for this extension.
+The extension accepts the following configuration options:
+
+attributespath (optional):: Specifies the path to a local YAML file that contains global attributes. If this is provided, the extension will load attributes from this file first. If this path is not provided or no valid attributes are found in the file, the extension will fall back to loading attributes from the `shared` component.
 
 ==== Registration example
 
-```yaml
+```yml
 antora:
   extensions:
   - require: '@redpanda-data/docs-extensions-and-macros/extensions/add-global-attributes'
+    attributespath: './local-attributes.yml'
 ```
 
+In this example, the `attributespath` option points to a local YAML file (`./local-attributes.yml`), which contains the global attributes. The extension will load attributes from this file first before falling back to the `shared` component.
+
 === Produce redirects (customization of core Antora)
 
 This extension replaces the default https://gitlab.com/antora/antora/-/tree/v3.1.x/packages/redirect-producer[`produceRedirects()` function] in Antora to handle redirect loops caused by https://docs.antora.org/antora/latest/page/page-aliases/[page aliases]. Normally, page aliases in Antora are used to resolve outdated links without causing issues. However, with https://docs.antora.org/antora/latest/playbook/urls-html-extension-style/#html-extension-style-key[`indexify`], the same URL may inadvertently be used for both the source and target of a redirect, leading to loops. This problem is https://antora.zulipchat.com/#narrow/stream/282400-users/topic/Redirect.20Loop.20Issue.20with.20Page.20Renaming.20and.20Indexify/near/433691700[recognized as a bug] in core Antora. For example, creating a page alias for `modules/manage/security/authorization.adoc` to point to `modules/manage/security/authorization/index.adoc' can lead to a redirect loop where `manage/security/authorization/` points to `manage/security/authorization/`. Furthermore, omitting the alias would lead to `xref not found` errors because Antora relies on the alias to resolve the old xrefs. This extension is necessary until such behaviors are natively supported or fixed in Antora core.
@@ -305,16 +310,33 @@ This extension does not require any environment variables.
 
 ==== Configuration options
 
-There are no configurable options for this extension.
+The extension accepts the following configuration options:
+
+termspath (optional):: Specifies the path to a local directory containing term files (in `.adoc` format). If this path is provided, the extension will attempt to load terms from this directory first. If this path is not provided or no valid terms are found in the specified directory, the extension will fall back to loading terms from the `shared` component.
+
+Term files should follow the following structure:
+
+```asciidoc
+:category: Documentation
+:hover-text: This is a description of the term.
+:link: https://example.com
+
+== Term Title
+
+This is the detailed description of the term.
+```
 
 ==== Registration example
 
-```yaml
+```yml
 antora:
   extensions:
-  - '@redpanda-data/docs-extensions-and-macros/extensions/aggregate-terms'
+  - require: '@redpanda-data/docs-extensions-and-macros/extensions/aggregate-terms'
+    termspath: './local-terms/'
 ```
 
+In this example, the `termspath` option points to a local directory (./local-terms/), where the term files are stored. The extension will load terms from this directory first before falling back to the `shared` component.
+
 === Unlisted pages
 
 This extension identifies and logs any pages that aren't listed in the navigation (nav) file of each version of each component. It then optionally adds these unlisted pages to the end of the navigation tree under a configurable heading.

package/extensions/add-global-attributes.js

@@ -4,33 +4,83 @@
  *    - require: ./extensions/add-global-attributes.js
 */
 
+const fs = require('fs');
+const path = require('path');
+const yaml = require('js-yaml');
+const _ = require('lodash');
+
+const ATTRIBUTES_PATH = 'modules/ROOT/partials/';  // Default path within the 'shared' component
+
 module.exports.register = function ({ config }) {
-  const yaml = require('js-yaml');
   const logger = this.getLogger('global-attributes-extension');
-  const chalk = require('chalk')
-  const _ = require('lodash');
+  const chalk = require('chalk');
+
+  /**
+   * Load global attributes from a specified local file if provided.
+   */
+  function loadLocalAttributes(siteCatalog, localAttributesFile) {
+    try {
+      const resolvedPath = path.resolve(localAttributesFile);
+      if (!fs.existsSync(resolvedPath)) {
+        logger.warn(`Local attributes file "${localAttributesFile}" does not exist.`);
+        return false;  // Return false if the local file doesn't exist
+      }
+
+      const fileContents = fs.readFileSync(resolvedPath, 'utf8');
+      const fileAttributes = yaml.load(fileContents);
+
+      siteCatalog.attributeFile = _.merge({}, fileAttributes);
+      console.log(chalk.green(`Loaded global attributes from local file "${localAttributesFile}".`));
+      return true;  // Return true if the local attributes were successfully loaded
+
+    } catch (error) {
+      logger.error(`Error loading local attributes from "${localAttributesFile}": ${error.message}`);
+      return false;  // Return false if an error occurs
+    }
+  }
 
   this.on('contentAggregated', ({ siteCatalog, contentAggregate }) => {
     try {
-      for (const component of contentAggregate) {
-        if (component.name === 'shared') {
-          const attributeFiles = component.files.filter(file => file.path.startsWith('modules/ROOT/partials/') && file.path.endsWith('.yml'));
-          if (!attributeFiles.length) {
-            logger.warn("No YAML attributes files found in 'shared' component.");
-          } else {
-            siteCatalog.attributeFile = attributeFiles.reduce((acc, file) => {
-              const fileAttributes = yaml.load(file.contents.toString('utf8'));
-              return _.merge(acc, fileAttributes);
-            }, {});
-            console.log(chalk.green('Loaded global attributes from shared component.'));
+      let attributesLoaded = false;
+
+      // Try to load attributes from the local file if provided
+      if (config.attributespath) {
+        attributesLoaded = loadLocalAttributes(siteCatalog, config.attributespath);
+      }
+
+      // If no local attributes were loaded, fallback to the 'shared' component
+      if (!attributesLoaded) {
+        let sharedComponentFound = false;
+
+        for (const component of contentAggregate) {
+          if (component.name === 'shared') {
+            sharedComponentFound = true;
+            const attributeFiles = component.files.filter(file => (file.path.startsWith(ATTRIBUTES_PATH) && (file.path.endsWith('.yml') || file.path.endsWith('.yaml'))));
+
+            if (!attributeFiles.length) {
+              logger.warn(`No YAML attributes files found in 'shared' component in ${ATTRIBUTES_PATH}.`);
+            } else {
+              siteCatalog.attributeFile = attributeFiles.reduce((acc, file) => {
+                const fileAttributes = yaml.load(file.contents.toString('utf8'));
+                return _.merge(acc, fileAttributes);
+              }, {});
+              console.log(chalk.green('Loaded global attributes from shared component.'));
+            }
+            break;
           }
-          break;
+        }
+
+        // If no 'shared' component is found, log a warning
+        if (!sharedComponentFound) {
+          logger.warn("No component named 'shared' found in the content and no valid local attributes file provided. Global attributes will not be available. You may see Asciidoc warnings about missing attributes and some behavior may not work as expected.");
         }
       }
+
     } catch (error) {
       logger.error(`Error loading attributes: ${error.message}`);
     }
   })
+
   .on('contentClassified', async ({ siteCatalog, contentCatalog }) => {
     const components = await contentCatalog.getComponents();
     for (let i = 0; i < components.length; i++) {
@@ -39,7 +89,7 @@ module.exports.register = function ({ config }) {
         if (siteCatalog.attributeFile) {
           asciidoc.attributes = _.merge({}, siteCatalog.attributeFile, asciidoc.attributes);
         }
-      })
+      });
     }
-  })
-}
+  });
+};

package/extensions/aggregate-terms.js

@@ -4,10 +4,18 @@
  *    - require: ./extensions/aggregate-terms.js
 */
 
+const fs = require('fs');
+const path = require('path');
+
+const TERMS_PATH = 'modules/terms/partials/';  // Default path within the 'shared' component
+
 module.exports.register = function ({ config }) {
   const logger = this.getLogger('term-aggregation-extension');
   const chalk = require('chalk');
 
+  /**
+   * Function to process term content, extracting hover text and links.
+   */
   function processTermContent(termContent) {
     const hoverTextMatch = termContent.match(/:hover-text: (.*)/);
     const hoverText = hoverTextMatch ? hoverTextMatch[1] : '';
@@ -27,32 +35,91 @@ module.exports.register = function ({ config }) {
     return termContent;
   }
 
+  /**
+   * Load terms from a specified local path if provided.
+   */
+  function loadLocalTerms(siteCatalog, termsPath) {
+    try {
+      const resolvedPath = path.resolve(termsPath);
+      if (!fs.existsSync(resolvedPath)) {
+        logger.warn(`Local terms path "${termsPath}" does not exist.`);
+        return false;
+      }
+
+      const termFiles = fs.readdirSync(resolvedPath).filter(file => file.endsWith('.adoc'));
+      if (!termFiles.length) {
+        logger.warn(`No term files found in local path "${termsPath}".`);
+        return false;
+      }
+
+      termFiles.forEach(file => {
+        const filePath = path.join(resolvedPath, file);
+        const termContent = fs.readFileSync(filePath, 'utf8');
+        const categoryMatch = /:category: (.*)/.exec(termContent);
+        const category = categoryMatch ? categoryMatch[1] : 'Miscellaneous';
+
+        const formattedCategory = category.charAt(0).toUpperCase() + category.slice(1);
+
+        if (!siteCatalog.termsByCategory[formattedCategory]) {
+          siteCatalog.termsByCategory[formattedCategory] = [];
+        }
+
+        siteCatalog.termsByCategory[formattedCategory].push({ name: file, content: termContent });
+      });
+
+      console.log(chalk.green(`Categorized terms from local terms path "${termsPath}".`));
+      return true;
+
+    } catch (error) {
+      logger.error(`Error loading local terms from "${termsPath}": ${error.message}`);
+      return false;
+    }
+  }
+
   this.on('contentAggregated', ({ siteCatalog, contentAggregate }) => {
     try {
       siteCatalog.termsByCategory = {};
+      let termsLoaded = false;
+
+      // Try to load terms from the local path if provided
+      if (config.termspath) {
+        termsLoaded = loadLocalTerms(siteCatalog, config.termspath);
+      }
 
-      for (const component of contentAggregate) {
-        if (component.name === 'shared') {
-          const termFiles = component.files.filter(file => file.path.includes('modules/terms/partials/'));
+      // If no local terms were loaded, fallback to the 'shared' component
+      if (!termsLoaded) {
+        let sharedComponentFound = false;
 
-          termFiles.forEach(file => {
-            const termContent = file.contents.toString('utf8');
-            const categoryMatch = /:category: (.*)/.exec(termContent);
-            var category = categoryMatch ? categoryMatch[1] : 'Miscellaneous'; // Default category
+        for (const component of contentAggregate) {
+          if (component.name === 'shared') {
+            sharedComponentFound = true;
+            const termFiles = component.files.filter(file => file.path.includes(TERMS_PATH));
 
-            category = category.charAt(0).toUpperCase() + category.slice(1);
+            termFiles.forEach(file => {
+              const termContent = file.contents.toString('utf8');
+              const categoryMatch = /:category: (.*)/.exec(termContent);
+              const category = categoryMatch ? categoryMatch[1] : 'Miscellaneous';
 
-            if (!siteCatalog.termsByCategory[category]) {
-              siteCatalog.termsByCategory[category] = [];
-            }
+              const formattedCategory = category.charAt(0).toUpperCase() + category.slice(1);
 
-            siteCatalog.termsByCategory[category].push({ name: file.basename, content: termContent });
-          });
+              if (!siteCatalog.termsByCategory[formattedCategory]) {
+                siteCatalog.termsByCategory[formattedCategory] = [];
+              }
 
-          console.log(chalk.green('Categorized terms from shared component.'));
-          break;
+              siteCatalog.termsByCategory[formattedCategory].push({ name: file.basename, content: termContent });
+            });
+
+            console.log(chalk.green('Categorized terms from shared component.'));
+            break;
+          }
+        }
+
+        // If no 'shared' component is found, log a warning
+        if (!sharedComponentFound) {
+          logger.warn(`No component named 'shared' found in the content and no valid local terms path provided. Terms will not be added to the glossary.`);
         }
       }
+
     } catch (error) {
       logger.error(`Error categorizing terms: ${error.message}`);
     }

package/extensions/generate-rp-connect-info.js

@@ -1,16 +1,15 @@
 'use strict'
-const https = require('https');
+const fs = require('fs');
+const path = require('path');
 const Papa = require('papaparse');
 
-const CSV_PATH = 'redpanda_connect.csv'
+const CSV_PATH = 'internal/plugins/info.csv'
 const GITHUB_OWNER = 'redpanda-data'
-const GITHUB_REPO = 'rp-connect-docs'
+const GITHUB_REPO = 'connect'
 const GITHUB_REF = 'main'
-/* const csvUrl = 'https://localhost:3000/csv';
-process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'; */
 
-module.exports.register = function ({ config,contentCatalog }) {
-    const logger = this.getLogger('redpanda-connect-info-extension');
+module.exports.register = function ({ config }) {
+  const logger = this.getLogger('redpanda-connect-info-extension');
 
   async function loadOctokit() {
     const { Octokit } = await import('@octokit/rest');
@@ -20,21 +19,30 @@ module.exports.register = function ({ config,contentCatalog }) {
     });
   }
 
-  this.once('contentClassified', async ({ siteCatalog, contentCatalog }) => {
-    const redpandaConnect = contentCatalog.getComponents().find(component => component.name === 'redpanda-connect')
-    const redpandaCloud = contentCatalog.getComponents().find(component => component.name === 'redpanda-cloud')
-    if (!redpandaConnect) return
-    const pages = contentCatalog.getPages()
+  this.once('contentClassified', async ({ contentCatalog }) => {
+    const redpandaConnect = contentCatalog.getComponents().find(component => component.name === 'redpanda-connect');
+    const redpandaCloud = contentCatalog.getComponents().find(component => component.name === 'redpanda-cloud');
+    const preview = contentCatalog.getComponents().find(component => component.name === 'preview');
+    if (!redpandaConnect) return;
+    const pages = contentCatalog.getPages();
+
     try {
-      // Fetch CSV data and parse it
-      const csvData = await fetchCSV();
+      // Fetch CSV data (either from local file or GitHub)
+      const csvData = await fetchCSV(config.csvpath);
       const parsedData = Papa.parse(csvData, { header: true, skipEmptyLines: true });
-      const enrichedData = enrichCsvDataWithUrls(parsedData, pages, logger);
-      parsedData.data = enrichedData
-      if(redpandaConnect)
+      const enrichedData = translateCsvData(parsedData, pages, logger);
+      parsedData.data = enrichedData;
+
+      if (redpandaConnect) {
         redpandaConnect.latest.asciidoc.attributes.csvData = parsedData;
-      if(redpandaCloud)
+      }
+      if (redpandaCloud) {
         redpandaCloud.latest.asciidoc.attributes.csvData = parsedData;
+      }
+      // For previewing the data on our extensions site
+      if (preview) {
+        preview.latest.asciidoc.attributes.csvData = parsedData;
+      }
 
     } catch (error) {
       logger.error('Error fetching or parsing CSV data:', error.message);
@@ -42,7 +50,22 @@ module.exports.register = function ({ config,contentCatalog }) {
     }
   });
 
-  async function fetchCSV() {
+  // Check for local CSV file first. If not found, fetch from GitHub
+  async function fetchCSV(localCsvPath) {
+    if (localCsvPath && fs.existsSync(localCsvPath)) {
+      if (path.extname(localCsvPath).toLowerCase() !== '.csv') {
+        throw new Error(`Invalid file type: ${localCsvPath}. Expected a CSV file.`);
+      }
+      logger.info(`Loading CSV data from local file: ${localCsvPath}`);
+      return fs.readFileSync(localCsvPath, 'utf8');
+    } else {
+      logger.info('Local CSV file not found. Fetching from GitHub...');
+      return await fetchCsvFromGitHub();
+    }
+  }
+
+  // Fetch CSV data from GitHub
+  async function fetchCsvFromGitHub() {
     const octokit = await loadOctokit();
     try {
       const { data: fileContent } = await octokit.rest.repos.getContent({
@@ -54,33 +77,87 @@ module.exports.register = function ({ config,contentCatalog }) {
       return Buffer.from(fileContent.content, 'base64').toString('utf8');
     } catch (error) {
       console.error('Error fetching Redpanda Connect catalog from GitHub:', error);
-      return [];
+      return '';
     }
   }
 
-  function enrichCsvDataWithUrls(parsedData, connectPages, logger) {
+  /**
+   * Translates the parsed CSV data into our expected format.
+   * If "enterprise" is found in the `support` column, it is replaced with "certified" in the output.
+   *
+   * @param {object} parsedData - The CSV data parsed into an object.
+   * @param {array} pages - The list of pages to map the URLs (used for enrichment with URLs).
+   * @param {object} logger - The logger used for error handling.
+   *
+   * @returns {array} - The translated and enriched data.
+   */
+  function translateCsvData(parsedData, pages, logger) {
     return parsedData.data.map(row => {
       // Create a new object with trimmed keys and values
       const trimmedRow = Object.fromEntries(
         Object.entries(row).map(([key, value]) => [key.trim(), value.trim()])
       );
-      const connector = trimmedRow.connector;
+
+      // Map fields from the trimmed row to the desired output
+      const connector = trimmedRow.name;
       const type = trimmedRow.type;
-      let url = '';
-      for (const file of connectPages) {
+      const commercial_name = trimmedRow.commercial_name;
+      const available_connect_version = trimmedRow.version;
+      const deprecated = trimmedRow.deprecated.toLowerCase() === 'y' ? 'y' : 'n';
+      const is_cloud_supported = trimmedRow.cloud.toLowerCase() === 'y' ? 'y' : 'n';
+      const cloud_ai = trimmedRow.cloud_with_gpu.toLowerCase() === 'y' ? 'y' : 'n';
+      // Handle enterprise to certified conversion and set enterprise license flag
+      const originalSupport = trimmedRow.support.toLowerCase();
+      const support_level = originalSupport === 'enterprise' ? 'certified' : originalSupport;
+      const is_licensed = originalSupport === 'enterprise' ? 'Yes' : 'No';
+
+      // Redpanda Connect and Cloud enrichment URLs
+      let redpandaConnectUrl = '';
+      let redpandaCloudUrl = '';
+
+      // Look for both Redpanda Connect and Cloud URLs
+      for (const file of pages) {
+        const component = file.src.component;
         const filePath = file.path;
-        if (filePath.endsWith(`${connector}.adoc`) && filePath.includes(`pages/${type}s/`)) {
-          url = `../${type}s/${connector}`;
-          break;
+
+        if (
+          component === 'redpanda-connect' &&
+          filePath.endsWith(`/${connector}.adoc`) &&
+          filePath.includes(`pages/${type}s/`)
+        ) {
+          redpandaConnectUrl = file.pub.url;
+        }
+
+        // Only check for Redpanda Cloud URLs if cloud is supported
+        if (
+          is_cloud_supported === 'y' &&
+          component === 'redpanda-cloud' &&
+          filePath.endsWith(`/${connector}.adoc`) &&
+          filePath.includes(`${type}s/`)
+        ) {
+          redpandaCloudUrl = file.pub.url;
         }
       }
-      if (!url) {
-        logger.warn(`No matching URL found for connector: ${connector} of type: ${type}`);
+
+      // Log a warning if neither URL was found (only warn for missing cloud if it should support cloud)
+      if (!redpandaConnectUrl && (!redpandaCloudUrl && is_cloud_supported === 'y')) {
+        logger.info(`Docs missing for: ${connector} of type: ${type}`);
       }
+
+      // Return the translated and enriched row
       return {
-        ...trimmedRow,
-        url: url
+        connector,
+        type,
+        commercial_name,
+        available_connect_version,
+        support_level,  // "enterprise" is replaced with "certified"
+        deprecated,
+        is_cloud_supported,
+        cloud_ai,
+        is_licensed,  // "Yes" if the original support level was "enterprise"
+        redpandaConnectUrl,
+        redpandaCloudUrl,
       };
     });
   }
-}
+}

package/extensions/validate-attributes.js

@@ -11,7 +11,9 @@ module.exports.register = function ({ config }) {
 
   this.on('documentsConverted', async ({ contentCatalog, siteCatalog }) => {
     // Retrieve valid categories and subcategories from site attributes defined in add-global-attributes.js.
+    if (!siteCatalog.attributeFile) return logger.warn('No global attributes file available - skipping attribute validation. Check global-attributes-extension for errors')
     const validCategories = siteCatalog.attributeFile['page-valid-categories'];
+    if (!validCategories) return logger.warn('No page-valid-categories attribute found - skipping attribute validation')
     const categoryMap = createCategoryMap(validCategories);
     const pages = contentCatalog.findBy({ family: 'page' });
     pages.forEach((page) => {