@redpanda-data/docs-extensions-and-macros 3.4.0 → 3.4.1

package/extensions/find-related-docs.js

@@ -4,67 +4,86 @@ module.exports.register = function ({ config }) {
   const logger = this.getLogger('related-docs-extension');
 
   this.on('documentsConverted', async ({ contentCatalog, siteCatalog }) => {
-    const labs = contentCatalog.findBy({ component: 'redpanda-labs', family: 'page' });
-    labs.forEach((labPage) => {
-      const relatedDocs = []
-      const relatedLabs = []
-      const sourceAttributes = labPage.asciidoc.attributes
+    // Find the latest version of each component
+    const latestVersions = {};
+    contentCatalog.getComponents().forEach(component => {
+      latestVersions[component.name] = component.latest.version;
+    });
+
+    // Retrieve all documents and labs from the latest versions of each component
+    const allDocs = [];
+    const allLabs = contentCatalog.findBy({ component: 'redpanda-labs', family: 'page', version: latestVersions['redpanda-labs'] });
+
+    Object.keys(latestVersions).forEach(component => {
+      if (component === 'redpanda-labs') return;
+      allDocs.push(...contentCatalog.findBy({ component, family: 'page', version: latestVersions[component] }));
+    });
+    allLabs.forEach((labPage) => {
+      const relatedDocs = new Set();
+      const relatedLabs = new Set();
+      const sourceAttributes = labPage.asciidoc.attributes;
       const pageCategories = sourceAttributes['page-categories'];
       if (!pageCategories) return;
       const sourceCategoryList = pageCategories.split(',').map(c => c.trim());
-      const sourceDeploymentType = getDeploymentType(sourceAttributes)
-      const docs = contentCatalog.findBy({ component: 'ROOT', family: 'page' });
-      docs.forEach((docPage) => {
-        const related = findRelated(docPage, sourceCategoryList, sourceDeploymentType, logger)
-        related && relatedDocs.push(related)
-      })
-      labs.forEach((targetLabPage) => {
+      const sourceDeploymentType = getDeploymentType(sourceAttributes);
+      const docs = contentCatalog.findBy({ component: 'ROOT', family: 'page', version: latestVersions['ROOT'] });
+
+      allDocs.forEach((docPage) => {
+        const related = findRelated(docPage, sourceCategoryList, sourceDeploymentType, logger);
+        if (related) relatedDocs.add(JSON.stringify(related));
+      });
+
+      allLabs.forEach((targetLabPage) => {
         if (targetLabPage === labPage) return;
 
         const related = findRelated(targetLabPage, sourceCategoryList, sourceDeploymentType, logger);
-        if (related) relatedLabs.push(related);
+        if (related) relatedLabs.add(JSON.stringify(related));
       });
 
+      // Convert Sets back to arrays and remove duplicates
+      const uniqueRelatedDocs = Array.from(relatedDocs).map(item => JSON.parse(item));
+      const uniqueRelatedLabs = Array.from(relatedLabs).map(item => JSON.parse(item));
+
       // Store related docs and labs in the lab page attributes
-      if (relatedDocs.length > 0) {
-        labPage.asciidoc.attributes['page-related-docs'] = JSON.stringify(relatedDocs);
+      if (uniqueRelatedDocs.length > 0) {
+        labPage.asciidoc.attributes['page-related-docs'] = JSON.stringify(uniqueRelatedDocs);
       }
 
-      if (relatedLabs.length > 0) {
-        labPage.asciidoc.attributes['page-related-labs'] = JSON.stringify(relatedLabs);
+      if (uniqueRelatedLabs.length > 0) {
+        labPage.asciidoc.attributes['page-related-labs'] = JSON.stringify(uniqueRelatedLabs);
       }
 
-      if (relatedDocs.length > 0 || relatedLabs.length > 0) {
+      if (uniqueRelatedDocs.length > 0 || uniqueRelatedLabs.length > 0) {
         logger.info(`Set related docs and labs attributes for ${labPage.asciidoc.doctitle}`);
       }
-    })
-  })
-}
+    });
+  });
+};
 
 function findRelated(docPage, sourceCategoryList, sourceDeploymentType, logger) {
-  const targetAttributes = docPage.asciidoc.attributes
+  const targetAttributes = docPage.asciidoc.attributes;
   const pageCategories = targetAttributes['page-categories'];
   if (!pageCategories) return null;
   const targetCategoryList = pageCategories.split(',').map(c => c.trim());
-  const targetDeploymentType = getDeploymentType(targetAttributes)
-  const categoryMatch = hasMatchingCategory(sourceCategoryList, targetCategoryList)
-  if (categoryMatch && (!targetDeploymentType ||sourceDeploymentType === targetDeploymentType)) {
+  const targetDeploymentType = getDeploymentType(targetAttributes);
+  const categoryMatch = hasMatchingCategory(sourceCategoryList, targetCategoryList);
+  if (categoryMatch && (!targetDeploymentType || sourceDeploymentType === targetDeploymentType)) {
     return {
       title: docPage.asciidoc.doctitle,
       url: docPage.pub.url,
-    }
+    };
   }
-  return null
+  return null;
 }
 
-function getDeploymentType (attributes) {
+function getDeploymentType(attributes) {
   return attributes['env-kubernetes'] ? 'Kubernetes'
     : attributes['env-linux'] ? 'Linux'
       : attributes['env-docker'] ? 'Docker'
-        : attributes.cloud ? 'Redpanda Cloud' 
-        : ''
+        : attributes.cloud ? 'Redpanda Cloud'
+        : '';
 }
 
-function hasMatchingCategory (sourcePageCategories, targetPageCategories) {
-  return sourcePageCategories.every((category) => targetPageCategories.includes(category))
-}
+function hasMatchingCategory(sourcePageCategories, targetPageCategories) {
+  return sourcePageCategories.every(category => targetPageCategories.includes(category));
+}

package/extensions/unlisted-pages.js

@@ -1,54 +1,68 @@
+'use strict';
+
 module.exports.register = function ({ config }) {
-  const { addToNavigation, unlistedPagesHeading = 'Unlisted Pages' } = config
-  const logger = this.getLogger('unlisted-pages-extension')
-  this
-    .on('navigationBuilt', ({ navigationCatalog, contentCatalog }) => {
-      contentCatalog.getComponents().forEach(({ versions }) => {
-        versions.forEach(({ name: component, version, navigation: nav, url: defaultUrl }) => {
-          if (component === 'api') return;
-          if (!nav) return
-          const navEntriesByUrl = getNavEntriesByUrl(nav)
-          const unlistedPages = contentCatalog
-            .findBy({ component, version, family: 'page' })
-            .filter((page) => page.out)
-            .reduce((collector, page) => {
-              if ((page.pub.url in navEntriesByUrl) || page.pub.url === defaultUrl) return collector
-              logger.warn({ file: page.src, source: page.src.origin }, 'detected unlisted page')
-              return collector.concat(page)
-            }, [])
-          if (unlistedPages.length && component === 'redpanda-connect') {
-            // Some component pages for Redpanda Connect  are autogenerated. This function tries to add unlisted component pages to the nav in case a new one gets created without updating the nav.
-            addRedpandaConnectPagesToNav(nav[0].items, unlistedPages)
-          } else if (unlistedPages.length && addToNavigation) {
-            nav.push({
-              content: unlistedPagesHeading,
-              items: unlistedPages.map((page) => {
-                return { content: page.asciidoc.navtitle, url: page.pub.url, urlType: 'internal' }
-              }),
-              root: true,
-            })
-          }
-        })
-      })
-    })
-}
+  const { addToNavigation, unlistedPagesHeading = 'Unlisted Pages' } = config;
+  const logger = this.getLogger('unlisted-pages-extension');
+
+  this.on('navigationBuilt', ({ siteCatalog, contentCatalog }) => {
+    contentCatalog.getComponents().forEach(({ versions }) => {
+      versions.forEach(({ name: component, version, navigation: nav, url: defaultUrl }) => {
+        if (component === 'api') return;
+        if (!nav) return;
+        const currentComponent = contentCatalog.getComponent(component);
+        const prerelease = currentComponent && currentComponent.latestPrerelease ? currentComponent.latestPrerelease : false;
+
+        const navEntriesByUrl = getNavEntriesByUrl(nav);
+        const unlistedPages = contentCatalog
+          .findBy({ component, version, family: 'page' })
+          .reduce((collector, page) => {
+            if (siteCatalog.unpublishedPages.includes(page.pub.url)) {
+              logger.info({ file: page.src, source: page.src.origin }, 'removing unpublished page from nav tree');
+              removePageFromNav(nav, page.pub.url); // Remove the page from navigationCatalog
+              return collector; // Skip adding this page to the collector
+            }
 
-function getNavEntriesByUrl (items = [], accum = {}) {
+            if (!(page.pub.url in navEntriesByUrl) && page.pub.url !== defaultUrl) {
+              logger.warn({ file: page.src, source: page.src.origin }, 'detected unlisted page');
+              return collector.concat(page);
+            }
+
+            return collector;
+          }, []);
+
+        if (unlistedPages.length && component === 'redpanda-connect') {
+          // Some component pages for Redpanda Connect are autogenerated. This function tries to add unlisted component pages to the nav in case a new one gets created without updating the nav.
+          addRedpandaConnectPagesToNav(nav[0].items, unlistedPages);
+        } else if (unlistedPages.length && addToNavigation) {
+          nav.push({
+            content: unlistedPagesHeading,
+            items: unlistedPages.map((page) => {
+              return { content: page.asciidoc.navtitle, url: page.pub.url, urlType: 'internal' };
+            }),
+            root: true,
+          });
+        }
+      });
+    });
+  });
+};
+
+function getNavEntriesByUrl(items = [], accum = {}) {
   items.forEach((item) => {
-    if (item.urlType === 'internal') accum[item.url.split('#')[0]] = item
-    getNavEntriesByUrl(item.items, accum)
-  })
-  return accum
+    if (item.urlType === 'internal') accum[item.url.split('#')[0]] = item;
+    getNavEntriesByUrl(item.items, accum);
+  });
+  return accum;
 }
 
 function addRedpandaConnectPagesToNav(navItems, pages) {
   // get the Components nav section
   let componentsSection = navItems.find(item => item.content === 'Components');
-  if (!componentsSection) return
+  if (!componentsSection) return;
 
   pages.forEach(page => {
     const dirname = page.out.dirname;
-    if (!dirname.includes('/components/')) return
+    if (!dirname.includes('/components/')) return;
     const heading = page.asciidoc.attributes.doctitle;
     const pathParts = dirname.split('/').slice(2); // Get the type
     // get existing nav items inside the Components tree
@@ -66,4 +80,16 @@ function addRedpandaConnectPagesToNav(navItems, pages) {
 
     currentLevel.push({ content: page.asciidoc.navtitle || page.src.stem, url: page.pub.url, urlType: 'internal' });
   });
-}
+}
+
+function removePageFromNav(navItems, urlToRemove) {
+  // Remove the page from the navigation items
+  for (let i = navItems.length - 1; i >= 0; i--) {
+    const item = navItems[i];
+    if (item.url === urlToRemove) {
+      navItems.splice(i, 1);
+    } else if (item.items && item.items.length > 0) {
+      removePageFromNav(item.items, urlToRemove);
+    }
+  }
+}

package/extensions/unpublish-pages.js

@@ -1,9 +1,34 @@
+'use strict';
+
 module.exports.register = function () {
-  this.on('documentsConverted', ({ contentCatalog }) => {
-    contentCatalog.getPages((page) => page.out).forEach((page) => {
-      if (page.asciidoc?.attributes['page-unpublish'] != null || page.asciidoc?.attributes['page-layout'] === 'api-partial') {
-        delete page.out
+  this.on('documentsConverted', ({ siteCatalog, contentCatalog }) => {
+    // Get all pages that have an `out` property
+    const pages = contentCatalog.getPages((page) => page.out);
+    let componentName;
+    siteCatalog.unpublishedPages = []
+
+    // Iterate over each page
+    pages.forEach((page) => {
+      // Retrieve the component associated with the page
+      componentName = page.asciidoc?.attributes['page-component-name'];
+      const component = contentCatalog.getComponent(componentName);
+
+      // Check if the component has a latest prerelease
+      const prerelease = component && component.latestPrerelease ? component.latestPrerelease : false;
+
+      // Check the conditions for unpublishing the page
+      const shouldUnpublish = (
+        page.asciidoc?.attributes['page-unpublish'] ||
+        (page.asciidoc?.attributes['publish-only-during-beta']
+        && !prerelease // No beta version available, meaning the beta period has ended
+      )
+      );
+
+      // If any condition is met, unpublish the page
+      if (shouldUnpublish) {
+        siteCatalog.unpublishedPages.push(page.pub.url)
+        delete page.out;
       }
-    })
-  })
-}
+    });
+  });
+};

package/extensions/version-fetcher/get-latest-redpanda-version.js

@@ -27,15 +27,17 @@ module.exports = async () => {
       per_page: 50
     });
 
-    // Filter valid semver tags and sort them to find the highest version
+    // Filter valid semver tags, exclude drafts, and sort them to find the highest version
     const sortedReleases = releases.data
+      .filter(release => !release.draft)
       .map(release => release.tag_name.replace(/^v/, ''))
-      .filter(tag => semver.valid(tag) && !tag.match(/rc\d+$/))  // Exclude 'rc-{number}' suffixes
+      .filter(tag => semver.valid(tag))
       // Sort in descending order to get the highest version first
       .sort(semver.rcompare);
 
     if (sortedReleases.length > 0) {
-      const latestRedpandaReleaseVersion = sortedReleases[0];
+      const latestRedpandaReleaseVersion = sortedReleases.find(tag => !tag.includes('rc'));
+      const latestRcReleaseVersion = sortedReleases.find(tag => tag.includes('rc'));
 
       // Get the commit hash for the highest version tag
       const commitData = await github.rest.git.getRef({
@@ -45,13 +47,32 @@ module.exports = async () => {
       });
       const latestRedpandaReleaseCommitHash = commitData.data.object.sha;
 
-      return [latestRedpandaReleaseVersion, latestRedpandaReleaseCommitHash.substring(0, 7)];
+      let latestRcReleaseCommitHash = null;
+      if (latestRcReleaseVersion) {
+        const rcCommitData = await github.rest.git.getRef({
+          owner,
+          repo,
+          ref: `tags/v${latestRcReleaseVersion}`
+        });
+        latestRcReleaseCommitHash = rcCommitData.data.object.sha;
+      }
+
+      return {
+        latestRedpandaRelease: {
+          version: latestRedpandaReleaseVersion,
+          commitHash: latestRedpandaReleaseCommitHash.substring(0, 7)
+        },
+        latestRcRelease: latestRcReleaseVersion ? {
+          version: latestRcReleaseVersion,
+          commitHash: latestRcReleaseCommitHash.substring(0, 7)
+        } : null
+      };
     } else {
       console.log("No valid semver releases found for Redpanda.");
-      return [null, null];
+      return { latestRedpandaRelease: null, latestRcRelease: null };
     }
   } catch (error) {
     console.error('Failed to fetch Redpanda release information:', error);
-    return [null, null];
+    return { latestRedpandaRelease: null, latestRcRelease: null };
   }
 };

package/extensions/version-fetcher/set-latest-version.js

@@ -39,18 +39,23 @@ module.exports.register = function ({ config }) {
             asciidoc.attributes['latest-connect-version'] = `${LatestConnectVersion}@`
             logger.info(`Set Redpanda Connect version to ${LatestConnectVersion} in ${name} ${version}`)
           }
+          if (LatestRedpandaVersion && LatestRedpandaVersion.latestRcRelease) {
+            asciidoc.attributes['redpanda-beta-version'] = `${LatestRedpandaVersion.latestRcRelease.version}@`
+            asciidoc.attributes['redpanda-beta-commit'] = `${LatestRedpandaVersion.latestRcRelease.commitHash}@`
+            logger.info(`Updated to latest Redpanda RC version: ${LatestRedpandaVersion.latestRcRelease.version} with commit: ${LatestRedpandaVersion.latestRcRelease.commitHash}`)
+        }
         })
 
         if (!component.latest.asciidoc) {
           component.latest.asciidoc = { attributes: {} }
         }
 
-        if (LatestRedpandaVersion && semver.valid(LatestRedpandaVersion[0])) {
+        if (LatestRedpandaVersion && semver.valid(LatestRedpandaVersion.latestRedpandaRelease.version)) {
           let currentVersion = component.latest.asciidoc.attributes['full-version'] || '0.0.0'
-          if (semver.gt(LatestRedpandaVersion[0], currentVersion)) {
-            component.latest.asciidoc.attributes['full-version'] = `${LatestRedpandaVersion[0]}@`
-            component.latest.asciidoc.attributes['latest-release-commit'] = `${LatestRedpandaVersion[1]}@`
-            logger.info(`Updated to latest Redpanda version: ${LatestRedpandaVersion[0]} with commit: ${LatestRedpandaVersion[1]}`)
+          if (semver.gt(LatestRedpandaVersion.latestRedpandaRelease.version, currentVersion)) {
+            component.latest.asciidoc.attributes['full-version'] = `${LatestRedpandaVersion.latestRedpandaRelease.version}@`
+            component.latest.asciidoc.attributes['latest-release-commit'] = `${LatestRedpandaVersion.latestRedpandaRelease.commitHash}@`
+            logger.info(`Updated to latest Redpanda version: ${LatestRedpandaVersion.latestRedpandaRelease.version} with commit: ${LatestRedpandaVersion.latestRedpandaRelease.commitHash}`)
           }
         }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redpanda-data/docs-extensions-and-macros",
-  "version": "3.4.0",
+  "version": "3.4.1",
   "description": "Antora extensions and macros developed for Redpanda documentation.",
   "keywords": [
     "antora",