@redpanda-data/docs-extensions-and-macros 4.13.0 → 4.13.2

package/extensions/algolia-indexer/generate-index.js

@@ -1,10 +1,8 @@
 'use strict'
 
-const { parse }   = require('node-html-parser')
-const { decode }  = require('html-entities')
-const path        = require('path')
-const URL         = require('url')
-const chalk       = require('chalk')
+const { parse } = require('node-html-parser')
+const { decode } = require('html-entities')
+const path = require('path')
 
 // Create encoder once at module scope for efficiency
 const textEncoder = new TextEncoder()
@@ -20,16 +18,22 @@ const textEncoder = new TextEncoder()
  * @param {Object} contentCatalog - The Antora content catalog, with pages and metadata.
  * @param {Object} [config={}] - Configuration options
  * @param {Boolean} config.indexLatestOnly - If true, only index the latest version of any given page.
+ * @param {Array} config.excludes - CSS selectors for elements to exclude from indexing.
  * @param {Object} config.logger - Logger to use
  * @typedef {Object} SearchIndexData
  * @returns {SearchIndexData} A data object that contains the Algolia index
  */
 function generateIndex (playbook, contentCatalog, { indexLatestOnly = false, excludes = [], logger } = {}) {
-  if (!logger) logger = process.env.NODE_ENV === 'test' ? { info: () => undefined } : console
+  // Use provided logger or create a no-op logger for tests
+  if (!logger) {
+    logger = process.env.NODE_ENV === 'test'
+      ? { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} }
+      : console
+  }
 
   const algolia = {}
 
-  console.log(chalk.cyan('Indexing...'))
+  logger.info('Starting Algolia index generation...')
   const unixTimestamp = Math.floor(Date.now() / 1000)
 
   // Select indexable pages
@@ -37,37 +41,31 @@ function generateIndex (playbook, contentCatalog, { indexLatestOnly = false, exc
     if (!page.out || page.asciidoc?.attributes?.noindex != null) return
     return {}
   })
-  if (!pages.length) return {}
 
-  // Handle the site URL
-  let siteUrl = playbook.site.url
-  if (!siteUrl) {
-    siteUrl = ''
+  if (!pages.length) {
+    logger.warn('No pages found to index')
+    return {}
   }
-  if (siteUrl.charAt(siteUrl.length - 1) === '/') {
-    siteUrl = siteUrl.substr(0, siteUrl.length - 1)
+
+  // Handle the site URL
+  let siteUrl = playbook.site.url || ''
+  if (siteUrl.endsWith('/')) {
+    siteUrl = siteUrl.slice(0, -1)
   }
   const urlPath = extractUrlPath(siteUrl)
 
-  var algoliaCount = 0
+  let algoliaCount = 0
 
-  for (var i = 0; i < pages.length; i++) {
-    const page = pages[i]
+  for (const page of pages) {
     const root = parse(
       page.contents,
       {
         blockTextElements: {
-          code: true,
-        },
+          code: true
+        }
       }
     )
 
-    /* Skip pages marked as "noindex" for "robots"
-    const noindex = root.querySelector('meta[name=robots][content=noindex]')
-    if (noindex) {
-      continue
-    }*/
-
     // Compute a flag identifying if the current page is in the
     // "current" component version.
     // When indexLatestOnly is set, we only index the current version.
@@ -78,23 +76,25 @@ function generateIndex (playbook, contentCatalog, { indexLatestOnly = false, exc
 
     if (indexLatestOnly && !isCurrent) continue
 
-    // capture the component name and version
+    // Capture the component name and version
     const cname = component.name
-    const version = page.src.origin.descriptor.prerelease ? page.src.origin.descriptor.displayVersion : page.src.version;
-
-    // handle the page keywords
-    const kw = root.querySelector('meta[name=keywords]')
-    var keywords = []
-    if (kw) {
-      keywords = kw.getAttribute('content')
-      keywords = keywords ? keywords.split(/,\s*/) : []
+    const version = page.src.origin?.descriptor?.prerelease
+      ? page.src.origin.descriptor.displayVersion
+      : page.src.version
+
+    // Handle the page keywords
+    const kwElement = root.querySelector('meta[name=keywords]')
+    let keywords = []
+    if (kwElement) {
+      const kwContent = kwElement.getAttribute('content')
+      keywords = kwContent ? kwContent.split(/,\s*/) : []
     }
 
-    // gather page breadcrumbs
+    // Gather page breadcrumbs
     const breadcrumbs = []
     root.querySelectorAll('nav.breadcrumbs > ul > li a')
       .forEach((elem) => {
-        var url = path.resolve(
+        const url = path.resolve(
           path.join('/', page.out.dirname),
           elem.getAttribute('href')
         )
@@ -111,7 +111,7 @@ function generateIndex (playbook, contentCatalog, { indexLatestOnly = false, exc
       continue
     }
 
-    // handle titles
+    // Handle titles
     const h1 = article.querySelector('h1')
     if (!h1) {
       logger.warn(`No H1 in ${page.pub.url}...skipping`)
@@ -122,148 +122,180 @@ function generateIndex (playbook, contentCatalog, { indexLatestOnly = false, exc
 
     const titles = []
     article.querySelectorAll('h2,h3,h4,h5,h6').forEach((title) => {
-      var id = title.getAttribute('id')
+      const id = title.getAttribute('id')
       if (id) {
         titles.push({
           t: title.text,
-          h: id,
+          h: id
         })
       }
       title.remove()
     })
 
-    // exclude elements within the article that should not be indexed
-    excludes.forEach((excl) => {
-      if (!excl) return
-      article.querySelectorAll(excl).map((e) =>  e.remove())
-    })
+    // Exclude elements within the article that should not be indexed
+    for (const excl of excludes) {
+      if (!excl) continue
+      article.querySelectorAll(excl).forEach((e) => e.remove())
+    }
 
-    var intro = article.querySelector('p');
-    // decode any HTML entities
-    intro = decode(intro.rawText);
+    // FIXED: Handle potential null intro element
+    const introElement = article.querySelector('p')
+    const intro = introElement ? decode(introElement.rawText) : ''
 
-    // establish structure in the Algolia index
+    // Establish structure in the Algolia index
     if (!(cname in algolia)) algolia[cname] = {}
     if (!(version in algolia[cname])) algolia[cname][version] = []
 
     // Check if this is a properties reference page (or has many titles)
-    const isPropertiesPage = page.pub.url.includes('/properties/') || titles.length > 30;
+    const isPropertiesPage = page.pub.url.includes('/properties/') || titles.length > 30
 
     // Handle the article text
-    let text = '';
+    let text = ''
 
     if (!isPropertiesPage) {
       // For normal pages, index full text content
-      const contentElements = article.querySelectorAll('p, table, li');
-      let contentText = '';
-      let currentSize = 0;
+      const contentElements = article.querySelectorAll('p, table, li')
+      let contentText = ''
+      let currentSize = 0
       // Maximum size in bytes (Algolia's limit is 100KB, using 50KB for safety)
-      const MAX_SIZE = 50000;
-      
+      const MAX_SIZE = 50000
+
       for (const element of contentElements) {
-        let elementText = '';
+        let elementText = ''
         if (element.tagName === 'TABLE') {
           for (const tr of element.querySelectorAll('tr')) {
             for (const cell of tr.querySelectorAll('td, th')) {
-              elementText += cell.textContent + ' ';
+              elementText += cell.textContent + ' '
             }
           }
         } else {
-          elementText = element.textContent;
+          elementText = element.textContent
         }
-        
-        const elementSize = textEncoder.encode(elementText).length;
+
+        const elementSize = textEncoder.encode(elementText).length
         if (currentSize + elementSize > MAX_SIZE) {
-          break;
+          break
         }
-        
-        contentText += elementText;
-        currentSize += elementSize;
+
+        contentText += elementText
+        currentSize += elementSize
       }
-      
+
       text = contentText.replace(/\n/g, ' ')
         .replace(/\r/g, ' ')
         .replace(/\s+/g, ' ')
-        .trim();
+        .trim()
     } else {
       // For long pages, only use intro as text (property names are already in titles array)
-      text = intro;
-      logger.info(`Skipping full text indexing for long page: ${page.pub.url} (${titles.length} properties)`);
+      text = intro
+      logger.info(`Skipping full text indexing for long page: ${page.pub.url} (${titles.length} properties)`)
     }
 
-    let tag;
-    const title = (component.title || '').trim();
+    let tag
+    const title = (component.title || '').trim()
     if (title.toLowerCase() === 'home') {
-      // Collect all unique component titles except 'home', 'shared', 'search' using public API when available
+      // Collect all unique component titles except 'home', 'shared', 'search'
       const componentsList = typeof contentCatalog.getComponents === 'function'
         ? contentCatalog.getComponents()
         : Array.isArray(contentCatalog.components)
           ? contentCatalog.components
-          : Object.values(contentCatalog.components || contentCatalog._components || {});
+          : Object.values(contentCatalog.components || contentCatalog._components || {})
+
       // Find the latest version for Self-Managed (component title: 'Self-Managed')
-      let latestVersion = undefined;
-      const selfManaged = componentsList.find(c => (c.title || '').trim().toLowerCase() === 'self-managed');
-      if (selfManaged && selfManaged.latest && selfManaged.latest.version) {
-        latestVersion = selfManaged.latest.version;
-        if (latestVersion && !/^v/.test(latestVersion)) latestVersion = 'v' + latestVersion;
+      let selfManagedLatestVersion
+      const selfManaged = componentsList.find(c => (c.title || '').trim().toLowerCase() === 'self-managed')
+      if (selfManaged?.latest?.version) {
+        selfManagedLatestVersion = selfManaged.latest.version
+        if (selfManagedLatestVersion && !/^v/.test(selfManagedLatestVersion)) {
+          selfManagedLatestVersion = 'v' + selfManagedLatestVersion
+        }
       }
+
       const allComponentTitles = componentsList
         .map(c => (c.title || '').trim())
-        .filter(t => t && !['home', 'shared', 'search'].includes(t.toLowerCase()));
+        .filter(t => t && !['home', 'shared', 'search'].includes(t.toLowerCase()))
+
       if (!allComponentTitles.length) {
-        throw new Error('No component titles found for "home" page. Indexing aborted.');
+        throw new Error('No component titles found for "home" page. Indexing aborted.')
       }
-      tag = [...new Set(allComponentTitles)];
+
+      tag = [...new Set(allComponentTitles)]
       // For Self-Managed, append v<latest-version> to the tag
-      if (latestVersion) {
-        tag = tag.map(t => t.toLowerCase() === 'self-managed' ? `${t} ${latestVersion}` : t);
+      if (selfManagedLatestVersion) {
+        tag = tag.map(t => t.toLowerCase() === 'self-managed' ? `${t} ${selfManagedLatestVersion}` : t)
       }
     } else {
-      tag = `${title}${version ? ' v' + version : ''}`;
+      tag = `${title}${version ? ' v' + version : ''}`
     }
-    var indexItem;
-    const deployment = page.asciidoc?.attributes['env-kubernetes'] ? 'Kubernetes' : page.asciidoc?.attributes['env-linux'] ? 'Linux' : page.asciidoc?.attributes['env-docker'] ? 'Docker' : page.asciidoc?.attributes['page-cloud'] ? 'Redpanda Cloud' : ''
+
+    const deployment = page.asciidoc?.attributes['env-kubernetes']
+      ? 'Kubernetes'
+      : page.asciidoc?.attributes['env-linux']
+        ? 'Linux'
+        : page.asciidoc?.attributes['env-docker']
+          ? 'Docker'
+          : page.asciidoc?.attributes['page-cloud']
+            ? 'Redpanda Cloud'
+            : ''
 
     const categories = page.asciidoc?.attributes['page-categories']
-    ? page.asciidoc.attributes['page-categories'].split(',').map(category => category.trim())
-    : []
+      ? page.asciidoc.attributes['page-categories'].split(',').map(category => category.trim())
+      : []
 
-    var indexItem = {
+    const commercialNames = page.asciidoc?.attributes['page-commercial-names']
+      ? page.asciidoc.attributes['page-commercial-names'].split(',').map(name => name.trim())
+      : []
+
+    // FIXED: keywords now included in index item
+    const indexItem = {
       title: documentTitle,
       version: version,
       text: text,
       intro: intro,
       objectID: urlPath + page.pub.url,
       titles: titles,
+      keywords: keywords,
       categories: categories,
-      unixTimestamp: unixTimestamp,
+      commercialNames: commercialNames,
+      unixTimestamp: unixTimestamp
     }
 
     if (component.name !== 'redpanda-labs') {
-      indexItem.product = component.title;
-      indexItem.breadcrumbs = breadcrumbs;
-      indexItem.type = 'Doc';
-      indexItem._tags = Array.isArray(tag) ? tag : [tag];
+      indexItem.product = component.title
+      indexItem.breadcrumbs = breadcrumbs
+      indexItem.type = 'Doc'
+      indexItem._tags = Array.isArray(tag) ? tag : [tag]
     } else {
-      indexItem.deployment = deployment;
-      indexItem.type = 'Lab';
-      indexItem.interactive = false;
-      indexItem._tags = Array.isArray(tag) ? tag : [tag];
+      indexItem.deployment = deployment
+      indexItem.type = 'Lab'
+      indexItem.interactive = false
+      indexItem._tags = Array.isArray(tag) ? tag : [tag]
     }
+
     algolia[cname][version].push(indexItem)
     algoliaCount++
   }
+
+  logger.info(`Indexed ${algoliaCount} pages`)
   return algolia
 }
 
-// Extract the path from a URL
+/**
+ * Extract the path from a URL
+ * @param {string} url - The URL to extract path from
+ * @returns {string} The URL path
+ */
 function extractUrlPath (url) {
-  if (url) {
-    if (url.charAt() === '/') return url
-    const urlPath = URL.parse(url).pathname
+  if (!url) return ''
+  if (url.charAt(0) === '/') return url
+
+  try {
+    // FIXED: Use modern URL API instead of deprecated url.parse()
+    const urlPath = new URL(url).pathname
     return urlPath === '/' ? '' : urlPath
+  } catch {
+    return ''
   }
-  return ''
 }
 
 module.exports = generateIndex

package/extensions/algolia-indexer/index.js

@@ -5,14 +5,16 @@ const algoliasearch = require('algoliasearch')
 const http = require('http')
 const https = require('https')
 const _ = require('lodash')
-process.env.UV_THREADPOOL_SIZE=16
+
+// Increase thread pool size for better HTTP performance
+process.env.UV_THREADPOOL_SIZE = 16
 
 /**
  * Algolia indexing for an Antora documentation site.
  *
  * @module antora-algolia-indexer
  */
-function register({
+function register ({
   config: {
     indexLatestOnly,
     excludes,
@@ -21,29 +23,41 @@ function register({
 }) {
   const logger = this.getLogger('algolia-indexer-extension')
 
-  if (!process.env.ALGOLIA_ADMIN_API_KEY || !process.env.ALGOLIA_APP_ID || !process.env.ALGOLIA_INDEX_NAME) return
-
-  var client
-  var index
-
-  const httpAgent = new http.Agent({ keepAlive: true, maxSockets: 100 });
-  const httpsAgent = new https.Agent({ keepAlive: true, maxSockets: 100 });
+  // Validate required environment variables
+  const requiredEnvVars = ['ALGOLIA_ADMIN_API_KEY', 'ALGOLIA_APP_ID', 'ALGOLIA_INDEX_NAME']
+  const missingVars = requiredEnvVars.filter(v => !process.env[v])
 
-  // Connect and authenticate with Algolia using the custom agent
-  client = algoliasearch(process.env.ALGOLIA_APP_ID, process.env.ALGOLIA_ADMIN_API_KEY, {
-    httpAgent: httpAgent,
-    httpsAgent: httpsAgent
-  })
-  index = client.initIndex(process.env.ALGOLIA_INDEX_NAME)
+  if (missingVars.length > 0) {
+    logger.info(`Algolia indexing disabled - missing environment variables: ${missingVars.join(', ')}`)
+    return
+  }
 
+  // Validate unknown options
   if (Object.keys(unknownOptions).length) {
     const keys = Object.keys(unknownOptions)
     throw new Error(`Unrecognized option${keys.length > 1 ? 's' : ''} specified: ${keys.join(', ')}`)
   }
 
-  this.on('beforePublish', async ({ playbook, siteCatalog, contentCatalog }) => {
+  // Create HTTP agents with connection pooling
+  const httpAgent = new http.Agent({ keepAlive: true, maxSockets: 100 })
+  const httpsAgent = new https.Agent({ keepAlive: true, maxSockets: 100 })
+
+  // Connect and authenticate with Algolia
+  const client = algoliasearch(process.env.ALGOLIA_APP_ID, process.env.ALGOLIA_ADMIN_API_KEY, {
+    httpAgent: httpAgent,
+    httpsAgent: httpsAgent
+  })
+  const index = client.initIndex(process.env.ALGOLIA_INDEX_NAME)
+
+  this.on('beforePublish', async ({ playbook, contentCatalog }) => {
     const algolia = generateIndex(playbook, contentCatalog, { indexLatestOnly, excludes, logger })
-    let existingObjectsMap = new Map()
+
+    if (!algolia || Object.keys(algolia).length === 0) {
+      logger.warn('No content to index for Algolia')
+      return
+    }
+
+    const existingObjectsMap = new Map()
 
     // Save objects in a local cache to query later.
     // Avoids sending multiple requests.
@@ -58,8 +72,9 @@ function register({
           }
         }
       })
+      logger.info(`Loaded ${existingObjectsMap.size} existing objects from Algolia index`)
     } catch (err) {
-      logger.error(JSON.stringify(err))
+      logger.error(`Error browsing existing Algolia objects: ${JSON.stringify(err)}`)
     }
 
     let totalObjectsToUpdate = 0
@@ -90,46 +105,63 @@ function register({
           action: 'addObject',
           indexName: process.env.ALGOLIA_INDEX_NAME,
           body: object
-        }));
+        }))
 
         const updateObjectActions = objectsToUpdate.map(object => ({
           action: 'updateObject',
           indexName: process.env.ALGOLIA_INDEX_NAME,
           body: object
-        }));
+        }))
 
-        const batchActions = [...addObjectActions, ...updateObjectActions];
+        const batchActions = [...addObjectActions, ...updateObjectActions]
 
-        // Upload new records only if the objects have been updated or they are new.
-        // See https://www.algolia.com/doc/api-reference/api-methods/batch/?client=javascript
-        await client.multipleBatch(batchActions).then(() => {
-          logger.info('Batch operations completed successfully');
-        }).catch(error => {
-          logger.error(`Error uploading records to Algolia: ${error.message}`);
-        });
+        // FIXED: Only send batch if there are actions to perform
+        if (batchActions.length > 0) {
+          try {
+            await client.multipleBatch(batchActions)
+            logger.debug(`Batch completed: ${objectsToAdd.length} added, ${objectsToUpdate.length} updated for ${c}/${v}`)
+          } catch (error) {
+            logger.error(`Error uploading records to Algolia: ${error.message}`)
+          }
+        }
       }
     }
 
+    // Identify objects to delete (stale content)
     for (const [objectID, obj] of existingObjectsMap) {
-      if ((obj.type === 'Doc' && !obj.objectID.includes('/api/')) || (!obj.type) || (obj.type === 'Lab' && !obj.interactive)) {
+      // Only delete Doc pages (not API) and Labs that aren't interactive
+      const shouldDelete = (obj.type === 'Doc' && !obj.objectID.includes('/api/')) ||
+                          (!obj.type) ||
+                          (obj.type === 'Lab' && !obj.interactive)
+
+      if (shouldDelete) {
         objectsToDelete.push(objectID)
       }
     }
+
     if (objectsToDelete.length > 0) {
-      console.log(objectsToDelete)
-      await index.deleteObjects(objectsToDelete).then(() => {
-        console.log(`Deleted ${objectsToDelete.length} outdated records`);
-      }).catch(error => {
-        logger.error(`Error deleting records from Algolia: ${error.message}`);
-      });
+      logger.info(`Deleting ${objectsToDelete.length} outdated records...`)
+      logger.debug(`Objects to delete: ${JSON.stringify(objectsToDelete)}`)
+
+      try {
+        await index.deleteObjects(objectsToDelete)
+        logger.info(`Successfully deleted ${objectsToDelete.length} outdated records`)
+      } catch (error) {
+        logger.error(`Error deleting records from Algolia: ${error.message}`)
+      }
     }
 
-    logger.info('Updated records:' + totalObjectsToUpdate)
-    logger.info('New records:' + totalObjectsToAdd)
+    // Summary
+    logger.info(`Algolia sync complete: ${totalObjectsToAdd} added, ${totalObjectsToUpdate} updated, ${objectsToDelete.length} deleted`)
 
-    totalObjectsToAdd === 0 && totalObjectsToUpdate === 0 && logger.info('No new records uploaded or existing records updated')
+    if (totalObjectsToAdd === 0 && totalObjectsToUpdate === 0 && objectsToDelete.length === 0) {
+      logger.info('Index is up to date - no changes needed')
+    }
   })
 
+  // Cleanup HTTP agents on process exit
+  // NOTE: This registers a global handler. In watch mode, agents will persist
+  // between builds, which is generally fine for connection reuse.
   process.on('exit', () => {
     httpAgent.destroy()
     httpsAgent.destroy()

package/extensions/collect-bloblang-samples.js

@@ -4,12 +4,13 @@ module.exports.register = function () {
   const logger = this.getLogger('collect-bloblang-samples');
 
   this.on('contentClassified', ({ contentCatalog }) => {
+
     const collectExamples = (examples, componentName) => {
       const bloblangSamples = [];
       const seenTitles = new Set();
 
       examples
-        .filter((example) => example.src.relative.startsWith('playground/')) // Only include files in the 'bloblang' subdirectory
+        .filter((example) => example.src.relative.startsWith('playground/')) // Only include files in the 'playground' subdirectory
         .forEach((example) => {
           try {
             const content = example.contents.toString('utf8');