@redpanda-data/docs-extensions-and-macros 4.13.3 → 4.13.5

package/bin/doc-tools-mcp.js

@@ -15,6 +15,9 @@
  * - Telemetry: Usage tracking for adoption metrics
  */
 
+// Load environment variables from .env file if it exists
+require('dotenv').config();
+
 const fs = require('fs');
 const path = require('path');
 const { Server } = require('@modelcontextprotocol/sdk/server/index.js');

package/bin/doc-tools.js

@@ -2,6 +2,9 @@
 
 'use strict'
 
+// Load environment variables from .env file if it exists
+require('dotenv').config()
+
 const { spawnSync } = require('child_process')
 const os = require('os')
 const { Command, Option } = require('commander')

package/cli-utils/octokit-client.js

@@ -0,0 +1,36 @@
+'use strict'
+
+const { Octokit } = require('@octokit/rest')
+const { getGitHubToken } = require('./github-token')
+
+/**
+ * Shared Octokit client instance for GitHub API access
+ * Configured with optional authentication and retry logic
+ *
+ * This singleton instance is shared across all doc-tools modules to:
+ * - Avoid redundant initialization
+ * - Share rate limit tracking
+ * - Centralize GitHub API configuration
+ */
+
+// Get authentication token from environment
+const token = getGitHubToken()
+
+// Configure Octokit options
+const octokitOptions = {
+  userAgent: 'redpanda-docs-tools',
+  retry: {
+    enabled: true,
+    retries: 3
+  }
+}
+
+// Only add auth if token is available
+if (token) {
+  octokitOptions.auth = token
+}
+
+// Create singleton instance
+const octokit = new Octokit(octokitOptions)
+
+module.exports = octokit

package/extensions/generate-rp-connect-info.js

@@ -25,11 +25,9 @@ module.exports.register = function ({ config }) {
   // Use csvpath (legacy) or csvPath
   const localCsvPath = csvpath || null
 
-  async function loadOctokit () {
-    const { Octokit } = await import('@octokit/rest')
-    const { getGitHubToken } = require('../cli-utils/github-token')
-    const token = getGitHubToken()
-    return token ? new Octokit({ auth: token }) : new Octokit()
+  function loadOctokit () {
+    // Use shared Octokit client
+    return require('../cli-utils/octokit-client')
   }
 
   // Use 'on' and return the promise so Antora waits for async completion

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redpanda-data/docs-extensions-and-macros",
-  "version": "4.13.3",
+  "version": "4.13.5",
   "description": "Antora extensions and macros developed for Redpanda documentation.",
   "keywords": [
     "antora",
@@ -103,6 +103,7 @@
     "chalk": "4.1.2",
     "cheerio": "^1.1.2",
     "commander": "^14.0.0",
+    "dotenv": "^16.6.1",
     "glob": "^11.0.0",
     "gulp": "^4.0.2",
     "gulp-connect": "^5.7.0",

package/tools/cloud-regions/generate-cloud-regions.js

@@ -70,8 +70,8 @@ function displayClusterType(ct) {
  */
 async function fetchYaml({ owner, repo, path, ref = 'main', token }) {
   try {
-    const { Octokit } = await import('@octokit/rest');
-    const octokit = new Octokit(token ? { auth: token } : {});
+    // Use shared Octokit client
+    const octokit = require('../../cli-utils/octokit-client');
 
     console.log(`[cloud-regions] INFO: Fetching ${owner}/${repo}/${path}@${ref} via GitHub API`);
 

package/tools/fetch-from-github.js

@@ -1,23 +1,10 @@
 const fs = require('fs');
 const path = require('path');
 
-let octokitInstance = null;
-async function loadOctokit() {
-  if (!octokitInstance) {
-    const { Octokit } = await import('@octokit/rest');
-    octokitInstance = process.env.VBOT_GITHUB_API_TOKEN
-      ? new Octokit({
-          auth: process.env.VBOT_GITHUB_API_TOKEN,
-        })
-      : new Octokit();
-
-    if (!process.env.VBOT_GITHUB_API_TOKEN) {
-      console.info(
-        'No GitHub token found (VBOT_GITHUB_API_TOKEN).'
-      );
-    }
-  }
-  return octokitInstance;
+// Use shared Octokit client
+function loadOctokit() {
+  const octokit = require('../cli-utils/octokit-client');
+  return octokit;
 }
 
 async function saveFile(content, saveDir, filename) {

package/tools/get-console-version.js

@@ -23,13 +23,8 @@ module.exports = async function getConsoleVersion({ beta = false, fromAntora = f
     useBeta = getPrereleaseFromAntora();
   }
 
-  // Initialize GitHub client
-  const { getGitHubToken } = require('../cli-utils/github-token');
-  const { Octokit } = await import('@octokit/rest');
-  const token = getGitHubToken();
-  const octokit = token
-    ? new Octokit({ auth: token })
-    : new Octokit();
+  // Use shared Octokit client
+  const octokit = require('../cli-utils/octokit-client');
 
   // Fetch latest release info
   let data;

package/tools/get-redpanda-version.js

@@ -19,13 +19,8 @@ module.exports = async function getRedpandaVersion({ beta = false, fromAntora =
     useBeta = getPrereleaseFromAntora();
   }
 
-  // Load Octokit
-  const { getGitHubToken } = require('../cli-utils/github-token');
-  const { Octokit } = await import('@octokit/rest');
-  const token = getGitHubToken();
-  const octokit = token
-    ? new Octokit({ auth: token })
-    : new Octokit();
+  // Use shared Octokit client
+  const octokit = require('../cli-utils/octokit-client');
 
   // Fetch version data
   let data;

package/tools/redpanda-connect/connector-binary-analyzer.js

@@ -1,4 +1,4 @@
-const { Octokit } = require('@octokit/rest');
+const octokit = require('../../cli-utils/octokit-client');
 const { execSync, spawnSync } = require('child_process');
 const fs = require('fs');
 const path = require('path');
@@ -13,16 +13,6 @@ const https = require('https');
  * - Which connectors are self-hosted only
  */
 
-// Initialize Octokit with optional authentication
-const octokit = new Octokit({
-  auth: process.env.GITHUB_TOKEN,
-  userAgent: 'redpanda-docs-tools',
-  retry: {
-    enabled: true,
-    retries: 3
-  }
-});
-
 const REPO_OWNER = 'redpanda-data';
 const REPO_NAME = 'connect';
 

package/tools/redpanda-connect/helpers/buildConfigYaml.js

@@ -36,11 +36,16 @@ module.exports = function buildConfigYaml(type, connectorName, children, include
       return; // skip deprecated fields
     }
     if (!includeAdvanced && field.is_advanced) {
-      return; // skip advanced fields in “common” mode
+      return; // skip advanced fields in "common" mode
     }
 
-    if (field.type === 'object' && Array.isArray(field.children)) {
-      // Render nested object
+    // Check if this is an array-of-objects (e.g., client_certs[])
+    // These should render as empty arrays, not expanded object structures
+    if (field.kind === 'array' && field.type === 'object' && Array.isArray(field.children)) {
+      // Render as array leaf (e.g., "client_certs: []")
+      lines.push(renderLeafField(field, baseIndent));
+    } else if (field.type === 'object' && Array.isArray(field.children)) {
+      // Render nested object (plain object, not array)
       const nestedLines = renderObjectField(field, baseIndent);
       lines.push(...nestedLines);
     } else {

package/tools/redpanda-connect/helpers/renderObjectField.js

@@ -28,7 +28,12 @@ module.exports = function renderObjectField(field, indentLevel) {
     if (child.is_deprecated) {
       return; // skip entirely
     }
-    if (Array.isArray(child.children) && child.children.length > 0) {
+    // Check if this is an array-of-objects (e.g., client_certs[])
+    // These should render as empty arrays, not expanded object structures
+    if (child.kind === 'array' && child.type === 'object' && Array.isArray(child.children)) {
+      // Render as array leaf (e.g., "client_certs: []")
+      lines.push(renderLeafField(child, childIndent));
+    } else if (Array.isArray(child.children) && child.children.length > 0) {
       // Nested object → recurse
       lines.push(...renderObjectField(child, childIndent));
     } else {

package/tools/redpanda-connect/pr-summary-formatter.js

@@ -397,8 +397,9 @@ function generatePRSummary(diffData, binaryAnalysis = null, draftedConnectors =
         lines.push('');
         cloudSupportedNew.forEach(c => {
           lines.push(`- **${c.name}** (${c.type}, ${c.status})`);
-          if (c.description) {
-            const shortDesc = truncateToSentence(c.description, 2);
+          const desc = c.summary || c.description;
+          if (desc) {
+            const shortDesc = truncateToSentence(desc, 2);
             lines.push(`  - ${shortDesc}`);
           }
         });
@@ -410,8 +411,9 @@ function generatePRSummary(diffData, binaryAnalysis = null, draftedConnectors =
         lines.push('');
         selfHostedOnlyNew.forEach(c => {
           lines.push(`- **${c.name}** (${c.type}, ${c.status})`);
-          if (c.description) {
-            const shortDesc = truncateToSentence(c.description, 2);
+          const desc = c.summary || c.description;
+          if (desc) {
+            const shortDesc = truncateToSentence(desc, 2);
             lines.push(`  - ${shortDesc}`);
           }
         });
@@ -421,8 +423,9 @@ function generatePRSummary(diffData, binaryAnalysis = null, draftedConnectors =
       // No cloud support info, just list all
       diffData.details.newComponents.forEach(c => {
         lines.push(`- **${c.name}** (${c.type}, ${c.status})`);
-        if (c.description) {
-          const shortDesc = truncateToSentence(c.description, 2);
+        const desc = c.summary || c.description;
+        if (desc) {
+          const shortDesc = truncateToSentence(desc, 2);
           lines.push(`  - ${shortDesc}`);
         }
       });

package/tools/redpanda-connect/rpcn-connector-docs-handler.js

@@ -189,8 +189,9 @@ function updateWhatsNew ({ dataDir, oldVersion, newVersion, binaryAnalysis }) {
           for (const comp of comps) {
             section += `** xref:guides:bloblang/functions.adoc#${comp.name}[\`${comp.name}\`]`
             if (comp.status && comp.status !== 'stable') section += ` (${comp.status})`
-            if (comp.description) {
-              section += `: ${capToTwoSentences(comp.description)}`
+            const desc = comp.summary || comp.description
+            if (desc) {
+              section += `: ${capToTwoSentences(desc)}`
             } else {
               section += `\n+\n// TODO: Add description for ${comp.name} function`
             }
@@ -201,8 +202,9 @@ function updateWhatsNew ({ dataDir, oldVersion, newVersion, binaryAnalysis }) {
           for (const comp of comps) {
             section += `** xref:guides:bloblang/methods.adoc#${comp.name}[\`${comp.name}\`]`
             if (comp.status && comp.status !== 'stable') section += ` (${comp.status})`
-            if (comp.description) {
-              section += `: ${capToTwoSentences(comp.description)}`
+            const desc = comp.summary || comp.description
+            if (desc) {
+              section += `: ${capToTwoSentences(desc)}`
             } else {
               section += `\n+\n// TODO: Add description for ${comp.name} method`
             }
@@ -267,7 +269,8 @@ function updateWhatsNew ({ dataDir, oldVersion, newVersion, binaryAnalysis }) {
 
       for (const comp of diff.details.deprecatedComponents) {
         const typeLabel = comp.type.charAt(0).toUpperCase() + comp.type.slice(1)
-        const desc = comp.description ? capToTwoSentences(comp.description) : '-'
+        const descText = comp.summary || comp.description
+        const desc = descText ? capToTwoSentences(descText) : '-'
 
         if (comp.type === 'bloblang-functions') {
           section += `|xref:guides:bloblang/functions.adoc#${comp.name}[${comp.name}]\n`
@@ -913,6 +916,9 @@ async function handleRpcnConnectorDocs (options) {
         fs.unlinkSync(oldestPath)
         console.log(`🧹 Deleted old version from docs-data: ${oldestFile}`)
       }
+
+      // Reload newIndex after augmentation so diff generation uses augmented data
+      newIndex = JSON.parse(fs.readFileSync(dataFile, 'utf8'))
     } catch (err) {
       console.error(`Warning: Failed to augment data file: ${err.message}`)
     }
@@ -1066,7 +1072,9 @@ async function handleRpcnConnectorDocs (options) {
               validConnectors.push({
                 name: connector.name,
                 type: type.replace(/s$/, ''),
-                status: connector.status || connector.type || 'stable'
+                status: connector.status || connector.type || 'stable',
+                cloudOnly: connector.cloudOnly === true,
+                requiresCgo: connector.requiresCgo === true
               })
             }
           })
@@ -1113,14 +1121,175 @@ async function handleRpcnConnectorDocs (options) {
         cloudOnly: path.resolve(process.cwd(), 'modules/components/partials/components/cloud-only')
       }
 
-      const allMissing = validConnectors.filter(({ name, type }) => {
+      // Build a set of cloud-supported connectors (inCloud + cloudOnly, excluding self-hosted-only)
+      const cloudSupportedSet = new Set()
+      if (binaryAnalysis?.comparison) {
+        // inCloud = available in both OSS and Cloud
+        binaryAnalysis.comparison.inCloud?.forEach(c => {
+          cloudSupportedSet.add(`${c.type}:${c.name}`)
+        })
+        // cloudOnly = only available in Cloud (not in OSS)
+        binaryAnalysis.comparison.cloudOnly?.forEach(c => {
+          cloudSupportedSet.add(`${c.type}:${c.name}`)
+        })
+      } else {
+        // Fallback when binary analysis is unavailable:
+        // Check all connectors that have cloudSupported flag or assume all non-deprecated are cloud-supported
+        console.log('   ℹ️  Binary analysis unavailable - checking all non-deprecated connectors for cloud-docs')
+        const types = ['inputs', 'outputs', 'processors', 'caches', 'rate_limits', 'buffers', 'metrics', 'scanners', 'tracers']
+        types.forEach(type => {
+          if (Array.isArray(dataObj[type])) {
+            dataObj[type].forEach(connector => {
+              // Include if cloudSupported is explicitly true, or if it's null/undefined and not deprecated
+              const isCloudSupported = connector.cloudSupported === true ||
+                (connector.cloudSupported == null && connector.status !== 'deprecated')
+              if (isCloudSupported && connector.name) {
+                // Store type as plural to match binary analysis format
+                cloudSupportedSet.add(`${type}:${connector.name}`)
+              }
+            })
+          }
+        })
+      }
+
+      // Check for missing connector documentation in rp-connect-docs
+      const allMissing = validConnectors.filter(({ name, type, cloudOnly }) => {
         const relPath = path.join(`${type}s`, `${name}.adoc`)
-        const existsInAny = Object.values(roots).some(root =>
+
+        // For cloud-only connectors, ONLY check the cloud-only directory
+        if (cloudOnly) {
+          return !fs.existsSync(path.join(roots.cloudOnly, relPath))
+        }
+
+        // For regular connectors, check pages and partials (not cloud-only)
+        const existsInAny = [roots.pages, roots.partials].some(root =>
           fs.existsSync(path.join(root, relPath))
         )
         return !existsInAny
       })
 
+      // Check for cloud-supported connectors missing from cloud-docs repo (via GitHub API)
+      const missingFromCloudDocs = []
+      const cloudDocsErrors = []
+      if (cloudSupportedSet.size > 0 && options.checkCloudDocs !== false) {
+        console.log('\n   ℹ️  Checking cloud-docs repository for missing connector pages...')
+
+        // Use shared Octokit instance
+        const octokit = require('../../cli-utils/octokit-client')
+
+        try {
+          // Check each cloud-supported connector
+          // Filter to only check actual connector/component types that need individual pages
+          const connectorTypes = ['inputs', 'outputs', 'processors', 'caches', 'buffers', 'scanners', 'metrics', 'tracers']
+
+          for (const connectorKey of cloudSupportedSet) {
+            const [type, name] = connectorKey.split(':')
+
+            // Skip non-connector types (config, bloblang-functions, bloblang-methods, rate-limits)
+            if (!connectorTypes.includes(type)) {
+              continue
+            }
+
+            // Skip deprecated connectors - they don't need cloud-docs pages
+            if (Array.isArray(dataObj[type])) {
+              const connector = dataObj[type].find(c => c.name === name)
+              if (connector && connector.status === 'deprecated') {
+                continue
+              }
+            }
+
+            const cloudDocsPath = `modules/develop/pages/connect/components/${type}/${name}.adoc`
+
+            try {
+              await octokit.repos.getContent({
+                owner: 'redpanda-data',
+                repo: 'cloud-docs',
+                path: cloudDocsPath,
+                ref: 'main'
+              })
+              // File exists, no action needed
+            } catch (error) {
+              if (error.status === 404) {
+                // File doesn't exist in cloud-docs
+                missingFromCloudDocs.push({ type, name, path: cloudDocsPath })
+              } else {
+                // Non-404 error (auth, rate-limit, network, etc.)
+                // Try fallback: check raw URL without authentication
+                const rawUrl = `https://raw.githubusercontent.com/redpanda-data/cloud-docs/main/${cloudDocsPath}`
+                try {
+                  const https = require('https')
+                  const { URL } = require('url')
+                  const parsedUrl = new URL(rawUrl)
+
+                  await new Promise((resolve, reject) => {
+                    const req = https.request({
+                      hostname: parsedUrl.hostname,
+                      path: parsedUrl.pathname,
+                      method: 'HEAD',
+                      timeout: 5000
+                    }, (res) => {
+                      if (res.statusCode === 200) {
+                        resolve() // File exists
+                      } else if (res.statusCode === 404) {
+                        reject(new Error('404'))
+                      } else {
+                        reject(new Error(`Status ${res.statusCode}`))
+                      }
+                    })
+                    req.on('error', reject)
+                    req.on('timeout', () => {
+                      req.destroy()
+                      reject(new Error('Timeout'))
+                    })
+                    req.end()
+                  })
+                  // Fallback succeeded - file exists, no action needed
+                } catch (fallbackError) {
+                  if (fallbackError.message === '404') {
+                    // Confirmed missing via fallback
+                    missingFromCloudDocs.push({ type, name, path: cloudDocsPath })
+                  } else {
+                    // Both API and fallback failed
+                    cloudDocsErrors.push({
+                      type,
+                      name,
+                      path: cloudDocsPath,
+                      status: error.status || 'unknown',
+                      message: `API: ${error.message}; Fallback: ${fallbackError.message}`
+                    })
+                  }
+                }
+              }
+            }
+          }
+
+          // Report results
+          if (cloudDocsErrors.length > 0) {
+            console.log(`   ⚠️  Encountered ${cloudDocsErrors.length} error(s) while checking cloud-docs (check inconclusive):`)
+            cloudDocsErrors.forEach(({ type, name, status, message }) => {
+              console.log(`      • ${type}/${name} - Status ${status}: ${message}`)
+            })
+            console.log(`   ℹ️  Please resolve these errors (e.g., check GITHUB_TOKEN or VBOT_GITHUB_API_TOKEN, API rate limits, network connectivity)`)
+            if (missingFromCloudDocs.length > 0) {
+              console.log(`   ℹ️  Additionally, ${missingFromCloudDocs.length} connector(s) confirmed missing from cloud-docs:`)
+              missingFromCloudDocs.forEach(({ type, name }) => {
+                console.log(`      • ${type}/${name}`)
+              })
+            }
+          } else if (missingFromCloudDocs.length > 0) {
+            console.log(`   ⚠️  Found ${missingFromCloudDocs.length} cloud-supported connector(s) missing from cloud-docs:`)
+            missingFromCloudDocs.forEach(({ type, name }) => {
+              console.log(`      • ${type}/${name}`)
+            })
+            console.log(`   ℹ️  These connectors need pages added to https://github.com/redpanda-data/cloud-docs`)
+          } else {
+            console.log(`   ✓  All cloud-supported connectors have pages in cloud-docs`)
+          }
+        } catch (error) {
+          console.log(`   ⚠️  Could not check cloud-docs: ${error.message}`)
+        }
+      }
+
       const missingConnectors = allMissing.filter(c =>
         !c.name.includes('sql_driver') &&
         c.status !== 'deprecated'