@redpanda-data/docs-extensions-and-macros 4.13.4 → 4.13.6

package/bin/doc-tools-mcp.js

@@ -15,6 +15,9 @@
  * - Telemetry: Usage tracking for adoption metrics
  */
 
+// Load environment variables from .env file if it exists
+require('dotenv').config();
+
 const fs = require('fs');
 const path = require('path');
 const { Server } = require('@modelcontextprotocol/sdk/server/index.js');

package/bin/doc-tools.js

@@ -2,6 +2,9 @@
 
 'use strict'
 
+// Load environment variables from .env file if it exists
+require('dotenv').config()
+
 const { spawnSync } = require('child_process')
 const os = require('os')
 const { Command, Option } = require('commander')

package/cli-utils/github-token.js

@@ -9,14 +9,18 @@
  * Get GitHub token from environment variables
  * Checks multiple common variable names in priority order:
  * 1. REDPANDA_GITHUB_TOKEN - Custom Redpanda token
- * 2. GITHUB_TOKEN - GitHub Actions default
- * 3. GH_TOKEN - GitHub CLI default
+ * 2. ACTIONS_BOT_TOKEN - GitHub Actions bot token
+ * 3. GITHUB_TOKEN - GitHub Actions default
+ * 4. VBOT_GITHUB_API_TOKEN - Legacy bot token
+ * 5. GH_TOKEN - GitHub CLI default
  *
  * @returns {string|null} GitHub token or null if not found
  */
 function getGitHubToken() {
   return process.env.REDPANDA_GITHUB_TOKEN ||
+         process.env.ACTIONS_BOT_TOKEN ||
          process.env.GITHUB_TOKEN ||
+         process.env.VBOT_GITHUB_API_TOKEN ||
          process.env.GH_TOKEN ||
          null;
 }

package/cli-utils/octokit-client.js

@@ -0,0 +1,36 @@
+'use strict'
+
+const { Octokit } = require('@octokit/rest')
+const { getGitHubToken } = require('./github-token')
+
+/**
+ * Shared Octokit client instance for GitHub API access
+ * Configured with optional authentication and retry logic
+ *
+ * This singleton instance is shared across all doc-tools modules to:
+ * - Avoid redundant initialization
+ * - Share rate limit tracking
+ * - Centralize GitHub API configuration
+ */
+
+// Get authentication token from environment
+const token = getGitHubToken()
+
+// Configure Octokit options
+const octokitOptions = {
+  userAgent: 'redpanda-docs-tools',
+  retry: {
+    enabled: true,
+    retries: 3
+  }
+}
+
+// Only add auth if token is available
+if (token) {
+  octokitOptions.auth = token
+}
+
+// Create singleton instance
+const octokit = new Octokit(octokitOptions)
+
+module.exports = octokit

package/extensions/generate-rp-connect-info.js

@@ -25,11 +25,9 @@ module.exports.register = function ({ config }) {
   // Use csvpath (legacy) or csvPath
   const localCsvPath = csvpath || null
 
-  async function loadOctokit () {
-    const { Octokit } = await import('@octokit/rest')
-    const { getGitHubToken } = require('../cli-utils/github-token')
-    const token = getGitHubToken()
-    return token ? new Octokit({ auth: token }) : new Octokit()
+  function loadOctokit () {
+    // Use shared Octokit client
+    return require('../cli-utils/octokit-client')
   }
 
   // Use 'on' and return the promise so Antora waits for async completion

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redpanda-data/docs-extensions-and-macros",
-  "version": "4.13.4",
+  "version": "4.13.6",
   "description": "Antora extensions and macros developed for Redpanda documentation.",
   "keywords": [
     "antora",
@@ -103,6 +103,7 @@
     "chalk": "4.1.2",
     "cheerio": "^1.1.2",
     "commander": "^14.0.0",
+    "dotenv": "^16.6.1",
     "glob": "^11.0.0",
     "gulp": "^4.0.2",
     "gulp-connect": "^5.7.0",

package/tools/cloud-regions/generate-cloud-regions.js

@@ -70,8 +70,8 @@ function displayClusterType(ct) {
  */
 async function fetchYaml({ owner, repo, path, ref = 'main', token }) {
   try {
-    const { Octokit } = await import('@octokit/rest');
-    const octokit = new Octokit(token ? { auth: token } : {});
+    // Use shared Octokit client
+    const octokit = require('../../cli-utils/octokit-client');
 
     console.log(`[cloud-regions] INFO: Fetching ${owner}/${repo}/${path}@${ref} via GitHub API`);
 

package/tools/fetch-from-github.js

@@ -1,23 +1,10 @@
 const fs = require('fs');
 const path = require('path');
 
-let octokitInstance = null;
-async function loadOctokit() {
-  if (!octokitInstance) {
-    const { Octokit } = await import('@octokit/rest');
-    octokitInstance = process.env.VBOT_GITHUB_API_TOKEN
-      ? new Octokit({
-          auth: process.env.VBOT_GITHUB_API_TOKEN,
-        })
-      : new Octokit();
-
-    if (!process.env.VBOT_GITHUB_API_TOKEN) {
-      console.info(
-        'No GitHub token found (VBOT_GITHUB_API_TOKEN).'
-      );
-    }
-  }
-  return octokitInstance;
+// Use shared Octokit client
+function loadOctokit() {
+  const octokit = require('../cli-utils/octokit-client');
+  return octokit;
 }
 
 async function saveFile(content, saveDir, filename) {

package/tools/get-console-version.js

@@ -23,13 +23,8 @@ module.exports = async function getConsoleVersion({ beta = false, fromAntora = f
     useBeta = getPrereleaseFromAntora();
   }
 
-  // Initialize GitHub client
-  const { getGitHubToken } = require('../cli-utils/github-token');
-  const { Octokit } = await import('@octokit/rest');
-  const token = getGitHubToken();
-  const octokit = token
-    ? new Octokit({ auth: token })
-    : new Octokit();
+  // Use shared Octokit client
+  const octokit = require('../cli-utils/octokit-client');
 
   // Fetch latest release info
   let data;

package/tools/get-redpanda-version.js

@@ -19,13 +19,8 @@ module.exports = async function getRedpandaVersion({ beta = false, fromAntora =
     useBeta = getPrereleaseFromAntora();
   }
 
-  // Load Octokit
-  const { getGitHubToken } = require('../cli-utils/github-token');
-  const { Octokit } = await import('@octokit/rest');
-  const token = getGitHubToken();
-  const octokit = token
-    ? new Octokit({ auth: token })
-    : new Octokit();
+  // Use shared Octokit client
+  const octokit = require('../cli-utils/octokit-client');
 
   // Fetch version data
   let data;

package/tools/redpanda-connect/connector-binary-analyzer.js

@@ -1,4 +1,4 @@
-const { Octokit } = require('@octokit/rest');
+const octokit = require('../../cli-utils/octokit-client');
 const { execSync, spawnSync } = require('child_process');
 const fs = require('fs');
 const path = require('path');
@@ -13,16 +13,6 @@ const https = require('https');
  * - Which connectors are self-hosted only
  */
 
-// Initialize Octokit with optional authentication
-const octokit = new Octokit({
-  auth: process.env.GITHUB_TOKEN,
-  userAgent: 'redpanda-docs-tools',
-  retry: {
-    enabled: true,
-    retries: 3
-  }
-});
-
 const REPO_OWNER = 'redpanda-data';
 const REPO_NAME = 'connect';
 

package/tools/redpanda-connect/helpers/buildConfigYaml.js

@@ -36,11 +36,16 @@ module.exports = function buildConfigYaml(type, connectorName, children, include
       return; // skip deprecated fields
     }
     if (!includeAdvanced && field.is_advanced) {
-      return; // skip advanced fields in “common” mode
+      return; // skip advanced fields in "common" mode
     }
 
-    if (field.type === 'object' && Array.isArray(field.children)) {
-      // Render nested object
+    // Check if this is an array-of-objects (e.g., client_certs[])
+    // These should render as empty arrays, not expanded object structures
+    if (field.kind === 'array' && field.type === 'object' && Array.isArray(field.children)) {
+      // Render as array leaf (e.g., "client_certs: []")
+      lines.push(renderLeafField(field, baseIndent));
+    } else if (field.type === 'object' && Array.isArray(field.children)) {
+      // Render nested object (plain object, not array)
       const nestedLines = renderObjectField(field, baseIndent);
       lines.push(...nestedLines);
     } else {

package/tools/redpanda-connect/helpers/renderObjectField.js

@@ -28,7 +28,12 @@ module.exports = function renderObjectField(field, indentLevel) {
     if (child.is_deprecated) {
       return; // skip entirely
     }
-    if (Array.isArray(child.children) && child.children.length > 0) {
+    // Check if this is an array-of-objects (e.g., client_certs[])
+    // These should render as empty arrays, not expanded object structures
+    if (child.kind === 'array' && child.type === 'object' && Array.isArray(child.children)) {
+      // Render as array leaf (e.g., "client_certs: []")
+      lines.push(renderLeafField(child, childIndent));
+    } else if (Array.isArray(child.children) && child.children.length > 0) {
       // Nested object → recurse
       lines.push(...renderObjectField(child, childIndent));
     } else {

package/tools/redpanda-connect/rpcn-connector-docs-handler.js

@@ -1072,7 +1072,9 @@ async function handleRpcnConnectorDocs (options) {
               validConnectors.push({
                 name: connector.name,
                 type: type.replace(/s$/, ''),
-                status: connector.status || connector.type || 'stable'
+                status: connector.status || connector.type || 'stable',
+                cloudOnly: connector.cloudOnly === true,
+                requiresCgo: connector.requiresCgo === true
               })
             }
           })
@@ -1119,14 +1121,175 @@ async function handleRpcnConnectorDocs (options) {
         cloudOnly: path.resolve(process.cwd(), 'modules/components/partials/components/cloud-only')
       }
 
-      const allMissing = validConnectors.filter(({ name, type }) => {
+      // Build a set of cloud-supported connectors (inCloud + cloudOnly, excluding self-hosted-only)
+      const cloudSupportedSet = new Set()
+      if (binaryAnalysis?.comparison) {
+        // inCloud = available in both OSS and Cloud
+        binaryAnalysis.comparison.inCloud?.forEach(c => {
+          cloudSupportedSet.add(`${c.type}:${c.name}`)
+        })
+        // cloudOnly = only available in Cloud (not in OSS)
+        binaryAnalysis.comparison.cloudOnly?.forEach(c => {
+          cloudSupportedSet.add(`${c.type}:${c.name}`)
+        })
+      } else {
+        // Fallback when binary analysis is unavailable:
+        // Check all connectors that have cloudSupported flag or assume all non-deprecated are cloud-supported
+        console.log('   ℹ️  Binary analysis unavailable - checking all non-deprecated connectors for cloud-docs')
+        const types = ['inputs', 'outputs', 'processors', 'caches', 'rate_limits', 'buffers', 'metrics', 'scanners', 'tracers']
+        types.forEach(type => {
+          if (Array.isArray(dataObj[type])) {
+            dataObj[type].forEach(connector => {
+              // Include if cloudSupported is explicitly true, or if it's null/undefined and not deprecated
+              const isCloudSupported = connector.cloudSupported === true ||
+                (connector.cloudSupported == null && connector.status !== 'deprecated')
+              if (isCloudSupported && connector.name) {
+                // Store type as plural to match binary analysis format
+                cloudSupportedSet.add(`${type}:${connector.name}`)
+              }
+            })
+          }
+        })
+      }
+
+      // Check for missing connector documentation in rp-connect-docs
+      const allMissing = validConnectors.filter(({ name, type, cloudOnly }) => {
         const relPath = path.join(`${type}s`, `${name}.adoc`)
-        const existsInAny = Object.values(roots).some(root =>
+
+        // For cloud-only connectors, ONLY check the cloud-only directory
+        if (cloudOnly) {
+          return !fs.existsSync(path.join(roots.cloudOnly, relPath))
+        }
+
+        // For regular connectors, check pages and partials (not cloud-only)
+        const existsInAny = [roots.pages, roots.partials].some(root =>
           fs.existsSync(path.join(root, relPath))
         )
         return !existsInAny
       })
 
+      // Check for cloud-supported connectors missing from cloud-docs repo (via GitHub API)
+      const missingFromCloudDocs = []
+      const cloudDocsErrors = []
+      if (cloudSupportedSet.size > 0 && options.checkCloudDocs !== false) {
+        console.log('\n   INFO: Checking cloud-docs repository for missing connector pages...')
+
+        // Use shared Octokit instance
+        const octokit = require('../../cli-utils/octokit-client')
+
+        try {
+          // Optimization: Fetch entire directory tree in 1 API call instead of 471 individual calls
+          console.log('   Fetching cloud-docs directory tree (1 API call)...')
+
+          let existingFiles = new Set()
+
+          try {
+            // Get the tree for the components directory
+            const { data: tree } = await octokit.git.getTree({
+              owner: 'redpanda-data',
+              repo: 'cloud-docs',
+              tree_sha: 'main:modules/develop/pages/connect/components',
+              recursive: true
+            })
+
+            // Build a set of existing file paths for O(1) lookup
+            tree.tree.forEach(item => {
+              if (item.type === 'blob' && item.path.endsWith('.adoc')) {
+                existingFiles.add(item.path)
+              }
+            })
+
+            console.log(`   Loaded ${existingFiles.size} existing connector pages from cloud-docs`)
+          } catch (treeError) {
+            console.log(`   WARNING: Could not fetch tree (${treeError.status}), falling back to individual checks`)
+            // If tree API fails, fall back to individual checks (old behavior)
+            existingFiles = null
+          }
+
+          // Check each cloud-supported connector
+          // Filter to only check actual connector/component types that need individual pages
+          const connectorTypes = ['inputs', 'outputs', 'processors', 'caches', 'buffers', 'scanners', 'metrics', 'tracers']
+
+          for (const connectorKey of cloudSupportedSet) {
+            const [type, name] = connectorKey.split(':')
+
+            // Skip non-connector types (config, bloblang-functions, bloblang-methods, rate-limits)
+            if (!connectorTypes.includes(type)) {
+              continue
+            }
+
+            // Skip deprecated connectors - they don't need cloud-docs pages
+            if (Array.isArray(dataObj[type])) {
+              const connector = dataObj[type].find(c => c.name === name)
+              if (connector && connector.status === 'deprecated') {
+                continue
+              }
+            }
+
+            const relativePath = `${type}/${name}.adoc`
+            const fullPath = `modules/develop/pages/connect/components/${relativePath}`
+
+            // Fast path: Check against tree if we have it
+            if (existingFiles !== null) {
+              if (!existingFiles.has(relativePath)) {
+                missingFromCloudDocs.push({ type, name, path: fullPath })
+              }
+              continue
+            }
+
+            // Fallback path: Individual API calls (only if tree fetch failed)
+            try {
+              await octokit.repos.getContent({
+                owner: 'redpanda-data',
+                repo: 'cloud-docs',
+                path: fullPath,
+                ref: 'main'
+              })
+              // File exists, no action needed
+            } catch (error) {
+              if (error.status === 404) {
+                // File doesn't exist in cloud-docs
+                missingFromCloudDocs.push({ type, name, path: fullPath })
+              } else {
+                // Non-404 error - record as error
+                cloudDocsErrors.push({
+                  type,
+                  name,
+                  path: fullPath,
+                  status: error.status || 'unknown',
+                  message: error.message
+                })
+              }
+            }
+          }
+
+          // Report results
+          if (cloudDocsErrors.length > 0) {
+            console.log(`   WARNING: Encountered ${cloudDocsErrors.length} error(s) while checking cloud-docs (check inconclusive):`)
+            cloudDocsErrors.forEach(({ type, name, status, message }) => {
+              console.log(`      - ${type}/${name} - Status ${status}: ${message}`)
+            })
+            console.log(`   INFO: Please resolve these errors (e.g., check GITHUB_TOKEN or VBOT_GITHUB_API_TOKEN, API rate limits, network connectivity)`)
+            if (missingFromCloudDocs.length > 0) {
+              console.log(`   INFO: Additionally, ${missingFromCloudDocs.length} connector(s) confirmed missing from cloud-docs:`)
+              missingFromCloudDocs.forEach(({ type, name }) => {
+                console.log(`      - ${type}/${name}`)
+              })
+            }
+          } else if (missingFromCloudDocs.length > 0) {
+            console.log(`   WARNING: Found ${missingFromCloudDocs.length} cloud-supported connector(s) missing from cloud-docs:`)
+            missingFromCloudDocs.forEach(({ type, name }) => {
+              console.log(`      - ${type}/${name}`)
+            })
+            console.log(`   INFO: These connectors need pages added to https://github.com/redpanda-data/cloud-docs`)
+          } else {
+            console.log(`   All cloud-supported connectors have pages in cloud-docs`)
+          }
+        } catch (error) {
+          console.log(`   WARNING: Could not check cloud-docs: ${error.message}`)
+        }
+      }
+
       const missingConnectors = allMissing.filter(c =>
         !c.name.includes('sql_driver') &&
         c.status !== 'deprecated'