@redpanda-data/docs-extensions-and-macros 4.13.5 → 4.13.6

package/cli-utils/github-token.js

@@ -9,14 +9,18 @@
  * Get GitHub token from environment variables
  * Checks multiple common variable names in priority order:
  * 1. REDPANDA_GITHUB_TOKEN - Custom Redpanda token
- * 2. GITHUB_TOKEN - GitHub Actions default
- * 3. GH_TOKEN - GitHub CLI default
+ * 2. ACTIONS_BOT_TOKEN - GitHub Actions bot token
+ * 3. GITHUB_TOKEN - GitHub Actions default
+ * 4. VBOT_GITHUB_API_TOKEN - Legacy bot token
+ * 5. GH_TOKEN - GitHub CLI default
  *
  * @returns {string|null} GitHub token or null if not found
  */
 function getGitHubToken() {
   return process.env.REDPANDA_GITHUB_TOKEN ||
+         process.env.ACTIONS_BOT_TOKEN ||
          process.env.GITHUB_TOKEN ||
+         process.env.VBOT_GITHUB_API_TOKEN ||
          process.env.GH_TOKEN ||
          null;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redpanda-data/docs-extensions-and-macros",
-  "version": "4.13.5",
+  "version": "4.13.6",
   "description": "Antora extensions and macros developed for Redpanda documentation.",
   "keywords": [
     "antora",

package/tools/redpanda-connect/rpcn-connector-docs-handler.js

@@ -1172,12 +1172,40 @@ async function handleRpcnConnectorDocs (options) {
       const missingFromCloudDocs = []
       const cloudDocsErrors = []
       if (cloudSupportedSet.size > 0 && options.checkCloudDocs !== false) {
-        console.log('\n   ℹ️  Checking cloud-docs repository for missing connector pages...')
+        console.log('\n   INFO: Checking cloud-docs repository for missing connector pages...')
 
         // Use shared Octokit instance
         const octokit = require('../../cli-utils/octokit-client')
 
         try {
+          // Optimization: Fetch entire directory tree in 1 API call instead of 471 individual calls
+          console.log('   Fetching cloud-docs directory tree (1 API call)...')
+
+          let existingFiles = new Set()
+
+          try {
+            // Get the tree for the components directory
+            const { data: tree } = await octokit.git.getTree({
+              owner: 'redpanda-data',
+              repo: 'cloud-docs',
+              tree_sha: 'main:modules/develop/pages/connect/components',
+              recursive: true
+            })
+
+            // Build a set of existing file paths for O(1) lookup
+            tree.tree.forEach(item => {
+              if (item.type === 'blob' && item.path.endsWith('.adoc')) {
+                existingFiles.add(item.path)
+              }
+            })
+
+            console.log(`   Loaded ${existingFiles.size} existing connector pages from cloud-docs`)
+          } catch (treeError) {
+            console.log(`   WARNING: Could not fetch tree (${treeError.status}), falling back to individual checks`)
+            // If tree API fails, fall back to individual checks (old behavior)
+            existingFiles = null
+          }
+
           // Check each cloud-supported connector
           // Filter to only check actual connector/component types that need individual pages
           const connectorTypes = ['inputs', 'outputs', 'processors', 'caches', 'buffers', 'scanners', 'metrics', 'tracers']
@@ -1198,95 +1226,67 @@ async function handleRpcnConnectorDocs (options) {
               }
             }
 
-            const cloudDocsPath = `modules/develop/pages/connect/components/${type}/${name}.adoc`
+            const relativePath = `${type}/${name}.adoc`
+            const fullPath = `modules/develop/pages/connect/components/${relativePath}`
+
+            // Fast path: Check against tree if we have it
+            if (existingFiles !== null) {
+              if (!existingFiles.has(relativePath)) {
+                missingFromCloudDocs.push({ type, name, path: fullPath })
+              }
+              continue
+            }
 
+            // Fallback path: Individual API calls (only if tree fetch failed)
             try {
               await octokit.repos.getContent({
                 owner: 'redpanda-data',
                 repo: 'cloud-docs',
-                path: cloudDocsPath,
+                path: fullPath,
                 ref: 'main'
               })
               // File exists, no action needed
             } catch (error) {
               if (error.status === 404) {
                 // File doesn't exist in cloud-docs
-                missingFromCloudDocs.push({ type, name, path: cloudDocsPath })
+                missingFromCloudDocs.push({ type, name, path: fullPath })
               } else {
-                // Non-404 error (auth, rate-limit, network, etc.)
-                // Try fallback: check raw URL without authentication
-                const rawUrl = `https://raw.githubusercontent.com/redpanda-data/cloud-docs/main/${cloudDocsPath}`
-                try {
-                  const https = require('https')
-                  const { URL } = require('url')
-                  const parsedUrl = new URL(rawUrl)
-
-                  await new Promise((resolve, reject) => {
-                    const req = https.request({
-                      hostname: parsedUrl.hostname,
-                      path: parsedUrl.pathname,
-                      method: 'HEAD',
-                      timeout: 5000
-                    }, (res) => {
-                      if (res.statusCode === 200) {
-                        resolve() // File exists
-                      } else if (res.statusCode === 404) {
-                        reject(new Error('404'))
-                      } else {
-                        reject(new Error(`Status ${res.statusCode}`))
-                      }
-                    })
-                    req.on('error', reject)
-                    req.on('timeout', () => {
-                      req.destroy()
-                      reject(new Error('Timeout'))
-                    })
-                    req.end()
-                  })
-                  // Fallback succeeded - file exists, no action needed
-                } catch (fallbackError) {
-                  if (fallbackError.message === '404') {
-                    // Confirmed missing via fallback
-                    missingFromCloudDocs.push({ type, name, path: cloudDocsPath })
-                  } else {
-                    // Both API and fallback failed
-                    cloudDocsErrors.push({
-                      type,
-                      name,
-                      path: cloudDocsPath,
-                      status: error.status || 'unknown',
-                      message: `API: ${error.message}; Fallback: ${fallbackError.message}`
-                    })
-                  }
-                }
+                // Non-404 error - record as error
+                cloudDocsErrors.push({
+                  type,
+                  name,
+                  path: fullPath,
+                  status: error.status || 'unknown',
+                  message: error.message
+                })
               }
             }
           }
 
           // Report results
           if (cloudDocsErrors.length > 0) {
-            console.log(`   ⚠️  Encountered ${cloudDocsErrors.length} error(s) while checking cloud-docs (check inconclusive):`)
+            console.log(`   WARNING: Encountered ${cloudDocsErrors.length} error(s) while checking cloud-docs (check inconclusive):`)
             cloudDocsErrors.forEach(({ type, name, status, message }) => {
-              console.log(`      • ${type}/${name} - Status ${status}: ${message}`)
+              console.log(`      - ${type}/${name} - Status ${status}: ${message}`)
             })
-            console.log(`   ℹ️  Please resolve these errors (e.g., check GITHUB_TOKEN or VBOT_GITHUB_API_TOKEN, API rate limits, network connectivity)`)
+            console.log(`   INFO: Please resolve these errors (e.g., check GITHUB_TOKEN or VBOT_GITHUB_API_TOKEN, API rate limits, network connectivity)`)
             if (missingFromCloudDocs.length > 0) {
-              console.log(`   ℹ️  Additionally, ${missingFromCloudDocs.length} connector(s) confirmed missing from cloud-docs:`)
+              console.log(`   INFO: Additionally, ${missingFromCloudDocs.length} connector(s) confirmed missing from cloud-docs:`)
               missingFromCloudDocs.forEach(({ type, name }) => {
-                console.log(`      • ${type}/${name}`)
+                console.log(`      - ${type}/${name}`)
               })
             }
           } else if (missingFromCloudDocs.length > 0) {
-            console.log(`   ⚠️  Found ${missingFromCloudDocs.length} cloud-supported connector(s) missing from cloud-docs:`)
+            console.log(`   WARNING: Found ${missingFromCloudDocs.length} cloud-supported connector(s) missing from cloud-docs:`)
             missingFromCloudDocs.forEach(({ type, name }) => {
-              console.log(`      • ${type}/${name}`)
+              console.log(`      - ${type}/${name}`)
             })
-            console.log(`   ℹ️  These connectors need pages added to https://github.com/redpanda-data/cloud-docs`)
+            console.log(`   INFO: These connectors need pages added to https://github.com/redpanda-data/cloud-docs`)
           } else {
-            console.log(`   ✓  All cloud-supported connectors have pages in cloud-docs`)
+            console.log(`   All cloud-supported connectors have pages in cloud-docs`)
           }
         } catch (error) {
-          console.log(`   ⚠️  Could not check cloud-docs: ${error.message}`)
+          console.log(`   WARNING: Could not check cloud-docs: ${error.message}`)
         }
       }