@redpanda-data/docs-extensions-and-macros 5.12.3 → 5.12.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redpanda-data/docs-extensions-and-macros",
-  "version": "5.12.3",
+  "version": "5.12.4",
   "description": "Antora extensions and macros developed for Redpanda documentation.",
   "keywords": [
     "antora",

package/tools/redpanda-connect/connector-binary-analyzer.js

@@ -1,5 +1,5 @@
 const octokit = require('../../cli-utils/octokit-client');
-const { execSync, spawnSync } = require('child_process');
+const { execFileSync, spawnSync } = require('child_process');
 const fs = require('fs');
 const path = require('path');
 const os = require('os');
@@ -16,6 +16,31 @@ const https = require('https');
 const REPO_OWNER = 'redpanda-data';
 const REPO_NAME = 'connect';
 
+// Release asset prefixes per binary flavour. The plain OSS build is the one
+// `rpk connect install` puts on PATH, so fetching that asset directly yields
+// the same connector schema without passing a version through rpk's
+// --connect-version validation.
+const BINARY_ASSET_PREFIXES = {
+  oss: 'redpanda-connect',
+  cloud: 'redpanda-connect-cloud',
+  cgo: 'redpanda-connect-cgo'
+};
+
+/**
+ * Resolve a binary flavour to its release asset prefix.
+ * @param {string} binaryType - 'oss', 'cloud' or 'cgo'
+ * @returns {string} The asset name prefix
+ */
+function resolveAssetPrefix (binaryType) {
+  const prefix = BINARY_ASSET_PREFIXES[binaryType];
+  if (!prefix) {
+    throw new Error(
+      `Unknown binary type "${binaryType}" (expected one of ${Object.keys(BINARY_ASSET_PREFIXES).join(', ')})`
+    );
+  }
+  return prefix;
+}
+
 /**
  * Get platform-specific binary name
  * @returns {Object} Platform and arch info
@@ -46,10 +71,15 @@ function getPlatformInfo() {
 
 /**
  * Get the latest release version from GitHub
- * @param {string} binaryType - Type of binary ('cloud' or 'cgo')
+ * @param {string} binaryType - Type of binary ('oss', 'cloud' or 'cgo')
  * @returns {Promise<string>} Version string (e.g., "4.76.0")
  */
 async function getLatestVersion(binaryType = 'cloud') {
+  // Resolved outside the try so an unknown flavour surfaces its own message
+  // instead of being wrapped as a failure to reach the GitHub API. tools/ is
+  // published, so callers outside this repo see this error.
+  const prefix = resolveAssetPrefix(binaryType);
+
   try {
     const { data: releases } = await octokit.repos.listReleases({
       owner: REPO_OWNER,
@@ -57,12 +87,12 @@ async function getLatestVersion(binaryType = 'cloud') {
       per_page: 20
     });
 
-    const prefix = binaryType === 'cloud' ? 'redpanda-connect-cloud' : 'redpanda-connect-cgo';
-
-    // Find the latest release with the specified binary type
+    // Find the latest release with the specified binary type. Match the
+    // separator too: every prefix starts with 'redpanda-connect', so a bare
+    // startsWith on the OSS prefix would also match the cloud and cgo assets.
     for (const release of releases) {
       const asset = release.assets.find(a =>
-        a.name.startsWith(prefix) &&
+        a.name.startsWith(`${prefix}_`) &&
         a.name.includes('linux') &&
         a.name.includes('amd64')
       );
@@ -124,9 +154,99 @@ function downloadFile(url, destPath) {
   });
 }
 
+/**
+ * Directory that holds downloaded Connect binaries for the life of the process.
+ *
+ * Binaries are ~320 MB extracted, so they never belong in docs-data: the
+ * consuming repo commits that directory wholesale, and a run killed before its
+ * cleanup would leave a blob over GitHub's 100 MB file limit staged for the
+ * auto-docs PR. One directory for the whole run also lets downloadBinary's
+ * skip-if-present check serve repeat requests: the multi-release loop asks for
+ * every interior version twice (as the new side of one diff pair and the old
+ * side of the next), and a per-call directory made that a second download.
+ *
+ * @returns {string} Path to the process-wide binary cache directory
+ */
+let binaryCacheDir = null;
+function getBinaryCacheDir () {
+  if (binaryCacheDir && fs.existsSync(binaryCacheDir)) return binaryCacheDir;
+
+  binaryCacheDir = fs.mkdtempSync(path.join(os.tmpdir(), 'rpcn-binaries-'));
+  const created = binaryCacheDir;
+  process.on('exit', () => {
+    try {
+      fs.rmSync(created, { recursive: true, force: true });
+    } catch (err) {
+      // Nothing useful to do while exiting; the OS reclaims tmpdir anyway.
+    }
+  });
+  return binaryCacheDir;
+}
+
+// Cap the cache: a backfill can touch dozens of versions, but no more than a
+// handful of binaries are live at once (the two OSS versions of the current
+// diff pair plus the cloud and cgo binaries analysed alongside them). Without
+// a cap, sharing one directory would trade repeat downloads for unbounded disk.
+const BINARY_CACHE_LIMIT = 6;
+const binaryCacheUse = [];
+
+/**
+ * Record a binary as most recently used and evict the oldest beyond the cap.
+ * @param {string} destPath - Path to the binary just downloaded or reused
+ */
+function touchBinaryCache (destPath) {
+  const existing = binaryCacheUse.indexOf(destPath);
+  if (existing !== -1) binaryCacheUse.splice(existing, 1);
+  binaryCacheUse.push(destPath);
+
+  while (binaryCacheUse.length > BINARY_CACHE_LIMIT) {
+    const evicted = binaryCacheUse.shift();
+    try {
+      fs.rmSync(evicted, { force: true });
+    } catch (err) {
+      console.warn(`Warning: could not evict cached binary ${evicted}: ${err.message}`);
+    }
+  }
+}
+
+/**
+ * Extract the Connect binary out of a release tarball.
+ *
+ * Extraction goes into an empty directory of its own because the archive's
+ * inner file name is not the destination name, and the destination directory
+ * holds other flavours and versions whose names also contain
+ * 'redpanda-connect'. Scanning the shared directory instead would pick a
+ * neighbour in readdir order and analyse, say, the cloud binary as the cgo one.
+ *
+ * @param {string} tarballPath - Path to the downloaded .tar.gz
+ * @param {string} destPath - Path the extracted binary should end up at
+ * @returns {string} destPath
+ */
+function extractBinaryFromTarball (tarballPath, destPath) {
+  const extractDir = fs.mkdtempSync(`${destPath}.extract-`);
+  try {
+    execFileSync('tar', ['-xzf', tarballPath, '-C', extractDir], { stdio: 'ignore' });
+
+    const candidates = fs.readdirSync(extractDir)
+      .filter(name => name.includes('redpanda-connect') && !name.endsWith('.tar.gz'))
+      .filter(name => fs.statSync(path.join(extractDir, name)).isFile())
+      .sort();
+
+    if (candidates.length === 0) {
+      throw new Error('Binary not found after extraction');
+    }
+
+    fs.renameSync(path.join(extractDir, candidates[0]), destPath);
+    fs.chmodSync(destPath, 0o755);
+    return destPath;
+  } finally {
+    fs.rmSync(extractDir, { recursive: true, force: true });
+  }
+}
+
 /**
  * Download a binary from GitHub releases
- * @param {string} binaryType - 'cloud' or 'cgo'
+ * @param {string} binaryType - 'oss', 'cloud' or 'cgo'
  * @param {string} version - Version to download (e.g., "4.76.0")
  * @param {string} destDir - Destination directory
  * @returns {Promise<string>} Path to downloaded binary
@@ -136,13 +256,14 @@ async function downloadBinary(binaryType, version, destDir) {
   const platformInfo = binaryType === 'cgo'
     ? { platform: 'linux', arch: 'amd64' }
     : getPlatformInfo();
-  const prefix = binaryType === 'cloud' ? 'redpanda-connect-cloud' : 'redpanda-connect-cgo';
+  const prefix = resolveAssetPrefix(binaryType);
   const binaryName = `${prefix}-${version}-${platformInfo.platform}-${platformInfo.arch}`;
   const destPath = path.join(destDir, binaryName);
 
   // Skip download if already exists
   if (fs.existsSync(destPath)) {
     console.log(`✓ ${binaryType.toUpperCase()} binary already downloaded: ${binaryName}`);
+    touchBinaryCache(destPath);
     return destPath;
   }
 
@@ -170,35 +291,12 @@ async function downloadBinary(binaryType, version, destDir) {
 
     // Extract the binary
     console.log(`Extracting ${binaryType.toUpperCase()} binary...`);
-    execSync(`tar -xzf "${tarballPath}" -C "${destDir}"`, { stdio: 'ignore' });
-
-    // Find the extracted binary
-    const extractedFiles = fs.readdirSync(destDir);
-    let binaryPath = null;
-
-    for (const file of extractedFiles) {
-      const fullPath = path.join(destDir, file);
-      if (fs.statSync(fullPath).isFile() && file.includes('redpanda-connect') && !file.endsWith('.tar.gz')) {
-        binaryPath = fullPath;
-        break;
-      }
-    }
-
-    if (!binaryPath) {
-      throw new Error('Binary not found after extraction');
-    }
-
-    // Rename to standard name if needed
-    if (binaryPath !== destPath) {
-      fs.renameSync(binaryPath, destPath);
-    }
-
-    // Make executable
-    fs.chmodSync(destPath, 0o755);
+    extractBinaryFromTarball(tarballPath, destPath);
 
     // Clean up tarball
     fs.unlinkSync(tarballPath);
 
+    touchBinaryCache(destPath);
     console.log(`Done: Downloaded ${binaryType.toUpperCase()} binary: ${binaryName}`);
     return destPath;
   } catch (error) {
@@ -435,9 +533,10 @@ async function analyzeAllBinaries(ossVersion, cloudVersion = null, dataDir = nul
 
   const docsDataDir = dataDir || path.resolve(process.cwd(), 'docs-data');
 
-  // Use temp directory for binaries (not docs-data)
-  const os = require('os');
-  const binaryDir = fs.mkdtempSync(path.join(os.tmpdir(), 'rpcn-binaries-'));
+  // Use temp directory for binaries (not docs-data). Shared with every other
+  // download in this process, so a multi-release run fetches each cloud and cgo
+  // binary once instead of once per diff pair.
+  const binaryDir = getBinaryCacheDir();
 
   console.log(`📁 Using temp directory for binaries: ${binaryDir}`);
 
@@ -508,13 +607,8 @@ async function analyzeAllBinaries(ossVersion, cloudVersion = null, dataDir = nul
     console.log('⏩ Skipping cgo binary analysis');
   }
 
-  // Clean up temp binaries
-  try {
-    fs.rmSync(binaryDir, { recursive: true, force: true });
-    console.log(`🧹 Cleaned up temp binaries`);
-  } catch (err) {
-    console.warn(`Warning: Could not clean up temp directory: ${err.message}`);
-  }
+  // Binaries stay in the shared cache so the next diff pair can reuse them; the
+  // directory is removed when the process exits (see getBinaryCacheDir).
 
   // IMPORTANT: cgoOnly contains connectors that exist ONLY in the cgo binary
   // and NOT in the regular OSS binary (rpk connect list). These are the
@@ -566,6 +660,11 @@ module.exports = {
   analyzeAllBinaries,
   findCgoOnlyConnectors,
   getLatestVersion,
+  getPlatformInfo,
+  resolveAssetPrefix,
+  getBinaryCacheDir,
+  extractBinaryFromTarball,
+  downloadBinary,
   downloadCloudBinary,
   downloadCgoBinary,
   getConnectorList,

package/tools/redpanda-connect/generate-rpcn-connector-docs.js

@@ -5,7 +5,7 @@ const path = require('path');
 const handlebars = require('handlebars');
 const yaml = require('yaml');
 const helpers = require('./helpers');
-const { lostMetadataSections } = require('./metadata-utils.js');
+const { lostMetadataSections, resolvePageTypeDir } = require('./metadata-utils.js');
 
 // Register each helper under handlebars, verifying that it’s a function
 Object.entries(helpers).forEach(([name, fn]) => {
@@ -730,11 +730,7 @@ function backfillPageDescriptions (connectorData, { pagesRoot, dryRun = false }
 
   for (const [dataKey, items] of Object.entries(connectorData)) {
     if (!PAGE_TYPE_DIRS.has(dataKey) || !Array.isArray(items)) continue;
-    // The data uses 'rate-limits' while the pages directory is
-    // 'rate_limits'. Resolve whichever spelling exists on disk.
-    const typeDir = fs.existsSync(path.join(root, dataKey))
-      ? dataKey
-      : dataKey.replace(/-/g, '_');
+    const typeDir = resolvePageTypeDir(root, dataKey);
     for (const item of items) {
       if (!item || !item.name) continue;
       const pagePath = path.join(root, typeDir, `${item.name}.adoc`);

package/tools/redpanda-connect/metadata-utils.js

@@ -1,5 +1,8 @@
 'use strict';
 
+const fs = require('fs');
+const path = require('path');
+
 /**
  * Utilities for extracting the "== Metadata" section out of a connector's
  * `description` prose so it can be emitted as a regenerated partial (like the
@@ -87,6 +90,31 @@ function typeDirFor (item) {
   return type.endsWith('s') ? type : `${type}s`;
 }
 
+/**
+ * Resolve the on-disk page/partial directory for a connector data key.
+ *
+ * The Connect binary emits the rate-limit family as `rate-limits` while the
+ * directory in rp-connect-docs is `rate_limits`, so the data key and the
+ * directory name disagree for exactly one family. Resolve against what is
+ * actually on disk rather than encoding either spelling: a caller that guesses
+ * `rate-limits` reports every existing rate-limit page as missing and then
+ * drafts a duplicate beside it, in a directory update-nav.js's regex does not
+ * match, re-added on every run.
+ *
+ * @param {string} root directory the type directories live under
+ * @param {string} dataKey the data key, for example `inputs` or `rate-limits`
+ * @returns {string} the directory name that exists, preferring the data key
+ */
+function resolvePageTypeDir (root, dataKey) {
+  if (!root || !dataKey) return dataKey || '';
+  if (fs.existsSync(path.join(root, dataKey))) return dataKey;
+  const underscored = dataKey.replace(/-/g, '_');
+  if (underscored !== dataKey && fs.existsSync(path.join(root, underscored))) return underscored;
+  const hyphenated = dataKey.replace(/_/g, '-');
+  if (hyphenated !== dataKey && fs.existsSync(path.join(root, hyphenated))) return hyphenated;
+  return dataKey;
+}
+
 /**
  * Build the Antora include directive for a connector's metadata partial.
  * @param {object} item connector data with `type`/`typeDir` and `name`
@@ -170,6 +198,7 @@ function lostMetadataSections (oldContent, newContent) {
 
 module.exports = {
   locateMetadata,
+  resolvePageTypeDir,
   extractMetadata,
   typeDirFor,
   metadataIncludeLine,

package/tools/redpanda-connect/rpcn-connector-docs-handler.js

@@ -11,6 +11,7 @@ const { getRpkConnectVersion, printDeltaReport } = require('./report-delta')
 const { discoverIntermediateReleases, findCloudVersionForDate } = require('./github-release-utils')
 const { analyzeAllBinaries } = require('./connector-binary-analyzer.js')
 const parseCSVConnectors = require('./parse-csv-connectors.js')
+const { resolvePageTypeDir } = require('./metadata-utils.js')
 const semver = require('semver')
 
 /**
@@ -106,6 +107,23 @@ function capToTwoSentences (description) {
   return result.trim()
 }
 
+// Component families in a connector index, in the spelling the Connect binary
+// emits. Rate limits are 'rate-limits' there; 'rate_limits' is the docs page
+// directory spelling, which older snapshots also carry as an (always empty)
+// data key, so it is walked when present but never created. Every copy of this
+// list used to be the underscore form only, which meant rate-limit components
+// were never given cloudSupported or requiresCgo flags and any cgo-only or
+// cloud-only rate limit was dropped from the published snapshot.
+const COMPONENT_TYPE_KEYS = ['inputs', 'outputs', 'processors', 'caches', 'rate-limits',
+  'rate_limits', 'buffers', 'metrics', 'scanners', 'tracers']
+
+// 'config' components live in the same index but have no config/fields wrapper.
+const COMPONENT_TYPE_KEYS_WITH_CONFIG = [...COMPONENT_TYPE_KEYS, 'config']
+
+// Keys kept only for snapshots written before the spelling was fixed. Walked
+// if a snapshot has them, never added to one.
+const LEGACY_COMPONENT_TYPE_KEYS = new Set(['rate_limits'])
+
 /**
  * Remove platform metadata fields from connectors
  * @param {Array} connectors - Array of connector objects
@@ -146,11 +164,9 @@ function augmentConnectorData (connectorData, binaryAnalysis) {
   let addedCgoCount = 0
   let addedCloudOnlyCount = 0
 
-  const connectorTypes = ['inputs', 'outputs', 'processors', 'caches', 'rate_limits',
-    'buffers', 'metrics', 'scanners', 'tracers', 'config']
-
-  for (const type of connectorTypes) {
+  for (const type of COMPONENT_TYPE_KEYS_WITH_CONFIG) {
     if (!Array.isArray(augmentedData[type])) {
+      if (LEGACY_COMPONENT_TYPE_KEYS.has(type)) continue
       augmentedData[type] = []
     }
 
@@ -219,10 +235,7 @@ function augmentConnectorData (connectorData, binaryAnalysis) {
  */
 function buildCleanOssData (connectorIndex) {
   const cleanData = JSON.parse(JSON.stringify(connectorIndex))
-  const connectorTypes = ['inputs', 'outputs', 'processors', 'caches', 'rate_limits',
-    'buffers', 'metrics', 'scanners', 'tracers']
-
-  for (const type of connectorTypes) {
+  for (const type of COMPONENT_TYPE_KEYS) {
     if (Array.isArray(cleanData[type])) {
       // Keep only connectors from OSS rpk (have config/fields)
       // Remove augmentation-only connectors (added by previous binary analysis)
@@ -703,11 +716,52 @@ function logCollapsed (label, filesArray, maxToShow = 10) {
 }
 
 
+/**
+ * Why a stored connect-<version>.json cannot stand in for a fresh fetch.
+ *
+ * The handler writes augmentConnectorData() output back over the same file, so
+ * a stored snapshot is not necessarily raw binary output. Reusing an augmented
+ * one would re-augment already-augmented data: the cloud-only and cgo-only
+ * entries injected by the previous run are not in the new analysis, so they
+ * survive with their platform markers reset and a cgo-only connector ends up
+ * published as an ordinary OSS one. The version field is the only provenance
+ * the downstream templates get, so a file that disagrees with its own name
+ * cannot be trusted either.
+ *
+ * @param {Object} data - Parsed contents of connect-<version>.json
+ * @param {string} version - Version the file is supposed to describe
+ * @returns {string|null} Reason it cannot be reused, or null if it can
+ */
+function snapshotReuseBlocker (data, version) {
+  if (!data || typeof data !== 'object' || Array.isArray(data)) {
+    return 'file is not a connector index'
+  }
+
+  if (typeof data.version === 'string' && data.version !== version) {
+    return `file reports version ${data.version}`
+  }
+
+  const augmented = Object.values(data)
+    .filter(Array.isArray)
+    .some(components => components.some(c => c && typeof c === 'object' && (
+      c.cloudSupported !== undefined ||
+      c.requiresCgo !== undefined ||
+      c.cloudOnly !== undefined
+    )))
+
+  if (augmented) {
+    return 'file already carries cloud/cgo augmentation from an earlier run'
+  }
+
+  return null
+}
+
 /**
  * Load or fetch connector data for a specific version
  * @param {string} version - Version to load (e.g., "4.50.0")
  * @param {string} dataDir - Directory where JSON files are stored
  * @param {Object} options - Options for fetching if needed
+ * @param {boolean} [options.forceFresh] - Refetch even when a usable file exists
  * @returns {Promise<Object>} Parsed connector data
  */
 async function loadConnectorDataForVersion(version, dataDir, options = {}) {
@@ -715,51 +769,75 @@ async function loadConnectorDataForVersion(version, dataDir, options = {}) {
 
   // If forceFresh is set, always fetch even if file exists
   // This ensures we have accurate connector lists for diff comparison
+  let reason = options.forceFresh ? 'force refresh requested' : 'file not found';
+
   if (!options.forceFresh && fs.existsSync(dataFile)) {
-    console.log(`✓ Using existing data file: connect-${version}.json`);
-    const data = JSON.parse(fs.readFileSync(dataFile, 'utf8'));
-    return data;
+    let stored = null;
+    let blocker = null;
+    try {
+      stored = JSON.parse(fs.readFileSync(dataFile, 'utf8'));
+      blocker = snapshotReuseBlocker(stored, version);
+    } catch (err) {
+      blocker = `file could not be parsed (${err.message})`;
+    }
+
+    if (!blocker) {
+      console.log(`✓ Using existing data file: connect-${version}.json`);
+      return stored;
+    }
+
+    console.warn(`↻ Ignoring existing connect-${version}.json: ${blocker}`);
+    reason = blocker;
   }
 
   // Fetch fresh data
-  const reason = options.forceFresh ? 'force refresh requested' : 'file not found';
   console.log(`📥 Fetching data for ${version} (${reason})...`);
 
+  // Fetch the OSS release asset for this exact version instead of asking rpk to
+  // install it. rpk's validateVersion (rpk/pkg/cli/connect/install.go) matches
+  // --connect-version against a regex that caps each segment at two digits, so
+  // it rejects every version >= 4.100.0, which is every version worth asking
+  // for. The cap is lifted in rpk v26.1.15 and on dev, but v26.1.14 and the
+  // whole v26.2.x line still carry it, so which versions can be installed
+  // depends on which rpk happens to be on PATH. Fetching the asset removes that
+  // dependency instead of working around one rpk release, so this is the
+  // permanent mechanism rather than something to revert once rpk ships a fix:
+  // the plain OSS asset is the same build rpk would have installed, so the
+  // schema is identical, and this leaves the caller's own Connect installation
+  // untouched instead of forcing a managed plugin over the top of it.
   try {
-    // Try installing that specific version and fetching data
-    console.log(`   Installing Redpanda Connect version ${version}...`);
-    const installResult = spawnSync('rpk', ['connect', 'install', '--connect-version', version, '--force'], {
-      stdio: 'pipe'
-    });
-
-    if (installResult.status !== 0) {
-      throw new Error(`Failed to install Connect version ${version}`);
-    }
+    const { downloadBinary, getConnectorList, getBinaryCacheDir } = require('./connector-binary-analyzer.js');
 
-    // Fetch connector list
-    const tmpFile = path.join(dataDir, `connect-${version}.tmp.json`);
-    const fd = fs.openSync(tmpFile, 'w');
-    const listResult = spawnSync('rpk', ['connect', 'list', '--format', 'json-full'], {
-      stdio: ['ignore', fd, 'pipe']
-    });
-    fs.closeSync(fd);
-
-    if (listResult.status !== 0) {
-      throw new Error(`Failed to fetch connector list for version ${version}`);
+    // Binaries go to the process-wide temp cache, never to dataDir: the
+    // consuming repo commits dataDir, and the extracted binary is ~320 MB.
+    const binaryPath = await downloadBinary('oss', version, getBinaryCacheDir());
+    const parsed = getConnectorList(binaryPath);
+
+    if (!parsed || typeof parsed !== 'object') {
+      throw new Error('connector list was empty or not an object');
     }
 
-    // Parse and validate
-    const rawJson = fs.readFileSync(tmpFile, 'utf8');
-    const parsed = JSON.parse(rawJson);
+    // The asset name embeds the version and the release was fetched by tag, so
+    // a binary that reports a different version means the wrong asset was
+    // downloaded. Fail rather than publish a snapshot under the wrong name.
+    if (typeof parsed.version === 'string' && parsed.version !== version) {
+      throw new Error(`asset reports version ${parsed.version}, not ${version}`);
+    }
 
-    // Move to final location
+    // Write via a temp file so an interrupted run cannot leave a half-written
+    // snapshot that the next run would happily reuse.
+    const tmpFile = path.join(dataDir, `connect-${version}.tmp.json`);
+    fs.writeFileSync(tmpFile, JSON.stringify(parsed, null, 2));
     fs.renameSync(tmpFile, dataFile);
 
     console.log(`✓ Successfully fetched data for version ${version}`);
     return parsed;
   } catch (error) {
     console.error(`❌ Failed to fetch data for version ${version}: ${error.message}`);
-    throw new Error(`Cannot process version ${version} - data unavailable`);
+    // Carry the reason into the thrown error: this message is what reaches the
+    // failure summary and the PR body, where a bare "data unavailable" sends
+    // the reader back to the raw workflow log.
+    throw new Error(`Cannot process version ${version} - data unavailable (${error.message})`);
   }
 }
 
@@ -789,34 +867,38 @@ async function handleRpcnConnectorDocs (options) {
           console.error('Expected format: X.Y.Z (e.g., 4.50.0)')
           process.exit(1)
         }
-        console.log(`Installing Redpanda Connect version ${options.connectVersion}...`)
-        const installResult = spawnSync('rpk', ['connect', 'install', '--connect-version', options.connectVersion, '--force'], {
-          stdio: 'inherit'
-        })
-        if (installResult.status !== 0) {
-          throw new Error(`Failed to install Connect version ${options.connectVersion}`)
-        }
-        console.log(`Done: Installed Redpanda Connect version ${options.connectVersion}`)
         newVersion = options.connectVersion
+        // An explicit version comes from the release asset rather than from
+        // rpk, which rejects anything >= 4.100.0 on the rpk releases this
+        // pipeline runs. See loadConnectorDataForVersion. Reuses an existing
+        // snapshot when it is raw, unaugmented output for this same version,
+        // and leaves the caller's own Connect install alone either way.
+        console.log(`Fetching connector data for Connect ${newVersion}...`)
+        await loadConnectorDataForVersion(newVersion, dataDir)
+        dataFile = path.join(dataDir, `connect-${newVersion}.json`)
+        needsAugmentation = true
+        console.log(`Done: Fetched connector data for version ${newVersion}`)
       } else {
+        // No version asked for, so read whatever rpk has: this is install's
+        // unvalidated "latest" path, which the version regex never sees.
         newVersion = getRpkConnectVersion()
+        console.log(`Fetching connector data from Connect ${newVersion}...`)
+
+        const tmpFile = path.join(dataDir, `connect-${newVersion}.tmp.json`)
+        const finalFile = path.join(dataDir, `connect-${newVersion}.json`)
+
+        const fd = fs.openSync(tmpFile, 'w')
+        spawnSync('rpk', ['connect', 'list', '--format', 'json-full'], { stdio: ['ignore', fd, 'inherit'] })
+        fs.closeSync(fd)
+
+        const rawJson = fs.readFileSync(tmpFile, 'utf8')
+        const parsed = JSON.parse(rawJson)
+        fs.writeFileSync(finalFile, JSON.stringify(parsed, null, 2))
+        fs.unlinkSync(tmpFile)
+        dataFile = finalFile
+        needsAugmentation = true
+        console.log(`Done: Fetched connector data for version ${newVersion}`)
       }
-      console.log(`Fetching connector data from Connect ${newVersion}...`)
-
-      const tmpFile = path.join(dataDir, `connect-${newVersion}.tmp.json`)
-      const finalFile = path.join(dataDir, `connect-${newVersion}.json`)
-
-      const fd = fs.openSync(tmpFile, 'w')
-      const r = spawnSync('rpk', ['connect', 'list', '--format', 'json-full'], { stdio: ['ignore', fd, 'inherit'] })
-      fs.closeSync(fd)
-
-      const rawJson = fs.readFileSync(tmpFile, 'utf8')
-      const parsed = JSON.parse(rawJson)
-      fs.writeFileSync(finalFile, JSON.stringify(parsed, null, 2))
-      fs.unlinkSync(tmpFile)
-      dataFile = finalFile
-      needsAugmentation = true
-      console.log(`Done: Fetched connector data for version ${newVersion}`)
 
       // Fetch info.csv
       try {
@@ -1458,11 +1540,8 @@ async function handleRpcnConnectorDocs (options) {
 
       // Strip CGO/cloud-only connectors and metadata from old data
       oldIndexForDiff = JSON.parse(JSON.stringify(oldIndex))
-      const connectorTypes = ['inputs', 'outputs', 'processors', 'caches', 'rate_limits',
-        'buffers', 'metrics', 'scanners', 'tracers', 'config']
-
       let totalStripped = 0
-      for (const type of connectorTypes) {
+      for (const type of COMPONENT_TYPE_KEYS_WITH_CONFIG) {
         if (Array.isArray(oldIndexForDiff[type])) {
           const originalCount = oldIndexForDiff[type].length
 
@@ -1649,8 +1728,7 @@ async function handleRpcnConnectorDocs (options) {
       const dataObj = JSON.parse(rawData)
 
       const validConnectors = []
-      const types = ['inputs', 'outputs', 'processors', 'caches', 'rate_limits', 'buffers', 'metrics', 'scanners', 'tracers']
-      types.forEach(type => {
+      COMPONENT_TYPE_KEYS.forEach(type => {
         if (Array.isArray(dataObj[type])) {
           dataObj[type].forEach(connector => {
             if (connector.name) {
@@ -1721,8 +1799,7 @@ async function handleRpcnConnectorDocs (options) {
         // Fallback when binary analysis is unavailable:
         // Check all connectors that have cloudSupported flag or assume all non-deprecated are cloud-supported
         console.log('   ℹ️  Binary analysis unavailable - checking all non-deprecated connectors for cloud-docs')
-        const types = ['inputs', 'outputs', 'processors', 'caches', 'rate_limits', 'buffers', 'metrics', 'scanners', 'tracers']
-        types.forEach(type => {
+        COMPONENT_TYPE_KEYS.forEach(type => {
           if (Array.isArray(dataObj[type])) {
             dataObj[type].forEach(connector => {
               // Include if cloudSupported is explicitly true, or if it's null/undefined and not deprecated
@@ -1738,17 +1815,21 @@ async function handleRpcnConnectorDocs (options) {
       }
 
       // Check for missing connector documentation in rp-connect-docs
+      // `type` came from a data key, and the rate-limit family's data key
+      // ('rate-limits') is not its directory name ('rate_limits'). Resolve the
+      // directory against disk per root, or every existing rate-limit page is
+      // reported missing and drafted again beside itself.
       const allMissing = validConnectors.filter(({ name, type, cloudOnly }) => {
-        const relPath = path.join(`${type}s`, `${name}.adoc`)
+        const relPathIn = (root) => path.join(resolvePageTypeDir(root, `${type}s`), `${name}.adoc`)
 
         // For cloud-only connectors, ONLY check the cloud-only directory
         if (cloudOnly) {
-          return !fs.existsSync(path.join(roots.cloudOnly, relPath))
+          return !fs.existsSync(path.join(roots.cloudOnly, relPathIn(roots.cloudOnly)))
         }
 
         // For regular connectors, check pages and partials (not cloud-only)
         const existsInAny = [roots.pages, roots.partials].some(root =>
-          fs.existsSync(path.join(root, relPath))
+          fs.existsSync(path.join(root, relPathIn(root)))
         )
         return !existsInAny
       })
@@ -2081,6 +2162,8 @@ module.exports = {
   capToTwoSentences,
   augmentConnectorData,
   buildCleanOssData,
+  loadConnectorDataForVersion,
+  snapshotReuseBlocker,
   fieldAnchor,
   buildFieldsTable,
   buildChangedDefaultsTable