@redpanda-data/docs-extensions-and-macros 5.12.2 → 5.12.4

package/extensions/generate-fields-only-pages.js

@@ -11,6 +11,7 @@ const handlebars = require('handlebars')
 
 // Import and register Redpanda Connect helpers
 const helpers = require('../tools/redpanda-connect/helpers')
+const { flattenConnectFields } = require('../tools/redpanda-connect/helpers/flattenConnectFields')
 Object.entries(helpers).forEach(([name, fn]) => {
   if (typeof fn === 'function') {
     handlebars.registerHelper(name, fn)
@@ -25,18 +26,11 @@ handlebars.registerHelper('renderConnectFieldsTable', function (children) {
 
   const rows = []
 
-  function collectFields (fieldsList, pathPrefix = '') {
-    if (!Array.isArray(fieldsList)) return
-
-    fieldsList.forEach(field => {
-      if (field.is_deprecated || !field.name) return
-
-      const isArray = field.kind === 'array'
-      const nameWithArray = (typeof field.name === 'string' && isArray && !field.name.endsWith('[]'))
-        ? `${field.name}[]`
-        : field.name
-      const currentPath = pathPrefix ? `${pathPrefix}.${nameWithArray}` : nameWithArray
-
+  // The dotted paths, the `[]` array marker and the deprecated-field skipping
+  // all come from the shared walker, so this table, the reference-page
+  // headings and the connector delta report cannot drift apart.
+  flattenConnectFields(children, { arrayMarker: true, skipDeprecated: true })
+    .forEach(({ path: currentPath, field }) => {
       // Normalize type
       let displayType
       const isArrayTitle = typeof field.name === 'string' && field.name.endsWith('[]')
@@ -88,15 +82,7 @@ handlebars.registerHelper('renderConnectFieldsTable', function (children) {
         default: defaultValue || '-',
         description: desc || '-'
       })
-
-      // Recurse for children
-      if (field.children && Array.isArray(field.children) && field.children.length > 0) {
-        collectFields(field.children, currentPath)
-      }
     })
-  }
-
-  collectFields(children, '')
 
   if (rows.length === 0) return 'No configuration fields available.\n\n'
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redpanda-data/docs-extensions-and-macros",
-  "version": "5.12.2",
+  "version": "5.12.4",
   "description": "Antora extensions and macros developed for Redpanda documentation.",
   "keywords": [
     "antora",

package/tools/redpanda-connect/connector-binary-analyzer.js

@@ -1,5 +1,5 @@
 const octokit = require('../../cli-utils/octokit-client');
-const { execSync, spawnSync } = require('child_process');
+const { execFileSync, spawnSync } = require('child_process');
 const fs = require('fs');
 const path = require('path');
 const os = require('os');
@@ -16,6 +16,31 @@ const https = require('https');
 const REPO_OWNER = 'redpanda-data';
 const REPO_NAME = 'connect';
 
+// Release asset prefixes per binary flavour. The plain OSS build is the one
+// `rpk connect install` puts on PATH, so fetching that asset directly yields
+// the same connector schema without passing a version through rpk's
+// --connect-version validation.
+const BINARY_ASSET_PREFIXES = {
+  oss: 'redpanda-connect',
+  cloud: 'redpanda-connect-cloud',
+  cgo: 'redpanda-connect-cgo'
+};
+
+/**
+ * Resolve a binary flavour to its release asset prefix.
+ * @param {string} binaryType - 'oss', 'cloud' or 'cgo'
+ * @returns {string} The asset name prefix
+ */
+function resolveAssetPrefix (binaryType) {
+  const prefix = BINARY_ASSET_PREFIXES[binaryType];
+  if (!prefix) {
+    throw new Error(
+      `Unknown binary type "${binaryType}" (expected one of ${Object.keys(BINARY_ASSET_PREFIXES).join(', ')})`
+    );
+  }
+  return prefix;
+}
+
 /**
  * Get platform-specific binary name
  * @returns {Object} Platform and arch info
@@ -46,10 +71,15 @@ function getPlatformInfo() {
 
 /**
  * Get the latest release version from GitHub
- * @param {string} binaryType - Type of binary ('cloud' or 'cgo')
+ * @param {string} binaryType - Type of binary ('oss', 'cloud' or 'cgo')
  * @returns {Promise<string>} Version string (e.g., "4.76.0")
  */
 async function getLatestVersion(binaryType = 'cloud') {
+  // Resolved outside the try so an unknown flavour surfaces its own message
+  // instead of being wrapped as a failure to reach the GitHub API. tools/ is
+  // published, so callers outside this repo see this error.
+  const prefix = resolveAssetPrefix(binaryType);
+
   try {
     const { data: releases } = await octokit.repos.listReleases({
       owner: REPO_OWNER,
@@ -57,12 +87,12 @@ async function getLatestVersion(binaryType = 'cloud') {
       per_page: 20
     });
 
-    const prefix = binaryType === 'cloud' ? 'redpanda-connect-cloud' : 'redpanda-connect-cgo';
-
-    // Find the latest release with the specified binary type
+    // Find the latest release with the specified binary type. Match the
+    // separator too: every prefix starts with 'redpanda-connect', so a bare
+    // startsWith on the OSS prefix would also match the cloud and cgo assets.
     for (const release of releases) {
       const asset = release.assets.find(a =>
-        a.name.startsWith(prefix) &&
+        a.name.startsWith(`${prefix}_`) &&
         a.name.includes('linux') &&
         a.name.includes('amd64')
       );
@@ -124,9 +154,99 @@ function downloadFile(url, destPath) {
   });
 }
 
+/**
+ * Directory that holds downloaded Connect binaries for the life of the process.
+ *
+ * Binaries are ~320 MB extracted, so they never belong in docs-data: the
+ * consuming repo commits that directory wholesale, and a run killed before its
+ * cleanup would leave a blob over GitHub's 100 MB file limit staged for the
+ * auto-docs PR. One directory for the whole run also lets downloadBinary's
+ * skip-if-present check serve repeat requests: the multi-release loop asks for
+ * every interior version twice (as the new side of one diff pair and the old
+ * side of the next), and a per-call directory made that a second download.
+ *
+ * @returns {string} Path to the process-wide binary cache directory
+ */
+let binaryCacheDir = null;
+function getBinaryCacheDir () {
+  if (binaryCacheDir && fs.existsSync(binaryCacheDir)) return binaryCacheDir;
+
+  binaryCacheDir = fs.mkdtempSync(path.join(os.tmpdir(), 'rpcn-binaries-'));
+  const created = binaryCacheDir;
+  process.on('exit', () => {
+    try {
+      fs.rmSync(created, { recursive: true, force: true });
+    } catch (err) {
+      // Nothing useful to do while exiting; the OS reclaims tmpdir anyway.
+    }
+  });
+  return binaryCacheDir;
+}
+
+// Cap the cache: a backfill can touch dozens of versions, but no more than a
+// handful of binaries are live at once (the two OSS versions of the current
+// diff pair plus the cloud and cgo binaries analysed alongside them). Without
+// a cap, sharing one directory would trade repeat downloads for unbounded disk.
+const BINARY_CACHE_LIMIT = 6;
+const binaryCacheUse = [];
+
+/**
+ * Record a binary as most recently used and evict the oldest beyond the cap.
+ * @param {string} destPath - Path to the binary just downloaded or reused
+ */
+function touchBinaryCache (destPath) {
+  const existing = binaryCacheUse.indexOf(destPath);
+  if (existing !== -1) binaryCacheUse.splice(existing, 1);
+  binaryCacheUse.push(destPath);
+
+  while (binaryCacheUse.length > BINARY_CACHE_LIMIT) {
+    const evicted = binaryCacheUse.shift();
+    try {
+      fs.rmSync(evicted, { force: true });
+    } catch (err) {
+      console.warn(`Warning: could not evict cached binary ${evicted}: ${err.message}`);
+    }
+  }
+}
+
+/**
+ * Extract the Connect binary out of a release tarball.
+ *
+ * Extraction goes into an empty directory of its own because the archive's
+ * inner file name is not the destination name, and the destination directory
+ * holds other flavours and versions whose names also contain
+ * 'redpanda-connect'. Scanning the shared directory instead would pick a
+ * neighbour in readdir order and analyse, say, the cloud binary as the cgo one.
+ *
+ * @param {string} tarballPath - Path to the downloaded .tar.gz
+ * @param {string} destPath - Path the extracted binary should end up at
+ * @returns {string} destPath
+ */
+function extractBinaryFromTarball (tarballPath, destPath) {
+  const extractDir = fs.mkdtempSync(`${destPath}.extract-`);
+  try {
+    execFileSync('tar', ['-xzf', tarballPath, '-C', extractDir], { stdio: 'ignore' });
+
+    const candidates = fs.readdirSync(extractDir)
+      .filter(name => name.includes('redpanda-connect') && !name.endsWith('.tar.gz'))
+      .filter(name => fs.statSync(path.join(extractDir, name)).isFile())
+      .sort();
+
+    if (candidates.length === 0) {
+      throw new Error('Binary not found after extraction');
+    }
+
+    fs.renameSync(path.join(extractDir, candidates[0]), destPath);
+    fs.chmodSync(destPath, 0o755);
+    return destPath;
+  } finally {
+    fs.rmSync(extractDir, { recursive: true, force: true });
+  }
+}
+
 /**
  * Download a binary from GitHub releases
- * @param {string} binaryType - 'cloud' or 'cgo'
+ * @param {string} binaryType - 'oss', 'cloud' or 'cgo'
  * @param {string} version - Version to download (e.g., "4.76.0")
  * @param {string} destDir - Destination directory
  * @returns {Promise<string>} Path to downloaded binary
@@ -136,13 +256,14 @@ async function downloadBinary(binaryType, version, destDir) {
   const platformInfo = binaryType === 'cgo'
     ? { platform: 'linux', arch: 'amd64' }
     : getPlatformInfo();
-  const prefix = binaryType === 'cloud' ? 'redpanda-connect-cloud' : 'redpanda-connect-cgo';
+  const prefix = resolveAssetPrefix(binaryType);
   const binaryName = `${prefix}-${version}-${platformInfo.platform}-${platformInfo.arch}`;
   const destPath = path.join(destDir, binaryName);
 
   // Skip download if already exists
   if (fs.existsSync(destPath)) {
     console.log(`✓ ${binaryType.toUpperCase()} binary already downloaded: ${binaryName}`);
+    touchBinaryCache(destPath);
     return destPath;
   }
 
@@ -170,35 +291,12 @@ async function downloadBinary(binaryType, version, destDir) {
 
     // Extract the binary
     console.log(`Extracting ${binaryType.toUpperCase()} binary...`);
-    execSync(`tar -xzf "${tarballPath}" -C "${destDir}"`, { stdio: 'ignore' });
-
-    // Find the extracted binary
-    const extractedFiles = fs.readdirSync(destDir);
-    let binaryPath = null;
-
-    for (const file of extractedFiles) {
-      const fullPath = path.join(destDir, file);
-      if (fs.statSync(fullPath).isFile() && file.includes('redpanda-connect') && !file.endsWith('.tar.gz')) {
-        binaryPath = fullPath;
-        break;
-      }
-    }
-
-    if (!binaryPath) {
-      throw new Error('Binary not found after extraction');
-    }
-
-    // Rename to standard name if needed
-    if (binaryPath !== destPath) {
-      fs.renameSync(binaryPath, destPath);
-    }
-
-    // Make executable
-    fs.chmodSync(destPath, 0o755);
+    extractBinaryFromTarball(tarballPath, destPath);
 
     // Clean up tarball
     fs.unlinkSync(tarballPath);
 
+    touchBinaryCache(destPath);
     console.log(`Done: Downloaded ${binaryType.toUpperCase()} binary: ${binaryName}`);
     return destPath;
   } catch (error) {
@@ -435,9 +533,10 @@ async function analyzeAllBinaries(ossVersion, cloudVersion = null, dataDir = nul
 
   const docsDataDir = dataDir || path.resolve(process.cwd(), 'docs-data');
 
-  // Use temp directory for binaries (not docs-data)
-  const os = require('os');
-  const binaryDir = fs.mkdtempSync(path.join(os.tmpdir(), 'rpcn-binaries-'));
+  // Use temp directory for binaries (not docs-data). Shared with every other
+  // download in this process, so a multi-release run fetches each cloud and cgo
+  // binary once instead of once per diff pair.
+  const binaryDir = getBinaryCacheDir();
 
   console.log(`📁 Using temp directory for binaries: ${binaryDir}`);
 
@@ -508,13 +607,8 @@ async function analyzeAllBinaries(ossVersion, cloudVersion = null, dataDir = nul
     console.log('⏩ Skipping cgo binary analysis');
   }
 
-  // Clean up temp binaries
-  try {
-    fs.rmSync(binaryDir, { recursive: true, force: true });
-    console.log(`🧹 Cleaned up temp binaries`);
-  } catch (err) {
-    console.warn(`Warning: Could not clean up temp directory: ${err.message}`);
-  }
+  // Binaries stay in the shared cache so the next diff pair can reuse them; the
+  // directory is removed when the process exits (see getBinaryCacheDir).
 
   // IMPORTANT: cgoOnly contains connectors that exist ONLY in the cgo binary
   // and NOT in the regular OSS binary (rpk connect list). These are the
@@ -566,6 +660,11 @@ module.exports = {
   analyzeAllBinaries,
   findCgoOnlyConnectors,
   getLatestVersion,
+  getPlatformInfo,
+  resolveAssetPrefix,
+  getBinaryCacheDir,
+  extractBinaryFromTarball,
+  downloadBinary,
   downloadCloudBinary,
   downloadCgoBinary,
   getConnectorList,

package/tools/redpanda-connect/generate-rpcn-connector-docs.js

@@ -5,7 +5,7 @@ const path = require('path');
 const handlebars = require('handlebars');
 const yaml = require('yaml');
 const helpers = require('./helpers');
-const { lostMetadataSections } = require('./metadata-utils.js');
+const { lostMetadataSections, resolvePageTypeDir } = require('./metadata-utils.js');
 
 // Register each helper under handlebars, verifying that it’s a function
 Object.entries(helpers).forEach(([name, fn]) => {
@@ -730,11 +730,7 @@ function backfillPageDescriptions (connectorData, { pagesRoot, dryRun = false }
 
   for (const [dataKey, items] of Object.entries(connectorData)) {
     if (!PAGE_TYPE_DIRS.has(dataKey) || !Array.isArray(items)) continue;
-    // The data uses 'rate-limits' while the pages directory is
-    // 'rate_limits'. Resolve whichever spelling exists on disk.
-    const typeDir = fs.existsSync(path.join(root, dataKey))
-      ? dataKey
-      : dataKey.replace(/-/g, '_');
+    const typeDir = resolvePageTypeDir(root, dataKey);
     for (const item of items) {
       if (!item || !item.name) continue;
       const pagePath = path.join(root, typeDir, `${item.name}.adoc`);

package/tools/redpanda-connect/helpers/flattenConnectFields.js

@@ -0,0 +1,73 @@
+'use strict';
+
+/**
+ * Shared walker for Redpanda Connect field trees.
+ *
+ * This is a plain module, not a Handlebars helper, so it is deliberately not
+ * exported from helpers/index.js. It exists because the same recursion was
+ * written four times (the reference-page renderer, the fields-only table
+ * extension and the connector delta report), and the copies disagreed about
+ * what a field path looks like. The renderer is the authority: the path this
+ * module produces must match the `=== ` heading renderConnectFields emits,
+ * because the what's-new tables link to those headings.
+ */
+
+/**
+ * Field name as it appears in the rendered docs: an array-of-object field
+ * carries an `[]` marker so the path is a valid config path.
+ * @param {object} field - Field definition
+ * @returns {string} Display name for the field
+ */
+function connectFieldName (field) {
+  const name = field && field.name;
+  if (typeof name === 'string' && field.kind === 'array' && !name.endsWith('[]')) {
+    return `${name}[]`;
+  }
+  return name;
+}
+
+/**
+ * Walk a field tree depth-first and return one entry per field, in the order
+ * the fields appear in the source data.
+ *
+ * A field nested under an existing group (for example
+ * `sqs.zero_key_warn_interval` under aws_s3's pre-existing `sqs` group) is
+ * only visible to a set-based diff if the tree is flattened first: a map of
+ * top-level names alone treats that as no change at all, because `sqs` itself
+ * is not new.
+ *
+ * Nameless nodes are skipped along with their subtree, matching
+ * renderConnectFields: a node with no name gets no heading, so nothing under
+ * it is addressable in the published docs.
+ *
+ * @param {Array} children - Field definitions (each may itself have `children`)
+ * @param {object} [opts] - { arrayMarker, skipDeprecated, prefix }
+ * @param {boolean} [opts.arrayMarker=false] - Append `[]` to array-of-object names
+ * @param {boolean} [opts.skipDeprecated=false] - Skip deprecated fields and their subtrees
+ * @param {string} [opts.prefix=''] - Dotted path of the parent, if any
+ * @returns {Array<{path: string, name: string, parentPath: (string|null), field: object}>}
+ */
+function flattenConnectFields (children, opts = {}) {
+  const { arrayMarker = false, skipDeprecated = false, prefix = '' } = opts;
+  const result = [];
+
+  const walk = (list, currentPrefix, parentPath) => {
+    if (!Array.isArray(list)) return;
+    list.forEach(field => {
+      if (!field || !field.name) return;
+      if (skipDeprecated && field.is_deprecated) return;
+
+      const name = arrayMarker ? connectFieldName(field) : field.name;
+      const path = currentPrefix ? `${currentPrefix}.${name}` : `${name}`;
+
+      result.push({ path, name, parentPath, field });
+      walk(field.children, path, path);
+    });
+  };
+
+  walk(children, prefix, prefix || null);
+
+  return result;
+}
+
+module.exports = { flattenConnectFields, connectFieldName };

package/tools/redpanda-connect/helpers/renderConnectFields.js

@@ -1,6 +1,7 @@
 const yaml = require('yaml');
 const renderYamlList = require('./renderYamlList');
 const handlebars = require('handlebars');
+const { connectFieldName } = require('./flattenConnectFields');
 
 /**
  * Renders the children of a configuration object into AsciiDoc.
@@ -56,11 +57,10 @@ module.exports = function renderConnectFields(children, prefix = '') {
     }
 
     let block = '';
-    const isArray = child.kind === 'array';
-    // Only append [] if kind is array and name does not already end with []
-    const nameWithArray = (typeof child.name === 'string' && isArray && !child.name.endsWith('[]'))
-      ? `${child.name}[]`
-      : child.name;
+    // Only append [] if kind is array and name does not already end with [].
+    // Shared with the delta report and the fields-only table so all three
+    // producers agree on what a field path looks like.
+    const nameWithArray = connectFieldName(child);
     const currentPath = prefix
       ? `${prefix}.${nameWithArray}`
       : `${nameWithArray}`;

package/tools/redpanda-connect/metadata-utils.js

@@ -1,5 +1,8 @@
 'use strict';
 
+const fs = require('fs');
+const path = require('path');
+
 /**
  * Utilities for extracting the "== Metadata" section out of a connector's
  * `description` prose so it can be emitted as a regenerated partial (like the
@@ -87,6 +90,31 @@ function typeDirFor (item) {
   return type.endsWith('s') ? type : `${type}s`;
 }
 
+/**
+ * Resolve the on-disk page/partial directory for a connector data key.
+ *
+ * The Connect binary emits the rate-limit family as `rate-limits` while the
+ * directory in rp-connect-docs is `rate_limits`, so the data key and the
+ * directory name disagree for exactly one family. Resolve against what is
+ * actually on disk rather than encoding either spelling: a caller that guesses
+ * `rate-limits` reports every existing rate-limit page as missing and then
+ * drafts a duplicate beside it, in a directory update-nav.js's regex does not
+ * match, re-added on every run.
+ *
+ * @param {string} root directory the type directories live under
+ * @param {string} dataKey the data key, for example `inputs` or `rate-limits`
+ * @returns {string} the directory name that exists, preferring the data key
+ */
+function resolvePageTypeDir (root, dataKey) {
+  if (!root || !dataKey) return dataKey || '';
+  if (fs.existsSync(path.join(root, dataKey))) return dataKey;
+  const underscored = dataKey.replace(/-/g, '_');
+  if (underscored !== dataKey && fs.existsSync(path.join(root, underscored))) return underscored;
+  const hyphenated = dataKey.replace(/_/g, '-');
+  if (hyphenated !== dataKey && fs.existsSync(path.join(root, hyphenated))) return hyphenated;
+  return dataKey;
+}
+
 /**
  * Build the Antora include directive for a connector's metadata partial.
  * @param {object} item connector data with `type`/`typeDir` and `name`
@@ -170,6 +198,7 @@ function lostMetadataSections (oldContent, newContent) {
 
 module.exports = {
   locateMetadata,
+  resolvePageTypeDir,
   extractMetadata,
   typeDirFor,
   metadataIncludeLine,

package/tools/redpanda-connect/report-delta.js

@@ -1,4 +1,5 @@
 const { execSync } = require('child_process');
+const { flattenConnectFields } = require('./helpers/flattenConnectFields');
 
 /**
  * Generate a JSON diff report between two connector index objects.
@@ -54,22 +55,29 @@ function generateConnectorDiffJson(oldIndex, newIndex, opts = {}) {
     if (!(cKey in oldMap)) return;
     const oldFields = new Set(oldMap[cKey].fields || []);
     const newFieldsArr = newMap[cKey].fields || [];
-    newFieldsArr.forEach(fName => {
-      if (!oldFields.has(fName)) {
-        const [type, compName] = cKey.split(':');
-        let rawFieldObj = null;
-        if (type === 'config') {
-          rawFieldObj = (newMap[cKey].raw.children || []).find(f => f.name === fName);
-        } else {
-          rawFieldObj = (newMap[cKey].raw.config?.children || []).find(f => f.name === fName);
-        }
-        newFields.push({
-          component: cKey,
-          field: fName,
-          introducedIn: rawFieldObj && (rawFieldObj.introducedInVersion || rawFieldObj.version),
-          description: rawFieldObj && rawFieldObj.description
-        });
-      }
+    const added = collapseToTopmost(
+      newFieldsArr.filter(fName => !oldFields.has(fName)),
+      newMap[cKey].parents
+    );
+    added.forEach(fName => {
+      const rawFieldObj = newMap[cKey].fieldMap.get(fName) || null;
+      // A field that arrives already deprecated -- or sits under a
+      // deprecated ancestor group -- gets no heading on the reference page,
+      // so advertising it under "New field support" would link to a fragment
+      // that never exists.
+      if (isDeprecatedOrDescendsFromDeprecated(fName, newMap[cKey].fieldMap, newMap[cKey].parents)) return;
+      newFields.push({
+        component: cKey,
+        field: fName,
+        // Most fields carry no version of their own, so fall back to the
+        // version this diff is comparing against: the field is new as of
+        // exactly that release. Some fields do carry `version` (268 entries
+        // in connect-4.105.0), and that wins, because it can predate the
+        // span being compared: inputs:oracledb_cdc/snapshot_mode reports
+        // 4.99.0 in the 4.98.0 to 4.103.1 diff.
+        introducedIn: (rawFieldObj && (rawFieldObj.introducedInVersion || rawFieldObj.version)) || opts.newVersion || null,
+        description: rawFieldObj && rawFieldObj.description
+      });
     });
   });
 
@@ -79,13 +87,15 @@ function generateConnectorDiffJson(oldIndex, newIndex, opts = {}) {
     if (!(cKey in newMap)) return;
     const newFieldsSet = new Set(newMap[cKey].fields || []);
     const oldFieldsArr = oldMap[cKey].fields || [];
-    oldFieldsArr.forEach(fName => {
-      if (!newFieldsSet.has(fName)) {
-        removedFields.push({
-          component: cKey,
-          field: fName
-        });
-      }
+    const removed = collapseToTopmost(
+      oldFieldsArr.filter(fName => !newFieldsSet.has(fName)),
+      oldMap[cKey].parents
+    );
+    removed.forEach(fName => {
+      removedFields.push({
+        component: cKey,
+        field: fName
+      });
     });
   });
 
@@ -172,26 +182,11 @@ function generateConnectorDiffJson(oldIndex, newIndex, opts = {}) {
     // Check fields that exist in both versions
     const commonFields = oldFieldsArr.filter(f => newFieldsArr.includes(f));
     commonFields.forEach(fName => {
-      const [type, compName] = cKey.split(':');
+      const oldFieldObj = oldMap[cKey].fieldMap.get(fName) || null;
+      const newFieldObj = newMap[cKey].fieldMap.get(fName) || null;
 
-      // Get old field object
-      let oldFieldObj = null;
-      if (type === 'config') {
-        oldFieldObj = (oldMap[cKey].raw.children || []).find(f => f.name === fName);
-      } else {
-        oldFieldObj = (oldMap[cKey].raw.config?.children || []).find(f => f.name === fName);
-      }
-
-      // Get new field object
-      let newFieldObj = null;
-      if (type === 'config') {
-        newFieldObj = (newMap[cKey].raw.children || []).find(f => f.name === fName);
-      } else {
-        newFieldObj = (newMap[cKey].raw.config?.children || []).find(f => f.name === fName);
-      }
-
-      const oldDeprecated = oldFieldObj && (oldFieldObj.is_deprecated === true || oldFieldObj.deprecated === true || (oldFieldObj.status || '').toLowerCase() === 'deprecated');
-      const newDeprecated = newFieldObj && (newFieldObj.is_deprecated === true || newFieldObj.deprecated === true || (newFieldObj.status || '').toLowerCase() === 'deprecated');
+      const oldDeprecated = isDeprecatedField(oldFieldObj);
+      const newDeprecated = isDeprecatedField(newFieldObj);
 
       if (!oldDeprecated && newDeprecated) {
         deprecatedFields.push({
@@ -380,6 +375,67 @@ function discoverComponentKeys(obj) {
   return Object.keys(obj).filter(key => Array.isArray(obj[key]));
 }
 
+/**
+ * Whether a field is marked deprecated.
+ *
+ * Three markers have been used by the source data over time. Only
+ * `is_deprecated` appears in current releases (verified across connect-4.63.0,
+ * 4.104.0 and 4.105.0: 85, 75 and 75 fields, zero uses of the other two), and
+ * it is the one renderConnectFields keys on when it decides not to render a
+ * heading for a field.
+ *
+ * @param {object} field - Field definition, or null
+ * @returns {boolean} True when the field is deprecated
+ */
+function isDeprecatedField(field) {
+  if (!field) return false;
+  return field.is_deprecated === true ||
+    field.deprecated === true ||
+    (field.status || '').toLowerCase() === 'deprecated';
+}
+
+/**
+ * True if `fName` or any of its ancestor fields is deprecated.
+ *
+ * renderConnectFields never renders a field under a deprecated group (the
+ * whole group is hidden), so a leaf-only check reports a field added under an
+ * existing-but-deprecated group in "New field support" even though it never
+ * appears on the page. Walking the parent chain via `parents` is what
+ * catches that.
+ */
+function isDeprecatedOrDescendsFromDeprecated(fName, fieldMap, parents) {
+  let current = fName;
+  while (current) {
+    if (isDeprecatedField(fieldMap.get(current))) return true;
+    current = parents && parents.get(current);
+  }
+  return false;
+}
+
+/**
+ * Drop paths whose parent changed in the same way, keeping only the topmost
+ * field of each changed subtree.
+ *
+ * Adding or removing one group adds or removes every field under it, and the
+ * flattened field list contains all of them. Reporting each descendant turns
+ * a single change into one row per node: removing `batching` from
+ * outputs:redpanda_migrator would list `batching` plus batching.count,
+ * .byte_size, .period, .check and .processors, and the group row already says
+ * all of that. Only the fields added to or removed from a group that still
+ * exists are news.
+ *
+ * @param {Array<string>} changedPaths - Paths that are new (or removed)
+ * @param {Map<string, (string|null)>} parents - Path to parent path
+ * @returns {Array<string>} The topmost changed path of each subtree, in order
+ */
+function collapseToTopmost(changedPaths, parents) {
+  const changed = new Set(changedPaths);
+  return changedPaths.filter(path => {
+    const parent = parents && parents.get(path);
+    return !(parent && changed.has(parent));
+  });
+}
+
 function buildComponentMap(indexObj) {
   const map = {};
   const types = discoverComponentKeys(indexObj);
@@ -402,7 +458,14 @@ function buildComponentMap(indexObj) {
         }
       }
 
-      const fieldNames = childArray.map(f => f.name);
+      // arrayMarker keeps the paths identical to the headings
+      // renderConnectFields emits, so a field name in the what's-new table is
+      // a config path a reader can paste into YAML (`sasl[].aws.tcp`, not the
+      // invalid `sasl.aws.tcp`).
+      const flattened = flattenConnectFields(childArray, { arrayMarker: true });
+      const fieldNames = flattened.map(f => f.path);
+      const fieldMap = new Map(flattened.map(f => [f.path, f.field]));
+      const parents = new Map(flattened.map(f => [f.path, f.parentPath]));
 
       // Preserve platform metadata for accurate diff comparison
       const metadata = {
@@ -414,6 +477,8 @@ function buildComponentMap(indexObj) {
       map[lookupKey] = {
         raw: component,
         fields: fieldNames,
+        fieldMap: fieldMap,
+        parents: parents,
         metadata: metadata
       };
     });
@@ -499,8 +564,11 @@ function getRpkConnectVersion() {
  *  • which connectors/components are brand-new
  *  • which new fields appeared under existing connectors (including “config” entries)
  *  • for each new component/field, if the raw object contains “version” or “introducedInVersion” or “requiresVersion” metadata, print it
+ * @param {object} oldIndex - Previous version connector index
+ * @param {object} newIndex - Current version connector index
+ * @param {object} [opts] - { newVersion } used as the introducedIn fallback
  */
-function printDeltaReport(oldIndex, newIndex) {
+function printDeltaReport(oldIndex, newIndex, opts = {}) {
   const oldMap = buildComponentMap(oldIndex);
   const newMap = buildComponentMap(newIndex);
 
@@ -513,27 +581,24 @@ function printDeltaReport(oldIndex, newIndex) {
     if (!(cKey in oldMap)) return; // skip brand-new components here
     const oldFields = new Set(oldMap[cKey].fields || []);
     const newFieldsArr = newMap[cKey].fields || [];
-    newFieldsArr.forEach(fName => {
-      if (!oldFields.has(fName)) {
-        // fetch raw field metadata if available
-        const [type, compName] = cKey.split(':');
-        let rawFieldObj = null;
-        if (type === 'config') {
-          rawFieldObj = (newMap[cKey].raw.children || []).find(f => f.name === fName);
-        } else {
-          rawFieldObj = (newMap[cKey].raw.config?.children || []).find(f => f.name === fName);
-        }
-
-        let introducedIn = rawFieldObj && (rawFieldObj.introducedInVersion || rawFieldObj.version);
-        let requiresVer = rawFieldObj && rawFieldObj.requiresVersion;
-
-        newFields.push({
-          component: cKey,
-          field: fName,
-          introducedIn,
-          requiresVersion: requiresVer,
-        });
-      }
+    const added = collapseToTopmost(
+      newFieldsArr.filter(fName => !oldFields.has(fName)),
+      newMap[cKey].parents
+    );
+    added.forEach(fName => {
+      const rawFieldObj = newMap[cKey].fieldMap.get(fName) || null;
+      if (isDeprecatedField(rawFieldObj)) return;
+      // Same fallback as generateConnectorDiffJson, so the console report and
+      // the published table cannot disagree about the same input.
+      const introducedIn = (rawFieldObj && (rawFieldObj.introducedInVersion || rawFieldObj.version)) || opts.newVersion || null;
+      const requiresVer = rawFieldObj && rawFieldObj.requiresVersion;
+
+      newFields.push({
+        component: cKey,
+        field: fName,
+        introducedIn,
+        requiresVersion: requiresVer,
+      });
     });
   });
 
@@ -560,21 +625,10 @@ function printDeltaReport(oldIndex, newIndex) {
     const commonFields = oldFieldsArr.filter(f => newFieldsArr.includes(f));
 
     commonFields.forEach(fName => {
-      const [type, compName] = cKey.split(':');
-      let oldFieldObj = null;
-      if (type === 'config') {
-        oldFieldObj = (oldMap[cKey].raw.children || []).find(f => f.name === fName);
-      } else {
-        oldFieldObj = (oldMap[cKey].raw.config?.children || []).find(f => f.name === fName);
-      }
-      let newFieldObj = null;
-      if (type === 'config') {
-        newFieldObj = (newMap[cKey].raw.children || []).find(f => f.name === fName);
-      } else {
-        newFieldObj = (newMap[cKey].raw.config?.children || []).find(f => f.name === fName);
-      }
-      const oldDeprecated = oldFieldObj && (oldFieldObj.is_deprecated === true || oldFieldObj.deprecated === true || (oldFieldObj.status || '').toLowerCase() === 'deprecated');
-      const newDeprecated = newFieldObj && (newFieldObj.is_deprecated === true || newFieldObj.deprecated === true || (newFieldObj.status || '').toLowerCase() === 'deprecated');
+      const oldFieldObj = oldMap[cKey].fieldMap.get(fName) || null;
+      const newFieldObj = newMap[cKey].fieldMap.get(fName) || null;
+      const oldDeprecated = isDeprecatedField(oldFieldObj);
+      const newDeprecated = isDeprecatedField(newFieldObj);
       if (!oldDeprecated && newDeprecated) {
         deprecatedFieldsList.push({ component: cKey, field: fName });
       }

package/tools/redpanda-connect/rpcn-connector-docs-handler.js

@@ -11,6 +11,7 @@ const { getRpkConnectVersion, printDeltaReport } = require('./report-delta')
 const { discoverIntermediateReleases, findCloudVersionForDate } = require('./github-release-utils')
 const { analyzeAllBinaries } = require('./connector-binary-analyzer.js')
 const parseCSVConnectors = require('./parse-csv-connectors.js')
+const { resolvePageTypeDir } = require('./metadata-utils.js')
 const semver = require('semver')
 
 /**
@@ -106,6 +107,23 @@ function capToTwoSentences (description) {
   return result.trim()
 }
 
+// Component families in a connector index, in the spelling the Connect binary
+// emits. Rate limits are 'rate-limits' there; 'rate_limits' is the docs page
+// directory spelling, which older snapshots also carry as an (always empty)
+// data key, so it is walked when present but never created. Every copy of this
+// list used to be the underscore form only, which meant rate-limit components
+// were never given cloudSupported or requiresCgo flags and any cgo-only or
+// cloud-only rate limit was dropped from the published snapshot.
+const COMPONENT_TYPE_KEYS = ['inputs', 'outputs', 'processors', 'caches', 'rate-limits',
+  'rate_limits', 'buffers', 'metrics', 'scanners', 'tracers']
+
+// 'config' components live in the same index but have no config/fields wrapper.
+const COMPONENT_TYPE_KEYS_WITH_CONFIG = [...COMPONENT_TYPE_KEYS, 'config']
+
+// Keys kept only for snapshots written before the spelling was fixed. Walked
+// if a snapshot has them, never added to one.
+const LEGACY_COMPONENT_TYPE_KEYS = new Set(['rate_limits'])
+
 /**
  * Remove platform metadata fields from connectors
  * @param {Array} connectors - Array of connector objects
@@ -146,11 +164,9 @@ function augmentConnectorData (connectorData, binaryAnalysis) {
   let addedCgoCount = 0
   let addedCloudOnlyCount = 0
 
-  const connectorTypes = ['inputs', 'outputs', 'processors', 'caches', 'rate_limits',
-    'buffers', 'metrics', 'scanners', 'tracers', 'config']
-
-  for (const type of connectorTypes) {
+  for (const type of COMPONENT_TYPE_KEYS_WITH_CONFIG) {
     if (!Array.isArray(augmentedData[type])) {
+      if (LEGACY_COMPONENT_TYPE_KEYS.has(type)) continue
       augmentedData[type] = []
     }
 
@@ -219,10 +235,7 @@ function augmentConnectorData (connectorData, binaryAnalysis) {
  */
 function buildCleanOssData (connectorIndex) {
   const cleanData = JSON.parse(JSON.stringify(connectorIndex))
-  const connectorTypes = ['inputs', 'outputs', 'processors', 'caches', 'rate_limits',
-    'buffers', 'metrics', 'scanners', 'tracers']
-
-  for (const type of connectorTypes) {
+  for (const type of COMPONENT_TYPE_KEYS) {
     if (Array.isArray(cleanData[type])) {
       // Keep only connectors from OSS rpk (have config/fields)
       // Remove augmentation-only connectors (added by previous binary analysis)
@@ -413,7 +426,7 @@ function updateWhatsNew ({ dataDir, oldVersion, newVersion, binaryAnalysis }) {
       if (regularFields.length > 0) {
         section += '\n=== New field support\n\n'
         section += 'This release adds support for the following new fields:\n\n'
-        section += buildFieldsTable(regularFields, capToTwoSentences)
+        section += buildFieldsTable(regularFields, capToTwoSentences, { showIntroducedIn: true })
       }
     }
 
@@ -553,24 +566,35 @@ function fieldAnchor (fieldName) {
  * @param {Function} capFn - Caption function
  * @returns {string} AsciiDoc table
  */
-function buildFieldsTable (fields, capFn) {
+function buildFieldsTable (fields, capFn, opts = {}) {
+  const { showIntroducedIn = false } = opts
   const byField = {}
   for (const field of fields) {
     const [type, compName] = field.component.split(':')
-    if (!byField[field.field]) {
-      byField[field.field] = {
+    // Two components can gain the same field path in different releases
+    // (`urls` arrived in 3.58.0 for one and 4.23.0 for another). Merging those
+    // into a single row would publish the first version for both, so when the
+    // version is on show it is part of the row's identity.
+    const key = showIntroducedIn
+      ? `${field.field}\u0000${field.introducedIn || ''}`
+      : field.field
+    if (!byField[key]) {
+      byField[key] = {
+        field: field.field,
         description: field.description,
+        introducedIn: field.introducedIn,
         components: []
       }
     }
-    byField[field.field].components.push({ type, name: compName })
+    byField[key].components.push({ type, name: compName })
   }
 
-  let section = '[cols="1m,3,2a"]\n'
+  let section = showIntroducedIn ? '[cols="1m,3,2a,1m"]\n' : '[cols="1m,3,2a"]\n'
   section += '|===\n'
-  section += '|Field |Description |Affected components\n\n'
+  section += showIntroducedIn ? '|Field |Description |Affected components |Introduced in\n\n' : '|Field |Description |Affected components\n\n'
 
-  for (const [fieldName, info] of Object.entries(byField)) {
+  for (const info of Object.values(byField)) {
+    const fieldName = info.field
     const byType = {}
     for (const comp of info.components) {
       if (!byType[comp.type]) byType[comp.type] = []
@@ -595,7 +619,9 @@ function buildFieldsTable (fields, capFn) {
 
     section += `|${fieldName}\n`
     section += `|${desc}\n`
-    section += `|${componentList}\n\n`
+    section += `|${componentList}\n`
+    if (showIntroducedIn) section += `|${info.introducedIn || '-'}\n`
+    section += '\n'
   }
 
   section += '|===\n\n'
@@ -690,11 +716,52 @@ function logCollapsed (label, filesArray, maxToShow = 10) {
 }
 
 
+/**
+ * Why a stored connect-<version>.json cannot stand in for a fresh fetch.
+ *
+ * The handler writes augmentConnectorData() output back over the same file, so
+ * a stored snapshot is not necessarily raw binary output. Reusing an augmented
+ * one would re-augment already-augmented data: the cloud-only and cgo-only
+ * entries injected by the previous run are not in the new analysis, so they
+ * survive with their platform markers reset and a cgo-only connector ends up
+ * published as an ordinary OSS one. The version field is the only provenance
+ * the downstream templates get, so a file that disagrees with its own name
+ * cannot be trusted either.
+ *
+ * @param {Object} data - Parsed contents of connect-<version>.json
+ * @param {string} version - Version the file is supposed to describe
+ * @returns {string|null} Reason it cannot be reused, or null if it can
+ */
+function snapshotReuseBlocker (data, version) {
+  if (!data || typeof data !== 'object' || Array.isArray(data)) {
+    return 'file is not a connector index'
+  }
+
+  if (typeof data.version === 'string' && data.version !== version) {
+    return `file reports version ${data.version}`
+  }
+
+  const augmented = Object.values(data)
+    .filter(Array.isArray)
+    .some(components => components.some(c => c && typeof c === 'object' && (
+      c.cloudSupported !== undefined ||
+      c.requiresCgo !== undefined ||
+      c.cloudOnly !== undefined
+    )))
+
+  if (augmented) {
+    return 'file already carries cloud/cgo augmentation from an earlier run'
+  }
+
+  return null
+}
+
 /**
  * Load or fetch connector data for a specific version
  * @param {string} version - Version to load (e.g., "4.50.0")
  * @param {string} dataDir - Directory where JSON files are stored
  * @param {Object} options - Options for fetching if needed
+ * @param {boolean} [options.forceFresh] - Refetch even when a usable file exists
  * @returns {Promise<Object>} Parsed connector data
  */
 async function loadConnectorDataForVersion(version, dataDir, options = {}) {
@@ -702,51 +769,75 @@ async function loadConnectorDataForVersion(version, dataDir, options = {}) {
 
   // If forceFresh is set, always fetch even if file exists
   // This ensures we have accurate connector lists for diff comparison
+  let reason = options.forceFresh ? 'force refresh requested' : 'file not found';
+
   if (!options.forceFresh && fs.existsSync(dataFile)) {
-    console.log(`✓ Using existing data file: connect-${version}.json`);
-    const data = JSON.parse(fs.readFileSync(dataFile, 'utf8'));
-    return data;
+    let stored = null;
+    let blocker = null;
+    try {
+      stored = JSON.parse(fs.readFileSync(dataFile, 'utf8'));
+      blocker = snapshotReuseBlocker(stored, version);
+    } catch (err) {
+      blocker = `file could not be parsed (${err.message})`;
+    }
+
+    if (!blocker) {
+      console.log(`✓ Using existing data file: connect-${version}.json`);
+      return stored;
+    }
+
+    console.warn(`↻ Ignoring existing connect-${version}.json: ${blocker}`);
+    reason = blocker;
   }
 
   // Fetch fresh data
-  const reason = options.forceFresh ? 'force refresh requested' : 'file not found';
   console.log(`📥 Fetching data for ${version} (${reason})...`);
 
+  // Fetch the OSS release asset for this exact version instead of asking rpk to
+  // install it. rpk's validateVersion (rpk/pkg/cli/connect/install.go) matches
+  // --connect-version against a regex that caps each segment at two digits, so
+  // it rejects every version >= 4.100.0, which is every version worth asking
+  // for. The cap is lifted in rpk v26.1.15 and on dev, but v26.1.14 and the
+  // whole v26.2.x line still carry it, so which versions can be installed
+  // depends on which rpk happens to be on PATH. Fetching the asset removes that
+  // dependency instead of working around one rpk release, so this is the
+  // permanent mechanism rather than something to revert once rpk ships a fix:
+  // the plain OSS asset is the same build rpk would have installed, so the
+  // schema is identical, and this leaves the caller's own Connect installation
+  // untouched instead of forcing a managed plugin over the top of it.
   try {
-    // Try installing that specific version and fetching data
-    console.log(`   Installing Redpanda Connect version ${version}...`);
-    const installResult = spawnSync('rpk', ['connect', 'install', '--connect-version', version, '--force'], {
-      stdio: 'pipe'
-    });
-
-    if (installResult.status !== 0) {
-      throw new Error(`Failed to install Connect version ${version}`);
-    }
+    const { downloadBinary, getConnectorList, getBinaryCacheDir } = require('./connector-binary-analyzer.js');
 
-    // Fetch connector list
-    const tmpFile = path.join(dataDir, `connect-${version}.tmp.json`);
-    const fd = fs.openSync(tmpFile, 'w');
-    const listResult = spawnSync('rpk', ['connect', 'list', '--format', 'json-full'], {
-      stdio: ['ignore', fd, 'pipe']
-    });
-    fs.closeSync(fd);
-
-    if (listResult.status !== 0) {
-      throw new Error(`Failed to fetch connector list for version ${version}`);
+    // Binaries go to the process-wide temp cache, never to dataDir: the
+    // consuming repo commits dataDir, and the extracted binary is ~320 MB.
+    const binaryPath = await downloadBinary('oss', version, getBinaryCacheDir());
+    const parsed = getConnectorList(binaryPath);
+
+    if (!parsed || typeof parsed !== 'object') {
+      throw new Error('connector list was empty or not an object');
     }
 
-    // Parse and validate
-    const rawJson = fs.readFileSync(tmpFile, 'utf8');
-    const parsed = JSON.parse(rawJson);
+    // The asset name embeds the version and the release was fetched by tag, so
+    // a binary that reports a different version means the wrong asset was
+    // downloaded. Fail rather than publish a snapshot under the wrong name.
+    if (typeof parsed.version === 'string' && parsed.version !== version) {
+      throw new Error(`asset reports version ${parsed.version}, not ${version}`);
+    }
 
-    // Move to final location
+    // Write via a temp file so an interrupted run cannot leave a half-written
+    // snapshot that the next run would happily reuse.
+    const tmpFile = path.join(dataDir, `connect-${version}.tmp.json`);
+    fs.writeFileSync(tmpFile, JSON.stringify(parsed, null, 2));
     fs.renameSync(tmpFile, dataFile);
 
     console.log(`✓ Successfully fetched data for version ${version}`);
     return parsed;
   } catch (error) {
     console.error(`❌ Failed to fetch data for version ${version}: ${error.message}`);
-    throw new Error(`Cannot process version ${version} - data unavailable`);
+    // Carry the reason into the thrown error: this message is what reaches the
+    // failure summary and the PR body, where a bare "data unavailable" sends
+    // the reader back to the raw workflow log.
+    throw new Error(`Cannot process version ${version} - data unavailable (${error.message})`);
   }
 }
 
@@ -776,34 +867,38 @@ async function handleRpcnConnectorDocs (options) {
           console.error('Expected format: X.Y.Z (e.g., 4.50.0)')
           process.exit(1)
         }
-        console.log(`Installing Redpanda Connect version ${options.connectVersion}...`)
-        const installResult = spawnSync('rpk', ['connect', 'install', '--connect-version', options.connectVersion, '--force'], {
-          stdio: 'inherit'
-        })
-        if (installResult.status !== 0) {
-          throw new Error(`Failed to install Connect version ${options.connectVersion}`)
-        }
-        console.log(`Done: Installed Redpanda Connect version ${options.connectVersion}`)
         newVersion = options.connectVersion
+        // An explicit version comes from the release asset rather than from
+        // rpk, which rejects anything >= 4.100.0 on the rpk releases this
+        // pipeline runs. See loadConnectorDataForVersion. Reuses an existing
+        // snapshot when it is raw, unaugmented output for this same version,
+        // and leaves the caller's own Connect install alone either way.
+        console.log(`Fetching connector data for Connect ${newVersion}...`)
+        await loadConnectorDataForVersion(newVersion, dataDir)
+        dataFile = path.join(dataDir, `connect-${newVersion}.json`)
+        needsAugmentation = true
+        console.log(`Done: Fetched connector data for version ${newVersion}`)
       } else {
+        // No version asked for, so read whatever rpk has: this is install's
+        // unvalidated "latest" path, which the version regex never sees.
         newVersion = getRpkConnectVersion()
+        console.log(`Fetching connector data from Connect ${newVersion}...`)
+
+        const tmpFile = path.join(dataDir, `connect-${newVersion}.tmp.json`)
+        const finalFile = path.join(dataDir, `connect-${newVersion}.json`)
+
+        const fd = fs.openSync(tmpFile, 'w')
+        spawnSync('rpk', ['connect', 'list', '--format', 'json-full'], { stdio: ['ignore', fd, 'inherit'] })
+        fs.closeSync(fd)
+
+        const rawJson = fs.readFileSync(tmpFile, 'utf8')
+        const parsed = JSON.parse(rawJson)
+        fs.writeFileSync(finalFile, JSON.stringify(parsed, null, 2))
+        fs.unlinkSync(tmpFile)
+        dataFile = finalFile
+        needsAugmentation = true
+        console.log(`Done: Fetched connector data for version ${newVersion}`)
       }
-      console.log(`Fetching connector data from Connect ${newVersion}...`)
-
-      const tmpFile = path.join(dataDir, `connect-${newVersion}.tmp.json`)
-      const finalFile = path.join(dataDir, `connect-${newVersion}.json`)
-
-      const fd = fs.openSync(tmpFile, 'w')
-      const r = spawnSync('rpk', ['connect', 'list', '--format', 'json-full'], { stdio: ['ignore', fd, 'inherit'] })
-      fs.closeSync(fd)
-
-      const rawJson = fs.readFileSync(tmpFile, 'utf8')
-      const parsed = JSON.parse(rawJson)
-      fs.writeFileSync(finalFile, JSON.stringify(parsed, null, 2))
-      fs.unlinkSync(tmpFile)
-      dataFile = finalFile
-      needsAugmentation = true
-      console.log(`Done: Fetched connector data for version ${newVersion}`)
 
       // Fetch info.csv
       try {
@@ -1445,11 +1540,8 @@ async function handleRpcnConnectorDocs (options) {
 
       // Strip CGO/cloud-only connectors and metadata from old data
       oldIndexForDiff = JSON.parse(JSON.stringify(oldIndex))
-      const connectorTypes = ['inputs', 'outputs', 'processors', 'caches', 'rate_limits',
-        'buffers', 'metrics', 'scanners', 'tracers', 'config']
-
       let totalStripped = 0
-      for (const type of connectorTypes) {
+      for (const type of COMPONENT_TYPE_KEYS_WITH_CONFIG) {
         if (Array.isArray(oldIndexForDiff[type])) {
           const originalCount = oldIndexForDiff[type].length
 
@@ -1475,7 +1567,7 @@ async function handleRpcnConnectorDocs (options) {
       }
     }
 
-    printDeltaReport(oldIndexForDiff, newIndex)
+    printDeltaReport(oldIndexForDiff, newIndex, { newVersion })
 
     const { generateConnectorDiffJson } = require('./report-delta.js')
     diffJson = generateConnectorDiffJson(
@@ -1636,8 +1728,7 @@ async function handleRpcnConnectorDocs (options) {
       const dataObj = JSON.parse(rawData)
 
       const validConnectors = []
-      const types = ['inputs', 'outputs', 'processors', 'caches', 'rate_limits', 'buffers', 'metrics', 'scanners', 'tracers']
-      types.forEach(type => {
+      COMPONENT_TYPE_KEYS.forEach(type => {
         if (Array.isArray(dataObj[type])) {
           dataObj[type].forEach(connector => {
             if (connector.name) {
@@ -1708,8 +1799,7 @@ async function handleRpcnConnectorDocs (options) {
         // Fallback when binary analysis is unavailable:
         // Check all connectors that have cloudSupported flag or assume all non-deprecated are cloud-supported
         console.log('   ℹ️  Binary analysis unavailable - checking all non-deprecated connectors for cloud-docs')
-        const types = ['inputs', 'outputs', 'processors', 'caches', 'rate_limits', 'buffers', 'metrics', 'scanners', 'tracers']
-        types.forEach(type => {
+        COMPONENT_TYPE_KEYS.forEach(type => {
           if (Array.isArray(dataObj[type])) {
             dataObj[type].forEach(connector => {
               // Include if cloudSupported is explicitly true, or if it's null/undefined and not deprecated
@@ -1725,17 +1815,21 @@ async function handleRpcnConnectorDocs (options) {
       }
 
       // Check for missing connector documentation in rp-connect-docs
+      // `type` came from a data key, and the rate-limit family's data key
+      // ('rate-limits') is not its directory name ('rate_limits'). Resolve the
+      // directory against disk per root, or every existing rate-limit page is
+      // reported missing and drafted again beside itself.
       const allMissing = validConnectors.filter(({ name, type, cloudOnly }) => {
-        const relPath = path.join(`${type}s`, `${name}.adoc`)
+        const relPathIn = (root) => path.join(resolvePageTypeDir(root, `${type}s`), `${name}.adoc`)
 
         // For cloud-only connectors, ONLY check the cloud-only directory
         if (cloudOnly) {
-          return !fs.existsSync(path.join(roots.cloudOnly, relPath))
+          return !fs.existsSync(path.join(roots.cloudOnly, relPathIn(roots.cloudOnly)))
         }
 
         // For regular connectors, check pages and partials (not cloud-only)
         const existsInAny = [roots.pages, roots.partials].some(root =>
-          fs.existsSync(path.join(root, relPath))
+          fs.existsSync(path.join(root, relPathIn(root)))
         )
         return !existsInAny
       })
@@ -2068,6 +2162,8 @@ module.exports = {
   capToTwoSentences,
   augmentConnectorData,
   buildCleanOssData,
+  loadConnectorDataForVersion,
+  snapshotReuseBlocker,
   fieldAnchor,
   buildFieldsTable,
   buildChangedDefaultsTable