@redpanda-data/docs-extensions-and-macros 4.15.8 → 4.15.9

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redpanda-data/docs-extensions-and-macros",
-  "version": "4.15.8",
+  "version": "4.15.9",
   "description": "Antora extensions and macros developed for Redpanda documentation.",
   "keywords": [
     "antora",

package/tools/property-extractor/property_extractor.py

@@ -991,6 +991,14 @@ def _apply_override_to_existing_property(property_dict, override, overrides_file
     if "category" in override:
         property_dict["category"] = override["category"]
 
+    # Apply accepted_values override - replaces enum field to filter displayed values
+    # Use case: Exclude internal-only enum values (e.g., greedy mode for leader_balancer_mode)
+    if "accepted_values" in override:
+        if isinstance(override["accepted_values"], list):
+            property_dict["enum"] = override["accepted_values"]
+        else:
+            logger.warning(f"accepted_values for property must be an array")
+
 
 def _create_property_from_override(prop_name, override, overrides_file_path):
     """Create a new property from override specification."""
@@ -1025,9 +1033,10 @@ def _create_property_from_override(prop_name, override, overrides_file_path):
     
     # Add any other custom fields from override
     for key, value in override.items():
-        if key not in ["description", "type", "default", "config_scope", "version", 
-                       "example", "example_file", "example_yaml", "related_topics", 
-                       "is_deprecated", "visibility"]:
+        if key not in ["description", "type", "default", "config_scope", "version",
+                       "example", "example_file", "example_yaml", "related_topics",
+                       "is_deprecated", "visibility", "exclude_from_docs", "category",
+                       "accepted_values", "_comment"]:
             new_property[key] = value
 
     # Add exclude_from_docs if specified
@@ -1037,7 +1046,14 @@ def _create_property_from_override(prop_name, override, overrides_file_path):
     # Add category if specified
     if "category" in override:
         new_property["category"] = override["category"]
-    
+
+    # Add accepted_values as enum field if specified
+    if "accepted_values" in override:
+        if isinstance(override["accepted_values"], list):
+            new_property["enum"] = override["accepted_values"]
+        else:
+            logger.warning(f"accepted_values for property '{prop_name}' must be an array")
+
     return new_property
 
 

package/tools/redpanda-connect/connector-binary-analyzer.js

@@ -247,6 +247,7 @@ function getConnectorList(binaryPath) {
 
     let result;
     if (needsDocker) {
+      console.log(`   Using Docker to run Linux binary on ${os.platform()}`);
       // Use Docker to run Linux binaries on macOS/Windows
       const binaryDir = path.dirname(binaryPath);
       const binaryFile = path.basename(binaryPath);
@@ -268,6 +269,7 @@ function getConnectorList(binaryPath) {
         maxBuffer: 10 * 1024 * 1024 // 10MB buffer
       });
     } else {
+      console.log(`   Running natively on ${os.platform()}`);
       // Run natively
       result = spawnSync(binaryPath, ['list', '--format', 'json-full'], {
         stdio: ['ignore', 'pipe', 'ignore'],

package/tools/redpanda-connect/report-delta.js

@@ -2,6 +2,7 @@ const { execSync } = require('child_process');
 
 /**
  * Generate a JSON diff report between two connector index objects.
+ * Includes platform metadata (CGO, cloud-only) to detect transitions.
  * @param {object} oldIndex - Previous version connector index
  * @param {object} newIndex - Current version connector index
  * @param {object} opts - { oldVersion, newVersion, timestamp, binaryAnalysis, oldBinaryAnalysis }
@@ -11,31 +12,39 @@ function generateConnectorDiffJson(oldIndex, newIndex, opts = {}) {
   const oldMap = buildComponentMap(oldIndex);
   const newMap = buildComponentMap(newIndex);
 
-  // New components
+  // New components (include platform metadata)
   const newComponentKeys = Object.keys(newMap).filter(k => !(k in oldMap));
   const newComponents = newComponentKeys.map(key => {
     const [type, name] = key.split(':');
     const raw = newMap[key].raw;
+    const metadata = newMap[key].metadata || {};
     return {
       name,
       type,
       status: raw.status || raw.type || '',
       version: raw.version || raw.introducedInVersion || '',
-      description: raw.description || ''
+      description: raw.description || '',
+      requiresCgo: metadata.requiresCgo || false,
+      cloudOnly: metadata.cloudOnly || false,
+      cloudSupported: metadata.cloudSupported || false
     };
   });
 
-  // Removed components
+  // Removed components (include platform metadata to understand why removed)
   const removedComponentKeys = Object.keys(oldMap).filter(k => !(k in newMap));
   const removedComponents = removedComponentKeys.map(key => {
     const [type, name] = key.split(':');
     const raw = oldMap[key].raw;
+    const metadata = oldMap[key].metadata || {};
     return {
       name,
       type,
       status: raw.status || raw.type || '',
       version: raw.version || raw.introducedInVersion || '',
-      description: raw.description || ''
+      description: raw.description || '',
+      requiresCgo: metadata.requiresCgo || false,
+      cloudOnly: metadata.cloudOnly || false,
+      cloudSupported: metadata.cloudSupported || false
     };
   });
 
@@ -99,6 +108,57 @@ function generateConnectorDiffJson(oldIndex, newIndex, opts = {}) {
     }
   });
 
+  // Platform transitions (CGO, cloud support changes)
+  const platformTransitions = [];
+  Object.keys(newMap).forEach(cKey => {
+    if (!(cKey in oldMap)) return;
+
+    const oldMeta = oldMap[cKey].metadata || {};
+    const newMeta = newMap[cKey].metadata || {};
+    const [type, name] = cKey.split(':');
+
+    const transitions = [];
+
+    // CGO requirement changes
+    if (!oldMeta.requiresCgo && newMeta.requiresCgo) {
+      transitions.push('became_cgo_only');
+    } else if (oldMeta.requiresCgo && !newMeta.requiresCgo) {
+      transitions.push('no_longer_cgo_only');
+    }
+
+    // Cloud support changes
+    if (!oldMeta.cloudSupported && newMeta.cloudSupported) {
+      transitions.push('added_cloud_support');
+    } else if (oldMeta.cloudSupported && !newMeta.cloudSupported) {
+      transitions.push('removed_cloud_support');
+    }
+
+    // Cloud-only status changes
+    if (!oldMeta.cloudOnly && newMeta.cloudOnly) {
+      transitions.push('became_cloud_only');
+    } else if (oldMeta.cloudOnly && !newMeta.cloudOnly) {
+      transitions.push('no_longer_cloud_only');
+    }
+
+    if (transitions.length > 0) {
+      platformTransitions.push({
+        name,
+        type,
+        transitions,
+        oldPlatform: {
+          requiresCgo: oldMeta.requiresCgo || false,
+          cloudSupported: oldMeta.cloudSupported || false,
+          cloudOnly: oldMeta.cloudOnly || false
+        },
+        newPlatform: {
+          requiresCgo: newMeta.requiresCgo || false,
+          cloudSupported: newMeta.cloudSupported || false,
+          cloudOnly: newMeta.cloudOnly || false
+        }
+      });
+    }
+  });
+
   // Newly deprecated fields (exist in both versions but became deprecated)
   const deprecatedFields = [];
   // Changed default values
@@ -221,6 +281,7 @@ function generateConnectorDiffJson(oldIndex, newIndex, opts = {}) {
       deprecatedComponents: deprecatedComponents.length,
       deprecatedFields: deprecatedFields.length,
       changedDefaults: changedDefaults.length,
+      platformTransitions: platformTransitions.length,
       newBloblangMethods: newBloblangMethods.length,
       removedBloblangMethods: removedBloblangMethods.length,
       newBloblangFunctions: newBloblangFunctions.length,
@@ -236,6 +297,7 @@ function generateConnectorDiffJson(oldIndex, newIndex, opts = {}) {
       deprecatedComponents,
       deprecatedFields,
       changedDefaults,
+      platformTransitions,
       newBloblangMethods,
       removedBloblangMethods,
       newBloblangFunctions,
@@ -341,7 +403,19 @@ function buildComponentMap(indexObj) {
       }
 
       const fieldNames = childArray.map(f => f.name);
-      map[lookupKey] = { raw: component, fields: fieldNames };
+
+      // Preserve platform metadata for accurate diff comparison
+      const metadata = {
+        requiresCgo: component.requiresCgo || false,
+        cloudSupported: component.cloudSupported || false,
+        cloudOnly: component.cloudOnly || false
+      };
+
+      map[lookupKey] = {
+        raw: component,
+        fields: fieldNames,
+        metadata: metadata
+      };
     });
   });
 

package/tools/redpanda-connect/rpcn-connector-docs-handler.js

@@ -92,6 +92,19 @@ function capToTwoSentences (description) {
   return result.trim()
 }
 
+/**
+ * Remove platform metadata fields from connectors
+ * @param {Array} connectors - Array of connector objects
+ */
+function stripPlatformMetadata (connectors) {
+  if (!Array.isArray(connectors)) return
+  connectors.forEach(c => {
+    delete c.cloudSupported
+    delete c.requiresCgo
+    delete c.cloudOnly
+  })
+}
+
 /**
  * Update whats-new.adoc with new release information
  * @param {Object} params - Parameters
@@ -487,37 +500,6 @@ function logCollapsed (label, filesArray, maxToShow = 10) {
   console.log('')
 }
 
-/**
- * Strip augmentation fields from connector data to ensure clean comparisons
- * Removes cloudSupported, requiresCgo, cloudOnly fields and filters out cloud-only connectors
- * @param {object} data - Connector index data
- * @returns {object} Clean connector data without augmentation
- */
-function stripAugmentationFields(data) {
-  const cleanData = JSON.parse(JSON.stringify(data));
-  const connectorTypes = ['inputs', 'outputs', 'processors', 'caches', 'rate_limits',
-    'buffers', 'metrics', 'scanners', 'tracers', 'config', 'bloblang-methods'];
-
-  for (const type of connectorTypes) {
-    if (Array.isArray(cleanData[type])) {
-      // Remove connectors that were added by augmentation (cloudOnly or requiresCgo without OSS data)
-      cleanData[type] = cleanData[type].filter(c => {
-        // Keep if it's not marked as cloudOnly or requiresCgo
-        // OR if it has a config/fields (meaning it came from OSS, not just binary analysis)
-        return (!(c.cloudOnly || c.requiresCgo) || c.config || c.fields);
-      });
-
-      // Remove augmentation fields
-      cleanData[type].forEach(c => {
-        delete c.cloudSupported;
-        delete c.requiresCgo;
-        delete c.cloudOnly;
-      });
-    }
-  }
-
-  return cleanData;
-}
 
 /**
  * Load or fetch connector data for a specific version
@@ -529,15 +511,11 @@ function stripAugmentationFields(data) {
 async function loadConnectorDataForVersion(version, dataDir, options = {}) {
   const dataFile = path.join(dataDir, `connect-${version}.json`);
 
-  // If file exists, load it
+  // If file exists, load it (with platform metadata intact)
   if (fs.existsSync(dataFile)) {
     console.log(`✓ Using existing data file: connect-${version}.json`);
     const data = JSON.parse(fs.readFileSync(dataFile, 'utf8'));
-
-    // Strip augmentation fields to ensure clean comparisons
-    // Augmentation adds CGO/cloud-only components that shouldn't affect version diffs
-    const cleanData = stripAugmentationFields(data);
-    return cleanData;
+    return data;
   }
 
   // If not, fetch it
@@ -940,8 +918,8 @@ async function handleRpcnConnectorDocs (options) {
   let oldIndex = {}
   let oldVersion = null
   if (options.oldData && fs.existsSync(options.oldData)) {
-    // Strip augmentation fields to ensure clean comparisons
-    oldIndex = stripAugmentationFields(JSON.parse(fs.readFileSync(options.oldData, 'utf8')))
+    // Load with platform metadata intact for accurate diff
+    oldIndex = JSON.parse(fs.readFileSync(options.oldData, 'utf8'))
     const m = options.oldData.match(/connect-([\d.]+)\.json$/)
     if (m) oldVersion = m[1]
   } else {
@@ -959,30 +937,43 @@ async function handleRpcnConnectorDocs (options) {
       oldVersion = sortedVersions[0]
       const oldFile = `connect-${oldVersion}.json`
       const oldPath = path.join(dataDir, oldFile)
-      // Strip augmentation fields to ensure clean comparisons
-      oldIndex = stripAugmentationFields(JSON.parse(fs.readFileSync(oldPath, 'utf8')))
+      // Load with platform metadata intact for accurate diff
+      oldIndex = JSON.parse(fs.readFileSync(oldPath, 'utf8'))
       console.log(`📋 Using old version data: ${oldFile}`)
     } else {
       oldVersion = getAntoraValue('asciidoc.attributes.latest-connect-version')
       if (oldVersion) {
         const oldPath = path.join(dataDir, `connect-${oldVersion}.json`)
         if (fs.existsSync(oldPath)) {
-          // Strip augmentation fields to ensure clean comparisons
-          oldIndex = stripAugmentationFields(JSON.parse(fs.readFileSync(oldPath, 'utf8')))
+          // Load with platform metadata intact for accurate diff
+          oldIndex = JSON.parse(fs.readFileSync(oldPath, 'utf8'))
         }
       }
     }
   }
 
-  // Load and strip augmentation fields for clean comparisons
-  let newIndex = stripAugmentationFields(JSON.parse(fs.readFileSync(dataFile, 'utf8')))
+  // Load with platform metadata intact for accurate diff
+  let newIndex = JSON.parse(fs.readFileSync(dataFile, 'utf8'))
 
-  // Save a clean copy of OSS data for binary analysis (before augmentation)
-  // This ensures the binary analyzer compares actual binaries, not augmented data
+  // Save a clean copy of OSS data for binary analysis
+  // Binary analyzer needs pure OSS data without augmented CGO/cloud connectors
   const cleanOssDataPath = path.join(dataDir, `._connect-${newVersion}-clean.json`)
 
-  // Use the already-stripped newIndex for clean data
+  // Create clean version by removing augmented connectors
   const cleanData = JSON.parse(JSON.stringify(newIndex))
+  const connectorTypes = ['inputs', 'outputs', 'processors', 'caches', 'rate_limits',
+    'buffers', 'metrics', 'scanners', 'tracers']
+
+  for (const type of connectorTypes) {
+    if (Array.isArray(cleanData[type])) {
+      // Keep only connectors from OSS rpk (have config/fields)
+      // Remove augmentation-only connectors (added by previous binary analysis)
+      cleanData[type] = cleanData[type].filter(c => c.config || c.fields)
+
+      // Remove platform metadata from remaining connectors
+      stripPlatformMetadata(cleanData[type])
+    }
+  }
 
   fs.writeFileSync(cleanOssDataPath, JSON.stringify(cleanData, null, 2), 'utf8')
 
@@ -1031,8 +1022,6 @@ async function handleRpcnConnectorDocs (options) {
     }
   }
 
-  printDeltaReport(oldIndex, newIndex)
-
   // Binary analysis
   let oldBinaryAnalysis = null
 
@@ -1250,9 +1239,10 @@ async function handleRpcnConnectorDocs (options) {
         }
       }
 
-      // NOTE: We do NOT reload newIndex after augmentation
-      // Diff generation should use clean OSS data to avoid false positives from CGO/cloud-only components
-      // The augmented data is saved to disk but not used for version comparisons
+      // IMPORTANT: Reload newIndex with augmented data for unified diff
+      // The unified diff approach compares platform metadata to detect transitions
+      newIndex = connectorData
+      console.log(`✓ Reloaded newIndex with augmented data for diff comparison`)
     } catch (err) {
       console.error(`Warning: Failed to augment data file: ${err.message}`)
     }
@@ -1265,9 +1255,51 @@ async function handleRpcnConnectorDocs (options) {
   } else if (versionsMatch) {
     console.log(`⏭️  Skipping diff generation: versions match (${oldVersion} === ${newVersion})`)
   } else {
+    // FALLBACK: If binary analysis failed, strip CGO/cloud augmentation from old data
+    // to prevent false "removed" reports when comparing augmented old vs non-augmented new
+    let oldIndexForDiff = oldIndex
+    // Check if CGO analysis specifically failed (cgoIndex will be undefined if CGO binary couldn't be analyzed)
+    const cgoAnalysisFailed = !binaryAnalysis || !binaryAnalysis.ossVersion || !binaryAnalysis.cgoIndex
+    if (cgoAnalysisFailed) {
+      console.log('⚠️  Binary analysis unavailable - stripping CGO/cloud metadata from old data for clean comparison')
+
+      // Strip CGO/cloud-only connectors and metadata from old data
+      oldIndexForDiff = JSON.parse(JSON.stringify(oldIndex))
+      const connectorTypes = ['inputs', 'outputs', 'processors', 'caches', 'rate_limits',
+        'buffers', 'metrics', 'scanners', 'tracers']
+
+      let totalStripped = 0
+      for (const type of connectorTypes) {
+        if (Array.isArray(oldIndexForDiff[type])) {
+          const originalCount = oldIndexForDiff[type].length
+
+          // Remove connectors marked as CGO-only or cloud-only
+          // These shouldn't appear as "removed" when binary analysis is unavailable
+          oldIndexForDiff[type] = oldIndexForDiff[type].filter(c => {
+            return !(c.requiresCgo || c.cloudOnly)
+          })
+
+          const removed = originalCount - oldIndexForDiff[type].length
+          if (removed > 0) {
+            console.log(`   • Stripped ${removed} CGO/cloud connectors from ${type}`)
+            totalStripped += removed
+          }
+
+          // Remove platform metadata from remaining connectors
+          stripPlatformMetadata(oldIndexForDiff[type])
+        }
+      }
+
+      if (totalStripped > 0) {
+        console.log(`   ✓ Total stripped: ${totalStripped} CGO/cloud connectors`)
+      }
+    }
+
+    printDeltaReport(oldIndexForDiff, newIndex)
+
     const { generateConnectorDiffJson } = require('./report-delta.js')
     diffJson = generateConnectorDiffJson(
-      oldIndex,
+      oldIndexForDiff,
       newIndex,
       {
         oldVersion: oldVersion,