@redpanda-data/docs-extensions-and-macros 4.15.7 → 4.15.9

package/tools/redpanda-connect/report-delta.js

@@ -2,6 +2,7 @@ const { execSync } = require('child_process');
 
 /**
  * Generate a JSON diff report between two connector index objects.
+ * Includes platform metadata (CGO, cloud-only) to detect transitions.
  * @param {object} oldIndex - Previous version connector index
  * @param {object} newIndex - Current version connector index
  * @param {object} opts - { oldVersion, newVersion, timestamp, binaryAnalysis, oldBinaryAnalysis }
@@ -11,31 +12,39 @@ function generateConnectorDiffJson(oldIndex, newIndex, opts = {}) {
   const oldMap = buildComponentMap(oldIndex);
   const newMap = buildComponentMap(newIndex);
 
-  // New components
+  // New components (include platform metadata)
   const newComponentKeys = Object.keys(newMap).filter(k => !(k in oldMap));
   const newComponents = newComponentKeys.map(key => {
     const [type, name] = key.split(':');
     const raw = newMap[key].raw;
+    const metadata = newMap[key].metadata || {};
     return {
       name,
       type,
       status: raw.status || raw.type || '',
       version: raw.version || raw.introducedInVersion || '',
-      description: raw.description || ''
+      description: raw.description || '',
+      requiresCgo: metadata.requiresCgo || false,
+      cloudOnly: metadata.cloudOnly || false,
+      cloudSupported: metadata.cloudSupported || false
     };
   });
 
-  // Removed components
+  // Removed components (include platform metadata to understand why removed)
   const removedComponentKeys = Object.keys(oldMap).filter(k => !(k in newMap));
   const removedComponents = removedComponentKeys.map(key => {
     const [type, name] = key.split(':');
     const raw = oldMap[key].raw;
+    const metadata = oldMap[key].metadata || {};
     return {
       name,
       type,
       status: raw.status || raw.type || '',
       version: raw.version || raw.introducedInVersion || '',
-      description: raw.description || ''
+      description: raw.description || '',
+      requiresCgo: metadata.requiresCgo || false,
+      cloudOnly: metadata.cloudOnly || false,
+      cloudSupported: metadata.cloudSupported || false
     };
   });
 
@@ -99,6 +108,57 @@ function generateConnectorDiffJson(oldIndex, newIndex, opts = {}) {
     }
   });
 
+  // Platform transitions (CGO, cloud support changes)
+  const platformTransitions = [];
+  Object.keys(newMap).forEach(cKey => {
+    if (!(cKey in oldMap)) return;
+
+    const oldMeta = oldMap[cKey].metadata || {};
+    const newMeta = newMap[cKey].metadata || {};
+    const [type, name] = cKey.split(':');
+
+    const transitions = [];
+
+    // CGO requirement changes
+    if (!oldMeta.requiresCgo && newMeta.requiresCgo) {
+      transitions.push('became_cgo_only');
+    } else if (oldMeta.requiresCgo && !newMeta.requiresCgo) {
+      transitions.push('no_longer_cgo_only');
+    }
+
+    // Cloud support changes
+    if (!oldMeta.cloudSupported && newMeta.cloudSupported) {
+      transitions.push('added_cloud_support');
+    } else if (oldMeta.cloudSupported && !newMeta.cloudSupported) {
+      transitions.push('removed_cloud_support');
+    }
+
+    // Cloud-only status changes
+    if (!oldMeta.cloudOnly && newMeta.cloudOnly) {
+      transitions.push('became_cloud_only');
+    } else if (oldMeta.cloudOnly && !newMeta.cloudOnly) {
+      transitions.push('no_longer_cloud_only');
+    }
+
+    if (transitions.length > 0) {
+      platformTransitions.push({
+        name,
+        type,
+        transitions,
+        oldPlatform: {
+          requiresCgo: oldMeta.requiresCgo || false,
+          cloudSupported: oldMeta.cloudSupported || false,
+          cloudOnly: oldMeta.cloudOnly || false
+        },
+        newPlatform: {
+          requiresCgo: newMeta.requiresCgo || false,
+          cloudSupported: newMeta.cloudSupported || false,
+          cloudOnly: newMeta.cloudOnly || false
+        }
+      });
+    }
+  });
+
   // Newly deprecated fields (exist in both versions but became deprecated)
   const deprecatedFields = [];
   // Changed default values
@@ -163,6 +223,50 @@ function generateConnectorDiffJson(oldIndex, newIndex, opts = {}) {
     });
   });
 
+  // Detect new/removed Bloblang methods and functions
+  const oldMethods = new Set((oldIndex['bloblang-methods'] || []).filter(Boolean).map(m => m.name).filter(Boolean));
+  const newMethods = new Set((newIndex['bloblang-methods'] || []).filter(Boolean).map(m => m.name).filter(Boolean));
+  const oldFunctions = new Set((oldIndex['bloblang-functions'] || []).filter(Boolean).map(f => f.name).filter(Boolean));
+  const newFunctions = new Set((newIndex['bloblang-functions'] || []).filter(Boolean).map(f => f.name).filter(Boolean));
+
+  const newBloblangMethods = Array.from(newMethods).filter(m => !oldMethods.has(m)).sort();
+  const removedBloblangMethods = Array.from(oldMethods).filter(m => !newMethods.has(m)).sort();
+  const newBloblangFunctions = Array.from(newFunctions).filter(f => !oldFunctions.has(f)).sort();
+  const removedBloblangFunctions = Array.from(oldFunctions).filter(f => !newFunctions.has(f)).sort();
+
+  // Detect deprecated Bloblang methods and functions
+  const deprecatedBloblangMethods = [];
+  const deprecatedBloblangFunctions = [];
+
+  const oldMethodsMap = new Map((oldIndex['bloblang-methods'] || []).filter(Boolean).filter(m => m.name).map(m => [m.name, m]));
+  const newMethodsMap = new Map((newIndex['bloblang-methods'] || []).filter(Boolean).filter(m => m.name).map(m => [m.name, m]));
+  const oldFunctionsMap = new Map((oldIndex['bloblang-functions'] || []).filter(Boolean).filter(f => f.name).map(f => [f.name, f]));
+  const newFunctionsMap = new Map((newIndex['bloblang-functions'] || []).filter(Boolean).filter(f => f.name).map(f => [f.name, f]));
+
+  // Check methods for newly deprecated status
+  newMethodsMap.forEach((newMethod, name) => {
+    const oldMethod = oldMethodsMap.get(name);
+    if (oldMethod) {
+      const oldStatus = (oldMethod.status || '').toLowerCase();
+      const newStatus = (newMethod.status || '').toLowerCase();
+      if (oldStatus !== 'deprecated' && newStatus === 'deprecated') {
+        deprecatedBloblangMethods.push(name);
+      }
+    }
+  });
+
+  // Check functions for newly deprecated status
+  newFunctionsMap.forEach((newFunction, name) => {
+    const oldFunction = oldFunctionsMap.get(name);
+    if (oldFunction) {
+      const oldStatus = (oldFunction.status || '').toLowerCase();
+      const newStatus = (newFunction.status || '').toLowerCase();
+      if (oldStatus !== 'deprecated' && newStatus === 'deprecated') {
+        deprecatedBloblangFunctions.push(name);
+      }
+    }
+  });
+
   const result = {
     comparison: {
       oldVersion: opts.oldVersion || '',
@@ -176,7 +280,14 @@ function generateConnectorDiffJson(oldIndex, newIndex, opts = {}) {
       removedFields: removedFields.length,
       deprecatedComponents: deprecatedComponents.length,
       deprecatedFields: deprecatedFields.length,
-      changedDefaults: changedDefaults.length
+      changedDefaults: changedDefaults.length,
+      platformTransitions: platformTransitions.length,
+      newBloblangMethods: newBloblangMethods.length,
+      removedBloblangMethods: removedBloblangMethods.length,
+      newBloblangFunctions: newBloblangFunctions.length,
+      removedBloblangFunctions: removedBloblangFunctions.length,
+      deprecatedBloblangMethods: deprecatedBloblangMethods.length,
+      deprecatedBloblangFunctions: deprecatedBloblangFunctions.length
     },
     details: {
       newComponents,
@@ -185,7 +296,14 @@ function generateConnectorDiffJson(oldIndex, newIndex, opts = {}) {
       removedFields,
       deprecatedComponents,
       deprecatedFields,
-      changedDefaults
+      changedDefaults,
+      platformTransitions,
+      newBloblangMethods,
+      removedBloblangMethods,
+      newBloblangFunctions,
+      removedBloblangFunctions,
+      deprecatedBloblangMethods,
+      deprecatedBloblangFunctions
     }
   };
 
@@ -285,7 +403,19 @@ function buildComponentMap(indexObj) {
       }
 
       const fieldNames = childArray.map(f => f.name);
-      map[lookupKey] = { raw: component, fields: fieldNames };
+
+      // Preserve platform metadata for accurate diff comparison
+      const metadata = {
+        requiresCgo: component.requiresCgo || false,
+        cloudSupported: component.cloudSupported || false,
+        cloudOnly: component.cloudOnly || false
+      };
+
+      map[lookupKey] = {
+        raw: component,
+        fields: fieldNames,
+        metadata: metadata
+      };
     });
   });
 

package/tools/redpanda-connect/rpcn-connector-docs-handler.js

@@ -92,6 +92,19 @@ function capToTwoSentences (description) {
   return result.trim()
 }
 
+/**
+ * Remove platform metadata fields from connectors
+ * @param {Array} connectors - Array of connector objects
+ */
+function stripPlatformMetadata (connectors) {
+  if (!Array.isArray(connectors)) return
+  connectors.forEach(c => {
+    delete c.cloudSupported
+    delete c.requiresCgo
+    delete c.cloudOnly
+  })
+}
+
 /**
  * Update whats-new.adoc with new release information
  * @param {Object} params - Parameters
@@ -487,37 +500,6 @@ function logCollapsed (label, filesArray, maxToShow = 10) {
   console.log('')
 }
 
-/**
- * Strip augmentation fields from connector data to ensure clean comparisons
- * Removes cloudSupported, requiresCgo, cloudOnly fields and filters out cloud-only connectors
- * @param {object} data - Connector index data
- * @returns {object} Clean connector data without augmentation
- */
-function stripAugmentationFields(data) {
-  const cleanData = JSON.parse(JSON.stringify(data));
-  const connectorTypes = ['inputs', 'outputs', 'processors', 'caches', 'rate_limits',
-    'buffers', 'metrics', 'scanners', 'tracers', 'config', 'bloblang-methods'];
-
-  for (const type of connectorTypes) {
-    if (Array.isArray(cleanData[type])) {
-      // Remove connectors that were added by augmentation (cloudOnly or requiresCgo without OSS data)
-      cleanData[type] = cleanData[type].filter(c => {
-        // Keep if it's not marked as cloudOnly or requiresCgo
-        // OR if it has a config/fields (meaning it came from OSS, not just binary analysis)
-        return (!(c.cloudOnly || c.requiresCgo) || c.config || c.fields);
-      });
-
-      // Remove augmentation fields
-      cleanData[type].forEach(c => {
-        delete c.cloudSupported;
-        delete c.requiresCgo;
-        delete c.cloudOnly;
-      });
-    }
-  }
-
-  return cleanData;
-}
 
 /**
  * Load or fetch connector data for a specific version
@@ -529,15 +511,11 @@ function stripAugmentationFields(data) {
 async function loadConnectorDataForVersion(version, dataDir, options = {}) {
   const dataFile = path.join(dataDir, `connect-${version}.json`);
 
-  // If file exists, load it
+  // If file exists, load it (with platform metadata intact)
   if (fs.existsSync(dataFile)) {
     console.log(`✓ Using existing data file: connect-${version}.json`);
     const data = JSON.parse(fs.readFileSync(dataFile, 'utf8'));
-
-    // Strip augmentation fields to ensure clean comparisons
-    // Augmentation adds CGO/cloud-only components that shouldn't affect version diffs
-    const cleanData = stripAugmentationFields(data);
-    return cleanData;
+    return data;
   }
 
   // If not, fetch it
@@ -940,8 +918,8 @@ async function handleRpcnConnectorDocs (options) {
   let oldIndex = {}
   let oldVersion = null
   if (options.oldData && fs.existsSync(options.oldData)) {
-    // Strip augmentation fields to ensure clean comparisons
-    oldIndex = stripAugmentationFields(JSON.parse(fs.readFileSync(options.oldData, 'utf8')))
+    // Load with platform metadata intact for accurate diff
+    oldIndex = JSON.parse(fs.readFileSync(options.oldData, 'utf8'))
     const m = options.oldData.match(/connect-([\d.]+)\.json$/)
     if (m) oldVersion = m[1]
   } else {
@@ -959,30 +937,43 @@ async function handleRpcnConnectorDocs (options) {
       oldVersion = sortedVersions[0]
       const oldFile = `connect-${oldVersion}.json`
       const oldPath = path.join(dataDir, oldFile)
-      // Strip augmentation fields to ensure clean comparisons
-      oldIndex = stripAugmentationFields(JSON.parse(fs.readFileSync(oldPath, 'utf8')))
+      // Load with platform metadata intact for accurate diff
+      oldIndex = JSON.parse(fs.readFileSync(oldPath, 'utf8'))
       console.log(`📋 Using old version data: ${oldFile}`)
     } else {
       oldVersion = getAntoraValue('asciidoc.attributes.latest-connect-version')
       if (oldVersion) {
         const oldPath = path.join(dataDir, `connect-${oldVersion}.json`)
         if (fs.existsSync(oldPath)) {
-          // Strip augmentation fields to ensure clean comparisons
-          oldIndex = stripAugmentationFields(JSON.parse(fs.readFileSync(oldPath, 'utf8')))
+          // Load with platform metadata intact for accurate diff
+          oldIndex = JSON.parse(fs.readFileSync(oldPath, 'utf8'))
         }
       }
     }
   }
 
-  // Load and strip augmentation fields for clean comparisons
-  let newIndex = stripAugmentationFields(JSON.parse(fs.readFileSync(dataFile, 'utf8')))
+  // Load with platform metadata intact for accurate diff
+  let newIndex = JSON.parse(fs.readFileSync(dataFile, 'utf8'))
 
-  // Save a clean copy of OSS data for binary analysis (before augmentation)
-  // This ensures the binary analyzer compares actual binaries, not augmented data
+  // Save a clean copy of OSS data for binary analysis
+  // Binary analyzer needs pure OSS data without augmented CGO/cloud connectors
   const cleanOssDataPath = path.join(dataDir, `._connect-${newVersion}-clean.json`)
 
-  // Use the already-stripped newIndex for clean data
+  // Create clean version by removing augmented connectors
   const cleanData = JSON.parse(JSON.stringify(newIndex))
+  const connectorTypes = ['inputs', 'outputs', 'processors', 'caches', 'rate_limits',
+    'buffers', 'metrics', 'scanners', 'tracers']
+
+  for (const type of connectorTypes) {
+    if (Array.isArray(cleanData[type])) {
+      // Keep only connectors from OSS rpk (have config/fields)
+      // Remove augmentation-only connectors (added by previous binary analysis)
+      cleanData[type] = cleanData[type].filter(c => c.config || c.fields)
+
+      // Remove platform metadata from remaining connectors
+      stripPlatformMetadata(cleanData[type])
+    }
+  }
 
   fs.writeFileSync(cleanOssDataPath, JSON.stringify(cleanData, null, 2), 'utf8')
 
@@ -1031,8 +1022,6 @@ async function handleRpcnConnectorDocs (options) {
     }
   }
 
-  printDeltaReport(oldIndex, newIndex)
-
   // Binary analysis
   let oldBinaryAnalysis = null
 
@@ -1250,9 +1239,10 @@ async function handleRpcnConnectorDocs (options) {
         }
       }
 
-      // NOTE: We do NOT reload newIndex after augmentation
-      // Diff generation should use clean OSS data to avoid false positives from CGO/cloud-only components
-      // The augmented data is saved to disk but not used for version comparisons
+      // IMPORTANT: Reload newIndex with augmented data for unified diff
+      // The unified diff approach compares platform metadata to detect transitions
+      newIndex = connectorData
+      console.log(`✓ Reloaded newIndex with augmented data for diff comparison`)
     } catch (err) {
       console.error(`Warning: Failed to augment data file: ${err.message}`)
     }
@@ -1265,9 +1255,51 @@ async function handleRpcnConnectorDocs (options) {
   } else if (versionsMatch) {
     console.log(`⏭️  Skipping diff generation: versions match (${oldVersion} === ${newVersion})`)
   } else {
+    // FALLBACK: If binary analysis failed, strip CGO/cloud augmentation from old data
+    // to prevent false "removed" reports when comparing augmented old vs non-augmented new
+    let oldIndexForDiff = oldIndex
+    // Check if CGO analysis specifically failed (cgoIndex will be undefined if CGO binary couldn't be analyzed)
+    const cgoAnalysisFailed = !binaryAnalysis || !binaryAnalysis.ossVersion || !binaryAnalysis.cgoIndex
+    if (cgoAnalysisFailed) {
+      console.log('⚠️  Binary analysis unavailable - stripping CGO/cloud metadata from old data for clean comparison')
+
+      // Strip CGO/cloud-only connectors and metadata from old data
+      oldIndexForDiff = JSON.parse(JSON.stringify(oldIndex))
+      const connectorTypes = ['inputs', 'outputs', 'processors', 'caches', 'rate_limits',
+        'buffers', 'metrics', 'scanners', 'tracers']
+
+      let totalStripped = 0
+      for (const type of connectorTypes) {
+        if (Array.isArray(oldIndexForDiff[type])) {
+          const originalCount = oldIndexForDiff[type].length
+
+          // Remove connectors marked as CGO-only or cloud-only
+          // These shouldn't appear as "removed" when binary analysis is unavailable
+          oldIndexForDiff[type] = oldIndexForDiff[type].filter(c => {
+            return !(c.requiresCgo || c.cloudOnly)
+          })
+
+          const removed = originalCount - oldIndexForDiff[type].length
+          if (removed > 0) {
+            console.log(`   • Stripped ${removed} CGO/cloud connectors from ${type}`)
+            totalStripped += removed
+          }
+
+          // Remove platform metadata from remaining connectors
+          stripPlatformMetadata(oldIndexForDiff[type])
+        }
+      }
+
+      if (totalStripped > 0) {
+        console.log(`   ✓ Total stripped: ${totalStripped} CGO/cloud connectors`)
+      }
+    }
+
+    printDeltaReport(oldIndexForDiff, newIndex)
+
     const { generateConnectorDiffJson } = require('./report-delta.js')
     diffJson = generateConnectorDiffJson(
-      oldIndex,
+      oldIndexForDiff,
       newIndex,
       {
         oldVersion: oldVersion,

package/tools/redpanda-connect/templates/bloblang-function.hbs

@@ -1,15 +1,41 @@
 // This content is autogenerated. Do not edit manually. To override descriptions, use the doc-tools CLI with the --overrides option: https://redpandadata.atlassian.net/wiki/spaces/DOC/pages/1247543314/Generate+reference+docs+for+Redpanda+Connect
 
 = {{name}}
-
 {{#if signature}}
+
 *Signature*: `{{signature}}`
 {{/if}}
+{{#if (eq status "deprecated")}}
 
+[WARNING]
+====
+This {{#if signature}}function{{else}}method{{/if}} is deprecated and will be removed in a future version.
+====
+{{/if}}
 {{#if description}}
-{{description}}
+
+{{{ensurePeriod description}}}
 {{/if}}
 
+{{#if params}}
+{{#if params.named}}
+
+== Parameters
+
+[cols="1,1,3"]
+|===
+| Name | Type | Description
+
+{{#each params.named}}
+| `{{name}}`{{#if is_optional}} (optional){{/if}}
+| `{{type}}`
+| {{{description}}}
+
+{{/each}}
+|===
+
+{{/if}}
+{{/if}}
 
 {{#if examples}}
 == Examples

package/tools/redpanda-connect/templates/bloblang-functions-overview.hbs

@@ -0,0 +1,29 @@
+= Bloblang Functions
+// tag::single-source[]
+:description: A list of Bloblang functions
+
+// © 2024 Redpanda Data Inc.
+
+
+Functions can be placed anywhere and allow you to extract information from your environment, generate values, or access data from the underlying message being mapped:
+
+```bloblang
+root.doc.id = uuid_v4()
+root.doc.received_at = now()
+root.doc.host = hostname()
+```
+
+Functions support both named and nameless style arguments:
+
+```bloblang
+root.values_one = range(start: 0, stop: this.max, step: 2)
+root.values_two = range(0, this.max, 2)
+
+# In:  {"max":10}
+```
+{{#each functions}}
+
+include::redpanda-connect:components:partial$bloblang-functions/{{this}}.adoc[leveloffset=+1]
+{{/each}}
+
+// end::single-source[]

package/tools/redpanda-connect/templates/bloblang-methods-overview.hbs

@@ -0,0 +1,39 @@
+= Bloblang Methods
+// tag::single-source[]
+:description: A list of Bloblang methods
+
+// © 2024 Redpanda Data Inc.
+
+
+Methods provide most of the power in Bloblang as they allow you to augment values and can be added to any expression (including other methods):
+
+```bloblang
+root.doc.id = this.thing.id.string().catch(uuid_v4())
+root.doc.reduced_nums = this.thing.nums.map_each(num -> if num < 10 {
+  deleted()
+} else {
+  num - 10
+})
+root.has_good_taste = ["pikachu","mewtwo","magmar"].contains(this.user.fav_pokemon)
+
+# In:  {"thing":{"id":123,"nums":[5,12,18,7,25]},"user":{"fav_pokemon":"pikachu"}}
+```
+
+Methods support both named and nameless style arguments:
+
+```bloblang
+root.foo_one = this.(bar | baz).trim().replace_all(old: "dog", new: "cat")
+root.foo_two = this.(bar | baz).trim().replace_all("dog", "cat")
+
+# In:  {"bar":"  I love my dog  "}
+```
+{{#each categories}}
+
+== {{{toSentenceCase this.name}}}
+{{#each this.methods}}
+
+include::redpanda-connect:components:partial$bloblang-methods/{{this}}.adoc[leveloffset=+2]
+{{/each}}
+{{/each}}
+
+// end::single-source[]

package/tools/redpanda-connect/templates/connector.hbs

@@ -1,10 +1,9 @@
 = {{{name}}}
 // tag::single-source[]
 :type: {{type}}
-:status: {{status}}
 {{#if support}}:support: {{support}}{{/if}}
 :categories: [{{join categories ", "}}]
-:description: {{#if summary}}{{{summary}}}{{else if description}}{{description}}{{else}}TODO: Missing description. Add a description in the overrides file or source data.{{/if}}
+:description: {{#if summary}}{{{summary}}}{{else if description}}{{description}}{{else}}// TODO: Missing description. Add a description in the overrides file or source data.{{/if}}
 
 component_type_dropdown::[]
 

package/tools/redpanda-connect/templates/intro.hbs

@@ -9,5 +9,5 @@ endif::[]
 {{#if description}}
 {{{description}}}
 {{else}}
-TODO: Missing description. Add a description in the overrides file or source data.
+// TODO: Missing description. Add a description in the overrides file or source data.
 {{/if}}