@redpanda-data/docs-extensions-and-macros 4.8.1 → 4.9.1

package/tools/property-extractor/templates/property-cloud.hbs

@@ -0,0 +1,105 @@
+{{#if cloud_supported}}
+// tag::redpanda-cloud[]
+{{/if}}
+=== {{name}}
+{{#if version}}
+
+*Introduced in {{version}}*
+{{/if}}
+{{#if description}}
+
+{{{description}}}
+{{else}}
+
+No description available.
+{{/if}}
+{{#if is_enterprise}}
+
+ifndef::env-cloud[]
+*Enterprise license required*: `{{enterprise_value}}` (for license details, see xref:get-started:licensing/index.adoc[Redpanda Licensing])
+endif::[]
+{{/if}}
+{{#if cloud_byoc_only}}
+
+NOTE: This property is available only in Redpanda Cloud BYOC deployments.
+{{/if}}
+{{#if units}}
+
+*Unit:* {{units}}
+{{else}}
+{{#if (formatUnits name)}}
+
+*Unit:* {{formatUnits name}}
+{{/if}}
+{{/if}}
+{{#if (ne defined_in "src/v/config/node_config.cc")}}
+{{#if (ne needs_restart undefined)}}
+
+*Requires restart:* {{#if needs_restart}}Yes{{else}}No{{/if}}
+{{/if}}
+{{/if}}
+{{#if visibility}}
+
+// tag::self-managed-only[]
+*Visibility:* `{{visibility}}`
+// end::self-managed-only[]
+{{/if}}
+{{#if type}}
+
+*Type:* {{type}}
+{{/if}}
+{{#if (and minimum maximum)}}
+
+*Accepted values:* [`{{minimum}}`, `{{maximum}}`]
+{{else}}
+{{#if minimum}}
+
+*Minimum value:* `{{minimum}}`
+{{/if}}
+{{#if maximum}}
+
+*Maximum value:* `{{maximum}}`
+{{/if}}
+{{/if}}
+{{#if (ne default undefined)}}
+
+ifdef::env-cloud[]
+*Default:* Available in the Redpanda Cloud Console
+endif::[]
+ifndef::env-cloud[]
+*Default:* `{{formatPropertyValue default type}}`
+endif::[]
+{{/if}}
+
+// tag::self-managed-only[]
+*Nullable:* {{#if nullable}}Yes{{else}}No{{/if}}
+// end::self-managed-only[]
+{{#if example}}
+
+{{{renderPropertyExample this}}}
+{{/if}}
+{{#if related_topics}}
+
+*Related topics:*
+
+{{#each related_topics}}
+* {{{this}}}
+{{/each}}
+{{/if}}
+{{#if aliases}}
+
+// tag::self-managed-only[]
+*Aliases:* {{join aliases ", "}}
+// end::self-managed-only[]
+{{/if}}
+{{#if is_deprecated}}
+
+[WARNING]
+====
+This property is deprecated.
+====
+{{/if}}
+---
+{{#if cloud_supported}}
+// end::redpanda-cloud[]
+{{/if}}

package/tools/property-extractor/templates/property.hbs

@@ -2,14 +2,21 @@
 
 {{#if version}}
 *Introduced in {{version}}*
-
 {{/if}}
+
 {{#if description}}
 {{{description}}}
 {{else}}
 No description available.
 {{/if}}
 
+{{#if is_enterprise}}
+ifndef::env-cloud[]
+*Enterprise license required*: `{{enterprise_value}}` (for license details, see xref:get-started:licensing/index.adoc[Redpanda Licensing])
+endif::[]
+
+{{/if}}
+
 {{#if units}}
 *Unit:* {{units}}
 
@@ -56,6 +63,14 @@ No description available.
 {{{renderPropertyExample this}}}
 {{/if}}
 
+{{#if related_topics}}
+*Related topics:*
+
+{{#each related_topics}}
+* {{{this}}}
+{{/each}}
+{{/if}}
+
 {{#if aliases}}
 *Aliases:* {{join aliases ", "}}
 

package/tools/property-extractor/templates/topic-property-cloud.hbs

@@ -0,0 +1,97 @@
+{{#if cloud_supported}}
+// tag::redpanda-cloud[]
+{{/if}}
+=== {{name}}
+{{#if version}}
+
+*Introduced in {{version}}*
+{{/if}}
+{{#if description}}
+
+{{{description}}}
+{{else}}
+
+No description available.
+{{/if}}
+{{#if is_enterprise}}
+
+ifndef::env-cloud[]
+*Enterprise license required*: `{{enterprise_value}}` (for license details, see xref:get-started:licensing/index.adoc[Redpanda Licensing])
+endif::[]
+{{/if}}
+{{#if cloud_byoc_only}}
+
+NOTE: This property is only available in Redpanda Cloud BYOC deployments.
+{{/if}}
+{{#if type}}
+
+*Type:* {{type}}
+{{/if}}
+{{#if acceptable_values}}
+
+*Accepted values:* {{{acceptable_values}}}
+{{/if}}
+{{#if corresponding_cluster_property}}
+
+*Related cluster property:* xref:reference:cluster-properties.adoc#{{corresponding_cluster_property}}[{{corresponding_cluster_property}}]
+{{/if}}
+{{#if (and minimum maximum)}}
+
+*Accepted values:* [`{{minimum}}`, `{{maximum}}`]
+{{else}}
+{{#if minimum}}
+
+*Minimum value:* `{{minimum}}`
+{{/if}}
+{{#if maximum}}
+
+*Maximum value:* `{{maximum}}`
+{{/if}}
+{{/if}}
+{{#if (ne default undefined)}}
+{{#if cloud_supported}}
+
+ifdef::env-cloud[]
+*Default:* Available in the Redpanda Cloud Console
+endif::[]
+ifndef::env-cloud[]
+*Default:* `{{formatPropertyValue default type}}`
+endif::[]
+{{else}}
+
+*Default:* `{{formatPropertyValue default type}}`
+{{/if}}
+{{/if}}
+
+// tag::self-managed-only[]
+*Nullable:* {{#if nullable}}Yes{{else}}No{{/if}}
+// end::self-managed-only[]
+{{#if example}}
+
+{{{renderPropertyExample this}}}
+{{/if}}
+{{#if related_topics}}
+
+*Related topics:*
+
+{{#each related_topics}}
+* {{{this}}}
+{{/each}}
+{{/if}}
+{{#if aliases}}
+
+// tag::self-managed-only[]
+*Aliases:* {{join aliases ", "}}
+// end::self-managed-only[]
+{{/if}}
+{{#if is_deprecated}}
+
+[WARNING]
+====
+This property is deprecated.
+====
+{{/if}}
+---
+{{#if cloud_supported}}
+// end::redpanda-cloud[]
+{{/if}}

package/tools/property-extractor/templates/topic-property.hbs

@@ -1,59 +1,73 @@
 === {{name}}
-
 {{#if version}}
-*Introduced in {{version}}*
 
+*Introduced in {{version}}*
 {{/if}}
 {{#if description}}
+
 {{{description}}}
 {{else}}
+
 No description available.
 {{/if}}
+{{#if is_enterprise}}
 
+ifndef::env-cloud[]
+*Enterprise license required*: `{{enterprise_value}}` (for license details, see xref:get-started:licensing/index.adoc[Redpanda Licensing])
+endif::[]
+{{/if}}
 {{#if type}}
-*Type:* {{type}}
 
+*Type:* {{type}}
 {{/if}}
 {{#if acceptable_values}}
-*Accepted values:* {{{acceptable_values}}}
 
+*Accepted values:* {{{acceptable_values}}}
 {{/if}}
 {{#if corresponding_cluster_property}}
-*Related cluster property:* xref:reference:cluster-properties.adoc#{{corresponding_cluster_property}}[{{corresponding_cluster_property}}]
 
+*Related cluster property:* xref:reference:cluster-properties.adoc#{{corresponding_cluster_property}}[{{corresponding_cluster_property}}]
 {{/if}}
 {{#if (and minimum maximum)}}
-*Accepted values:* [`{{minimum}}`, `{{maximum}}`]
 
+*Accepted values:* [`{{minimum}}`, `{{maximum}}`]
 {{else}}
 {{#if minimum}}
-*Minimum value:* `{{minimum}}`
 
+*Minimum value:* `{{minimum}}`
 {{/if}}
 {{#if maximum}}
-*Maximum value:* `{{maximum}}`
 
+*Maximum value:* `{{maximum}}`
 {{/if}}
 {{/if}}
 {{#if (ne default undefined)}}
-*Default:* `{{formatPropertyValue default type}}`
 
+*Default:* `{{formatPropertyValue default type}}`
 {{/if}}
-*Nullable:* {{#if nullable}}Yes{{else}}No{{/if}}
 
+*Nullable:* {{#if nullable}}Yes{{else}}No{{/if}}
 {{#if example}}
+
 {{{renderPropertyExample this}}}
 {{/if}}
+{{#if related_topics}}
+
+*Related topics:*
 
+{{#each related_topics}}
+* {{{this}}}
+{{/each}}
+{{/if}}
 {{#if aliases}}
-*Aliases:* {{join aliases ", "}}
 
+*Aliases:* {{join aliases ", "}}
 {{/if}}
 {{#if is_deprecated}}
+
 [WARNING]
 ====
 This property is deprecated.
 ====
-
 {{/if}}
 ---

package/tools/property-extractor/transformers.py

@@ -1,13 +1,50 @@
 import re
+import logging
 from property_bag import PropertyBag
 from parser import normalize_string
 
+# Get logger for this module
+logger = logging.getLogger(__name__)
+
+# Import the process_enterprise_value function from property_extractor
+# Note: We import at function level to avoid circular imports since property_extractor
+# imports transformers.py. This pattern allows the EnterpriseTransformer to access
+# the centralized enterprise value processing logic without creating import cycles.
+def get_process_enterprise_value():
+    """
+    Lazily import and return the centralized `process_enterprise_value` function from `property_extractor`.
+    
+    Attempts to import `process_enterprise_value` and return it to avoid circular-import issues. If the import fails an error message is printed and None is returned.
+    
+    Returns:
+        Callable or None: The `process_enterprise_value` callable when available, otherwise `None`.
+    """
+    try:
+        from property_extractor import process_enterprise_value
+        return process_enterprise_value
+    except ImportError as e:
+        logger.error("Cannot import process_enterprise_value from property_extractor: %s", e)
+        return None
+
 
 class BasicInfoTransformer:
     def accepts(self, info, file_pair):
+        """
+        Always accepts the provided info and file_pair.
+        
+        Parameters:
+        	info (dict): Parsed metadata for a property (annotation/params/declaration).
+        	file_pair (object): Pair of source/implementation file metadata used by transformers.
+        
+        Returns:
+        	bool: Always returns True, indicating this transformer should be applied.
+        """
         return True
 
     def parse(self, property, info, file_pair):
+        if not info.get("params") or len(info["params"]) == 0:
+            return property
+            
         property["name"] = info["params"][0]["value"]
         property["defined_in"] = re.sub(
             r"^.*src/", "src/", str(file_pair.implementation)
@@ -427,13 +464,65 @@ class AliasTransformer:
 
 
 class EnterpriseTransformer:
+    """
+    Transforms enterprise property values from C++ expressions to user-friendly JSON.
+
+    This transformer processes enterprise values by delegating to the
+    centralized process_enterprise_value function which handles the full range of
+    C++ expression types found in enterprise property definitions.
+    """
     def accepts(self, info, file_pair):
+        """
+        Return True if the provided info indicates an enterprise-only property.
+        
+        Parameters:
+            info (dict): The metadata dictionary for a property. This function checks for a 'type' key whose string contains 'enterprise'.
+            file_pair: Unused; present for transformer interface compatibility.
+        
+        Returns:
+            bool: True when info contains a 'type' that includes 'enterprise', otherwise False.
+        """
         return bool(info.get('type') and 'enterprise' in info['type'])
 
     def parse(self, property, info, file_pair):
+        """
+        Mark a property as enterprise-only and attach its enterprise value.
+        
+        If an enterprise value is present in info['params'][0]['value'], this method attempts to process it using the shared
+        process_enterprise_value helper (loaded via get_process_enterprise_value()). If the processor is unavailable or raises
+        an exception, the raw enterprise value is used.
+        
+        Side effects:
+        - Sets property["enterprise_value"] to the processed or raw value.
+        - Sets property["is_enterprise"] = True.
+        - Removes the first element from info['params'].
+        
+        Parameters:
+            property (dict): Property bag to modify and return.
+            info (dict): Parsed metadata; must have a non-None 'params' list for processing.
+            file_pair: Unused here but accepted for transformer API compatibility.
+        
+        Returns:
+            dict: The updated property bag.
+        """
         if info['params'] is not None:
             enterpriseValue = info['params'][0]['value']
-            property['enterprise_value'] = enterpriseValue
+            
+            # Get the processing function
+            process_enterprise_value = get_process_enterprise_value()
+            if process_enterprise_value is None:
+                property["enterprise_value"] = enterpriseValue
+                property['is_enterprise'] = True
+                del info['params'][0]
+                return property
+            
+            try:
+                processed_value = process_enterprise_value(enterpriseValue)
+                property["enterprise_value"] = processed_value
+            except Exception:
+                # Fallback to raw value if processing fails
+                property["enterprise_value"] = enterpriseValue
+
             property['is_enterprise'] = True
             del info['params'][0]
             return property
@@ -462,7 +551,14 @@ class MetaParamTransformer:
         iterable_params = info['params']
         for param in iterable_params:
             if isinstance(param['value'], str) and param['value'].startswith("meta{"):
-                meta_content = param['value'].strip("meta{ }").strip()
+                # Extract content between meta{ and } using explicit slicing
+                param_value = param['value']
+                if param_value.endswith('}'):
+                    meta_content = param_value[5:-1].strip()  # Remove "meta{" and "}"
+                else:
+                    # Handle malformed meta{ without closing }
+                    meta_content = param_value[5:].strip()  # Remove "meta{" only
+                
                 meta_dict = {}
                 for item in meta_content.split(','):
                     item = item.strip()

package/tools/redpanda-connect/generate-rpcn-connector-docs.js

@@ -209,6 +209,31 @@ function resolveReferences(obj, root) {
  * When generating full drafts, components with a `status` of `'deprecated'` are skipped.
  */
 async function generateRpcnConnectorDocs(options) {
+  // Types and output folders for bloblang function/method partials
+  const bloblangTypes = [
+    { key: 'bloblang-functions', folder: 'bloblang-functions' },
+    { key: 'bloblang-methods', folder: 'bloblang-methods' }
+  ];
+  // Recursively mark is_beta on any field/component with description starting with BETA:
+  function markBeta(obj) {
+    if (!obj || typeof obj !== 'object') return;
+    if (Array.isArray(obj)) {
+      obj.forEach(markBeta);
+      return;
+    }
+    // Mark as beta if description starts with 'BETA:' (case-insensitive, trims leading whitespace)
+    if (typeof obj.description === 'string' && /^\s*BETA:\s*/i.test(obj.description)) {
+      obj.is_beta = true;
+    }
+    // Recurse into children/config/fields
+    if (Array.isArray(obj.children)) obj.children.forEach(markBeta);
+    if (obj.config && Array.isArray(obj.config.children)) obj.config.children.forEach(markBeta);
+    // For connector/component arrays
+    for (const key of Object.keys(obj)) {
+      if (Array.isArray(obj[key])) obj[key].forEach(markBeta);
+    }
+  }
+
   const {
     data,
     overrides,
@@ -216,6 +241,7 @@ async function generateRpcnConnectorDocs(options) {
     templateIntro,
     templateFields,
     templateExamples,
+    templateBloblang,
     writeFullDrafts
   } = options;
 
@@ -223,16 +249,38 @@ async function generateRpcnConnectorDocs(options) {
   const raw = fs.readFileSync(data, 'utf8');
   const ext = path.extname(data).toLowerCase();
   const dataObj = ext === '.json' ? JSON.parse(raw) : yaml.parse(raw);
+  // Mark beta fields/components before overrides
+  markBeta(dataObj);
 
   // Apply overrides if provided
   if (overrides) {
     const ovRaw = fs.readFileSync(overrides, 'utf8');
     const ovObj = JSON.parse(ovRaw);
-    
     // Resolve any $ref references in the overrides
     const resolvedOverrides = resolveReferences(ovObj, ovObj);
-    
     mergeOverrides(dataObj, resolvedOverrides);
+
+    // Special: merge bloblang_methods and bloblang_functions from overrides into main data
+    for (const [overrideKey, mainKey] of [
+      ['bloblang_methods', 'bloblang-methods'],
+      ['bloblang_functions', 'bloblang-functions']
+    ]) {
+      if (Array.isArray(resolvedOverrides[overrideKey])) {
+        if (!Array.isArray(dataObj[mainKey])) dataObj[mainKey] = [];
+        // Merge by name
+        const mainArr = dataObj[mainKey];
+        const overrideArr = resolvedOverrides[overrideKey];
+        for (const overrideItem of overrideArr) {
+          if (!overrideItem.name) continue;
+          const idx = mainArr.findIndex(i => i.name === overrideItem.name);
+          if (idx !== -1) {
+            mainArr[idx] = { ...mainArr[idx], ...overrideItem };
+          } else {
+            mainArr.push(overrideItem);
+          }
+        }
+      }
+    }
   }
 
   // Compile the “main” template (used when writeFullDrafts = true)
@@ -260,6 +308,7 @@ async function generateRpcnConnectorDocs(options) {
   const fieldsOutRoot   = path.join(outputRoot, 'fields');
   const examplesOutRoot = path.join(outputRoot, 'examples');
   const draftsRoot      = path.join(outputRoot, 'drafts');
+  const configExamplesRoot = path.resolve(process.cwd(), 'modules/components/examples');
 
   if (!writeFullDrafts) {
     fs.mkdirSync(fieldsOutRoot,   { recursive: true });
@@ -332,6 +381,51 @@ async function generateRpcnConnectorDocs(options) {
     }
   }
 
+  // Bloblang function/method partials (only if includeBloblang is true)
+  if (options.includeBloblang) {
+    for (const { key, folder } of bloblangTypes) {
+      const items = dataObj[key];
+      if (!Array.isArray(items)) continue;
+      const outRoot = path.join(outputRoot, folder);
+      fs.mkdirSync(outRoot, { recursive: true });
+      // Use custom or default template
+      const bloblangTemplatePath = templateBloblang || path.resolve(__dirname, './templates/bloblang-function.hbs');
+      const bloblangTemplate = handlebars.compile(fs.readFileSync(bloblangTemplatePath, 'utf8'));
+      for (const fn of items) {
+        if (!fn.name) continue;
+        const adoc = bloblangTemplate(fn);
+        const outPath = path.join(outRoot, `${fn.name}.adoc`);
+        fs.writeFileSync(outPath, adoc, 'utf8');
+        partialsWritten++;
+        partialFiles.push(path.relative(process.cwd(), outPath));
+      }
+    }
+  }
+
+  // Common/Advanced config snippet YAMLs in modules/components/examples
+  const commonConfig = helpers.commonConfig;
+  const advancedConfig = helpers.advancedConfig;
+  for (const [type, items] of Object.entries(dataObj)) {
+    if (!Array.isArray(items)) continue;
+    for (const item of items) {
+      if (!item.name || !item.config || !Array.isArray(item.config.children)) continue;
+  // Common config
+  const commonYaml = commonConfig(type, item.name, item.config.children);
+  const commonPath = path.join(configExamplesRoot, 'common', type, `${item.name}.yaml`);
+  fs.mkdirSync(path.dirname(commonPath), { recursive: true });
+  fs.writeFileSync(commonPath, commonYaml.toString(), 'utf8');
+  partialsWritten++;
+  partialFiles.push(path.relative(process.cwd(), commonPath));
+  // Advanced config
+  const advYaml = advancedConfig(type, item.name, item.config.children);
+  const advPath = path.join(configExamplesRoot, 'advanced', type, `${item.name}.yaml`);
+  fs.mkdirSync(path.dirname(advPath), { recursive: true });
+  fs.writeFileSync(advPath, advYaml.toString(), 'utf8');
+  partialsWritten++;
+  partialFiles.push(path.relative(process.cwd(), advPath));
+    }
+  }
+
   return {
     partialsWritten,
     draftsWritten,

package/tools/redpanda-connect/helpers/bloblangExample.js

@@ -0,0 +1,42 @@
+// Bloblang example formatting helper for Handlebars
+function bloblangExample(example) {
+  if (typeof example === 'object' && example !== null && example.mapping) {
+    let codeBlock = '';
+    if (example.summary && example.summary.trim()) {
+      codeBlock += `# ${example.summary.trim().replace(/\n/g, '\n# ')}\n\n`;
+    }
+    if (typeof example.mapping === 'string') {
+      codeBlock += example.mapping.trim() + '\n';
+    }
+    if (Array.isArray(example.results)) {
+      for (const pair of example.results) {
+        if (Array.isArray(pair) && pair.length === 2) {
+          codeBlock += `\n# In:  ${pair[0]}\n# Out: ${pair[1]}\n`;
+        }
+      }
+    }
+    return `[,coffeescript]\n----\n${codeBlock.trim()}\n----\n`;
+  } else {
+    let exStr = '';
+    if (typeof example === 'string') {
+      exStr = example;
+    } else if (typeof example === 'object' && example !== null) {
+      if (example.code) {
+        exStr = example.code;
+      } else if (example.example) {
+        exStr = example.example;
+      } else {
+        try {
+          exStr = require('yaml').stringify(example).trim();
+        } catch {
+          exStr = JSON.stringify(example, null, 2);
+        }
+      }
+    } else {
+      exStr = String(example);
+    }
+    return `[source,coffeescript]\n----\n${exStr}\n----\n`;
+  }
+}
+
+module.exports = bloblangExample;

package/tools/redpanda-connect/helpers/index.js

@@ -4,10 +4,10 @@ module.exports = {
   uppercase:       require('./uppercase.js'),
   eq:              require('./eq.js'),
   ne:              require('./ne.js'),
-  join:              require('./join.js'),
+  join:            require('./join.js'),
   or:              require('./or.js'),
-  toYaml:              require('./toYaml.js'),
-  isObject:              require('./isObject.js'),
+  toYaml:          require('./toYaml.js'),
+  isObject:        require('./isObject.js'),
   renderYamlList:  require('./renderYamlList.js'),
   renderConnectFields:    require('./renderConnectFields.js'),
   renderConnectExamples:  require('./renderConnectExamples.js'),
@@ -16,4 +16,5 @@ module.exports = {
   buildConfigYaml:        require('./buildConfigYaml.js'),
   commonConfig:           require('./commonConfig.js'),
   advancedConfig:         require('./advancedConfig.js'),
+  bloblangExample:        require('./bloblangExample.js'),
 };