@redpanda-data/docs-extensions-and-macros 4.8.0 → 4.9.0

package/tools/property-extractor/requirements.txt

@@ -1,2 +1,4 @@
 tree_sitter==0.21.1
 setuptools>=42.0.0
+pyyaml>=6.0
+requests>=2.32.5

package/tools/property-extractor/templates/deprecated-properties.hbs

@@ -0,0 +1,25 @@
+{{#if deprecated}}
+= Deprecated Configuration Properties
+:description: This is an exhaustive list of all the deprecated properties.
+
+This is an exhaustive list of all the deprecated properties.
+
+{{#if brokerProperties}}
+== Broker properties
+
+{{#each brokerProperties}}
+{{> deprecated-property}}
+
+{{/each}}
+{{/if}}
+
+{{#if clusterProperties}}
+== Cluster properties
+
+{{#each clusterProperties}}
+{{> deprecated-property}}
+
+{{/each}}
+{{/if}}
+
+{{/if}}

package/tools/property-extractor/templates/deprecated-property.hbs

@@ -0,0 +1,7 @@
+=== `{{name}}`
+
+{{#if description}}
+{{{description}}}
+{{else}}
+No description available.
+{{/if}}

package/tools/property-extractor/templates/property-cloud.hbs

@@ -0,0 +1,105 @@
+{{#if cloud_supported}}
+// tag::redpanda-cloud[]
+{{/if}}
+=== {{name}}
+{{#if version}}
+
+*Introduced in {{version}}*
+{{/if}}
+{{#if description}}
+
+{{{description}}}
+{{else}}
+
+No description available.
+{{/if}}
+{{#if is_enterprise}}
+
+ifndef::env-cloud[]
+*Enterprise license required*: `{{enterprise_value}}` (for license details, see xref:get-started:licensing/index.adoc[Redpanda Licensing])
+endif::[]
+{{/if}}
+{{#if cloud_byoc_only}}
+
+NOTE: This property is available only in Redpanda Cloud BYOC deployments.
+{{/if}}
+{{#if units}}
+
+*Unit:* {{units}}
+{{else}}
+{{#if (formatUnits name)}}
+
+*Unit:* {{formatUnits name}}
+{{/if}}
+{{/if}}
+{{#if (ne defined_in "src/v/config/node_config.cc")}}
+{{#if (ne needs_restart undefined)}}
+
+*Requires restart:* {{#if needs_restart}}Yes{{else}}No{{/if}}
+{{/if}}
+{{/if}}
+{{#if visibility}}
+
+// tag::self-managed-only[]
+*Visibility:* `{{visibility}}`
+// end::self-managed-only[]
+{{/if}}
+{{#if type}}
+
+*Type:* {{type}}
+{{/if}}
+{{#if (and minimum maximum)}}
+
+*Accepted values:* [`{{minimum}}`, `{{maximum}}`]
+{{else}}
+{{#if minimum}}
+
+*Minimum value:* `{{minimum}}`
+{{/if}}
+{{#if maximum}}
+
+*Maximum value:* `{{maximum}}`
+{{/if}}
+{{/if}}
+{{#if (ne default undefined)}}
+
+ifdef::env-cloud[]
+*Default:* Available in the Redpanda Cloud Console
+endif::[]
+ifndef::env-cloud[]
+*Default:* `{{formatPropertyValue default type}}`
+endif::[]
+{{/if}}
+
+// tag::self-managed-only[]
+*Nullable:* {{#if nullable}}Yes{{else}}No{{/if}}
+// end::self-managed-only[]
+{{#if example}}
+
+{{{renderPropertyExample this}}}
+{{/if}}
+{{#if related_topics}}
+
+*Related topics:*
+
+{{#each related_topics}}
+* {{{this}}}
+{{/each}}
+{{/if}}
+{{#if aliases}}
+
+// tag::self-managed-only[]
+*Aliases:* {{join aliases ", "}}
+// end::self-managed-only[]
+{{/if}}
+{{#if is_deprecated}}
+
+[WARNING]
+====
+This property is deprecated.
+====
+{{/if}}
+---
+{{#if cloud_supported}}
+// end::redpanda-cloud[]
+{{/if}}

package/tools/property-extractor/templates/property-page.hbs

@@ -0,0 +1,22 @@
+= {{pageTitle}}
+{{#if pageAliases}}
+:page-aliases: {{join pageAliases ", "}}
+{{/if}}
+:description: {{description}}
+
+{{{intro}}}
+
+{{#if sectionTitle}}
+== {{sectionTitle}}
+{{/if}}
+
+{{#if sectionIntro}}
+{{{sectionIntro}}}
+{{/if}}
+
+{{#each groups}}
+{{#each this.properties}}
+{{> (lookup ../this "template")}}
+
+{{/each}}
+{{/each}}

package/tools/property-extractor/templates/property.hbs

@@ -0,0 +1,85 @@
+=== {{name}}
+
+{{#if version}}
+*Introduced in {{version}}*
+{{/if}}
+
+{{#if description}}
+{{{description}}}
+{{else}}
+No description available.
+{{/if}}
+
+{{#if is_enterprise}}
+ifndef::env-cloud[]
+*Enterprise license required*: `{{enterprise_value}}` (for license details, see xref:get-started:licensing/index.adoc[Redpanda Licensing])
+endif::[]
+
+{{/if}}
+
+{{#if units}}
+*Unit:* {{units}}
+
+{{else}}
+{{#if (formatUnits name)}}
+*Unit:* {{formatUnits name}}
+
+{{/if}}
+{{/if}}
+{{#if (ne defined_in "src/v/config/node_config.cc")}}
+{{#if (ne needs_restart undefined)}}
+*Requires restart:* {{#if needs_restart}}Yes{{else}}No{{/if}}
+
+{{/if}}
+{{/if}}
+{{#if visibility}}
+*Visibility:* `{{visibility}}`
+
+{{/if}}
+{{#if type}}
+*Type:* {{type}}
+
+{{/if}}
+{{#if (and minimum maximum)}}
+*Accepted values:* [`{{minimum}}`, `{{maximum}}`]
+
+{{else}}
+{{#if minimum}}
+*Minimum value:* `{{minimum}}`
+
+{{/if}}
+{{#if maximum}}
+*Maximum value:* `{{maximum}}`
+
+{{/if}}
+{{/if}}
+{{#if (ne default undefined)}}
+*Default:* `{{formatPropertyValue default type}}`
+
+{{/if}}
+*Nullable:* {{#if nullable}}Yes{{else}}No{{/if}}
+
+{{#if example}}
+{{{renderPropertyExample this}}}
+{{/if}}
+
+{{#if related_topics}}
+*Related topics:*
+
+{{#each related_topics}}
+* {{{this}}}
+{{/each}}
+{{/if}}
+
+{{#if aliases}}
+*Aliases:* {{join aliases ", "}}
+
+{{/if}}
+{{#if is_deprecated}}
+[WARNING]
+====
+This property is deprecated.
+====
+
+{{/if}}
+---

package/tools/property-extractor/templates/topic-property-cloud.hbs

@@ -0,0 +1,97 @@
+{{#if cloud_supported}}
+// tag::redpanda-cloud[]
+{{/if}}
+=== {{name}}
+{{#if version}}
+
+*Introduced in {{version}}*
+{{/if}}
+{{#if description}}
+
+{{{description}}}
+{{else}}
+
+No description available.
+{{/if}}
+{{#if is_enterprise}}
+
+ifndef::env-cloud[]
+*Enterprise license required*: `{{enterprise_value}}` (for license details, see xref:get-started:licensing/index.adoc[Redpanda Licensing])
+endif::[]
+{{/if}}
+{{#if cloud_byoc_only}}
+
+NOTE: This property is only available in Redpanda Cloud BYOC deployments.
+{{/if}}
+{{#if type}}
+
+*Type:* {{type}}
+{{/if}}
+{{#if acceptable_values}}
+
+*Accepted values:* {{{acceptable_values}}}
+{{/if}}
+{{#if corresponding_cluster_property}}
+
+*Related cluster property:* xref:reference:cluster-properties.adoc#{{corresponding_cluster_property}}[{{corresponding_cluster_property}}]
+{{/if}}
+{{#if (and minimum maximum)}}
+
+*Accepted values:* [`{{minimum}}`, `{{maximum}}`]
+{{else}}
+{{#if minimum}}
+
+*Minimum value:* `{{minimum}}`
+{{/if}}
+{{#if maximum}}
+
+*Maximum value:* `{{maximum}}`
+{{/if}}
+{{/if}}
+{{#if (ne default undefined)}}
+{{#if cloud_supported}}
+
+ifdef::env-cloud[]
+*Default:* Available in the Redpanda Cloud Console
+endif::[]
+ifndef::env-cloud[]
+*Default:* `{{formatPropertyValue default type}}`
+endif::[]
+{{else}}
+
+*Default:* `{{formatPropertyValue default type}}`
+{{/if}}
+{{/if}}
+
+// tag::self-managed-only[]
+*Nullable:* {{#if nullable}}Yes{{else}}No{{/if}}
+// end::self-managed-only[]
+{{#if example}}
+
+{{{renderPropertyExample this}}}
+{{/if}}
+{{#if related_topics}}
+
+*Related topics:*
+
+{{#each related_topics}}
+* {{{this}}}
+{{/each}}
+{{/if}}
+{{#if aliases}}
+
+// tag::self-managed-only[]
+*Aliases:* {{join aliases ", "}}
+// end::self-managed-only[]
+{{/if}}
+{{#if is_deprecated}}
+
+[WARNING]
+====
+This property is deprecated.
+====
+{{/if}}
+---
+{{#if cloud_supported}}
+// end::redpanda-cloud[]
+{{/if}}

package/tools/property-extractor/templates/topic-property.hbs

@@ -0,0 +1,73 @@
+=== {{name}}
+{{#if version}}
+
+*Introduced in {{version}}*
+{{/if}}
+{{#if description}}
+
+{{{description}}}
+{{else}}
+
+No description available.
+{{/if}}
+{{#if is_enterprise}}
+
+ifndef::env-cloud[]
+*Enterprise license required*: `{{enterprise_value}}` (for license details, see xref:get-started:licensing/index.adoc[Redpanda Licensing])
+endif::[]
+{{/if}}
+{{#if type}}
+
+*Type:* {{type}}
+{{/if}}
+{{#if acceptable_values}}
+
+*Accepted values:* {{{acceptable_values}}}
+{{/if}}
+{{#if corresponding_cluster_property}}
+
+*Related cluster property:* xref:reference:cluster-properties.adoc#{{corresponding_cluster_property}}[{{corresponding_cluster_property}}]
+{{/if}}
+{{#if (and minimum maximum)}}
+
+*Accepted values:* [`{{minimum}}`, `{{maximum}}`]
+{{else}}
+{{#if minimum}}
+
+*Minimum value:* `{{minimum}}`
+{{/if}}
+{{#if maximum}}
+
+*Maximum value:* `{{maximum}}`
+{{/if}}
+{{/if}}
+{{#if (ne default undefined)}}
+
+*Default:* `{{formatPropertyValue default type}}`
+{{/if}}
+
+*Nullable:* {{#if nullable}}Yes{{else}}No{{/if}}
+{{#if example}}
+
+{{{renderPropertyExample this}}}
+{{/if}}
+{{#if related_topics}}
+
+*Related topics:*
+
+{{#each related_topics}}
+* {{{this}}}
+{{/each}}
+{{/if}}
+{{#if aliases}}
+
+*Aliases:* {{join aliases ", "}}
+{{/if}}
+{{#if is_deprecated}}
+
+[WARNING]
+====
+This property is deprecated.
+====
+{{/if}}
+---

package/tools/property-extractor/transformers.py

@@ -1,13 +1,50 @@
 import re
+import logging
 from property_bag import PropertyBag
 from parser import normalize_string
 
+# Get logger for this module
+logger = logging.getLogger(__name__)
+
+# Import the process_enterprise_value function from property_extractor
+# Note: We import at function level to avoid circular imports since property_extractor
+# imports transformers.py. This pattern allows the EnterpriseTransformer to access
+# the centralized enterprise value processing logic without creating import cycles.
+def get_process_enterprise_value():
+    """
+    Lazily import and return the centralized `process_enterprise_value` function from `property_extractor`.
+    
+    Attempts to import `process_enterprise_value` and return it to avoid circular-import issues. If the import fails an error message is printed and None is returned.
+    
+    Returns:
+        Callable or None: The `process_enterprise_value` callable when available, otherwise `None`.
+    """
+    try:
+        from property_extractor import process_enterprise_value
+        return process_enterprise_value
+    except ImportError as e:
+        logger.error("Cannot import process_enterprise_value from property_extractor: %s", e)
+        return None
+
 
 class BasicInfoTransformer:
     def accepts(self, info, file_pair):
+        """
+        Always accepts the provided info and file_pair.
+        
+        Parameters:
+        	info (dict): Parsed metadata for a property (annotation/params/declaration).
+        	file_pair (object): Pair of source/implementation file metadata used by transformers.
+        
+        Returns:
+        	bool: Always returns True, indicating this transformer should be applied.
+        """
         return True
 
     def parse(self, property, info, file_pair):
+        if not info.get("params") or len(info["params"]) == 0:
+            return property
+            
         property["name"] = info["params"][0]["value"]
         property["defined_in"] = re.sub(
             r"^.*src/", "src/", str(file_pair.implementation)
@@ -36,13 +73,52 @@ class IsNullableTransformer:
 
 
 class IsArrayTransformer:
+    """
+    Detects properties that should be treated as arrays based on their C++ type declarations.
+    
+    This transformer identifies two types of array properties:
+    1. std::vector<T> - Standard C++ vectors
+    2. one_or_many_property<T> - Redpanda's custom type that accepts either a single value or an array
+    
+    The one_or_many_property type is used in Redpanda configuration for properties like 'admin' 
+    and 'admin_api_tls' where users can specify either:
+    - A single object: admin: {address: "127.0.0.1", port: 9644}
+    - An array of objects: admin: [{address: "127.0.0.1", port: 9644}, {address: "0.0.0.0", port: 9645}]
+    
+    When detected, these properties are marked with:
+    - type: "array"
+    - items: {type: <inner_type>} where <inner_type> is extracted from T
+    """
+    
+    # Class-level constants for array type patterns
+    ARRAY_PATTERN_STD_VECTOR = "std::vector"
+    ARRAY_PATTERN_ONE_OR_MANY = "one_or_many_property"
+    
     def __init__(self, type_transformer):
         self.type_transformer = type_transformer
 
     def accepts(self, info, file_pair):
-        return "std::vector" in info["declaration"]
+        """
+        Check if this property declaration represents an array type.
+        
+        Returns True for:
+        - std::vector<T> declarations (standard C++ vectors)
+        - one_or_many_property<T> declarations (Redpanda's flexible array type)
+        """
+        return (self.ARRAY_PATTERN_STD_VECTOR in info["declaration"] or 
+                self.ARRAY_PATTERN_ONE_OR_MANY in info["declaration"])
 
     def parse(self, property, info, file_pair):
+        """
+        Transform the property to indicate it's an array type.
+        
+        Sets:
+        - property["type"] = "array"
+        - property["items"]["type"] = <extracted_inner_type>
+        
+        The inner type is extracted by the type_transformer, which handles
+        removing the wrapper (std::vector<> or one_or_many_property<>) to get T.
+        """
         property["type"] = "array"
         property["items"] = PropertyBag()
         property["items"]["type"] = self.type_transformer.get_type_from_declaration(
@@ -94,10 +170,35 @@ class VisibilityTransformer:
 
 
 class TypeTransformer:
+    
+    # Class-level constants for type pattern matching
+    # Shared with IsArrayTransformer for consistency
+    ARRAY_PATTERN_STD_VECTOR = "std::vector"
+    ARRAY_PATTERN_ONE_OR_MANY = "one_or_many_property"
+    OPTIONAL_PATTERN = "std::optional"
+    
     def accepts(self, info, file_pair):
         return True
 
     def get_cpp_type_from_declaration(self, declaration):
+        """
+        Extract the inner type from C++ property declarations.
+        
+        This method handles various C++ template types and extracts the core type T from:
+        - property<T> -> T
+        - std::optional<T> -> T
+        - std::vector<T> -> T  
+        - one_or_many_property<T> -> T (Redpanda's flexible array type)
+        
+        For one_or_many_property, this is crucial because it allows the same property
+        to accept either a single value or an array of values in the configuration.
+        Examples:
+        - one_or_many_property<model::broker_endpoint> -> model::broker_endpoint
+        - one_or_many_property<endpoint_tls_config> -> endpoint_tls_config
+        
+        The extracted type is then used to determine the JSON schema type and
+        for resolving default values from the definitions.
+        """
         one_line_declaration = declaration.replace("\n", "").strip()
         raw_type = (
             re.sub(r"^.*property<(.+)>.*", "\\1", one_line_declaration)
@@ -105,11 +206,19 @@ class TypeTransformer:
             .replace(",", "")
         )
 
-        if "std::optional" in raw_type:
+        if self.OPTIONAL_PATTERN in raw_type:
             raw_type = re.sub(".*std::optional<(.+)>.*", "\\1", raw_type)
 
-        if "std::vector" in raw_type:
+        if self.ARRAY_PATTERN_STD_VECTOR in raw_type:
             raw_type = re.sub(".*std::vector<(.+)>.*", "\\1", raw_type)
+        
+        # Handle one_or_many_property<T> - extract the inner type T
+        # This is essential for Redpanda's flexible configuration properties
+        # that can accept either single values or arrays
+        # Check and extract from raw_type for consistency with other type extractors
+        if self.ARRAY_PATTERN_ONE_OR_MANY in raw_type:
+            raw_type = re.sub(".*one_or_many_property<(.+)>.*", "\\1", raw_type)
+            raw_type = raw_type.split()[0].replace(",", "")
 
         return raw_type
 
@@ -282,6 +391,10 @@ class FriendlyDefaultTransformer:
       - std::chrono::milliseconds(10)
       - std::nullopt
     """
+    
+    # Class-level constants for pattern matching in default values
+    ARRAY_PATTERN_STD_VECTOR = "std::vector"
+    
     def accepts(self, info, file_pair):
         return info.get("params") and len(info["params"]) > 3
 
@@ -308,7 +421,7 @@ class FriendlyDefaultTransformer:
                 return property
 
         # Transform std::vector defaults.
-        if "std::vector" in default:
+        if self.ARRAY_PATTERN_STD_VECTOR in default:
             m = re.search(r'\{([^}]+)\}', default)
             if m:
                 contents = m.group(1).strip()
@@ -351,13 +464,65 @@ class AliasTransformer:
 
 
 class EnterpriseTransformer:
+    """
+    Transforms enterprise property values from C++ expressions to user-friendly JSON.
+
+    This transformer processes enterprise values by delegating to the
+    centralized process_enterprise_value function which handles the full range of
+    C++ expression types found in enterprise property definitions.
+    """
     def accepts(self, info, file_pair):
+        """
+        Return True if the provided info indicates an enterprise-only property.
+        
+        Parameters:
+            info (dict): The metadata dictionary for a property. This function checks for a 'type' key whose string contains 'enterprise'.
+            file_pair: Unused; present for transformer interface compatibility.
+        
+        Returns:
+            bool: True when info contains a 'type' that includes 'enterprise', otherwise False.
+        """
         return bool(info.get('type') and 'enterprise' in info['type'])
 
     def parse(self, property, info, file_pair):
+        """
+        Mark a property as enterprise-only and attach its enterprise value.
+        
+        If an enterprise value is present in info['params'][0]['value'], this method attempts to process it using the shared
+        process_enterprise_value helper (loaded via get_process_enterprise_value()). If the processor is unavailable or raises
+        an exception, the raw enterprise value is used.
+        
+        Side effects:
+        - Sets property["enterprise_value"] to the processed or raw value.
+        - Sets property["is_enterprise"] = True.
+        - Removes the first element from info['params'].
+        
+        Parameters:
+            property (dict): Property bag to modify and return.
+            info (dict): Parsed metadata; must have a non-None 'params' list for processing.
+            file_pair: Unused here but accepted for transformer API compatibility.
+        
+        Returns:
+            dict: The updated property bag.
+        """
         if info['params'] is not None:
             enterpriseValue = info['params'][0]['value']
-            property['enterprise_value'] = enterpriseValue
+            
+            # Get the processing function
+            process_enterprise_value = get_process_enterprise_value()
+            if process_enterprise_value is None:
+                property["enterprise_value"] = enterpriseValue
+                property['is_enterprise'] = True
+                del info['params'][0]
+                return property
+            
+            try:
+                processed_value = process_enterprise_value(enterpriseValue)
+                property["enterprise_value"] = processed_value
+            except Exception:
+                # Fallback to raw value if processing fails
+                property["enterprise_value"] = enterpriseValue
+
             property['is_enterprise'] = True
             del info['params'][0]
             return property
@@ -386,7 +551,14 @@ class MetaParamTransformer:
         iterable_params = info['params']
         for param in iterable_params:
             if isinstance(param['value'], str) and param['value'].startswith("meta{"):
-                meta_content = param['value'].strip("meta{ }").strip()
+                # Extract content between meta{ and } using explicit slicing
+                param_value = param['value']
+                if param_value.endswith('}'):
+                    meta_content = param_value[5:-1].strip()  # Remove "meta{" and "}"
+                else:
+                    # Handle malformed meta{ without closing }
+                    meta_content = param_value[5:].strip()  # Remove "meta{" only
+                
                 meta_dict = {}
                 for item in meta_content.split(','):
                     item = item.strip()