@redpanda-data/docs-extensions-and-macros 4.8.0 → 4.8.1

package/tools/property-extractor/generate-handlebars-docs.js

@@ -0,0 +1,344 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const handlebars = require('handlebars');
+const helpers = require('./helpers');
+
+// Register all helpers
+Object.entries(helpers).forEach(([name, fn]) => {
+  if (typeof fn !== 'function') {
+    console.error(`❌ Helper "${name}" is not a function`);
+    process.exit(1);
+  }
+  handlebars.registerHelper(name, fn);
+});
+
+/**
+ * Configuration mapping for different property types
+ */
+const PROPERTY_CONFIG = {
+  broker: {
+    pageTitle: 'Broker Configuration Properties',
+    pageAliases: ['reference:node-properties.adoc', 'reference:node-configuration-sample.adoc'],
+    description: 'Reference of broker configuration properties.',
+    intro: `Broker configuration properties are applied individually to each broker in a cluster. You can find and modify these properties in the \`redpanda.yaml\` configuration file.
+
+For information on how to edit broker properties, see xref:manage:cluster-maintenance/node-property-configuration.adoc[].
+
+NOTE: All broker properties require that you restart Redpanda for any update to take effect.`,
+    sectionTitle: 'Broker configuration',
+    groups: [
+      {
+        filter: (prop) => prop.config_scope === 'broker' && !prop.is_deprecated
+      }
+    ],
+    filename: 'broker-properties.adoc'
+  },
+  cluster: {
+    pageTitle: 'Cluster Configuration Properties',
+    pageAliases: ['reference:tunable-properties.adoc', 'reference:cluster-properties.adoc'],
+    description: 'Cluster configuration properties list.',
+    intro: `Cluster configuration properties are the same for all brokers in a cluster, and are set at the cluster level.
+
+For information on how to edit cluster properties, see xref:manage:cluster-maintenance/cluster-property-configuration.adoc[] or xref:manage:kubernetes/k-cluster-property-configuration.adoc[].
+
+NOTE: Some cluster properties require that you restart the cluster for any updates to take effect. See the specific property details to identify whether or not a restart is required.`,
+    sectionTitle: 'Cluster configuration',
+    groups: [
+      {
+        filter: (prop) => prop.config_scope === 'cluster' && !prop.is_deprecated
+      }
+    ],
+    filename: 'cluster-properties.adoc'
+  },
+  'object-storage': {
+    pageTitle: 'Object Storage Properties',
+    description: 'Reference of object storage properties.',
+    intro: `Object storage properties are a type of cluster property. For information on how to edit cluster properties, see xref:manage:cluster-maintenance/cluster-property-configuration.adoc[].
+
+NOTE: Some object storage properties require that you restart the cluster for any updates to take effect. See the specific property details to identify whether or not a restart is required.`,
+    sectionTitle: 'Object storage configuration',
+    sectionIntro: 'Object storage properties should only be set if you enable xref:manage:tiered-storage.adoc[Tiered Storage].',
+    groups: [
+      {
+        filter: (prop) => prop.name && (
+          prop.name.includes('cloud_storage') ||
+          prop.name.includes('s3_') ||
+          prop.name.includes('azure_') ||
+          prop.name.includes('gcs_') ||
+          prop.name.includes('archival_') ||
+          prop.name.includes('remote_') ||
+          prop.name.includes('tiered_')
+        ) && !prop.is_deprecated
+      }
+    ],
+    filename: 'object-storage-properties.adoc'
+  },
+  topic: {
+    pageTitle: 'Topic Configuration Properties',
+    pageAliases: ['reference:topic-properties.adoc'],
+    description: 'Reference of topic configuration properties.',
+    intro: `A topic-level property sets a Redpanda or Kafka configuration for a particular topic.
+
+Many topic-level properties have corresponding xref:manage:cluster-maintenance/cluster-property-configuration.adoc[cluster properties] that set a default value for all topics of a cluster. To customize the value for a topic, you can set a topic-level property that overrides the value of the corresponding cluster property.
+
+NOTE: All topic properties take effect immediately after being set.`,
+    sectionTitle: 'Topic configuration',
+    groups: [
+      {
+        filter: (prop) => prop.config_scope === 'topic' && !prop.is_deprecated,
+        template: 'topic-property'
+      }
+    ],
+    filename: 'topic-properties.adoc'
+  }
+};
+
+// "src/v/kafka/server/handlers/topics/types.cc": "topic"
+
+/**
+ * Gets template path, checking environment variables for custom paths first
+ */
+function getTemplatePath(defaultPath, envVar) {
+  const customPath = process.env[envVar];
+  if (customPath && fs.existsSync(customPath)) {
+    console.log(`📄 Using custom template: ${customPath}`);
+    return customPath;
+  }
+  return defaultPath;
+}
+
+/**
+ * Registers Handlebars partials from template files
+ */
+function registerPartials() {
+  const templatesDir = path.join(__dirname, 'templates');
+  
+  // Register property partial
+  const propertyTemplatePath = getTemplatePath(
+    path.join(templatesDir, 'property.hbs'),
+    'TEMPLATE_PROPERTY'
+  );
+  const propertyTemplate = fs.readFileSync(propertyTemplatePath, 'utf8');
+  handlebars.registerPartial('property', propertyTemplate);
+  
+  // Register topic property partial
+  const topicPropertyTemplatePath = getTemplatePath(
+    path.join(templatesDir, 'topic-property.hbs'),
+    'TEMPLATE_TOPIC_PROPERTY'
+  );
+  const topicPropertyTemplate = fs.readFileSync(topicPropertyTemplatePath, 'utf8');
+  handlebars.registerPartial('topic-property', topicPropertyTemplate);
+  
+  // Register deprecated property partial
+  const deprecatedPropertyTemplatePath = getTemplatePath(
+    path.join(templatesDir, 'deprecated-property.hbs'),
+    'TEMPLATE_DEPRECATED_PROPERTY'
+  );
+  const deprecatedPropertyTemplate = fs.readFileSync(deprecatedPropertyTemplatePath, 'utf8');
+  handlebars.registerPartial('deprecated-property', deprecatedPropertyTemplate);
+}
+
+/**
+ * Generates documentation for a specific property type
+ */
+function generatePropertyDocs(properties, config, outputDir) {
+  const templatePath = getTemplatePath(
+    path.join(__dirname, 'templates', 'property-page.hbs'),
+    'TEMPLATE_PROPERTY_PAGE'
+  );
+  const template = handlebars.compile(fs.readFileSync(templatePath, 'utf8'));
+
+  // Filter and group properties according to configuration
+  const groups = config.groups.map(group => {
+    const filteredProperties = Object.values(properties)
+      .filter(prop => group.filter(prop))
+      .sort((a, b) => String(a.name || '').localeCompare(String(b.name || '')));
+
+    return {
+      title: group.title,
+      intro: group.intro,
+      properties: filteredProperties,
+      template: group.template || 'property' // Default to 'property' template
+    };
+  }).filter(group => group.properties.length > 0);
+
+  const data = {
+    ...config,
+    groups
+  };
+
+  const output = template(data);
+  const outputPath = path.join(outputDir, config.filename);
+  
+  fs.mkdirSync(path.dirname(outputPath), { recursive: true });
+  fs.writeFileSync(outputPath, output, 'utf8');
+  
+  console.log(`✅ Generated ${outputPath}`);
+  return groups.reduce((total, group) => total + group.properties.length, 0);
+}
+
+/**
+ * Generates deprecated properties documentation
+ */
+function generateDeprecatedDocs(properties, outputDir) {
+  const templatePath = getTemplatePath(
+    path.join(__dirname, 'templates', 'deprecated-properties.hbs'),
+    'TEMPLATE_DEPRECATED'
+  );
+  const template = handlebars.compile(fs.readFileSync(templatePath, 'utf8'));
+
+  const deprecatedProperties = Object.values(properties).filter(prop => prop.is_deprecated);
+  
+  const brokerProperties = deprecatedProperties
+    .filter(prop => prop.config_scope === 'broker')
+    .sort((a, b) => String(a.name || '').localeCompare(String(b.name || '')));
+    
+  const clusterProperties = deprecatedProperties
+    .filter(prop => prop.config_scope === 'cluster')
+    .sort((a, b) => String(a.name || '').localeCompare(String(b.name || '')));
+
+  const data = {
+    deprecated: deprecatedProperties.length > 0,
+    brokerProperties: brokerProperties.length > 0 ? brokerProperties : null,
+    clusterProperties: clusterProperties.length > 0 ? clusterProperties : null
+  };
+
+  const output = template(data);
+  const outputPath = path.join(outputDir, 'deprecated', 'partials', 'deprecated-properties.adoc');
+  
+  fs.mkdirSync(path.dirname(outputPath), { recursive: true });
+  fs.writeFileSync(outputPath, output, 'utf8');
+  
+  console.log(`✅ Generated ${outputPath}`);
+  return deprecatedProperties.length;
+}
+
+/**
+ * Main function to generate all property documentation
+ */
+function generateAllDocs(inputFile, outputDir) {
+  // Register partials
+  registerPartials();
+
+  // Read input JSON
+  const data = JSON.parse(fs.readFileSync(inputFile, 'utf8'));
+  const properties = data.properties || {};
+
+  let totalProperties = 0;
+  let totalBrokerProperties = 0;
+  let totalClusterProperties = 0;
+  let totalObjectStorageProperties = 0;
+  let totalTopicProperties = 0;
+
+  // Generate each type of documentation
+  for (const [type, config] of Object.entries(PROPERTY_CONFIG)) {
+    const count = generatePropertyDocs(properties, config, path.join(outputDir, 'pages'));
+    totalProperties += count;
+    
+    if (type === 'broker') totalBrokerProperties = count;
+    else if (type === 'cluster') totalClusterProperties = count;
+    else if (type === 'object-storage') totalObjectStorageProperties = count;
+    else if (type === 'topic') totalTopicProperties = count;
+  }
+
+  // Generate deprecated properties documentation
+  const deprecatedCount = generateDeprecatedDocs(properties, path.join(outputDir, 'pages'));
+
+  // Generate summary file
+  const allPropertiesContent = Object.keys(properties).sort().join('\n');
+  fs.writeFileSync(path.join(outputDir, 'all_properties.txt'), allPropertiesContent, 'utf8');
+
+  // Generate error reports
+  generateErrorReports(properties, outputDir);
+
+  console.log(`📊 Generation Summary:`);
+  console.log(`   Total properties read: ${Object.keys(properties).length}`);
+  console.log(`   Total Broker properties: ${totalBrokerProperties}`);
+  console.log(`   Total Cluster properties: ${totalClusterProperties}`);
+  console.log(`   Total Object Storage properties: ${totalObjectStorageProperties}`);
+  console.log(`   Total Topic properties: ${totalTopicProperties}`);
+  console.log(`   Total Deprecated properties: ${deprecatedCount}`);
+
+  return {
+    totalProperties: Object.keys(properties).length,
+    brokerProperties: totalBrokerProperties,
+    clusterProperties: totalClusterProperties,
+    objectStorageProperties: totalObjectStorageProperties,
+    topicProperties: totalTopicProperties,
+    deprecatedProperties: deprecatedCount
+  };
+}
+
+/**
+ * Generate error reports for properties with missing or invalid data
+ */
+function generateErrorReports(properties, outputDir) {
+  const errorDir = path.join(outputDir, 'error');
+  fs.mkdirSync(errorDir, { recursive: true });
+
+  const emptyDescriptions = [];
+  const deprecatedProperties = [];
+
+  Object.values(properties).forEach(prop => {
+    if (!prop.description || prop.description.trim() === '') {
+      emptyDescriptions.push(prop.name);
+    }
+    if (prop.is_deprecated) {
+      deprecatedProperties.push(prop.name);
+    }
+  });
+
+  // Write error reports
+  if (emptyDescriptions.length > 0) {
+    fs.writeFileSync(
+      path.join(errorDir, 'empty_description.txt'),
+      emptyDescriptions.join('\n'),
+      'utf8'
+    );
+    const percentage = ((emptyDescriptions.length / Object.keys(properties).length) * 100).toFixed(2);
+    console.log(`You have ${emptyDescriptions.length} properties with empty description. Percentage of errors: ${percentage}%. Data written in 'empty_description.txt'.`);
+  }
+
+  if (deprecatedProperties.length > 0) {
+    fs.writeFileSync(
+      path.join(errorDir, 'deprecated_properties.txt'),
+      deprecatedProperties.join('\n'),
+      'utf8'
+    );
+    const percentage = ((deprecatedProperties.length / Object.keys(properties).length) * 100).toFixed(2);
+    console.log(`You have ${deprecatedProperties.length} deprecated properties. Percentage of errors: ${percentage}%. Data written in 'deprecated_properties.txt'.`);
+  }
+}
+
+module.exports = {
+  generateAllDocs,
+  generatePropertyDocs,
+  generateDeprecatedDocs,
+  PROPERTY_CONFIG
+};
+
+// CLI interface
+if (require.main === module) {
+  const args = process.argv.slice(2);
+  if (args.length < 2) {
+    console.error('Usage: node generate-handlebars-docs.js <input-file> <output-dir>');
+    process.exit(1);
+  }
+
+  const [inputFile, outputDir] = args;
+  
+  if (!fs.existsSync(inputFile)) {
+    console.error(`❌ Input file not found: ${inputFile}`);
+    process.exit(1);
+  }
+
+  try {
+    generateAllDocs(inputFile, outputDir);
+    console.log('✅ Documentation generation completed successfully');
+  } catch (error) {
+    console.error(`❌ Error generating documentation: ${error.message}`);
+    process.exit(1);
+  }
+}

package/tools/property-extractor/helpers/and.js

@@ -0,0 +1,10 @@
+/**
+ * Handlebars helper for logical AND
+ * @param {...*} args - Values to check
+ * @returns {boolean} True if all values are truthy per JavaScript semantics
+ */
+module.exports = function and(...args) {
+  // Remove the last argument which is the Handlebars options object
+  const values = args.slice(0, -1);
+  return values.every(val => Boolean(val));
+};

package/tools/property-extractor/helpers/eq.js

@@ -0,0 +1,9 @@
+/**
+ * Handlebars helper for equality comparison
+ * @param {*} a - First value
+ * @param {*} b - Second value
+ * @returns {boolean} True if values are equal
+ */
+module.exports = function eq(a, b) {
+  return a === b;
+};

package/tools/property-extractor/helpers/formatPropertyValue.js

@@ -0,0 +1,128 @@
+const handlebars = require('handlebars');
+
+/**
+ * Helper function to format a value (used in object/array formatting)
+ */
+function formatValue(val) {
+  if (typeof val === 'string') {
+    return `"${val}"`;
+  } else if (typeof val === 'boolean') {
+    return val ? 'true' : 'false';
+  } else if (val === null || val === undefined) {
+    return 'null';
+  } else if (Array.isArray(val)) {
+    return "[" + val.map(v => formatValue(v)).join(", ") + "]";
+  } else if (typeof val === 'object' && val !== null) {
+    return JSON.stringify(val);
+  } else {
+    return String(val);
+  }
+}
+
+/**
+ * Process C++ internal representations and convert them to user-friendly formats
+ * This matches the Python process_defaults function logic
+ */
+function processDefaults(inputString, suffix) {
+  if (typeof inputString !== 'string') {
+    return inputString;
+  }
+
+  // Test for ip:port in vector: std::vector<net::unresolved_address>({{...}})
+  const vectorMatch = inputString.match(/std::vector<net::unresolved_address>\(\{\{("([\d.]+)",\s*(\d+))\}\}\)/);
+  if (vectorMatch) {
+    const ip = vectorMatch[2];
+    const port = vectorMatch[3];
+    return [`${ip}:${port}`];
+  }
+
+  // Test for ip:port in single-string: net::unresolved_address("127.0.0.1", 9092)
+  const brokerMatch = inputString.match(/net::unresolved_address\("([\d.]+)",\s*(\d+)\)/);
+  if (brokerMatch) {
+    const ip = brokerMatch[1];
+    const port = brokerMatch[2];
+    return `${ip}:${port}`;
+  }
+
+  // Handle std::nullopt
+  if (inputString.includes('std::nullopt')) {
+    return inputString.replace(/std::nullopt/g, 'null');
+  }
+
+  // Handle time units and other patterns would go here...
+  // For now, return the original string
+  return inputString;
+}
+
+/**
+ * Formats a property value for display, matching Python legacy format exactly
+ * @param {*} value - The value to format
+ * @param {string} type - The property type
+ * @returns {handlebars.SafeString} Formatted value
+ */
+module.exports = function formatPropertyValue(value, type) {
+  if (value === null || value === undefined || value === '') {
+    return new handlebars.SafeString('null');
+  }
+
+  if (typeof value === 'boolean') {
+    return new handlebars.SafeString(value ? 'true' : 'false');
+  }
+
+  if (typeof value === 'object' && !Array.isArray(value)) {
+    // Format object defaults with Python-style syntax: {key: "value", key2: value2}
+    const pairs = [];
+    for (const [k, v] of Object.entries(value)) {
+      // Process each value for C++ representations
+      let processedValue = v;
+      if (typeof v === 'string') {
+        processedValue = processDefaults(v, null);
+      }
+      pairs.push(`${k}: ${formatValue(processedValue)}`);
+    }
+    return new handlebars.SafeString(`{${pairs.join(', ')}}`);
+  }
+
+  if (Array.isArray(value)) {
+    // Handle array defaults to match Python format
+    if (value.length === 0) {
+      return new handlebars.SafeString('[]');
+    } else {
+      // Format each array element
+      const formattedElements = [];
+      for (const item of value) {
+        if (typeof item === 'object' && item !== null) {
+          // Format object within array
+          const pairs = [];
+          for (const [k, v] of Object.entries(item)) {
+            // Process each value for C++ representations
+            let processedValue = v;
+            if (typeof v === 'string') {
+              processedValue = processDefaults(v, null);
+            }
+            pairs.push(`${k}: ${formatValue(processedValue)}`);
+          }
+          formattedElements.push(`{${pairs.join(', ')}}`);
+        } else {
+          formattedElements.push(String(item));
+        }
+      }
+      return new handlebars.SafeString(`[${formattedElements.join(', ')}]`);
+    }
+  }
+
+  // For other types, handle strings vs non-strings differently
+  let result;
+  if (typeof value === 'string') {
+    // Keep strings as-is (preserve original characters and casing)
+    result = value;
+  } else {
+    // Convert non-string values to String without altering case/quotes
+    result = String(value);
+  }
+  
+  // Apply C++ processing
+  result = processDefaults(result, null);
+  
+  return new handlebars.SafeString(result);
+};

package/tools/property-extractor/helpers/formatUnits.js

@@ -0,0 +1,26 @@
+/**
+ * Formats units for display based on property name suffix
+ * @param {string} name - Property name that might contain unit suffixes
+ * @returns {string} Formatted unit description
+ */
+module.exports = function formatUnits(name) {
+  const suffixToUnit = {
+    'ms': 'milliseconds',
+    'sec': 'seconds',
+    'seconds': 'seconds',
+    'bytes': 'bytes',
+    'buf': 'bytes',
+    'partitions': 'number of partitions per topic',
+    'percent': 'percent',
+    'bps': 'bytes per second',
+    'fraction': 'fraction'
+  };
+
+  if (!name) return '';
+
+  // Extract the last part after splitting on underscores (like Python implementation)
+  const parts = name.split('_');
+  const suffix = parts[parts.length - 1];
+  
+  return suffixToUnit[suffix] || '';
+};

package/tools/property-extractor/helpers/index.js

@@ -0,0 +1,13 @@
+'use strict';
+
+module.exports = {
+  join: require('./join.js'),
+  eq: require('./eq.js'),
+  ne: require('./ne.js'),
+  and: require('./and.js'),
+  or: require('./or.js'),
+  not: require('./not.js'),
+  formatPropertyValue: require('./formatPropertyValue.js'),
+  renderPropertyExample: require('./renderPropertyExample.js'),
+  formatUnits: require('./formatUnits.js'),
+};

package/tools/property-extractor/helpers/join.js

@@ -0,0 +1,18 @@
+/**
+ * Handlebars helper to join an array with a separator
+ * @param {Array} array - The array to join
+ * @param {string} separator - The separator to use
+ * @returns {string} The joined string
+ */
+module.exports = function join(array, separator) {
+  if (!Array.isArray(array)) {
+    return '';
+  }
+  
+  // Detect and ignore Handlebars options object
+  if (separator && typeof separator === 'object' && (separator.hash !== undefined || separator.data !== undefined)) {
+    separator = undefined;
+  }
+  
+  return array.join(separator || ', ');
+};

package/tools/property-extractor/helpers/ne.js

@@ -0,0 +1,9 @@
+/**
+ * Handlebars helper for inequality comparison
+ * @param {*} a - First value
+ * @param {*} b - Second value
+ * @returns {boolean} True if values are not equal
+ */
+module.exports = function ne(a, b) {
+  return a !== b;
+};

package/tools/property-extractor/helpers/not.js

@@ -0,0 +1,8 @@
+/**
+ * Handlebars helper for logical NOT
+ * @param {*} value - Value to negate
+ * @returns {boolean} True if value is falsy
+ */
+module.exports = function not(value) {
+  return !value;
+};

package/tools/property-extractor/helpers/or.js

@@ -0,0 +1,10 @@
+/**
+ * Handlebars helper for logical OR
+ * @param {...*} args - Values to check
+ * @returns {boolean} True if any value is truthy
+ */
+module.exports = function or(...args) {
+  // Remove the last argument which is the Handlebars options object
+  const values = args.slice(0, -1);
+  return values.some(val => !!val);
+};

package/tools/property-extractor/helpers/renderPropertyExample.js

@@ -0,0 +1,42 @@
+const handlebars = require('handlebars');
+
+/**
+ * Renders an example for a property based on its format
+ * @param {Object} property - The property object containing example data
+ * @returns {handlebars.SafeString} Formatted example block
+ */
+module.exports = function renderPropertyExample(property) {
+  if (!property.example) {
+    return new handlebars.SafeString('');
+  }
+
+  let exampleContent = '';
+
+  // Handle different example formats
+  if (typeof property.example === 'string') {
+    // Check if it's already a complete AsciiDoc example
+    if (property.example.includes('.Example') || property.example.includes('[,yaml]')) {
+      exampleContent = property.example;
+    } else {
+      // Simple string example - wrap it
+      exampleContent = `.Example\n[,yaml]\n----\n${property.name}: ${property.example}\n----`;
+    }
+  } else if (Array.isArray(property.example)) {
+    // Multiline array example
+    exampleContent = property.example.join('\n');
+  } else if (typeof property.example === 'object' && property.example.title) {
+    // Structured example with title and content
+    exampleContent = `.${property.example.title}\n`;
+    if (property.example.description) {
+      exampleContent += `${property.example.description}\n\n`;
+    }
+    if (property.example.config) {
+      exampleContent += `[,yaml]\n----\n${JSON.stringify(property.example.config, null, 2)}\n----`;
+    }
+  } else {
+    // Fallback: JSON stringify the example
+    exampleContent = `.Example\n[,yaml]\n----\n${property.name}: ${JSON.stringify(property.example, null, 2)}\n----`;
+  }
+
+  return new handlebars.SafeString('\n' + exampleContent + '\n');
+};