@redpanda-data/docs-extensions-and-macros 4.8.0 → 4.8.1

package/bin/doc-tools.js

@@ -98,7 +98,7 @@ function requireCmd(cmd, help, versionFlag = '--version') {
  * @param {number} [minMinor=10] - Minimum required minor version of Python.
  */
 function requirePython(minMajor = 3, minMinor = 10) {
-  const candidates = ['python3', 'python'];
+  const candidates = ['python3', 'python', 'python3.12', 'python3.11', 'python3.10'];
   for (const p of candidates) {
     try {
       const out = execSync(`${p} --version`, { encoding: 'utf8' }).trim();
@@ -234,12 +234,16 @@ For more details, visit: https://github.com/norwoodj/helm-docs
 /**
  * Ensures all dependencies required for generating property documentation are installed.
  *
- * Checks for the presence of `make`, Python 3.10 or newer, C++ compiler, and C++ standard library headers.
+ * Checks for the presence of `make`, Python 3.10 or newer, Node.js, C++ compiler, and C++ standard library headers.
  * Exits the process with an error message if any dependency is missing.
  */
 function verifyPropertyDependencies() {
   requireCmd('make', 'Your OS package manager');
   requirePython();
+  
+  // Check for Node.js (required for Handlebars templates)
+  requireCmd('node', 'https://nodejs.org/en/download/ or use your package manager (e.g., brew install node)');
+  requireCmd('npm', 'Usually installed with Node.js');
 
   // Check for C++ compiler
   let cppCompiler = null;
@@ -463,25 +467,23 @@ function runClusterDocs(mode, tag, options) {
   if (r.status !== 0) process.exit(r.status);
 }
 
-// helper to diff two autogenerated directories
-function diffDirs(kind, oldTag, newTag) {
-  const oldDir = path.join('autogenerated', oldTag, kind);
-  const newDir = path.join('autogenerated', newTag, kind);
-  const diffDir = path.join('autogenerated', 'diffs', kind, `${oldTag}_to_${newTag}`);
+// helper to diff two temporary directories
+function diffDirs(kind, oldTag, newTag, oldTempDir, newTempDir) {
+  const diffDir = path.join('tmp', 'diffs', kind, `${oldTag}_to_${newTag}`);
   const patch = path.join(diffDir, 'changes.patch');
 
-  if (!fs.existsSync(oldDir)) {
-    console.error(`❌ Cannot diff: missing ${oldDir}`);
+  if (!fs.existsSync(oldTempDir)) {
+    console.error(`❌ Cannot diff: missing ${oldTempDir}`);
     process.exit(1);
   }
-  if (!fs.existsSync(newDir)) {
-    console.error(`❌ Cannot diff: missing ${newDir}`);
+  if (!fs.existsSync(newTempDir)) {
+    console.error(`❌ Cannot diff: missing ${newTempDir}`);
     process.exit(1);
   }
 
   fs.mkdirSync(diffDir, { recursive: true });
 
-  const cmd = `diff -ru "${oldDir}" "${newDir}" > "${patch}" || true`;
+  const cmd = `diff -ru "${oldTempDir}" "${newTempDir}" > "${patch}" || true`;
   const res = spawnSync(cmd, { stdio: 'inherit', shell: true });
 
   if (res.error) {
@@ -489,6 +491,10 @@ function diffDirs(kind, oldTag, newTag) {
     process.exit(1);
   }
   console.log(`✅ Wrote patch: ${patch}`);
+  
+  // Clean up temporary directories
+  fs.rmSync(oldTempDir, { recursive: true, force: true });
+  fs.rmSync(newTempDir, { recursive: true, force: true });
 }
 
 automation
@@ -747,15 +753,56 @@ automation
   .description('Generate JSON and AsciiDoc documentation for Redpanda configuration properties')
   .option('--tag <tag>', 'Git tag or branch to extract from', 'dev')
   .option('--diff <oldTag>', 'Also diff autogenerated properties from <oldTag> → <tag>')
+  .option('--overrides <path>', 'Optional JSON file with property description overrides')
+  .option('--output-dir <dir>', 'Where to write all generated files', 'modules/reference')
+  .option('--template-property-page <path>', 'Custom Handlebars template for property page layout')
+  .option('--template-property <path>', 'Custom Handlebars template for individual property sections')
+  .option('--template-deprecated <path>', 'Custom Handlebars template for deprecated properties page')
+  .option('--template-deprecated-property <path>', 'Custom Handlebars template for individual deprecated property sections')
   .action((options) => {
     verifyPropertyDependencies();
 
     const newTag = options.tag;
     const oldTag = options.diff;
+    const overridesPath = options.overrides;
+    const outputDir = options.outputDir;
     const cwd = path.resolve(__dirname, '../tools/property-extractor');
-    const make = (tag) => {
-      console.log(`⏳ Building property docs for ${tag}…`);
-      const r = spawnSync('make', ['build', `TAG=${tag}`], { cwd, stdio: 'inherit' });
+    
+    const make = (tag, overrides, templates = {}, outputDir = 'modules/reference/', tempDir = null) => {
+      console.log(`⏳ Building property docs for ${tag}${tempDir ? ' (for diff)' : ''}…`);
+      const args = ['build', `TAG=${tag}`];
+      
+      // Pass all paths as environment variables for consistency
+      const env = { ...process.env };
+      if (overrides) {
+        env.OVERRIDES = path.resolve(overrides);
+      }
+      if (templates.propertyPage) {
+        env.TEMPLATE_PROPERTY_PAGE = path.resolve(templates.propertyPage);
+      }
+      if (templates.property) {
+        env.TEMPLATE_PROPERTY = path.resolve(templates.property);
+      }
+      if (templates.deprecated) {
+        env.TEMPLATE_DEPRECATED = path.resolve(templates.deprecated);
+      }
+      if (templates.deprecatedProperty) {
+        env.TEMPLATE_DEPRECATED_PROPERTY = path.resolve(templates.deprecatedProperty);
+      }
+      
+      if (tempDir) {
+        // For diff purposes, generate to temporary directory
+        env.OUTPUT_ASCIIDOC_DIR = path.resolve(tempDir);
+        env.OUTPUT_JSON_DIR = path.resolve(tempDir, 'examples');
+        env.OUTPUT_AUTOGENERATED_DIR = path.resolve(tempDir);
+      } else {
+        // Normal generation - go directly to final destination
+        // Let Makefile calculate OUTPUT_ASCIIDOC_DIR as OUTPUT_AUTOGENERATED_DIR/pages
+        env.OUTPUT_JSON_DIR = path.resolve(outputDir, 'examples');
+        env.OUTPUT_AUTOGENERATED_DIR = path.resolve(outputDir);
+      }
+      
+      const r = spawnSync('make', args, { cwd, stdio: 'inherit', env });
       if (r.error) {
         console.error(`❌ ${r.error.message}`);
         process.exit(1);
@@ -763,50 +810,38 @@ automation
       if (r.status !== 0) process.exit(r.status);
     };
 
-    if (oldTag) {
-      const oldDir = path.join('autogenerated', oldTag, 'properties');
-      if (!fs.existsSync(oldDir)) make(oldTag);
-    }
-
-    make(newTag);
-
-    if (oldTag) {
-      diffDirs('properties', oldTag, newTag);
-    }
-
-    process.exit(0);
-  });
-
-automation
-  .command('topic-property-docs')
-  .description('Generate JSON and AsciiDoc documentation for Redpanda topic configuration properties')
-  .option('--tag <tag>', 'Git tag or branch to extract from', 'dev')
-  .option('--diff <oldTag>', 'Also diff autogenerated topic properties from <oldTag> → <tag>')
-  .action((options) => {
-    verifyPropertyDependencies();
-
-    const newTag = options.tag;
-    const oldTag = options.diff;
-    const cwd = path.resolve(__dirname, '../tools/property-extractor');
-    const make = (tag) => {
-      console.log(`⏳ Building topic property docs for ${tag}…`);
-      const r = spawnSync('make', ['topic-properties', `TAG=${tag}`], { cwd, stdio: 'inherit' });
-      if (r.error) {
-        console.error(`❌ ${r.error.message}`);
-        process.exit(1);
-      }
-      if (r.status !== 0) process.exit(r.status);
+    // Collect template options
+    const templates = {
+      propertyPage: options.templatePropertyPage,
+      property: options.templateProperty,
+      deprecated: options.templateDeprecated,
+      deprecatedProperty: options.templateDeprecatedProperty
     };
 
+    let oldTempDir = null;
+    let newTempDir = null;
+
     if (oldTag) {
-      const oldDir = path.join('autogenerated', oldTag, 'properties');
-      if (!fs.existsSync(oldDir)) make(oldTag);
+      // Generate old version to temporary directory for diff
+      oldTempDir = path.join('tmp', 'diff', `${oldTag}-properties`);
+      fs.mkdirSync(oldTempDir, { recursive: true });
+      make(oldTag, overridesPath, templates, outputDir, oldTempDir);
     }
 
-    make(newTag);
-
     if (oldTag) {
-      diffDirs('properties', oldTag, newTag);
+      // Generate new version to temporary directory for diff
+      newTempDir = path.join('tmp', 'diff', `${newTag}-properties`);
+      fs.mkdirSync(newTempDir, { recursive: true });
+      make(newTag, overridesPath, templates, outputDir, newTempDir);
+      
+      // Then generate new version to final destination
+      make(newTag, overridesPath, templates, outputDir);
+      
+      // Compare the temporary directories
+      diffDirs('properties', oldTag, newTag, oldTempDir, newTempDir);
+    } else {
+      // No diff requested, just generate to final destination
+      make(newTag, overridesPath, templates, outputDir);
     }
 
     process.exit(0);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redpanda-data/docs-extensions-and-macros",
-  "version": "4.8.0",
+  "version": "4.8.1",
   "description": "Antora extensions and macros developed for Redpanda documentation.",
   "keywords": [
     "antora",

package/tools/property-extractor/Makefile

@@ -1,4 +1,4 @@
-.PHONY: build venv clean redpanda-git treesitter topic-properties generate-docs check
+.PHONY: build venv clean redpanda-git treesitter generate-docs check
 
 # --- Main build: venv, fetch code, build parser, extract & docgen ---
 build: venv redpanda-git treesitter
@@ -18,11 +18,11 @@ TAG       ?= dev
 # Derive a “major.minor” or rc identifier from TAG for folder naming
 VERSION := $(shell \
   if echo "$(TAG)" | grep -qE '^v?[0-9]+\.[0-9]+'; then \
-    echo "$(TAG)" \
-      | sed -E 's/^v?([0-9]+\.[0-9]+)(\.[0-9]+)?(-rc[0-9]+)?.*/\1\3/' \
-      | sed 's/-rc/rc/'; \
+	echo "$(TAG)" \
+	  | sed -E 's/^v?([0-9]+\.[0-9]+)(\.[0-9]+)?(-rc[0-9]+)?.*/\1\3/' \
+	  | sed 's/-rc/rc/'; \
   else \
-    echo "$(TAG)"; \
+	echo "$(TAG)"; \
   fi)
 
 # Paths
@@ -34,9 +34,33 @@ REDPANDA_SRC  := $(TMP_ROOT)/redpanda
 TREESITTER_DIR:= $(TOOL_ROOT)/tree-sitter/tree-sitter-cpp
 VENV          := $(TOOL_ROOT)/tmp/redpanda-property-extractor-venv
 PYTHON        := $(VENV)/bin/python
-OUTPUT_DIR    := $(REPO_ROOT)/autogenerated/$(TAG)/properties
 TREE_SITTER   := npx tree-sitter
 
+# Output directory configuration (can be overridden by environment variables)
+OUTPUT_AUTOGENERATED_DIR ?= $(REPO_ROOT)/modules/reference
+OUTPUT_ASCIIDOC_DIR      ?= $(OUTPUT_AUTOGENERATED_DIR)/pages
+OUTPUT_JSON_DIR          ?= $(OUTPUT_AUTOGENERATED_DIR)/examples
+
+# --- Main build: venv, fetch code, build parser, extract & docgen ---
+build: venv redpanda-git treesitter
+	@echo "🔧 Building with Redpanda tag: $(TAG)"
+	@mkdir -p $(TOOL_ROOT)/gen
+	@if [ -n "$(OVERRIDES)" ] && [ ! -f "$(OVERRIDES)" ]; then \
+	  echo "❌ Error: Overrides file '$(OVERRIDES)' does not exist."; \
+	  echo "   Please check the path or omit the OVERRIDES variable to continue without overrides."; \
+	  exit 1; \
+	fi
+	@cd $(TOOL_ROOT) && \
+	  $(PYTHON) -W ignore::FutureWarning property_extractor.py \
+		--recursive \
+		--path $(REDPANDA_SRC) \
+		--output gen/properties-output.json \
+		--enhanced-output gen/$(TAG)-properties.json \
+		$(if $(OVERRIDES),$(if $(shell [ -f "$(OVERRIDES)" ] && echo exists),--overrides $(OVERRIDES),),)
+	@echo "✅ JSON generated at $(TOOL_ROOT)/gen/properties-output.json"
+	@echo "✅ Enhanced JSON (with overrides) generated at $(TOOL_ROOT)/gen/$(TAG)-properties.json"
+	@$(MAKE) generate-docs
+
 # --- Ensure Python venv & dependencies ---
 venv: $(TOOL_ROOT)/requirements.txt
 	@if [ ! -d "$(VENV)" ]; then \
@@ -84,34 +108,38 @@ treesitter:
 	@echo "🔧 Generating parser in $(TREESITTER_DIR)…"
 	@cd "$(TREESITTER_DIR)" && npm install --silent && $(TREE_SITTER) generate
 
-# --- Turn the JSON into AsciiDoc pages under autogen/<version>/properties ---
-generate-docs:
-	@echo "📝 Generating AsciiDoc pages in $(OUTPUT_DIR)…"
-	@mkdir -p "$(OUTPUT_DIR)"
-	@cd $(TOOL_ROOT) && \
-	  $(PYTHON) json-to-asciidoc/generate_docs.py --output-dir "$(OUTPUT_DIR)"
-	@echo "📄 Copying properties-output.json to $(OUTPUT_DIR)…"
-	@cp "$(TOOL_ROOT)/gen/properties-output.json" "$(OUTPUT_DIR)/"
-	@echo "✅ Docs generated at $(OUTPUT_DIR)"
+# --- Install Node.js dependencies for Handlebars ---
+node-deps:
+	@echo "📦 Installing Node.js dependencies…"
+	@cd $(TOOL_ROOT) && npm install --silent
+
+# --- Turn the JSON into AsciiDoc pages under the specified output directories ---
+generate-docs: node-deps
+	@echo "📝 Generating AsciiDoc pages in $(OUTPUT_ASCIIDOC_DIR)…"
+	@mkdir -p "$(OUTPUT_ASCIIDOC_DIR)"
+	@mkdir -p "$(OUTPUT_JSON_DIR)"
+	@# Use the enhanced properties file (with overrides) for documentation generation if it exists
+	@if [ -f "$(TOOL_ROOT)/gen/$(TAG)-properties.json" ]; then \
+	  cd $(TOOL_ROOT) && \
+	    node generate-handlebars-docs.js "gen/$(TAG)-properties.json" "$(OUTPUT_AUTOGENERATED_DIR)"; \
+	else \
+	  cd $(TOOL_ROOT) && \
+	    node generate-handlebars-docs.js "gen/properties-output.json" "$(OUTPUT_AUTOGENERATED_DIR)"; \
+	fi
+	@echo "📄 Copying properties JSON files to $(OUTPUT_JSON_DIR)…"
+	@if [ -f "$(TOOL_ROOT)/gen/$(TAG)-properties.json" ]; then \
+	  cp "$(TOOL_ROOT)/gen/$(TAG)-properties.json" "$(OUTPUT_JSON_DIR)/"; \
+	fi
+	@echo "✅ Docs generated at $(OUTPUT_AUTOGENERATED_DIR)"
 
 # --- Debug helper to print all the key paths/vars ---
 check:
-	@echo "MODULE_ROOT:   $(MODULE_ROOT)"
-	@echo "TOOL_ROOT:     $(TOOL_ROOT)"
-	@echo "REDPANDA_SRC:  $(REDPANDA_SRC)"
-	@echo "TREESITTER:    $(TREESITTER_DIR)"
-	@echo "VENV:          $(VENV)"
-	@echo "PYTHON:        $(PYTHON)"
-	@echo "OUTPUT_DIR:    $(OUTPUT_DIR)"
-
-# --- Extract topic properties ---
-topic-properties: venv redpanda-git treesitter
-	@echo "🔧 Extracting topic properties with Redpanda tag: $(TAG)"
-	@mkdir -p $(TOOL_ROOT)/gen
-	@mkdir -p "$(OUTPUT_DIR)"
-	@cd $(TOOL_ROOT) && \
-	  $(PYTHON) topic_property_extractor.py \
-	    --source-path $(REDPANDA_SRC) \
-	    --output-json "$(OUTPUT_DIR)/topic-properties-output.json" \
-	    --output-adoc "$(OUTPUT_DIR)/topic-properties.adoc"
-	@echo "✅ Topic properties extracted"
+	@echo "MODULE_ROOT:              $(MODULE_ROOT)"
+	@echo "TOOL_ROOT:                $(TOOL_ROOT)"
+	@echo "REDPANDA_SRC:             $(REDPANDA_SRC)"
+	@echo "TREESITTER:               $(TREESITTER_DIR)"
+	@echo "VENV:                     $(VENV)"
+	@echo "PYTHON:                   $(PYTHON)"
+	@echo "OUTPUT_ASCIIDOC_DIR:      $(OUTPUT_ASCIIDOC_DIR)"
+	@echo "OUTPUT_JSON_DIR:          $(OUTPUT_JSON_DIR)"
+	@echo "OUTPUT_AUTOGENERATED_DIR: $(OUTPUT_AUTOGENERATED_DIR)"