@redpanda-data/docs-extensions-and-macros 4.8.0 → 4.9.0

package/bin/doc-tools.js

@@ -98,7 +98,7 @@ function requireCmd(cmd, help, versionFlag = '--version') {
  * @param {number} [minMinor=10] - Minimum required minor version of Python.
  */
 function requirePython(minMajor = 3, minMinor = 10) {
-  const candidates = ['python3', 'python'];
+  const candidates = ['python3', 'python', 'python3.12', 'python3.11', 'python3.10'];
   for (const p of candidates) {
     try {
       const out = execSync(`${p} --version`, { encoding: 'utf8' }).trim();
@@ -234,12 +234,16 @@ For more details, visit: https://github.com/norwoodj/helm-docs
 /**
  * Ensures all dependencies required for generating property documentation are installed.
  *
- * Checks for the presence of `make`, Python 3.10 or newer, C++ compiler, and C++ standard library headers.
+ * Checks for the presence of `make`, Python 3.10 or newer, Node.js, C++ compiler, and C++ standard library headers.
  * Exits the process with an error message if any dependency is missing.
  */
 function verifyPropertyDependencies() {
   requireCmd('make', 'Your OS package manager');
   requirePython();
+  
+  // Check for Node.js (required for Handlebars templates)
+  requireCmd('node', 'https://nodejs.org/en/download/ or use your package manager (e.g., brew install node)');
+  requireCmd('npm', 'Usually installed with Node.js');
 
   // Check for C++ compiler
   let cppCompiler = null;
@@ -455,33 +459,135 @@ const commonOptions = {
   consoleDockerRepo: 'console',
 };
 
+/**
+ * Run the cluster documentation generator script for a specific release/tag.
+ *
+ * Invokes the external `generate-cluster-docs.sh` script with the provided mode, tag,
+ * and Docker-related options. The script's stdout/stderr are forwarded to the current
+ * process; if the script exits with a non-zero status, this function will terminate
+ * the Node.js process with that status code.
+ *
+ * @param {string} mode - Operation mode passed to the script (e.g., "generate" or "clean").
+ * @param {string} tag - Release tag or version to generate docs for.
+ * @param {Object} options - Runtime options.
+ * @param {string} options.dockerRepo - Docker repository used by the script.
+ * @param {string} options.consoleTag - Console image tag passed to the script.
+ * @param {string} options.consoleDockerRepo - Console Docker repository used by the script.
+ */
 function runClusterDocs(mode, tag, options) {
   const script = path.join(__dirname, '../cli-utils/generate-cluster-docs.sh');
   const args = [mode, tag, options.dockerRepo, options.consoleTag, options.consoleDockerRepo];
   console.log(`⏳ Running ${script} with arguments: ${args.join(' ')}`);
-  const r = spawnSync('bash', [script, ...args], { stdio: 'inherit', shell: true });
+  const r = spawnSync('bash', [script, ...args], { stdio: 'inherit' });
   if (r.status !== 0) process.exit(r.status);
 }
 
-// helper to diff two autogenerated directories
-function diffDirs(kind, oldTag, newTag) {
-  const oldDir = path.join('autogenerated', oldTag, kind);
-  const newDir = path.join('autogenerated', newTag, kind);
-  const diffDir = path.join('autogenerated', 'diffs', kind, `${oldTag}_to_${newTag}`);
-  const patch = path.join(diffDir, 'changes.patch');
+/**
+ * Generate a detailed JSON report describing property changes between two releases.
+ *
+ * Looks for `<oldTag>-properties.json` and `<newTag>-properties.json` in
+ * `modules/reference/examples`. If both files exist, invokes the external
+ * property comparison tool to produce `property-changes-<oldTag>-to-<newTag>.json`
+ * in the provided output directory.
+ *
+ * If either input JSON is missing the function logs a message and returns without
+ * error. Any errors from the comparison tool are logged; the function does not
+ * throw.
+ *
+ * @param {string} oldTag - Release tag or identifier for the "old" properties set.
+ * @param {string} newTag - Release tag or identifier for the "new" properties set.
+ * @param {string} outputDir - Directory where the comparison report will be written.
+ */
+function generatePropertyComparisonReport(oldTag, newTag, outputDir) {
+  try {
+    console.log(`\n📊 Generating detailed property comparison report...`);
+    
+    // Look for the property JSON files in modules/reference/examples
+    const repoRoot = findRepoRoot();
+    const examplesDir = path.join(repoRoot, 'modules', 'reference', 'examples');
+    const oldJsonPath = path.join(examplesDir, `${oldTag}-properties.json`);
+    const newJsonPath = path.join(examplesDir, `${newTag}-properties.json`);
+    
+    // Check if JSON files exist
+    if (!fs.existsSync(oldJsonPath)) {
+      console.log(`⚠️  Old properties JSON not found at: ${oldJsonPath}`);
+      console.log(`   Skipping detailed property comparison.`);
+      return;
+    }
+    
+    if (!fs.existsSync(newJsonPath)) {
+      console.log(`⚠️  New properties JSON not found at: ${newJsonPath}`);
+      console.log(`   Skipping detailed property comparison.`);
+      return;
+    }
+    
+    // Ensure output directory exists (use absolute path)
+    const absoluteOutputDir = path.resolve(outputDir);
+    fs.mkdirSync(absoluteOutputDir, { recursive: true });
+    
+    // Run the property comparison tool with descriptive filename
+    const propertyExtractorDir = path.resolve(__dirname, '../tools/property-extractor');
+    const compareScript = path.join(propertyExtractorDir, 'compare-properties.js');
+    const reportFilename = `property-changes-${oldTag}-to-${newTag}.json`;
+    const reportPath = path.join(absoluteOutputDir, reportFilename);
+    const args = [compareScript, oldJsonPath, newJsonPath, oldTag, newTag, absoluteOutputDir, reportFilename];
+    
+    const result = spawnSync('node', args, { 
+      stdio: 'inherit',
+      cwd: propertyExtractorDir 
+    });
+    
+    if (result.error) {
+      console.error(`❌ Property comparison failed: ${result.error.message}`);
+    } else if (result.status !== 0) {
+      console.error(`❌ Property comparison exited with code: ${result.status}`);
+    } else {
+      console.log(`✅ Property comparison report saved to: ${reportPath}`);
+    }
+  } catch (error) {
+    console.error(`❌ Error generating property comparison: ${error.message}`);
+  }
+}
+
+/**
+ * Create a unified diff patch between two temporary directories and clean them up.
+ *
+ * Ensures both source directories exist, writes a recursive unified diff
+ * (changes.patch) to tmp/diffs/<kind>/<oldTag>_to_<newTag>/, and removes the
+ * provided temporary directories. On missing inputs or if the diff subprocess
+ * fails to spawn, the process exits with a non-zero status.
+ *
+ * @param {string} kind - Logical category for the diff (e.g., "metrics" or "rpk"); used in the output path.
+ * @param {string} oldTag - Identifier for the "old" version (used in the output path).
+ * @param {string} newTag - Identifier for the "new" version (used in the output path).
+ * @param {string} oldTempDir - Path to the existing temporary directory containing the old output; must exist.
+ * @param {string} newTempDir - Path to the existing temporary directory containing the new output; must exist.
+ */
+function diffDirs(kind, oldTag, newTag, oldTempDir, newTempDir) {
+  // Backwards compatibility: if temp directories not provided, use autogenerated paths
+  if (!oldTempDir) {
+    oldTempDir = path.join('autogenerated', oldTag, kind);
+  }
+  if (!newTempDir) {
+    newTempDir = path.join('autogenerated', newTag, kind);
+  }
 
-  if (!fs.existsSync(oldDir)) {
-    console.error(`❌ Cannot diff: missing ${oldDir}`);
+  const diffDir = path.join('tmp', 'diffs', kind, `${oldTag}_to_${newTag}`);
+
+  if (!fs.existsSync(oldTempDir)) {
+    console.error(`❌ Cannot diff: missing ${oldTempDir}`);
     process.exit(1);
   }
-  if (!fs.existsSync(newDir)) {
-    console.error(`❌ Cannot diff: missing ${newDir}`);
+  if (!fs.existsSync(newTempDir)) {
+    console.error(`❌ Cannot diff: missing ${newTempDir}`);
     process.exit(1);
   }
 
   fs.mkdirSync(diffDir, { recursive: true });
 
-  const cmd = `diff -ru "${oldDir}" "${newDir}" > "${patch}" || true`;
+  // Generate traditional patch for metrics and rpk
+  const patch = path.join(diffDir, 'changes.patch');
+  const cmd = `diff -ru "${oldTempDir}" "${newTempDir}" > "${patch}" || true`;
   const res = spawnSync(cmd, { stdio: 'inherit', shell: true });
 
   if (res.error) {
@@ -489,6 +595,36 @@ function diffDirs(kind, oldTag, newTag) {
     process.exit(1);
   }
   console.log(`✅ Wrote patch: ${patch}`);
+  
+  // Safety guard: only clean up directories that are explicitly passed as temp directories
+  // For backwards compatibility with autogenerated paths, don't clean up automatically
+  const tmpRoot = path.resolve('tmp') + path.sep;
+  const workspaceRoot = path.resolve('.') + path.sep;
+  
+  // Only clean up if directories were explicitly provided as temp directories
+  // (indicated by having all 5 parameters) and they're in the tmp/ directory
+  const explicitTempDirs = arguments.length >= 5;
+  
+  if (explicitTempDirs) {
+    [oldTempDir, newTempDir].forEach(dirPath => {
+      const resolvedPath = path.resolve(dirPath) + path.sep;
+      const isInTmp = resolvedPath.startsWith(tmpRoot);
+      const isInWorkspace = resolvedPath.startsWith(workspaceRoot);
+      
+      if (isInWorkspace && isInTmp) {
+        try {
+          fs.rmSync(dirPath, { recursive: true, force: true });
+          console.log(`🧹 Cleaned up temporary directory: ${dirPath}`);
+        } catch (err) {
+          console.warn(`⚠️  Warning: Could not clean up directory ${dirPath}: ${err.message}`);
+        }
+      } else {
+        console.log(`ℹ️  Skipping cleanup of directory outside tmp/: ${dirPath}`);
+      }
+    });
+  } else {
+    console.log(`ℹ️  Using autogenerated directories - skipping cleanup for safety`);
+  }
 }
 
 automation
@@ -747,50 +883,85 @@ automation
   .description('Generate JSON and AsciiDoc documentation for Redpanda configuration properties')
   .option('--tag <tag>', 'Git tag or branch to extract from', 'dev')
   .option('--diff <oldTag>', 'Also diff autogenerated properties from <oldTag> → <tag>')
+  .option('--overrides <path>', 'Optional JSON file with property description overrides')
+  .option('--output-dir <dir>', 'Where to write all generated files', 'modules/reference')
+  .option('--cloud-support', 'Enable cloud support metadata by fetching configuration from the cloudv2 repository (requires GITHUB_TOKEN, GH_TOKEN, or REDPANDA_GITHUB_TOKEN)')
+  .option('--template-property-page <path>', 'Custom Handlebars template for property page layout')
+  .option('--template-property <path>', 'Custom Handlebars template for individual property sections')
+  .option('--template-topic-property <path>', 'Custom Handlebars template for individual topic property sections')
+  .option('--template-deprecated <path>', 'Custom Handlebars template for deprecated properties page')
+  .option('--template-deprecated-property <path>', 'Custom Handlebars template for individual deprecated property sections')
   .action((options) => {
     verifyPropertyDependencies();
 
-    const newTag = options.tag;
-    const oldTag = options.diff;
-    const cwd = path.resolve(__dirname, '../tools/property-extractor');
-    const make = (tag) => {
-      console.log(`⏳ Building property docs for ${tag}…`);
-      const r = spawnSync('make', ['build', `TAG=${tag}`], { cwd, stdio: 'inherit' });
-      if (r.error) {
-        console.error(`❌ ${r.error.message}`);
+    // Validate cloud support dependencies if requested
+    if (options.cloudSupport) {
+      console.log('🔍 Validating cloud support dependencies...');
+      
+      // Check for GITHUB_TOKEN, GH_TOKEN, or REDPANDA_GITHUB_TOKEN
+      const token = process.env.GITHUB_TOKEN || process.env.GH_TOKEN || process.env.REDPANDA_GITHUB_TOKEN;
+      if (!token) {
+        console.error('❌ Cloud support requires GITHUB_TOKEN, GH_TOKEN, or REDPANDA_GITHUB_TOKEN environment variable');
+        console.error('   Set up GitHub token:');
+        console.error('   1. Go to https://github.com/settings/tokens');
+        console.error('   2. Generate token with "repo" scope');
+        console.error('   3. Set: export GITHUB_TOKEN=your_token_here');
+        console.error('   Or: export GH_TOKEN=your_token_here');
+        console.error('   Or: export REDPANDA_GITHUB_TOKEN=your_token_here');
         process.exit(1);
       }
-      if (r.status !== 0) process.exit(r.status);
-    };
-
-    if (oldTag) {
-      const oldDir = path.join('autogenerated', oldTag, 'properties');
-      if (!fs.existsSync(oldDir)) make(oldTag);
-    }
-
-    make(newTag);
-
-    if (oldTag) {
-      diffDirs('properties', oldTag, newTag);
+      
+      console.log('📦 Cloud support enabled - Python dependencies will be validated during execution');
+      if (process.env.VIRTUAL_ENV) {
+        console.log(`   Using virtual environment: ${process.env.VIRTUAL_ENV}`);
+      }
+      console.log('   Required packages: pyyaml, requests');
+      console.log('✅ GitHub token validated');
     }
 
-    process.exit(0);
-  });
-
-automation
-  .command('topic-property-docs')
-  .description('Generate JSON and AsciiDoc documentation for Redpanda topic configuration properties')
-  .option('--tag <tag>', 'Git tag or branch to extract from', 'dev')
-  .option('--diff <oldTag>', 'Also diff autogenerated topic properties from <oldTag> → <tag>')
-  .action((options) => {
-    verifyPropertyDependencies();
-
     const newTag = options.tag;
     const oldTag = options.diff;
+    const overridesPath = options.overrides;
+    const outputDir = options.outputDir;
     const cwd = path.resolve(__dirname, '../tools/property-extractor');
-    const make = (tag) => {
-      console.log(`⏳ Building topic property docs for ${tag}…`);
-      const r = spawnSync('make', ['topic-properties', `TAG=${tag}`], { cwd, stdio: 'inherit' });
+    
+    const make = (tag, overrides, templates = {}, outputDir = 'modules/reference/', tempDir = null, cloudSupport = false) => {
+      console.log(`⏳ Building property docs for ${tag}${tempDir ? ' (for diff)' : ''}…`);
+      const args = ['build', `TAG=${tag}`];
+      
+      // Pass all paths as environment variables for consistency
+      const env = { ...process.env };
+      if (overrides) {
+        env.OVERRIDES = path.resolve(overrides);
+      }
+      if (cloudSupport) {
+        env.CLOUD_SUPPORT = '1';
+      }
+      if (templates.propertyPage) {
+        env.TEMPLATE_PROPERTY_PAGE = path.resolve(templates.propertyPage);
+      }
+      if (templates.property) {
+        env.TEMPLATE_PROPERTY = path.resolve(templates.property);
+      }
+      if (templates.topicProperty) {
+        env.TEMPLATE_TOPIC_PROPERTY = path.resolve(templates.topicProperty);
+      }
+      if (templates.deprecated) {
+        env.TEMPLATE_DEPRECATED = path.resolve(templates.deprecated);
+      }
+      if (templates.deprecatedProperty) {
+        env.TEMPLATE_DEPRECATED_PROPERTY = path.resolve(templates.deprecatedProperty);
+      }
+      
+      if (tempDir) {
+        env.OUTPUT_JSON_DIR = path.resolve(tempDir, 'examples');
+        env.OUTPUT_AUTOGENERATED_DIR = path.resolve(tempDir);
+      } else {
+        env.OUTPUT_JSON_DIR = path.resolve(outputDir, 'examples');
+        env.OUTPUT_AUTOGENERATED_DIR = path.resolve(outputDir);
+      }
+      
+      const r = spawnSync('make', args, { cwd, stdio: 'inherit', env });
       if (r.error) {
         console.error(`❌ ${r.error.message}`);
         process.exit(1);
@@ -798,15 +969,26 @@ automation
       if (r.status !== 0) process.exit(r.status);
     };
 
+    // Collect template options
+    const templates = {
+      propertyPage: options.templatePropertyPage,
+      property: options.templateProperty,
+      topicProperty: options.templateTopicProperty,
+      deprecated: options.templateDeprecated,
+      deprecatedProperty: options.templateDeprecatedProperty
+    };
+
     if (oldTag) {
-      const oldDir = path.join('autogenerated', oldTag, 'properties');
-      if (!fs.existsSync(oldDir)) make(oldTag);
+      // Generate old version directly to final destination so its JSON is available for comparison
+      make(oldTag, overridesPath, templates, outputDir, null, options.cloudSupport);
     }
 
-    make(newTag);
+    // Generate new version to final destination
+    make(newTag, overridesPath, templates, outputDir, null, options.cloudSupport);
 
     if (oldTag) {
-      diffDirs('properties', oldTag, newTag);
+      // Generate property comparison report using the JSON files now in modules/reference/examples
+      generatePropertyComparisonReport(oldTag, newTag, 'modules/reference');
     }
 
     process.exit(0);
@@ -1015,9 +1197,9 @@ automation
     const { generateCloudRegions } = require('../tools/cloud-regions/generate-cloud-regions.js');
 
     try {
-      const token = process.env.GITHUB_TOKEN || process.env.GH_TOKEN;
+      const token = process.env.GITHUB_TOKEN || process.env.GH_TOKEN || process.env.REDPANDA_GITHUB_TOKEN;
       if (!token) {
-        throw new Error('GITHUB_TOKEN environment variable is required to fetch from private cloudv2-infra repo.');
+        throw new Error('GITHUB_TOKEN, GH_TOKEN, or REDPANDA_GITHUB_TOKEN environment variable is required to fetch from private cloudv2-infra repo.');
       }
       const fmt = (options.format || 'md').toLowerCase();
       let templatePath = undefined;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redpanda-data/docs-extensions-and-macros",
-  "version": "4.8.0",
+  "version": "4.9.0",
   "description": "Antora extensions and macros developed for Redpanda documentation.",
   "keywords": [
     "antora",

package/tools/property-extractor/Makefile

@@ -1,16 +1,4 @@
-.PHONY: build venv clean redpanda-git treesitter topic-properties generate-docs check
-
-# --- Main build: venv, fetch code, build parser, extract & docgen ---
-build: venv redpanda-git treesitter
-	@echo "🔧 Building with Redpanda tag: $(TAG)"
-	@mkdir -p $(TOOL_ROOT)/gen
-	@cd $(TOOL_ROOT) && \
-	  $(PYTHON) -W ignore::FutureWarning property_extractor.py \
-	    --recursive \
-	    --path $(REDPANDA_SRC) \
-	    --output gen/properties-output.json
-	@echo "✅ Cluster properties JSON generated at $(TOOL_ROOT)/gen/properties-output.json"
-	@$(MAKE) generate-docs
+.PHONY: build venv clean redpanda-git treesitter generate-docs check
 
 # Default tag (can be overridden via `make TAG=v25.1.1`)
 TAG       ?= dev
@@ -18,11 +6,11 @@ TAG       ?= dev
 # Derive a “major.minor” or rc identifier from TAG for folder naming
 VERSION := $(shell \
   if echo "$(TAG)" | grep -qE '^v?[0-9]+\.[0-9]+'; then \
-    echo "$(TAG)" \
-      | sed -E 's/^v?([0-9]+\.[0-9]+)(\.[0-9]+)?(-rc[0-9]+)?.*/\1\3/' \
-      | sed 's/-rc/rc/'; \
+	echo "$(TAG)" \
+	  | sed -E 's/^v?([0-9]+\.[0-9]+)(\.[0-9]+)?(-rc[0-9]+)?.*/\1\3/' \
+	  | sed 's/-rc/rc/'; \
   else \
-    echo "$(TAG)"; \
+	echo "$(TAG)"; \
   fi)
 
 # Paths
@@ -34,19 +22,45 @@ REDPANDA_SRC  := $(TMP_ROOT)/redpanda
 TREESITTER_DIR:= $(TOOL_ROOT)/tree-sitter/tree-sitter-cpp
 VENV          := $(TOOL_ROOT)/tmp/redpanda-property-extractor-venv
 PYTHON        := $(VENV)/bin/python
-OUTPUT_DIR    := $(REPO_ROOT)/autogenerated/$(TAG)/properties
 TREE_SITTER   := npx tree-sitter
 
+# Output directory configuration (can be overridden by environment variables)
+OUTPUT_AUTOGENERATED_DIR ?= $(REPO_ROOT)/modules/reference
+OUTPUT_ASCIIDOC_DIR      ?= $(OUTPUT_AUTOGENERATED_DIR)/pages
+OUTPUT_JSON_DIR          ?= $(OUTPUT_AUTOGENERATED_DIR)/examples
+
+# --- Main build: venv, fetch code, build parser, extract & docgen ---
+build: venv redpanda-git treesitter
+	@echo "🔧 Building with Redpanda tag: $(TAG)"
+	@mkdir -p $(TOOL_ROOT)/gen
+	@if [ -n "$(OVERRIDES)" ] && [ ! -f "$(OVERRIDES)" ]; then \
+	  echo "❌ Error: Overrides file '$(OVERRIDES)' does not exist."; \
+	  echo "   Please check the path or omit the OVERRIDES variable to continue without overrides."; \
+	  exit 1; \
+	fi
+	@cd $(TOOL_ROOT) && \
+	  $(PYTHON) -W ignore property_extractor.py \
+		--recursive \
+		--path $(REDPANDA_SRC) \
+		--output gen/properties-output.json \
+		--enhanced-output gen/$(TAG)-properties.json \
+		$(if $(OVERRIDES),$(if $(shell [ -f "$(OVERRIDES)" ] && echo exists),--overrides $(OVERRIDES),),) \
+		$(if $(CLOUD_SUPPORT),--cloud-support,)
+	@echo "✅ JSON generated at $(TOOL_ROOT)/gen/properties-output.json"
+	@echo "✅ Enhanced JSON (with overrides) generated at $(TOOL_ROOT)/gen/$(TAG)-properties.json"
+	@$(MAKE) generate-docs
+
 # --- Ensure Python venv & dependencies ---
 venv: $(TOOL_ROOT)/requirements.txt
 	@if [ ! -d "$(VENV)" ]; then \
 	  echo "🐍 Creating virtual environment in $(VENV)..."; \
 	  python3 -m venv $(VENV); \
-	  $(VENV)/bin/pip install --upgrade pip --quiet; \
-	  $(VENV)/bin/pip install --no-cache-dir -r $<; \
 	else \
 	  echo "🐍 Virtual environment already exists at $(VENV)"; \
-	fi
+	fi; \
+	echo "🔄 Upgrading pip and installing requirements..."; \
+	$(VENV)/bin/pip install --upgrade pip --quiet; \
+	$(VENV)/bin/pip install --no-cache-dir -r $<;
 
 # --- Clean out all generated state ---
 clean:
@@ -82,36 +96,40 @@ treesitter:
 	  git fetch --tags -q && \
 	  git checkout -q v0.20.5
 	@echo "🔧 Generating parser in $(TREESITTER_DIR)…"
-	@cd "$(TREESITTER_DIR)" && npm install --silent && $(TREE_SITTER) generate
+	@cd "$(TREESITTER_DIR)" && export CFLAGS="-Wno-unused-but-set-variable" && npm install --silent && export CFLAGS="-Wno-unused-but-set-variable" && $(TREE_SITTER) generate
 
-# --- Turn the JSON into AsciiDoc pages under autogen/<version>/properties ---
-generate-docs:
-	@echo "📝 Generating AsciiDoc pages in $(OUTPUT_DIR)…"
-	@mkdir -p "$(OUTPUT_DIR)"
-	@cd $(TOOL_ROOT) && \
-	  $(PYTHON) json-to-asciidoc/generate_docs.py --output-dir "$(OUTPUT_DIR)"
-	@echo "📄 Copying properties-output.json to $(OUTPUT_DIR)…"
-	@cp "$(TOOL_ROOT)/gen/properties-output.json" "$(OUTPUT_DIR)/"
-	@echo "✅ Docs generated at $(OUTPUT_DIR)"
+# --- Install Node.js dependencies for Handlebars ---
+node-deps:
+	@echo "📦 Installing Node.js dependencies…"
+	@cd $(TOOL_ROOT) && npm install --silent
+
+# --- Turn the JSON into AsciiDoc pages under the specified output directories ---
+generate-docs: node-deps
+	@echo "📝 Generating AsciiDoc pages in $(OUTPUT_ASCIIDOC_DIR)…"
+	@mkdir -p "$(OUTPUT_ASCIIDOC_DIR)"
+	@mkdir -p "$(OUTPUT_JSON_DIR)"
+	@# Use the enhanced properties file (with overrides) for documentation generation if it exists
+	@if [ -f "$(TOOL_ROOT)/gen/$(TAG)-properties.json" ]; then \
+	  cd $(TOOL_ROOT) && \
+	    node generate-handlebars-docs.js "gen/$(TAG)-properties.json" "$(OUTPUT_AUTOGENERATED_DIR)"; \
+	else \
+	  cd $(TOOL_ROOT) && \
+	    node generate-handlebars-docs.js "gen/properties-output.json" "$(OUTPUT_AUTOGENERATED_DIR)"; \
+	fi
+	@echo "📄 Copying properties JSON files to $(OUTPUT_JSON_DIR)…"
+	@if [ -f "$(TOOL_ROOT)/gen/$(TAG)-properties.json" ]; then \
+	  cp "$(TOOL_ROOT)/gen/$(TAG)-properties.json" "$(OUTPUT_JSON_DIR)/"; \
+	fi
+	@echo "✅ Docs generated at $(OUTPUT_AUTOGENERATED_DIR)"
 
 # --- Debug helper to print all the key paths/vars ---
 check:
-	@echo "MODULE_ROOT:   $(MODULE_ROOT)"
-	@echo "TOOL_ROOT:     $(TOOL_ROOT)"
-	@echo "REDPANDA_SRC:  $(REDPANDA_SRC)"
-	@echo "TREESITTER:    $(TREESITTER_DIR)"
-	@echo "VENV:          $(VENV)"
-	@echo "PYTHON:        $(PYTHON)"
-	@echo "OUTPUT_DIR:    $(OUTPUT_DIR)"
-
-# --- Extract topic properties ---
-topic-properties: venv redpanda-git treesitter
-	@echo "🔧 Extracting topic properties with Redpanda tag: $(TAG)"
-	@mkdir -p $(TOOL_ROOT)/gen
-	@mkdir -p "$(OUTPUT_DIR)"
-	@cd $(TOOL_ROOT) && \
-	  $(PYTHON) topic_property_extractor.py \
-	    --source-path $(REDPANDA_SRC) \
-	    --output-json "$(OUTPUT_DIR)/topic-properties-output.json" \
-	    --output-adoc "$(OUTPUT_DIR)/topic-properties.adoc"
-	@echo "✅ Topic properties extracted"
+	@echo "MODULE_ROOT:              $(MODULE_ROOT)"
+	@echo "TOOL_ROOT:                $(TOOL_ROOT)"
+	@echo "REDPANDA_SRC:             $(REDPANDA_SRC)"
+	@echo "TREESITTER:               $(TREESITTER_DIR)"
+	@echo "VENV:                     $(VENV)"
+	@echo "PYTHON:                   $(PYTHON)"
+	@echo "OUTPUT_ASCIIDOC_DIR:      $(OUTPUT_ASCIIDOC_DIR)"
+	@echo "OUTPUT_JSON_DIR:          $(OUTPUT_JSON_DIR)"
+	@echo "OUTPUT_AUTOGENERATED_DIR: $(OUTPUT_AUTOGENERATED_DIR)"