@redpanda-data/docs-extensions-and-macros 4.8.1 → 4.9.1

package/bin/doc-tools.js

@@ -459,18 +459,120 @@ const commonOptions = {
   consoleDockerRepo: 'console',
 };
 
+/**
+ * Run the cluster documentation generator script for a specific release/tag.
+ *
+ * Invokes the external `generate-cluster-docs.sh` script with the provided mode, tag,
+ * and Docker-related options. The script's stdout/stderr are forwarded to the current
+ * process; if the script exits with a non-zero status, this function will terminate
+ * the Node.js process with that status code.
+ *
+ * @param {string} mode - Operation mode passed to the script (e.g., "generate" or "clean").
+ * @param {string} tag - Release tag or version to generate docs for.
+ * @param {Object} options - Runtime options.
+ * @param {string} options.dockerRepo - Docker repository used by the script.
+ * @param {string} options.consoleTag - Console image tag passed to the script.
+ * @param {string} options.consoleDockerRepo - Console Docker repository used by the script.
+ */
 function runClusterDocs(mode, tag, options) {
   const script = path.join(__dirname, '../cli-utils/generate-cluster-docs.sh');
   const args = [mode, tag, options.dockerRepo, options.consoleTag, options.consoleDockerRepo];
   console.log(`⏳ Running ${script} with arguments: ${args.join(' ')}`);
-  const r = spawnSync('bash', [script, ...args], { stdio: 'inherit', shell: true });
+  const r = spawnSync('bash', [script, ...args], { stdio: 'inherit' });
   if (r.status !== 0) process.exit(r.status);
 }
 
-// helper to diff two temporary directories
+/**
+ * Generate a detailed JSON report describing property changes between two releases.
+ *
+ * Looks for `<oldTag>-properties.json` and `<newTag>-properties.json` in
+ * `modules/reference/examples`. If both files exist, invokes the external
+ * property comparison tool to produce `property-changes-<oldTag>-to-<newTag>.json`
+ * in the provided output directory.
+ *
+ * If either input JSON is missing the function logs a message and returns without
+ * error. Any errors from the comparison tool are logged; the function does not
+ * throw.
+ *
+ * @param {string} oldTag - Release tag or identifier for the "old" properties set.
+ * @param {string} newTag - Release tag or identifier for the "new" properties set.
+ * @param {string} outputDir - Directory where the comparison report will be written.
+ */
+function generatePropertyComparisonReport(oldTag, newTag, outputDir) {
+  try {
+    console.log(`\n📊 Generating detailed property comparison report...`);
+    
+    // Look for the property JSON files in modules/reference/examples
+    const repoRoot = findRepoRoot();
+    const examplesDir = path.join(repoRoot, 'modules', 'reference', 'examples');
+    const oldJsonPath = path.join(examplesDir, `${oldTag}-properties.json`);
+    const newJsonPath = path.join(examplesDir, `${newTag}-properties.json`);
+    
+    // Check if JSON files exist
+    if (!fs.existsSync(oldJsonPath)) {
+      console.log(`⚠️  Old properties JSON not found at: ${oldJsonPath}`);
+      console.log(`   Skipping detailed property comparison.`);
+      return;
+    }
+    
+    if (!fs.existsSync(newJsonPath)) {
+      console.log(`⚠️  New properties JSON not found at: ${newJsonPath}`);
+      console.log(`   Skipping detailed property comparison.`);
+      return;
+    }
+    
+    // Ensure output directory exists (use absolute path)
+    const absoluteOutputDir = path.resolve(outputDir);
+    fs.mkdirSync(absoluteOutputDir, { recursive: true });
+    
+    // Run the property comparison tool with descriptive filename
+    const propertyExtractorDir = path.resolve(__dirname, '../tools/property-extractor');
+    const compareScript = path.join(propertyExtractorDir, 'compare-properties.js');
+    const reportFilename = `property-changes-${oldTag}-to-${newTag}.json`;
+    const reportPath = path.join(absoluteOutputDir, reportFilename);
+    const args = [compareScript, oldJsonPath, newJsonPath, oldTag, newTag, absoluteOutputDir, reportFilename];
+    
+    const result = spawnSync('node', args, { 
+      stdio: 'inherit',
+      cwd: propertyExtractorDir 
+    });
+    
+    if (result.error) {
+      console.error(`❌ Property comparison failed: ${result.error.message}`);
+    } else if (result.status !== 0) {
+      console.error(`❌ Property comparison exited with code: ${result.status}`);
+    } else {
+      console.log(`✅ Property comparison report saved to: ${reportPath}`);
+    }
+  } catch (error) {
+    console.error(`❌ Error generating property comparison: ${error.message}`);
+  }
+}
+
+/**
+ * Create a unified diff patch between two temporary directories and clean them up.
+ *
+ * Ensures both source directories exist, writes a recursive unified diff
+ * (changes.patch) to tmp/diffs/<kind>/<oldTag>_to_<newTag>/, and removes the
+ * provided temporary directories. On missing inputs or if the diff subprocess
+ * fails to spawn, the process exits with a non-zero status.
+ *
+ * @param {string} kind - Logical category for the diff (e.g., "metrics" or "rpk"); used in the output path.
+ * @param {string} oldTag - Identifier for the "old" version (used in the output path).
+ * @param {string} newTag - Identifier for the "new" version (used in the output path).
+ * @param {string} oldTempDir - Path to the existing temporary directory containing the old output; must exist.
+ * @param {string} newTempDir - Path to the existing temporary directory containing the new output; must exist.
+ */
 function diffDirs(kind, oldTag, newTag, oldTempDir, newTempDir) {
+  // Backwards compatibility: if temp directories not provided, use autogenerated paths
+  if (!oldTempDir) {
+    oldTempDir = path.join('autogenerated', oldTag, kind);
+  }
+  if (!newTempDir) {
+    newTempDir = path.join('autogenerated', newTag, kind);
+  }
+
   const diffDir = path.join('tmp', 'diffs', kind, `${oldTag}_to_${newTag}`);
-  const patch = path.join(diffDir, 'changes.patch');
 
   if (!fs.existsSync(oldTempDir)) {
     console.error(`❌ Cannot diff: missing ${oldTempDir}`);
@@ -483,6 +585,8 @@ function diffDirs(kind, oldTag, newTag, oldTempDir, newTempDir) {
 
   fs.mkdirSync(diffDir, { recursive: true });
 
+  // Generate traditional patch for metrics and rpk
+  const patch = path.join(diffDir, 'changes.patch');
   const cmd = `diff -ru "${oldTempDir}" "${newTempDir}" > "${patch}" || true`;
   const res = spawnSync(cmd, { stdio: 'inherit', shell: true });
 
@@ -492,9 +596,35 @@ function diffDirs(kind, oldTag, newTag, oldTempDir, newTempDir) {
   }
   console.log(`✅ Wrote patch: ${patch}`);
   
-  // Clean up temporary directories
-  fs.rmSync(oldTempDir, { recursive: true, force: true });
-  fs.rmSync(newTempDir, { recursive: true, force: true });
+  // Safety guard: only clean up directories that are explicitly passed as temp directories
+  // For backwards compatibility with autogenerated paths, don't clean up automatically
+  const tmpRoot = path.resolve('tmp') + path.sep;
+  const workspaceRoot = path.resolve('.') + path.sep;
+  
+  // Only clean up if directories were explicitly provided as temp directories
+  // (indicated by having all 5 parameters) and they're in the tmp/ directory
+  const explicitTempDirs = arguments.length >= 5;
+  
+  if (explicitTempDirs) {
+    [oldTempDir, newTempDir].forEach(dirPath => {
+      const resolvedPath = path.resolve(dirPath) + path.sep;
+      const isInTmp = resolvedPath.startsWith(tmpRoot);
+      const isInWorkspace = resolvedPath.startsWith(workspaceRoot);
+      
+      if (isInWorkspace && isInTmp) {
+        try {
+          fs.rmSync(dirPath, { recursive: true, force: true });
+          console.log(`🧹 Cleaned up temporary directory: ${dirPath}`);
+        } catch (err) {
+          console.warn(`⚠️  Warning: Could not clean up directory ${dirPath}: ${err.message}`);
+        }
+      } else {
+        console.log(`ℹ️  Skipping cleanup of directory outside tmp/: ${dirPath}`);
+      }
+    });
+  } else {
+    console.log(`ℹ️  Using autogenerated directories - skipping cleanup for safety`);
+  }
 }
 
 automation
@@ -546,6 +676,7 @@ automation
   .description('Generate RPCN connector docs and diff changes since the last version')
   .option('-d, --data-dir <path>', 'Directory where versioned connect JSON files live', path.resolve(process.cwd(), 'docs-data'))
   .option('--old-data <path>', 'Optional override for old data file (for diff)')
+  .option('--update-whats-new', 'Update whats-new.adoc with new section from diff JSON')
   .option('-f, --fetch-connectors', 'Fetch latest connector data using rpk')
   .option('-m, --draft-missing', 'Generate full-doc drafts for connectors missing in output')
   .option('--csv <path>', 'Path to connector metadata CSV file', 'internal/plugins/info.csv')
@@ -553,7 +684,9 @@ automation
   .option('--template-intro <path>', 'Intro section partial template', path.resolve(__dirname, '../tools/redpanda-connect/templates/intro.hbs'))
   .option('--template-fields <path>', 'Fields section partial template', path.resolve(__dirname, '../tools/redpanda-connect/templates/fields-partials.hbs'))
   .option('--template-examples <path>', 'Examples section partial template', path.resolve(__dirname, '../tools/redpanda-connect/templates/examples-partials.hbs'))
+  .option('--template-bloblang <path>', 'Custom Handlebars template for bloblang function/method partials')
   .option('--overrides <path>', 'Optional JSON file with overrides')
+  .option('--include-bloblang', 'Include Bloblang functions and methods in generation')
   .action(async (options) => {
     requireTool('rpk', {
       versionFlag: '--version',
@@ -614,7 +747,9 @@ automation
         templateIntro: options.templateIntro,
         templateFields: options.templateFields,
         templateExamples: options.templateExamples,
-        writeFullDrafts: false
+        templateBloblang: options.templateBloblang,
+        writeFullDrafts: false,
+        includeBloblang: !!options.includeBloblang
       });
       partialsWritten = result.partialsWritten;
       partialFiles    = result.partialFiles;
@@ -696,10 +831,13 @@ automation
     }
 
     let oldIndex = {};
+    let oldVersion = null;
     if (options.oldData && fs.existsSync(options.oldData)) {
       oldIndex = JSON.parse(fs.readFileSync(options.oldData, 'utf8'));
+      const m = options.oldData.match(/connect-([\d.]+)\.json$/);
+      if (m) oldVersion = m[1];
     } else {
-      const oldVersion = getAntoraValue('asciidoc.attributes.latest-connect-version');
+      oldVersion = getAntoraValue('asciidoc.attributes.latest-connect-version');
       if (oldVersion) {
         const oldPath = path.join(dataDir, `connect-${oldVersion}.json`);
         if (fs.existsSync(oldPath)) {
@@ -711,6 +849,21 @@ automation
     const newIndex = JSON.parse(fs.readFileSync(dataFile, 'utf8'));
     printDeltaReport(oldIndex, newIndex);
 
+    // Generate JSON diff file for whats-new.adoc
+    const { generateConnectorDiffJson } = require('../tools/redpanda-connect/report-delta.js');
+    const diffJson = generateConnectorDiffJson(
+      oldIndex,
+      newIndex,
+      {
+        oldVersion: oldVersion || '',
+        newVersion,
+        timestamp
+      }
+    );
+    const diffPath = path.join(dataDir, `connect-diff-${(oldVersion || 'unknown')}_to_${newVersion}.json`);
+    fs.writeFileSync(diffPath, JSON.stringify(diffJson, null, 2), 'utf8');
+    console.log(`✅ Connector diff JSON written to: ${diffPath}`);
+
     function logCollapsed(label, filesArray, maxToShow = 10) {
       console.log(`  • ${label}: ${filesArray.length} total`);
       const sample = filesArray.slice(0, maxToShow);
@@ -742,6 +895,135 @@ automation
       logCollapsed('Draft files', draftFiles, 5);
     }
 
+    // Optionally update whats-new.adoc
+    if (options.updateWhatsNew) {
+      try {
+        const whatsNewPath = path.join(findRepoRoot(), 'modules/get-started/pages/whats-new.adoc');
+        if (!fs.existsSync(whatsNewPath)) {
+          console.error(`❌ Unable to update release notes: 'whats-new.adoc' was not found at: ${whatsNewPath}\nPlease ensure this file exists and is tracked in your repository.`);
+          return;
+        }
+        // Find the diff JSON file we just wrote
+        const diffPath = path.join(dataDir, `connect-diff-${(oldVersion || 'unknown')}_to_${newVersion}.json`);
+        if (!fs.existsSync(diffPath)) {
+          console.error(`❌ Unable to update release notes: The connector diff JSON was not found at: ${diffPath}\nPlease ensure the diff was generated successfully before updating release notes.`);
+          return;
+        }
+        let diff;
+        try {
+          diff = JSON.parse(fs.readFileSync(diffPath, 'utf8'));
+        } catch (jsonErr) {
+          console.error(`❌ Unable to parse connector diff JSON at ${diffPath}: ${jsonErr.message}\nPlease check the file for syntax errors or corruption.`);
+          return;
+        }
+        let whatsNewContent;
+        try {
+          whatsNewContent = fs.readFileSync(whatsNewPath, 'utf8');
+        } catch (readErr) {
+          console.error(`❌ Unable to read whats-new.adoc at ${whatsNewPath}: ${readErr.message}\nPlease check file permissions and try again.`);
+          return;
+        }
+  const whatsNew = whatsNewContent;
+        // Regex to find section for this version
+        const versionTitle = `== Version ${diff.comparison.newVersion}`;
+        const versionRe = new RegExp(`^== Version ${diff.comparison.newVersion.replace(/[-.]/g, '\\$&')}(?:\\r?\\n|$)`, 'm');
+        const match = versionRe.exec(whatsNew);
+        let startIdx = match ? match.index : -1;
+        let endIdx = -1;
+        if (startIdx !== -1) {
+          // Find the start of the next version section
+          const rest = whatsNew.slice(startIdx + 1);
+          const nextMatch = /^== Version /m.exec(rest);
+          endIdx = nextMatch ? startIdx + 1 + nextMatch.index : whatsNew.length;
+        }
+        // Compose new section
+        let section = `\n== Version ${diff.comparison.newVersion}\n\n=== Component updates\n\n`;
+        // Add link to full release notes for this connector version after version heading and before component updates
+        let releaseNotesLink = '';
+        if (diff.comparison && diff.comparison.newVersion) {
+          releaseNotesLink = `link:https://github.com/redpanda-data/connect/releases/tag/v${diff.comparison.newVersion}[See the full release notes^].\n\n`;
+        }
+        section = `\n== Version ${diff.comparison.newVersion}\n\n${releaseNotesLink}=== Component updates\n\n`;
+        // New components
+        if (diff.details.newComponents && diff.details.newComponents.length) {
+          section += 'This release adds the following new components:\n\n';
+          // Group by type
+          const byType = {};
+          for (const comp of diff.details.newComponents) {
+            if (!byType[comp.type]) byType[comp.type] = [];
+            byType[comp.type].push(comp);
+          }
+          for (const [type, comps] of Object.entries(byType)) {
+            section += `* ${type.charAt(0).toUpperCase() + type.slice(1)}:\n`;
+            for (const comp of comps) {
+              section += `** xref:components:${type}/${comp.name}.adoc[\`${comp.name}\`]`;
+              if (comp.status) section += ` (${comp.status})`;
+              if (comp.description) section += `: ${comp.description}`;
+              section += '\n';
+            }
+          }
+        }
+
+        // New fields
+        if (diff.details.newFields && diff.details.newFields.length) {
+          section += '\nThis release adds support for the following new fields:\n\n';
+          // Group new fields by component type
+          const fieldsByType = {};
+          for (const field of diff.details.newFields) {
+            // component: "inputs:kafka", field: "timely_nacks_maximum_wait"
+            const [type, compName] = field.component.split(':');
+            if (!fieldsByType[type]) fieldsByType[type] = [];
+            fieldsByType[type].push({
+              compName,
+              field: field.field,
+              description: field.description || '',
+            });
+          }
+          for (const [type, fields] of Object.entries(fieldsByType)) {
+            section += `* ${type.charAt(0).toUpperCase() + type.slice(1)} components\n`;
+            // Group by component name
+            const byComp = {};
+            for (const f of fields) {
+              if (!byComp[f.compName]) byComp[f.compName] = [];
+              byComp[f.compName].push(f);
+            }
+            for (const [comp, compFields] of Object.entries(byComp)) {
+              section += `** xref:components:${type}/${comp}.adoc['${comp}']`;
+              if (compFields.length === 1) {
+                const f = compFields[0];
+                section += `: xref:components:${type}/${comp}.adoc#${f.field}['${f.field}']`;
+                if (f.description) section += ` - ${f.description}`;
+                section += '\n';
+              } else {
+                section += '\n';
+                for (const f of compFields) {
+                  section += `*** xref:components:${type}/${comp}.adoc#${f.field}['${f.field}']`;
+                  if (f.description) section += ` - ${f.description}`;
+                  section += '\n';
+                }
+              }
+            }
+          }
+        }
+        let updated;
+        if (startIdx !== -1) {
+          // Replace the existing section
+          updated = whatsNew.slice(0, startIdx) + section + '\n' + whatsNew.slice(endIdx);
+          console.log(`♻️  whats-new.adoc: replaced section for Version ${diff.comparison.newVersion}`);
+        } else {
+          // Insert above first version heading
+          const versionHeading = /^== Version /m;
+          const firstMatch = versionHeading.exec(whatsNew);
+          let insertIdx = firstMatch ? firstMatch.index : 0;
+          updated = whatsNew.slice(0, insertIdx) + section + '\n' + whatsNew.slice(insertIdx);
+          console.log(`✅ whats-new.adoc updated with Version ${diff.comparison.newVersion}`);
+        }
+        fs.writeFileSync(whatsNewPath, updated, 'utf8');
+      } catch (err) {
+        console.error(`❌ Failed to update whats-new.adoc: ${err.message}`);
+      }
+    }
+
     console.log('\n📄 Summary:');
     console.log(`   • Run time: ${timestamp}`);
     console.log(`   • Version used: ${newVersion}`);
@@ -755,20 +1037,47 @@ automation
   .option('--diff <oldTag>', 'Also diff autogenerated properties from <oldTag> → <tag>')
   .option('--overrides <path>', 'Optional JSON file with property description overrides')
   .option('--output-dir <dir>', 'Where to write all generated files', 'modules/reference')
+  .option('--cloud-support', 'Enable cloud support metadata by fetching configuration from the cloudv2 repository (requires GITHUB_TOKEN, GH_TOKEN, or REDPANDA_GITHUB_TOKEN)')
   .option('--template-property-page <path>', 'Custom Handlebars template for property page layout')
   .option('--template-property <path>', 'Custom Handlebars template for individual property sections')
+  .option('--template-topic-property <path>', 'Custom Handlebars template for individual topic property sections')
   .option('--template-deprecated <path>', 'Custom Handlebars template for deprecated properties page')
   .option('--template-deprecated-property <path>', 'Custom Handlebars template for individual deprecated property sections')
   .action((options) => {
     verifyPropertyDependencies();
 
+    // Validate cloud support dependencies if requested
+    if (options.cloudSupport) {
+      console.log('🔍 Validating cloud support dependencies...');
+      
+      // Check for GITHUB_TOKEN, GH_TOKEN, or REDPANDA_GITHUB_TOKEN
+      const token = process.env.GITHUB_TOKEN || process.env.GH_TOKEN || process.env.REDPANDA_GITHUB_TOKEN;
+      if (!token) {
+        console.error('❌ Cloud support requires GITHUB_TOKEN, GH_TOKEN, or REDPANDA_GITHUB_TOKEN environment variable');
+        console.error('   Set up GitHub token:');
+        console.error('   1. Go to https://github.com/settings/tokens');
+        console.error('   2. Generate token with "repo" scope');
+        console.error('   3. Set: export GITHUB_TOKEN=your_token_here');
+        console.error('   Or: export GH_TOKEN=your_token_here');
+        console.error('   Or: export REDPANDA_GITHUB_TOKEN=your_token_here');
+        process.exit(1);
+      }
+      
+      console.log('📦 Cloud support enabled - Python dependencies will be validated during execution');
+      if (process.env.VIRTUAL_ENV) {
+        console.log(`   Using virtual environment: ${process.env.VIRTUAL_ENV}`);
+      }
+      console.log('   Required packages: pyyaml, requests');
+      console.log('✅ GitHub token validated');
+    }
+
     const newTag = options.tag;
     const oldTag = options.diff;
     const overridesPath = options.overrides;
     const outputDir = options.outputDir;
     const cwd = path.resolve(__dirname, '../tools/property-extractor');
     
-    const make = (tag, overrides, templates = {}, outputDir = 'modules/reference/', tempDir = null) => {
+    const make = (tag, overrides, templates = {}, outputDir = 'modules/reference/', tempDir = null, cloudSupport = false) => {
       console.log(`⏳ Building property docs for ${tag}${tempDir ? ' (for diff)' : ''}…`);
       const args = ['build', `TAG=${tag}`];
       
@@ -777,12 +1086,18 @@ automation
       if (overrides) {
         env.OVERRIDES = path.resolve(overrides);
       }
+      if (cloudSupport) {
+        env.CLOUD_SUPPORT = '1';
+      }
       if (templates.propertyPage) {
         env.TEMPLATE_PROPERTY_PAGE = path.resolve(templates.propertyPage);
       }
       if (templates.property) {
         env.TEMPLATE_PROPERTY = path.resolve(templates.property);
       }
+      if (templates.topicProperty) {
+        env.TEMPLATE_TOPIC_PROPERTY = path.resolve(templates.topicProperty);
+      }
       if (templates.deprecated) {
         env.TEMPLATE_DEPRECATED = path.resolve(templates.deprecated);
       }
@@ -791,13 +1106,9 @@ automation
       }
       
       if (tempDir) {
-        // For diff purposes, generate to temporary directory
-        env.OUTPUT_ASCIIDOC_DIR = path.resolve(tempDir);
         env.OUTPUT_JSON_DIR = path.resolve(tempDir, 'examples');
         env.OUTPUT_AUTOGENERATED_DIR = path.resolve(tempDir);
       } else {
-        // Normal generation - go directly to final destination
-        // Let Makefile calculate OUTPUT_ASCIIDOC_DIR as OUTPUT_AUTOGENERATED_DIR/pages
         env.OUTPUT_JSON_DIR = path.resolve(outputDir, 'examples');
         env.OUTPUT_AUTOGENERATED_DIR = path.resolve(outputDir);
       }
@@ -814,34 +1125,22 @@ automation
     const templates = {
       propertyPage: options.templatePropertyPage,
       property: options.templateProperty,
+      topicProperty: options.templateTopicProperty,
       deprecated: options.templateDeprecated,
       deprecatedProperty: options.templateDeprecatedProperty
     };
 
-    let oldTempDir = null;
-    let newTempDir = null;
-
     if (oldTag) {
-      // Generate old version to temporary directory for diff
-      oldTempDir = path.join('tmp', 'diff', `${oldTag}-properties`);
-      fs.mkdirSync(oldTempDir, { recursive: true });
-      make(oldTag, overridesPath, templates, outputDir, oldTempDir);
+      // Generate old version directly to final destination so its JSON is available for comparison
+      make(oldTag, overridesPath, templates, outputDir, null, options.cloudSupport);
     }
 
+    // Generate new version to final destination
+    make(newTag, overridesPath, templates, outputDir, null, options.cloudSupport);
+
     if (oldTag) {
-      // Generate new version to temporary directory for diff
-      newTempDir = path.join('tmp', 'diff', `${newTag}-properties`);
-      fs.mkdirSync(newTempDir, { recursive: true });
-      make(newTag, overridesPath, templates, outputDir, newTempDir);
-      
-      // Then generate new version to final destination
-      make(newTag, overridesPath, templates, outputDir);
-      
-      // Compare the temporary directories
-      diffDirs('properties', oldTag, newTag, oldTempDir, newTempDir);
-    } else {
-      // No diff requested, just generate to final destination
-      make(newTag, overridesPath, templates, outputDir);
+      // Generate property comparison report using the JSON files now in modules/reference/examples
+      generatePropertyComparisonReport(oldTag, newTag, 'modules/reference');
     }
 
     process.exit(0);
@@ -1050,9 +1349,9 @@ automation
     const { generateCloudRegions } = require('../tools/cloud-regions/generate-cloud-regions.js');
 
     try {
-      const token = process.env.GITHUB_TOKEN || process.env.GH_TOKEN;
+      const token = process.env.GITHUB_TOKEN || process.env.GH_TOKEN || process.env.REDPANDA_GITHUB_TOKEN;
       if (!token) {
-        throw new Error('GITHUB_TOKEN environment variable is required to fetch from private cloudv2-infra repo.');
+        throw new Error('GITHUB_TOKEN, GH_TOKEN, or REDPANDA_GITHUB_TOKEN environment variable is required to fetch from private cloudv2-infra repo.');
       }
       const fmt = (options.format || 'md').toLowerCase();
       let templatePath = undefined;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redpanda-data/docs-extensions-and-macros",
-  "version": "4.8.1",
+  "version": "4.9.1",
   "description": "Antora extensions and macros developed for Redpanda documentation.",
   "keywords": [
     "antora",

package/tools/property-extractor/Makefile

@@ -1,17 +1,5 @@
 .PHONY: build venv clean redpanda-git treesitter generate-docs check
 
-# --- Main build: venv, fetch code, build parser, extract & docgen ---
-build: venv redpanda-git treesitter
-	@echo "🔧 Building with Redpanda tag: $(TAG)"
-	@mkdir -p $(TOOL_ROOT)/gen
-	@cd $(TOOL_ROOT) && \
-	  $(PYTHON) -W ignore::FutureWarning property_extractor.py \
-	    --recursive \
-	    --path $(REDPANDA_SRC) \
-	    --output gen/properties-output.json
-	@echo "✅ Cluster properties JSON generated at $(TOOL_ROOT)/gen/properties-output.json"
-	@$(MAKE) generate-docs
-
 # Default tag (can be overridden via `make TAG=v25.1.1`)
 TAG       ?= dev
 
@@ -51,12 +39,13 @@ build: venv redpanda-git treesitter
 	  exit 1; \
 	fi
 	@cd $(TOOL_ROOT) && \
-	  $(PYTHON) -W ignore::FutureWarning property_extractor.py \
+	  $(PYTHON) -W ignore property_extractor.py \
 		--recursive \
 		--path $(REDPANDA_SRC) \
 		--output gen/properties-output.json \
 		--enhanced-output gen/$(TAG)-properties.json \
-		$(if $(OVERRIDES),$(if $(shell [ -f "$(OVERRIDES)" ] && echo exists),--overrides $(OVERRIDES),),)
+		$(if $(OVERRIDES),$(if $(shell [ -f "$(OVERRIDES)" ] && echo exists),--overrides $(OVERRIDES),),) \
+		$(if $(CLOUD_SUPPORT),--cloud-support,)
 	@echo "✅ JSON generated at $(TOOL_ROOT)/gen/properties-output.json"
 	@echo "✅ Enhanced JSON (with overrides) generated at $(TOOL_ROOT)/gen/$(TAG)-properties.json"
 	@$(MAKE) generate-docs
@@ -66,11 +55,12 @@ venv: $(TOOL_ROOT)/requirements.txt
 	@if [ ! -d "$(VENV)" ]; then \
 	  echo "🐍 Creating virtual environment in $(VENV)..."; \
 	  python3 -m venv $(VENV); \
-	  $(VENV)/bin/pip install --upgrade pip --quiet; \
-	  $(VENV)/bin/pip install --no-cache-dir -r $<; \
 	else \
 	  echo "🐍 Virtual environment already exists at $(VENV)"; \
-	fi
+	fi; \
+	echo "🔄 Upgrading pip and installing requirements..."; \
+	$(VENV)/bin/pip install --upgrade pip --quiet; \
+	$(VENV)/bin/pip install --no-cache-dir -r $<;
 
 # --- Clean out all generated state ---
 clean:
@@ -106,7 +96,7 @@ treesitter:
 	  git fetch --tags -q && \
 	  git checkout -q v0.20.5
 	@echo "🔧 Generating parser in $(TREESITTER_DIR)…"
-	@cd "$(TREESITTER_DIR)" && npm install --silent && $(TREE_SITTER) generate
+	@cd "$(TREESITTER_DIR)" && export CFLAGS="-Wno-unused-but-set-variable" && npm install --silent && export CFLAGS="-Wno-unused-but-set-variable" && $(TREE_SITTER) generate
 
 # --- Install Node.js dependencies for Handlebars ---
 node-deps: