@redpanda-data/docs-extensions-and-macros 4.4.2 → 4.6.0

package/bin/doc-tools.js

@@ -1,16 +1,38 @@
 #!/usr/bin/env node
 
 const { execSync, spawnSync } = require('child_process');
+const os = require('os');
 const { Command } = require('commander');
 const path = require('path');
+const yaml = require('yaml');
 const fs = require('fs');
+const handlebars = require('handlebars');
+const { determineDocsBranch } = require('../cli-utils/self-managed-docs-branch.js');
+const fetchFromGithub = require('../tools/fetch-from-github.js');
+const { urlToXref } = require('../cli-utils/convert-doc-links.js');
+const { generateRpcnConnectorDocs } = require('../tools/redpanda-connect/generate-rpcn-connector-docs.js');
+const parseCSVConnectors = require('../tools/redpanda-connect/parse-csv-connectors.js');
+const { getAntoraValue, setAntoraValue } = require('../cli-utils/antora-utils');
+const {
+  getRpkConnectVersion,
+  printDeltaReport
+} = require('../tools/redpanda-connect/report-delta');
 
+/**
+ * Searches upward from a starting directory to locate the repository root.
+ *
+ * Traverses parent directories from the specified start path, returning the first directory containing either a `.git` folder or a `package.json` file. Exits the process with an error if no such directory is found.
+ *
+ * @param {string} [start] - The directory to begin the search from. Defaults to the current working directory.
+ * @returns {string} The absolute path to the repository root directory.
+ */
 function findRepoRoot(start = process.cwd()) {
   let dir = start;
   while (dir !== path.parse(dir).root) {
-    // marker could be a .git folder or package.json or anything you choose
-    if (fs.existsSync(path.join(dir, '.git')) ||
-        fs.existsSync(path.join(dir, 'package.json'))) {
+    if (
+      fs.existsSync(path.join(dir, '.git')) ||
+      fs.existsSync(path.join(dir, 'package.json'))
+    ) {
       return dir;
     }
     dir = path.dirname(dir);
@@ -21,107 +43,233 @@ function findRepoRoot(start = process.cwd()) {
 
 // --------------------------------------------------------------------
 // Dependency check functions
-// --------------------------------------------------------------------
-function checkDependency(command, versionArg, name, helpURL) {
-  try {
-    execSync(`${command} ${versionArg}`, { stdio: 'ignore' });
-  } catch (error) {
-    console.error(`Error: ${name} is required but not found or not working properly.
-Please install ${name} and try again.
-For more info, see: ${helpURL}`);
-    process.exit(1);
-  }
+
+/**
+ * Prints an error message to stderr and exits the process with a non-zero status.
+ *
+ * @param {string} msg - The error message to display before exiting.
+ */
+function fail(msg) {
+  console.error(`❌ ${msg}`);
+  process.exit(1);
 }
 
-function checkCommandExists(command) {
+/**
+ * Ensures that a specified command-line tool is installed and operational.
+ *
+ * Attempts to execute the tool with a version flag to verify its presence. If the tool is missing or fails to run, the process exits with an error message and optional installation hint.
+ *
+ * @param {string} cmd - The name of the tool to check (e.g., 'docker', 'helm-docs').
+ * @param {object} [opts] - Optional settings.
+ * @param {string} [opts.versionFlag='--version'] - The flag used to test the tool's execution.
+ * @param {string} [opts.help] - An optional hint or installation instruction shown on failure.
+ */
+function requireTool(cmd, { versionFlag = '--version', help = '' } = {}) {
   try {
-    execSync(`which ${command}`, { stdio: 'ignore' });
-    return true;
-  } catch (error) {
-    console.error(`Error: \`${command}\` is required but not found. Please install \`${command}\` and try again.`);
-    return false;
+    execSync(`${cmd} ${versionFlag}`, { stdio: 'ignore' });
+  } catch {
+    const hint = help ? `\n→ ${help}` : '';
+    fail(`'${cmd}' is required but not found.${hint}`);
   }
 }
 
-function checkMake() {
-  if (!checkCommandExists('make')) {
-    console.error('Error: `make` is required but not found. Please install `make` to use the automation Makefile. For help, see: https://www.google.com/search?q=how+to+install+make');
-    process.exit(1);
-  }
+/**
+ * Ensures that a command-line tool is installed by checking if it responds to a specified flag.
+ *
+ * @param {string} cmd - The name of the command-line tool to check.
+ * @param {string} [help] - Optional help text to display if the tool is not found.
+ * @param {string} [versionFlag='--version'] - The flag to use when checking if the tool is installed.
+ *
+ * @throws {Error} If the specified command is not found or does not respond to the specified flag.
+ */
+function requireCmd(cmd, help, versionFlag = '--version') {
+  requireTool(cmd, { versionFlag, help });
 }
 
-function checkPython() {
-  const candidates = ['python3', 'python'];
-  let found = false;
+// --------------------------------------------------------------------
+// Special validators
 
-  for (const cmd of candidates) {
+/**
+ * Ensures that Python with a minimum required version is installed and available in the system PATH.
+ *
+ * Checks for either `python3` or `python` executables and verifies that the version is at least the specified minimum (default: 3.10). Exits the process with an error message if the requirement is not met.
+ *
+ * @param {number} [minMajor=3] - Minimum required major version of Python.
+ * @param {number} [minMinor=10] - Minimum required minor version of Python.
+ */
+function requirePython(minMajor = 3, minMinor = 10) {
+  const candidates = ['python3', 'python'];
+  for (const p of candidates) {
     try {
-      const versionOutput = execSync(`${cmd} --version`, {
-        encoding: 'utf8',
-        stdio: ['pipe', 'pipe', 'ignore']
-      }).trim();
-      // versionOutput looks like "Python 3.x.y"
-      const versionString = versionOutput.split(' ')[1];
-      const [major, minor] = versionString.split('.').map(Number);
-      if (major > 3 || (major === 3 && minor >= 10)) {
-        found = true;
-        break;
-      } else {
-        console.error(`Error: Python 3.10 or higher is required. Detected version: ${versionString}`);
-        process.exit(1);
+      const out = execSync(`${p} --version`, { encoding: 'utf8' }).trim();
+      const [maj, min] = out.split(' ')[1].split('.').map(Number);
+      if (maj > minMajor || (maj === minMajor && min >= minMinor)) {
+        return; // success
       }
     } catch {
-      // this candidate didn’t exist or errored—try the next one
+      /* ignore & try next */
     }
   }
-  if (!found) {
-    console.error('Error: Python 3.10 or higher is required but not found.\nPlease install Python and ensure `python3 --version` or `python --version` returns at least 3.10: https://www.geeksforgeeks.org/how-to-install-python-on-mac/');
-    process.exit(1);
-  }
+  fail(
+    `Python ${minMajor}.${minMinor}+ not found or too old.
+→ Install from your package manager or https://python.org`
+  );
 }
 
-function checkCompiler() {
-  const gccInstalled = checkCommandExists('gcc');
-  const clangInstalled = checkCommandExists('clang');
-  if (!gccInstalled && !clangInstalled) {
-    console.error('Error: A C++ compiler (such as gcc or clang) is required but not found. Please install one: https://osxdaily.com/2023/05/02/how-install-gcc-mac/');
-    process.exit(1);
-  }
-}
-
-function checkDocker() {
-  checkDependency('docker', '--version', 'Docker', 'https://docs.docker.com/get-docker/');
+/**
+ * Ensures that the Docker CLI is installed and the Docker daemon is running.
+ *
+ * @throws {Error} If Docker is not installed or the Docker daemon is not running.
+ */
+function requireDockerDaemon() {
+  requireTool('docker', { help: 'https://docs.docker.com/get-docker/' });
   try {
     execSync('docker info', { stdio: 'ignore' });
-  } catch (error) {
-    console.error('Error: Docker daemon appears to be not running. Please start Docker.');
-    process.exit(1);
+  } catch {
+    fail('Docker daemon does not appear to be running. Please start Docker.');
   }
 }
 
+// --------------------------------------------------------------------
+// Grouped checks
+
+/**
+ * Ensures that required dependencies for generating CRD documentation are installed.
+ *
+ * Verifies the presence of the {@link git} and {@link crd-ref-docs} command-line tools, exiting the process with an error message if either is missing.
+ */
+function verifyCrdDependencies() {
+  requireCmd('git', 'Install Git: https://git-scm.com/downloads');
+  requireCmd(
+    'crd-ref-docs',
+    `
+The 'crd-ref-docs' command is required but was not found.
+
+To install it, follow these steps (for macOS):
+
+1. Determine your architecture:
+   Run: \`uname -m\`
+
+2. Download and install:
+
+- For Apple Silicon (M1/M2/M3):
+  curl -fLO https://github.com/elastic/crd-ref-docs/releases/download/v0.1.0/crd-ref-docs_0.1.0_Darwin_arm64.tar.gz
+  tar -xzf crd-ref-docs_0.1.0_Darwin_arm64.tar.gz
+  chmod +x crd-ref-docs
+  sudo mv crd-ref-docs /usr/local/bin/
+
+- For Intel (x86_64):
+  curl -fLO https://github.com/elastic/crd-ref-docs/releases/download/v0.1.0/crd-ref-docs_0.1.0_Darwin_x86_64.tar.gz
+  tar -xzf crd-ref-docs_0.1.0_Darwin_x86_64.tar.gz
+  chmod +x crd-ref-docs
+  sudo mv crd-ref-docs /usr/local/bin/
+
+For more details, visit: https://github.com/elastic/crd-ref-docs
+    `.trim()
+  );
+  requireCmd(
+    'go',
+    `
+The 'go' command (Golang) is required but was not found.
+
+To install it on macOS:
+
+Option 1: Install via Homebrew (recommended):
+  brew install go
+
+Option 2: Download directly from the official site:
+  1. Visit: https://go.dev/dl/
+  2. Download the appropriate installer for macOS.
+  3. Run the installer and follow the instructions.
+
+After installation, verify it works:
+  go version
+
+For more details, see: https://go.dev/doc/install
+    `.trim(),
+    'version'
+  );
+}
+
+/**
+ * Ensures that all required tools for Helm documentation generation are installed.
+ *
+ * Checks for the presence of `helm-docs`, `pandoc`, and `git`, exiting the process with an error if any are missing.
+ */
+function verifyHelmDependencies() {
+  requireCmd(
+    'helm-docs',
+    `
+The 'helm-docs' command is required but was not found.
+
+To install it, follow these steps (for macOS):
+
+1. Determine your architecture:
+   Run: \`uname -m\`
+
+2. Download and install:
+
+- For Apple Silicon (M1/M2/M3):
+  curl -fLO https://github.com/norwoodj/helm-docs/releases/download/v1.11.0/helm-docs_1.11.0_Darwin_arm64.tar.gz
+  tar -xzf helm-docs_1.11.0_Darwin_arm64.tar.gz
+  chmod +x helm-docs
+  sudo mv helm-docs /usr/local/bin/
+
+- For Intel (x86_64):
+  curl -fLO https://github.com/norwoodj/helm-docs/releases/download/v1.11.0/helm-docs_1.11.0_Darwin_x86_64.tar.gz
+  tar -xzf helm-docs_1.11.0_Darwin_x86_64.tar.gz
+  chmod +x helm-docs
+  sudo mv helm-docs /usr/local/bin/
+
+Alternatively, if you use Homebrew:
+  brew install norwoodj/tap/helm-docs
+
+For more details, visit: https://github.com/norwoodj/helm-docs
+    `.trim()
+  );
+  requireCmd('pandoc', 'brew install pandoc or https://pandoc.org');
+  requireCmd('git', 'Install Git: https://git-scm.com/downloads');
+}
+
+/**
+ * Ensures all dependencies required for generating property documentation are installed.
+ *
+ * Checks for the presence of `make`, Python 3.10 or newer, and at least one C++ compiler (`gcc` or `clang`). Exits the process with an error message if any dependency is missing.
+ */
 function verifyPropertyDependencies() {
-  checkMake();
-  checkPython();
-  checkCompiler();
+  requireCmd('make', 'Your OS package manager');
+  requirePython();
+  try {
+    execSync('gcc --version', { stdio: 'ignore' });
+  } catch {
+    try {
+      execSync('clang --version', { stdio: 'ignore' });
+    } catch {
+      fail('A C++ compiler (gcc or clang) is required.');
+    }
+  }
 }
 
+/**
+ * Ensures all required dependencies for generating Redpanda metrics documentation are installed.
+ *
+ * Verifies that Python 3.10+, `curl`, and `tar` are available, and that the Docker daemon is running.
+ *
+ * @throws {Error} If any required dependency is missing or the Docker daemon is not running.
+ */
 function verifyMetricsDependencies() {
-  checkPython();
-  if (!checkCommandExists('curl') || !checkCommandExists('tar')) {
-    // `checkCommandExists` already prints a helpful message.
-    process.exit(1);
-  }
-  checkDocker();
+  requirePython();
+  requireCmd('curl');
+  requireCmd('tar');
+  requireDockerDaemon();
 }
+
 // --------------------------------------------------------------------
 // Main CLI Definition
 // --------------------------------------------------------------------
 const programCli = new Command();
 
-programCli
-  .name('doc-tools')
-  .description('Redpanda Document Automation CLI')
-  .version('1.0.1');
+programCli.name('doc-tools').description('Redpanda Document Automation CLI').version('1.1.0');
 
 // Top-level commands.
 programCli
@@ -142,7 +290,7 @@ programCli
     try {
       await require('../tools/get-redpanda-version.js')(options);
     } catch (err) {
-      console.error(err);
+      console.error(`❌ ${err.message}`);
       process.exit(1);
     }
   });
@@ -156,41 +304,103 @@ programCli
     try {
       await require('../tools/get-console-version.js')(options);
     } catch (err) {
-      console.error(err);
+      console.error(`❌ ${err.message}`);
+      process.exit(1);
+    }
+  });
+
+programCli
+  .command('link-readme')
+  .description('Symlink a README.adoc into docs/modules/<module>/pages/')
+  .requiredOption('-s, --subdir <subdir>', 'Relative path to the lab project subdirectory')
+  .requiredOption('-t, --target <filename>', 'Name of the target AsciiDoc file in pages/')
+  .action((options) => {
+    const repoRoot = findRepoRoot();
+    const normalized = options.subdir.replace(/\/+$/, '');
+    const moduleName = normalized.split('/')[0];
+
+    const projectDir = path.join(repoRoot, normalized);
+    const pagesDir = path.join(repoRoot, 'docs', 'modules', moduleName, 'pages');
+    const sourceFile = path.join(projectDir, 'README.adoc');
+    const destLink = path.join(pagesDir, options.target);
+
+    if (!fs.existsSync(projectDir)) {
+      console.error(`❌ Project directory not found: ${projectDir}`);
+      process.exit(1);
+    }
+    if (!fs.existsSync(sourceFile)) {
+      console.error(`❌ README.adoc not found in ${projectDir}`);
+      process.exit(1);
+    }
+
+    fs.mkdirSync(pagesDir, { recursive: true });
+    const relPath = path.relative(pagesDir, sourceFile);
+
+    try {
+      if (fs.existsSync(destLink)) {
+        const stat = fs.lstatSync(destLink);
+        if (stat.isSymbolicLink()) fs.unlinkSync(destLink);
+        else fail(`Destination already exists and is not a symlink: ${destLink}`);
+      }
+      fs.symlinkSync(relPath, destLink);
+      console.log(`✅ Linked ${relPath} → ${destLink}`);
+    } catch (err) {
+      fail(`Failed to create symlink: ${err.message}`);
+    }
+  });
+
+programCli
+  .command('fetch')
+  .description('Fetch a file or directory from GitHub and save it locally')
+  .requiredOption('-o, --owner <owner>', 'GitHub repo owner or org')
+  .requiredOption('-r, --repo <repo>', 'GitHub repo name')
+  .requiredOption('-p, --remote-path <path>', 'Path in the repo to fetch')
+  .requiredOption('-d, --save-dir <dir>', 'Local directory to save into')
+  .option('-f, --filename <name>', 'Custom filename to save as')
+  .action(async (options) => {
+    try {
+      await fetchFromGithub(
+        options.owner,
+        options.repo,
+        options.remotePath,
+        options.saveDir,
+        options.filename
+      );
+      console.log(`✅ Fetched to ${options.saveDir}`);
+    } catch (err) {
+      console.error(`❌ ${err.message}`);
       process.exit(1);
     }
   });
 
 // Create an "automation" subcommand group.
-const automation = new Command('generate')
-  .description('Run docs automations (properties, metrics, and rpk docs generation)');
+const automation = new Command('generate').description('Run docs automations');
 
 // --------------------------------------------------------------------
-// Automation Subcommands: Delegate to a unified Bash script internally.
+// Automation subcommands
 // --------------------------------------------------------------------
 
 // Common options for both automation tasks.
 const commonOptions = {
-  tag: 'latest',
   dockerRepo: 'redpanda',
   consoleTag: 'latest',
-  consoleDockerRepo: 'console'
+  consoleDockerRepo: 'console',
 };
 
 function runClusterDocs(mode, tag, options) {
   const script = path.join(__dirname, '../cli-utils/generate-cluster-docs.sh');
-  const args   = [ mode, tag, options.dockerRepo, options.consoleTag, options.consoleDockerRepo ];
-  console.log(`Running ${script} with arguments: ${args.join(' ')}`);
-  const r = spawnSync('bash', [ script, ...args ], { stdio: 'inherit', shell: true });
+  const args = [mode, tag, options.dockerRepo, options.consoleTag, options.consoleDockerRepo];
+  console.log(`⏳ Running ${script} with arguments: ${args.join(' ')}`);
+  const r = spawnSync('bash', [script, ...args], { stdio: 'inherit', shell: true });
   if (r.status !== 0) process.exit(r.status);
 }
 
 // helper to diff two autogenerated directories
 function diffDirs(kind, oldTag, newTag) {
-  const oldDir  = path.join('autogenerated', oldTag, kind);
-  const newDir  = path.join('autogenerated', newTag, kind);
+  const oldDir = path.join('autogenerated', oldTag, kind);
+  const newDir = path.join('autogenerated', newTag, kind);
   const diffDir = path.join('autogenerated', 'diffs', kind, `${oldTag}_to_${newTag}`);
-  const patch   = path.join(diffDir, 'changes.patch');
+  const patch = path.join(diffDir, 'changes.patch');
 
   if (!fs.existsSync(oldDir)) {
     console.error(`❌ Cannot diff: missing ${oldDir}`);
@@ -215,11 +425,23 @@ function diffDirs(kind, oldTag, newTag) {
 
 automation
   .command('metrics-docs')
-  .description('Extract Redpanda metrics and generate JSON/AsciiDoc docs')
-  .option('--tag <tag>', 'Redpanda tag (default: latest)', commonOptions.tag)
-  .option('--docker-repo <repo>', '...', commonOptions.dockerRepo)
-  .option('--console-tag <tag>', '...', commonOptions.consoleTag)
-  .option('--console-docker-repo <repo>', '...', commonOptions.consoleDockerRepo)
+  .description('Generate JSON and AsciiDoc documentation for Redpanda metrics')
+  .requiredOption('-t, --tag <tag>', 'Redpanda version to use when starting Redpanda in Docker')
+  .option(
+    '--docker-repo <repo>',
+    'Docker repository to use when starting Redpanda in Docker',
+    commonOptions.dockerRepo
+  )
+  .option(
+    '--console-tag <tag>',
+    'Redpanda Console version to use when starting Redpanda Console in Docker',
+    commonOptions.consoleTag
+  )
+  .option(
+    '--console-docker-repo <repo>',
+    'Docker repository to use when starting Redpanda Console in Docker',
+    commonOptions.consoleDockerRepo
+  )
   .option('--diff <oldTag>', 'Also diff autogenerated metrics from <oldTag> → <tag>')
   .action((options) => {
     verifyMetricsDependencies();
@@ -245,21 +467,216 @@ automation
     process.exit(0);
   });
 
+  automation
+  .command('rpcn-connector-docs')
+  .description('Generate RPCN connector docs and diff changes since the last version')
+  .option('-d, --data-dir <path>', 'Directory where versioned connect JSON files live', path.resolve(process.cwd(), 'docs-data'))
+  .option('--old-data <path>', 'Optional override for old data file (for diff)')
+  .option('-f, --fetch-connectors', 'Fetch latest connector data using rpk')
+  .option('-m, --draft-missing', 'Generate full-doc drafts for connectors missing in output')
+  .option('--csv <path>', 'Path to connector metadata CSV file', 'internal/plugins/info.csv')
+  .option('--template-main <path>', 'Main Handlebars template', path.resolve(__dirname, '../tools/redpanda-connect/templates/connector.hbs'))
+  .option('--template-intro <path>', 'Intro section partial template', path.resolve(__dirname, '../tools/redpanda-connect/templates/intro.hbs'))
+  .option('--template-fields <path>', 'Fields section partial template', path.resolve(__dirname, '../tools/redpanda-connect/templates/fields-partials.hbs'))
+  .option('--template-examples <path>', 'Examples section partial template', path.resolve(__dirname, '../tools/redpanda-connect/templates/examples-partials.hbs'))
+  .option('--overrides <path>', 'Optional JSON file with overrides')
+  .action(async (options) => {
+    let success = true;
+    const dataDir = path.resolve(process.cwd(), options.dataDir);
+    fs.mkdirSync(dataDir, { recursive: true });
+
+    const timestamp = new Date().toISOString();
+
+    let newVersion;
+    let dataFile;
+    if (options.fetchConnectors) {
+      try {
+        execSync('rpk --version', { stdio: 'ignore' });
+        newVersion = getRpkConnectVersion();
+        const tmpFile = path.join(dataDir, `connect-${newVersion}.tmp.json`);
+        const finalFile = path.join(dataDir, `connect-${newVersion}.json`);
+
+        const fd = fs.openSync(tmpFile, 'w');
+        const r = spawnSync('rpk', ['connect', 'list', '--format', 'json-full'], { stdio: ['ignore', fd, 'inherit'] });
+        fs.closeSync(fd);
+
+        const rawJson = fs.readFileSync(tmpFile, 'utf8');
+        const parsed = JSON.parse(rawJson);
+        fs.writeFileSync(finalFile, JSON.stringify(parsed, null, 2));
+        fs.unlinkSync(tmpFile);
+        dataFile = finalFile;
+        console.log(`✅ Fetched and saved: ${finalFile}`);
+      } catch (err) {
+        console.error(`❌ Failed to fetch connectors: ${err.message}`);
+        success = false;
+      }
+    } else {
+      const candidates = fs.readdirSync(dataDir).filter(f => /^connect-\d+\.\d+\.\d+\.json$/.test(f));
+      if (candidates.length === 0) {
+        console.error('❌ No connect-<version>.json found. Use --fetch-connectors.');
+        process.exit(1);
+      }
+      candidates.sort();
+      dataFile = path.join(dataDir, candidates[candidates.length - 1]);
+      newVersion = candidates[candidates.length - 1].match(/connect-(\d+\.\d+\.\d+)\.json/)[1];
+    }
+
+    console.log('⏳ Generating connector partials...');
+    let partialsWritten, partialFiles, draftsWritten, draftFiles;
+
+    try {
+      const result = await generateRpcnConnectorDocs({
+        data: dataFile,
+        overrides: options.overrides,
+        template: options.templateMain,
+        templateIntro: options.templateIntro,
+        templateFields: options.templateFields,
+        templateExamples: options.templateExamples,
+        writeFullDrafts: false
+      });
+      partialsWritten = result.partialsWritten;
+      partialFiles    = result.partialFiles;
+    } catch (err) {
+      console.error(`❌ Failed to generate partials: ${err.message}`);
+      success = false;
+    }
+
+    if (options.draftMissing) {
+      console.log('⏳ Drafting missing connectors...');
+      try {
+        const connectorList = await parseCSVConnectors(options.csv, console);
+        const validConnectors = connectorList.filter(row => row.name && row.type);
+
+        const pagesRoot = path.resolve(process.cwd(), 'modules/components/pages');
+        const allMissing = validConnectors.filter(({ name, type }) => {
+          if (!name || !type) {
+            console.warn(`⚠️  Skipping invalid connector entry:`, { name, type });
+            return false;
+          }
+          const expected = path.join(pagesRoot, `${type}s`, `${name}.adoc`);
+          return !fs.existsSync(expected);
+        });
+
+        const missingConnectors = allMissing.filter(({ name }) => !name.includes('sql_driver'));
+
+        if (missingConnectors.length === 0) {
+          console.log('✅ All connectors (excluding sql_drivers) already have docs—nothing to draft.');
+        } else {
+          console.log(`⏳ Docs missing for ${missingConnectors.length} connectors:`);
+          missingConnectors.forEach(({ name, type }) => {
+            console.log(`   • ${type}/${name}`);
+          });
+          console.log('');
+
+          const rawData = fs.readFileSync(dataFile, 'utf8');
+          const dataObj = JSON.parse(rawData);
+
+          const filteredDataObj = {};
+          for (const [key, arr] of Object.entries(dataObj)) {
+            if (!Array.isArray(arr)) {
+              filteredDataObj[key] = arr;
+              continue;
+            }
+            filteredDataObj[key] = arr.filter(component =>
+              missingConnectors.some(m => m.name === component.name && `${m.type}s` === key)
+            );
+          }
+
+          const tempDataPath = path.join(dataDir, '._filtered_connect_data.json');
+          fs.writeFileSync(tempDataPath, JSON.stringify(filteredDataObj, null, 2), 'utf8');
+
+          const draftResult = await generateRpcnConnectorDocs({
+            data: tempDataPath,
+            overrides: options.overrides,
+            template: options.templateMain,
+            templateFields: options.templateFields,
+            templateExamples: options.templateExamples,
+            templateIntro: options.templateIntro,
+            writeFullDrafts: true
+          });
+
+          fs.unlinkSync(tempDataPath);
+          draftsWritten = draftResult.draftsWritten;
+          draftFiles   = draftResult.draftFiles;
+        }
+      } catch (err) {
+        console.error(`❌ Could not draft missing: ${err.message}`);
+        success = false;
+      }
+    }
+
+    let oldIndex = {};
+    if (options.oldData && fs.existsSync(options.oldData)) {
+      oldIndex = JSON.parse(fs.readFileSync(options.oldData, 'utf8'));
+    } else {
+      const oldVersion = getAntoraValue('asciidoc.attributes.latest-connect-version');
+      if (oldVersion) {
+        const oldPath = path.join(dataDir, `connect-${oldVersion}.json`);
+        if (fs.existsSync(oldPath)) {
+          oldIndex = JSON.parse(fs.readFileSync(oldPath, 'utf8'));
+        }
+      }
+    }
+
+    const newIndex = JSON.parse(fs.readFileSync(dataFile, 'utf8'));
+    printDeltaReport(oldIndex, newIndex);
+
+    function logCollapsed(label, filesArray, maxToShow = 10) {
+      console.log(`  • ${label}: ${filesArray.length} total`);
+      const sample = filesArray.slice(0, maxToShow);
+      sample.forEach(fp => console.log(`    – ${fp}`));
+      const remaining = filesArray.length - sample.length;
+      if (remaining > 0) {
+        console.log(`    … plus ${remaining} more`);
+      }
+      console.log('');
+    }
+
+    const wrote = setAntoraValue('asciidoc.attributes.latest-connect-version', newVersion);
+    if (wrote) {
+      console.log(`✅ Updated Antora version: ${newVersion}`);
+    }
+
+    console.log('📊 Generation Report:');
+    console.log(`   • Partial files: ${partialsWritten}`);
+    // Split “partials” into fields vs examples by checking the path substring.
+    const fieldsPartials   = partialFiles.filter(fp => fp.includes('/fields/'));
+    const examplesPartials = partialFiles.filter(fp => fp.includes('/examples/'));
+
+    // Show only up to 10 of each
+    logCollapsed('Fields partials', fieldsPartials,   10);
+    logCollapsed('Examples partials', examplesPartials, 10);
+
+    if (options.draftMissing) {
+      console.log(`   • Full drafts:   ${draftsWritten}`);
+      logCollapsed('Draft files', draftFiles, 5);
+    }
+
+    console.log('\n📄 Summary:');
+    console.log(`   • Run time: ${timestamp}`);
+    console.log(`   • Version used: ${newVersion}`);
+
+    process.exit(success ? 0 : 1);
+  });
+
 automation
   .command('property-docs')
-  .description('Extract properties from Redpanda source')
-  .option('--tag <tag>', 'Git tag or branch to extract from (default: dev)', 'dev')
+  .description('Generate JSON and AsciiDoc documentation for Redpanda configuration properties')
+  .option('--tag <tag>', 'Git tag or branch to extract from', 'dev')
   .option('--diff <oldTag>', 'Also diff autogenerated properties from <oldTag> → <tag>')
   .action((options) => {
     verifyPropertyDependencies();
 
     const newTag = options.tag;
     const oldTag = options.diff;
-    const cwd    = path.resolve(__dirname, '../tools/property-extractor');
-    const make   = (tag) => {
+    const cwd = path.resolve(__dirname, '../tools/property-extractor');
+    const make = (tag) => {
       console.log(`⏳ Building property docs for ${tag}…`);
       const r = spawnSync('make', ['build', `TAG=${tag}`], { cwd, stdio: 'inherit' });
-      if (r.error  ) { console.error(r.error); process.exit(1); }
+      if (r.error) {
+        console.error(`❌ ${r.error.message}`);
+        process.exit(1);
+      }
       if (r.status !== 0) process.exit(r.status);
     };
 
@@ -279,11 +696,23 @@ automation
 
 automation
   .command('rpk-docs')
-  .description('Generate documentation for rpk commands')
-  .option('--tag <tag>', 'Redpanda tag (default: latest)', commonOptions.tag)
-  .option('--docker-repo <repo>', '...', commonOptions.dockerRepo)
-  .option('--console-tag <tag>', '...', commonOptions.consoleTag)
-  .option('--console-docker-repo <repo>', '...', commonOptions.consoleDockerRepo)
+  .description('Generate AsciiDoc documentation for rpk CLI commands')
+  .requiredOption('-t, --tag <tag>', 'Redpanda version to use when starting Redpanda in Docker')
+  .option(
+    '--docker-repo <repo>',
+    'Docker repository to use when starting Redpanda in Docker',
+    commonOptions.dockerRepo
+  )
+  .option(
+    '--console-tag <tag>',
+    'Redpanda Console version to use when starting Redpanda Console in Docker',
+    commonOptions.consoleTag
+  )
+  .option(
+    '--console-docker-repo <repo>',
+    'Docker repository to use when starting Redpanda Console in Docker',
+    commonOptions.consoleDockerRepo
+  )
   .option('--diff <oldTag>', 'Also diff autogenerated rpk docs from <oldTag> → <tag>')
   .action((options) => {
     verifyMetricsDependencies();
@@ -309,70 +738,298 @@ automation
     process.exit(0);
   });
 
-  programCli
-  .command('link-readme')
-  .description('Symlink a README.adoc into docs/modules/<module>/pages/')
-  .requiredOption('-s, --subdir <subdir>', 'Relative path to the lab project subdirectory')
-  .requiredOption('-t, --target <filename>', 'Name of the target AsciiDoc file in pages/')
-  .action((options) => {
-    const repoRoot = findRepoRoot();
-    const normalized = options.subdir.replace(/\/+$/, '');
-    const moduleName = normalized.split('/')[0];
+automation
+  .command('helm-spec')
+  .description(
+    `Generate AsciiDoc documentation for one or more Helm charts (supports local dirs or GitHub URLs)`
+  )
+  .option(
+    '--chart-dir <dir|url>',
+    'Chart directory (contains Chart.yaml) or a root containing multiple charts, or a GitHub URL',
+    'https://github.com/redpanda-data/redpanda-operator/charts'
+  )
+  .requiredOption('-t, --tag <tag>', 'Branch or tag to clone when using a GitHub URL for the chart-dir')
+  .option('--readme <file>', 'Relative README.md path inside each chart dir', 'README.md')
+  .option('--output-dir <dir>', 'Where to write all generated AsciiDoc files', 'modules/reference/pages')
+  .option('--output-suffix <suffix>', 'Suffix to append to each chart name (including extension)', '-helm-spec.adoc')
+  .action((opts) => {
+    verifyHelmDependencies();
 
-    const projectDir = path.join(repoRoot, normalized);
-    const pagesDir   = path.join(repoRoot, 'docs', 'modules', moduleName, 'pages');
-    const sourceFile = path.join(projectDir, 'README.adoc');
-    const destLink   = path.join(pagesDir, options.target);
+    // Prepare chart-root (local or GitHub)
+    let root = opts.chartDir;
+    let tmpClone = null;
 
-    if (!fs.existsSync(projectDir)) {
-      console.error(`❌ Project directory not found: ${projectDir}`);
+    if (/^https?:\/\/github\.com\//.test(root)) {
+      if (!opts.tag) {
+        console.error('❌ When using a GitHub URL you must pass --tag');
+        process.exit(1);
+      }
+      const u = new URL(root);
+      const parts = u.pathname.replace(/\.git$/, '').split('/').filter(Boolean);
+      if (parts.length < 2) {
+        console.error(`❌ Invalid GitHub URL: ${root}`);
+        process.exit(1);
+      }
+      const [owner, repo, ...sub] = parts;
+      const repoUrl = `https://${u.host}/${owner}/${repo}.git`;
+      const ref = opts.tag;
+
+      console.log(`⏳ Verifying ${repoUrl}@${ref}…`);
+      const ok =
+        spawnSync(
+          'git',
+          ['ls-remote', '--exit-code', repoUrl, `refs/heads/${ref}`, `refs/tags/${ref}`],
+          { stdio: 'ignore' }
+        ).status === 0;
+      if (!ok) {
+        console.error(`❌ ${ref} not found on ${repoUrl}`);
+        process.exit(1);
+      }
+
+      tmpClone = fs.mkdtempSync(path.join(os.tmpdir(), 'helm-'));
+      console.log(`⏳ Cloning ${repoUrl}@${ref} → ${tmpClone}`);
+      if (
+        spawnSync('git', ['clone', '--depth', '1', '--branch', ref, repoUrl, tmpClone], {
+          stdio: 'inherit',
+        }).status !== 0
+      ) {
+        console.error('❌ git clone failed');
+        process.exit(1);
+      }
+      root = sub.length ? path.join(tmpClone, sub.join('/')) : tmpClone;
+    }
+
+    // Discover charts
+    if (!fs.existsSync(root) || !fs.statSync(root).isDirectory()) {
+      console.error(`❌ Chart root not found: ${root}`);
       process.exit(1);
     }
-    if (!fs.existsSync(sourceFile)) {
-      console.error(`❌ README.adoc not found in ${projectDir}`);
+    let charts = [];
+    if (fs.existsSync(path.join(root, 'Chart.yaml'))) {
+      charts = [root];
+    } else {
+      charts = fs
+        .readdirSync(root)
+        .map((n) => path.join(root, n))
+        .filter((p) => fs.existsSync(path.join(p, 'Chart.yaml')));
+    }
+    if (charts.length === 0) {
+      console.error(`❌ No charts found under: ${root}`);
       process.exit(1);
     }
 
-    fs.mkdirSync(pagesDir, { recursive: true });
-    const relPath = path.relative(pagesDir, sourceFile);
+    // Ensure output-dir exists
+    const outDir = path.resolve(opts.outputDir);
+    fs.mkdirSync(outDir, { recursive: true });
 
-    try {
-      fs.symlinkSync(relPath, destLink);
-      console.log(`✔️  Linked ${relPath} → ${destLink}`);
-    } catch (err) {
-      console.error(`❌ Failed to create symlink: ${err.message}`);
-      process.exit(1);
+    // Process each chart
+    for (const chartPath of charts) {
+      const name = path.basename(chartPath);
+      console.log(`⏳ Processing chart "${name}"…`);
+
+      // Regenerate README.md
+      console.log(`⏳ helm-docs in ${chartPath}`);
+      let r = spawnSync('helm-docs', { cwd: chartPath, stdio: 'inherit' });
+      if (r.status !== 0) process.exit(r.status);
+
+      // Convert Markdown → AsciiDoc
+      const md = path.join(chartPath, opts.readme);
+      if (!fs.existsSync(md)) {
+        console.error(`❌ README not found: ${md}`);
+        process.exit(1);
+      }
+      const outFile = path.join(outDir, `k-${name}${opts.outputSuffix}`);
+      console.log(`⏳ pandoc ${md} → ${outFile}`);
+      fs.mkdirSync(path.dirname(outFile), { recursive: true });
+      r = spawnSync('pandoc', [md, '-t', 'asciidoc', '-o', outFile], { stdio: 'inherit' });
+      if (r.status !== 0) process.exit(r.status);
+
+      // Post-process tweaks
+      let doc = fs.readFileSync(outFile, 'utf8');
+      const xrefRe = /https:\/\/docs\.redpanda\.com[^\s\]\[\)"]+(?:\[[^\]]*\])?/g;
+      doc = doc
+        .replace(/(\[\d+\])\]\./g, '$1\\].')
+        .replace(/^== # (.*)$/gm, '= $1')
+        .replace(/^== description: (.*)$/gm, ':description: $1')
+        .replace(xrefRe, (match) => {
+          let urlPart = match;
+          let bracketPart = '';
+          const m = match.match(/^([^\[]+)(\[[^\]]*\])$/);
+          if (m) {
+            urlPart = m[1];
+            bracketPart = m[2];
+          }
+          if (urlPart.endsWith('#')) {
+            return match;
+          }
+          try {
+            const xref = urlToXref(urlPart);
+            return bracketPart ? `${xref}${bracketPart}` : `${xref}[]`;
+          } catch (err) {
+            console.warn(`⚠️ urlToXref failed on ${urlPart}: ${err.message}`);
+            return match;
+          }
+        });
+      fs.writeFileSync(outFile, doc, 'utf8');
+
+      console.log(`✅ Wrote ${outFile}`);
     }
+
+    // Cleanup
+    if (tmpClone) fs.rmSync(tmpClone, { recursive: true, force: true });
   });
 
-programCli
-.command('fetch')
-.description('Fetch a file or directory from GitHub and save locally')
-.requiredOption('-o, --owner <owner>', 'GitHub repo owner or org')
-.requiredOption('-r, --repo <repo>', 'GitHub repo name')
-.requiredOption('-p, --remote-path <path>', 'Path in the repo to fetch')
-.requiredOption('-d, --save-dir <dir>', 'Local directory to save into')
-.option('-f, --filename <name>', 'Custom filename to save as')
-.action(async (options) => {
-  try {
-    const fetchFromGithub = await require('../tools/fetch-from-github.js');
-    // options.owner, options.repo, options.remotePath, options.saveDir, options.filename
+automation
+  .command('crd-spec')
+  .description('Generate Asciidoc documentation for Kubernetes CRD references')
+  .requiredOption('-t, --tag <operatorTag>', 'Operator release tag or branch, such as operator/v25.1.2')
+  .option(
+    '-s, --source-path <src>',
+    'CRD Go types dir or GitHub URL',
+    'https://github.com/redpanda-data/redpanda-operator/operator/api/redpanda/v1alpha2'
+  )
+  .option('-d, --depth <n>', 'How many levels deep', '10')
+  .option('--templates-dir <dir>', 'Asciidoctor templates dir', '.github/crd-config/templates/asciidoctor/operator')
+  .option('--output <file>', 'Where to write the generated AsciiDoc file', 'modules/reference/pages/k-crd.adoc')
+  .action(async (opts) => {
+    verifyCrdDependencies();
+
+    // Fetch upstream config
+    const configTmp = fs.mkdtempSync(path.join(os.tmpdir(), 'crd-config-'));
+    console.log(`⏳ Fetching crd-ref-docs-config.yaml from redpanda-operator@main…`);
     await fetchFromGithub(
-      options.owner,
-      options.repo,
-      options.remotePath,
-      options.saveDir,
-      options.filename
+      'redpanda-data',
+      'redpanda-operator',
+      'operator/crd-ref-docs-config.yaml',
+      configTmp,
+      'crd-ref-docs-config.yaml'
     );
-  } catch (err) {
-    console.error('❌', err.message);
-    process.exit(1);
-  }
-});
+    const configPath = path.join(configTmp, 'crd-ref-docs-config.yaml');
 
+    // Detect docs repo context
+    const repoRoot = findRepoRoot();
+    const pkg = JSON.parse(fs.readFileSync(path.join(repoRoot, 'package.json'), 'utf8'));
+    const inDocs =
+      pkg.name === 'redpanda-docs-playbook' ||
+      (pkg.repository && pkg.repository.url.includes('redpanda-data/docs'));
+    let docsBranch = null;
 
-// Attach the automation group to the main program.
-programCli.addCommand(automation);
+    if (!inDocs) {
+      console.warn('⚠️ Not inside redpanda-data/docs; skipping branch suggestion.');
+    } else {
+      try {
+        docsBranch = await determineDocsBranch(opts.tag);
+        console.log(`✅ Detected docs repo; you should commit to branch '${docsBranch}'.`);
+      } catch (err) {
+        console.error(`❌ Unable to determine docs branch: ${err.message}`);
+        process.exit(1);
+      }
+    }
 
-programCli.parse(process.argv);
+    // Validate templates
+    if (!fs.existsSync(opts.templatesDir)) {
+      console.error(`❌ Templates directory not found: ${opts.templatesDir}`);
+      process.exit(1);
+    }
+
+    // Prepare source (local folder or GitHub URL)
+    let localSrc = opts.sourcePath;
+    let tmpSrc;
+    if (/^https?:\/\/github\.com\//.test(opts.sourcePath)) {
+      const u = new URL(opts.sourcePath);
+      const parts = u.pathname.split('/').filter(Boolean);
+      if (parts.length < 2) {
+        console.error(`❌ Invalid GitHub URL: ${opts.sourcePath}`);
+        process.exit(1);
+      }
+      const [owner, repo, ...subpathParts] = parts;
+      const repoUrl = `https://${u.host}/${owner}/${repo}`;
+      const subpath = subpathParts.join('/');
+      console.log(`⏳ Verifying "${opts.tag}" in ${repoUrl}…`);
+      const ok =
+        spawnSync('git', ['ls-remote', '--exit-code', repoUrl, `refs/tags/${opts.tag}`, `refs/heads/${opts.tag}`], {
+          stdio: 'ignore',
+        }).status === 0;
+      if (!ok) {
+        console.error(`❌ Tag or branch "${opts.tag}" not found on ${repoUrl}`);
+        process.exit(1);
+      }
+      tmpSrc = fs.mkdtempSync(path.join(os.tmpdir(), 'crd-src-'));
+      console.log(`⏳ Cloning ${repoUrl}@${opts.tag} → ${tmpSrc}`);
+      if (
+        spawnSync('git', ['clone', '--depth', '1', '--branch', opts.tag, repoUrl, tmpSrc], {
+          stdio: 'inherit',
+        }).status !== 0
+      ) {
+        console.error(`❌ git clone failed`);
+        process.exit(1);
+      }
+      localSrc = subpath ? path.join(tmpSrc, subpath) : tmpSrc;
+      if (!fs.existsSync(localSrc)) {
+        console.error(`❌ Subdirectory not found in repo: ${subpath}`);
+        process.exit(1);
+      }
+    }
+
+    // Ensure output directory exists
+    const outputDir = path.dirname(opts.output);
+    if (!fs.existsSync(outputDir)) {
+      fs.mkdirSync(outputDir, { recursive: true });
+    }
 
+    // Run crd-ref-docs
+    const args = [
+      '--source-path',
+      localSrc,
+      '--max-depth',
+      opts.depth,
+      '--templates-dir',
+      opts.templatesDir,
+      '--config',
+      configPath,
+      '--renderer',
+      'asciidoctor',
+      '--output-path',
+      opts.output,
+    ];
+    console.log(`⏳ Running crd-ref-docs ${args.join(' ')}`);
+    if (spawnSync('crd-ref-docs', args, { stdio: 'inherit' }).status !== 0) {
+      console.error(`❌ crd-ref-docs failed`);
+      process.exit(1);
+    }
+
+    let doc = fs.readFileSync(opts.output, 'utf8');
+    const xrefRe = /https:\/\/docs\.redpanda\.com[^\s\]\[\)"]+(?:\[[^\]]*\])?/g;
+    doc = doc.replace(xrefRe, (match) => {
+      let urlPart = match;
+      let bracketPart = '';
+      const m = match.match(/^([^\[]+)(\[[^\]]*\])$/);
+      if (m) {
+        urlPart = m[1];
+        bracketPart = m[2];
+      }
+      if (urlPart.endsWith('#')) {
+        return match;
+      }
+      try {
+        const xref = urlToXref(urlPart);
+        return bracketPart ? `${xref}${bracketPart}` : `${xref}[]`;
+      } catch (err) {
+        console.warn(`⚠️ urlToXref failed on ${urlPart}: ${err.message}`);
+        return match;
+      }
+    });
+    fs.writeFileSync(opts.output, doc, 'utf8');
+
+    // Cleanup
+    if (tmpSrc) fs.rmSync(tmpSrc, { recursive: true, force: true });
+    fs.rmSync(configTmp, { recursive: true, force: true });
+
+    console.log(`✅ CRD docs generated at ${opts.output}`);
+    if (inDocs) {
+      console.log(`ℹ️ Don't forget to commit your changes on branch '${docsBranch}'.`);
+    }
+  });
+
+programCli.addCommand(automation);
+programCli.parse(process.argv);