@redpanda-data/docs-extensions-and-macros 4.5.0 → 4.6.0

package/bin/doc-tools.js

@@ -1,14 +1,22 @@
 #!/usr/bin/env node
 
 const { execSync, spawnSync } = require('child_process');
-const os                    = require('os');
+const os = require('os');
 const { Command } = require('commander');
 const path = require('path');
+const yaml = require('yaml');
 const fs = require('fs');
-const {determineDocsBranch} = require('../cli-utils/self-managed-docs-branch.js')
+const handlebars = require('handlebars');
+const { determineDocsBranch } = require('../cli-utils/self-managed-docs-branch.js');
 const fetchFromGithub = require('../tools/fetch-from-github.js');
 const { urlToXref } = require('../cli-utils/convert-doc-links.js');
-
+const { generateRpcnConnectorDocs } = require('../tools/redpanda-connect/generate-rpcn-connector-docs.js');
+const parseCSVConnectors = require('../tools/redpanda-connect/parse-csv-connectors.js');
+const { getAntoraValue, setAntoraValue } = require('../cli-utils/antora-utils');
+const {
+  getRpkConnectVersion,
+  printDeltaReport
+} = require('../tools/redpanda-connect/report-delta');
 
 /**
  * Searches upward from a starting directory to locate the repository root.
@@ -21,9 +29,10 @@ const { urlToXref } = require('../cli-utils/convert-doc-links.js');
 function findRepoRoot(start = process.cwd()) {
   let dir = start;
   while (dir !== path.parse(dir).root) {
-    // marker could be a .git folder or package.json or anything you choose
-    if (fs.existsSync(path.join(dir, '.git')) ||
-        fs.existsSync(path.join(dir, 'package.json'))) {
+    if (
+      fs.existsSync(path.join(dir, '.git')) ||
+      fs.existsSync(path.join(dir, 'package.json'))
+    ) {
       return dir;
     }
     dir = path.dirname(dir);
@@ -34,6 +43,7 @@ function findRepoRoot(start = process.cwd()) {
 
 // --------------------------------------------------------------------
 // Dependency check functions
+
 /**
  * Prints an error message to stderr and exits the process with a non-zero status.
  *
@@ -68,7 +78,7 @@ function requireTool(cmd, { versionFlag = '--version', help = '' } = {}) {
  *
  * @param {string} cmd - The name of the command-line tool to check.
  * @param {string} [help] - Optional help text to display if the tool is not found.
- * @param {string} [versionFlag='--help'] - The flag to use when checking if the tool is installed.
+ * @param {string} [versionFlag='--version'] - The flag to use when checking if the tool is installed.
  *
  * @throws {Error} If the specified command is not found or does not respond to the specified flag.
  */
@@ -76,9 +86,9 @@ function requireCmd(cmd, help, versionFlag = '--version') {
   requireTool(cmd, { versionFlag, help });
 }
 
-
 // --------------------------------------------------------------------
 // Special validators
+
 /**
  * Ensures that Python with a minimum required version is installed and available in the system PATH.
  *
@@ -87,7 +97,6 @@ function requireCmd(cmd, help, versionFlag = '--version') {
  * @param {number} [minMajor=3] - Minimum required major version of Python.
  * @param {number} [minMinor=10] - Minimum required minor version of Python.
  */
-
 function requirePython(minMajor = 3, minMinor = 10) {
   const candidates = ['python3', 'python'];
   for (const p of candidates) {
@@ -101,8 +110,10 @@ function requirePython(minMajor = 3, minMinor = 10) {
       /* ignore & try next */
     }
   }
-  fail(`Python ${minMajor}.${minMinor}+ not found or too old.
-→ Install from your package manager or https://python.org`);
+  fail(
+    `Python ${minMajor}.${minMinor}+ not found or too old.
+→ Install from your package manager or https://python.org`
+  );
 }
 
 /**
@@ -121,65 +132,64 @@ function requireDockerDaemon() {
 
 // --------------------------------------------------------------------
 // Grouped checks
+
 /**
  * Ensures that required dependencies for generating CRD documentation are installed.
  *
  * Verifies the presence of the {@link git} and {@link crd-ref-docs} command-line tools, exiting the process with an error message if either is missing.
  */
-
 function verifyCrdDependencies() {
-  requireCmd('git',             'Install Git: https://git-scm.com/downloads');
+  requireCmd('git', 'Install Git: https://git-scm.com/downloads');
   requireCmd(
     'crd-ref-docs',
     `
-  The 'crd-ref-docs' command is required but was not found.
+The 'crd-ref-docs' command is required but was not found.
 
-  To install it, follow these steps (for macOS):
+To install it, follow these steps (for macOS):
 
-  1. Determine your architecture:
-     Run: \`uname -m\`
+1. Determine your architecture:
+   Run: \`uname -m\`
 
-  2. Download and install:
+2. Download and install:
 
-  - For Apple Silicon (M1/M2/M3):
-    curl -fLO https://github.com/elastic/crd-ref-docs/releases/download/v0.1.0/crd-ref-docs_0.1.0_Darwin_arm64.tar.gz
-    tar -xzf crd-ref-docs_0.1.0_Darwin_arm64.tar.gz
-    chmod +x crd-ref-docs
-    sudo mv crd-ref-docs /usr/local/bin/
+- For Apple Silicon (M1/M2/M3):
+  curl -fLO https://github.com/elastic/crd-ref-docs/releases/download/v0.1.0/crd-ref-docs_0.1.0_Darwin_arm64.tar.gz
+  tar -xzf crd-ref-docs_0.1.0_Darwin_arm64.tar.gz
+  chmod +x crd-ref-docs
+  sudo mv crd-ref-docs /usr/local/bin/
 
-  - For Intel (x86_64):
-    curl -fLO https://github.com/elastic/crd-ref-docs/releases/download/v0.1.0/crd-ref-docs_0.1.0_Darwin_x86_64.tar.gz
-    tar -xzf crd-ref-docs_0.1.0_Darwin_x86_64.tar.gz
-    chmod +x crd-ref-docs
-    sudo mv crd-ref-docs /usr/local/bin/
+- For Intel (x86_64):
+  curl -fLO https://github.com/elastic/crd-ref-docs/releases/download/v0.1.0/crd-ref-docs_0.1.0_Darwin_x86_64.tar.gz
+  tar -xzf crd-ref-docs_0.1.0_Darwin_x86_64.tar.gz
+  chmod +x crd-ref-docs
+  sudo mv crd-ref-docs /usr/local/bin/
 
-  For more details, visit: https://github.com/elastic/crd-ref-docs
+For more details, visit: https://github.com/elastic/crd-ref-docs
     `.trim()
   );
   requireCmd(
     'go',
     `
-  The 'go' command (Golang) is required but was not found.
+The 'go' command (Golang) is required but was not found.
 
-  To install it on macOS:
+To install it on macOS:
 
-  Option 1: Install via Homebrew (recommended):
-    brew install go
+Option 1: Install via Homebrew (recommended):
+  brew install go
 
-  Option 2: Download directly from the official site:
-    1. Visit: https://go.dev/dl/
-    2. Download the appropriate installer for macOS.
-    3. Run the installer and follow the instructions.
+Option 2: Download directly from the official site:
+  1. Visit: https://go.dev/dl/
+  2. Download the appropriate installer for macOS.
+  3. Run the installer and follow the instructions.
 
-  After installation, verify it works:
-    go version
+After installation, verify it works:
+  go version
 
-  For more details, see: https://go.dev/doc/install
+For more details, see: https://go.dev/doc/install
     `.trim(),
-  'version'
+    'version'
   );
-
-  }
+}
 
 /**
  * Ensures that all required tools for Helm documentation generation are installed.
@@ -190,35 +200,35 @@ function verifyHelmDependencies() {
   requireCmd(
     'helm-docs',
     `
-  The 'helm-docs' command is required but was not found.
+The 'helm-docs' command is required but was not found.
 
-  To install it, follow these steps (for macOS):
+To install it, follow these steps (for macOS):
 
-  1. Determine your architecture:
-     Run: \`uname -m\`
+1. Determine your architecture:
+   Run: \`uname -m\`
 
-  2. Download and install:
+2. Download and install:
 
-  - For Apple Silicon (M1/M2/M3):
-    curl -fLO https://github.com/norwoodj/helm-docs/releases/download/v1.11.0/helm-docs_1.11.0_Darwin_arm64.tar.gz
-    tar -xzf helm-docs_1.11.0_Darwin_arm64.tar.gz
-    chmod +x helm-docs
-    sudo mv helm-docs /usr/local/bin/
+- For Apple Silicon (M1/M2/M3):
+  curl -fLO https://github.com/norwoodj/helm-docs/releases/download/v1.11.0/helm-docs_1.11.0_Darwin_arm64.tar.gz
+  tar -xzf helm-docs_1.11.0_Darwin_arm64.tar.gz
+  chmod +x helm-docs
+  sudo mv helm-docs /usr/local/bin/
 
-  - For Intel (x86_64):
-    curl -fLO https://github.com/norwoodj/helm-docs/releases/download/v1.11.0/helm-docs_1.11.0_Darwin_x86_64.tar.gz
-    tar -xzf helm-docs_1.11.0_Darwin_x86_64.tar.gz
-    chmod +x helm-docs
-    sudo mv helm-docs /usr/local/bin/
+- For Intel (x86_64):
+  curl -fLO https://github.com/norwoodj/helm-docs/releases/download/v1.11.0/helm-docs_1.11.0_Darwin_x86_64.tar.gz
+  tar -xzf helm-docs_1.11.0_Darwin_x86_64.tar.gz
+  chmod +x helm-docs
+  sudo mv helm-docs /usr/local/bin/
 
-  Alternatively, if you use Homebrew:
-    brew install norwoodj/tap/helm-docs
+Alternatively, if you use Homebrew:
+  brew install norwoodj/tap/helm-docs
 
-  For more details, visit: https://github.com/norwoodj/helm-docs
+For more details, visit: https://github.com/norwoodj/helm-docs
     `.trim()
   );
-    requireCmd('pandoc',     'brew install pandoc or https://pandoc.org');
-  requireCmd('git',             'install Git: https://git-scm.com/downloads');
+  requireCmd('pandoc', 'brew install pandoc or https://pandoc.org');
+  requireCmd('git', 'Install Git: https://git-scm.com/downloads');
 }
 
 /**
@@ -227,13 +237,16 @@ function verifyHelmDependencies() {
  * Checks for the presence of `make`, Python 3.10 or newer, and at least one C++ compiler (`gcc` or `clang`). Exits the process with an error message if any dependency is missing.
  */
 function verifyPropertyDependencies() {
-  requireCmd('make',       'your OS package manager');
+  requireCmd('make', 'Your OS package manager');
   requirePython();
-  // at least one compiler:
-  try { execSync('gcc --version', { stdio: 'ignore' }); }
-  catch {
-    try { execSync('clang --version', { stdio: 'ignore' }); }
-    catch { fail('A C++ compiler (gcc or clang) is required.'); }
+  try {
+    execSync('gcc --version', { stdio: 'ignore' });
+  } catch {
+    try {
+      execSync('clang --version', { stdio: 'ignore' });
+    } catch {
+      fail('A C++ compiler (gcc or clang) is required.');
+    }
   }
 }
 
@@ -250,15 +263,13 @@ function verifyMetricsDependencies() {
   requireCmd('tar');
   requireDockerDaemon();
 }
+
 // --------------------------------------------------------------------
 // Main CLI Definition
 // --------------------------------------------------------------------
 const programCli = new Command();
 
-programCli
-  .name('doc-tools')
-  .description('Redpanda Document Automation CLI')
-  .version('1.1.0');
+programCli.name('doc-tools').description('Redpanda Document Automation CLI').version('1.1.0');
 
 // Top-level commands.
 programCli
@@ -279,7 +290,7 @@ programCli
     try {
       await require('../tools/get-redpanda-version.js')(options);
     } catch (err) {
-      console.error(err);
+      console.error(`❌ ${err.message}`);
       process.exit(1);
     }
   });
@@ -293,12 +304,12 @@ programCli
     try {
       await require('../tools/get-console-version.js')(options);
     } catch (err) {
-      console.error(err);
+      console.error(`❌ ${err.message}`);
       process.exit(1);
     }
   });
 
-  programCli
+programCli
   .command('link-readme')
   .description('Symlink a README.adoc into docs/modules/<module>/pages/')
   .requiredOption('-s, --subdir <subdir>', 'Relative path to the lab project subdirectory')
@@ -309,9 +320,9 @@ programCli
     const moduleName = normalized.split('/')[0];
 
     const projectDir = path.join(repoRoot, normalized);
-    const pagesDir   = path.join(repoRoot, 'docs', 'modules', moduleName, 'pages');
+    const pagesDir = path.join(repoRoot, 'docs', 'modules', moduleName, 'pages');
     const sourceFile = path.join(projectDir, 'README.adoc');
-    const destLink   = path.join(pagesDir, options.target);
+    const destLink = path.join(pagesDir, options.target);
 
     if (!fs.existsSync(projectDir)) {
       console.error(`❌ Project directory not found: ${projectDir}`);
@@ -332,38 +343,38 @@ programCli
         else fail(`Destination already exists and is not a symlink: ${destLink}`);
       }
       fs.symlinkSync(relPath, destLink);
-      console.log(`✔️  Linked ${relPath} → ${destLink}`);
+      console.log(`✅ Linked ${relPath} → ${destLink}`);
     } catch (err) {
       fail(`Failed to create symlink: ${err.message}`);
     }
   });
 
 programCli
-.command('fetch')
-.description('Fetch a file or directory from GitHub and save it locally')
-.requiredOption('-o, --owner <owner>', 'GitHub repo owner or org')
-.requiredOption('-r, --repo <repo>', 'GitHub repo name')
-.requiredOption('-p, --remote-path <path>', 'Path in the repo to fetch')
-.requiredOption('-d, --save-dir <dir>', 'Local directory to save into')
-.option('-f, --filename <name>', 'Custom filename to save as')
-.action(async (options) => {
-  try {
-    await fetchFromGithub(
-      options.owner,
-      options.repo,
-      options.remotePath,
-      options.saveDir,
-      options.filename
-    );
-  } catch (err) {
-    console.error('❌', err.message);
-    process.exit(1);
-  }
-});
+  .command('fetch')
+  .description('Fetch a file or directory from GitHub and save it locally')
+  .requiredOption('-o, --owner <owner>', 'GitHub repo owner or org')
+  .requiredOption('-r, --repo <repo>', 'GitHub repo name')
+  .requiredOption('-p, --remote-path <path>', 'Path in the repo to fetch')
+  .requiredOption('-d, --save-dir <dir>', 'Local directory to save into')
+  .option('-f, --filename <name>', 'Custom filename to save as')
+  .action(async (options) => {
+    try {
+      await fetchFromGithub(
+        options.owner,
+        options.repo,
+        options.remotePath,
+        options.saveDir,
+        options.filename
+      );
+      console.log(`✅ Fetched to ${options.saveDir}`);
+    } catch (err) {
+      console.error(`❌ ${err.message}`);
+      process.exit(1);
+    }
+  });
 
 // Create an "automation" subcommand group.
-const automation = new Command('generate')
-  .description('Run docs automations (properties, metrics, rpk docs, and Kubernetes)');
+const automation = new Command('generate').description('Run docs automations');
 
 // --------------------------------------------------------------------
 // Automation subcommands
@@ -373,23 +384,23 @@ const automation = new Command('generate')
 const commonOptions = {
   dockerRepo: 'redpanda',
   consoleTag: 'latest',
-  consoleDockerRepo: 'console'
+  consoleDockerRepo: 'console',
 };
 
 function runClusterDocs(mode, tag, options) {
   const script = path.join(__dirname, '../cli-utils/generate-cluster-docs.sh');
-  const args   = [ mode, tag, options.dockerRepo, options.consoleTag, options.consoleDockerRepo ];
-  console.log(`Running ${script} with arguments: ${args.join(' ')}`);
-  const r = spawnSync('bash', [ script, ...args ], { stdio: 'inherit', shell: true });
+  const args = [mode, tag, options.dockerRepo, options.consoleTag, options.consoleDockerRepo];
+  console.log(`⏳ Running ${script} with arguments: ${args.join(' ')}`);
+  const r = spawnSync('bash', [script, ...args], { stdio: 'inherit', shell: true });
   if (r.status !== 0) process.exit(r.status);
 }
 
 // helper to diff two autogenerated directories
 function diffDirs(kind, oldTag, newTag) {
-  const oldDir  = path.join('autogenerated', oldTag, kind);
-  const newDir  = path.join('autogenerated', newTag, kind);
+  const oldDir = path.join('autogenerated', oldTag, kind);
+  const newDir = path.join('autogenerated', newTag, kind);
   const diffDir = path.join('autogenerated', 'diffs', kind, `${oldTag}_to_${newTag}`);
-  const patch   = path.join(diffDir, 'changes.patch');
+  const patch = path.join(diffDir, 'changes.patch');
 
   if (!fs.existsSync(oldDir)) {
     console.error(`❌ Cannot diff: missing ${oldDir}`);
@@ -415,11 +426,22 @@ function diffDirs(kind, oldTag, newTag) {
 automation
   .command('metrics-docs')
   .description('Generate JSON and AsciiDoc documentation for Redpanda metrics')
-  .requiredOption('-t, --tag <tag>',
-    'Redpanda version to use when starting Redpanda in Docker')
-  .option('--docker-repo <repo>', 'Docker repository to use when starting Redpanda in Docker', commonOptions.dockerRepo)
-  .option('--console-tag <tag>', 'Redpanda Console version to use when starting Redpanda Console in Docker', commonOptions.consoleTag)
-  .option('--console-docker-repo <repo>', 'Docker repository to use when starting Redpanda Console in Docker', commonOptions.consoleDockerRepo)
+  .requiredOption('-t, --tag <tag>', 'Redpanda version to use when starting Redpanda in Docker')
+  .option(
+    '--docker-repo <repo>',
+    'Docker repository to use when starting Redpanda in Docker',
+    commonOptions.dockerRepo
+  )
+  .option(
+    '--console-tag <tag>',
+    'Redpanda Console version to use when starting Redpanda Console in Docker',
+    commonOptions.consoleTag
+  )
+  .option(
+    '--console-docker-repo <repo>',
+    'Docker repository to use when starting Redpanda Console in Docker',
+    commonOptions.consoleDockerRepo
+  )
   .option('--diff <oldTag>', 'Also diff autogenerated metrics from <oldTag> → <tag>')
   .action((options) => {
     verifyMetricsDependencies();
@@ -445,6 +467,198 @@ automation
     process.exit(0);
   });
 
+  automation
+  .command('rpcn-connector-docs')
+  .description('Generate RPCN connector docs and diff changes since the last version')
+  .option('-d, --data-dir <path>', 'Directory where versioned connect JSON files live', path.resolve(process.cwd(), 'docs-data'))
+  .option('--old-data <path>', 'Optional override for old data file (for diff)')
+  .option('-f, --fetch-connectors', 'Fetch latest connector data using rpk')
+  .option('-m, --draft-missing', 'Generate full-doc drafts for connectors missing in output')
+  .option('--csv <path>', 'Path to connector metadata CSV file', 'internal/plugins/info.csv')
+  .option('--template-main <path>', 'Main Handlebars template', path.resolve(__dirname, '../tools/redpanda-connect/templates/connector.hbs'))
+  .option('--template-intro <path>', 'Intro section partial template', path.resolve(__dirname, '../tools/redpanda-connect/templates/intro.hbs'))
+  .option('--template-fields <path>', 'Fields section partial template', path.resolve(__dirname, '../tools/redpanda-connect/templates/fields-partials.hbs'))
+  .option('--template-examples <path>', 'Examples section partial template', path.resolve(__dirname, '../tools/redpanda-connect/templates/examples-partials.hbs'))
+  .option('--overrides <path>', 'Optional JSON file with overrides')
+  .action(async (options) => {
+    let success = true;
+    const dataDir = path.resolve(process.cwd(), options.dataDir);
+    fs.mkdirSync(dataDir, { recursive: true });
+
+    const timestamp = new Date().toISOString();
+
+    let newVersion;
+    let dataFile;
+    if (options.fetchConnectors) {
+      try {
+        execSync('rpk --version', { stdio: 'ignore' });
+        newVersion = getRpkConnectVersion();
+        const tmpFile = path.join(dataDir, `connect-${newVersion}.tmp.json`);
+        const finalFile = path.join(dataDir, `connect-${newVersion}.json`);
+
+        const fd = fs.openSync(tmpFile, 'w');
+        const r = spawnSync('rpk', ['connect', 'list', '--format', 'json-full'], { stdio: ['ignore', fd, 'inherit'] });
+        fs.closeSync(fd);
+
+        const rawJson = fs.readFileSync(tmpFile, 'utf8');
+        const parsed = JSON.parse(rawJson);
+        fs.writeFileSync(finalFile, JSON.stringify(parsed, null, 2));
+        fs.unlinkSync(tmpFile);
+        dataFile = finalFile;
+        console.log(`✅ Fetched and saved: ${finalFile}`);
+      } catch (err) {
+        console.error(`❌ Failed to fetch connectors: ${err.message}`);
+        success = false;
+      }
+    } else {
+      const candidates = fs.readdirSync(dataDir).filter(f => /^connect-\d+\.\d+\.\d+\.json$/.test(f));
+      if (candidates.length === 0) {
+        console.error('❌ No connect-<version>.json found. Use --fetch-connectors.');
+        process.exit(1);
+      }
+      candidates.sort();
+      dataFile = path.join(dataDir, candidates[candidates.length - 1]);
+      newVersion = candidates[candidates.length - 1].match(/connect-(\d+\.\d+\.\d+)\.json/)[1];
+    }
+
+    console.log('⏳ Generating connector partials...');
+    let partialsWritten, partialFiles, draftsWritten, draftFiles;
+
+    try {
+      const result = await generateRpcnConnectorDocs({
+        data: dataFile,
+        overrides: options.overrides,
+        template: options.templateMain,
+        templateIntro: options.templateIntro,
+        templateFields: options.templateFields,
+        templateExamples: options.templateExamples,
+        writeFullDrafts: false
+      });
+      partialsWritten = result.partialsWritten;
+      partialFiles    = result.partialFiles;
+    } catch (err) {
+      console.error(`❌ Failed to generate partials: ${err.message}`);
+      success = false;
+    }
+
+    if (options.draftMissing) {
+      console.log('⏳ Drafting missing connectors...');
+      try {
+        const connectorList = await parseCSVConnectors(options.csv, console);
+        const validConnectors = connectorList.filter(row => row.name && row.type);
+
+        const pagesRoot = path.resolve(process.cwd(), 'modules/components/pages');
+        const allMissing = validConnectors.filter(({ name, type }) => {
+          if (!name || !type) {
+            console.warn(`⚠️  Skipping invalid connector entry:`, { name, type });
+            return false;
+          }
+          const expected = path.join(pagesRoot, `${type}s`, `${name}.adoc`);
+          return !fs.existsSync(expected);
+        });
+
+        const missingConnectors = allMissing.filter(({ name }) => !name.includes('sql_driver'));
+
+        if (missingConnectors.length === 0) {
+          console.log('✅ All connectors (excluding sql_drivers) already have docs—nothing to draft.');
+        } else {
+          console.log(`⏳ Docs missing for ${missingConnectors.length} connectors:`);
+          missingConnectors.forEach(({ name, type }) => {
+            console.log(`   • ${type}/${name}`);
+          });
+          console.log('');
+
+          const rawData = fs.readFileSync(dataFile, 'utf8');
+          const dataObj = JSON.parse(rawData);
+
+          const filteredDataObj = {};
+          for (const [key, arr] of Object.entries(dataObj)) {
+            if (!Array.isArray(arr)) {
+              filteredDataObj[key] = arr;
+              continue;
+            }
+            filteredDataObj[key] = arr.filter(component =>
+              missingConnectors.some(m => m.name === component.name && `${m.type}s` === key)
+            );
+          }
+
+          const tempDataPath = path.join(dataDir, '._filtered_connect_data.json');
+          fs.writeFileSync(tempDataPath, JSON.stringify(filteredDataObj, null, 2), 'utf8');
+
+          const draftResult = await generateRpcnConnectorDocs({
+            data: tempDataPath,
+            overrides: options.overrides,
+            template: options.templateMain,
+            templateFields: options.templateFields,
+            templateExamples: options.templateExamples,
+            templateIntro: options.templateIntro,
+            writeFullDrafts: true
+          });
+
+          fs.unlinkSync(tempDataPath);
+          draftsWritten = draftResult.draftsWritten;
+          draftFiles   = draftResult.draftFiles;
+        }
+      } catch (err) {
+        console.error(`❌ Could not draft missing: ${err.message}`);
+        success = false;
+      }
+    }
+
+    let oldIndex = {};
+    if (options.oldData && fs.existsSync(options.oldData)) {
+      oldIndex = JSON.parse(fs.readFileSync(options.oldData, 'utf8'));
+    } else {
+      const oldVersion = getAntoraValue('asciidoc.attributes.latest-connect-version');
+      if (oldVersion) {
+        const oldPath = path.join(dataDir, `connect-${oldVersion}.json`);
+        if (fs.existsSync(oldPath)) {
+          oldIndex = JSON.parse(fs.readFileSync(oldPath, 'utf8'));
+        }
+      }
+    }
+
+    const newIndex = JSON.parse(fs.readFileSync(dataFile, 'utf8'));
+    printDeltaReport(oldIndex, newIndex);
+
+    function logCollapsed(label, filesArray, maxToShow = 10) {
+      console.log(`  • ${label}: ${filesArray.length} total`);
+      const sample = filesArray.slice(0, maxToShow);
+      sample.forEach(fp => console.log(`    – ${fp}`));
+      const remaining = filesArray.length - sample.length;
+      if (remaining > 0) {
+        console.log(`    … plus ${remaining} more`);
+      }
+      console.log('');
+    }
+
+    const wrote = setAntoraValue('asciidoc.attributes.latest-connect-version', newVersion);
+    if (wrote) {
+      console.log(`✅ Updated Antora version: ${newVersion}`);
+    }
+
+    console.log('📊 Generation Report:');
+    console.log(`   • Partial files: ${partialsWritten}`);
+    // Split “partials” into fields vs examples by checking the path substring.
+    const fieldsPartials   = partialFiles.filter(fp => fp.includes('/fields/'));
+    const examplesPartials = partialFiles.filter(fp => fp.includes('/examples/'));
+
+    // Show only up to 10 of each
+    logCollapsed('Fields partials', fieldsPartials,   10);
+    logCollapsed('Examples partials', examplesPartials, 10);
+
+    if (options.draftMissing) {
+      console.log(`   • Full drafts:   ${draftsWritten}`);
+      logCollapsed('Draft files', draftFiles, 5);
+    }
+
+    console.log('\n📄 Summary:');
+    console.log(`   • Run time: ${timestamp}`);
+    console.log(`   • Version used: ${newVersion}`);
+
+    process.exit(success ? 0 : 1);
+  });
+
 automation
   .command('property-docs')
   .description('Generate JSON and AsciiDoc documentation for Redpanda configuration properties')
@@ -455,11 +669,14 @@ automation
 
     const newTag = options.tag;
     const oldTag = options.diff;
-    const cwd    = path.resolve(__dirname, '../tools/property-extractor');
-    const make   = (tag) => {
+    const cwd = path.resolve(__dirname, '../tools/property-extractor');
+    const make = (tag) => {
       console.log(`⏳ Building property docs for ${tag}…`);
       const r = spawnSync('make', ['build', `TAG=${tag}`], { cwd, stdio: 'inherit' });
-      if (r.error  ) { console.error(r.error); process.exit(1); }
+      if (r.error) {
+        console.error(`❌ ${r.error.message}`);
+        process.exit(1);
+      }
       if (r.status !== 0) process.exit(r.status);
     };
 
@@ -480,11 +697,22 @@ automation
 automation
   .command('rpk-docs')
   .description('Generate AsciiDoc documentation for rpk CLI commands')
-  .requiredOption('-t, --tag <tag>',
-    'Redpanda version to use when starting Redpanda in Docker')
-  .option('--docker-repo <repo>', 'Docker repository to use when starting Redpanda in Docker', commonOptions.dockerRepo)
-  .option('--console-tag <tag>', 'Redpanda Console version to use when starting Redpanda Console in Docker', commonOptions.consoleTag)
-  .option('--console-docker-repo <repo>', 'Docker repository to use when starting Redpanda Console in Docker', commonOptions.consoleDockerRepo)
+  .requiredOption('-t, --tag <tag>', 'Redpanda version to use when starting Redpanda in Docker')
+  .option(
+    '--docker-repo <repo>',
+    'Docker repository to use when starting Redpanda in Docker',
+    commonOptions.dockerRepo
+  )
+  .option(
+    '--console-tag <tag>',
+    'Redpanda Console version to use when starting Redpanda Console in Docker',
+    commonOptions.consoleTag
+  )
+  .option(
+    '--console-docker-repo <repo>',
+    'Docker repository to use when starting Redpanda Console in Docker',
+    commonOptions.consoleDockerRepo
+  )
   .option('--diff <oldTag>', 'Also diff autogenerated rpk docs from <oldTag> → <tag>')
   .action((options) => {
     verifyMetricsDependencies();
@@ -512,181 +740,163 @@ automation
 
 automation
   .command('helm-spec')
-  .description(`Generate AsciiDoc documentation for one or more Helm charts (supports local dirs or GitHub URLs)`)
+  .description(
+    `Generate AsciiDoc documentation for one or more Helm charts (supports local dirs or GitHub URLs)`
+  )
   .option(
     '--chart-dir <dir|url>',
     'Chart directory (contains Chart.yaml) or a root containing multiple charts, or a GitHub URL',
     'https://github.com/redpanda-data/redpanda-operator/charts'
   )
-  .requiredOption('-t, --tag <tag>',
-    'Branch or tag to clone when using a GitHub URL for the chart-dir')
-  .option(
-    '--readme <file>',
-    'Relative README.md path inside each chart dir',
-    'README.md'
-  )
-  .option(
-    '--output-dir <dir>',
-    'Where to write all generated AsciiDoc files',
-    'modules/reference/pages'
-  )
-  .option(
-    '--output-suffix <suffix>',
-    'Suffix to append to each chart name (including extension)',
-    '-helm-spec.adoc'
-  )
-  .action(opts => {
-    verifyHelmDependencies()
+  .requiredOption('-t, --tag <tag>', 'Branch or tag to clone when using a GitHub URL for the chart-dir')
+  .option('--readme <file>', 'Relative README.md path inside each chart dir', 'README.md')
+  .option('--output-dir <dir>', 'Where to write all generated AsciiDoc files', 'modules/reference/pages')
+  .option('--output-suffix <suffix>', 'Suffix to append to each chart name (including extension)', '-helm-spec.adoc')
+  .action((opts) => {
+    verifyHelmDependencies();
 
-    // Prepare chart-root (local or GitHub) ───────────────────────
-    let root = opts.chartDir
-    let tmpClone = null
+    // Prepare chart-root (local or GitHub)
+    let root = opts.chartDir;
+    let tmpClone = null;
 
     if (/^https?:\/\/github\.com\//.test(root)) {
       if (!opts.tag) {
-        console.error('❌ When using a GitHub URL you must pass --tag')
-        process.exit(1)
+        console.error('❌ When using a GitHub URL you must pass --tag');
+        process.exit(1);
       }
-      const u     = new URL(root)
-      const parts = u.pathname.replace(/\.git$/, '').split('/').filter(Boolean)
+      const u = new URL(root);
+      const parts = u.pathname.replace(/\.git$/, '').split('/').filter(Boolean);
       if (parts.length < 2) {
-        console.error(`❌ Invalid GitHub URL: ${root}`)
-        process.exit(1)
+        console.error(`❌ Invalid GitHub URL: ${root}`);
+        process.exit(1);
       }
-      const [owner, repo, ...sub] = parts
-      const repoUrl = `https://${u.host}/${owner}/${repo}.git`
-      const ref     = opts.tag
-
-      console.log(`🔎 Verifying ${repoUrl}@${ref}…`)
-      const ok = spawnSync('git', [
-        'ls-remote','--exit-code', repoUrl,
-        `refs/heads/${ref}`, `refs/tags/${ref}`
-      ], { stdio:'ignore' }).status === 0
+      const [owner, repo, ...sub] = parts;
+      const repoUrl = `https://${u.host}/${owner}/${repo}.git`;
+      const ref = opts.tag;
+
+      console.log(`⏳ Verifying ${repoUrl}@${ref}…`);
+      const ok =
+        spawnSync(
+          'git',
+          ['ls-remote', '--exit-code', repoUrl, `refs/heads/${ref}`, `refs/tags/${ref}`],
+          { stdio: 'ignore' }
+        ).status === 0;
       if (!ok) {
-        console.error(`❌ ${ref} not found on ${repoUrl}`)
-        process.exit(1)
+        console.error(`❌ ${ref} not found on ${repoUrl}`);
+        process.exit(1);
       }
 
-      tmpClone = fs.mkdtempSync(path.join(os.tmpdir(), 'helm-'))
-      console.log(`⏳ Cloning ${repoUrl}@${ref} → ${tmpClone}`)
-      if (spawnSync('git', [
-        'clone','--depth','1','--branch',ref,
-        repoUrl, tmpClone
-      ], { stdio:'inherit' }).status !== 0) {
-        console.error('❌ git clone failed')
-        process.exit(1)
+      tmpClone = fs.mkdtempSync(path.join(os.tmpdir(), 'helm-'));
+      console.log(`⏳ Cloning ${repoUrl}@${ref} → ${tmpClone}`);
+      if (
+        spawnSync('git', ['clone', '--depth', '1', '--branch', ref, repoUrl, tmpClone], {
+          stdio: 'inherit',
+        }).status !== 0
+      ) {
+        console.error('❌ git clone failed');
+        process.exit(1);
       }
-      root = sub.length ? path.join(tmpClone, sub.join('/')) : tmpClone
+      root = sub.length ? path.join(tmpClone, sub.join('/')) : tmpClone;
     }
 
-    // Discover charts ─────────────────────────────────────────────
+    // Discover charts
     if (!fs.existsSync(root) || !fs.statSync(root).isDirectory()) {
-      console.error(`❌ Chart root not found: ${root}`)
-      process.exit(1)
+      console.error(`❌ Chart root not found: ${root}`);
+      process.exit(1);
     }
-    // single-chart?
-    let charts = []
-    if (fs.existsSync(path.join(root,'Chart.yaml'))) {
-      charts = [root]
+    let charts = [];
+    if (fs.existsSync(path.join(root, 'Chart.yaml'))) {
+      charts = [root];
     } else {
-      charts = fs.readdirSync(root)
-        .map(n => path.join(root,n))
-        .filter(p => fs.existsSync(path.join(p,'Chart.yaml')))
+      charts = fs
+        .readdirSync(root)
+        .map((n) => path.join(root, n))
+        .filter((p) => fs.existsSync(path.join(p, 'Chart.yaml')));
     }
     if (charts.length === 0) {
-      console.error(`❌ No charts found under: ${root}`)
-      process.exit(1)
+      console.error(`❌ No charts found under: ${root}`);
+      process.exit(1);
     }
 
-    // Ensure output-dir exists ────────────────────────────────────
-    const outDir = path.resolve(opts.outputDir)
-    fs.mkdirSync(outDir, { recursive: true })
+    // Ensure output-dir exists
+    const outDir = path.resolve(opts.outputDir);
+    fs.mkdirSync(outDir, { recursive: true });
 
-    // Process each chart ─────────────────────────────────────────
+    // Process each chart
     for (const chartPath of charts) {
-      const name = path.basename(chartPath)
-      console.log(`\n🔨 Processing chart "${name}"…`)
+      const name = path.basename(chartPath);
+      console.log(`⏳ Processing chart "${name}"…`);
 
       // Regenerate README.md
-      console.log(`  ⏳ helm-docs in ${chartPath}`)
-      let r = spawnSync('helm-docs', { cwd: chartPath, stdio: 'inherit' })
-      if (r.status !== 0) process.exit(r.status)
+      console.log(`⏳ helm-docs in ${chartPath}`);
+      let r = spawnSync('helm-docs', { cwd: chartPath, stdio: 'inherit' });
+      if (r.status !== 0) process.exit(r.status);
 
       // Convert Markdown → AsciiDoc
-      const md = path.join(chartPath, opts.readme)
+      const md = path.join(chartPath, opts.readme);
       if (!fs.existsSync(md)) {
-        console.error(`❌ README not found: ${md}`)
-        process.exit(1)
+        console.error(`❌ README not found: ${md}`);
+        process.exit(1);
       }
-      const outFile = path.join(outDir, `k-${name}${opts.outputSuffix}`)
-      console.log(`  ⏳ pandoc ${md} → ${outFile}`)
-      fs.mkdirSync(path.dirname(outFile), { recursive: true })
-      r = spawnSync('pandoc', [ md, '-t', 'asciidoc', '-o', outFile ], { stdio:'inherit' })
-      if (r.status !== 0) process.exit(r.status)
+      const outFile = path.join(outDir, `k-${name}${opts.outputSuffix}`);
+      console.log(`⏳ pandoc ${md} → ${outFile}`);
+      fs.mkdirSync(path.dirname(outFile), { recursive: true });
+      r = spawnSync('pandoc', [md, '-t', 'asciidoc', '-o', outFile], { stdio: 'inherit' });
+      if (r.status !== 0) process.exit(r.status);
 
       // Post-process tweaks
-      let doc = fs.readFileSync(outFile, 'utf8')
+      let doc = fs.readFileSync(outFile, 'utf8');
       const xrefRe = /https:\/\/docs\.redpanda\.com[^\s\]\[\)"]+(?:\[[^\]]*\])?/g;
       doc = doc
         .replace(/(\[\d+\])\]\./g, '$1\\].')
         .replace(/^== # (.*)$/gm, '= $1')
         .replace(/^== description: (.*)$/gm, ':description: $1')
-        .replace(xrefRe, match => {
-          // split off any [bracketed text]
+        .replace(xrefRe, (match) => {
           let urlPart = match;
           let bracketPart = '';
           const m = match.match(/^([^\[]+)(\[[^\]]*\])$/);
           if (m) {
-            urlPart = m[1];         // the pure URL
-            bracketPart = m[2];     // the “[text]”
+            urlPart = m[1];
+            bracketPart = m[2];
           }
-          // if it ends in “#” we leave it alone
           if (urlPart.endsWith('#')) {
             return match;
           }
           try {
             const xref = urlToXref(urlPart);
-            // re-attach the bracket text, or append empty [] for bare URLs
-            return bracketPart
-              ? `${xref}${bracketPart}`
-              : `${xref}[]`;
+            return bracketPart ? `${xref}${bracketPart}` : `${xref}[]`;
           } catch (err) {
             console.warn(`⚠️ urlToXref failed on ${urlPart}: ${err.message}`);
             return match;
           }
         });
-      fs.writeFileSync(outFile, doc, 'utf8')
+      fs.writeFileSync(outFile, doc, 'utf8');
 
-      console.log(`✅ Wrote ${outFile}`)
+      console.log(`✅ Wrote ${outFile}`);
     }
 
-    // Cleanup ───────────────────────────────────────────────────
-    if (tmpClone) fs.rmSync(tmpClone, { recursive: true, force: true })
-  })
+    // Cleanup
+    if (tmpClone) fs.rmSync(tmpClone, { recursive: true, force: true });
+  });
 
 automation
   .command('crd-spec')
   .description('Generate Asciidoc documentation for Kubernetes CRD references')
-  .requiredOption('-t, --tag <operatorTag>',
-    'Operator release tag or branch, such as operator/v25.1.2')
-  .option('-s, --source-path <src>',
+  .requiredOption('-t, --tag <operatorTag>', 'Operator release tag or branch, such as operator/v25.1.2')
+  .option(
+    '-s, --source-path <src>',
     'CRD Go types dir or GitHub URL',
-    'https://github.com/redpanda-data/redpanda-operator/operator/api/redpanda/v1alpha2')
-  .option('-d, --depth <n>',
-    'How many levels deep',
-    '10')
-  .option('--templates-dir <dir>',
-    'Asciidoctor templates dir',
-    '.github/crd-config/templates/asciidoctor/operator')
-  .option('--output <file>',
-    'Where to write the generated AsciiDoc file',
-    'modules/reference/pages/k-crd.adoc')
-  .action(async opts => {
+    'https://github.com/redpanda-data/redpanda-operator/operator/api/redpanda/v1alpha2'
+  )
+  .option('-d, --depth <n>', 'How many levels deep', '10')
+  .option('--templates-dir <dir>', 'Asciidoctor templates dir', '.github/crd-config/templates/asciidoctor/operator')
+  .option('--output <file>', 'Where to write the generated AsciiDoc file', 'modules/reference/pages/k-crd.adoc')
+  .action(async (opts) => {
     verifyCrdDependencies();
 
-    // Fetch upstream config ──────────────────────────────────────────
+    // Fetch upstream config
     const configTmp = fs.mkdtempSync(path.join(os.tmpdir(), 'crd-config-'));
-    console.log('🔧 Fetching crd-ref-docs-config.yaml from redpanda-operator@main…');
+    console.log(`⏳ Fetching crd-ref-docs-config.yaml from redpanda-operator@main…`);
     await fetchFromGithub(
       'redpanda-data',
       'redpanda-operator',
@@ -696,36 +906,37 @@ automation
     );
     const configPath = path.join(configTmp, 'crd-ref-docs-config.yaml');
 
-    // Detect docs repo context ───────────────────────────────────────
+    // Detect docs repo context
     const repoRoot = findRepoRoot();
-    const pkg      = JSON.parse(fs.readFileSync(path.join(repoRoot,'package.json'),'utf8'));
-    const inDocs   = pkg.name === 'redpanda-docs-playbook'
-                   || (pkg.repository && pkg.repository.url.includes('redpanda-data/docs'));
+    const pkg = JSON.parse(fs.readFileSync(path.join(repoRoot, 'package.json'), 'utf8'));
+    const inDocs =
+      pkg.name === 'redpanda-docs-playbook' ||
+      (pkg.repository && pkg.repository.url.includes('redpanda-data/docs'));
     let docsBranch = null;
 
     if (!inDocs) {
-      console.warn('⚠️  Not inside redpanda-data/docs; skipping branch suggestion.');
+      console.warn('⚠️ Not inside redpanda-data/docs; skipping branch suggestion.');
     } else {
       try {
         docsBranch = await determineDocsBranch(opts.tag);
-        console.log(`Detected docs repo; you should commit to branch '${docsBranch}'.`);
+        console.log(`✅ Detected docs repo; you should commit to branch '${docsBranch}'.`);
       } catch (err) {
         console.error(`❌ Unable to determine docs branch: ${err.message}`);
         process.exit(1);
       }
     }
 
-    // Validate templates ─────────────────────────────────────────────
+    // Validate templates
     if (!fs.existsSync(opts.templatesDir)) {
       console.error(`❌ Templates directory not found: ${opts.templatesDir}`);
       process.exit(1);
     }
 
-    // Prepare source (local folder or GitHub URL) ───────────────────
+    // Prepare source (local folder or GitHub URL)
     let localSrc = opts.sourcePath;
     let tmpSrc;
     if (/^https?:\/\/github\.com\//.test(opts.sourcePath)) {
-      const u     = new URL(opts.sourcePath);
+      const u = new URL(opts.sourcePath);
       const parts = u.pathname.split('/').filter(Boolean);
       if (parts.length < 2) {
         console.error(`❌ Invalid GitHub URL: ${opts.sourcePath}`);
@@ -734,97 +945,91 @@ automation
       const [owner, repo, ...subpathParts] = parts;
       const repoUrl = `https://${u.host}/${owner}/${repo}`;
       const subpath = subpathParts.join('/');
-      // Verify tag/branch exists
-      console.log(`🔎 Verifying "${opts.tag}" in ${repoUrl}…`);
-      const ok = spawnSync('git', [
-        'ls-remote','--exit-code', repoUrl,
-        `refs/tags/${opts.tag}`, `refs/heads/${opts.tag}`
-      ], { stdio:'ignore' }).status === 0;
+      console.log(`⏳ Verifying "${opts.tag}" in ${repoUrl}…`);
+      const ok =
+        spawnSync('git', ['ls-remote', '--exit-code', repoUrl, `refs/tags/${opts.tag}`, `refs/heads/${opts.tag}`], {
+          stdio: 'ignore',
+        }).status === 0;
       if (!ok) {
         console.error(`❌ Tag or branch "${opts.tag}" not found on ${repoUrl}`);
         process.exit(1);
       }
-      // Clone
       tmpSrc = fs.mkdtempSync(path.join(os.tmpdir(), 'crd-src-'));
       console.log(`⏳ Cloning ${repoUrl}@${opts.tag} → ${tmpSrc}`);
-      if (spawnSync('git', ['clone','--depth','1','--branch',opts.tag,repoUrl,tmpSrc],{stdio:'inherit'}).status !== 0) {
-        console.error('❌ git clone failed'); process.exit(1);
+      if (
+        spawnSync('git', ['clone', '--depth', '1', '--branch', opts.tag, repoUrl, tmpSrc], {
+          stdio: 'inherit',
+        }).status !== 0
+      ) {
+        console.error(`❌ git clone failed`);
+        process.exit(1);
       }
-      // Point at subfolder if any
       localSrc = subpath ? path.join(tmpSrc, subpath) : tmpSrc;
       if (!fs.existsSync(localSrc)) {
-        console.error(`❌ Subdirectory not found in repo: ${subpath}`); process.exit(1);
+        console.error(`❌ Subdirectory not found in repo: ${subpath}`);
+        process.exit(1);
       }
     }
 
-    // Ensure output directory exists ────────────────────────────────
+    // Ensure output directory exists
     const outputDir = path.dirname(opts.output);
     if (!fs.existsSync(outputDir)) {
       fs.mkdirSync(outputDir, { recursive: true });
     }
 
-    // Run crd-ref-docs ───────────────────────────────────────────────
+    // Run crd-ref-docs
     const args = [
-      '--source-path',   localSrc,
-      '--max-depth',     opts.depth,
-      '--templates-dir', opts.templatesDir,
-      '--config',        configPath,
-      '--renderer',      'asciidoctor',
-      '--output-path',   opts.output
+      '--source-path',
+      localSrc,
+      '--max-depth',
+      opts.depth,
+      '--templates-dir',
+      opts.templatesDir,
+      '--config',
+      configPath,
+      '--renderer',
+      'asciidoctor',
+      '--output-path',
+      opts.output,
     ];
     console.log(`⏳ Running crd-ref-docs ${args.join(' ')}`);
-    if (spawnSync('crd-ref-docs', args, { stdio:'inherit' }).status !== 0) {
-      console.error('❌ crd-ref-docs failed'); process.exit(1);
+    if (spawnSync('crd-ref-docs', args, { stdio: 'inherit' }).status !== 0) {
+      console.error(`❌ crd-ref-docs failed`);
+      process.exit(1);
     }
 
-    let doc = fs.readFileSync(opts.output, 'utf8')
-    // this regex matches:
-    //  - a docs.redpanda.com URL
-    //  - optionally followed immediately by [some text]
-    // It will not include trailing ] in the URL match itself.
+    let doc = fs.readFileSync(opts.output, 'utf8');
     const xrefRe = /https:\/\/docs\.redpanda\.com[^\s\]\[\)"]+(?:\[[^\]]*\])?/g;
-
-    doc = doc.replace(xrefRe, match => {
-      // split off any [bracketed text]
+    doc = doc.replace(xrefRe, (match) => {
       let urlPart = match;
       let bracketPart = '';
       const m = match.match(/^([^\[]+)(\[[^\]]*\])$/);
       if (m) {
-        urlPart = m[1];         // the pure URL
-        bracketPart = m[2];     // the “[text]”
+        urlPart = m[1];
+        bracketPart = m[2];
       }
-
-      // if it ends in “#” we leave it alone
       if (urlPart.endsWith('#')) {
         return match;
       }
-
       try {
         const xref = urlToXref(urlPart);
-        // re-attach the bracket text, or append empty [] for bare URLs
-        return bracketPart
-          ? `${xref}${bracketPart}`
-          : `${xref}[]`;
+        return bracketPart ? `${xref}${bracketPart}` : `${xref}[]`;
       } catch (err) {
         console.warn(`⚠️ urlToXref failed on ${urlPart}: ${err.message}`);
         return match;
       }
     });
-    fs.writeFileSync(opts.output, doc, 'utf8')
+    fs.writeFileSync(opts.output, doc, 'utf8');
 
-    // Cleanup ────────────────────────────────────────────────────────
-    if (tmpSrc)    fs.rmSync(tmpSrc,    { recursive: true, force: true });
+    // Cleanup
+    if (tmpSrc) fs.rmSync(tmpSrc, { recursive: true, force: true });
     fs.rmSync(configTmp, { recursive: true, force: true });
 
     console.log(`✅ CRD docs generated at ${opts.output}`);
     if (inDocs) {
-      console.log(`➡️  Don't forget to commit your changes on branch '${docsBranch}'.`);
+      console.log(`ℹ️ Don't forget to commit your changes on branch '${docsBranch}'.`);
     }
   });
 
-
-// Attach the automation group to the main program.
 programCli.addCommand(automation);
-
 programCli.parse(process.argv);
-