@redpanda-data/docs-extensions-and-macros 4.3.0 → 4.4.0

package/bin/doc-tools.js

@@ -0,0 +1,328 @@
+#!/usr/bin/env node
+
+const { execSync, spawnSync } = require('child_process');
+const { Command } = require('commander');
+const path = require('path');
+const fs = require('fs');
+
+// --------------------------------------------------------------------
+// Dependency check functions
+// --------------------------------------------------------------------
+function checkDependency(command, versionArg, name, helpURL) {
+  try {
+    execSync(`${command} ${versionArg}`, { stdio: 'ignore' });
+  } catch (error) {
+    console.error(`Error: ${name} is required but not found or not working properly.
+Please install ${name} and try again.
+For more info, see: ${helpURL}`);
+    process.exit(1);
+  }
+}
+
+function checkCommandExists(command) {
+  try {
+    execSync(`which ${command}`, { stdio: 'ignore' });
+    return true;
+  } catch (error) {
+    console.error(`Error: \`${command}\` is required but not found. Please install \`${command}\` and try again.`);
+    return false;
+  }
+}
+
+function checkMake() {
+  if (!checkCommandExists('make')) {
+    console.error('Error: `make` is required but not found. Please install `make` to use the automation Makefile. For help, see: https://www.google.com/search?q=how+to+install+make');
+    process.exit(1);
+  }
+}
+
+function checkPython() {
+  const candidates = ['python3', 'python'];
+  let found = false;
+
+  for (const cmd of candidates) {
+    try {
+      const versionOutput = execSync(`${cmd} --version`, {
+        encoding: 'utf8',
+        stdio: ['pipe', 'pipe', 'ignore']
+      }).trim();
+      // versionOutput looks like "Python 3.x.y"
+      const versionString = versionOutput.split(' ')[1];
+      const [major, minor] = versionString.split('.').map(Number);
+      if (major > 3 || (major === 3 && minor >= 10)) {
+        found = true;
+        break;
+      } else {
+        console.error(`Error: Python 3.10 or higher is required. Detected version: ${versionString}`);
+        process.exit(1);
+      }
+    } catch {
+      // this candidate didn’t exist or errored—try the next one
+    }
+  }
+  if (!found) {
+    console.error('Error: Python 3.10 or higher is required but not found.\nPlease install Python and ensure `python3 --version` or `python --version` returns at least 3.10: https://www.geeksforgeeks.org/how-to-install-python-on-mac/');
+    process.exit(1);
+  }
+}
+
+function checkCompiler() {
+  const gccInstalled = checkCommandExists('gcc');
+  const clangInstalled = checkCommandExists('clang');
+  if (!gccInstalled && !clangInstalled) {
+    console.error('Error: A C++ compiler (such as gcc or clang) is required but not found. Please install one: https://osxdaily.com/2023/05/02/how-install-gcc-mac/');
+    process.exit(1);
+  }
+}
+
+function checkDocker() {
+  checkDependency('docker', '--version', 'Docker', 'https://docs.docker.com/get-docker/');
+  try {
+    execSync('docker info', { stdio: 'ignore' });
+  } catch (error) {
+    console.error('Error: Docker daemon appears to be not running. Please start Docker.');
+    process.exit(1);
+  }
+}
+
+function verifyPropertyDependencies() {
+  checkMake();
+  checkPython();
+  checkCompiler();
+}
+
+function verifyMetricsDependencies() {
+  checkPython();
+  if (!checkCommandExists('curl') || !checkCommandExists('tar')) {
+    // `checkCommandExists` already prints a helpful message.
+    process.exit(1);
+  }
+  checkDocker();
+}
+// --------------------------------------------------------------------
+// Main CLI Definition
+// --------------------------------------------------------------------
+const programCli = new Command();
+
+programCli
+  .name('doc-tools')
+  .description('Redpanda Document Automation CLI')
+  .version('1.0.0');
+
+// Top-level commands.
+programCli
+  .command('install-test-dependencies')
+  .description('Install packages for doc test workflows')
+  .action(() => {
+    const scriptPath = path.join(__dirname, '../cli-utils/install-test-dependencies.sh');
+    const result = spawnSync(scriptPath, { stdio: 'inherit', shell: true });
+    process.exit(result.status);
+  });
+
+programCli
+  .command('get-redpanda-version')
+  .description('Print the latest Redpanda version')
+  .option('--beta', 'Return the latest RC (beta) version if available')
+  .option('--from-antora', 'Read prerelease flag from local antora.yml')
+  .action(async (options) => {
+    try {
+      await require('../tools/get-redpanda-version.js')(options);
+    } catch (err) {
+      console.error(err);
+      process.exit(1);
+    }
+  });
+
+programCli
+  .command('get-console-version')
+  .description('Print the latest Console version')
+  .option('--beta', 'Return the latest beta version if available')
+  .option('--from-antora', 'Read prerelease flag from local antora.yml')
+  .action(async (options) => {
+    try {
+      await require('../tools/get-console-version.js')(options);
+    } catch (err) {
+      console.error(err);
+      process.exit(1);
+    }
+  });
+
+// Create an "automation" subcommand group.
+const automation = new Command('generate')
+  .description('Run docs automations (properties, metrics, and rpk docs generation)');
+
+// --------------------------------------------------------------------
+// Automation Subcommands: Delegate to a unified Bash script internally.
+// --------------------------------------------------------------------
+
+// Common options for both automation tasks.
+const commonOptions = {
+  tag: 'latest',
+  dockerRepo: 'redpanda',
+  consoleTag: 'latest',
+  consoleDockerRepo: 'console'
+};
+
+function runClusterDocs(mode, tag, options) {
+  const script = path.join(__dirname, '../cli-utils/generate-cluster-docs.sh');
+  const args   = [ mode, tag, options.dockerRepo, options.consoleTag, options.consoleDockerRepo ];
+  console.log(`Running ${script} with arguments: ${args.join(' ')}`);
+  const r = spawnSync('bash', [ script, ...args ], { stdio: 'inherit', shell: true });
+  if (r.status !== 0) process.exit(r.status);
+}
+
+// helper to diff two autogenerated directories
+function diffDirs(kind, oldTag, newTag) {
+  const oldDir  = path.join('autogenerated', oldTag, kind);
+  const newDir  = path.join('autogenerated', newTag, kind);
+  const diffDir = path.join('autogenerated', 'diffs', kind, `${oldTag}_to_${newTag}`);
+  const patch   = path.join(diffDir, 'changes.patch');
+
+  if (!fs.existsSync(oldDir)) {
+    console.error(`❌ Cannot diff: missing ${oldDir}`);
+    process.exit(1);
+  }
+  if (!fs.existsSync(newDir)) {
+    console.error(`❌ Cannot diff: missing ${newDir}`);
+    process.exit(1);
+  }
+
+  fs.mkdirSync(diffDir, { recursive: true });
+
+  const cmd = `diff -ru "${oldDir}" "${newDir}" > "${patch}" || true`;
+  const res = spawnSync(cmd, { stdio: 'inherit', shell: true });
+
+  if (res.error) {
+    console.error(`❌ diff failed: ${res.error.message}`);
+    process.exit(1);
+  }
+  console.log(`✅ Wrote patch: ${patch}`);
+}
+
+automation
+  .command('metrics-docs')
+  .description('Extract Redpanda metrics and generate JSON/AsciiDoc docs')
+  .option('--tag <tag>', 'Redpanda tag (default: latest)', commonOptions.tag)
+  .option('--docker-repo <repo>', '...', commonOptions.dockerRepo)
+  .option('--console-tag <tag>', '...', commonOptions.consoleTag)
+  .option('--console-docker-repo <repo>', '...', commonOptions.consoleDockerRepo)
+  .option('--diff <oldTag>', 'Also diff autogenerated metrics from <oldTag> → <tag>')
+  .action((options) => {
+    verifyMetricsDependencies();
+
+    const newTag = options.tag;
+    const oldTag = options.diff;
+
+    if (oldTag) {
+      const oldDir = path.join('autogenerated', oldTag, 'metrics');
+      if (!fs.existsSync(oldDir)) {
+        console.log(`⏳ Generating metrics docs for old tag ${oldTag}…`);
+        runClusterDocs('metrics', oldTag, options);
+      }
+    }
+
+    console.log(`⏳ Generating metrics docs for new tag ${newTag}…`);
+    runClusterDocs('metrics', newTag, options);
+
+    if (oldTag) {
+      diffDirs('metrics', oldTag, newTag);
+    }
+
+    process.exit(0);
+  });
+
+automation
+  .command('property-docs')
+  .description('Extract properties from Redpanda source')
+  .option('--tag <tag>', 'Git tag or branch to extract from (default: dev)', 'dev')
+  .option('--diff <oldTag>', 'Also diff autogenerated properties from <oldTag> → <tag>')
+  .action((options) => {
+    verifyPropertyDependencies();
+
+    const newTag = options.tag;
+    const oldTag = options.diff;
+    const cwd    = path.resolve(__dirname, '../tools/property-extractor');
+    const make   = (tag) => {
+      console.log(`⏳ Building property docs for ${tag}…`);
+      const r = spawnSync('make', ['build', `TAG=${tag}`], { cwd, stdio: 'inherit' });
+      if (r.error  ) { console.error(r.error); process.exit(1); }
+      if (r.status !== 0) process.exit(r.status);
+    };
+
+    if (oldTag) {
+      const oldDir = path.join('autogenerated', oldTag, 'properties');
+      if (!fs.existsSync(oldDir)) make(oldTag);
+    }
+
+    make(newTag);
+
+    if (oldTag) {
+      diffDirs('properties', oldTag, newTag);
+    }
+
+    process.exit(0);
+  });
+
+automation
+  .command('rpk-docs')
+  .description('Generate documentation for rpk commands')
+  .option('--tag <tag>', 'Redpanda tag (default: latest)', commonOptions.tag)
+  .option('--docker-repo <repo>', '...', commonOptions.dockerRepo)
+  .option('--console-tag <tag>', '...', commonOptions.consoleTag)
+  .option('--console-docker-repo <repo>', '...', commonOptions.consoleDockerRepo)
+  .option('--diff <oldTag>', 'Also diff autogenerated rpk docs from <oldTag> → <tag>')
+  .action((options) => {
+    verifyMetricsDependencies();
+
+    const newTag = options.tag;
+    const oldTag = options.diff;
+
+    if (oldTag) {
+      const oldDir = path.join('autogenerated', oldTag, 'rpk');
+      if (!fs.existsSync(oldDir)) {
+        console.log(`⏳ Generating rpk docs for old tag ${oldTag}…`);
+        runClusterDocs('rpk', oldTag, options);
+      }
+    }
+
+    console.log(`⏳ Generating rpk docs for new tag ${newTag}…`);
+    runClusterDocs('rpk', newTag, options);
+
+    if (oldTag) {
+      diffDirs('rpk', oldTag, newTag);
+    }
+
+    process.exit(0);
+  });
+
+programCli
+.command('fetch')
+.description('Fetch a file or directory from GitHub and save locally')
+.requiredOption('-o, --owner <owner>', 'GitHub repo owner or org')
+.requiredOption('-r, --repo <repo>', 'GitHub repo name')
+.requiredOption('-p, --remote-path <path>', 'Path in the repo to fetch')
+.requiredOption('-d, --save-dir <dir>', 'Local directory to save into')
+.option('-f, --filename <name>', 'Custom filename to save as')
+.action(async (options) => {
+  try {
+    const fetchFromGithub = await require('../tools/fetch-from-github.js');
+    // options.owner, options.repo, options.remotePath, options.saveDir, options.filename
+    await fetchFromGithub(
+      options.owner,
+      options.repo,
+      options.remotePath,
+      options.saveDir,
+      options.filename
+    );
+  } catch (err) {
+    console.error('❌', err.message);
+    process.exit(1);
+  }
+});
+
+
+// Attach the automation group to the main program.
+programCli.addCommand(automation);
+
+programCli.parse(process.argv);
+

package/cli-utils/add-caret-external-links.py

@@ -0,0 +1,68 @@
+import os
+import re
+
+# Define the regular expression pattern to match the links
+pattern = r'(https://[^\]]+)\[([^\]]+)\](?!\^)'
+
+# Function to process a single file
+def process_file(file_path):
+    with open(file_path, 'r', encoding='utf-8') as file:
+        content = file.read()
+
+    def replace_link(match):
+        link = match.group(1)
+        text = match.group(2)
+        if text.endswith('^'):
+            return match.group(0)  # No modification if caret is already present
+        else:
+            return f"{link}[{text}^]"
+
+    lines = content.split('\n')
+    updated_lines = []
+    for line in lines:
+        if re.search(pattern, line):
+            line = re.sub(pattern, replace_link, line)
+        updated_lines.append(line)
+
+    # Write the updated content back to the file
+-    with open(file_path, 'w', encoding='utf-8') as file:
+-        file.write('\n'.join(updated_lines))
++    try:
++        with open(file_path, 'w', encoding='utf-8') as file:
++            file.write('\n'.join(updated_lines))
++    except Exception as e:
++        print(f"Error writing to {file_path}: {e}")
++        return False
++    return True
+# Get the directory of the current script
+script_directory = os.path.dirname(os.path.abspath(__file__))
+
+# Construct the directory path for the 'modules' directory
+directory_path = os.path.join(script_directory, '..', 'modules')
+
+# List of excluded file paths (relative paths)
+# List of excluded file paths (relative paths)
+exclusion_list = [
+    os.path.join('reference', 'pages', 'redpanda-operator', 'crd.adoc'),
+    os.path.join('reference', 'pages', 'k-console-helm-spec.adoc'),
+    os.path.join('reference', 'pages', 'crd.adoc'),
+    os.path.join('reference', 'pages', 'k-redpanda-helm-spec.adoc'),
+    os.path.join('reference', 'partials', 'bundle-contents-k8s.adoc'),
+    os.path.join('reference', 'partials', 'bundle-contents-linux.adoc'),
+]
+
+# Function to process all .adoc files in a directory
+def process_directory(directory_path):
+    for root, _, files in os.walk(directory_path):
+        for file in files:
+            if file.endswith('.adoc'):
+                file_path = os.path.join(root, file)
+                relative_file_path = os.path.relpath(file_path, directory_path)
+                if relative_file_path not in exclusion_list:
+                    if process_file(file_path):
+                        print(f"Processed: {file_path}")
+                    else:
+                        print(f"Failed to process: {file_path}")
+
+# Call the function with the constructed directory path
+process_directory(directory_path)

package/cli-utils/beta-from-antora.js

@@ -0,0 +1,27 @@
+const fs = require('fs');
+const path = require('path');
+const yaml = require('js-yaml');
+
+/**
+ * Look for antora.yml in the current working directory
+ * (the project's root), load it if present, and return
+ * its `prerelease` value (boolean). If missing or on error,
+ * returns false.
+ */
+function getPrereleaseFromAntora() {
+  const antoraPath = path.join(process.cwd(), 'antora.yml');
+  if (!fs.existsSync(antoraPath)) {
+    return false;
+  }
+
+  try {
+    const fileContents = fs.readFileSync(antoraPath, 'utf8');
+    const antoraConfig = yaml.load(fileContents);
+    return antoraConfig.prerelease === true;
+  } catch (error) {
+    console.error('Error reading antora.yml:', error.message);
+    return false;
+  }
+}
+
+module.exports = { getPrereleaseFromAntora };

package/cli-utils/generate-cluster-docs.sh

@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Check if Docker is installed and running
+if ! command -v docker &> /dev/null; then
+  echo "❌ Docker is not installed or not in PATH. Please install Docker to continue."
+  exit 1
+fi
+
+# Check if Docker daemon is running
+if ! docker info &> /dev/null; then
+  echo "❌ Docker daemon is not running. Please start Docker to continue."
+  exit 1
+fi
+
+# Remember where we started so we can always come back
+ORIGINAL_PWD="$(pwd)"
+
+# All "cli-utils…" calls should be relative to this script’s dir
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+MODE="${1:-metrics}"
+TAG="${2:-latest}"
+DOCKER_REPO="${3:-redpanda}"
+CONSOLE_TAG="${4:-latest}"
+CONSOLE_REPO="${5:-console}"
+
+# if it's an RC tag, switch Docker repo
+shopt -s nocasematch
+if [[ "$TAG" =~ rc[0-9]+ ]]; then
+  DOCKER_REPO="redpanda-unstable"
+fi
+shopt -u nocasematch
+
+if [[ "$TAG" == "latest" ]]; then
+  MAJOR_MINOR="latest"
+else
+  MAJOR_MINOR="$(echo "$TAG" | sed -E 's/^v?([0-9]+\.[0-9]+).*$/\1/')"
+fi
+
+export REDPANDA_VERSION="$TAG"
+export REDPANDA_DOCKER_REPO="$DOCKER_REPO"
+export REDPANDA_CONSOLE_VERSION="$CONSOLE_TAG"
+export REDPANDA_CONSOLE_DOCKER_REPO="$CONSOLE_REPO"
+
+# Start up the cluster
+"$SCRIPT_DIR"/start-cluster.sh "$TAG"
+
+# Wait for it to settle
+if [[ "$MODE" == "metrics" ]]; then
+  echo "Waiting 300 seconds for metrics to be available…"
+  sleep 300
+else
+  echo "Waiting 30 seconds for cluster to be ready…"
+  sleep 30
+fi
+
+# Go back to where we were
+cd "$ORIGINAL_PWD"
+
+# Ensure Python venv (always create under cli-utils/venv)
+"$SCRIPT_DIR"/python-venv.sh \
+  "$SCRIPT_DIR"/venv \
+  "$SCRIPT_DIR"/../tools/metrics/requirements.txt
+
+if [[ "$MODE" == "metrics" ]]; then
+  "$SCRIPT_DIR"/venv/bin/python \
+    "$SCRIPT_DIR"/../tools/metrics/metrics.py \
+    "$TAG"
+else
+  "$SCRIPT_DIR"/venv/bin/python \
+    "$SCRIPT_DIR"/../tools/gen-rpk-ascii.py \
+    "$TAG"
+fi
+
+echo "✅ Redpanda cluster docs generated successfully!"
+
+# Tear down the cluster
+cd "$SCRIPT_DIR"/../docker-compose
+docker compose down --volumes
+
+# Return to the original directory
+cd "$ORIGINAL_PWD"

package/cli-utils/install-test-dependencies.sh

@@ -0,0 +1,158 @@
+#!/bin/bash
+set -e
+
+# Function to install Node.js
+install_node() {
+  if command -v node &>/dev/null; then
+    echo "Node.js is already installed. Version: $(node -v)"
+  else
+    echo "Installing Node.js..."
+    curl -fsSL https://fnm.vercel.app/install | bash || { echo "Failed to install fnm"; exit 1; }
+    # Load fnm into the current shell
+    export PATH=$HOME/.fnm:$PATH
+    eval "$(fnm env)" || { echo "Failed to load fnm environment"; exit 1; }
+    fnm install --lts || { echo "Failed to install Node.js"; exit 1; }
+    fnm use --lts || { echo "Failed to use Node.js"; exit 1; }
+    echo "Node.js version: $(node -v)"
+  fi
+}
+
+# Function to install Rust
+install_rust() {
+  if command -v rustc &>/dev/null; then
+    echo "Rust is already installed. Version: $(rustc --version)"
+  else
+    echo "Installing Rust..."
+    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y || { echo "Failed to install Rust"; exit 1; }
+    source $HOME/.cargo/env || { echo "Failed to load Rust environment"; exit 1; }
+    echo "Rust version: $(rustc --version)"
+  fi
+}
+
+# Function to check if expect and jq are installed and install them if they're not
+ensure_dependencies_installed() {
+
+    if ! command -v expect &> /dev/null; then
+        echo "Expect is not installed. Trying to install..."
+        missing_deps=1
+
+        # Detect OS
+        case "$(uname -s)" in
+            Linux)
+                echo "Detected Linux."
+                sudo apt-get update && sudo apt-get install expect -y || sudo yum install expect -y || { echo "Failed to install expect"; exit 1; }
+                ;;
+            Darwin)
+                echo "Detected macOS."
+                # Assumes Homebrew is installed. If not, it attempts to install Homebrew first.
+                if ! command -v brew &> /dev/null; then
+                    echo "Homebrew not found."
+                    exit 1
+                fi
+                brew install expect || { echo "Failed to install expect"; exit 1; }
+                ;;
+            *)
+                echo "Unsupported operating system. Please install expect manually."
+                exit 1
+                ;;
+        esac
+    fi
+
+    if ! command -v jq &> /dev/null; then
+        echo "jq is not installed. Trying to install..."
+
+        # Install jq based on OS
+        case "$(uname -s)" in
+            Linux)
+                sudo apt-get install jq -y || sudo yum install jq -y || { echo "Failed to install jq"; exit 1; }
+                ;;
+            Darwin)
+                brew install jq || { echo "Failed to install jq"; exit 1; }
+                ;;
+            *)
+                echo "Unsupported operating system. Please install jq manually."
+                exit 1
+                ;;
+        esac
+    fi
+
+    install_node
+    install_rust
+}
+
+# Ensure expect and jq are installed
+ensure_dependencies_installed
+
+# Function to check rpk installation and display its version
+check_rpk_installed() {
+    if command -v rpk &>/dev/null; then
+        echo "rpk is already installed. Version information:"
+        rpk --version
+        return 0
+    else
+        return 1
+    fi
+}
+
+# Determine OS and architecture
+OS="$(uname -s)"
+ARCH="$(uname -m)"
+
+# Check if rpk is already installed
+if check_rpk_installed; then
+    exit 0
+fi
+
+# Check if running on macOS and use Homebrew to install rpk
+if [ "${OS}" == "Darwin" ]; then
+    echo "Detected macOS. Attempting to install rpk using Homebrew..."
+
+    # Check if Homebrew is installed
+    if ! command -v brew &>/dev/null; then
+        echo "Homebrew not found."
+        exit 1
+    fi
+
+    # Install rpk
+    brew install redpanda-data/tap/redpanda || { echo "Failed to install rpk via Homebrew"; exit 1; }
+
+    # Verify installation
+    echo "rpk has been installed. Version information:"
+    rpk --version
+    exit 0
+fi
+
+# For Linux systems
+if [ "${OS}" == "Linux" ]; then
+    FILENAME="rpk-linux-amd64.zip"
+    URL_BASE="https://github.com/redpanda-data/redpanda/releases"
+
+    # Download latest version of rpk
+    echo "Downloading ${FILENAME}..."
+    curl -Lf --retry 3 -O "${URL_BASE}/latest/download/${FILENAME}" \
+        || { echo "Failed to download rpk"; exit 1; }
+
+    # Ensure the target directory exists
+    mkdir -p $HOME/.local/bin || { echo "Failed to create directory"; exit 1; }
+
+    # Unzip the rpk binary to the target directory
+    unzip -o "${FILENAME}" -d $HOME/.local/bin || { echo "Failed to unzip rpk"; exit 1; }
+
+    # Remove the downloaded archive
+    rm "${FILENAME}" || { echo "Failed to remove downloaded archive"; exit 1; }
+
+    # Add the target directory to PATH for the current session
+    export PATH=$HOME/.local/bin:$PATH
+
+    # Add the target directory to PATH for future sessions
+    echo 'export PATH=$HOME/.local/bin:$PATH' >> ~/.bashrc
+    source ~/.bashrc
+
+    # Verify installation
+    echo "rpk has been installed. Version information:"
+    rpk --version
+    exit 0
+fi
+
+echo "Unsupported operating system: ${OS}"
+exit 1

package/cli-utils/python-venv.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+#
+#   Create a new venv at $1 and install from $2
+
+set -euo pipefail
+
+VENV_DIR="${1:-venv}"
+REQ_FILE="${2:-requirements.txt}"
+
+echo "Recreating Python venv at $VENV_DIR..."
+rm -rf "$VENV_DIR"
+python3 -m venv "$VENV_DIR"
+"$VENV_DIR/bin/pip" install --upgrade pip --quiet
+
+if [[ -f "$REQ_FILE" ]]; then
+  echo "Installing $REQ_FILE..."
+  "$VENV_DIR/bin/pip" install -r "$REQ_FILE" --quiet
+else
+  echo "⚠️  Requirements file not found at $REQ_FILE"
+fi