@redpanda-data/docs-extensions-and-macros 4.2.5 → 4.4.0

package/cli-utils/add-caret-external-links.py

@@ -0,0 +1,68 @@
+import os
+import re
+
+# Define the regular expression pattern to match the links
+pattern = r'(https://[^\]]+)\[([^\]]+)\](?!\^)'
+
+# Function to process a single file
+def process_file(file_path):
+    with open(file_path, 'r', encoding='utf-8') as file:
+        content = file.read()
+
+    def replace_link(match):
+        link = match.group(1)
+        text = match.group(2)
+        if text.endswith('^'):
+            return match.group(0)  # No modification if caret is already present
+        else:
+            return f"{link}[{text}^]"
+
+    lines = content.split('\n')
+    updated_lines = []
+    for line in lines:
+        if re.search(pattern, line):
+            line = re.sub(pattern, replace_link, line)
+        updated_lines.append(line)
+
+    # Write the updated content back to the file
+-    with open(file_path, 'w', encoding='utf-8') as file:
+-        file.write('\n'.join(updated_lines))
++    try:
++        with open(file_path, 'w', encoding='utf-8') as file:
++            file.write('\n'.join(updated_lines))
++    except Exception as e:
++        print(f"Error writing to {file_path}: {e}")
++        return False
++    return True
+# Get the directory of the current script
+script_directory = os.path.dirname(os.path.abspath(__file__))
+
+# Construct the directory path for the 'modules' directory
+directory_path = os.path.join(script_directory, '..', 'modules')
+
+# List of excluded file paths (relative paths)
+# List of excluded file paths (relative paths)
+exclusion_list = [
+    os.path.join('reference', 'pages', 'redpanda-operator', 'crd.adoc'),
+    os.path.join('reference', 'pages', 'k-console-helm-spec.adoc'),
+    os.path.join('reference', 'pages', 'crd.adoc'),
+    os.path.join('reference', 'pages', 'k-redpanda-helm-spec.adoc'),
+    os.path.join('reference', 'partials', 'bundle-contents-k8s.adoc'),
+    os.path.join('reference', 'partials', 'bundle-contents-linux.adoc'),
+]
+
+# Function to process all .adoc files in a directory
+def process_directory(directory_path):
+    for root, _, files in os.walk(directory_path):
+        for file in files:
+            if file.endswith('.adoc'):
+                file_path = os.path.join(root, file)
+                relative_file_path = os.path.relpath(file_path, directory_path)
+                if relative_file_path not in exclusion_list:
+                    if process_file(file_path):
+                        print(f"Processed: {file_path}")
+                    else:
+                        print(f"Failed to process: {file_path}")
+
+# Call the function with the constructed directory path
+process_directory(directory_path)

package/cli-utils/beta-from-antora.js

@@ -0,0 +1,27 @@
+const fs = require('fs');
+const path = require('path');
+const yaml = require('js-yaml');
+
+/**
+ * Look for antora.yml in the current working directory
+ * (the project's root), load it if present, and return
+ * its `prerelease` value (boolean). If missing or on error,
+ * returns false.
+ */
+function getPrereleaseFromAntora() {
+  const antoraPath = path.join(process.cwd(), 'antora.yml');
+  if (!fs.existsSync(antoraPath)) {
+    return false;
+  }
+
+  try {
+    const fileContents = fs.readFileSync(antoraPath, 'utf8');
+    const antoraConfig = yaml.load(fileContents);
+    return antoraConfig.prerelease === true;
+  } catch (error) {
+    console.error('Error reading antora.yml:', error.message);
+    return false;
+  }
+}
+
+module.exports = { getPrereleaseFromAntora };

package/cli-utils/generate-cluster-docs.sh

@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Check if Docker is installed and running
+if ! command -v docker &> /dev/null; then
+  echo "❌ Docker is not installed or not in PATH. Please install Docker to continue."
+  exit 1
+fi
+
+# Check if Docker daemon is running
+if ! docker info &> /dev/null; then
+  echo "❌ Docker daemon is not running. Please start Docker to continue."
+  exit 1
+fi
+
+# Remember where we started so we can always come back
+ORIGINAL_PWD="$(pwd)"
+
+# All "cli-utils…" calls should be relative to this script’s dir
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+MODE="${1:-metrics}"
+TAG="${2:-latest}"
+DOCKER_REPO="${3:-redpanda}"
+CONSOLE_TAG="${4:-latest}"
+CONSOLE_REPO="${5:-console}"
+
+# if it's an RC tag, switch Docker repo
+shopt -s nocasematch
+if [[ "$TAG" =~ rc[0-9]+ ]]; then
+  DOCKER_REPO="redpanda-unstable"
+fi
+shopt -u nocasematch
+
+if [[ "$TAG" == "latest" ]]; then
+  MAJOR_MINOR="latest"
+else
+  MAJOR_MINOR="$(echo "$TAG" | sed -E 's/^v?([0-9]+\.[0-9]+).*$/\1/')"
+fi
+
+export REDPANDA_VERSION="$TAG"
+export REDPANDA_DOCKER_REPO="$DOCKER_REPO"
+export REDPANDA_CONSOLE_VERSION="$CONSOLE_TAG"
+export REDPANDA_CONSOLE_DOCKER_REPO="$CONSOLE_REPO"
+
+# Start up the cluster
+"$SCRIPT_DIR"/start-cluster.sh "$TAG"
+
+# Wait for it to settle
+if [[ "$MODE" == "metrics" ]]; then
+  echo "Waiting 300 seconds for metrics to be available…"
+  sleep 300
+else
+  echo "Waiting 30 seconds for cluster to be ready…"
+  sleep 30
+fi
+
+# Go back to where we were
+cd "$ORIGINAL_PWD"
+
+# Ensure Python venv (always create under cli-utils/venv)
+"$SCRIPT_DIR"/python-venv.sh \
+  "$SCRIPT_DIR"/venv \
+  "$SCRIPT_DIR"/../tools/metrics/requirements.txt
+
+if [[ "$MODE" == "metrics" ]]; then
+  "$SCRIPT_DIR"/venv/bin/python \
+    "$SCRIPT_DIR"/../tools/metrics/metrics.py \
+    "$TAG"
+else
+  "$SCRIPT_DIR"/venv/bin/python \
+    "$SCRIPT_DIR"/../tools/gen-rpk-ascii.py \
+    "$TAG"
+fi
+
+echo "✅ Redpanda cluster docs generated successfully!"
+
+# Tear down the cluster
+cd "$SCRIPT_DIR"/../docker-compose
+docker compose down --volumes
+
+# Return to the original directory
+cd "$ORIGINAL_PWD"

package/cli-utils/install-test-dependencies.sh

@@ -0,0 +1,158 @@
+#!/bin/bash
+set -e
+
+# Function to install Node.js
+install_node() {
+  if command -v node &>/dev/null; then
+    echo "Node.js is already installed. Version: $(node -v)"
+  else
+    echo "Installing Node.js..."
+    curl -fsSL https://fnm.vercel.app/install | bash || { echo "Failed to install fnm"; exit 1; }
+    # Load fnm into the current shell
+    export PATH=$HOME/.fnm:$PATH
+    eval "$(fnm env)" || { echo "Failed to load fnm environment"; exit 1; }
+    fnm install --lts || { echo "Failed to install Node.js"; exit 1; }
+    fnm use --lts || { echo "Failed to use Node.js"; exit 1; }
+    echo "Node.js version: $(node -v)"
+  fi
+}
+
+# Function to install Rust
+install_rust() {
+  if command -v rustc &>/dev/null; then
+    echo "Rust is already installed. Version: $(rustc --version)"
+  else
+    echo "Installing Rust..."
+    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y || { echo "Failed to install Rust"; exit 1; }
+    source $HOME/.cargo/env || { echo "Failed to load Rust environment"; exit 1; }
+    echo "Rust version: $(rustc --version)"
+  fi
+}
+
+# Function to check if expect and jq are installed and install them if they're not
+ensure_dependencies_installed() {
+
+    if ! command -v expect &> /dev/null; then
+        echo "Expect is not installed. Trying to install..."
+        missing_deps=1
+
+        # Detect OS
+        case "$(uname -s)" in
+            Linux)
+                echo "Detected Linux."
+                sudo apt-get update && sudo apt-get install expect -y || sudo yum install expect -y || { echo "Failed to install expect"; exit 1; }
+                ;;
+            Darwin)
+                echo "Detected macOS."
+                # Assumes Homebrew is installed. If not, it attempts to install Homebrew first.
+                if ! command -v brew &> /dev/null; then
+                    echo "Homebrew not found."
+                    exit 1
+                fi
+                brew install expect || { echo "Failed to install expect"; exit 1; }
+                ;;
+            *)
+                echo "Unsupported operating system. Please install expect manually."
+                exit 1
+                ;;
+        esac
+    fi
+
+    if ! command -v jq &> /dev/null; then
+        echo "jq is not installed. Trying to install..."
+
+        # Install jq based on OS
+        case "$(uname -s)" in
+            Linux)
+                sudo apt-get install jq -y || sudo yum install jq -y || { echo "Failed to install jq"; exit 1; }
+                ;;
+            Darwin)
+                brew install jq || { echo "Failed to install jq"; exit 1; }
+                ;;
+            *)
+                echo "Unsupported operating system. Please install jq manually."
+                exit 1
+                ;;
+        esac
+    fi
+
+    install_node
+    install_rust
+}
+
+# Ensure expect and jq are installed
+ensure_dependencies_installed
+
+# Function to check rpk installation and display its version
+check_rpk_installed() {
+    if command -v rpk &>/dev/null; then
+        echo "rpk is already installed. Version information:"
+        rpk --version
+        return 0
+    else
+        return 1
+    fi
+}
+
+# Determine OS and architecture
+OS="$(uname -s)"
+ARCH="$(uname -m)"
+
+# Check if rpk is already installed
+if check_rpk_installed; then
+    exit 0
+fi
+
+# Check if running on macOS and use Homebrew to install rpk
+if [ "${OS}" == "Darwin" ]; then
+    echo "Detected macOS. Attempting to install rpk using Homebrew..."
+
+    # Check if Homebrew is installed
+    if ! command -v brew &>/dev/null; then
+        echo "Homebrew not found."
+        exit 1
+    fi
+
+    # Install rpk
+    brew install redpanda-data/tap/redpanda || { echo "Failed to install rpk via Homebrew"; exit 1; }
+
+    # Verify installation
+    echo "rpk has been installed. Version information:"
+    rpk --version
+    exit 0
+fi
+
+# For Linux systems
+if [ "${OS}" == "Linux" ]; then
+    FILENAME="rpk-linux-amd64.zip"
+    URL_BASE="https://github.com/redpanda-data/redpanda/releases"
+
+    # Download latest version of rpk
+    echo "Downloading ${FILENAME}..."
+    curl -Lf --retry 3 -O "${URL_BASE}/latest/download/${FILENAME}" \
+        || { echo "Failed to download rpk"; exit 1; }
+
+    # Ensure the target directory exists
+    mkdir -p $HOME/.local/bin || { echo "Failed to create directory"; exit 1; }
+
+    # Unzip the rpk binary to the target directory
+    unzip -o "${FILENAME}" -d $HOME/.local/bin || { echo "Failed to unzip rpk"; exit 1; }
+
+    # Remove the downloaded archive
+    rm "${FILENAME}" || { echo "Failed to remove downloaded archive"; exit 1; }
+
+    # Add the target directory to PATH for the current session
+    export PATH=$HOME/.local/bin:$PATH
+
+    # Add the target directory to PATH for future sessions
+    echo 'export PATH=$HOME/.local/bin:$PATH' >> ~/.bashrc
+    source ~/.bashrc
+
+    # Verify installation
+    echo "rpk has been installed. Version information:"
+    rpk --version
+    exit 0
+fi
+
+echo "Unsupported operating system: ${OS}"
+exit 1

package/cli-utils/python-venv.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+#
+#   Create a new venv at $1 and install from $2
+
+set -euo pipefail
+
+VENV_DIR="${1:-venv}"
+REQ_FILE="${2:-requirements.txt}"
+
+echo "Recreating Python venv at $VENV_DIR..."
+rm -rf "$VENV_DIR"
+python3 -m venv "$VENV_DIR"
+"$VENV_DIR/bin/pip" install --upgrade pip --quiet
+
+if [[ -f "$REQ_FILE" ]]; then
+  echo "Installing $REQ_FILE..."
+  "$VENV_DIR/bin/pip" install -r "$REQ_FILE" --quiet
+else
+  echo "⚠️  Requirements file not found at $REQ_FILE"
+fi

package/cli-utils/start-cluster.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Usage: start-cluster.sh <tag>
+TAG="${1:-latest}"
+
+# Where this script lives (cli-utils)
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# One level up is the package root, where we expect docker‑compose/
+PACKAGE_ROOT="$(cd "$SCRIPT_DIR"/.. && pwd)"
+QUICKSTART_DIR="$PACKAGE_ROOT/docker-compose"
+
+# Remember where the user called us from
+CALLER_PWD="$(pwd)"
+
+# Default quickstart version
+MAJOR_MINOR="latest"
+if [[ "$TAG" != "latest" ]]; then
+  MAJOR_MINOR="$(echo "$TAG" | sed -E 's/^v?([0-9]+\.[0-9]+).*$/\1/')"
+fi
+
+# Fetch quickstart into package root if needed
+if [[ ! -d "$QUICKSTART_DIR" ]]; then
+  echo "📥 Fetching Redpanda quickstart for ${MAJOR_MINOR}…"
+  if [[ "$TAG" == "latest" ]]; then
+	  curl -sSLf --retry 3 https://docs.redpanda.com/redpanda-quickstart.tar.gz \
+	  | tar -C "$PACKAGE_ROOT" -xzf -
+  else
+    curl -sSLf --retry 3 "https://docs.redpanda.com/${MAJOR_MINOR}-redpanda-quickstart.tar.gz" \
+      | tar -C "$PACKAGE_ROOT" -xzf -
+  fi
+
+  if [[ ! -d "$QUICKSTART_DIR" ]]; then
+    echo "❌ Expected '$QUICKSTART_DIR' but none was found after extraction."
+    exit 1
+  fi
+fi
+
+# Switch into the quickstart dir and (re)start the cluster
+cd "$QUICKSTART_DIR"
+
+if docker compose ps | grep -q Up; then
+  echo "🛑 Stopping existing cluster…"
+  docker compose down --volumes
+fi
+
+echo "▶️  Starting Redpanda cluster…"
+docker compose up -d
+
+# Return to original directory
+cd "$CALLER_PWD"
+echo "✅ Cluster is up (version: ${TAG})"

package/docker-compose/bootstrap.yml

@@ -0,0 +1,67 @@
+# =================================================================
+# This file defines initial cluster properties for a Redpanda cluster.
+# Some of these settings are intended for quickstart development and evaluation
+# and are not suitable for production environments.
+#
+# For more information on bootstrap files, see:
+# https://docs.redpanda.com/current/deploy/deployment-option/self-hosted/manual/production/production-deployment/#configure-a-bootstrap-file
+# =================================================================
+
+#
+# Enable SASL authentication for the Kafka and Admin APIs.
+# https://docs.redpanda.com/current/reference/properties/cluster-properties/#admin_api_require_auth
+admin_api_require_auth: true
+# At least one superuser is required to be able to create other SASL users
+# https://docs.redpanda.com/current/reference/properties/cluster-properties/#superusers
+superusers:
+  - superuser
+# https://docs.redpanda.com/current/reference/properties/cluster-properties/#enable_sasl
+enable_sasl: true
+# Allow topics to be created on first access.
+# https://docs.redpanda.com/current/reference/properties/cluster-properties/#auto_create_topics_enabled
+auto_create_topics_enabled: true
+# Enable data transforms.
+# https://docs.redpanda.com/current/develop/data-transforms/how-transforms-work/
+data_transforms_enabled: true
+# Enable audit logging (enterprise feature).
+# https://docs.redpanda.com/current/manage/audit-logging/
+audit_enabled: true
+# Enable Tiered Storage (enterprise feature).
+# https://docs.redpanda.com/current/manage/tiered-storage/
+cloud_storage_enabled: true
+cloud_storage_region: local
+cloud_storage_access_key: minio
+cloud_storage_secret_key: redpandaTieredStorage7
+cloud_storage_api_endpoint: minio
+cloud_storage_api_endpoint_port: 9000
+cloud_storage_disable_tls: true
+cloud_storage_bucket: redpanda
+cloud_storage_enable_remote_write: true
+cloud_storage_enable_remote_read: true
+# Forces segments to be uploaded to Tiered Storage faster for the purposes of the quickstart
+# https://docs.redpanda.com/current/reference/properties/object-storage-properties/#cloud_storage_segment_max_upload_interval_sec
+cloud_storage_segment_max_upload_interval_sec: 60
+# Continuous Data Balancing (enterprise feature) continuously monitors your node and rack availability and disk usage. This enables self-healing clusters that dynamically balance partitions, ensuring smooth operations and optimal cluster performance.
+# https://docs.redpanda.com/current/manage/cluster-maintenance/continuous-data-balancing/
+partition_autobalancing_mode: continuous
+# Enable Redpanda to collect consumer group metrics.
+# https://docs.redpanda.com/current/reference/properties/cluster-properties/#enable_consumer_group_metrics
+enable_consumer_group_metrics:
+  - "group"
+  - "partition"
+  - "consumer_lag"
+# Lower the interval for the autogeneration of consumer group metrics.
+# https://docs.redpanda.com/current/reference/properties/cluster-properties/#consumer_group_lag_collection_interval_sec
+consumer_group_lag_collection_interval_sec: 60
+# Enable Redpanda to collect host metrics.
+# https://docs.redpanda.com/current/reference/properties/cluster-properties/#enable_host_metrics
+enable_host_metrics: true
+# Enable for Iceberg metrics
+iceberg_enabled: true
+# Set up Iceberg REST catalog configuration
+iceberg_catalog_type: rest
+iceberg_rest_catalog_endpoint: http://catalog:8181
+# Credentials are required, but the catalog ignores them
+iceberg_rest_catalog_client_id: catalog
+iceberg_rest_catalog_client_secret: catalog123
+iceberg_catalog_commit_interval_ms: 5000