@redpanda-data/docs-extensions-and-macros 4.6.0 → 4.6.1

package/bin/doc-tools.js

@@ -269,7 +269,11 @@ function verifyMetricsDependencies() {
 // --------------------------------------------------------------------
 const programCli = new Command();
 
-programCli.name('doc-tools').description('Redpanda Document Automation CLI').version('1.1.0');
+const pkg = require('../package.json');
+programCli
+  .name('doc-tools')
+  .description('Redpanda Document Automation CLI')
+  .version(pkg.version);
 
 // Top-level commands.
 programCli

package/cli-utils/generate-cluster-docs.sh

@@ -1,23 +1,34 @@
 #!/usr/bin/env bash
 set -euo pipefail
+IFS=$'\n\t'
 
-# Check if Docker is installed and running
+###############################################################################
+# Pre-flight: Ensure Docker is available and running
+###############################################################################
 if ! command -v docker &> /dev/null; then
   echo "❌ Docker is not installed or not in PATH. Please install Docker to continue."
   exit 1
 fi
 
-# Check if Docker daemon is running
 if ! docker info &> /dev/null; then
   echo "❌ Docker daemon is not running. Please start Docker to continue."
   exit 1
 fi
 
-# Remember where we started so we can always come back
-ORIGINAL_PWD="$(pwd)"
+###############################################################################
+# Load overrides from an optional .env file in the current directory
+###############################################################################
+if [[ -f .env ]]; then
+  # shellcheck disable=SC2046
+  export $(grep -Ev '^#' .env | xargs)
+fi
 
-# All "cli-utils…" calls should be relative to this script’s dir
+###############################################################################
+# Environment setup
+###############################################################################
+ORIGINAL_PWD="$(pwd)"
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_NAME="${PROJECT_NAME:-redpanda_quickstart}"
 
 MODE="${1:-metrics}"
 TAG="${2:-latest}"
@@ -25,16 +36,15 @@ DOCKER_REPO="${3:-redpanda}"
 CONSOLE_TAG="${4:-latest}"
 CONSOLE_REPO="${5:-console}"
 
-# if it's an RC tag, switch Docker repo
+# Adjust Docker repo for release candidates
 shopt -s nocasematch
 if [[ "$TAG" =~ rc[0-9]+ ]]; then
   DOCKER_REPO="redpanda-unstable"
 fi
 shopt -u nocasematch
 
-if [[ "$TAG" == "latest" ]]; then
-  MAJOR_MINOR="latest"
-else
+MAJOR_MINOR="latest"
+if [[ "$TAG" != "latest" ]]; then
   MAJOR_MINOR="$(echo "$TAG" | sed -E 's/^v?([0-9]+\.[0-9]+).*$/\1/')"
 fi
 
@@ -43,41 +53,43 @@ export REDPANDA_DOCKER_REPO="$DOCKER_REPO"
 export REDPANDA_CONSOLE_VERSION="$CONSOLE_TAG"
 export REDPANDA_CONSOLE_DOCKER_REPO="$CONSOLE_REPO"
 
-# Start up the cluster
-"$SCRIPT_DIR"/start-cluster.sh "$TAG"
+###############################################################################
+# Start Redpanda cluster
+###############################################################################
+"$SCRIPT_DIR/start-cluster.sh" "$TAG"
 
-# Wait for it to settle
+# Wait for the cluster to settle
 if [[ "$MODE" == "metrics" ]]; then
-  echo "Waiting 300 seconds for metrics to be available…"
+  echo "⏳ Waiting 300 seconds for metrics to be available…"
   sleep 300
 else
-  echo "Waiting 30 seconds for cluster to be ready…"
+  echo "⏳ Waiting 30 seconds for cluster to be ready…"
   sleep 30
 fi
 
-# Go back to where we were
-cd "$ORIGINAL_PWD"
-
-# Ensure Python venv (always create under cli-utils/venv)
-"$SCRIPT_DIR"/python-venv.sh \
-  "$SCRIPT_DIR"/venv \
-  "$SCRIPT_DIR"/../tools/metrics/requirements.txt
+###############################################################################
+# Python virtual environment setup
+###############################################################################
+"$SCRIPT_DIR/python-venv.sh" \
+  "$SCRIPT_DIR/venv" \
+  "$SCRIPT_DIR/../tools/metrics/requirements.txt"
 
+###############################################################################
+# Run documentation generator
+###############################################################################
 if [[ "$MODE" == "metrics" ]]; then
-  "$SCRIPT_DIR"/venv/bin/python \
-    "$SCRIPT_DIR"/../tools/metrics/metrics.py \
-    "$TAG"
+  "$SCRIPT_DIR/venv/bin/python" \
+    "$SCRIPT_DIR/../tools/metrics/metrics.py" "$TAG"
 else
-  "$SCRIPT_DIR"/venv/bin/python \
-    "$SCRIPT_DIR"/../tools/gen-rpk-ascii.py \
-    "$TAG"
+  "$SCRIPT_DIR/venv/bin/python" \
+    "$SCRIPT_DIR/../tools/gen-rpk-ascii.py" "$TAG"
 fi
 
 echo "✅ $MODE docs generated successfully!"
 
 # Tear down the cluster
 cd "$SCRIPT_DIR"/../docker-compose
-docker compose down --volumes
+docker compose -p "$PROJECT_NAME" down --volumes
 
 # Return to the original directory
-cd "$ORIGINAL_PWD"
+cd "$ORIGINAL_PWD" || exit 1

package/cli-utils/start-cluster.sh

@@ -1,63 +1,103 @@
 #!/usr/bin/env bash
 set -euo pipefail
+IFS=$'\n\t'
 
-# Usage: start-cluster.sh <tag>
-TAG="${1:-latest}"
+###############################################################################
+# Set START_CLUSTER_LOG=/path/to/logfile to capture output
+###############################################################################
+if [[ -n "${START_CLUSTER_LOG:-}" ]]; then
+  exec > >(tee -a "$START_CLUSTER_LOG") 2>&1
+fi
 
-# Where this script lives (cli-utils)
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+###############################################################################
+# Prevent concurrent runs with a simple lockfile
+###############################################################################
+LOCKFILE="/tmp/start-cluster.lock"
+if [[ -e "$LOCKFILE" ]]; then
+  echo "❗ Another start-cluster.sh is already running. Remove $LOCKFILE if stale." >&2
+  exit 1
+fi
+trap 'rm -f "$LOCKFILE"' EXIT
+touch "$LOCKFILE"
 
-# One level up is the package root, where we expect docker‑compose/
-PACKAGE_ROOT="$(cd "$SCRIPT_DIR"/.. && pwd)"
-QUICKSTART_DIR="$PACKAGE_ROOT/docker-compose"
+###############################################################################
+# Input parameters and defaults
+###############################################################################
+TAG="${1:-${CLUSTER_TAG:-latest}}"
+PROJECT_NAME="${PROJECT_NAME:-redpanda_quickstart}"
 
-# Remember where the user called us from
+###############################################################################
+# Directory discovery
+###############################################################################
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PACKAGE_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+QUICKSTART_DIR="$PACKAGE_ROOT/docker-compose"
 CALLER_PWD="$(pwd)"
 
-# Default quickstart version
+###############################################################################
+# Determine major.minor for quickstart override
+###############################################################################
 MAJOR_MINOR="latest"
 if [[ "$TAG" != "latest" ]]; then
   MAJOR_MINOR="$(echo "$TAG" | sed -E 's/^v?([0-9]+\.[0-9]+).*$/\1/')"
 fi
 
-# Conditionally override quickstart dir for >= 25.1
 if [[ "$MAJOR_MINOR" =~ ^([0-9]+)\.([0-9]+)$ ]]; then
   major="${BASH_REMATCH[1]}"
   minor="${BASH_REMATCH[2]}"
-
   if (( major > 25 )) || (( major == 25 && minor >= 1 )); then
     QUICKSTART_DIR="$PACKAGE_ROOT/docker-compose/25.1"
   fi
 fi
 
-# Fetch quickstart into package root if needed
+###############################################################################
+# Fetch quickstart bundle if missing
+###############################################################################
 if [[ ! -d "$QUICKSTART_DIR" ]]; then
   echo "📥 Fetching Redpanda quickstart for ${MAJOR_MINOR}…"
+  url="https://docs.redpanda.com"
   if [[ "$TAG" == "latest" ]]; then
-    curl -sSLf --retry 3 https://docs.redpanda.com/redpanda-quickstart.tar.gz \
-      | tar -C "$PACKAGE_ROOT" -xzf -
+    url="$url/redpanda-quickstart.tar.gz"
   else
-    curl -sSLf --retry 3 "https://docs.redpanda.com/${MAJOR_MINOR}-redpanda-quickstart.tar.gz" \
-      | tar -C "$PACKAGE_ROOT" -xzf -
-  fi
-
-  if [[ ! -d "$QUICKSTART_DIR" ]]; then
-    echo "❌ Expected '$QUICKSTART_DIR' but none was found after extraction."
-    exit 1
+    url="$url/${MAJOR_MINOR}-redpanda-quickstart.tar.gz"
   fi
+  curl -sSLf --retry 3 "$url" | tar -C "$PACKAGE_ROOT" -xzf -
+  [[ -d "$QUICKSTART_DIR" ]] || { echo "❌ Expected '$QUICKSTART_DIR' but none was found."; exit 1; }
 fi
 
-# Switch into the quickstart dir and (re)start the cluster
-cd "$QUICKSTART_DIR"
+###############################################################################
+# Move into compose directory and clean up existing cluster
+###############################################################################
+cd "$QUICKSTART_DIR" || { echo "❌ Cannot cd to '$QUICKSTART_DIR'"; exit 1; }
 
-if docker compose ps | grep -q Up; then
-  echo "🛑 Stopping existing cluster…"
-  docker compose down --volumes
+if docker compose -p "$PROJECT_NAME" ps -q | grep -q .; then
+  echo "🛑 Cleaning up existing cluster…"
+  docker compose -p "$PROJECT_NAME" down --volumes
+else
+  echo "No running containers to remove for project \"$PROJECT_NAME\"."
 fi
 
-echo "▶️  Starting Redpanda cluster…"
-docker compose up -d
+###############################################################################
+# Remove globally conflicting containers (dynamic list + legacy minio)
+###############################################################################
+services=$(docker compose -p "$PROJECT_NAME" config --services 2>/dev/null || true)
+services+=" minio"   # ensure legacy /minio container is handled
+
+for svc in $services; do
+  if docker ps -a --format '{{.Names}}' | grep -wq "$svc"; then
+    echo "🧹 Removing existing container: $svc"
+    docker rm -f "$svc"
+  fi
+done
+
+###############################################################################
+# Start cluster
+###############################################################################
+echo "▶️  Starting Redpanda cluster (version: ${TAG})…"
+docker compose -p "$PROJECT_NAME" up -d
 
-# Return to original directory
-cd "$CALLER_PWD"
+###############################################################################
+# Return to caller and report success
+###############################################################################
+cd "$CALLER_PWD" || exit 1
 echo "✅ Cluster is up (version: ${TAG})"

package/extensions/generate-rp-connect-info.js

@@ -82,14 +82,14 @@ module.exports.register = function ({ config }) {
   }
 
   /**
-   * Translates the parsed CSV data into our expected format.
-   * If "enterprise" is found in the `support` column, it is replaced with "certified" in the output.
+   * Transforms and enriches parsed CSV connector data with normalized fields and documentation URLs.
    *
-   * @param {object} parsedData - The CSV data parsed into an object.
-   * @param {array} pages - The list of pages to map the URLs (used for enrichment with URLs).
-   * @param {object} logger - The logger used for error handling.
+   * Each row is trimmed, mapped to expected output fields, and enriched with documentation URLs for Redpanda Connect and Cloud components if available. The `support` field is normalized, and licensing information is derived. Logs a warning if documentation URLs are missing for non-deprecated, non-SQL driver connectors that indicate cloud support.
    *
-   * @returns {array} - The translated and enriched data.
+   * @param {object} parsedData - Parsed CSV data containing connector rows.
+   * @param {array} pages - Array of page objects used to resolve documentation URLs.
+   * @param {object} logger - Logger instance for warning about missing documentation.
+   * @returns {array} Array of enriched connector objects with normalized fields and URLs.
    */
   function translateCsvData(parsedData, pages, logger) {
     return parsedData.data.map(row => {
@@ -139,12 +139,17 @@ module.exports.register = function ({ config }) {
         }
       }
 
-      // Log a warning if neither URL was found (only warn for missing cloud if it should support cloud)
-      // Ignore sql_driver connectors because they are not real connectors and only used as a utility for grouping sql driver types.
-      if (!connector.includes('sql_driver') && !redpandaConnectUrl && (!redpandaCloudUrl && is_cloud_supported === 'y')) {
+      // Log a warning if neither URL was found and the component is not deprecated
+      if (
+        deprecated !== 'y' &&
+        !connector.includes('sql_driver') &&
+        !redpandaConnectUrl &&
+        (!redpandaCloudUrl && is_cloud_supported === 'y')
+      ) {
         logger.warn(`Docs missing for: ${connector} of type: ${type}`);
       }
 
+
       // Return the translated and enriched row
       return {
         connector,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redpanda-data/docs-extensions-and-macros",
-  "version": "4.6.0",
+  "version": "4.6.1",
   "description": "Antora extensions and macros developed for Redpanda documentation.",
   "keywords": [
     "antora",

package/tools/redpanda-connect/generate-rpcn-connector-docs.js

@@ -54,6 +54,10 @@ function mergeOverrides(target, overrides) {
               item[field] = overrideItem[field];
             }
           });
+          // Copy through selfManagedOnly flag
+          if (Object.hasOwn(overrideItem, 'selfManagedOnly')) {
+            item.selfManagedOnly = overrideItem.selfManagedOnly;
+          }
           // Recurse for nested children
           item = mergeOverrides(item, overrideItem);
         }
@@ -75,6 +79,26 @@ function mergeOverrides(target, overrides) {
   return target;
 }
 
+/**
+ * Generates documentation files for RPCN connectors using Handlebars templates.
+ *
+ * Depending on the {@link writeFullDrafts} flag, generates either partial documentation files for connector fields and examples, or full draft documentation for each connector component. Supports merging override data and skips draft generation for components marked as deprecated.
+ *
+ * @param {Object} options - Configuration options for documentation generation.
+ * @param {string} options.data - Path to the connector data file (JSON or YAML).
+ * @param {string} [options.overrides] - Optional path to a JSON file with override data.
+ * @param {string} options.template - Path to the main Handlebars template.
+ * @param {string} [options.templateIntro] - Path to the intro partial template (used in full draft mode).
+ * @param {string} [options.templateFields] - Path to the fields partial template.
+ * @param {string} [options.templateExamples] - Path to the examples partial template.
+ * @param {boolean} options.writeFullDrafts - If true, generates full draft documentation; otherwise, generates partials.
+ * @returns {Promise<Object>} An object summarizing the number and paths of generated partials and drafts.
+ *
+ * @throws {Error} If reading or parsing input files fails, or if template rendering fails for a component.
+ *
+ * @remark
+ * When generating full drafts, components with a `status` of `'deprecated'` are skipped.
+ */
 async function generateRpcnConnectorDocs(options) {
   const {
     data,
@@ -171,6 +195,10 @@ async function generateRpcnConnectorDocs(options) {
       }
 
       if (writeFullDrafts) {
+        if (String(item.status || '').toLowerCase() === 'deprecated') {
+          console.log(`Skipping draft for deprecated component: ${type}/${name}`);
+          continue;
+        }
         let content;
         try {
           content = compiledTemplate(item);

package/tools/redpanda-connect/helpers/renderConnectFields.js

@@ -1,10 +1,7 @@
-'use strict';
-
 const yaml = require('yaml');
 const renderYamlList = require('./renderYamlList');
 const handlebars = require('handlebars');
 
-
 /**
  * Renders the children of a configuration object into AsciiDoc.
  *
@@ -27,120 +24,125 @@ module.exports = function renderConnectFields(children, prefix = '') {
   prefix = typeof prefix === 'string' ? prefix : '';
 
   sorted.forEach(child => {
-    if (child.is_deprecated) {
-      return;
+    if (child.is_deprecated || !child.name) return;
+
+    // Normalize type
+    let displayType;
+    if (child.type === 'string' && child.kind === 'array') {
+      displayType = 'array';
+    } else if (child.type === 'unknown' && child.kind === 'map') {
+      displayType = 'object';
+    } else {
+      displayType = child.type;
     }
+
+    let block = '';
     const isArray = child.kind === 'array';
-    if (!child.name) return;
     const currentPath = prefix
       ? `${prefix}.${child.name}${isArray ? '[]' : ''}`
       : `${child.name}${isArray ? '[]' : ''}`;
 
-    output += `=== \`${currentPath}\`\n\n`;
+    block += `=== \`${currentPath}\`\n\n`;
 
     if (child.description) {
-      output += `${child.description}\n\n`;
+      block += `${child.description}\n\n`;
     }
-
-    if (child.is_secret === true) {
-      output += `include::redpanda-connect:components:partial$secret_warning.adoc[]\n\n`;
+    if (child.is_secret) {
+      block += `include::redpanda-connect:components:partial$secret_warning.adoc[]\n\n`;
     }
-
     if (child.version) {
-      output += `ifndef::env-cloud[]\nRequires version ${child.version} or later.\nendif::[]\n\n`;
+      block += `ifndef::env-cloud[]\nRequires version ${child.version} or later.\nendif::[]\n\n`;
     }
 
-    output += `*Type*: \`${child.type}\`\n\n`;
+    block += `*Type*: \`${displayType}\`\n\n`;
 
+    // Default
     if (child.type !== 'object' && child.default !== undefined) {
       if (typeof child.default !== 'object') {
         const display = child.default === '' ? '""' : String(child.default);
-        output += `*Default*: \`${display}\`\n\n`;
+        block += `*Default*: \`${display}\`\n\n`;
       } else {
         const defYaml = yaml.stringify(child.default).trim();
-        output += `*Default*:\n[source,yaml]\n----\n${defYaml}\n----\n\n`;
+        block += `*Default*:\n[source,yaml]\n----\n${defYaml}\n----\n\n`;
       }
     }
 
-    if (
-      child.annotated_options &&
-      Array.isArray(child.annotated_options) &&
-      child.annotated_options.length > 0
-    ) {
-      output += '[cols="1m,2a"]\n';
-      output += '|===\n';
-      output += '|Option |Summary\n\n';
-      child.annotated_options.forEach(optionPair => {
-        if (Array.isArray(optionPair) && optionPair.length >= 2) {
-          output += `|${optionPair[0]}\n|${optionPair[1]}\n\n`;
-        }
+    // Annotated options
+    if (child.annotated_options && child.annotated_options.length) {
+      block += `[cols=\"1m,2a\"]\n|===\n|Option |Summary\n\n`;
+      child.annotated_options.forEach(([opt, summary]) => {
+        block += `|${opt}\n|${summary}\n\n`;
       });
-      output += '|===\n\n';
+      block += `|===\n\n`;
     }
 
-    if (child.options && Array.isArray(child.options) && child.options.length > 0) {
-      output += `*Options*: ${child.options.map(opt => `\`${opt}\``).join(', ')}\n\n`;
+    // Options list
+    if (child.options && child.options.length) {
+      block += `*Options*: ${child.options.map(opt => `\`${opt}\``).join(', ')}\n\n`;
     }
 
+    // Examples
     if (child.examples && child.examples.length) {
-      output += '[source,yaml]\n----\n';
-      output += '# Examples:\n';
-
+      block += `[source,yaml]\n----\n# Examples:\n`;
       if (child.type === 'string') {
         if (child.kind === 'array') {
-          output += renderYamlList(child.name, child.examples);
+          block += renderYamlList(child.name, child.examples);
         } else {
           child.examples.forEach(example => {
             if (typeof example === 'string' && example.includes('\n')) {
-              output += `${child.name}: |-\n`;
-              const indentedLines = example.split('\n').map(line => '  ' + line).join('\n');
-              output += `${indentedLines}\n`;
+              block += `${child.name}: |-\n`;
+              block += example.split('\n').map(line => '  ' + line).join('\n') + '\n';
             } else {
-              output += `${child.name}: ${example}\n`;
+              block += `${child.name}: \`${example}\`\n`;
             }
           });
-          output += '\n';
+          block += '\n';
         }
       } else if (child.type === 'processor') {
         if (child.kind === 'array') {
-          output += renderYamlList(child.name, child.examples);
+          block += renderYamlList(child.name, child.examples);
         } else {
           child.examples.forEach(example => {
-            output += `${child.name}: ${example}\n`;
+            block += `${child.name}: \`${String(example)}\`\n`;
           });
-          output += '\n';
+          block += '\n';
         }
       } else if (child.type === 'object') {
         if (child.kind === 'array') {
-          output += renderYamlList(child.name, child.examples);
+          block += renderYamlList(child.name, child.examples);
         } else {
           child.examples.forEach(example => {
             if (typeof example === 'object') {
               const snippet = yaml.stringify(example).trim();
-              const lines = snippet.split('\n');
-              // Prefix two spaces to every line
-              const formattedLines = lines.map(line => '  ' + line).join('\n');
-              output += `${child.name}:\n${formattedLines}\n`;
+              block += `${child.name}:\n`;
+              block += snippet.split('\n').map(line => '  ' + line).join('\n') + '\n';
             } else {
-              output += `${child.name}: ${example}\n`;
+              block += `${child.name}: \`${String(example)}\`\n`;
             }
           });
-          output += '\n';
+          block += '\n';
         }
       } else {
         child.examples.forEach(example => {
-          output += `${child.name}: ${example}\n`;
+          block += `${child.name}: \`${String(example)}\`\n`;
         });
-        output += '\n';
+        block += '\n';
       }
+      block += `----\n\n`;
+    }
 
-      output += '----\n\n';
+    // Nested
+    if (child.children && child.children.length) {
+      block += renderConnectFields(child.children, currentPath);
     }
 
-    if (child.children && Array.isArray(child.children) && child.children.length > 0) {
-      output += renderConnectFields(child.children, currentPath);
+    // Cloud guard
+    if (child.selfManagedOnly) {
+      output += `ifndef::env-cloud[]\n${block}endif::[]\n\n`;
+    } else {
+      output += block;
     }
   });
 
   return new handlebars.SafeString(output);
-}
+};

package/tools/redpanda-connect/templates/intro.hbs

@@ -1,5 +1,3 @@
-// This content is autogenerated. Do not edit manually. To override descriptions or summaries, use the doc-tools CLI with the --overrides option: https://redpandadata.atlassian.net/wiki/spaces/DOC/pages/1247543314/Generate+reference+docs+for+Redpanda+Connect
-
 {{{summary}}}
 
 {{#if version}}