@redpanda-data/docs-extensions-and-macros 4.3.0 → 4.4.0

package/tools/get-console-version.js

@@ -0,0 +1,53 @@
+#!/usr/bin/env node
+
+const yaml = require('js-yaml');
+const fs = require('fs');
+const path = require('path');
+const GetLatestConsoleVersion = require('../extensions/version-fetcher/get-latest-console-version.js');
+const { getPrereleaseFromAntora } = require('../cli-utils/beta-from-antora.js');
+
+/**
+ * Fetches and prints the latest Console version and Docker repo.
+ * @param {Object} options
+ * @param {boolean} options.beta - Return beta version if available
+ * @param {boolean} options.fromAntora - Derive beta flag from antora.yml
+ */
+module.exports = async function getConsoleVersion({ beta = false, fromAntora = false }) {
+  const owner = 'redpanda-data';
+  const repo = 'console';
+  const CONSOLE_DOCKER_REPO = repo;
+
+  // Determine whether to use beta based on antora.yml or flag
+  let useBeta = beta;
+  if (fromAntora) {
+    useBeta = getPrereleaseFromAntora();
+  }
+
+  // Initialize GitHub client
+  const { Octokit } = await import('@octokit/rest');
+  const octokit = process.env.REDPANDA_GITHUB_TOKEN
+    ? new Octokit({ auth: process.env.REDPANDA_GITHUB_TOKEN })
+    : new Octokit();
+
+  // Fetch latest release info
+  let data;
+  try {
+    data = await GetLatestConsoleVersion(octokit, owner, repo);
+  } catch (err) {
+    console.error('Failed to fetch Console version:', err.message);
+    process.exit(1);
+  }
+
+  if (!data) {
+    console.error('No version data returned for Console');
+    process.exit(1);
+  }
+
+  // Select the version
+  const version = useBeta
+    ? (data.latestBetaRelease || data.latestStableRelease)
+    : data.latestStableRelease;
+
+  console.log(`CONSOLE_VERSION=${version}`);
+  console.log(`CONSOLE_DOCKER_REPO=${CONSOLE_DOCKER_REPO}`);
+};

package/tools/get-redpanda-version.js

@@ -0,0 +1,53 @@
+#!/usr/bin/env node
+
+const GetLatestRedpandaVersion = require('../extensions/version-fetcher/get-latest-redpanda-version.js');
+const { getPrereleaseFromAntora } = require('../cli-utils/beta-from-antora.js');
+
+/**
+ * Fetches and prints the latest Redpanda version and Docker repository.
+ * @param {Object} options
+ * @param {boolean} options.beta - Whether to prefer RC (beta) releases
+ * @param {boolean} options.fromAntora - Whether to derive beta flag from antora.yml
+ */
+module.exports = async function getRedpandaVersion({ beta = false, fromAntora = false }) {
+  const owner = 'redpanda-data';
+  const repo = 'redpanda';
+
+  // Determine whether to treat this as a beta (RC) release
+  let useBeta = beta;
+  if (fromAntora) {
+    useBeta = getPrereleaseFromAntora();
+  }
+
+  // Load Octokit
+  const { Octokit } = await import('@octokit/rest');
+  const octokit = process.env.REDPANDA_GITHUB_TOKEN
+    ? new Octokit({ auth: process.env.REDPANDA_GITHUB_TOKEN })
+    : new Octokit();
+
+  // Fetch version data
+  let data;
+  try {
+    data = await GetLatestRedpandaVersion(octokit, owner, repo);
+  } catch (err) {
+    console.error('Failed to fetch the latest Redpanda version:', err.message);
+    process.exit(1);
+  }
+
+  if (!data) {
+    console.error('No version data returned for Redpanda');
+    process.exit(1);
+  }
+
+  // Determine the version string
+  const stableVersion = data.latestRedpandaRelease.version;
+  const rc = data.latestRcRelease;
+  const version = useBeta && rc ? rc.version : stableVersion;
+
+  // Determine the Docker repository
+  const dockerRepo = (useBeta && rc) ? 'redpanda-unstable' : 'redpanda';
+
+  // Output for downstream consumption
+  console.log(`REDPANDA_VERSION=${version}`);
+  console.log(`REDPANDA_DOCKER_REPO=${dockerRepo}`);
+};

package/tools/metrics/metrics.py

@@ -0,0 +1,199 @@
+import os
+import sys
+import requests
+import re
+import json
+import logging
+
+'''
+## How it works
+
+Fetches metrics from the brokers and parses out the `# HELP` lines, creating:
+
+- `metrics.json`
+- `metrics.adoc`
+
+These output files are stored in a versioned folder under `docs/autogenerated/<version>/metrics`, based on the major.minor version provided as the first argument.
+
+## Prerequisites
+
+- **Python 3** & **pip** on your system.
+- A Redpanda cluster running with the `public_metrics` and `metrics` endpoints exposed at http://localhost:19644/
+'''
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
+
+def fetch_metrics(url):
+    """Fetch metrics from the given URL."""
+    try:
+        response = requests.get(url)
+        response.raise_for_status()
+        return response.text
+    except requests.RequestException as e:
+        logging.error(f"Error fetching metrics from {url}: {e}")
+        return None
+
+def parse_metrics(metrics_text):
+    """
+    Parse Prometheus exposition text into a dict:
+      metric_name → { description, type, unit, labels: [<label_keys>] }
+
+    - Strips empty `{}` on unlabelled samples
+    - Propagates HELP/TYPE from base metrics onto _bucket, _count, _sum
+    - Works regardless of # HELP / # TYPE order
+    - Captures # UNIT metadata if present
+    """
+
+    lines = metrics_text.splitlines()
+
+    # Gather HELP/TYPE metadata in any order
+    # Gather HELP/TYPE metadata in any order
+    meta = {}  # name → { 'description': str, 'type': str, 'unit': str }
+    for line in lines:
+        if line.startswith("# HELP"):
+            m = re.match(r"# HELP\s+(\S+)\s+(.+)", line)
+            if m:
+                name, desc = m.groups()
+                meta.setdefault(name, {})['description'] = desc
+        elif line.startswith("# TYPE"):
+            m = re.match(r"# TYPE\s+(\S+)\s+(\S+)", line)
+            if m:
+                name, mtype = m.groups()
+                meta.setdefault(name, {})['type'] = mtype
+        elif line.startswith("# UNIT"):
+            m = re.match(r"# UNIT\s+(\S+)\s+(.+)", line)
+            if m:
+                name, unit = m.groups()
+                meta.setdefault(name, {})['unit'] = unit
+    # Collect label keys from _every_ sample line
+    label_map = {}  # name → set(label_keys)
+    for line in lines:
+        if line.startswith("#"):
+            continue
+
+        # labelled: foo{a="1",b="2"}  42
+        m_lbl = re.match(r"^(\S+)\{(.+?)\}\s+(.+)", line)
+        if m_lbl:
+            name, labels_str, _ = m_lbl.groups()
+            keys = [kv.split("=",1)[0] for kv in labels_str.split(",")]
+        else:
+            # unlabelled, maybe with stray {}: foo{} 42  or just foo 42
+            m_unlbl = re.match(r"^(\S+?)(?:\{\})?\s+(.+)", line)
+            if not m_unlbl:
+                continue
+            name, _ = m_unlbl.groups()
+            keys = []
+
+        label_map.setdefault(name, set()).update(keys)
+
+    # Propagate HELP/TYPE from base histograms/summaries
+    for series in list(label_map):
+        for suffix in ("_bucket", "_count", "_sum"):
+            if series.endswith(suffix):
+                base = series[:-len(suffix)]
+                if base in meta:
+                    meta.setdefault(series, {}).update(meta[base])
+                break
+
+    # Merge into final metrics dict, with warnings
+    metrics = {}
+    all_names = set(meta) | set(label_map)
+    for name in sorted(all_names):
+        desc = meta.get(name, {}).get("description")
+        mtype = meta.get(name, {}).get("type")
+        unit = meta.get(name, {}).get("unit")
+        labels = sorted(label_map.get(name, []))
+
+        if desc is None:
+            logging.warning(f"Metric '{name}' has samples but no # HELP.")
+            desc = ""
+        if mtype is None:
+            logging.warning(f"Metric '{name}' has no # TYPE entry.")
+
+        metrics[name] = {
+            "description": desc,
+            "type": mtype,
+            "unit": unit,
+            "labels": labels
+        }
+
+    logging.info(f"Extracted {len(metrics)} metrics.")
+    return metrics
+
+def output_asciidoc(metrics, adoc_file):
+    """Output metrics as AsciiDoc."""
+    with open(adoc_file, "w") as f:
+        for name, data in metrics.items():
+            f.write(f"=== {name}\n\n")
+            f.write(f"{data['description']}\n\n")
+            f.write(f"*Type*: {data['type']}")
+            if data.get("unit"):
+                f.write(f"\n\n*Unit*: {data['unit']}")
+            if data["labels"]:
+                f.write("\n\n*Labels*:\n")
+                for label in data["labels"]:
+                    f.write(f"\n- `{label}`")
+            f.write("\n\n---\n\n")
+    logging.info(f"AsciiDoc output written to {adoc_file}")
+
+def output_json(metrics, json_file):
+    """Output metrics as JSON."""
+    with open(json_file, "w") as f:
+        json.dump(metrics, f, indent=4)
+    logging.info(f"JSON output written to {json_file}")
+
+def ensure_directory_exists(directory):
+    """Ensure the given directory exists."""
+    if not os.path.exists(directory):
+        os.makedirs(directory)
+
+if __name__ == "__main__":
+    # Expect the major.minor version to be provided as the first argument.
+    if len(sys.argv) < 2:
+        logging.error("Major.minor version must be provided as the first argument. Exiting.")
+        sys.exit(1)
+
+    tag_modified = sys.argv[1].strip()
+
+    # Resolve the base autogenerated folder at the repo root
+    repo_root = os.getcwd()
+    gen_path = os.path.join(repo_root, "autogenerated")
+    if not os.path.isdir(gen_path):
+        logging.error(f"autogenerated folder not found at: {gen_path}")
+        sys.exit(1)
+
+    # Build the output directory using the already provided tag_modified.
+    output_dir = os.path.join(gen_path, tag_modified, "metrics")
+    ensure_directory_exists(output_dir)
+
+    METRICS_URL = "http://localhost:19644/public_metrics/"
+    metrics_text = fetch_metrics(METRICS_URL)
+    if not metrics_text:
+        logging.error("No public metrics retrieved. Exiting.")
+        sys.exit(1)
+
+    public_metrics = parse_metrics(metrics_text)
+
+    # Fetch internal metrics if available.
+    INTERNAL_METRICS_URL = "http://localhost:19644/metrics/"
+    internal_metrics_text = fetch_metrics(INTERNAL_METRICS_URL)
+    if internal_metrics_text:
+        internal_metrics = parse_metrics(internal_metrics_text)
+    else:
+        logging.error("No internal metrics retrieved.")
+        internal_metrics = {}
+
+    # Merge public and internal metrics.
+    merged_metrics = {
+        "public": public_metrics,
+        "internal": internal_metrics
+    }
+
+    # Define output file paths.
+    JSON_OUTPUT_FILE = os.path.join(output_dir, "metrics.json")
+    ASCIIDOC_OUTPUT_FILE = os.path.join(output_dir, "metrics.adoc")
+    INTERNAL_ASCIIDOC_OUTPUT_FILE = os.path.join(output_dir, "internal-metrics.adoc")
+
+    output_json(merged_metrics, JSON_OUTPUT_FILE)
+    output_asciidoc(public_metrics, ASCIIDOC_OUTPUT_FILE)
+    output_asciidoc(internal_metrics, INTERNAL_ASCIIDOC_OUTPUT_FILE)

package/tools/metrics/requirements.txt

@@ -0,0 +1 @@
+requests==2.32.3

package/tools/property-extractor/Makefile

@@ -0,0 +1,99 @@
+.PHONY: build venv clean redpanda-git treesitter generate-docs check
+
+# Default tag (can be overridden via `make TAG=v25.1.1`)
+TAG       ?= dev
+
+# Derive a “major.minor” or rc identifier from TAG for folder naming
+VERSION := $(shell \
+  if echo "$(TAG)" | grep -qE '^v?[0-9]+\.[0-9]+'; then \
+    echo "$(TAG)" \
+      | sed -E 's/^v?([0-9]+\.[0-9]+)(\.[0-9]+)?(-rc[0-9]+)?.*/\1\3/' \
+      | sed 's/-rc/rc/'; \
+  else \
+    echo "$(TAG)"; \
+  fi)
+
+# Paths
+REPO_ROOT     := $(shell git rev-parse --show-toplevel)
+MODULE_ROOT   := $(shell cd "$(dir $(realpath $(lastword $(MAKEFILE_LIST))))"/../.. && pwd)
+TOOL_ROOT     := $(MODULE_ROOT)/tools/property-extractor
+TMP_ROOT      := $(TOOL_ROOT)/tmp
+REDPANDA_SRC  := $(TMP_ROOT)/redpanda
+TREESITTER_DIR:= $(TOOL_ROOT)/tree-sitter/tree-sitter-cpp
+VENV          := $(TOOL_ROOT)/tmp/redpanda-property-extractor-venv
+PYTHON        := $(VENV)/bin/python
+OUTPUT_DIR    := $(REPO_ROOT)/autogenerated/$(TAG)/properties
+TREE_SITTER   := npx tree-sitter
+
+# --- Main build: venv, fetch code, build parser, extract & docgen ---
+build: venv redpanda-git treesitter
+	@echo "🔧 Building with Redpanda tag: $(TAG)"
+	@mkdir -p $(TOOL_ROOT)/gen
+	@cd $(TOOL_ROOT) && \
+	  $(PYTHON) -W ignore::FutureWarning property_extractor.py \
+	    --recursive \
+	    --path $(REDPANDA_SRC) \
+	    --output gen/properties-output.json
+	@echo "✅ JSON generated at $(TOOL_ROOT)/gen/properties-output.json"
+	@$(MAKE) generate-docs
+
+# --- Ensure Python venv & dependencies ---
+venv: $(TOOL_ROOT)/requirements.txt
+	@if [ ! -d "$(VENV)" ]; then \
+	  echo "🐍 Creating virtual environment in $(VENV)..."; \
+	  python3 -m venv $(VENV); \
+	  $(VENV)/bin/pip install --upgrade pip --quiet; \
+	  $(VENV)/bin/pip install --no-cache-dir -r $<; \
+	else \
+	  echo "🐍 Virtual environment already exists at $(VENV)"; \
+	fi
+
+# --- Clean out all generated state ---
+clean:
+	@echo "🧹 Cleaning up…"
+	@rm -rf $(VENV) $(TMP_ROOT) $(REPO_ROOT)/autogenerated
+
+# --- Clone or update Redpanda, checkout TAG or default branch ---
+redpanda-git:
+	@mkdir -p "$(TMP_ROOT)"
+	@echo "🔄 Cloning/updating Redpanda into $(REDPANDA_SRC)…"
+	@if [ -d "$(REDPANDA_SRC)" ]; then \
+	  git -C "$(REDPANDA_SRC)" fetch --all --tags -q; \
+	else \
+	  git clone -q https://github.com/redpanda-data/redpanda.git "$(REDPANDA_SRC)"; \
+	fi; \
+	if git -C "$(REDPANDA_SRC)" rev-parse --verify -q "$(TAG)" >/dev/null; then \
+	  echo "🔖 Checking out '$(TAG)'"; \
+	  git -C "$(REDPANDA_SRC)" checkout -q "$(TAG)"; \
+	else \
+	  DEFAULT_BRANCH=$$(git -C "$(REDPANDA_SRC)" symbolic-ref --short refs/remotes/origin/HEAD); \
+	  echo "⚠️ Tag '$(TAG)' not found; falling back to '$${DEFAULT_BRANCH}'"; \
+	  git -C "$(REDPANDA_SRC)" checkout -q "$${DEFAULT_BRANCH}"; \
+	fi
+
+# --- Clone Tree-sitter grammar & generate parser ---
+treesitter:
+	@echo "🌲 Ensuring tree-sitter-cpp grammar…"
+	@if [ ! -d "$(TREESITTER_DIR)" ]; then \
+	  git clone https://github.com/tree-sitter/tree-sitter-cpp.git "$(TREESITTER_DIR)"; \
+	fi
+	@echo "🔧 Generating parser in $(TREESITTER_DIR)…"
+	@cd "$(TREESITTER_DIR)" && npm install --silent && $(TREE_SITTER) generate
+
+# --- Turn the JSON into AsciiDoc pages under autogen/<version>/properties ---
+generate-docs:
+	@echo "📝 Generating AsciiDoc pages in $(OUTPUT_DIR)…"
+	@mkdir -p "$(OUTPUT_DIR)"
+	@cd $(TOOL_ROOT) && \
+	  $(PYTHON) json-to-asciidoc/generate_docs.py --output-dir "$(OUTPUT_DIR)"
+	@echo "✅ Docs generated at $(OUTPUT_DIR)"
+
+# --- Debug helper to print all the key paths/vars ---
+check:
+	@echo "MODULE_ROOT:   $(MODULE_ROOT)"
+	@echo "TOOL_ROOT:     $(TOOL_ROOT)"
+	@echo "REDPANDA_SRC:  $(REDPANDA_SRC)"
+	@echo "TREESITTER:    $(TREESITTER_DIR)"
+	@echo "VENV:          $(VENV)"
+	@echo "PYTHON:        $(PYTHON)"
+	@echo "OUTPUT_DIR:    $(OUTPUT_DIR)"

package/tools/property-extractor/README.adoc

@@ -0,0 +1,206 @@
+= Redpanda Property Generator
+
+The Redpanda Property Generator is a CLI tool designed to extract properties from Redpanda's source code and generate a JSON output with their definitions as well as Asciidoc pages.
+
+== Prerequisites
+
+Ensure the following prerequisites are installed on your system:
+
+- https://www.python.org/downloads/[Python 3.10 or higher]
+- A C++ compiler (such as `gcc`, `clang`)
+- https://www.google.com/search?q=how+to+install+make[`make` utility] (to use the Makefile for automation)
++
+To ensure `make` is available:
++
+[,bash]
+----
+make --version
+----
+
+== Install
+
+. Clone the repository:
++
+[,bash]
+----
+git clone https://github.com/redpanda-data/docs-extensions-and-macros.git
+cd docs-extensions-and-macros
+----
+
+== Generate properties
+
+. Run the build process:
++
+[,bash]
+----
+cd tools/property-extractor
+make build
+----
++
+This command:
++
+- Sets up the Python virtual environment (`venv`).
+- Checks out the Redpanda source code to the specified branch or tag.
+- Runs the extractor to generate a JSON file at `gen/properties-output.json`.
+- Runs the docs generator to generate Asciidoc pages from the `properties-output.json`.
+
+. Locate the generated files:
++
+[,bash]
+----
+ls gen/properties-output.json
+ls output
+----
+
+To clean the environment and generated files:
+
+[,bash]
+----
+make clean
+----
+
+== Run the extractor manually
+
+To run the extractor tool directly:
+
+[,bash]
+----
+./property_extractor.py --path <path-to-redpanda-source> [options]
+----
+
+=== Command options
+
+|===
+| Option | Description
+
+| `--path <path>`
+| Path to the Redpanda source directory to extract properties from (required).
+| `--recursive`
+| Recursively scan the provided path for header (`*.h`) and implementation (`*.cc`) file pairs.
+| `--output <output>`
+| Path to the output JSON file. If not provided, the output will be printed to the console.
+| `--definitions <definitions>`
+| Path to the `definitions.json` file for type definitions (default: included `definitions.json`).
+| `-v`, `--verbose`
+| Enable verbose logging for debugging purposes.
+|===
+
+=== Example command
+
+[,bash]
+----
+./property_extractor.py --path ./tmp/redpanda --recursive --output autogenerated/properties.json
+----
+
+=== How it works
+
+. The tool identifies pairs of header (`*.h`) and implementation (`*.cc`) files in the specified Redpanda source directory. This ensures that both the declaration and definition of properties are available.
+
+. Tree-sitter is used to parse the C{plus}{plus} source code and create abstract syntax trees (ASTs). Both the Tree-sitter C++ library (via a Git submodule) and its Python bindings (`tree_sitter`) are required for this step.
+
+. Custom logic in `property_extractor.py` processes the ASTs to extract property definitions from specific files like:
++
+- `src/v/config/configuration.cc`
+- `src/v/kafka/client/configuration.cc`
+
+. Extracted properties are processed by a series of transformers to enrich and normalize the data. For example:
++
+- `BasicInfoTransformer`: Extracts names and metadata.
+- `VisibilityTransformer`: Determines visibility (e.g., public or private).
+- `IsNullableTransformer`: Detects if a property is nullable.
+
+. The `definitions.json` file is merged into the output, linking property types to their descriptions.
+
+=== JSON output
+
+The final JSON contains:
+
+- `properties`: Extracted properties with metadata.
+- `definitions`: Type definitions, merged from `definitions.json`.
+
+Example JSON structure:
+
+[,json]
+----
+{
+    "properties": {
+        "example_property": {
+            "type": "string",
+            "description": "An example property."
+        }
+    },
+    "definitions": {
+        "string": {
+            "description": "A string type."
+        }
+    }
+}
+----
+
+=== Custom definitions
+
+You can provide a custom `definitions.json` file:
+
+[,bash]
+----
+./property_extractor.py --path ./tmp/redpanda --definitions custom-definitions.json --output autogenerated/custom-output.json
+----
+
+=== Debugging
+
+Enable verbose logging to see detailed information:
+
+[,bash]
+----
+./property_extractor.py --path ./tmp/redpanda --verbose
+----
+
+== Run the docs generator manually
+
+. Make sure you have the `autogenerated/properties-output.json` file, relative to the `Makefile` location.
+
+. Run the script:
++
+[,bash]
+----
+python3 generate_docs.py
+----
+
+The script will process the JSON and generate AsciiDoc files in the `output/pages/` directory.
+
+=== Output files
+
+The following files will be generated:
+
+- Broker Properties: `output/pages/broker-properties.adoc`
+- Cluster Properties: `output/pages/cluster-properties.adoc`
+- Object Storage Properties: `output/pages/object-storage-properties.adoc`
+- Deprecated Properties: `output/pages/deprecated/partials/deprecated-properties.adoc`
+
+=== Error reports
+
+If the script encounters issues, it will generate error files in the `output/error/` directory:
+
+- `empty_description.txt`: Properties without descriptions.
+- `empty_type.txt`: Properties without types.
+- `max_without_min.txt`: Properties with a maximum value but no minimum.
+- `min_without_max.txt`: Properties with a minimum value but no maximum.
+
+The console output will summarize the errors and property statistics.
+
+=== How it works
+
+. Input parsing:
+   - The script loads the JSON file from the `autogenerated/` directory.
+   - Properties are categorized into groups based on their `defined_in` field or specific naming conventions such as the `cloud_` prefix.
+
+. Validation:
+   - Validates fields like `description`, `type`, `maximum`, and `minimum`.
+   - Identifies missing or inconsistent data and logs these to error files.
+
+. Documentation generation:
+   - Creates AsciiDoc files with categorized properties, including metadata such as type, default value, visibility, and restart requirements.
+   - Appends appropriate titles, introductions, and formatting for each group.
+
+. Error reporting: Generates error reports for easy debugging and correction of the input JSON.
+