@redpanda-data/docs-extensions-and-macros 4.3.0 → 4.4.1

package/docker-compose/transform/transform.go

@@ -0,0 +1,122 @@
+package main
+// This data transform filters records based on a customizable regex pattern.
+// If a record's key or value
+// (determined by an environment variable) matches the specified regex,
+// the record is forwarded to the output.
+// Otherwise, it is dropped.
+//
+// Usage:
+// 1. Provide the following environment variables in your Docker or configuration setup:
+//    - PATTERN     : (required) a regular expression that determines what you want to match.
+//    - MATCH_VALUE : (optional) a boolean to decide whether to check the record value. If false,
+//                   the record key is checked. Default is false.
+//
+// Example environment variables:
+//   PATTERN=".*\\.edu$"
+//   MATCH_VALUE="true"
+//
+// Logs:
+// This transform logs information about each record and whether it matched.
+// The logs appear in the _redpanda.transform_logs topic, so you can debug how your records are being processed.
+//
+// Build instructions:
+//   go mod tidy
+//   rpk transform build
+//
+// For more details on building transforms with the Redpanda SDK, see:
+// https://docs.redpanda.com/current/develop/data-transforms
+//
+
+import (
+	"log"
+	"os"
+	"regexp"
+	"strings"
+
+	"github.com/redpanda-data/redpanda/src/transform-sdk/go/transform"
+)
+
+var (
+	re         *regexp.Regexp
+	checkValue bool
+)
+
+func isTrueVar(v string) bool {
+	switch strings.ToLower(v) {
+	case "yes", "ok", "1", "true":
+		return true
+	default:
+		return false
+	}
+}
+
+// The main() function runs only once at startup. It performs all initialization steps:
+// - Reads and compiles the regex pattern.
+// - Determines whether to match on the key or value.
+// - Registers the doRegexFilter() function to process records.
+func main() {
+	// Set logging preferences, including timestamp and UTC time.
+	log.SetPrefix("[regex-transform] ")
+	log.SetFlags(log.Ldate | log.Ltime | log.LUTC | log.Lmicroseconds)
+
+	// Start logging the transformation process
+	log.Println("Starting transform...")
+
+	// Read the PATTERN environment variable to get the regex pattern.
+	pattern, ok := os.LookupEnv("PATTERN")
+	if !ok {
+		log.Fatal("Missing PATTERN environment variable")
+	}
+	// Log the regex pattern being used.
+	log.Printf("Using PATTERN: %q\n", pattern)
+	// Compile the regex pattern for later use.
+	re = regexp.MustCompile(pattern)
+
+	// Read the MATCH_VALUE environment variable to determine whether to check the record's value.
+	mk, ok := os.LookupEnv("MATCH_VALUE")
+	checkValue = ok && isTrueVar(mk)
+	log.Printf("MATCH_VALUE set to: %t\n", checkValue)
+
+	log.Println("Initialization complete, waiting for records...")
+
+	// Listen for records to be written, calling doRegexFilter() for each record.
+	transform.OnRecordWritten(doRegexFilter)
+}
+
+// The doRegexFilter() function executes each time a new record is written.
+// It checks whether the record's key or value (based on MATCH_VALUE) matches the compiled regex.
+// If it matches, the record is forwarded, if not, it's dropped.
+func doRegexFilter(e transform.WriteEvent, w transform.RecordWriter) error {
+	// This stores the data to be checked (either the key or value).
+	var dataToCheck []byte
+
+	// Depending on the MATCH_VALUE environment variable, decide whether to check the record's key or value.
+	if checkValue {
+		// Use the value of the record if MATCH_VALUE is true.
+		dataToCheck = e.Record().Value
+		log.Printf("Checking record value: %s\n", string(dataToCheck))
+	} else {
+		// Use the key of the record if MATCH_VALUE is false.
+		dataToCheck = e.Record().Key
+		log.Printf("Checking record key: %s\n", string(dataToCheck))
+	}
+
+	// If there is no key or value to check, log and skip the record.
+	if dataToCheck == nil {
+		log.Println("Record has no key/value to check, skipping.")
+		return nil
+	}
+
+	// Check if the data matches the regex pattern.
+	pass := re.Match(dataToCheck)
+	if pass {
+		// If the record matches the pattern, log and write the record to the output topic.
+		log.Printf("Record matched pattern, passing through. Key: %s, Value: %s\n", string(e.Record().Key), string(e.Record().Value))
+		return w.Write(e.Record())
+	} else {
+		// If the record does not match the pattern, log and drop the record.
+		log.Printf("Record did not match pattern, dropping. Key: %s, Value: %s\n", string(e.Record().Key), string(e.Record().Value))
+		// Do not write the record if it doesn't match the pattern.
+		return nil
+	}
+}

package/docker-compose/transform/transform.yaml

@@ -0,0 +1,33 @@
+# Transform metadata used by the rpk transform build command.
+# This metadata file tells rpk:
+# 1) The transform’s display name, which also becomes the base for the .wasm file name.
+# 2) A brief description of what it does.
+# 3) Defaults for environment variables.
+# 4) Input and output topics (if you want to define them here rather than in the deploy command).
+
+# Human-readable name of the transform. rpk transform build uses this for the generated .wasm file.
+name: regex
+
+description: |
+  Filters the input topic to records that only match a regular expression.
+
+  Regular expressions are implemented using Go's regexp library, which uses the syntax of RE2.
+  See the RE2 wiki for allowed syntax: https://github.com/google/re2/wiki/Syntax
+
+  Environment variables:
+  - PATTERN: The regular expression that will match against records (required).
+  - MATCH_VALUE: By default, the regex matches keys, but if set to "true", the regex matches values.
+
+# By default, no input topic is set here. (You can set it in your deploy command if preferred.)
+input-topic: ""
+
+# By default, no output topic is set here. (You can set it in your deploy command if preferred.)
+output-topic: ""
+
+# Indicates the specific TinyGo environment used to compile your transform.
+language: tinygo-no-goroutines
+
+env:
+  # The PATTERN variable must be provided at deploy time.
+  # Example: --var=PATTERN=".*@example.com"
+  PATTERN: '<required>'

package/extensions/replace-attributes-in-attachments.js

@@ -182,7 +182,7 @@ function getDynamicReplacements(componentVersion, logger) {
   const versionNum = formatVersion(componentVersion.version || '', semver);
   const is24_3plus =
     versionNum && semver.gte(versionNum, '24.3.0') && componentVersion.title === 'Self-Managed';
-  const useTagAttributes = isPrerelease || is24_3plus;
+  const useTagAttributes = isPrerelease || is24_3plus || componentVersion.title === 'Labs';
 
   // Derive Redpanda / Console versions
   const redpandaVersion = isPrerelease

package/extensions/util/compute-out.js

@@ -0,0 +1,38 @@
+'use strict'
+
+const { posix: path } = require('node:path')
+
+function computeOut (src) {
+  const { component, version, module: module_, family, relative } = src
+  const outRelative = family === 'page' ? relative.replace(/\.adoc$/, '.html') : relative
+  const { dir: dirname, base: basename, ext: extname, name: stem } = path.parse(outRelative)
+  const componentVersion = this.getComponentVersion(component, version)
+  const versionSegment = componentVersion
+  const outDirSegments = []
+  const moduleRootPathSegments = []
+  if (component !== 'ROOT') outDirSegments.push(component)
+  if (versionSegment) outDirSegments.push(versionSegment)
+  if (module_ !== 'ROOT') outDirSegments.push(module_)
+  const outModuleDirSegments = outDirSegments.slice()
+  if (family !== 'page') {
+    outDirSegments.push(`_${family}s`)
+    moduleRootPathSegments.push('..')
+  }
+  if (dirname) {
+    outDirSegments.push(dirname)
+    for (const _ of dirname.split('/')) moduleRootPathSegments.push('..')
+  }
+  const rootPathSegments = moduleRootPathSegments.slice()
+  for (const _ of outModuleDirSegments) rootPathSegments.push('..')
+  const outDirname = outDirSegments.join('/')
+  const result = {
+    dirname: outDirname,
+    basename,
+    path: outDirname + '/' + basename,
+    moduleRootPath: moduleRootPathSegments.length ? moduleRootPathSegments.join('/') : '.',
+    rootPath: rootPathSegments.length ? rootPathSegments.join('/') : '.',
+  }
+  return result
+}
+
+module.exports = computeOut

package/extensions/util/create-asciidoc-file.js

@@ -0,0 +1,15 @@
+'use strict'
+
+const computeOut = require('./compute-out')
+const { posix: path } = require('node:path')
+
+function createAsciiDocFile (contentCatalog, file) {
+  file.mediaType = 'text/asciidoc'
+  const src = file.src
+  const out = computeOut.call(contentCatalog, src)
+  const pub = { url: '/' + out.path, moduleRootPath: out.moduleRootPath, rootPath: out.rootPath }
+  contentCatalog.removeFile((file = contentCatalog.addFile(Object.assign(file, { path: out.path, out: null, pub: pub }))))
+  return file
+}
+
+module.exports = createAsciiDocFile

package/macros/data-template.js

@@ -22,8 +22,8 @@ const jsonpath = require('jsonpath-plus');
 const yaml = require('yaml');
 // For synchronous HTTP fetching.
 const request = require('sync-request');
-const computeOut = require('../util/compute-out.js');
-const createAsciiDocFile = require('../util/create-asciidoc-file.js');
+const computeOut = require('../extensions/util/compute-out.js');
+const createAsciiDocFile = require('../extensions/util/create-asciidoc-file.js');
 
 // In-memory cache for external resources (avoid repeated network calls)
 const externalCache = new Map();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redpanda-data/docs-extensions-and-macros",
-  "version": "4.3.0",
+  "version": "4.4.1",
   "description": "Antora extensions and macros developed for Redpanda documentation.",
   "keywords": [
     "antora",
@@ -12,7 +12,13 @@
   "author": {
     "name": "Redpanda Docs Team"
   },
+  "bin": {
+    "doc-tools": "./bin/doc-tools.js"
+  },
   "scripts": {
+    "install-test-dependencies": "doc-tools install-test-dependencies",
+    "get-redpanda-version": "doc-tools get-redpanda-version",
+    "get-console-version": "doc-tools get-console-version",
     "build": "antora --to-dir docs --fetch local-antora-playbook.yml",
     "serve": "wds --node-resolve --open preview/test/ --watch --root-dir docs"
   },
@@ -55,8 +61,13 @@
   },
   "files": [
     "extensions",
+    "extension-utils",
     "asciidoc-extensions",
-    "macros"
+    "macros",
+    "bin",
+    "cli-utils",
+    "tools",
+    "docker-compose"
   ],
   "license": "ISC",
   "repository": {
@@ -84,7 +95,8 @@
     "semver": "^7.6.0",
     "sync-request": "^6.1.0",
     "tar": "^7.4.3",
-    "yaml": "^2.7.0"
+    "tree-sitter": "^0.22.4",
+    "yaml": "^2.7.1"
   },
   "devDependencies": {
     "@antora/cli": "3.1.4",

package/tools/docusaurus-to-antora-conversion-scripts/convert-docs.sh

@@ -0,0 +1,114 @@
+#!/bin/bash
+
+if ! command -v pandoc &> /dev/null; then
+  echo "Error: Pandoc is not installed."
+  echo "Please visit https://pandoc.org/installing.html to install Pandoc."
+  exit 1
+fi
+
+if ! command -v kramdoc &> /dev/null; then
+  echo "Error: Kramdoc is not installed."
+  echo "Please install kramdoc using: gem install kramdown-asciidoc"
+  exit 1
+fi
+
+SOURCE_DIRECTORY="$1"
+
+if [ -z "$SOURCE_DIRECTORY" ]; then
+  echo "Error: Source directory not provided."
+  echo "Usage: ./your_script.sh /path/to/your/source_directory"
+  exit 1
+fi
+
+OUTPUT_DIRECTORY="$(cd "$(dirname "$0")/../modules" && pwd)"
+
+# Create the output and partials directories if they don't exist
+mkdir -p "$OUTPUT_DIRECTORY"
+
+function remove_leading_tabs() {
+  local mdx_file="$1"
+  local content="$(cat "$mdx_file")"
+
+  # Remove leading tabs in the <Tabs> elements
+  local updated_content="$(echo "$content" | perl -0777 -pe 's/(\s*)<TabItem([\s\S]*?)>([\s\S]*?)<\/TabItem>/sprintf("%s<TabItem%s>%s<\/TabItem>", $1, $2, $3 =~ s!^\t!!rmsg)/ge')"
+
+  # Write the updated content back to the file
+  echo "$updated_content" > "$mdx_file"
+}
+
+function preprocess_markdown() {
+  local markdown_file="$1"
+  node "$(dirname "$0")/pre-process-markdown.js" "$markdown_file"
+}
+
+# Convert a Markdown file to AsciiDoc and add the description
+function convert_markdown_to_asciidoc() {
+  local markdown_file="$1"
+  local output_file="$2"
+  # Remove leading tabs from <Tab> elements
+  remove_leading_tabs "$markdown_file"
+
+  # Preprocess the markdown file
+  preprocess_markdown "$markdown_file"
+
+  local content="$(cat "$markdown_file")"
+
+  local output_file_dir="$(dirname "$output_file")"
+  mkdir -p "$output_file_dir"
+
+  # Extract the content of the meta description tag
+  local description="$(echo "$content" | sed -n 's/.*<meta name="description" content="\([^"]*\)".*/\1/p')"
+
+  # Remove the head element from the source Markdown file and save it
+  local cleaned_content="$(echo "$content" | sed '/<head>/,/<\/head>/d')"
+  # Remove the head element from the source Markdown file and save it
+  local cleaned_content
+  cleaned_content=$(echo "$content" | sed '/<head>/,/<\/head>/d')
+  local cleaned_file
+  cleaned_file=$(mktemp)
+  echo "$cleaned_content" > "$cleaned_file"
+
+  # Convert the cleaned Markdown file to AsciiDoc using Kramdoc
+  local asciidoc_content
+  asciidoc_content=$(kramdoc -o - "$cleaned_file")
+
+  # Clean up temporary file
+  rm -f "$cleaned_file"
+
+  # Insert the description attribute on the second line of the AsciiDoc content
+  asciidoc_content="$(echo "$asciidoc_content" | awk -v desc="$description" 'NR==1{print; print ":description: " desc ""; next} 1')"
+
+  # Write the updated AsciiDoc content to the output file
+  echo "$asciidoc_content" > "$output_file"
+
+  echo "Converted: $markdown_file -> $output_file"
+}
+
+# Convert all Markdown files in the source directory
+# Initialize counters
+success_count=0
+failure_count=0
+
+while IFS= read -r -d '' markdown_file; do
+  output_file="$(echo "$markdown_file" \
+    | sed "s|$SOURCE_DIRECTORY|$OUTPUT_DIRECTORY|" \
+    | sed 's|\.mdx$|.adoc|' \
+    | sed 's|\(.*\)/\(.*\)|\1/pages/\2|')"
+
+  if convert_markdown_to_asciidoc "$markdown_file" "$output_file"; then
+    # Run the Node.js script to process the output file
+    if node "$(dirname "$0")/post-process-asciidoc.js" "$output_file"; then
+      success_count=$((success_count + 1))
+    else
+      echo "Error: Failed to post-process ${output_file}"
+      failure_count=$((failure_count + 1))
+    fi
+  else
+    echo "Error: Failed to convert ${markdown_file}"
+    failure_count=$((failure_count + 1))
+  fi
+done < <(find "$SOURCE_DIRECTORY" -name "*.mdx" -print0)
+
+echo "Conversion complete. Success: ${success_count}, Failures: ${failure_count}"
+
+echo "All Markdown files converted to AsciiDoc."

package/tools/docusaurus-to-antora-conversion-scripts/get-file-changes.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -euo pipefail
+echo "Please enter the name of the first branch:"
+read branch1
+echo "Please enter the name of the second branch:"
+read branch2
+
+git fetch
+git diff --summary $branch1..$branch2 -- ./modules/

package/tools/docusaurus-to-antora-conversion-scripts/post-process-asciidoc.js

@@ -0,0 +1,63 @@
+const fs = require('fs');
+const path = require('path');
+
+function processFile(file) {
+  let content;
+  try {
+    content = fs.readFileSync(file, 'utf-8');
+  } catch (err) {
+    console.error(`Error reading file ${file}: ${err.message}`);
+    return;
+  }
+
+  const newContent = content.replace(
+    /link:(\.\.\/)+([\w/.-]+)(#?[\w/.-]*)(\[.+?\])/g,
+    (match, dots, linkPath, anchor, linkText) => {
+      const depth = dots.match(/\.\.\//g).length;
+      const pathParts = linkPath.split('/');
+      // Ensure we don't go beyond the available path parts
+      const startIndex = Math.max(0, pathParts.length - depth);
+      const newPath = pathParts.slice(0, startIndex).join(':');
+      return `xref:${newPath}:${pathParts[pathParts.length - 1]}.adoc${anchor || ''}${linkText}`;
+    }
+  );
+
+  try {
+    fs.writeFileSync(file, newContent, 'utf-8');
+  } catch (err) {
+    console.error(`Error writing file ${file}: ${err.message}`);
+  }
+}
+
+function processDirectory(directory) {
+  const files = fs.readdirSync(directory);
+
+  files.forEach((file) => {
+    const filePath = path.join(directory, file);
+    const stat = fs.statSync(filePath);
+
+    if (stat.isFile() && path.extname(file) === '.adoc') {
+      processFile(filePath);
+    } else if (stat.isDirectory()) {
+      processDirectory(filePath);
+    }
+  });
+}
+
+const inputPath = process.argv[2];
+
+if (!inputPath) {
+  console.error('No input path provided');
+  process.exit(1);
+}
+
+const stat = fs.statSync(inputPath);
+
+if (stat.isFile()) {
+  processFile(inputPath);
+} else if (stat.isDirectory()) {
+  processDirectory(inputPath);
+} else {
+  console.error('Input path is neither a file nor a directory');
+  process.exit(1);
+}

package/tools/docusaurus-to-antora-conversion-scripts/pre-process-markdown.js

@@ -0,0 +1,108 @@
+const fs = require('fs');
+const { execSync } = require('child_process');
+const pandoc = require('node-pandoc');
+// Fail fast if required CLIs are missing
+['pandoc', 'kramdoc'].forEach(cmd => {
+  try {
+    execSync(`command -v ${cmd}`, { stdio: 'ignore' });
+  } catch {
+    console.error(`Required dependency "${cmd}" not found in PATH`);
+    process.exit(1);
+  }
+});
+const os = require('os');
+const path = require('path');
+
+function convertHtmlTableToAsciiDoc(htmlTable) {
+  return new Promise((resolve, reject) => {
+    pandoc(htmlTable, '-f html -t asciidoc', (err, result) => {
+      if (err) {
+        console.error(`Error converting HTML table to AsciiDoc: ${err.message}`);
+        resolve(htmlTable);
+      } else {
+        resolve(result);
+      }
+    });
+  });
+}
+
+function markdownToAsciidoc(markdown) {
+  const tempMarkdownPath = path.join(os.tmpdir(), 'temp_markdown.md');
+  fs.writeFileSync(tempMarkdownPath, markdown, 'utf-8');
+
+  let result;
+  try {
+    const command = `kramdoc -o - "${tempMarkdownPath}"`;
+    result = execSync(command, { encoding: 'utf-8' });
+  } catch (err) {
+    console.error(`Error converting Markdown to AsciiDoc: ${err.message}`);
+    result = markdown;
+  } finally {
+    fs.unlinkSync(tempMarkdownPath);
+  }
+  return result;
+}
+
+function processTabs(match) {
+  const tabItems = [...match.matchAll(/\s?<TabItem[^>]*value="([^"]+)"[^>]*label="([^"]+)"[^>]*>([\s\S]*?)<\/TabItem>/g)];
+
+  let result = ['\n<!--\n[tabs]'];
+  result.push('=====');
+  for (const tabItem of tabItems) {
+    const [_, value, label, content] = tabItem;
+    result.push(`${label}::`);
+    result.push('+');
+    result.push('--');
+    const asciidocContent = markdownToAsciidoc(content.trim(), '');
+    result.push(asciidocContent);
+    result.push('--');
+  }
+
+  result.push('=====');
+  result.push('-->');
+  return result.join('\n');
+}
+
+function processDetails(match) {
+  const detailsRegex = /<details>(?:\r?\n)<summary>([\s\S]*?)<\/summary>(?:\r?\n)([\s\S]*?)(?:\r?\n)<\/details>/g;
+
+  return match.replace(detailsRegex, (match, title, content) => {
+    const asciidocTitle = `.${title.trim()}`;
+    const asciidocBlock = `[%collapsible%]\n====\n${content.trim()}\n====`;
+
+    return `<!--\n${asciidocTitle}\n${asciidocBlock}\n-->`;
+  });
+}
+
+async function convertFile(file) {
+  const content = fs.readFileSync(file, 'utf-8');
+
+  var newContent = content.replace(/<Tabs>([\s\S]*?)<\/Tabs>/g, processTabs);
+  newContent = newContent.replace(/<details>([\s\S]*?)<\/details>/g, processDetails);
+
+  const htmlTableMatches = newContent.match(/\s?(<table>((.|\n)*?)<\/table>)/g);
+  if (htmlTableMatches) {
+    for (const htmlTableMatch of htmlTableMatches) {
+      const tableRegex = /(<table>((.|\n)*?)<\/table>)/;
+      const tableMatch = htmlTableMatch.match(tableRegex);
+      if (tableMatch) {
+        const htmlTable = tableMatch[0];
+        const asciidocTable = await convertHtmlTableToAsciiDoc(htmlTable);
+        newContent = newContent.replace(htmlTableMatch, `\n<!--\n${asciidocTable}\n-->`);
+      }
+    }
+  }
+
+  fs.writeFileSync(file, newContent, 'utf-8');
+}
+
+const inputFile = process.argv[2];
+if (!inputFile) {
+  console.error('No input file provided');
+  process.exit(1);
+}
+
+convertFile(inputFile).catch((error) => {
+  console.error(`Error processing file: ${error.message}`);
+  process.exit(1);
+});

package/tools/fetch-from-github.js

@@ -0,0 +1,63 @@
+const fs = require('fs');
+const path = require('path');
+
+let octokitInstance = null;
+async function loadOctokit() {
+  if (!octokitInstance) {
+    const { Octokit } = await import('@octokit/rest');
+    octokitInstance = process.env.VBOT_GITHUB_API_TOKEN
+      ? new Octokit({
+          auth: process.env.VBOT_GITHUB_API_TOKEN,
+        })
+      : new Octokit();
+
+    if (!process.env.VBOT_GITHUB_API_TOKEN) {
+      console.warn(
+        'Warning: No GitHub token found (VBOT_GITHUB_API_TOKEN). API rate limits will be restricted.'
+      );
+    }
+  }
+  return octokitInstance;
+}
+
+async function saveFile(content, saveDir, filename) {
+  await fs.promises.mkdir(saveDir, { recursive: true });
+  const target = path.join(saveDir, filename);
+  await fs.promises.writeFile(target, content);
+  console.log(`Saved: ${target}`);
+}
+
+async function fetchFromGithub(owner, repo, remotePath, saveDir, customFilename) {
+  const octokit = await loadOctokit();
+
+  try {
+    const resp = await octokit.repos.getContent({ owner, repo, path: remotePath });
+    if (Array.isArray(resp.data)) {
+      // directory
+      for (const item of resp.data) {
+        if (item.type === 'file') {
+          await fetchFromGithub(owner, repo, item.path, saveDir, customFilename);
+        } else if (item.type === 'dir') {
+          // For directories, maintain the directory structure
+          const nestedDir = path.join(saveDir, path.basename(item.path));
+          await fetchFromGithub(owner, repo, item.path, nestedDir);
+        }
+      }
+    } else {
+      // single file
+      const content = Buffer.from(resp.data.content, 'base64').toString();
+      const filename = customFilename || path.basename(resp.data.path);
+      await saveFile(content, saveDir, filename);
+    }
+  } catch (error) {
+    if (error.status === 403 && error.message.includes('rate limit')) {
+      throw new Error(`GitHub API rate limit exceeded. Consider using a token via VBOT_GITHUB_API_TOKEN environment variable.`);
+    } else if (error.status === 404) {
+      throw new Error(`Path not found: ${remotePath} in ${owner}/${repo}`);
+    } else {
+      throw new Error(`Failed to fetch from GitHub: ${error.message}`);
+    }
+  }
+}
+
+module.exports = fetchFromGithub