@redpanda-data/docs-extensions-and-macros 4.4.2 → 4.6.0

package/docker-compose/25.1/transform/transform.go

@@ -0,0 +1,122 @@
+package main
+// This data transform filters records based on a customizable regex pattern.
+// If a record's key or value
+// (determined by an environment variable) matches the specified regex,
+// the record is forwarded to the output.
+// Otherwise, it is dropped.
+//
+// Usage:
+// 1. Provide the following environment variables in your Docker or configuration setup:
+//    - PATTERN     : (required) a regular expression that determines what you want to match.
+//    - MATCH_VALUE : (optional) a boolean to decide whether to check the record value. If false,
+//                   the record key is checked. Default is false.
+//
+// Example environment variables:
+//   PATTERN=".*\\.edu$"
+//   MATCH_VALUE="true"
+//
+// Logs:
+// This transform logs information about each record and whether it matched.
+// The logs appear in the _redpanda.transform_logs topic, so you can debug how your records are being processed.
+//
+// Build instructions:
+//   go mod tidy
+//   rpk transform build
+//
+// For more details on building transforms with the Redpanda SDK, see:
+// https://docs.redpanda.com/current/develop/data-transforms
+//
+
+import (
+	"log"
+	"os"
+	"regexp"
+	"strings"
+
+	"github.com/redpanda-data/redpanda/src/transform-sdk/go/transform"
+)
+
+var (
+	re         *regexp.Regexp
+	checkValue bool
+)
+
+func isTrueVar(v string) bool {
+	switch strings.ToLower(v) {
+	case "yes", "ok", "1", "true":
+		return true
+	default:
+		return false
+	}
+}
+
+// The main() function runs only once at startup. It performs all initialization steps:
+// - Reads and compiles the regex pattern.
+// - Determines whether to match on the key or value.
+// - Registers the doRegexFilter() function to process records.
+func main() {
+	// Set logging preferences, including timestamp and UTC time.
+	log.SetPrefix("[regex-transform] ")
+	log.SetFlags(log.Ldate | log.Ltime | log.LUTC | log.Lmicroseconds)
+
+	// Start logging the transformation process
+	log.Println("Starting transform...")
+
+	// Read the PATTERN environment variable to get the regex pattern.
+	pattern, ok := os.LookupEnv("PATTERN")
+	if !ok {
+		log.Fatal("Missing PATTERN environment variable")
+	}
+	// Log the regex pattern being used.
+	log.Printf("Using PATTERN: %q\n", pattern)
+	// Compile the regex pattern for later use.
+	re = regexp.MustCompile(pattern)
+
+	// Read the MATCH_VALUE environment variable to determine whether to check the record's value.
+	mk, ok := os.LookupEnv("MATCH_VALUE")
+	checkValue = ok && isTrueVar(mk)
+	log.Printf("MATCH_VALUE set to: %t\n", checkValue)
+
+	log.Println("Initialization complete, waiting for records...")
+
+	// Listen for records to be written, calling doRegexFilter() for each record.
+	transform.OnRecordWritten(doRegexFilter)
+}
+
+// The doRegexFilter() function executes each time a new record is written.
+// It checks whether the record's key or value (based on MATCH_VALUE) matches the compiled regex.
+// If it matches, the record is forwarded, if not, it's dropped.
+func doRegexFilter(e transform.WriteEvent, w transform.RecordWriter) error {
+	// This stores the data to be checked (either the key or value).
+	var dataToCheck []byte
+
+	// Depending on the MATCH_VALUE environment variable, decide whether to check the record's key or value.
+	if checkValue {
+		// Use the value of the record if MATCH_VALUE is true.
+		dataToCheck = e.Record().Value
+		log.Printf("Checking record value: %s\n", string(dataToCheck))
+	} else {
+		// Use the key of the record if MATCH_VALUE is false.
+		dataToCheck = e.Record().Key
+		log.Printf("Checking record key: %s\n", string(dataToCheck))
+	}
+
+	// If there is no key or value to check, log and skip the record.
+	if dataToCheck == nil {
+		log.Println("Record has no key/value to check, skipping.")
+		return nil
+	}
+
+	// Check if the data matches the regex pattern.
+	pass := re.Match(dataToCheck)
+	if pass {
+		// If the record matches the pattern, log and write the record to the output topic.
+		log.Printf("Record matched pattern, passing through. Key: %s, Value: %s\n", string(e.Record().Key), string(e.Record().Value))
+		return w.Write(e.Record())
+	} else {
+		// If the record does not match the pattern, log and drop the record.
+		log.Printf("Record did not match pattern, dropping. Key: %s, Value: %s\n", string(e.Record().Key), string(e.Record().Value))
+		// Do not write the record if it doesn't match the pattern.
+		return nil
+	}
+}

package/docker-compose/25.1/transform/transform.yaml

@@ -0,0 +1,33 @@
+# Transform metadata used by the rpk transform build command.
+# This metadata file tells rpk:
+# 1) The transform’s display name, which also becomes the base for the .wasm file name.
+# 2) A brief description of what it does.
+# 3) Defaults for environment variables.
+# 4) Input and output topics (if you want to define them here rather than in the deploy command).
+
+# Human-readable name of the transform. rpk transform build uses this for the generated .wasm file.
+name: regex
+
+description: |
+  Filters the input topic to records that only match a regular expression.
+
+  Regular expressions are implemented using Go's regexp library, which uses the syntax of RE2.
+  See the RE2 wiki for allowed syntax: https://github.com/google/re2/wiki/Syntax
+
+  Environment variables:
+  - PATTERN: The regular expression that will match against records (required).
+  - MATCH_VALUE: By default, the regex matches keys, but if set to "true", the regex matches values.
+
+# By default, no input topic is set here. (You can set it in your deploy command if preferred.)
+input-topic: ""
+
+# By default, no output topic is set here. (You can set it in your deploy command if preferred.)
+output-topic: ""
+
+# Indicates the specific TinyGo environment used to compile your transform.
+language: tinygo-no-goroutines
+
+env:
+  # The PATTERN variable must be provided at deploy time.
+  # Example: --var=PATTERN=".*@example.com"
+  PATTERN: '<required>'

package/docker-compose/bootstrap.yml

@@ -44,18 +44,6 @@ cloud_storage_segment_max_upload_interval_sec: 60
 # Continuous Data Balancing (enterprise feature) continuously monitors your node and rack availability and disk usage. This enables self-healing clusters that dynamically balance partitions, ensuring smooth operations and optimal cluster performance.
 # https://docs.redpanda.com/current/manage/cluster-maintenance/continuous-data-balancing/
 partition_autobalancing_mode: continuous
-# Enable Redpanda to collect consumer group metrics.
-# https://docs.redpanda.com/current/reference/properties/cluster-properties/#enable_consumer_group_metrics
-enable_consumer_group_metrics:
-  - "group"
-  - "partition"
-  - "consumer_lag"
-# Lower the interval for the autogeneration of consumer group metrics.
-# https://docs.redpanda.com/current/reference/properties/cluster-properties/#consumer_group_lag_collection_interval_sec
-consumer_group_lag_collection_interval_sec: 60
-# Enable Redpanda to collect host metrics.
-# https://docs.redpanda.com/current/reference/properties/cluster-properties/#enable_host_metrics
-enable_host_metrics: true
 # Enable for Iceberg metrics
 iceberg_enabled: true
 # Set up Iceberg REST catalog configuration

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redpanda-data/docs-extensions-and-macros",
-  "version": "4.4.2",
+  "version": "4.6.0",
   "description": "Antora extensions and macros developed for Redpanda documentation.",
   "keywords": [
     "antora",
@@ -20,7 +20,8 @@
     "get-redpanda-version": "doc-tools get-redpanda-version",
     "get-console-version": "doc-tools get-console-version",
     "build": "antora --to-dir docs --fetch local-antora-playbook.yml",
-    "serve": "wds --node-resolve --open preview/test/ --watch --root-dir docs"
+    "serve": "wds --node-resolve --open preview/test/ --watch --root-dir docs",
+    "test": "jest"
   },
   "contributors": [
     {
@@ -56,8 +57,7 @@
     "./macros/glossary": "./macros/glossary.js",
     "./macros/rp-connect-components": "./macros/rp-connect-components.js",
     "./macros/config-ref": "./macros/config-ref.js",
-    "./macros/helm-ref": "./macros/helm-ref.js",
-    "./macros/data-template": "./macros/data-template.js"
+    "./macros/helm-ref": "./macros/helm-ref.js"
   },
   "files": [
     "extensions",
@@ -101,6 +101,7 @@
   "devDependencies": {
     "@antora/cli": "3.1.4",
     "@antora/site-generator": "3.1.4",
-    "@web/dev-server": "^0.2.5"
+    "@web/dev-server": "^0.2.5",
+    "jest": "^29.7.0"
   }
 }

package/tools/fetch-from-github.js

@@ -12,8 +12,8 @@ async function loadOctokit() {
       : new Octokit();
 
     if (!process.env.VBOT_GITHUB_API_TOKEN) {
-      console.warn(
-        'Warning: No GitHub token found (VBOT_GITHUB_API_TOKEN). API rate limits will be restricted.'
+      console.info(
+        'No GitHub token found (VBOT_GITHUB_API_TOKEN).'
       );
     }
   }

package/tools/redpanda-connect/generate-rpcn-connector-docs.js

@@ -0,0 +1,205 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const handlebars = require('handlebars');
+const yaml = require('yaml');
+const helpers = require('./helpers');
+
+// Register each helper under handlebars, verifying that it’s a function
+Object.entries(helpers).forEach(([name, fn]) => {
+  if (typeof fn !== 'function') {
+    console.error(`❌ Helper "${name}" is not a function`);
+    process.exit(1);
+  }
+  handlebars.registerHelper(name, fn);
+});
+
+// Default “main” template (connector.hbs) which invokes partials {{> intro}}, {{> fields}}, {{> examples}}
+const DEFAULT_TEMPLATE = path.resolve(__dirname, './templates/connector.hbs');
+
+/**
+ * Reads a file at `filePath` and registers it as a Handlebars partial called `name`.
+ * Throws if the file cannot be read.
+ */
+function registerPartial(name, filePath) {
+  const resolved = path.resolve(filePath);
+  let source;
+  try {
+    source = fs.readFileSync(resolved, 'utf8');
+  } catch (err) {
+    throw new Error(`Unable to read "${name}" template at ${resolved}: ${err.message}`);
+  }
+  handlebars.registerPartial(name, source);
+}
+
+/**
+ * Deep-merge `overrides` into `target`. Only 'description', 'type',
+ * plus nested array/object entries get overridden; other keys remain intact.
+ */
+function mergeOverrides(target, overrides) {
+  if (!overrides || typeof overrides !== 'object') return target;
+  if (!target || typeof target !== 'object') {
+    throw new Error('Target must be a valid object');
+  }
+  for (const key in overrides) {
+    if (Array.isArray(target[key]) && Array.isArray(overrides[key])) {
+      // Merge two parallel arrays by matching items on `.name`
+      target[key] = target[key].map(item => {
+        const overrideItem = overrides[key].find(o => o.name === item.name);
+        if (overrideItem) {
+          // Overwrite description/type if present
+          ['description', 'type'].forEach(field => {
+            if (Object.hasOwn(overrideItem, field)) {
+              item[field] = overrideItem[field];
+            }
+          });
+          // Recurse for nested children
+          item = mergeOverrides(item, overrideItem);
+        }
+        return item;
+      });
+    } else if (
+      typeof target[key] === 'object' &&
+      typeof overrides[key] === 'object' &&
+      !Array.isArray(target[key]) &&
+      !Array.isArray(overrides[key])
+    ) {
+      // Deep-merge plain objects
+      target[key] = mergeOverrides(target[key], overrides[key]);
+    } else if (['description', 'type'].includes(key) && Object.hasOwn(overrides, key)) {
+      // Overwrite the primitive
+      target[key] = overrides[key];
+    }
+  }
+  return target;
+}
+
+async function generateRpcnConnectorDocs(options) {
+  const {
+    data,
+    overrides,
+    template,            // main Handlebars template (for full-draft mode)
+    templateIntro,
+    templateFields,
+    templateExamples,
+    writeFullDrafts
+  } = options;
+
+  // Read connector index (JSON or YAML)
+  const raw = fs.readFileSync(data, 'utf8');
+  const ext = path.extname(data).toLowerCase();
+  const dataObj = ext === '.json' ? JSON.parse(raw) : yaml.parse(raw);
+
+  // Apply overrides if provided
+  if (overrides) {
+    const ovRaw = fs.readFileSync(overrides, 'utf8');
+    const ovObj = JSON.parse(ovRaw);
+    mergeOverrides(dataObj, ovObj);
+  }
+
+  // Compile the “main” template (used when writeFullDrafts = true)
+  const compiledTemplate = handlebars.compile(fs.readFileSync(template, 'utf8'));
+
+  // Determine which templates to use for “fields” and “examples”
+  // If templateFields is not provided, fall back to the single `template`.
+  // If templateExamples is not provided, skip examples entirely.
+  const fieldsTemplatePath   = templateFields   || template;
+  const examplesTemplatePath = templateExamples || null;
+
+  // Register partials
+  if (!writeFullDrafts) {
+    if (fieldsTemplatePath) {
+      registerPartial('fields', fieldsTemplatePath);
+    }
+    if (examplesTemplatePath) {
+      registerPartial('examples', examplesTemplatePath);
+    }
+  } else {
+    registerPartial('intro', templateIntro);
+  }
+
+  const outputRoot     = path.resolve(process.cwd(), 'modules/components/partials');
+  const fieldsOutRoot   = path.join(outputRoot, 'fields');
+  const examplesOutRoot = path.join(outputRoot, 'examples');
+  const draftsRoot      = path.join(outputRoot, 'drafts');
+
+  if (!writeFullDrafts) {
+    fs.mkdirSync(fieldsOutRoot,   { recursive: true });
+    fs.mkdirSync(examplesOutRoot, { recursive: true });
+  }
+
+  let partialsWritten = 0;
+  let draftsWritten   = 0;
+  const partialFiles  = [];
+  const draftFiles    = [];
+
+  for (const [type, items] of Object.entries(dataObj)) {
+    if (!Array.isArray(items)) continue;
+
+    for (const item of items) {
+      if (!item.name) continue;
+      const name = item.name;
+
+      if (!writeFullDrafts) {
+        // Render fields using the registered “fields” partial
+        const fieldsOut = handlebars
+          .compile('{{> fields children=config.children}}')(item);
+
+        // Render examples only if an examples template was provided
+        let examplesOut = '';
+        if (examplesTemplatePath) {
+          examplesOut = handlebars
+            .compile('{{> examples examples=examples}}')(item);
+        }
+
+        if (fieldsOut.trim()) {
+          const fPath = path.join(fieldsOutRoot, type, `${name}.adoc`);
+          fs.mkdirSync(path.dirname(fPath), { recursive: true });
+          fs.writeFileSync(fPath, fieldsOut);
+          partialsWritten++;
+          partialFiles.push(path.relative(process.cwd(), fPath));
+        }
+
+        if (examplesOut.trim()) {
+          const ePath = path.join(examplesOutRoot, type, `${name}.adoc`);
+          fs.mkdirSync(path.dirname(ePath), { recursive: true });
+          fs.writeFileSync(ePath, examplesOut);
+          partialsWritten++;
+          partialFiles.push(path.relative(process.cwd(), ePath));
+        }
+      }
+
+      if (writeFullDrafts) {
+        let content;
+        try {
+          content = compiledTemplate(item);
+        } catch (err) {
+          throw new Error(`Template render failed for component "${name}": ${err.message}`);
+        }
+
+        const draftSubdir = name === 'gateway'
+          ? path.join(draftsRoot, 'cloud-only')
+          : draftsRoot;
+
+        const destFile = path.join(draftSubdir, `${name}.adoc`);
+        fs.mkdirSync(path.dirname(destFile), { recursive: true });
+        fs.writeFileSync(destFile, content, 'utf8');
+        draftsWritten++;
+        draftFiles.push(path.relative(process.cwd(), destFile));
+      }
+    }
+  }
+
+  return {
+    partialsWritten,
+    draftsWritten,
+    partialFiles,
+    draftFiles
+  };
+}
+
+module.exports = {
+  generateRpcnConnectorDocs,
+  mergeOverrides
+};

package/tools/redpanda-connect/helpers/advancedConfig.js

@@ -0,0 +1,17 @@
+const buildConfigYaml = require('./buildConfigYaml.js');
+const handlebars = require('handlebars');
+
+/**
+ * Handlebars helper “advancedConfig”.  Omits only deprecated.
+ *
+ * Usage in template:
+ *   {{advancedConfig this.type this.name this.config.children}}
+ */
+module.exports = function advancedConfig(type, connectorName, children) {
+  if (typeof type !== 'string' || typeof connectorName !== 'string' || !Array.isArray(children)) {
+    return '';
+  }
+
+  const yamlText = buildConfigYaml(type, connectorName, children, /*includeAdvanced=*/ true);
+  return new handlebars.SafeString(yamlText);
+}

package/tools/redpanda-connect/helpers/buildConfigYaml.js

@@ -0,0 +1,53 @@
+const renderLeafField = require('./renderLeafField');
+const renderObjectField = require('./renderObjectField');
+
+/**
+ * Builds either “Common” or “Advanced” YAML for one connector.
+ *
+ * - type            = “input” or “output” (or whatever type)
+ * - connectorName   = such as “amqp_1”
+ * - children        = the array of field‐definitions (entry.config.children)
+ * - includeAdvanced = if false → only fields where is_advanced !== true
+ *                      if true  → all fields (except deprecated)
+ *
+ * Structure produced:
+ *
+ *   type:
+ *     label: ""
+ *     connectorName:
+ *       ...child fields (with comments for “no default”)
+ */
+module.exports = function buildConfigYaml(type, connectorName, children, includeAdvanced) {
+  const lines = [];
+
+  // “type:” top‐level
+  lines.push(`${type}:`);
+
+  // Two‐space indent for “label”
+  lines.push(`  label: ""`);
+
+  // Two‐space indent for connectorName heading
+  lines.push(`  ${connectorName}:`);
+
+  // Four‐space indent for children
+  const baseIndent = 4;
+  children.forEach(field => {
+    if (field.is_deprecated) {
+      return; // skip deprecated fields
+    }
+    if (!includeAdvanced && field.is_advanced) {
+      return; // skip advanced fields in “common” mode
+    }
+
+    if (field.type === 'object' && Array.isArray(field.children)) {
+      // Render nested object
+      const nestedLines = renderObjectField(field, baseIndent);
+      lines.push(...nestedLines);
+    } else {
+      // Render a scalar or array leaf
+      lines.push(renderLeafField(field, baseIndent));
+    }
+  });
+
+  return lines.join('\n');
+}

package/tools/redpanda-connect/helpers/commonConfig.js

@@ -0,0 +1,31 @@
+const buildConfigYaml = require('./buildConfigYaml.js');
+const handlebars = require('handlebars');
+
+/**
+ * Handlebars helper “commonConfig”.  Omits deprecated + advanced.
+ *
+ * Usage in template:
+ *   {{commonConfig this.type this.name this.config.children}}
+ */
+module.exports = function commonConfig(type, connectorName, children) {
+  if (typeof type !== 'string' || !type.trim()) {
+    console.warn('commonConfig: type must be a non-empty string');
+    return '';
+  }
+  if (typeof connectorName !== 'string' || !connectorName.trim()) {
+    console.warn('commonConfig: connectorName must be a non-empty string');
+    return '';
+  }
+  if (!Array.isArray(children)) {
+    console.warn('commonConfig: children must be an array');
+    return '';
+  }
+
+  try {
+    const yamlText = buildConfigYaml(type, connectorName, children, /*includeAdvanced=*/ false);
+    return new handlebars.SafeString(yamlText);
+  } catch (error) {
+    console.error('Error in commonConfig helper:', error);
+    return new handlebars.SafeString('<!-- Error generating configuration -->');
+  }
+}

package/tools/redpanda-connect/helpers/eq.js

@@ -0,0 +1,10 @@
+/**
+ * Checks if two values are equal.
+ *
+ * @param {*} a - The first value.
+ * @param {*} b - The second value.
+ * @returns {boolean} True if the values are equal.
+ */
+module.exports = function eq(a, b) {
+  return a === b;
+}

package/tools/redpanda-connect/helpers/index.js

@@ -0,0 +1,19 @@
+'use strict';
+
+module.exports = {
+  uppercase:       require('./uppercase.js'),
+  eq:              require('./eq.js'),
+  ne:              require('./ne.js'),
+  join:              require('./join.js'),
+  or:              require('./or.js'),
+  toYaml:              require('./toYaml.js'),
+  isObject:              require('./isObject.js'),
+  renderYamlList:  require('./renderYamlList.js'),
+  renderConnectFields:    require('./renderConnectFields.js'),
+  renderConnectExamples:  require('./renderConnectExamples.js'),
+  renderLeafField:        require('./renderLeafField.js'),
+  renderObjectField:      require('./renderObjectField.js'),
+  buildConfigYaml:        require('./buildConfigYaml.js'),
+  commonConfig:           require('./commonConfig.js'),
+  advancedConfig:         require('./advancedConfig.js'),
+};

package/tools/redpanda-connect/helpers/isObject.js

@@ -0,0 +1 @@
+module.exports = v => v !== null && typeof v === 'object' && !Array.isArray(v)

package/tools/redpanda-connect/helpers/join.js

@@ -0,0 +1,6 @@
+module.exports = function join(array, separator) {
+  if (!Array.isArray(array)) {
+    return '';
+  }
+  return array.join(separator);
+}

package/tools/redpanda-connect/helpers/ne.js

@@ -0,0 +1,10 @@
+/**
+ * Checks if two values are not equal.
+ *
+ * @param {*} a - The first value.
+ * @param {*} b - The second value.
+ * @returns {boolean} True if the values are not equal.
+ */
+module.exports = function ne(a, b) {
+  return a !== b;
+}

package/tools/redpanda-connect/helpers/or.js

@@ -0,0 +1,4 @@
+module.exports = function (...args) {
+    const options = args.pop(); // Last argument is always the options object
+    return args.some(Boolean);
+};

package/tools/redpanda-connect/helpers/renderConnectExamples.js

@@ -0,0 +1,37 @@
+const handlebars = require('handlebars');
+
+/**
+ * Renders a list of examples.
+ *
+ * @param {Array<Object>} examples - An array of example objects.
+ * @returns {handlebars.SafeString} The rendered SafeString containing the examples.
+ */
+module.exports = function renderConnectExamples(examples) {
+  if (!examples || !Array.isArray(examples) || examples.length === 0) {
+    return '';
+  }
+  let output = '';
+  examples.forEach(example => {
+    if (example.title) {
+      const sanitizedTitle = example.title.replace(/[=]/g, '\\=');
+      output += `=== ${sanitizedTitle}\n\n`;
+    }
+    if (example.summary) {
+      output += `${example.summary}\n\n`;
+    }
+    if (example.config) {
+      if (typeof example.config !== 'string') {
+        console.warn('Example config must be a string, skipping');
+        return;
+      }
+      const configContent = example.config.trim();
+      if (configContent.includes('----')) {
+        console.warn('Example config contains AsciiDoc delimiters, this may break rendering');
+      }
+      output += '[source,yaml]\n----\n';
+      output += configContent + '\n';
+      output += '----\n\n';
+    }
+  });
+  return new handlebars.SafeString(output);
+}