@redpanda-data/docs-extensions-and-macros 4.4.1 → 4.5.0

package/docker-compose/25.1/generate-profiles.yaml

@@ -0,0 +1,77 @@
+input:
+  # Use the 'generate' input
+  # https://docs.redpanda.com/redpanda-connect/components/inputs/generate/
+  generate:
+    # The interval at which new records are generated.
+    interval: 1s
+    # The mapping section defines how each generated record is structured.
+    # The language used here is called Bloblang.
+    # https://docs.redpanda.com/redpanda-connect/guides/bloblang/about/
+    mapping: |
+      # Generate a fake first name using the 'first_name' faker function.
+      let first_name = fake("first_name")
+
+      # Generate a fake last name using the 'last_name' faker function.
+      let last_name  = fake("last_name")
+
+      # Define possible subscription levels for users.
+      let subscription_levels = ["Free", "Basic", "Premium"]
+
+      # Define possible notification channels for user preferences.
+      let notifications = ["email", "sms", "push" ]
+
+      # Define supported languages for user preferences.
+      let languages = ["en", "es", "fr", "de", "zh", "jp"]
+
+      # Assign a unique user ID using a UUID digit generator.
+      root.user_id = fake("uuid_digit")
+
+      # Assign the generated first name to the 'first_name' field.
+      root.first_name = $first_name
+
+      # Assign the generated last name to the 'last_name' field.
+      root.last_name = $last_name
+
+      # Construct the user's email by combining the first initial, last name, and a fake domain name.
+      # The email is converted to lowercase for consistency.
+      root.email = ($first_name.slice(0,1) + $last_name + "@" + fake("domain_name")).lowercase()
+
+      # Assign a fake registration date using the 'date' faker function.
+      root.registration_date = fake("date")
+
+      # Assign the current timestamp as the last login time.
+      root.last_login = now()
+
+      # Randomly assign a subscription level by selecting an index from the 'subscription_levels' array.
+      root.subscription_level = $subscription_levels.index(random_int(min: 0, max: 2))
+
+      # Randomly assign a language preference by selecting an index from the 'languages' array.
+      root.preferences.language = $languages.index(random_int(min: 0, max: 5))
+
+      # Randomly assign a notification preference by selecting an index from the 'notifications' array.
+      root.preferences.notifications = $notifications.index(random_int(min: 0, max: 2))
+pipeline:
+  processors:
+    - mapping: |
+        # Set the target topic for the generated records to 'profiles'.
+        meta topic = "profiles"
+
+        # Assign the entire record (root) to be sent to the specified topic.
+        root = this
+output:
+  # Use the 'kafka_franz' output to send the result back to Redpanda
+  # https://docs.redpanda.com/redpanda-connect/components/outputs/kafka_franz/
+  kafka_franz:
+    # Define the list of seed brokers for the Kafka cluster.
+    seed_brokers: [ "localhost:19092", "localhost:29092", "localhost:39092"]
+    # Dynamically assign the topic based on the metadata specified in the processors.
+    # In this case, it resolves to the 'profiles' topic.
+    topic: ${! metadata("topic") }
+    # Configure SASL authentication to securely connect to the Kafka brokers.
+    sasl:
+      - # Specify the SASL mechanism to use for authentication.
+        mechanism: SCRAM-SHA-256
+        # The password for the SASL authentication.
+        password: secretpassword
+        # The username for the SASL authentication.
+        username: superuser

package/docker-compose/25.1/rpk-profile.yaml

@@ -0,0 +1,24 @@
+# This file configures `rpk` to connect to a remote Redpanda cluster running in the same local network as `rpk`.
+
+# Configuration for connecting to the Kafka API of the Redpanda cluster.
+kafka_api:
+  # SASL (Simple Authentication and Security Layer) settings for authentication.
+  sasl:
+    user: superuser # The username used for authentication
+    password: secretpassword # The password associated with the username
+    mechanism: scram-sha-256 # Authentication mechanism; SCRAM-SHA-256 provides secure password-based authentication
+  # List of Kafka brokers in the Redpanda cluster.
+  # These brokers ensure high availability and fault tolerance for Kafka-based communication.
+  brokers:
+    - 127.0.0.1:19092 # Broker 1: Accessible on localhost, port 19092
+    - 127.0.0.1:29092 # Broker 2: Accessible on localhost, port 29092
+    - 127.0.0.1:39092 # Broker 3: Accessible on localhost, port 39092
+
+# Configuration for connecting to the Redpanda Admin API.
+# The Admin API allows you to perform administrative tasks such as managing configurations, monitoring, and scaling.
+admin_api:
+   # List of Admin API endpoints for managing the cluster.
+  addresses:
+    - 127.0.0.1:19644 # Admin API for Broker 1: Accessible on localhost, port 19644
+    - 127.0.0.1:29644 # Admin API for Broker 2: Accessible on localhost, port 29644
+    - 127.0.0.1:39644 # Admin API for Broker 3: Accessible on localhost, port 39644

package/docker-compose/25.1/transactions-schema.json

@@ -0,0 +1,37 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "Transactions",
+  "type": "object",
+  "properties": {
+    "email": {
+      "type": "string",
+      "format": "email",
+      "description": "The email address of the user involved in the transaction."
+    },
+    "index": {
+      "type": "integer",
+      "description": "A numeric index associated with the transaction."
+    },
+    "price": {
+      "type": "string",
+      "pattern": "^[A-Z]{3} \\d+(?:\\.\\d{2})?$",
+      "description": "A string representing the price of the product, including a currency code (ISO 4217) and an amount with two decimal places by default."
+    },
+    "product_url": {
+      "type": "string",
+      "format": "uri",
+      "description": "A URL that points to the product involved in the transaction."
+    },
+    "timestamp": {
+      "type": "string",
+      "format": "date-time",
+      "description": "The timestamp of when the transaction occurred, formatted in ISO 8601."
+    },
+    "user_id": {
+      "type": "integer",
+      "description": "A numeric identifier for the user."
+    }
+  },
+  "required": ["email", "index", "price", "product_url", "timestamp", "user_id"],
+  "additionalProperties": false
+}

package/docker-compose/25.1/transactions.md

@@ -0,0 +1,46 @@
+# Transactions Topic Documentation
+
+This document provides an overview of the `transactions` topic in the Redpanda cluster. The topic is designed to capture autogenerated transaction events with various attributes.
+
+## Schema Overview
+
+Each message in the `transactions` topic adheres to the following JSON schema:
+
+```json
+{
+    "email": "string",
+    "index": "integer",
+    "price": "string",
+    "product_url": "string",
+    "timestamp": "string",
+    "user_id": "integer"
+}
+```
+
+- **email**: The email address of the user involved in the transaction.
+- **index**: A numeric index associated with the transaction. This could represent the position or order of the transaction in a sequence.
+- **price**: A string representing the price of the product. It includes a currency code (e.g., "XXX") followed by the amount.
+- **product_url**: A URL that points to the product involved in the transaction.
+- **timestamp**: The timestamp of when the transaction occurred, formatted in ISO 8601.
+- **user_id**: A numeric identifier for the user. This is typically a unique ID assigned to each user in the system.
+
+## Example message
+
+```json
+{
+    "email": "wzieme@ykczius.edu",
+    "index": 0,
+    "price": "XXX 5651308.100000",
+    "product_url": "http://yjomdta.top/DxvGsCn.php",
+    "timestamp": "2024-08-16T15:51:19.799474084Z",
+    "user_id": 1
+}
+```
+
+## Use cases
+
+You can use the `transactions` topic for various purposes, including:
+
+- **Analytics**: Tracking and analyzing user transactions to understand buying behavior, popular products, etc.
+- **Monitoring**: Observing transaction patterns to detect anomalies, such as unusual spikes or drops in transaction volume.
+- **Data Processing**: Feeding transaction data into other systems, such as a data warehouse or real-time processing pipelines, for further processing and analysis.

package/docker-compose/25.1/transform/README.adoc

@@ -0,0 +1,73 @@
+= Modify the Wasm Transform in the Quickstart
+
+This directory contains the Go source code (`transform.go`) for the data transform that is used in the Redpanda Self-Managed quickstart.
+If you're following the quickstart, you *do not* need to modify or rebuild this code. The Docker Compose configuration automatically deploys a pre-built transform called `regex.wasm`.
+
+However, if you want to customize the data transform logic, continue reading.
+
+== Why customize the transform?
+
+- **Custom filtering**: Filter by a different regex or apply multiple conditions.
+- **Data manipulation**: Transform records before writing them out. For example, redacting sensitive data or combining fields.
+- **Extended functionality**: Add advanced logging, error handling, or multi-topic routing.
+
+== Prerequisites
+
+You need the following:
+
+- At least Go 1.20 installed.
++
+[source,bash]
+----
+go version
+----
+
+- The Redpanda CLI (`rpk`) installed.
+
+- A running Redpanda cluster. If you're using the local quickstart with Docker Compose, ensure the cluster is up and running. Or, point `rpk` to another Redpanda environment.
+
+== Modify and deploy your transform
+
+. Open link:transform.go[transform.go] and make your changes. For example:
++
+--
+- Change the regex logic to handle different use cases.
+- Add environment variables to control new features.
+- Extend the `doRegexFilter()` function to manipulate records.
+--
+
+. Compile your Go code into a `.wasm` file:
++
+[source,bash]
+----
+rpk transform build
+----
++
+This command compiles your Go source and produces a `.wasm` file that you can deploy to Redpanda.
+
+. Deploy the new transform.
++
+If your Docker Compose setup already has a service to deploy the transform, you can restart that service.
++
+Otherwise, you can deploy your updated `.wasm` manually using `rpk transform deploy`.
+
+. Produce messages into the input topic. For example:
++
+[source,bash]
+----
+echo '{"key":"alice@university.edu","value":"test message"}' | rpk topic produce logins
+----
+
+. Consume from the output topic. For example:
++
+[source,bash]
+----
+rpk topic consume edu-filtered-domains --num 1
+----
+
+== Suggested reading
+
+- link:https://docs.redpanda.com/current/reference/rpk/[Redpanda `rpk` CLI Reference^].
+- link:https://docs.redpanda.com/current/develop/data-transforms/build/[Develop Data Transforms^].
+- https://golang.org/ref/mod[Go Modules^] for managing dependencies and builds in Go.
+- https://docs.docker.com/compose/[Docker Compose^] for customizing your environment.

package/docker-compose/25.1/transform/go.mod

@@ -0,0 +1,5 @@
+module regex
+
+go 1.20
+
+require github.com/redpanda-data/redpanda/src/transform-sdk/go/transform v1.1.0

package/docker-compose/25.1/transform/go.sum

@@ -0,0 +1,2 @@
+github.com/redpanda-data/redpanda/src/transform-sdk/go/transform v1.1.0 h1:KxgHJZsHsrT3YX7DMpu/vJN4TZN3KFm1jzrCFLyOepA=
+github.com/redpanda-data/redpanda/src/transform-sdk/go/transform v1.1.0/go.mod h1:QGgiwwf/BIsD1b7EiyQ/Apzw+RLSpasRDdpOCiefQFQ=

package/docker-compose/25.1/transform/transform.go

@@ -0,0 +1,122 @@
+package main
+// This data transform filters records based on a customizable regex pattern.
+// If a record's key or value
+// (determined by an environment variable) matches the specified regex,
+// the record is forwarded to the output.
+// Otherwise, it is dropped.
+//
+// Usage:
+// 1. Provide the following environment variables in your Docker or configuration setup:
+//    - PATTERN     : (required) a regular expression that determines what you want to match.
+//    - MATCH_VALUE : (optional) a boolean to decide whether to check the record value. If false,
+//                   the record key is checked. Default is false.
+//
+// Example environment variables:
+//   PATTERN=".*\\.edu$"
+//   MATCH_VALUE="true"
+//
+// Logs:
+// This transform logs information about each record and whether it matched.
+// The logs appear in the _redpanda.transform_logs topic, so you can debug how your records are being processed.
+//
+// Build instructions:
+//   go mod tidy
+//   rpk transform build
+//
+// For more details on building transforms with the Redpanda SDK, see:
+// https://docs.redpanda.com/current/develop/data-transforms
+//
+
+import (
+	"log"
+	"os"
+	"regexp"
+	"strings"
+
+	"github.com/redpanda-data/redpanda/src/transform-sdk/go/transform"
+)
+
+var (
+	re         *regexp.Regexp
+	checkValue bool
+)
+
+func isTrueVar(v string) bool {
+	switch strings.ToLower(v) {
+	case "yes", "ok", "1", "true":
+		return true
+	default:
+		return false
+	}
+}
+
+// The main() function runs only once at startup. It performs all initialization steps:
+// - Reads and compiles the regex pattern.
+// - Determines whether to match on the key or value.
+// - Registers the doRegexFilter() function to process records.
+func main() {
+	// Set logging preferences, including timestamp and UTC time.
+	log.SetPrefix("[regex-transform] ")
+	log.SetFlags(log.Ldate | log.Ltime | log.LUTC | log.Lmicroseconds)
+
+	// Start logging the transformation process
+	log.Println("Starting transform...")
+
+	// Read the PATTERN environment variable to get the regex pattern.
+	pattern, ok := os.LookupEnv("PATTERN")
+	if !ok {
+		log.Fatal("Missing PATTERN environment variable")
+	}
+	// Log the regex pattern being used.
+	log.Printf("Using PATTERN: %q\n", pattern)
+	// Compile the regex pattern for later use.
+	re = regexp.MustCompile(pattern)
+
+	// Read the MATCH_VALUE environment variable to determine whether to check the record's value.
+	mk, ok := os.LookupEnv("MATCH_VALUE")
+	checkValue = ok && isTrueVar(mk)
+	log.Printf("MATCH_VALUE set to: %t\n", checkValue)
+
+	log.Println("Initialization complete, waiting for records...")
+
+	// Listen for records to be written, calling doRegexFilter() for each record.
+	transform.OnRecordWritten(doRegexFilter)
+}
+
+// The doRegexFilter() function executes each time a new record is written.
+// It checks whether the record's key or value (based on MATCH_VALUE) matches the compiled regex.
+// If it matches, the record is forwarded, if not, it's dropped.
+func doRegexFilter(e transform.WriteEvent, w transform.RecordWriter) error {
+	// This stores the data to be checked (either the key or value).
+	var dataToCheck []byte
+
+	// Depending on the MATCH_VALUE environment variable, decide whether to check the record's key or value.
+	if checkValue {
+		// Use the value of the record if MATCH_VALUE is true.
+		dataToCheck = e.Record().Value
+		log.Printf("Checking record value: %s\n", string(dataToCheck))
+	} else {
+		// Use the key of the record if MATCH_VALUE is false.
+		dataToCheck = e.Record().Key
+		log.Printf("Checking record key: %s\n", string(dataToCheck))
+	}
+
+	// If there is no key or value to check, log and skip the record.
+	if dataToCheck == nil {
+		log.Println("Record has no key/value to check, skipping.")
+		return nil
+	}
+
+	// Check if the data matches the regex pattern.
+	pass := re.Match(dataToCheck)
+	if pass {
+		// If the record matches the pattern, log and write the record to the output topic.
+		log.Printf("Record matched pattern, passing through. Key: %s, Value: %s\n", string(e.Record().Key), string(e.Record().Value))
+		return w.Write(e.Record())
+	} else {
+		// If the record does not match the pattern, log and drop the record.
+		log.Printf("Record did not match pattern, dropping. Key: %s, Value: %s\n", string(e.Record().Key), string(e.Record().Value))
+		// Do not write the record if it doesn't match the pattern.
+		return nil
+	}
+}

package/docker-compose/25.1/transform/transform.yaml

@@ -0,0 +1,33 @@
+# Transform metadata used by the rpk transform build command.
+# This metadata file tells rpk:
+# 1) The transform’s display name, which also becomes the base for the .wasm file name.
+# 2) A brief description of what it does.
+# 3) Defaults for environment variables.
+# 4) Input and output topics (if you want to define them here rather than in the deploy command).
+
+# Human-readable name of the transform. rpk transform build uses this for the generated .wasm file.
+name: regex
+
+description: |
+  Filters the input topic to records that only match a regular expression.
+
+  Regular expressions are implemented using Go's regexp library, which uses the syntax of RE2.
+  See the RE2 wiki for allowed syntax: https://github.com/google/re2/wiki/Syntax
+
+  Environment variables:
+  - PATTERN: The regular expression that will match against records (required).
+  - MATCH_VALUE: By default, the regex matches keys, but if set to "true", the regex matches values.
+
+# By default, no input topic is set here. (You can set it in your deploy command if preferred.)
+input-topic: ""
+
+# By default, no output topic is set here. (You can set it in your deploy command if preferred.)
+output-topic: ""
+
+# Indicates the specific TinyGo environment used to compile your transform.
+language: tinygo-no-goroutines
+
+env:
+  # The PATTERN variable must be provided at deploy time.
+  # Example: --var=PATTERN=".*@example.com"
+  PATTERN: '<required>'

package/docker-compose/bootstrap.yml

@@ -44,18 +44,6 @@ cloud_storage_segment_max_upload_interval_sec: 60
 # Continuous Data Balancing (enterprise feature) continuously monitors your node and rack availability and disk usage. This enables self-healing clusters that dynamically balance partitions, ensuring smooth operations and optimal cluster performance.
 # https://docs.redpanda.com/current/manage/cluster-maintenance/continuous-data-balancing/
 partition_autobalancing_mode: continuous
-# Enable Redpanda to collect consumer group metrics.
-# https://docs.redpanda.com/current/reference/properties/cluster-properties/#enable_consumer_group_metrics
-enable_consumer_group_metrics:
-  - "group"
-  - "partition"
-  - "consumer_lag"
-# Lower the interval for the autogeneration of consumer group metrics.
-# https://docs.redpanda.com/current/reference/properties/cluster-properties/#consumer_group_lag_collection_interval_sec
-consumer_group_lag_collection_interval_sec: 60
-# Enable Redpanda to collect host metrics.
-# https://docs.redpanda.com/current/reference/properties/cluster-properties/#enable_host_metrics
-enable_host_metrics: true
 # Enable for Iceberg metrics
 iceberg_enabled: true
 # Set up Iceberg REST catalog configuration

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redpanda-data/docs-extensions-and-macros",
-  "version": "4.4.1",
+  "version": "4.5.0",
   "description": "Antora extensions and macros developed for Redpanda documentation.",
   "keywords": [
     "antora",

package/tools/fetch-from-github.js

@@ -12,8 +12,8 @@ async function loadOctokit() {
       : new Octokit();
 
     if (!process.env.VBOT_GITHUB_API_TOKEN) {
-      console.warn(
-        'Warning: No GitHub token found (VBOT_GITHUB_API_TOKEN). API rate limits will be restricted.'
+      console.info(
+        'No GitHub token found (VBOT_GITHUB_API_TOKEN).'
       );
     }
   }