@redpanda-data/docs-extensions-and-macros 4.3.0 → 4.4.0

package/cli-utils/start-cluster.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Usage: start-cluster.sh <tag>
+TAG="${1:-latest}"
+
+# Where this script lives (cli-utils)
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# One level up is the package root, where we expect docker‑compose/
+PACKAGE_ROOT="$(cd "$SCRIPT_DIR"/.. && pwd)"
+QUICKSTART_DIR="$PACKAGE_ROOT/docker-compose"
+
+# Remember where the user called us from
+CALLER_PWD="$(pwd)"
+
+# Default quickstart version
+MAJOR_MINOR="latest"
+if [[ "$TAG" != "latest" ]]; then
+  MAJOR_MINOR="$(echo "$TAG" | sed -E 's/^v?([0-9]+\.[0-9]+).*$/\1/')"
+fi
+
+# Fetch quickstart into package root if needed
+if [[ ! -d "$QUICKSTART_DIR" ]]; then
+  echo "📥 Fetching Redpanda quickstart for ${MAJOR_MINOR}…"
+  if [[ "$TAG" == "latest" ]]; then
+	  curl -sSLf --retry 3 https://docs.redpanda.com/redpanda-quickstart.tar.gz \
+	  | tar -C "$PACKAGE_ROOT" -xzf -
+  else
+    curl -sSLf --retry 3 "https://docs.redpanda.com/${MAJOR_MINOR}-redpanda-quickstart.tar.gz" \
+      | tar -C "$PACKAGE_ROOT" -xzf -
+  fi
+
+  if [[ ! -d "$QUICKSTART_DIR" ]]; then
+    echo "❌ Expected '$QUICKSTART_DIR' but none was found after extraction."
+    exit 1
+  fi
+fi
+
+# Switch into the quickstart dir and (re)start the cluster
+cd "$QUICKSTART_DIR"
+
+if docker compose ps | grep -q Up; then
+  echo "🛑 Stopping existing cluster…"
+  docker compose down --volumes
+fi
+
+echo "▶️  Starting Redpanda cluster…"
+docker compose up -d
+
+# Return to original directory
+cd "$CALLER_PWD"
+echo "✅ Cluster is up (version: ${TAG})"

package/docker-compose/bootstrap.yml

@@ -0,0 +1,67 @@
+# =================================================================
+# This file defines initial cluster properties for a Redpanda cluster.
+# Some of these settings are intended for quickstart development and evaluation
+# and are not suitable for production environments.
+#
+# For more information on bootstrap files, see:
+# https://docs.redpanda.com/current/deploy/deployment-option/self-hosted/manual/production/production-deployment/#configure-a-bootstrap-file
+# =================================================================
+
+#
+# Enable SASL authentication for the Kafka and Admin APIs.
+# https://docs.redpanda.com/current/reference/properties/cluster-properties/#admin_api_require_auth
+admin_api_require_auth: true
+# At least one superuser is required to be able to create other SASL users
+# https://docs.redpanda.com/current/reference/properties/cluster-properties/#superusers
+superusers:
+  - superuser
+# https://docs.redpanda.com/current/reference/properties/cluster-properties/#enable_sasl
+enable_sasl: true
+# Allow topics to be created on first access.
+# https://docs.redpanda.com/current/reference/properties/cluster-properties/#auto_create_topics_enabled
+auto_create_topics_enabled: true
+# Enable data transforms.
+# https://docs.redpanda.com/current/develop/data-transforms/how-transforms-work/
+data_transforms_enabled: true
+# Enable audit logging (enterprise feature).
+# https://docs.redpanda.com/current/manage/audit-logging/
+audit_enabled: true
+# Enable Tiered Storage (enterprise feature).
+# https://docs.redpanda.com/current/manage/tiered-storage/
+cloud_storage_enabled: true
+cloud_storage_region: local
+cloud_storage_access_key: minio
+cloud_storage_secret_key: redpandaTieredStorage7
+cloud_storage_api_endpoint: minio
+cloud_storage_api_endpoint_port: 9000
+cloud_storage_disable_tls: true
+cloud_storage_bucket: redpanda
+cloud_storage_enable_remote_write: true
+cloud_storage_enable_remote_read: true
+# Forces segments to be uploaded to Tiered Storage faster for the purposes of the quickstart
+# https://docs.redpanda.com/current/reference/properties/object-storage-properties/#cloud_storage_segment_max_upload_interval_sec
+cloud_storage_segment_max_upload_interval_sec: 60
+# Continuous Data Balancing (enterprise feature) continuously monitors your node and rack availability and disk usage. This enables self-healing clusters that dynamically balance partitions, ensuring smooth operations and optimal cluster performance.
+# https://docs.redpanda.com/current/manage/cluster-maintenance/continuous-data-balancing/
+partition_autobalancing_mode: continuous
+# Enable Redpanda to collect consumer group metrics.
+# https://docs.redpanda.com/current/reference/properties/cluster-properties/#enable_consumer_group_metrics
+enable_consumer_group_metrics:
+  - "group"
+  - "partition"
+  - "consumer_lag"
+# Lower the interval for the autogeneration of consumer group metrics.
+# https://docs.redpanda.com/current/reference/properties/cluster-properties/#consumer_group_lag_collection_interval_sec
+consumer_group_lag_collection_interval_sec: 60
+# Enable Redpanda to collect host metrics.
+# https://docs.redpanda.com/current/reference/properties/cluster-properties/#enable_host_metrics
+enable_host_metrics: true
+# Enable for Iceberg metrics
+iceberg_enabled: true
+# Set up Iceberg REST catalog configuration
+iceberg_catalog_type: rest
+iceberg_rest_catalog_endpoint: http://catalog:8181
+# Credentials are required, but the catalog ignores them
+iceberg_rest_catalog_client_id: catalog
+iceberg_rest_catalog_client_secret: catalog123
+iceberg_catalog_commit_interval_ms: 5000

package/docker-compose/docker-compose.yml

@@ -0,0 +1,414 @@
+name: redpanda-cluster
+networks:
+  redpanda_network:
+    driver: bridge
+volumes:
+  redpanda-0: null
+  redpanda-1: null
+  redpanda-2: null
+  minio: null
+services:
+  ##################
+  # Redpanda Brokers #
+  ##################
+  redpanda-0:
+    command:
+      - redpanda
+      - start
+      - --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:19092
+      # Address the broker advertises to clients that connect to the Kafka API.
+      # Use the internal addresses to connect to the Redpanda brokers
+      # from inside the same Docker network.
+      # Use the external addresses to connect to the Redpanda brokers
+      # from outside the Docker network.
+      - --advertise-kafka-addr internal://redpanda-0:9092,external://localhost:19092
+      - --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:18082
+      # Address the broker advertises to clients that connect to the HTTP Proxy.
+      - --advertise-pandaproxy-addr internal://redpanda-0:8082,external://localhost:18082
+      - --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:18081
+      # Redpanda brokers use the RPC API to communicate with each other internally.
+      - --rpc-addr redpanda-0:33145
+      - --advertise-rpc-addr redpanda-0:33145
+      # Mode dev-container uses well-known configuration properties for development in containers.
+      - --mode dev-container
+      # Tells Seastar (the framework Redpanda uses under the hood) to use 1 core on the system.
+      - --smp 1
+      - --default-log-level=info
+    image: docker.redpanda.com/redpandadata/${REDPANDA_DOCKER_REPO:-redpanda}:${REDPANDA_VERSION:-latest}
+    container_name: redpanda-0
+    # Sets the username and password of the bootstrap SCRAM superuser
+    # See https://docs.redpanda.com/current/deploy/deployment-option/self-hosted/manual/production/production-deployment/#bootstrap-a-user-account
+    environment:
+      RP_BOOTSTRAP_USER: "superuser:secretpassword"
+    volumes:
+      - redpanda-0:/var/lib/redpanda/data
+      - ./bootstrap.yml:/etc/redpanda/.bootstrap.yaml
+    networks:
+      - redpanda_network
+    ports:
+      - 18081:18081
+      - 18082:18082
+      - 19092:19092
+      - 19644:9644
+    healthcheck:
+      test: ["CMD", "rpk", "cluster", "info", "-X", "user=superuser", "-X", "pass=secretpassword"]
+      interval: 10s
+      timeout: 15s
+      retries: 10
+    depends_on:
+      minio:
+        condition: service_healthy
+  redpanda-1:
+    command:
+      - redpanda
+      - start
+      - --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:29092
+      - --advertise-kafka-addr internal://redpanda-1:9092,external://localhost:29092
+      - --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:28082
+      - --advertise-pandaproxy-addr internal://redpanda-1:8082,external://localhost:28082
+      - --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:28081
+      - --rpc-addr redpanda-1:33145
+      - --advertise-rpc-addr redpanda-1:33145
+      - --mode dev-container
+      - --smp 1
+      - --default-log-level=info
+      - --seeds redpanda-0:33145
+    image: docker.redpanda.com/redpandadata/${REDPANDA_DOCKER_REPO:-redpanda}:${REDPANDA_VERSION:-latest}
+    container_name: redpanda-1
+    environment:
+      RP_BOOTSTRAP_USER: "superuser:secretpassword"
+    volumes:
+      - redpanda-1:/var/lib/redpanda/data
+      - ./bootstrap.yml:/etc/redpanda/.bootstrap.yaml
+    networks:
+      - redpanda_network
+    ports:
+      - 28081:28081
+      - 28082:28082
+      - 29092:29092
+      - 29644:9644
+    depends_on:
+      - redpanda-0
+      - minio
+  redpanda-2:
+    command:
+      - redpanda
+      - start
+      - --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:39092
+      - --advertise-kafka-addr internal://redpanda-2:9092,external://localhost:39092
+      - --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:38082
+      - --advertise-pandaproxy-addr internal://redpanda-2:8082,external://localhost:38082
+      - --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:38081
+      - --rpc-addr redpanda-2:33145
+      - --advertise-rpc-addr redpanda-2:33145
+      - --mode dev-container
+      - --smp 1
+      - --default-log-level=info
+      - --seeds redpanda-0:33145
+    image: docker.redpanda.com/redpandadata/${REDPANDA_DOCKER_REPO:-redpanda}:${REDPANDA_VERSION:-latest}
+    container_name: redpanda-2
+    environment:
+      RP_BOOTSTRAP_USER: "superuser:secretpassword"
+    volumes:
+      - redpanda-2:/var/lib/redpanda/data
+      - ./bootstrap.yml:/etc/redpanda/.bootstrap.yaml
+    networks:
+      - redpanda_network
+    ports:
+      - 38081:38081
+      - 38082:38082
+      - 39092:39092
+      - 39644:9644
+    depends_on:
+      - redpanda-0
+      - minio
+  ####################
+  # Redpanda Console #
+  ####################
+  console:
+    container_name: redpanda-console
+    image: docker.redpanda.com/redpandadata/${CONSOLE_DOCKER_REPO:-console}:${REDPANDA_CONSOLE_VERSION:-latest}
+    networks:
+      - redpanda_network
+    entrypoint: /bin/sh
+    command: -c 'echo "$$CONSOLE_CONFIG_FILE" > /tmp/config.yml && /app/console'
+    volumes:
+      - ./config:/tmp/config/
+    environment:
+      CONFIG_FILEPATH: ${CONFIG_FILEPATH:-/tmp/config.yml}
+      CONSOLE_CONFIG_FILE: |
+        # Configure a connection to the Redpanda cluster
+        # See https://docs.redpanda.com/current/console/config/connect-to-redpanda/
+        kafka:
+          brokers: ["redpanda-0:9092"]
+          sasl:
+            enabled: true
+            impersonateUser: true
+        schemaRegistry:
+          enabled: true
+          urls: ["http://redpanda-0:8081","http://redpanda-1:8081","http://redpanda-2:8081"]
+          authentication:
+            impersonateUser: true
+        redpanda:
+          adminApi:
+            enabled: true
+            urls: ["http://redpanda-0:9644","http://redpanda-1:9644","http://redpanda-2:9644"]
+            authentication:
+              basic:
+                username: superuser
+                password: secretpassword
+              impersonateUser: false
+        console:
+          # Configures Redpanda Console to fetch topic documentation from GitHub and display it in the UI.
+          # See https://docs.redpanda.com/current/console/config/topic-documentation/
+          topicDocumentation:
+            enabled: true
+            git:
+              enabled: true
+              repository:
+                url: https://github.com/redpanda-data/docs
+                branch: main
+                baseDirectory: tests/docker-compose
+        authentication:
+          jwtSigningKey: vazxnT+ZHtxKslK6QlDGovcYnSjTk/lKMmZ+mHrBVE+YdVDkLgSuP6AszAKe9Gvq
+          basic:
+            enabled: true
+        authorization:
+          roleBindings:
+          - roleName: admin
+            users:
+              - loginType: basic
+                name: superuser
+    ports:
+      - 8080:8080
+    depends_on:
+      redpanda-0:
+        condition: service_healthy
+  ####################
+  # Redpanda Connect #
+  ####################
+  connect:
+    container_name: redpanda-connect
+    image: docker.redpanda.com/redpandadata/connect
+    networks:
+      - redpanda_network
+    entrypoint: /bin/sh
+    depends_on:
+      redpanda-0:
+        condition: service_healthy
+    command: -c 'echo "$$CONNECT_CFG_FILE" > /tmp/connect.yml; /redpanda-connect -c /tmp/connect.yml'
+    environment:
+      # This Redpanda Connect configuration creates fake data,
+      # processes it, and writes the output to a set of topics.
+      #
+      # Input:
+      #  - Uses Redpanda Connect's generate input to generate fake data.
+      # See https://docs.redpanda.com/redpanda-connect/components/inputs/generate/
+      # Pipeline:
+      #  - Bloblang mapping to batch each input and map 1 message to 'logins'
+      #    topic, and a random number (1-3) of messages to 'transaction' topic
+      #  - Unarchive processor to parse the JSON array and extract each
+      #    element into its own message.
+      # See https://docs.redpanda.com/redpanda-connect/guides/bloblang/about/
+      # Output:
+      #  - kafka_franz output to write the messages to the Redpanda brokers.
+      # See https://docs.redpanda.com/redpanda-connect/components/outputs/kafka_franz/
+      CONNECT_CFG_FILE: |
+        input:
+          generate:
+            interval: 1s
+            mapping: |
+              let first_name = fake("first_name")
+              let last_name  = fake("last_name")
+
+              root.user_id    = counter()
+              root.name       = $$first_name + " " + $$last_name
+              root.email      = ($$first_name.slice(0,1) + $$last_name + "@" + fake("domain_name")).lowercase()
+              root.ip         = fake("ipv4")
+              root.login_time = now()
+        pipeline:
+          processors:
+            - mapping: |
+                root = range(0, random_int(min:2, max:4)).map_each(cust -> this)
+            - unarchive:
+                format: "json_array"
+            - mapping: |
+                if batch_index() == 0 {
+                  meta topic = "logins"
+                  root = this
+                } else {
+                  meta topic       = "transactions"
+                  root.user_id     = this.user_id
+                  root.email       = this.email
+                  root.index       = batch_index() - 1
+                  root.product_url = fake("url")
+                  root.price       = fake("amount_with_currency")
+                  root.timestamp   = now()
+                }
+        output:
+          kafka_franz:
+            seed_brokers: [ "redpanda-0:9092" ]
+            topic: $${! metadata("topic") }
+            sasl:
+              - mechanism: SCRAM-SHA-256
+                password: secretpassword
+                username: superuser
+  ####################
+  # rpk container to create the edu-filtered-domains topic #
+  # See https://docs.redpanda.com/current/reference/rpk/rpk-topic/rpk-topic-create/
+  ####################
+  createtopic:
+    command:
+      - topic
+      - create
+      - edu-filtered-domains
+      - -X user=superuser
+      - -X pass=secretpassword
+      - -X brokers=redpanda-0:9092
+      - --topic-config
+      - redpanda.iceberg.mode=key_value
+    image: docker.redpanda.com/redpandadata/${REDPANDA_DOCKER_REPO:-redpanda}:${REDPANDA_VERSION:-latest}
+    networks:
+      - redpanda_network
+    depends_on:
+      redpanda-0:
+        condition: service_healthy
+  ####################
+  # rpk container to register the schema #
+  # See https://docs.redpanda.com/current/manage/schema-reg/schema-reg-api/
+  ####################
+  registerschema:
+    command:
+      - registry
+      - schema
+      - create
+      - transactions
+      - --schema
+      - /etc/redpanda/transactions-schema.json
+      - -X user=superuser
+      - -X pass=secretpassword
+      - -X registry.hosts=redpanda-0:8081
+    image: docker.redpanda.com/redpandadata/${REDPANDA_DOCKER_REPO:-redpanda}:${REDPANDA_VERSION:-latest}
+    # Mount the local directory that contains your schema to the container.
+    volumes:
+      - ./transactions-schema.json:/etc/redpanda/transactions-schema.json
+    networks:
+      - redpanda_network
+    depends_on:
+      redpanda-0:
+        condition: service_healthy
+  ####################
+  # rpk container to deploy a consumer group #
+  # See https://docs.redpanda.com/current/reference/rpk/rpk-topic/rpk-topic-consume/
+  ####################
+  consumergroup:
+    command:
+      - topic
+      - consume
+      - transactions
+      - --group
+      - transactions-consumer
+      - -X user=superuser
+      - -X pass=secretpassword
+      - -X brokers=redpanda-0:9092
+    image: docker.redpanda.com/redpandadata/${REDPANDA_DOCKER_REPO:-redpanda}:${REDPANDA_VERSION:-latest}
+    networks:
+      - redpanda_network
+    depends_on:
+      - deploytransform
+  ####################
+  # rpk container to deploy the pre-built data transform #
+  # See https://docs.redpanda.com/current/develop/data-transforms/deploy/
+  ####################
+  deploytransform:
+    command:
+      - transform
+      - deploy
+      - --file=/etc/redpanda/regex.wasm
+      - --name=regex
+      - --input-topic=logins
+      - --output-topic=edu-filtered-domains
+      - --var=PATTERN="[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.edu"
+      - --var=MATCH_VALUE=true
+      - -X user=superuser
+      - -X pass=secretpassword
+      - -X admin.hosts=redpanda-0:9644
+    image: docker.redpanda.com/redpandadata/${REDPANDA_DOCKER_REPO:-redpanda}:${REDPANDA_VERSION:-latest}
+    volumes:
+      - ./transform/regex.wasm:/etc/redpanda/regex.wasm
+    networks:
+      - redpanda_network
+    depends_on:
+      redpanda-0:
+        condition: service_healthy
+      createtopic:
+        condition: service_started
+  ####################
+  # MinIO for Tiered Storage #
+  # See https://min.io/
+  #
+  # NOTE: MinIO is included in this quickstart for development and evaluation purposes only.
+  # It is not supported for production deployments of Redpanda.
+  #
+  # For production environments, use one of the supported object storage providers:
+  # https://docs.redpanda.com/current/deploy/deployment-option/self-hosted/manual/production/requirements/#object-storage-providers-for-tiered-storage
+  ####################
+  minio:
+    container_name: minio
+    image: minio/minio
+    command: server --console-address ":9001" /data
+    ports:
+      - 9000:9000
+      - 9001:9001
+    environment:
+      MINIO_ROOT_USER: minio
+      MINIO_ROOT_PASSWORD: redpandaTieredStorage7
+      MINIO_SERVER_URL: "http://minio:9000"
+      MINIO_REGION_NAME: local
+      MINIO_DOMAIN: minio
+    volumes:
+      - minio:/data
+    networks:
+      redpanda_network:
+        aliases:
+          - redpanda.minio
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/ready"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+  mc:
+    depends_on:
+      minio:
+        condition: service_healthy
+    image: minio/mc
+    container_name: mc
+    networks:
+      - redpanda_network
+    environment:
+      - AWS_ACCESS_KEY_ID=minio
+      - AWS_SECRET_ACCESS_KEY=redpandaTieredStorage7
+      - AWS_REGION=local
+    entrypoint: >
+      /bin/sh -c "
+      until (/usr/bin/mc config host add minio http://minio:9000 minio redpandaTieredStorage7) do echo '...waiting...' && sleep 1; done;
+      /usr/bin/mc mb minio/redpanda;
+      /usr/bin/mc policy set public minio/redpanda;
+      tail -f /dev/null
+      "
+  catalog:
+    image: tabulario/iceberg-rest
+    container_name: iceberg-rest
+    networks:
+      - redpanda_network
+    depends_on:
+      - minio
+    ports:
+      - 8181:8181
+    environment:
+      - AWS_ACCESS_KEY_ID=minio
+      - AWS_SECRET_ACCESS_KEY=redpandaTieredStorage7
+      - AWS_REGION=local
+      - CATALOG_WAREHOUSE=s3://redpanda/
+      - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO
+      - CATALOG_S3_ENDPOINT=http://minio:9000

package/docker-compose/generate-profiles.yaml

@@ -0,0 +1,77 @@
+input:
+  # Use the 'generate' input
+  # https://docs.redpanda.com/redpanda-connect/components/inputs/generate/
+  generate:
+    # The interval at which new records are generated.
+    interval: 1s
+    # The mapping section defines how each generated record is structured.
+    # The language used here is called Bloblang.
+    # https://docs.redpanda.com/redpanda-connect/guides/bloblang/about/
+    mapping: |
+      # Generate a fake first name using the 'first_name' faker function.
+      let first_name = fake("first_name")
+
+      # Generate a fake last name using the 'last_name' faker function.
+      let last_name  = fake("last_name")
+
+      # Define possible subscription levels for users.
+      let subscription_levels = ["Free", "Basic", "Premium"]
+
+      # Define possible notification channels for user preferences.
+      let notifications = ["email", "sms", "push" ]
+
+      # Define supported languages for user preferences.
+      let languages = ["en", "es", "fr", "de", "zh", "jp"]
+
+      # Assign a unique user ID using a UUID digit generator.
+      root.user_id = fake("uuid_digit")
+
+      # Assign the generated first name to the 'first_name' field.
+      root.first_name = $first_name
+
+      # Assign the generated last name to the 'last_name' field.
+      root.last_name = $last_name
+
+      # Construct the user's email by combining the first initial, last name, and a fake domain name.
+      # The email is converted to lowercase for consistency.
+      root.email = ($first_name.slice(0,1) + $last_name + "@" + fake("domain_name")).lowercase()
+
+      # Assign a fake registration date using the 'date' faker function.
+      root.registration_date = fake("date")
+
+      # Assign the current timestamp as the last login time.
+      root.last_login = now()
+
+      # Randomly assign a subscription level by selecting an index from the 'subscription_levels' array.
+      root.subscription_level = $subscription_levels.index(random_int(min: 0, max: 2))
+
+      # Randomly assign a language preference by selecting an index from the 'languages' array.
+      root.preferences.language = $languages.index(random_int(min: 0, max: 5))
+
+      # Randomly assign a notification preference by selecting an index from the 'notifications' array.
+      root.preferences.notifications = $notifications.index(random_int(min: 0, max: 2))
+pipeline:
+  processors:
+    - mapping: |
+        # Set the target topic for the generated records to 'profiles'.
+        meta topic = "profiles"
+
+        # Assign the entire record (root) to be sent to the specified topic.
+        root = this
+output:
+  # Use the 'kafka_franz' output to send the result back to Redpanda
+  # https://docs.redpanda.com/redpanda-connect/components/outputs/kafka_franz/
+  kafka_franz:
+    # Define the list of seed brokers for the Kafka cluster.
+    seed_brokers: [ "localhost:19092", "localhost:29092", "localhost:39092"]
+    # Dynamically assign the topic based on the metadata specified in the processors.
+    # In this case, it resolves to the 'profiles' topic.
+    topic: ${! metadata("topic") }
+    # Configure SASL authentication to securely connect to the Kafka brokers.
+    sasl:
+      - # Specify the SASL mechanism to use for authentication.
+        mechanism: SCRAM-SHA-256
+        # The password for the SASL authentication.
+        password: secretpassword
+        # The username for the SASL authentication.
+        username: superuser

package/docker-compose/rpk-profile.yaml

@@ -0,0 +1,24 @@
+# This file configures `rpk` to connect to a remote Redpanda cluster running in the same local network as `rpk`.
+
+# Configuration for connecting to the Kafka API of the Redpanda cluster.
+kafka_api:
+  # SASL (Simple Authentication and Security Layer) settings for authentication.
+  sasl:
+    user: superuser # The username used for authentication
+    password: secretpassword # The password associated with the username
+    mechanism: scram-sha-256 # Authentication mechanism; SCRAM-SHA-256 provides secure password-based authentication
+  # List of Kafka brokers in the Redpanda cluster.
+  # These brokers ensure high availability and fault tolerance for Kafka-based communication.
+  brokers:
+    - 127.0.0.1:19092 # Broker 1: Accessible on localhost, port 19092
+    - 127.0.0.1:29092 # Broker 2: Accessible on localhost, port 29092
+    - 127.0.0.1:39092 # Broker 3: Accessible on localhost, port 39092
+
+# Configuration for connecting to the Redpanda Admin API.
+# The Admin API allows you to perform administrative tasks such as managing configurations, monitoring, and scaling.
+admin_api:
+   # List of Admin API endpoints for managing the cluster.
+  addresses:
+    - 127.0.0.1:19644 # Admin API for Broker 1: Accessible on localhost, port 19644
+    - 127.0.0.1:29644 # Admin API for Broker 2: Accessible on localhost, port 29644
+    - 127.0.0.1:39644 # Admin API for Broker 3: Accessible on localhost, port 39644

package/docker-compose/transactions-schema.json

@@ -0,0 +1,37 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "Transactions",
+  "type": "object",
+  "properties": {
+    "email": {
+      "type": "string",
+      "format": "email",
+      "description": "The email address of the user involved in the transaction."
+    },
+    "index": {
+      "type": "integer",
+      "description": "A numeric index associated with the transaction."
+    },
+    "price": {
+      "type": "string",
+      "pattern": "^[A-Z]{3} \\d+(?:\\.\\d{2})?$",
+      "description": "A string representing the price of the product, including a currency code (ISO 4217) and an amount with two decimal places by default."
+    },
+    "product_url": {
+      "type": "string",
+      "format": "uri",
+      "description": "A URL that points to the product involved in the transaction."
+    },
+    "timestamp": {
+      "type": "string",
+      "format": "date-time",
+      "description": "The timestamp of when the transaction occurred, formatted in ISO 8601."
+    },
+    "user_id": {
+      "type": "integer",
+      "description": "A numeric identifier for the user."
+    }
+  },
+  "required": ["email", "index", "price", "product_url", "timestamp", "user_id"],
+  "additionalProperties": false
+}