karafka-rdkafka 0.20.0.rc5-arm64-darwin → 0.21.0-arm64-darwin

data/docker-compose-ssl.yml

@@ -0,0 +1,35 @@
+services:
+  kafka:
+    container_name: kafka
+    image: confluentinc/cp-kafka:8.0.0
+    ports:
+      - 9092:9092 # Support PLAINTEXT so we can run one docker setup for SSL and PLAINTEXT
+      - 9093:9093
+    volumes:
+      - ./ssl:/etc/kafka/secrets
+    environment:
+      CLUSTER_ID: kafka-docker-cluster-1
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_PROCESS_ROLES: broker,controller
+      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
+      KAFKA_LISTENERS: PLAINTEXT://:9092,SSL://:9093,CONTROLLER://:9094
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,SSL:SSL
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://127.0.0.1:9092,SSL://127.0.0.1:9093
+      KAFKA_BROKER_ID: 1
+      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@127.0.0.1:9094
+      ALLOW_PLAINTEXT_LISTENER: 'yes'
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
+      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+      KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
+      KAFKA_AUTHORIZER_CLASS_NAME: org.apache.kafka.metadata.authorizer.StandardAuthorizer
+
+      # SSL Configuration
+      KAFKA_SSL_KEYSTORE_FILENAME: kafka.server.keystore.jks
+      KAFKA_SSL_KEYSTORE_CREDENTIALS: kafka_keystore_creds
+      KAFKA_SSL_KEY_CREDENTIALS: kafka_ssl_key_creds
+      KAFKA_SSL_TRUSTSTORE_FILENAME: kafka.server.truststore.jks
+      KAFKA_SSL_TRUSTSTORE_CREDENTIALS: kafka_truststore_creds
+      KAFKA_SSL_CLIENT_AUTH: none
+      KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: ""

data/ext/build_common.sh

@@ -19,7 +19,7 @@ readonly CYRUS_SASL_VERSION="2.1.28"
 readonly ZLIB_VERSION="1.3.1"
 readonly ZSTD_VERSION="1.5.7"
 readonly KRB5_VERSION="1.21.3"
-readonly LIBRDKAFKA_VERSION="2.8.0"
+readonly LIBRDKAFKA_VERSION="2.11.0"
 
 # SHA256 checksums for supply chain security
 # Update these when upgrading versions
@@ -29,7 +29,7 @@ declare -A CHECKSUMS=(
     ["zlib-1.3.1.tar.gz"]="9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23"
     ["zstd-${ZSTD_VERSION}.tar.gz"]="eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3"
     ["krb5-${KRB5_VERSION}.tar.gz"]="b7a4cd5ead67fb08b980b21abd150ff7217e85ea320c9ed0c6dadd304840ad35"
-    ["librdkafka-${LIBRDKAFKA_VERSION}.tar.gz"]="5bd1c46f63265f31c6bfcedcde78703f77d28238eadf23821c2b43fc30be3e25"
+    ["librdkafka-${LIBRDKAFKA_VERSION}.tar.gz"]="592a823dc7c09ad4ded1bc8f700da6d4e0c88ffaf267815c6f25e7450b9395ca"
 )
 
 # Colors for output
@@ -94,12 +94,25 @@ secure_download() {
     local url="$1"
     local filename="$2"
 
+    # Check if file already exists in current directory (may have been already downloaded)
     if [ -f "$filename" ]; then
         log "File $filename already exists, verifying checksum..."
         verify_checksum "$filename"
         return 0
     fi
 
+    # Check dist directory relative to script location
+    local script_dir
+    script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+    local dist_file="$script_dir/../dist/$filename"
+
+    if [ -f "$dist_file" ]; then
+        log "Using distributed $filename from dist/"
+        cp "$dist_file" "$filename"
+        verify_checksum "$filename"
+        return 0
+    fi
+
     log "Downloading $filename from $url..."
 
     # Use platform-appropriate download command
@@ -344,7 +357,9 @@ get_zstd_url() {
 }
 
 get_krb5_url() {
-    echo "https://kerberos.org/dist/krb5/${KRB5_VERSION%.*}/krb5-${KRB5_VERSION}.tar.gz"
+    # Using MIT mirror since kerberos.org is down
+    # echo "https://kerberos.org/dist/krb5/${KRB5_VERSION%.*}/krb5-${KRB5_VERSION}.tar.gz"
+    echo "https://web.mit.edu/kerberos/dist/krb5/${KRB5_VERSION%.*}/krb5-${KRB5_VERSION}.tar.gz"
 }
 
 # Export functions and variables that scripts will need

data/ext/build_linux_aarch64_gnu.sh

@@ -0,0 +1,326 @@
+#!/usr/bin/env bash
+#
+# Build self-contained librdkafka.so for Linux aarch64 GNU with checksum verification
+# Usage: ./build_linux_aarch64_gnu.sh
+#
+# Expected directory structure:
+#   ext/build_linux_aarch64_gnu.sh            (this script)
+#   ext/build_common.sh                       (shared functions)
+#   dist/librdkafka-*.tar.gz                  (librdkafka source tarball)
+#   dist/patches/*.patch                      (optional Ruby-specific patches)
+#
+set -euo pipefail
+
+# Source common functions and constants
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$SCRIPT_DIR/build_common.sh"
+
+# Platform-specific paths
+DIST_DIR="$SCRIPT_DIR/../dist"
+PATCHES_DIR="$DIST_DIR/patches"
+BUILD_DIR="$(pwd)/build-tmp"
+DEPS_PREFIX="/tmp"
+
+# Check common dependencies
+check_common_dependencies
+
+# Linux-specific dependency check
+log "Checking Linux aarch64 GNU-specific build dependencies..."
+command -v gcc &> /dev/null || error "gcc not found. Install with: apt-get install build-essential"
+
+# Verify we're on ARM64 or cross-compiling
+ARCH=$(uname -m)
+if [[ "$ARCH" != "aarch64" && "$ARCH" != "arm64" ]]; then
+    warn "Not running on aarch64 ($ARCH detected). Ensure you have aarch64 cross-compilation tools if needed."
+fi
+
+# Auto-detect librdkafka tarball
+log "Looking for librdkafka tarball in $DIST_DIR..."
+LIBRDKAFKA_TARBALL=$(find_librdkafka_tarball "$DIST_DIR")
+log "Found librdkafka tarball: $LIBRDKAFKA_TARBALL"
+
+# Verify librdkafka tarball checksum if available
+verify_librdkafka_checksum "$LIBRDKAFKA_TARBALL"
+
+# Find patches
+PATCHES_FOUND=()
+find_patches "$PATCHES_DIR" PATCHES_FOUND
+
+security_log "Starting secure build with checksum verification enabled"
+log "Building self-contained librdkafka.so for Linux aarch64 GNU"
+log "Dependencies to build:"
+log "  - OpenSSL: $OPENSSL_VERSION"
+log "  - Cyrus SASL: $CYRUS_SASL_VERSION"
+log "  - MIT Kerberos: $KRB5_VERSION"
+log "  - zlib: $ZLIB_VERSION"
+log "  - ZStd: $ZSTD_VERSION"
+log "librdkafka source: $LIBRDKAFKA_TARBALL"
+log "Build directory: $BUILD_DIR"
+
+# Create build directory
+mkdir -p "$BUILD_DIR"
+cd "$BUILD_DIR"
+
+# Build OpenSSL
+log "Building OpenSSL $OPENSSL_VERSION for ARM64..."
+OPENSSL_PREFIX="$DEPS_PREFIX/static-openssl-$OPENSSL_VERSION-arm64"
+OPENSSL_TARBALL="openssl-$OPENSSL_VERSION.tar.gz"
+OPENSSL_DIR="openssl-$OPENSSL_VERSION"
+
+secure_download "$(get_openssl_url)" "$OPENSSL_TARBALL"
+extract_if_needed "$OPENSSL_TARBALL" "$OPENSSL_DIR"
+cd "$OPENSSL_DIR"
+
+# Check if OpenSSL lib directory exists (lib or lib64)
+if [ ! -f "$OPENSSL_PREFIX/lib/libssl.a" ] && [ ! -f "$OPENSSL_PREFIX/lib64/libssl.a" ]; then
+    log "Configuring and building OpenSSL for ARM64..."
+    export CFLAGS="-fPIC"
+    # Use linux-aarch64 target for ARM64
+    ./Configure linux-aarch64 \
+        no-shared \
+        no-dso \
+        --prefix="$OPENSSL_PREFIX"
+    make clean || true
+    make -j$(get_cpu_count)
+    make install
+    unset CFLAGS
+    log "OpenSSL built successfully"
+else
+    log "OpenSSL already built, skipping..."
+fi
+
+# Determine OpenSSL lib directory
+if [ -f "$OPENSSL_PREFIX/lib64/libssl.a" ]; then
+    OPENSSL_LIB_DIR="$OPENSSL_PREFIX/lib64"
+else
+    OPENSSL_LIB_DIR="$OPENSSL_PREFIX/lib"
+fi
+log "OpenSSL libraries in: $OPENSSL_LIB_DIR"
+
+cd "$BUILD_DIR"
+
+# Build MIT Kerberos (krb5)
+log "Building MIT Kerberos $KRB5_VERSION for ARM64..."
+KRB5_PREFIX="$DEPS_PREFIX/static-krb5-$KRB5_VERSION-arm64"
+KRB5_TARBALL="krb5-$KRB5_VERSION.tar.gz"
+KRB5_DIR="krb5-$KRB5_VERSION"
+
+secure_download "$(get_krb5_url)" "$KRB5_TARBALL"
+extract_if_needed "$KRB5_TARBALL" "$KRB5_DIR"
+cd "$KRB5_DIR/src"
+
+if [ ! -f "$KRB5_PREFIX/lib/libgssapi_krb5.a" ]; then
+    log "Configuring and building MIT Kerberos for ARM64..."
+    make clean 2>/dev/null || true
+    ./configure --disable-shared --enable-static --prefix="$KRB5_PREFIX" \
+        --without-ldap --without-tcl --without-keyutils \
+        --disable-rpath --without-system-verto \
+        CFLAGS="-fPIC" CXXFLAGS="-fPIC"
+
+    # Build everything except the problematic kadmin tools
+    log "Building Kerberos (will ignore kadmin build failures)..."
+    make -j$(get_cpu_count) || {
+        log "Full build failed (expected due to kadmin), continuing with libraries..."
+        # The libraries should be built even if kadmin fails
+        true
+    }
+
+    # Install what was successfully built
+    make install || {
+        log "Full install failed, installing individual components..."
+        # Try to install the core libraries manually
+        make install-mkdirs 2>/dev/null || true
+        make -C util install 2>/dev/null || true
+        make -C lib install 2>/dev/null || true
+        make -C plugins/kdb/db2 install 2>/dev/null || true
+    }
+
+    # Verify we got the essential libraries
+    if [ ! -f "$KRB5_PREFIX/lib/libgssapi_krb5.a" ]; then
+        error "Failed to build essential Kerberos libraries"
+    fi
+
+    log "MIT Kerberos libraries built successfully"
+else
+    log "MIT Kerberos already built, skipping..."
+fi
+
+cd "$BUILD_DIR"
+
+# Build SASL
+log "Building Cyrus SASL $CYRUS_SASL_VERSION for ARM64..."
+SASL_PREFIX="$DEPS_PREFIX/static-sasl-$CYRUS_SASL_VERSION-arm64"
+SASL_TARBALL="cyrus-sasl-$CYRUS_SASL_VERSION.tar.gz"
+SASL_DIR="cyrus-sasl-$CYRUS_SASL_VERSION"
+
+secure_download "$(get_sasl_url)" "$SASL_TARBALL"
+extract_if_needed "$SASL_TARBALL" "$SASL_DIR"
+cd "$SASL_DIR"
+
+if [ ! -f "$SASL_PREFIX/lib/libsasl2.a" ]; then
+    log "Configuring and building SASL for ARM64..."
+    make clean 2>/dev/null || true
+    ./configure --disable-shared --enable-static --prefix="$SASL_PREFIX" \
+        --without-dblib --disable-gdbm \
+        --enable-gssapi="$KRB5_PREFIX" \
+        CFLAGS="-fPIC" CXXFLAGS="-fPIC" \
+        CPPFLAGS="-I$KRB5_PREFIX/include" \
+        LDFLAGS="-L$KRB5_PREFIX/lib"
+    make -j$(get_cpu_count)
+    make install
+    log "SASL built successfully"
+else
+    log "SASL already built, skipping..."
+fi
+
+cd "$BUILD_DIR"
+
+# Build zlib
+log "Building zlib $ZLIB_VERSION for ARM64..."
+ZLIB_PREFIX="$DEPS_PREFIX/static-zlib-$ZLIB_VERSION-arm64"
+ZLIB_TARBALL="zlib-$ZLIB_VERSION.tar.gz"
+ZLIB_DIR="zlib-$ZLIB_VERSION"
+
+secure_download "$(get_zlib_url)" "$ZLIB_TARBALL"
+extract_if_needed "$ZLIB_TARBALL" "$ZLIB_DIR"
+cd "$ZLIB_DIR"
+
+if [ ! -f "$ZLIB_PREFIX/lib/libz.a" ]; then
+    log "Configuring and building zlib for ARM64..."
+    make clean 2>/dev/null || true
+    export CFLAGS="-fPIC"
+    ./configure --prefix="$ZLIB_PREFIX" --static
+    make -j$(get_cpu_count)
+    make install
+    unset CFLAGS
+    log "zlib built successfully"
+else
+    log "zlib already built, skipping..."
+fi
+
+cd "$BUILD_DIR"
+
+# Build ZStd
+log "Building ZStd $ZSTD_VERSION for ARM64..."
+ZSTD_PREFIX="$DEPS_PREFIX/static-zstd-$ZSTD_VERSION-arm64"
+ZSTD_TARBALL="zstd-$ZSTD_VERSION.tar.gz"
+ZSTD_DIR="zstd-$ZSTD_VERSION"
+
+secure_download "$(get_zstd_url)" "$ZSTD_TARBALL"
+extract_if_needed "$ZSTD_TARBALL" "$ZSTD_DIR"
+cd "$ZSTD_DIR"
+
+if [ ! -f "$ZSTD_PREFIX/lib/libzstd.a" ]; then
+    log "Building ZStd for ARM64..."
+    make clean 2>/dev/null || true
+    make lib-mt CFLAGS="-fPIC" PREFIX="$ZSTD_PREFIX" -j$(get_cpu_count)
+    # Use standard install target - install-pc may not exist in all versions
+    make install PREFIX="$ZSTD_PREFIX"
+    log "ZStd built successfully"
+else
+    log "ZStd already built, skipping..."
+fi
+
+cd "$BUILD_DIR"
+
+# Extract and patch librdkafka
+log "Extracting librdkafka..."
+tar xzf "$LIBRDKAFKA_TARBALL"
+cd "librdkafka-$LIBRDKAFKA_VERSION"
+
+# Fix permissions and apply patches
+fix_configure_permissions
+apply_patches PATCHES_FOUND
+
+# Configure librdkafka
+log "Configuring librdkafka for ARM64..."
+
+if [ -f configure ]; then
+    log "Using standard configure (autotools)"
+    # Export environment variables for configure to pick up
+    export CPPFLAGS="-I$KRB5_PREFIX/include"
+    export LDFLAGS="-L$KRB5_PREFIX/lib"
+
+    ./configure --enable-static --disable-shared --disable-curl \
+        --enable-gssapi
+
+    # Clean up environment variables
+    unset CPPFLAGS LDFLAGS
+else
+    error "No configure script found (checked: configure.self, configure)"
+fi
+
+# Build librdkafka
+log "Compiling librdkafka for ARM64..."
+make clean || true
+make -j$(get_cpu_count)
+
+# Verify librdkafka.a exists
+if [ ! -f src/librdkafka.a ]; then
+    error "librdkafka.a not found after build"
+fi
+
+log "librdkafka.a built successfully"
+
+# Create self-contained shared library
+log "Creating self-contained librdkafka.so for ARM64..."
+
+# Write the export map
+cat > export.map <<'EOF'
+{
+  global:
+    rd_kafka_*;
+  local:
+    *;
+};
+EOF
+
+# Link everything statically, expose only rd_kafka_* symbols
+aarch64-linux-gnu-gcc -shared -fPIC \
+    -Wl,--version-script=export.map \
+    -Wl,--whole-archive src/librdkafka.a -Wl,--no-whole-archive \
+    -o librdkafka.so \
+    "$SASL_PREFIX/lib/libsasl2.a" \
+    "$KRB5_PREFIX/lib/libgssapi_krb5.a" \
+    "$KRB5_PREFIX/lib/libkrb5.a" \
+    "$KRB5_PREFIX/lib/libk5crypto.a" \
+    "$KRB5_PREFIX/lib/libcom_err.a" \
+    "$KRB5_PREFIX/lib/libkrb5support.a" \
+    "$OPENSSL_LIB_DIR/libssl.a" \
+    "$OPENSSL_LIB_DIR/libcrypto.a" \
+    "$ZLIB_PREFIX/lib/libz.a" \
+    "$ZSTD_PREFIX/lib/libzstd.a" \
+    -lpthread -lm -ldl -lresolv
+
+if [ ! -f librdkafka.so ]; then
+    error "Failed to create librdkafka.so"
+fi
+
+log "librdkafka.so created successfully"
+
+# Verify the build
+log "Verifying build..."
+file librdkafka.so
+
+log "Checking dependencies with ldd:"
+ldd librdkafka.so
+
+log "Checking for external dependencies (should only show system libraries):"
+EXTERNAL_DEPS=$(nm -D librdkafka.so | grep " U " | grep -v "@GLIBC" || true)
+if [ -n "$EXTERNAL_DEPS" ]; then
+    error "Found external dependencies - library is not self-contained: $EXTERNAL_DEPS"
+else
+    log "✅ No external dependencies found - library is self-contained!"
+fi
+
+# Copy to output directory
+OUTPUT_DIR="$SCRIPT_DIR"
+cp librdkafka.so "$OUTPUT_DIR/"
+log "librdkafka.so copied to: $OUTPUT_DIR/librdkafka.so"
+
+# Print summaries
+print_security_summary
+print_build_summary "Linux" "aarch64 GNU" "$OUTPUT_DIR" "librdkafka.so"
+
+# Cleanup
+cleanup_build_dir "$BUILD_DIR"

data/ext/build_linux_x86_64_gnu.sh

@@ -258,7 +258,18 @@ log "librdkafka.a built successfully"
 # Create self-contained shared library
 log "Creating self-contained librdkafka.so..."
 
-gcc -shared -fPIC -Wl,--whole-archive src/librdkafka.a -Wl,--no-whole-archive \
+echo '
+{
+  global:
+    rd_kafka_*;
+  local:
+    *;
+};
+' > export.map
+
+gcc -shared -fPIC \
+    -Wl,--version-script=export.map \
+    -Wl,--whole-archive src/librdkafka.a -Wl,--no-whole-archive \
     -o librdkafka.so \
     "$SASL_PREFIX/lib/libsasl2.a" \
     "$KRB5_PREFIX/lib/libgssapi_krb5.a" \
@@ -285,12 +296,12 @@ file librdkafka.so
 log "Checking dependencies with ldd:"
 ldd librdkafka.so
 
-log "Checking for external dependencies (should only show system libraries):"
-EXTERNAL_DEPS=$(nm -D librdkafka.so | grep " U " | grep -v "@GLIBC" || true)
-if [ -n "$EXTERNAL_DEPS" ]; then
-    error "Found external dependencies - library is not self-contained: $EXTERNAL_DEPS"
+log "Checking for non-system library dependencies:"
+NON_SYSTEM_DEPS=$(ldd librdkafka.so | grep -v -E "(linux-vdso|ld-linux|libc\.so|libpthread\.so|libm\.so|libdl\.so)" || true)
+if [ -n "$NON_SYSTEM_DEPS" ]; then
+    error "Found non-system dependencies: $NON_SYSTEM_DEPS"
 else
-    log "✅ No external dependencies found - library is self-contained!"
+    log "✅ Only system dependencies found - library is portable!"
 fi
 
 # Copy to output directory

data/ext/build_linux_x86_64_musl.sh

@@ -625,10 +625,20 @@ do
     fi
 done
 
+echo '
+{
+  global:
+    rd_kafka_*;
+  local:
+    *;
+};
+' > export.map
+
 gcc -shared -fPIC \
-    -Wl,--whole-archive src/librdkafka.a -Wl,--no-whole-archive \
-    -o librdkafka.so \
-    -Wl,-Bstatic \
+  -Wl,--version-script=export.map \
+  -Wl,--whole-archive src/librdkafka.a -Wl,--no-whole-archive \
+  -o librdkafka.so \
+  -Wl,-Bstatic \
     "$SASL_PREFIX/lib/libsasl2.a" \
     "$KRB5_PREFIX/lib/libgssapi_krb5.a" \
     "$KRB5_PREFIX/lib/libkrb5.a" \
@@ -639,11 +649,11 @@ gcc -shared -fPIC \
     "$OPENSSL_LIB_DIR/libcrypto.a" \
     "$ZLIB_PREFIX/lib/libz.a" \
     "$ZSTD_PREFIX/lib/libzstd.a" \
-    -Wl,-Bdynamic \
-    -lpthread -lm -ldl -lc \
-    -static-libgcc \
-    -Wl,--as-needed \
-    -Wl,--no-undefined
+  -Wl,-Bdynamic \
+  -lpthread -lm -ldl -lc \
+  -static-libgcc \
+  -Wl,--as-needed \
+  -Wl,--no-undefined
 
 if [ ! -f librdkafka.so ]; then
     error "Failed to create librdkafka.so"

data/ext/build_macos_arm64.sh

@@ -448,6 +448,12 @@ log "Creating self-contained librdkafka.dylib with Kerberos support..."
 
 # Create self-contained shared library by linking all static dependencies (NOW INCLUDING KERBEROS)
 # This is the macOS equivalent of your Linux gcc -shared command
+
+# Write symbol export file (macOS equivalent of export.map)
+cat > export_symbols.txt <<'EOF'
+_rd_kafka_*
+EOF
+
 clang -dynamiclib -fPIC \
     -Wl,-force_load,src/librdkafka.a \
     -Wl,-force_load,"$SASL_PREFIX/lib/libsasl2.a" \
@@ -462,6 +468,7 @@ clang -dynamiclib -fPIC \
     -Wl,-force_load,"$ZSTD_PREFIX/lib/libzstd.a" \
     -o librdkafka.dylib \
     -lpthread -lc -arch $ARCH -lresolv \
+    -framework GSS -framework Kerberos \
     -install_name @rpath/librdkafka.dylib \
     -Wl,-undefined,dynamic_lookup
 

data/ext/generate-ssl-certs.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+
+#==============================================================================
+# Kafka SSL Certificate Generator
+#==============================================================================
+#
+# DESCRIPTION:
+#   Generates SSL certificates for testing Kafka with SSL/TLS encryption.
+#   Creates both Java KeyStore (JKS) files for Kafka server and PEM files
+#   for client applications like rdkafka.
+#
+# PURPOSE:
+#   - Test SSL connectivity between Kafka clients and brokers
+#   - Validate rdkafka SSL integration
+#   - Enable encrypted communication for development/testing environments
+#
+# USAGE:
+#   ./ext/generate-ssl-certs.sh
+#   docker compose -f docker-compose-ssl.yml up
+#
+# REQUIREMENTS:
+#   - OpenSSL (for certificate generation)
+#   - Java keytool (usually included with JDK/JRE)
+#   - Write permissions in current directory
+#
+# OUTPUT FILES (created in ./ssl/ directory):
+#   ├── kafka.server.keystore.jks    # Kafka server's private key and certificate
+#   ├── kafka.server.truststore.jks  # Trusted CA certificates for Kafka
+#   ├── kafka_keystore_creds         # Password file for keystore
+#   ├── kafka_truststore_creds       # Password file for truststore
+#   ├── kafka_ssl_key_creds          # Password file for SSL keys
+#   ├── ca-cert                      # CA certificate (for rdkafka clients)
+#   └── ca-cert.pem                  # CA certificate in PEM format
+#
+# CONFIGURATION:
+#   - Certificate validity: 365 days
+#   - Password: "confluent" (all certificates use same password for simplicity)
+#   - Subject: CN=localhost (suitable for local testing)
+#   - CA Subject: CN=localhost-ca
+#
+# DOCKER COMPOSE INTEGRATION:
+#   Use with docker-compose-ssl.yml that mounts ./ssl directory to
+#   /etc/kafka/secrets inside the Kafka container.
+#
+# RDKAFKA CLIENT CONFIGURATION:
+#   security.protocol=SSL
+#   ssl.ca.location=./ssl/ca-cert
+#   ssl.endpoint.identification.algorithm=none  # For localhost testing
+#
+# NOTES:
+#   - Safe to run multiple times (cleans up existing files)
+#   - Certificates are self-signed and suitable for testing only
+#   - For production, use certificates signed by a trusted CA
+#   - All passwords are set to "confluent" for simplicity
+#
+#==============================================================================
+
+# Create ssl directory and clean up any existing files
+mkdir -p ssl
+cd ssl
+
+# Clean up existing files
+rm -f kafka.server.keystore.jks kafka.server.truststore.jks
+rm -f kafka_keystore_creds kafka_truststore_creds kafka_ssl_key_creds
+rm -f ca-key ca-cert cert-file cert-signed ca-cert.srl ca-cert.pem
+
+echo "Cleaned up existing SSL files..."
+
+# Set variables
+VALIDITY_DAYS=365
+PASSWORD="confluent"  # Use a simpler, well-known password
+DNAME="CN=localhost,OU=Test,O=Test,L=Test,ST=Test,C=US"
+
+# Create password files (all same password for simplicity)
+echo "$PASSWORD" > kafka_keystore_creds
+echo "$PASSWORD" > kafka_truststore_creds
+echo "$PASSWORD" > kafka_ssl_key_creds
+
+# Step 1: Generate CA key and certificate
+openssl req -new -x509 -keyout ca-key -out ca-cert -days $VALIDITY_DAYS -subj "/CN=localhost-ca/OU=Test/O=Test/L=Test/S=Test/C=US" -passin pass:$PASSWORD -passout pass:$PASSWORD
+
+# Step 2: Create truststore and import the CA certificate
+keytool -keystore kafka.server.truststore.jks -alias CARoot -import -file ca-cert -storepass $PASSWORD -keypass $PASSWORD -noprompt
+
+# Step 3: Create keystore
+keytool -keystore kafka.server.keystore.jks -alias localhost -validity $VALIDITY_DAYS -genkey -keyalg RSA -dname "$DNAME" -storepass $PASSWORD -keypass $PASSWORD
+
+# Step 4: Create certificate signing request
+keytool -keystore kafka.server.keystore.jks -alias localhost -certreq -file cert-file -storepass $PASSWORD -keypass $PASSWORD
+
+# Step 5: Sign the certificate with the CA
+openssl x509 -req -CA ca-cert -CAkey ca-key -in cert-file -out cert-signed -days $VALIDITY_DAYS -CAcreateserial -passin pass:$PASSWORD
+
+# Step 6: Import CA certificate into keystore
+keytool -keystore kafka.server.keystore.jks -alias CARoot -import -file ca-cert -storepass $PASSWORD -keypass $PASSWORD -noprompt
+
+# Step 7: Import signed certificate into keystore
+keytool -keystore kafka.server.keystore.jks -alias localhost -import -file cert-signed -storepass $PASSWORD -keypass $PASSWORD -noprompt
+
+# Export CA certificate to PEM format for rdkafka
+cp ca-cert ca-cert.pem
+
+# Clean up intermediate files (but keep ca-cert.pem for rdkafka)
+rm ca-key cert-file cert-signed
+
+echo "SSL certificates generated successfully!"
+echo "Password: $PASSWORD"
+echo ""
+echo "For rdkafka, use ca-cert.pem or ca-cert files"

data/karafka-rdkafka.gemspec

@@ -41,15 +41,17 @@ Gem::Specification.new do |gem|
   end
 
   gem.add_dependency 'ffi', '~> 1.15'
+  gem.add_dependency 'json', '> 2.0'
   gem.add_dependency 'logger'
   gem.add_dependency 'mini_portile2', '~> 2.6'
-  gem.add_dependency 'ostruct'
   gem.add_dependency 'rake', '> 12'
 
+  gem.add_development_dependency 'ostruct'
   gem.add_development_dependency 'pry'
   gem.add_development_dependency 'rspec', '~> 3.5'
   gem.add_development_dependency 'rake'
   gem.add_development_dependency 'simplecov'
+  gem.add_development_dependency 'warning'
 
   gem.metadata = {
     'funding_uri' => 'https://karafka.io/#become-pro',