karafka-rdkafka 0.21.0 → 0.22.0

data/.github/workflows/push_linux_x86_64_musl.yml

@@ -1,79 +0,0 @@
-name: Push Linux x86_64 musl Platform Gem
-on:
-  push:
-    tags:
-      - v*
-permissions:
-  contents: read
-env:
-  BUNDLE_RETRY: 6
-  BUNDLE_JOBS: 4
-jobs:
-  build:
-    if: github.repository_owner == 'karafka'
-    timeout-minutes: 30
-    runs-on: ubuntu-latest
-    environment: deployment
-    container:
-      # Same as CI, we build on the oldest possible for ABI compatibility
-      image: alpine:3.18@sha256:de0eb0b3f2a47ba1eb89389859a9bd88b28e82f5826b6969ad604979713c2d4f # renovate: ignore
-    steps:
-      - name: Install dependencies
-        run: |
-          apk add --no-cache git curl ca-certificates build-base linux-headers \
-            pkgconf perl autoconf automake libtool bison flex file bash wget zstd-dev \
-            openssl-dev cyrus-sasl-dev cyrus-sasl cyrus-sasl-login \
-            cyrus-sasl-crammd5 cyrus-sasl-digestmd5 cyrus-sasl-gssapiv2 cyrus-sasl-scram \
-            krb5-libs openssl zlib zlib-dev zstd-libs
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-        with:
-          fetch-depth: 0
-      - name: Configure git safe directory
-        run: git config --global --add safe.directory /__w/karafka-rdkafka/karafka-rdkafka
-      - name: Cache build-tmp directory
-        uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
-        with:
-          path: ext/build-tmp-musl
-          key: build-tmp-musl-${{ runner.os }}-${{ hashFiles('ext/*.sh', 'ext/Rakefile') }}-v2
-      - name: Build precompiled librdkafka.so
-        run: |
-          cd ext
-          bash ./build_linux_x86_64_musl.sh
-      - name: Upload precompiled library
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
-        with:
-          name: librdkafka-precompiled-musl
-          path: ext/
-          retention-days: 1
-
-  push:
-    if: github.repository_owner == 'karafka'
-    timeout-minutes: 30
-    runs-on: ubuntu-latest
-    needs: build
-    environment: deployment
-    permissions:
-      contents: write
-      id-token: write
-    steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-        with:
-          fetch-depth: 0
-      - name: Download precompiled library
-        uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
-        with:
-          name: librdkafka-precompiled-musl
-          path: ext/
-      - name: Set up Ruby
-        uses: ruby/setup-ruby@2a7b30092b0caf9c046252510f9273b4875f3db9 # v1.254.0
-        with:
-          ruby-version: '3.4'
-          bundler-cache: false
-      - name: Configure trusted publishing credentials
-        uses: rubygems/configure-rubygems-credentials@bc6dd217f8a4f919d6835fcfefd470ef821f5c44 # v1.0.0
-      - name: Build and push platform-specific gem
-        run: |
-          gem build *.gemspec
-          gem push *.gem
-        env:
-          RUBY_PLATFORM: 'x86_64-linux-musl'

data/.github/workflows/push_macos_arm64.yml

@@ -1,54 +0,0 @@
-name: Push macOS ARM64 Platform Gem
-
-on:
-  push:
-    tags:
-      - v*
-
-permissions:
-  contents: read
-
-env:
-  BUNDLE_RETRY: 6
-  BUNDLE_JOBS: 4
-
-jobs:
-  push:
-    if: github.repository_owner == 'karafka'
-    timeout-minutes: 30
-    runs-on: macos-14
-    environment: deployment
-    permissions:
-      contents: write
-      id-token: write
-    steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-        with:
-          fetch-depth: 0
-      - name: Install Bash 4+ and Kerberos
-        run: |
-          brew install bash
-          brew list krb5 &>/dev/null || brew install krb5
-          echo "/opt/homebrew/bin" >> $GITHUB_PATH
-      - name: Cache build-tmp directory
-        uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
-        with:
-          path: ext/build-tmp-macos
-          key: build-tmp-${{ runner.os }}-${{ hashFiles('ext/*.sh', 'ext/Rakefile') }}-v2
-      - name: Set up Ruby
-        uses: ruby/setup-ruby@2a7b30092b0caf9c046252510f9273b4875f3db9 # v1.254.0
-        with:
-          ruby-version: '3.4'
-          bundler-cache: false
-      - name: Build precompiled librdkafka for macOS ARM64
-        run: |
-          cd ext
-          /opt/homebrew/bin/bash ./build_macos_arm64.sh
-      - name: Configure trusted publishing credentials
-        uses: rubygems/configure-rubygems-credentials@bc6dd217f8a4f919d6835fcfefd470ef821f5c44 # v1.0.0
-      - name: Build and push platform-specific gem
-        run: |
-          gem build *.gemspec
-          gem push *.gem
-        env:
-          RUBY_PLATFORM: 'arm64-darwin'

data/.github/workflows/push_ruby.yml

@@ -1,37 +0,0 @@
-name: Push Ruby Platform Gem
-
-on:
-  push:
-    tags:
-      - v*
-
-permissions:
-  contents: read
-
-jobs:
-  push:
-    if: github.repository_owner == 'karafka'
-    runs-on: ubuntu-latest
-    environment: deployment
-
-    permissions:
-      contents: write
-      id-token: write
-
-    steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-        with:
-          fetch-depth: 0
-
-      - name: Set up Ruby
-        uses: ruby/setup-ruby@2a7b30092b0caf9c046252510f9273b4875f3db9 # v1.254.0
-        with:
-          bundler-cache: false
-
-      - name: Build rdkafka-ruby
-        run: |
-          set -e
-          bundle install
-          cd ext && bundle exec rake
-
-      - uses: rubygems/release-gem@a25424ba2ba8b387abc8ef40807c2c85b96cbe32 # v1.1.1

data/.github/workflows/trigger-wiki-refresh.yml

@@ -1,30 +0,0 @@
-name: Trigger Wiki Refresh
-
-on:
-  release:
-    types: [published]
-  push:
-    branches: [main]
-
-jobs:
-  trigger-wiki-refresh:
-    runs-on: ubuntu-latest
-    environment: wiki-trigger
-    if: github.repository_owner == 'karafka'
-    steps:
-      - name: Trigger wiki refresh
-        uses: peter-evans/repository-dispatch@ff45666b9427631e3450c54a1bcbee4d9ff4d7c0 # v3.0.0
-        with:
-          token: ${{ secrets.WIKI_REPO_TOKEN }}
-          repository: karafka/wiki
-          event-type: sync-trigger
-          client-payload: |
-            {
-              "repository": "${{ github.repository }}",
-              "event_name": "${{ github.event_name }}",
-              "release_tag": "${{ github.event.release.tag_name || '' }}",
-              "release_name": "${{ github.event.release.name || '' }}",
-              "commit_sha": "${{ github.sha }}",
-              "commit_message": "Trigger Wiki Refresh",
-              "triggered_by": "${{ github.actor }}"
-            }

data/.github/workflows/verify-action-pins.yml

@@ -1,16 +0,0 @@
-name: Verify Action Pins
-on:
-  pull_request:
-    paths:
-      - '.github/workflows/**'
-jobs:
-  verify_action_pins:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-      - name: Check SHA pins
-        run: |
-          if grep -E -r "uses: .*/.*@(v[0-9]+|main|master)($|[[:space:]]|$)" --include="*.yml" --include="*.yaml" .github/workflows/ | grep -v "#"; then
-            echo "::error::Actions should use SHA pins, not tags or branch names"
-            exit 1
-          fi

data/.gitignore

@@ -1,16 +0,0 @@
-# Ignore bundler config.
-/.bundle
-
-Gemfile.lock
-ext/ports
-ext/tmp
-ext/librdkafka.*
-ext/build-tmp
-*.gem
-.yardoc
-doc
-coverage
-vendor
-.idea/
-out/
-ssl/

data/.rspec

@@ -1,3 +0,0 @@
---require spec_helper
---format documentation
---pattern spec/lib/**/*_spec.rb

data/.ruby-gemset

@@ -1 +0,0 @@
-rdkafka-ruby

data/.ruby-version

@@ -1 +0,0 @@
-3.4.5

data/.yardopts

@@ -1,2 +0,0 @@
---no-private
---markup=markdown

data/ext/README.md

@@ -1,19 +0,0 @@
-# Ext
-
-This gem depends on the `librdkafka` C library. It is downloaded, stored in
-`dist/` directory, and checked into source control.
-
-To update the `librdkafka` version follow the following steps:
-
-* Go to https://github.com/confluentinc/librdkafka/releases to get the new
-  version number and asset checksum for `tar.gz`.
-* Change the version in `lib/rdkafka/version.rb`
-* Change the `sha256` in `lib/rdkafka/version.rb`
-* Run `bundle exec rake dist:download` in the `ext` directory to download the
-  new release and place it in the `dist/` for you
-* Run `bundle exec rake` in the `ext` directory to build the new version
-* Run `docker-compose pull` in the main gem directory to ensure the docker
-  images used by the tests and run `docker-compose up`
-* Finally, run `bundle exec rspec` in the main gem directory to execute
-  the test suite to detect any regressions that may have been introduced
-  by the update

data/ext/build_common.sh

@@ -1,376 +0,0 @@
-#!/usr/bin/env bash
-#
-# Common functions and constants for librdkafka builds
-# This file should be sourced by platform-specific build scripts
-#
-# Usage: source "$(dirname "${BASH_SOURCE[0]}")/build_common.sh"
-#
-
-# Prevent multiple sourcing
-if [[ "${BUILD_COMMON_SOURCED:-}" == "1" ]]; then
-    return 0
-fi
-
-BUILD_COMMON_SOURCED=1
-
-# Version constants - update these to upgrade dependencies
-readonly OPENSSL_VERSION="3.0.16"
-readonly CYRUS_SASL_VERSION="2.1.28"
-readonly ZLIB_VERSION="1.3.1"
-readonly ZSTD_VERSION="1.5.7"
-readonly KRB5_VERSION="1.21.3"
-readonly LIBRDKAFKA_VERSION="2.11.0"
-
-# SHA256 checksums for supply chain security
-# Update these when upgrading versions
-declare -A CHECKSUMS=(
-    ["openssl-${OPENSSL_VERSION}.tar.gz"]="57e03c50feab5d31b152af2b764f10379aecd8ee92f16c985983ce4a99f7ef86"
-    ["cyrus-sasl-${CYRUS_SASL_VERSION}.tar.gz"]="7ccfc6abd01ed67c1a0924b353e526f1b766b21f42d4562ee635a8ebfc5bb38c"
-    ["zlib-1.3.1.tar.gz"]="9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23"
-    ["zstd-${ZSTD_VERSION}.tar.gz"]="eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3"
-    ["krb5-${KRB5_VERSION}.tar.gz"]="b7a4cd5ead67fb08b980b21abd150ff7217e85ea320c9ed0c6dadd304840ad35"
-    ["librdkafka-${LIBRDKAFKA_VERSION}.tar.gz"]="592a823dc7c09ad4ded1bc8f700da6d4e0c88ffaf267815c6f25e7450b9395ca"
-)
-
-# Colors for output
-readonly RED='\033[0;31m'
-readonly GREEN='\033[0;32m'
-readonly YELLOW='\033[1;33m'
-readonly BLUE='\033[0;34m'
-readonly NC='\033[0m' # No Color
-
-# Logging functions
-log() {
-    echo -e "${GREEN}[$(date '+%Y-%m-%d %H:%M:%S')] $1${NC}"
-}
-
-warn() {
-    echo -e "${YELLOW}[WARNING] $1${NC}"
-}
-
-error() {
-    echo -e "${RED}[ERROR] $1${NC}"
-    exit 1
-}
-
-security_log() {
-    echo -e "${BLUE}[SECURITY] $1${NC}"
-}
-
-# Function to verify checksums
-verify_checksum() {
-    local file="$1"
-    local expected_checksum="${CHECKSUMS[$file]}"
-
-    if [ -z "$expected_checksum" ]; then
-        error "No checksum defined for $file - this is a security risk!"
-    fi
-
-    security_log "Verifying checksum for $file..."
-    local actual_checksum
-
-    # Use platform-appropriate checksum command
-    if command -v sha256sum &> /dev/null; then
-        actual_checksum=$(sha256sum "$file" | cut -d' ' -f1)
-    elif command -v shasum &> /dev/null; then
-        actual_checksum=$(shasum -a 256 "$file" | cut -d' ' -f1)
-    else
-        error "No SHA256 checksum utility found (tried sha256sum, shasum)"
-    fi
-
-    if [ "$actual_checksum" = "$expected_checksum" ]; then
-        security_log "✅ Checksum verified for $file"
-        return 0
-    else
-        error "❌ CHECKSUM MISMATCH for $file!
-Expected: $expected_checksum
-Actual:   $actual_checksum
-This could indicate a supply chain attack or corrupted download!"
-    fi
-}
-
-# Function to securely download and verify files
-secure_download() {
-    local url="$1"
-    local filename="$2"
-
-    # Check if file already exists in current directory (may have been already downloaded)
-    if [ -f "$filename" ]; then
-        log "File $filename already exists, verifying checksum..."
-        verify_checksum "$filename"
-        return 0
-    fi
-
-    # Check dist directory relative to script location
-    local script_dir
-    script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-    local dist_file="$script_dir/../dist/$filename"
-
-    if [ -f "$dist_file" ]; then
-        log "Using distributed $filename from dist/"
-        cp "$dist_file" "$filename"
-        verify_checksum "$filename"
-        return 0
-    fi
-
-    log "Downloading $filename from $url..."
-
-    # Use platform-appropriate download command
-    if command -v wget &> /dev/null; then
-        # Linux - use wget with security options
-        if ! wget --secure-protocol=TLSv1_2 \
-                 --https-only \
-                 --timeout=30 \
-                 --tries=3 \
-                 --progress=bar \
-                 "$url" \
-                 -O "$filename"; then
-            error "Failed to download $filename from $url"
-        fi
-    elif command -v curl &> /dev/null; then
-        # macOS/fallback - use curl with security options
-        if ! curl -L \
-                 --tlsv1.2 \
-                 --connect-timeout 30 \
-                 --max-time 300 \
-                 --retry 3 \
-                 --progress-bar \
-                 "$url" \
-                 -o "$filename"; then
-            error "Failed to download $filename from $url"
-        fi
-    else
-        error "No download utility found (tried wget, curl)"
-    fi
-
-    # Verify checksum immediately after download
-    verify_checksum "$filename"
-}
-
-# Function to detect CPU count for parallel builds
-get_cpu_count() {
-    if command -v nproc &> /dev/null; then
-        nproc
-    elif command -v sysctl &> /dev/null; then
-        sysctl -n hw.ncpu
-    else
-        echo "4"  # fallback
-    fi
-}
-
-# Function to auto-detect librdkafka tarball
-find_librdkafka_tarball() {
-    local dist_dir="$1"
-    local tarball="$dist_dir/librdkafka-${LIBRDKAFKA_VERSION}.tar.gz"
-
-    if [ ! -f "$tarball" ]; then
-        error "librdkafka-${LIBRDKAFKA_VERSION}.tar.gz not found in $dist_dir"
-    fi
-
-    echo "$tarball"
-}
-
-# Function to find and validate patches
-find_patches() {
-    local patches_dir="$1"
-    local -n patches_array=$2  # nameref to output array
-
-    patches_array=()
-
-    if [ -d "$patches_dir" ]; then
-        while IFS= read -r -d '' patch; do
-            patches_array+=("$patch")
-        done < <(find "$patches_dir" -name "*.patch" -type f -print0 | sort -z)
-
-        if [ ${#patches_array[@]} -gt 0 ]; then
-            log "Found ${#patches_array[@]} patches to apply:"
-            for patch in "${patches_array[@]}"; do
-                log "  - $(basename "$patch")"
-            done
-        else
-            log "No patches found in $patches_dir"
-        fi
-    else
-        log "No patches directory found: $patches_dir"
-    fi
-}
-
-# Function to apply patches
-apply_patches() {
-    local -n patches_array=$1  # nameref to patches array
-
-    if [ ${#patches_array[@]} -gt 0 ]; then
-        log "Applying Ruby-specific patches..."
-        for patch in "${patches_array[@]}"; do
-            log "Applying patch: $(basename "$patch")"
-            if patch -p1 < "$patch"; then
-                log "✅ Successfully applied $(basename "$patch")"
-            else
-                error "❌ Failed to apply patch: $(basename "$patch")"
-            fi
-        done
-        log "All patches applied successfully"
-    fi
-}
-
-# Function to verify librdkafka tarball checksum if available
-verify_librdkafka_checksum() {
-    local tarball="$1"
-    local filename
-    filename=$(basename "$tarball")
-
-    if [ -n "${CHECKSUMS[$filename]:-}" ]; then
-        local current_dir
-        current_dir=$(pwd)
-        cd "$(dirname "$tarball")"
-        verify_checksum "$filename"
-        cd "$current_dir"
-    else
-        warn "No checksum defined for $filename - consider adding one for security"
-    fi
-}
-
-# Function to set execute permissions on configure scripts
-fix_configure_permissions() {
-    log "Setting execute permissions on configure scripts..."
-    chmod +x configure* 2>/dev/null || true
-    chmod +x mklove/modules/configure.* 2>/dev/null || true
-}
-
-# Function to print security summary
-print_security_summary() {
-    security_log "🔒 SECURITY VERIFICATION COMPLETE"
-    security_log "All dependencies downloaded and verified with SHA256 checksums"
-    security_log "Supply chain integrity maintained throughout build process"
-}
-
-# Function to print build summary
-print_build_summary() {
-    local platform="$1"
-    local arch="$2"
-    local output_dir="$3"
-    local library_name="$4"
-
-    log "Build completed successfully!"
-    log "📦 Self-contained librdkafka built for $platform $arch:"
-    log "   ✅ OpenSSL $OPENSSL_VERSION (SSL/TLS support) - checksum verified"
-    log "   ✅ Cyrus SASL $CYRUS_SASL_VERSION (authentication for AWS MSK) - checksum verified"
-    log "   ✅ MIT Kerberos $KRB5_VERSION (GSSAPI/Kerberos authentication) - checksum verified"
-    log "   ✅ zlib $ZLIB_VERSION (compression) - checksum verified"
-    log "   ✅ ZStd $ZSTD_VERSION (high-performance compression) - checksum verified"
-    log ""
-    log "🎯 Ready for deployment on $platform systems"
-    log "☁️  Compatible with AWS MSK and other secured Kafka clusters"
-    log "🔐 Supply chain security: All dependencies cryptographically verified"
-    log ""
-    log "Location: $output_dir/$library_name"
-}
-
-# Function to clean up build directory with user prompt (except .tar.gz files in CI)
-cleanup_build_dir() {
-    local build_dir="$1"
-
-    if [ "${CI:-}" = "true" ]; then
-        # In CI: remove everything except .tar.gz files without prompting
-        echo "CI detected: cleaning up $build_dir (preserving .tar.gz files for caching)"
-
-        # First, find and move all .tar.gz files to a temp location
-        temp_dir=$(mktemp -d)
-        find "$build_dir" -name "*.tar.gz" -exec mv {} "$temp_dir/" \; 2>/dev/null || true
-
-        # Remove everything in build_dir
-        rm -rf "$build_dir"/* 2>/dev/null || true
-        rm -rf "$build_dir"/.* 2>/dev/null || true
-
-        # Move .tar.gz files back
-        mv "$temp_dir"/* "$build_dir/" 2>/dev/null || true
-        rmdir "$temp_dir" 2>/dev/null || true
-
-        log "Build directory cleaned up (preserved .tar.gz files)"
-    else
-        # Interactive mode: prompt user
-        echo
-        read -p "Remove build directory $build_dir? (y/N): " -n 1 -r
-        echo
-        if [[ $REPLY =~ ^[Yy]$ ]]; then
-            rm -rf "$build_dir"
-            log "Build directory cleaned up"
-        fi
-    fi
-}
-
-# Function to validate build environment
-check_common_dependencies() {
-    log "Checking common build dependencies..."
-
-    local missing_tools=()
-
-    command -v tar &> /dev/null || missing_tools+=("tar")
-    command -v make &> /dev/null || missing_tools+=("make")
-    command -v patch &> /dev/null || missing_tools+=("patch")
-
-    # Check for download tools
-    if ! command -v wget &> /dev/null && ! command -v curl &> /dev/null; then
-        missing_tools+=("wget or curl")
-    fi
-
-    # Check for checksum tools
-    if ! command -v sha256sum &> /dev/null && ! command -v shasum &> /dev/null; then
-        missing_tools+=("sha256sum or shasum")
-    fi
-
-    if [ ${#missing_tools[@]} -gt 0 ]; then
-        error "Missing required tools: ${missing_tools[*]}"
-    fi
-
-    log "✅ Common build tools found"
-}
-
-# Function to extract tarball if directory doesn't exist
-extract_if_needed() {
-    local tarball="$1"
-    local expected_dir="$2"
-
-    if [ ! -d "$expected_dir" ]; then
-        log "Extracting $(basename "$tarball")..."
-        tar xzf "$tarball"
-    else
-        log "Directory $expected_dir already exists, skipping extraction"
-    fi
-}
-
-# Download URLs for dependencies
-get_openssl_url() {
-    echo "https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz"
-}
-
-get_sasl_url() {
-    echo "https://github.com/cyrusimap/cyrus-sasl/releases/download/cyrus-sasl-${CYRUS_SASL_VERSION}/cyrus-sasl-${CYRUS_SASL_VERSION}.tar.gz"
-}
-
-get_zlib_url() {
-    echo "https://github.com/madler/zlib/releases/download/v${ZLIB_VERSION}/zlib-${ZLIB_VERSION}.tar.gz"
-}
-
-get_zstd_url() {
-    echo "https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz"
-}
-
-get_krb5_url() {
-    # Using MIT mirror since kerberos.org is down
-    # echo "https://kerberos.org/dist/krb5/${KRB5_VERSION%.*}/krb5-${KRB5_VERSION}.tar.gz"
-    echo "https://web.mit.edu/kerberos/dist/krb5/${KRB5_VERSION%.*}/krb5-${KRB5_VERSION}.tar.gz"
-}
-
-# Export functions and variables that scripts will need
-export -f log warn error security_log
-export -f verify_checksum secure_download get_cpu_count
-export -f find_librdkafka_tarball find_patches apply_patches
-export -f verify_librdkafka_checksum fix_configure_permissions
-export -f print_security_summary print_build_summary cleanup_build_dir
-export -f check_common_dependencies extract_if_needed
-export -f get_openssl_url get_sasl_url get_zlib_url get_zstd_url get_krb5_url
-
-# Export constants
-export OPENSSL_VERSION CYRUS_SASL_VERSION ZLIB_VERSION ZSTD_VERSION KRB5_VERSION
-export RED GREEN YELLOW BLUE NC