rdkafka 0.21.0 → 0.22.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 027ae6ca4a08d2ed55825e411a38a538b4950f73c73de9599a51b4055e853721
-  data.tar.gz: 38020387566a559027cc0495b392b87423d3a5a8a2d18d9944edeb7294f76560
+  metadata.gz: 6bdfd8912cec56ebe8303d0ad14f3ab7ab2e5e0565e9418a6265a0e48172bc31
+  data.tar.gz: 646f8a174c1019a0417a571f5894c745701c5d670a0585beb40200a2ffe08193
 SHA512:
-  metadata.gz: 70e254d58fac9e28366e5d186cff73624624797a716b186d7c057a683bed2358464b2029560c6b951c97a09bd4d0dc6e9b62824fd9b4f787b2f8b3228f5ea729
-  data.tar.gz: 3955754c8441d9eb2153a0b6885bba279e047e41d437425462a7858c620c08868212733c975a0bc1dc543a82cbe6094cf9207b14030227f8ecb4f6fd3fb5a6f1
+  metadata.gz: 9d0711b10d02a1c6792fb21206d7729c958fadc186da5b6d8382d69b0d98183cddf23c123ea1602dd890f39603cd44341b864726c4a737966148fffb7e5e5dbd
+  data.tar.gz: deeacf4c205c164d2bd2d786f4a512f62caf8aeda6d6fcc11c24d2d665f4b7bd0a7f44447dfa1ebffc1fe54e7a9cdf36c7bbd4ae71c01e167b46b36a298d7482

data/.github/CODEOWNERS

@@ -0,0 +1,3 @@
+/.github @mensfeld
+/.github/workflows/ @mensfeld
+/.github/actions/ @mensfeld

data/.github/workflows/ci_linux_x86_64_gnu.yml

@@ -0,0 +1,271 @@
+# CI Strategy: Comprehensive Testing of Build and Precompiled Flows
+#
+# This workflow tests both compilation-from-source and precompiled binary distribution
+# strategies across multiple Ubuntu and Ruby versions to ensure broad compatibility and
+# reliability.
+#
+# WHY WE TEST BOTH UBUNTU 22.04 AND 24.04:
+# - Different system library versions (OpenSSL, zlib, libsasl2, libzstd, etc.)
+# - Different GCC compiler versions that affect native extension compilation
+# - Different glibc versions that can impact binary compatibility
+# - Real-world deployment scenarios where users run on various Ubuntu LTS versions
+# - Different Ruby versions
+#
+# COMPILATION FLOW (build_install + specs_install):
+# - Tests that librdkafka compiles correctly from source on each Ubuntu version
+# - Validates that mini_portile2 can successfully build native dependencies
+# - Ensures Ruby native extensions link properly with system libraries
+# - Verifies that the same codebase works across different toolchain versions
+#
+# PRECOMPILED FLOW (build_precompiled + specs_precompiled):
+# - Tests our precompiled static libraries work on different Ubuntu versions
+# - Validates that statically-linked binaries are truly portable across environments
+# - Ensures precompiled libraries don't have unexpected system dependencies
+# - Verifies that removing build tools doesn't break precompiled binary usage
+#
+# ARTIFACT ISOLATION:
+# - Each Ubuntu version gets separate artifacts (rdkafka-built-gem-22.04, etc.)
+# - Prevents cross-contamination of OS-specific compiled extensions
+# - Ensures test accuracy by matching build and test environments
+#
+# This comprehensive approach catches issues that single-platform testing would miss,
+# such as system library incompatibilities, compiler-specific bugs, or static linking
+# problems that only manifest on specific Ubuntu versions.
+
+name: CI Linux x86_64 GNU
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  pull_request:
+    branches: [ main, master ]
+  push:
+    branches: [ main, master ]
+  schedule:
+    - cron: '0 1 * * *'
+
+permissions:
+  contents: read
+
+env:
+  BUNDLE_RETRY: 6
+  BUNDLE_JOBS: 4
+
+jobs:
+  specs_install:
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby:
+          - '3.5.0-preview1'
+          - '3.4'
+          - '3.3'
+          - '3.2'
+          - '3.1'
+          - 'jruby-10.0'
+        ubuntu-version: ['22.04', '24.04']
+        include:
+          - ruby: '3.4'
+            coverage: 'true'
+          - ruby: 'jruby-10.0'
+            continue-on-error: true
+    runs-on: ubuntu-${{ matrix.ubuntu-version }}
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@a4effe49ee8ee5b8b5091268c473a4628afb5651 # v1.245.0
+        with:
+          ruby-version: ${{matrix.ruby}}
+          bundler-cache: false
+      - name: Start Kafka with Docker Compose
+        run: |
+          docker compose up -d
+          echo "Waiting for Kafka to be ready..."
+
+          sleep 10
+
+          echo "=== Container status ==="
+          docker compose ps kafka
+
+          for i in {1..30}; do
+            echo "=== Attempt $i/30 ==="
+
+            echo "Testing kafka-topics command..."
+            if docker compose exec -T kafka kafka-topics --bootstrap-server localhost:9092 --list >/dev/null 2>&1; then
+              echo "Kafka topics command succeeded!"
+              break
+            else
+              echo "Kafka topics command failed (exit code: $?)"
+            fi
+
+            echo "Sleeping 2 seconds..."
+            sleep 2
+          done
+      - name: Install dependencies
+        env:
+          RDKAFKA_EXT_PATH: ${{ github.workspace }}/ext
+        run: |
+          # Only install gems that aren't Ruby-version specific
+          bundle install
+      - name: Build gem with mini_portile
+        run: |
+          set -e
+          cd ext && bundle exec rake
+          cd ..
+      - name: Run all specs
+        env:
+          GITHUB_COVERAGE: ${{matrix.coverage}}
+          RDKAFKA_EXT_PATH: ${{ github.workspace }}/ext
+        continue-on-error: ${{ matrix.continue-on-error || false }}
+        run: |
+          bundle exec rspec
+
+  build_precompiled:
+    timeout-minutes: 30
+    # We precompile on older Ubuntu and check compatibility by running specs since we aim to
+    # release only one precompiled version for all supported Ubuntu versions
+    # This is why we do not want Renovate to update it automatically
+    runs-on: ubuntu-22.04 # renovate: ignore
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
+      - name: Install build dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            build-essential \
+            gcc \
+            g++ \
+            make \
+            tar \
+            gzip \
+            wget \
+            curl \
+            file \
+            pkg-config \
+            autoconf \
+            automake \
+            libtool \
+            python3 \
+            git \
+            ca-certificates \
+            patch \
+            libsasl2-dev \
+            libssl-dev \
+            zlib1g-dev \
+            libzstd-dev \
+            bison \
+            flex \
+            perl \
+            binutils-dev
+      - name: Cache build-tmp directory
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: ext/build-tmp
+          key: build-tmp-${{ runner.os }}-${{ hashFiles('ext/*.sh', 'ext/Rakefile') }}-v2
+      - name: Build precompiled librdkafka.so
+        run: |
+          cd ext
+          ./build_linux_x86_64_gnu.sh
+      - name: Upload precompiled library
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: librdkafka-precompiled-linux
+          path: ext/
+          retention-days: 1
+
+  specs_precompiled:
+    timeout-minutes: 30
+    needs: build_precompiled
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby:
+          - '3.5.0-preview1'
+          - '3.4'
+          - '3.3'
+          - '3.2'
+          - '3.1'
+        ubuntu-version: ['22.04', '24.04']
+        include:
+          - ruby: '3.4'
+            ubuntu-version: '24.04'
+            coverage: 'true'
+
+    runs-on: ubuntu-${{ matrix.ubuntu-version }}
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
+      - name: Download precompiled library
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          name: librdkafka-precompiled-linux
+          path: ext/
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@a4effe49ee8ee5b8b5091268c473a4628afb5651 # v1.245.0
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: false
+      - name: Start Kafka with Docker Compose
+        run: |
+          docker compose up -d
+          echo "Waiting for Kafka to be ready..."
+
+          sleep 10
+
+          echo "=== Container status ==="
+          docker compose ps kafka
+          for i in {1..30}; do
+            echo "=== Attempt $i/30 ==="
+
+            echo "Testing kafka-topics command..."
+            if docker compose exec -T kafka kafka-topics --bootstrap-server localhost:9092 --list >/dev/null 2>&1; then
+              echo "Kafka topics command succeeded!"
+              break
+            else
+              echo "Kafka topics command failed (exit code: $?)"
+            fi
+
+            echo "Sleeping 2 seconds..."
+            sleep 2
+          done
+      - name: Install bundle with precompiled library
+        env:
+          GITHUB_COVERAGE: ${{ matrix.coverage }}
+          RDKAFKA_EXT_PATH: ${{ github.workspace }}/ext
+        run: |
+          bundle install
+          echo "Bundle install completed with precompiled library"
+      - name: Remove build dependencies to test static linking
+        continue-on-error: true
+        run: |
+          echo "Removing build dependencies to verify precompiled library is truly self-contained..."
+
+          # Remove packages one by one to avoid dependency conflicts
+          packages_to_remove="build-essential gcc g++ make patch tar wget libsasl2-dev libssl-dev zlib1g-dev libzstd-dev"
+
+          for package in $packages_to_remove; do
+            if dpkg -l | grep -q "^ii.*$package "; then
+              echo "Removing $package..."
+              sudo dpkg --remove --force-depends $package 2>/dev/null || echo "Could not remove $package"
+            else
+              echo "$package is not installed"
+            fi
+          done
+
+          echo "Build dependencies removal completed"
+          echo "Remaining build tools:"
+          which gcc g++ make 2>/dev/null || echo "No build tools found in PATH (good!)"
+      - name: Run specs with precompiled library
+        env:
+          GITHUB_COVERAGE: ${{ matrix.coverage }}
+          RDKAFKA_EXT_PATH: ${{ github.workspace }}/ext
+        run: |
+          bundle exec rspec

data/.github/workflows/ci_linux_x86_64_musl.yml

@@ -0,0 +1,194 @@
+# Why We Build and Run Without Caching Native Extensions
+#
+# We intentionally compile the native librdkafka library fresh in each test job
+# rather than caching or pre-building it for several reasons:
+#
+# 1. Architecture Compatibility
+#    - Pre-built native libraries (.so files) are architecture-specific
+#    - Can cause "Exec format error" when build/runtime environments differ
+#    - Building in the same container guarantees compatibility
+#
+# 2. Container Image Variations
+#    - Different Ruby Alpine images may have subtle differences in:
+#      * Base system libraries, compiler toolchains, musl libc versions
+#    - These differences can cause pre-built libraries to fail at runtime
+#
+# 3. Simplicity and Reliability
+#    - Single source of truth: everything builds and runs in same environment
+#    - No artifact management complexity or potential upload/download failures
+#    - Easier debugging when issues are contained in one job
+#
+# Trade-offs: Slightly longer CI times (~2-3 min per job) but much more reliable
+# than dealing with architecture mismatches and artifact corruption issues.
+
+name: CI Linux x86_64 musl
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  pull_request:
+    branches: [ main, master ]
+  push:
+    branches: [ main, master ]
+  schedule:
+    - cron: '0 1 * * *'
+
+permissions:
+  contents: read
+
+env:
+  BUNDLE_RETRY: 6
+  BUNDLE_JOBS: 4
+
+jobs:
+  specs_install:
+    timeout-minutes: 30
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - ruby: '3.1'
+            alpine_version: '3.21'
+          - ruby: '3.2'
+            alpine_version: '3.21'
+          - ruby: '3.2'
+            alpine_version: '3.22'
+          - ruby: '3.3'
+            alpine_version: '3.21'
+          - ruby: '3.3'
+            alpine_version: '3.22'
+          - ruby: '3.4'
+            alpine_version: '3.21'
+            coverage: 'true'
+          - ruby: '3.4'
+            alpine_version: '3.22'
+            coverage: 'true'
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
+
+      - name: Start Kafka with Docker Compose
+        run: |
+          docker compose up -d
+          echo "Waiting for Kafka to be ready..."
+          sleep 10
+          for i in {1..30}; do
+            if docker compose exec -T kafka kafka-topics --bootstrap-server localhost:9092 --list >/dev/null 2>&1; then
+              echo "Kafka topics command succeeded!"
+              break
+            fi
+            sleep 2
+          done
+
+      - name: Run all specs
+        env:
+          GITHUB_COVERAGE: ${{ matrix.coverage }}
+        run: |
+          docker run --rm \
+            --network host \
+            -v "${{ github.workspace }}:/workspace" \
+            -w /workspace \
+            -e "GITHUB_COVERAGE=${{ matrix.coverage }}" \
+            ruby:${{ matrix.ruby }}-alpine${{ matrix.alpine_version }} \
+            sh -c 'apk add --no-cache git curl ca-certificates build-base linux-headers \
+              pkgconf perl autoconf automake libtool bison flex file \
+              ruby-dev ruby-bundler bash zstd-dev zlib zlib-dev openssl-dev \
+              cyrus-sasl-dev cyrus-sasl cyrus-sasl-login \
+              cyrus-sasl-crammd5 cyrus-sasl-digestmd5 cyrus-sasl-gssapiv2 cyrus-sasl-scram \
+              krb5-libs openssl zlib zstd-libs && \
+              git config --global --add safe.directory /workspace && \
+              bundle config set --local path vendor/bundle && \
+              bundle install && \
+              cd ext && bundle exec rake && \
+              cd .. && \
+              bundle exec ruby -S rspec'
+
+  build_precompiled:
+    timeout-minutes: 45
+    runs-on: ubuntu-latest
+    container:
+      # Similar to GNU, we build on the oldest for ABI compatibility
+      image: alpine:3.18@sha256:de0eb0b3f2a47ba1eb89389859a9bd88b28e82f5826b6969ad604979713c2d4f # renovate: ignore
+    steps:
+      - name: Install dependencies
+        run: |
+          apk add --no-cache git curl ca-certificates build-base linux-headers \
+            pkgconf perl autoconf automake libtool bison flex file bash wget zstd-dev \
+            openssl-dev cyrus-sasl-dev cyrus-sasl cyrus-sasl-login \
+            cyrus-sasl-crammd5 cyrus-sasl-digestmd5 cyrus-sasl-gssapiv2 cyrus-sasl-scram \
+            krb5-libs openssl zlib zlib-dev zstd-libs
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
+      - name: Configure git safe directory
+        run: git config --global --add safe.directory /__w/karafka-rdkafka/karafka-rdkafka
+      - name: Cache build-tmp directory
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: ext/build-tmp-musl
+          key: build-tmp-musl-${{ runner.os }}-${{ hashFiles('ext/*.sh', 'ext/Rakefile') }}-v2
+      - name: Build precompiled librdkafka.so
+        run: |
+          cd ext
+          bash ./build_linux_x86_64_musl.sh
+      - name: Upload precompiled library
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: librdkafka-precompiled-musl
+          path: ext/
+          retention-days: 1
+
+  specs_precompiled:
+    timeout-minutes: 30
+    runs-on: ubuntu-latest
+    needs: build_precompiled
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby:
+          - '3.4'
+          - '3.3'
+          - '3.2'
+          - '3.1'
+        include:
+          - ruby: '3.4'
+            coverage: 'true'
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
+      - name: Download precompiled library
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          name: librdkafka-precompiled-musl
+          path: ext/
+      - name: Start Kafka with Docker Compose
+        run: |
+          docker compose up -d
+          echo "Waiting for Kafka to be ready..."
+          sleep 10
+
+          for i in {1..30}; do
+            if docker compose exec -T kafka kafka-topics --bootstrap-server localhost:9092 --list >/dev/null 2>&1; then
+              echo "Kafka topics command succeeded!"
+              break
+            fi
+            sleep 2
+          done
+      - name: Run specs with precompiled library
+        env:
+          GITHUB_COVERAGE: ${{ matrix.coverage }}
+          RDKAFKA_EXT_PATH: ${{ github.workspace }}/ext
+        run: |
+          docker run --rm \
+            --network host \
+            -v "${{ github.workspace }}:/workspace" \
+            -w /workspace \
+            -e "GITHUB_COVERAGE=${{ matrix.coverage }}" \
+            -e "RDKAFKA_EXT_PATH=/workspace/ext" \
+            ruby:${{ matrix.ruby }}-alpine \
+            sh -c 'apk add --no-cache git build-base linux-headers bash && git config --global --add safe.directory /workspace && bundle config set --local path vendor/bundle && bundle install && bundle exec ruby -S rspec'