karafka-rdkafka 0.21.0.rc1 → 0.21.0.rc2

data/.github/workflows/ci_linux_ubuntu_x86_64_gnu_complementary.yml

@@ -0,0 +1,294 @@
+# CI Strategy: Complementary Testing for Ubuntu Precompiled Binaries and Extended Compatibility
+#
+# This workflow runs complementary tests that don't need to block PRs but are essential
+# for catching regressions in precompiled binary distribution, edge case Ruby versions,
+# and system library compatibility. It complements the main CI by testing scenarios
+# that are stable-but-critical on a nightly schedule.
+#
+# WHY COMPLEMENTARY TESTING:
+# - Precompiled binary testing is stable (rarely breaks from code changes)
+# - Edge Ruby versions (3.1, 3.5-preview, JRuby) need testing but shouldn't block PRs
+# - Integration specs test OS library compatibility, which changes with system updates
+# - These tests catch regressions from external changes (system libs, Ruby updates)
+# - Running every 3 days to prevent these slower tests from blocking PR velocity
+# - Manual triggering allows testing workflow changes before they go into schedule
+#
+# PRECOMPILED BINARY TESTING (build_precompiled + specs_precompiled):
+# - Tests static library distribution works across Ubuntu versions
+# - Validates precompiled libraries don't have unexpected system dependencies
+# - Ensures binary compatibility between different Ubuntu LTS versions
+# - Tests SSL/TLS connectivity with precompiled binaries
+# - Validates that removing build tools doesn't break precompiled usage
+#
+# EXTENDED COMPATIBILITY TESTING:
+# - Edge case Ruby versions: 3.1, 3.5.0-preview1, JRuby-10.0
+# - Integration specs testing system library compatibility
+# - Both compilation and precompiled flows for comprehensive coverage
+# - Cross-platform compatibility validation
+#
+# SCHEDULING STRATEGY:
+# - Runs every 3 days at 2 AM to catch system/library changes from base image updates
+# - Triggers on workflow file changes to validate CI modifications
+# - Manual dispatch available for ad-hoc regression testing
+# - Separate artifact naming prevents interference with main CI
+#
+# This approach ensures comprehensive coverage while keeping PR CI fast and focused
+# on code-related issues rather than infrastructure/system regressions.
+
+name: CI Linux Complementary Ubuntu x86_64 GNU
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  schedule:
+    - cron: '0 2 */3 * *'
+  workflow_dispatch:
+  push:
+    paths:
+      - '.github/workflows/ci_linux_ubuntu_x86_64_gnu_complementary.yml'
+    branches: [ main, master ]
+  pull_request:
+    paths:
+      - '.github/workflows/ci_linux_ubuntu_x86_64_gnu_complementary.yml'
+    branches: [ main, master ]
+
+permissions:
+  contents: read
+
+env:
+  BUNDLE_RETRY: 6
+  BUNDLE_JOBS: 4
+
+jobs:
+  build_precompiled:
+    timeout-minutes: 30
+    # We precompile on older Ubuntu and check compatibility by running specs since we aim to
+    # release only one precompiled version for all supported Ubuntu versions
+    # This is why we do not want Renovate to update it automatically
+    runs-on: ubuntu-22.04 # renovate: ignore
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
+      - name: Install build dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            build-essential \
+            gcc \
+            g++ \
+            make \
+            tar \
+            gzip \
+            wget \
+            curl \
+            file \
+            pkg-config \
+            autoconf \
+            automake \
+            libtool \
+            python3 \
+            git \
+            ca-certificates \
+            patch \
+            libsasl2-dev \
+            libssl-dev \
+            zlib1g-dev \
+            libzstd-dev \
+            bison \
+            flex \
+            perl \
+            binutils-dev
+      - name: Cache build-tmp directory
+        uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
+        with:
+          path: ext/build-tmp
+          key: build-tmp-${{ runner.os }}-${{ hashFiles('ext/*.sh', 'ext/Rakefile') }}-v2
+      - name: Build precompiled librdkafka.so
+        run: |
+          cd ext
+          ./build_linux_x86_64_gnu.sh
+      - name: Upload precompiled library
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: librdkafka-precompiled-ubuntu-complementary
+          path: ext/
+          retention-days: 1
+
+  specs_install:
+    timeout-minutes: 50
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby:
+          - '3.5.0-preview1'
+          - '3.4'
+          - '3.3'
+          - '3.2'
+          - '3.1'
+          - 'jruby-10.0'
+        ubuntu-version: ['22.04', '24.04']
+        include:
+          - ruby: '3.4'
+            coverage: 'true'
+          - ruby: 'jruby-10.0'
+            continue-on-error: true
+    runs-on: ubuntu-${{ matrix.ubuntu-version }}
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@2a7b30092b0caf9c046252510f9273b4875f3db9 # v1.254.0
+        with:
+          ruby-version: ${{matrix.ruby}}
+          bundler-cache: false
+      - name: Start Kafka with Docker Compose
+        run: |
+          ./ext/generate-ssl-certs.sh
+          docker compose -f docker-compose-ssl.yml up -d
+          echo "Waiting for Kafka to be ready..."
+
+          sleep 10
+
+          echo "=== Container status ==="
+          docker compose ps kafka
+
+          for i in {1..30}; do
+            echo "=== Attempt $i/30 ==="
+
+            echo "Testing kafka-topics command..."
+            if docker compose exec -T kafka kafka-topics --bootstrap-server localhost:9092 --list >/dev/null 2>&1; then
+              echo "Kafka topics command succeeded!"
+              break
+            else
+              echo "Kafka topics command failed (exit code: $?)"
+            fi
+
+            echo "Sleeping 2 seconds..."
+            sleep 2
+          done
+      - name: Install dependencies
+        env:
+          RDKAFKA_EXT_PATH: ${{ github.workspace }}/ext
+        continue-on-error: ${{ matrix.continue-on-error || false }}
+        run: |
+          bundle install
+      - name: Build gem with mini_portile
+        continue-on-error: ${{ matrix.continue-on-error || false }}
+        run: |
+          set -e
+          cd ext && bundle exec rake
+          cd ..
+      - name: Run all specs in SSL (compiled flow)
+        env:
+          RDKAFKA_EXT_PATH: ${{ github.workspace }}/ext
+          KAFKA_SSL_ENABLED: "true"
+        continue-on-error: ${{ matrix.continue-on-error || false }}
+        run: |
+          bundle exec rspec
+
+      - name: Run integration specs (compiled flow)
+        continue-on-error: ${{ matrix.continue-on-error || false }}
+        run: |
+          for file in $(ls spec/integrations/*_spec.rb); do
+            echo "Running $file with Ruby ${{ matrix.ruby }}";
+            bundle exec ruby "$file" || exit 1;
+          done
+
+  specs_precompiled:
+    timeout-minutes: 30
+    needs: build_precompiled
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby:
+          - '3.5.0-preview1'
+          - '3.4'
+          - '3.3'
+          - '3.2'
+          - '3.1'
+        ubuntu-version: ['22.04', '24.04']
+
+    runs-on: ubuntu-${{ matrix.ubuntu-version }}
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
+      - name: Download precompiled library
+        uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
+        with:
+          name: librdkafka-precompiled-ubuntu-complementary
+          path: ext/
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@2a7b30092b0caf9c046252510f9273b4875f3db9 # v1.254.0
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: false
+      - name: Start Kafka with Docker Compose
+        run: |
+          ./ext/generate-ssl-certs.sh
+          docker compose -f docker-compose-ssl.yml up -d
+          echo "Waiting for Kafka to be ready..."
+
+          sleep 10
+
+          echo "=== Container status ==="
+          docker compose ps kafka
+          for i in {1..30}; do
+            echo "=== Attempt $i/30 ==="
+
+            echo "Testing kafka-topics command..."
+            if docker compose exec -T kafka kafka-topics --bootstrap-server localhost:9092 --list >/dev/null 2>&1; then
+              echo "Kafka topics command succeeded!"
+              break
+            else
+              echo "Kafka topics command failed (exit code: $?)"
+            fi
+
+            echo "Sleeping 2 seconds..."
+            sleep 2
+          done
+      - name: Install bundle with precompiled library
+        env:
+          RDKAFKA_EXT_PATH: ${{ github.workspace }}/ext
+        run: |
+          bundle install
+          echo "Bundle install completed with precompiled library"
+      - name: Remove build dependencies to test static linking
+        continue-on-error: true
+        run: |
+          echo "Removing build dependencies to verify precompiled library is truly self-contained..."
+
+          # Remove packages one by one to avoid dependency conflicts
+          packages_to_remove="build-essential gcc g++ make patch tar wget libsasl2-dev libssl-dev zlib1g-dev libzstd-dev"
+
+          for package in $packages_to_remove; do
+            if dpkg -l | grep -q "^ii.*$package "; then
+              echo "Removing $package..."
+              sudo dpkg --remove --force-depends $package 2>/dev/null || echo "Could not remove $package"
+            else
+              echo "$package is not installed"
+            fi
+          done
+
+          echo "Build dependencies removal completed"
+          echo "Remaining build tools:"
+          which gcc g++ make 2>/dev/null || echo "No build tools found in PATH (good!)"
+      - name: Run specs with precompiled library and SSL
+        env:
+          RDKAFKA_EXT_PATH: ${{ github.workspace }}/ext
+          KAFKA_SSL_ENABLED: "true"
+        run: |
+          bundle exec rspec
+
+      - name: Run integration specs (precompiled flow)
+        env:
+          RDKAFKA_EXT_PATH: ${{ github.workspace }}/ext
+        run: |
+          for file in $(ls spec/integrations/*_spec.rb); do
+            echo "Running $file with Ruby ${{ matrix.ruby }} (precompiled)";
+            bundle exec ruby "$file" || exit 1;
+          done

data/.github/workflows/ci_macos_arm64.yml

@@ -23,7 +23,7 @@ env:
 
 jobs:
   specs_install:
-    timeout-minutes: 30
+    timeout-minutes: 45
     strategy:
       fail-fast: false
       matrix:
@@ -54,7 +54,7 @@ jobs:
           brew list krb5 &>/dev/null || brew install krb5
           echo "/opt/homebrew/bin" >> $GITHUB_PATH
       - name: Set up Ruby
-        uses: ruby/setup-ruby@a4effe49ee8ee5b8b5091268c473a4628afb5651 # v1.245.0
+        uses: ruby/setup-ruby@2a7b30092b0caf9c046252510f9273b4875f3db9 # v1.254.0
         with:
           ruby-version: ${{matrix.ruby}}
           bundler-cache: false
@@ -150,7 +150,7 @@ jobs:
           brew list krb5 &>/dev/null || brew install krb5
           echo "/opt/homebrew/bin" >> $GITHUB_PATH
       - name: Cache build-tmp directory
-        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
         with:
           path: ext/build-tmp-macos
           key: build-tmp-${{ runner.os }}-${{ hashFiles('ext/*.sh', 'ext/Rakefile') }}-v2
@@ -193,12 +193,12 @@ jobs:
         with:
           fetch-depth: 0
       - name: Download precompiled library
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
         with:
           name: librdkafka-precompiled-macos
           path: ext/
       - name: Set up Ruby
-        uses: ruby/setup-ruby@a4effe49ee8ee5b8b5091268c473a4628afb5651 # v1.245.0
+        uses: ruby/setup-ruby@2a7b30092b0caf9c046252510f9273b4875f3db9 # v1.254.0
         with:
           ruby-version: ${{ matrix.ruby }}
           bundler-cache: false

data/.github/workflows/push_linux_aarch64_gnu.yml

@@ -0,0 +1,65 @@
+name: Push Linux aarch64 GNU Platform Gem
+
+on:
+  push:
+    tags:
+      - v*
+
+permissions:
+  contents: read
+
+env:
+  BUNDLE_RETRY: 6
+  BUNDLE_JOBS: 4
+
+jobs:
+  push:
+    if: github.repository_owner == 'karafka'
+    timeout-minutes: 30
+    # Same as CI, we build on the oldest possible for ABI compatibility
+    runs-on: ubuntu-22.04-arm # renovate: ignore
+    environment: deployment
+    permissions:
+      contents: write
+      id-token: write
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
+      - name: Install build dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            build-essential \
+            gcc \
+            make \
+            patch \
+            tar \
+            wget \
+            ca-certificates \
+            libsasl2-dev \
+            libssl-dev \
+            zlib1g-dev \
+            libzstd-dev
+      - name: Cache build-tmp directory
+        uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
+        with:
+          path: ext/build-tmp
+          key: build-tmp-${{ runner.os }}-${{ hashFiles('ext/*.sh') }}
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@2a7b30092b0caf9c046252510f9273b4875f3db9 # v1.254.0
+        with:
+          ruby-version: '3.4'
+          bundler-cache: false
+      - name: Build precompiled librdkafka.so
+        run: |
+          cd ext
+          ./build_linux_aarch64_gnu.sh
+      - name: Configure trusted publishing credentials
+        uses: rubygems/configure-rubygems-credentials@bc6dd217f8a4f919d6835fcfefd470ef821f5c44 # v1.0.0
+      - name: Build and push platform-specific gem
+        run: |
+          gem build *.gemspec
+          gem push *.gem
+        env:
+          RUBY_PLATFORM: 'aarch64-linux-gnu'

data/.github/workflows/push_linux_x86_64_gnu.yml

@@ -42,12 +42,12 @@ jobs:
             zlib1g-dev \
             libzstd-dev
       - name: Cache build-tmp directory
-        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
         with:
           path: ext/build-tmp
           key: build-tmp-${{ runner.os }}-${{ hashFiles('ext/*.sh') }}
       - name: Set up Ruby
-        uses: ruby/setup-ruby@a4effe49ee8ee5b8b5091268c473a4628afb5651 # v1.245.0
+        uses: ruby/setup-ruby@2a7b30092b0caf9c046252510f9273b4875f3db9 # v1.254.0
         with:
           ruby-version: '3.4'
           bundler-cache: false

data/.github/workflows/push_linux_x86_64_musl.yml

@@ -31,7 +31,7 @@ jobs:
       - name: Configure git safe directory
         run: git config --global --add safe.directory /__w/karafka-rdkafka/karafka-rdkafka
       - name: Cache build-tmp directory
-        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
         with:
           path: ext/build-tmp-musl
           key: build-tmp-musl-${{ runner.os }}-${{ hashFiles('ext/*.sh', 'ext/Rakefile') }}-v2
@@ -60,12 +60,12 @@ jobs:
         with:
           fetch-depth: 0
       - name: Download precompiled library
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
         with:
           name: librdkafka-precompiled-musl
           path: ext/
       - name: Set up Ruby
-        uses: ruby/setup-ruby@a4effe49ee8ee5b8b5091268c473a4628afb5651 # v1.245.0
+        uses: ruby/setup-ruby@2a7b30092b0caf9c046252510f9273b4875f3db9 # v1.254.0
         with:
           ruby-version: '3.4'
           bundler-cache: false

data/.github/workflows/push_macos_arm64.yml

@@ -31,12 +31,12 @@ jobs:
           brew list krb5 &>/dev/null || brew install krb5
           echo "/opt/homebrew/bin" >> $GITHUB_PATH
       - name: Cache build-tmp directory
-        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
         with:
           path: ext/build-tmp-macos
           key: build-tmp-${{ runner.os }}-${{ hashFiles('ext/*.sh', 'ext/Rakefile') }}-v2
       - name: Set up Ruby
-        uses: ruby/setup-ruby@a4effe49ee8ee5b8b5091268c473a4628afb5651 # v1.245.0
+        uses: ruby/setup-ruby@2a7b30092b0caf9c046252510f9273b4875f3db9 # v1.254.0
         with:
           ruby-version: '3.4'
           bundler-cache: false

data/.github/workflows/push_ruby.yml

@@ -24,7 +24,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Ruby
-        uses: ruby/setup-ruby@472790540115ce5bd69d399a020189a8c87d641f # v1.247.0
+        uses: ruby/setup-ruby@2a7b30092b0caf9c046252510f9273b4875f3db9 # v1.254.0
         with:
           bundler-cache: false
 

data/.github/workflows/trigger-wiki-refresh.yml

@@ -0,0 +1,30 @@
+name: Trigger Wiki Refresh
+
+on:
+  release:
+    types: [published]
+  push:
+    branches: [main]
+
+jobs:
+  trigger-wiki-refresh:
+    runs-on: ubuntu-latest
+    environment: wiki-trigger
+    if: github.repository_owner == 'karafka'
+    steps:
+      - name: Trigger wiki refresh
+        uses: peter-evans/repository-dispatch@ff45666b9427631e3450c54a1bcbee4d9ff4d7c0 # v3.0.0
+        with:
+          token: ${{ secrets.WIKI_REPO_TOKEN }}
+          repository: karafka/wiki
+          event-type: sync-trigger
+          client-payload: |
+            {
+              "repository": "${{ github.repository }}",
+              "event_name": "${{ github.event_name }}",
+              "release_tag": "${{ github.event.release.tag_name || '' }}",
+              "release_name": "${{ github.event.release.name || '' }}",
+              "commit_sha": "${{ github.sha }}",
+              "commit_message": "Trigger Wiki Refresh",
+              "triggered_by": "${{ github.actor }}"
+            }

data/.github/workflows/verify-action-pins.yml

@@ -4,7 +4,7 @@ on:
     paths:
       - '.github/workflows/**'
 jobs:
-  verify:
+  verify_action_pins:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

data/.gitignore

@@ -13,3 +13,4 @@ coverage
 vendor
 .idea/
 out/
+ssl/

data/.rspec

@@ -1,2 +1,3 @@
 --require spec_helper
 --format documentation
+--pattern spec/lib/**/*_spec.rb

data/CHANGELOG.md

@@ -1,12 +1,18 @@
 # Rdkafka Changelog
 
 ## 0.21.0 (Unreleased)
-- [Enhancement] Bump librdkafka to 2.11.0
+- [Enhancement] Support explicit Debian testing due to lib issues.
+- [Enhancement] Support ARM64 Gnu precompilation.
+- [Enhancement] Bump librdkafka to 2.11.0.
+- [Enhancement] Improve what symbols are exposed outside of the precompiled extensions.
+- [Enhancement] Introduce an integration suite layer for non RSpec specs execution.
+- [Fix] Add `json` gem as a dependency (was missing but used).
 
 ## 0.20.1 (2025-07-17)
 - [Enhancement] Drastically increase number of platforms in the integration suite
 - [Fix] Support Ubuntu `22.04` and older Alpine precompiled versions
 - [Fix] FFI::DynamicLibrary.load_library': Could not open library
+- [Change] Add new CI action to trigger auto-doc refresh.
 
 ## 0.20.0 (2025-07-17)
 - **[Feature]** Add precompiled `x86_64-linux-gnu` setup.