karafka-rdkafka 0.20.0.rc5-arm64-darwin → 0.21.0-arm64-darwin

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c9b0ba518573eb693cdaee10342aaf781e87725cd7805baaffa922dbbfff120f
-  data.tar.gz: c82f0b574b88c7fcf9a2fd013c6256a715368bdbf30918a8e0cb7ea9ea5d6451
+  metadata.gz: daf3fb477ad6b404d4c4a5f01b6496f33d31ceccfcc47f8307fd6ebd5c7451ed
+  data.tar.gz: 6735011385e395dbebdd6f41d6df9bd7d889f5c5f4db8d6d73b4d072880c8c9f
 SHA512:
-  metadata.gz: 7d8c87b873d9b9783daa73d91d1962d8d906d4aa2826d646793cf9e729efe0e321a7539d0564333975c29752f93831f8b922f6af69c383bbdb2add68648d39fd
-  data.tar.gz: d4f27fb07b5bed6245a11fa8baa4d93ab021d28bdf6ed8a08f0572954af57bb7bbb1330a806a94d6a45be89c49dc73f7026ac8ee32e6dda12826a52106ee5b36
+  metadata.gz: e1de6d06aee4f4c2006d0aa342e5da5624b8073d708022bbb7ce46bb24cb0dc8a32480865585e0e57ca4397928e782744a2be148ce41cb2a7933fd76e1979dd2
+  data.tar.gz: f8ffc91157f499d77c80e89664ec6a4fecb1b149affc0b321c0736ba67ea63a54860af0a14be175a1aeb0363db16984f2043fc701786acc5b4d451a7eebf609d

data/.github/workflows/{ci_linux_x86_64_musl.yml → ci_linux_alpine_x86_64_musl.yml}

@@ -1,4 +1,27 @@
-name: CI Linux x86_64 musl
+# Why We Build and Run Without Caching Native Extensions
+#
+# We intentionally compile the native librdkafka library fresh in each test job
+# rather than caching or pre-building it for several reasons:
+#
+# 1. Architecture Compatibility
+#    - Pre-built native libraries (.so files) are architecture-specific
+#    - Can cause "Exec format error" when build/runtime environments differ
+#    - Building in the same container guarantees compatibility
+#
+# 2. Container Image Variations
+#    - Different Ruby Alpine images may have subtle differences in:
+#      * Base system libraries, compiler toolchains, musl libc versions
+#    - These differences can cause pre-built libraries to fail at runtime
+#
+# 3. Simplicity and Reliability
+#    - Single source of truth: everything builds and runs in same environment
+#    - No artifact management complexity or potential upload/download failures
+#    - Easier debugging when issues are contained in one job
+#
+# Trade-offs: Slightly longer CI times (~2-3 min per job) but much more reliable
+# than dealing with architecture mismatches and artifact corruption issues.
+
+name: CI Linux Alpine x86_64 musl
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -20,70 +43,39 @@ env:
   BUNDLE_JOBS: 4
 
 jobs:
-  build_install:
-    timeout-minutes: 30
-    runs-on: ubuntu-latest
-    container:
-      image: alpine:3.22@sha256:8a1f59ffb675680d47db6337b49d22281a139e9d709335b492be023728e11715
-    steps:
-      - name: Install dependencies
-        run: |
-          apk add --no-cache git curl ca-certificates build-base linux-headers \
-            pkgconf perl autoconf automake libtool bison flex file \
-            ruby ruby-dev ruby-bundler bash zstd-dev zlib zlib-dev openssl-dev cyrus-sasl-dev
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-        with:
-          fetch-depth: 0
-      - name: Configure git safe directory
-        run: git config --global --add safe.directory /__w/karafka-rdkafka/karafka-rdkafka
-      - name: Build gem with mini_portile
-        run: |
-          set -e
-          bundle config set --local path 'vendor/bundle'
-          bundle install
-          cd ext && bundle exec rake
-          cd ..
-      - name: Upload built gem and bundle
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
-        with:
-          name: rdkafka-built-gem-musl
-          path: |
-            vendor/bundle/
-            .bundle/
-            ext/
-            lib/
-          retention-days: 1
-
   specs_install:
-    timeout-minutes: 30
+    timeout-minutes: 45
     runs-on: ubuntu-latest
-    needs: build_install
     strategy:
       fail-fast: false
       matrix:
-        ruby:
-          - '3.4'
-          - '3.3'
-          - '3.2'
-          - '3.1'
         include:
+          - ruby: '3.1'
+            alpine_version: '3.21'
+          - ruby: '3.2'
+            alpine_version: '3.21'
+          - ruby: '3.2'
+            alpine_version: '3.22'
+          - ruby: '3.3'
+            alpine_version: '3.21'
+          - ruby: '3.3'
+            alpine_version: '3.22'
+          - ruby: '3.4'
+            alpine_version: '3.21'
+            coverage: 'true'
           - ruby: '3.4'
+            alpine_version: '3.22'
             coverage: 'true'
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           fetch-depth: 0
-      - name: Download built gem
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
-        with:
-          name: rdkafka-built-gem-musl
-          path: ./
+
       - name: Start Kafka with Docker Compose
         run: |
-          docker compose up -d
+          docker compose -f docker-compose.yml up -d
           echo "Waiting for Kafka to be ready..."
           sleep 10
-
           for i in {1..30}; do
             if docker compose exec -T kafka kafka-topics --bootstrap-server localhost:9092 --list >/dev/null 2>&1; then
               echo "Kafka topics command succeeded!"
@@ -91,39 +83,36 @@ jobs:
             fi
             sleep 2
           done
-      - name: Run all specs
+
+      - name: Run all specs in PLAINTEXT
         env:
           GITHUB_COVERAGE: ${{ matrix.coverage }}
-          RDKAFKA_EXT_PATH: ${{ github.workspace }}/ext
         run: |
           docker run --rm \
             --network host \
             -v "${{ github.workspace }}:/workspace" \
             -w /workspace \
             -e "GITHUB_COVERAGE=${{ matrix.coverage }}" \
-            -e "RDKAFKA_EXT_PATH=/workspace/ext" \
-            ruby:${{ matrix.ruby }}-alpine \
-            sh -c 'apk add --no-cache git build-base linux-headers bash \
-              cyrus-sasl \
-              cyrus-sasl-login \
-              cyrus-sasl-crammd5 \
-              cyrus-sasl-digestmd5 \
-              cyrus-sasl-gssapiv2 \
-              cyrus-sasl-scram \
-              krb5-libs \
-              openssl \
-              zlib \
-              zlib-dev \
-              zstd-libs && \
+            ruby:${{ matrix.ruby }}-alpine${{ matrix.alpine_version }} \
+            sh -c 'apk add --no-cache git curl ca-certificates build-base linux-headers \
+              pkgconf perl autoconf automake libtool bison flex file \
+              ruby-dev ruby-bundler bash zstd-dev zlib zlib-dev openssl-dev \
+              cyrus-sasl-dev cyrus-sasl cyrus-sasl-login \
+              cyrus-sasl-crammd5 cyrus-sasl-digestmd5 cyrus-sasl-gssapiv2 cyrus-sasl-scram \
+              krb5-libs openssl zlib zstd-libs && \
               git config --global --add safe.directory /workspace && \
               bundle config set --local path vendor/bundle && \
               bundle install && \
+              cd ext && bundle exec rake && \
+              cd .. && \
               bundle exec ruby -S rspec'
+
   build_precompiled:
     timeout-minutes: 45
     runs-on: ubuntu-latest
     container:
-      image: alpine:3.22@sha256:8a1f59ffb675680d47db6337b49d22281a139e9d709335b492be023728e11715
+      # Similar to GNU, we build on the oldest for ABI compatibility
+      image: alpine:3.18@sha256:de0eb0b3f2a47ba1eb89389859a9bd88b28e82f5826b6969ad604979713c2d4f # renovate: ignore
     steps:
       - name: Install dependencies
         run: |
@@ -138,7 +127,7 @@ jobs:
       - name: Configure git safe directory
         run: git config --global --add safe.directory /__w/karafka-rdkafka/karafka-rdkafka
       - name: Cache build-tmp directory
-        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
         with:
           path: ext/build-tmp-musl
           key: build-tmp-musl-${{ runner.os }}-${{ hashFiles('ext/*.sh', 'ext/Rakefile') }}-v2
@@ -173,13 +162,13 @@ jobs:
         with:
           fetch-depth: 0
       - name: Download precompiled library
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
         with:
           name: librdkafka-precompiled-musl
           path: ext/
       - name: Start Kafka with Docker Compose
         run: |
-          docker compose up -d
+          docker compose -f docker-compose.yml up -d
           echo "Waiting for Kafka to be ready..."
           sleep 10
 
@@ -190,7 +179,7 @@ jobs:
             fi
             sleep 2
           done
-      - name: Run specs with precompiled library
+      - name: Run specs with precompiled library and PLAINTEXT
         env:
           GITHUB_COVERAGE: ${{ matrix.coverage }}
           RDKAFKA_EXT_PATH: ${{ github.workspace }}/ext
@@ -202,4 +191,7 @@ jobs:
             -e "GITHUB_COVERAGE=${{ matrix.coverage }}" \
             -e "RDKAFKA_EXT_PATH=/workspace/ext" \
             ruby:${{ matrix.ruby }}-alpine \
-            sh -c 'apk add --no-cache git build-base linux-headers bash && git config --global --add safe.directory /workspace && bundle config set --local path vendor/bundle && bundle install && bundle exec ruby -S rspec'
+            sh -c 'apk add --no-cache git build-base linux-headers bash && \
+              git config --global --add safe.directory /workspace && \
+              bundle config set --local path vendor/bundle && \
+              bundle install && bundle exec ruby -S rspec'

data/.github/workflows/ci_linux_alpine_x86_64_musl_complementary.yml

@@ -0,0 +1,264 @@
+# CI Strategy: Complementary Testing for SSL and System Library Regressions on Alpine/musl
+#
+# This workflow runs complementary tests that don't need to block PRs but are essential
+# for catching regressions in SSL functionality and musl libc/Alpine system library compatibility.
+# It complements the main CI by testing stable-but-critical functionality on a nightly
+# schedule and when workflow changes are made.
+#
+# WHY COMPLEMENTARY TESTING FOR ALPINE/MUSL:
+# - SSL specs have been stable for 3+ years and rarely break due to code changes
+# - Integration specs test musl libc and Alpine package compatibility
+# - These tests catch regressions from external changes (OpenSSL updates, Alpine package updates)
+# - Running every 3 days to prevent these slower tests from blocking PR velocity
+# - Manual triggering allows testing workflow changes before they go into schedule
+#
+# SSL TESTING (specs_install + specs_precompiled):
+# - Tests SSL/TLS connectivity with Kafka using docker-compose-ssl.yml on Alpine
+# - Validates certificate handling and SSL handshakes across Ruby versions on musl
+# - Ensures SSL works with both compiled-from-source and precompiled flows on Alpine
+# - Catches OpenSSL version compatibility issues and SSL library regressions on musl
+# - Tests real SSL scenarios that mirror Alpine-based production deployments
+#
+# INTEGRATION TESTING (integration specs in both jobs):
+# - Tests musl libc and Alpine system library compatibility without requiring Kafka infrastructure
+# - Validates libssl, libsasl2, libzstd, zlib integration across Alpine versions
+# - Ensures native extensions work with different Alpine package versions
+# - Catches regressions from Alpine package updates and musl libc changes
+# - Tests both compilation and precompiled library compatibility on Alpine
+#
+# SCHEDULING STRATEGY:
+# - Runs every 3 days at 5 AM to catch system/library changes from Alpine base image updates
+# - Triggers on workflow file changes to validate CI modifications
+# - Manual dispatch available for ad-hoc regression testing
+# - Separate artifact naming prevents interference with main CI
+#
+# This approach ensures comprehensive coverage while keeping PR CI fast and focused
+# on code-related issues rather than infrastructure/system regressions on Alpine.
+
+name: CI Linux Complementary Alpine x86_64 musl
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  schedule:
+    - cron: '0 5 */3 * *'
+  workflow_dispatch:
+  push:
+    paths:
+      - '.github/workflows/ci_linux_alpine_x86_64_musl_complementary.yml'
+    branches: [ main, master ]
+  pull_request:
+    paths:
+      - '.github/workflows/ci_linux_alpine_x86_64_musl_complementary.yml'
+    branches: [ main, master ]
+
+permissions:
+  contents: read
+
+env:
+  BUNDLE_RETRY: 6
+  BUNDLE_JOBS: 4
+
+jobs:
+  build_precompiled:
+    timeout-minutes: 45
+    runs-on: ubuntu-latest
+    container:
+      # Similar to GNU, we build on the oldest for ABI compatibility
+      image: alpine:3.18@sha256:de0eb0b3f2a47ba1eb89389859a9bd88b28e82f5826b6969ad604979713c2d4f # renovate: ignore
+    steps:
+      - name: Install dependencies
+        run: |
+          apk add --no-cache git curl ca-certificates build-base linux-headers \
+            pkgconf perl autoconf automake libtool bison flex file bash wget zstd-dev \
+            openssl-dev cyrus-sasl-dev cyrus-sasl cyrus-sasl-login \
+            cyrus-sasl-crammd5 cyrus-sasl-digestmd5 cyrus-sasl-gssapiv2 cyrus-sasl-scram \
+            krb5-libs openssl zlib zlib-dev zstd-libs
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
+      - name: Configure git safe directory
+        run: git config --global --add safe.directory /__w/karafka-rdkafka/karafka-rdkafka
+      - name: Cache build-tmp directory
+        uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
+        with:
+          path: ext/build-tmp-musl
+          key: build-tmp-musl-${{ runner.os }}-${{ hashFiles('ext/*.sh', 'ext/Rakefile') }}-v2
+      - name: Build precompiled librdkafka.so
+        run: |
+          cd ext
+          bash ./build_linux_x86_64_musl.sh
+      - name: Upload precompiled library
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: librdkafka-precompiled-musl-complementary
+          path: ext/
+          retention-days: 1
+
+  specs_install:
+    timeout-minutes: 45
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - ruby: '3.4'
+            alpine_version: '3.22'
+          - ruby: '3.3'
+            alpine_version: '3.21'
+          - ruby: '3.2'
+            alpine_version: '3.21'
+          - ruby: '3.1'
+            alpine_version: '3.21'
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
+
+      - name: Start Kafka with Docker Compose
+        run: |
+          ./ext/generate-ssl-certs.sh
+          docker compose -f docker-compose-ssl.yml up -d
+          echo "Waiting for Kafka to be ready..."
+          sleep 10
+          for i in {1..30}; do
+            if docker compose exec -T kafka kafka-topics --bootstrap-server localhost:9092 --list >/dev/null 2>&1; then
+              echo "Kafka topics command succeeded!"
+              break
+            fi
+            sleep 2
+          done
+
+      - name: Run all specs in SSL (compiled flow)
+        env:
+          KAFKA_SSL_ENABLED: "true"
+        run: |
+          docker run --rm \
+            --network host \
+            -v "${{ github.workspace }}:/workspace" \
+            -w /workspace \
+            -e "KAFKA_SSL_ENABLED=true" \
+            ruby:${{ matrix.ruby }}-alpine${{ matrix.alpine_version }} \
+            sh -c 'apk add --no-cache git curl ca-certificates build-base linux-headers \
+              pkgconf perl autoconf automake libtool bison flex file \
+              ruby-dev ruby-bundler bash zstd-dev zlib zlib-dev openssl-dev \
+              cyrus-sasl-dev cyrus-sasl cyrus-sasl-login \
+              cyrus-sasl-crammd5 cyrus-sasl-digestmd5 cyrus-sasl-gssapiv2 cyrus-sasl-scram \
+              krb5-libs openssl zlib zstd-libs openjdk17-jre-headless && \
+              git config --global --add safe.directory /workspace && \
+              bundle config set --local path vendor/bundle && \
+              bundle install && \
+              cd ext && bundle exec rake && \
+              cd .. && \
+              echo "=== SSL Library Versions ===" && \
+              openssl version && \
+              apk list --installed | grep -E "(openssl|cyrus-sasl)" && \
+              echo "=== Running SSL Specs (Compiled) ===" && \
+              bundle exec ruby -S rspec'
+
+      - name: Run integration specs (compiled flow)
+        run: |
+          docker run --rm \
+            -v "${{ github.workspace }}:/workspace" \
+            -w /workspace \
+            ruby:${{ matrix.ruby }}-alpine${{ matrix.alpine_version }} \
+            sh -c 'apk add --no-cache git curl ca-certificates build-base linux-headers \
+              pkgconf perl autoconf automake libtool bison flex file \
+              ruby-dev ruby-bundler bash zstd-dev zlib zlib-dev openssl-dev \
+              cyrus-sasl-dev cyrus-sasl cyrus-sasl-login \
+              cyrus-sasl-crammd5 cyrus-sasl-digestmd5 cyrus-sasl-gssapiv2 cyrus-sasl-scram \
+              krb5-libs openssl zlib zstd-libs libcurl curl-dev && \
+              git config --global --add safe.directory /workspace && \
+              bundle config set --local path vendor/bundle && \
+              bundle install && \
+              cd ext && bundle exec rake && \
+              cd .. && \
+              echo "=== Alpine/musl Library Versions ===" && \
+              openssl version && \
+              apk list --installed | grep -E "(openssl|cyrus-sasl|zstd|zlib)" && \
+              echo "=== Running Integration Specs (Compiled) ===" && \
+              for file in $(ls spec/integrations/*_spec.rb); do \
+                echo "Running $file with Ruby ${{ matrix.ruby }} on Alpine ${{ matrix.alpine_version }}"; \
+                bundle exec ruby "$file" || exit 1; \
+              done'
+
+  specs_precompiled:
+    timeout-minutes: 30
+    needs: build_precompiled
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby:
+          - '3.4'
+          - '3.3'
+          - '3.2'
+          - '3.1'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
+      - name: Download precompiled library
+        uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
+        with:
+          name: librdkafka-precompiled-musl-complementary
+          path: ext/
+      - name: Start Kafka with Docker Compose
+        run: |
+          ./ext/generate-ssl-certs.sh
+          docker compose -f docker-compose-ssl.yml up -d
+          echo "Waiting for Kafka to be ready..."
+          sleep 10
+
+          for i in {1..30}; do
+            if docker compose exec -T kafka kafka-topics --bootstrap-server localhost:9092 --list >/dev/null 2>&1; then
+              echo "Kafka topics command succeeded!"
+              break
+            fi
+            sleep 2
+          done
+
+      - name: Run specs with precompiled library and SSL
+        env:
+          RDKAFKA_EXT_PATH: ${{ github.workspace }}/ext
+          KAFKA_SSL_ENABLED: "true"
+        run: |
+          docker run --rm \
+            --network host \
+            -v "${{ github.workspace }}:/workspace" \
+            -w /workspace \
+            -e "RDKAFKA_EXT_PATH=/workspace/ext" \
+            -e "KAFKA_SSL_ENABLED=true" \
+            ruby:${{ matrix.ruby }}-alpine \
+            sh -c 'apk add --no-cache git build-base linux-headers bash openjdk17-jre-headless && \
+              git config --global --add safe.directory /workspace && \
+              bundle config set --local path vendor/bundle && bundle install && \
+              apk list --installed | grep -E "(openssl|cyrus-sasl)" && \
+              bundle exec ruby -S rspec'
+
+      - name: Run integration specs (precompiled flow)
+        env:
+          RDKAFKA_EXT_PATH: ${{ github.workspace }}/ext
+        run: |
+          docker run --rm \
+            -v "${{ github.workspace }}:/workspace" \
+            -w /workspace \
+            -e "RDKAFKA_EXT_PATH=/workspace/ext" \
+            ruby:${{ matrix.ruby }}-alpine \
+            sh -c 'apk add --no-cache git curl ca-certificates build-base linux-headers \
+              pkgconf perl autoconf automake libtool bison flex file \
+              ruby-dev ruby-bundler bash zstd-dev zlib zlib-dev openssl-dev \
+              cyrus-sasl-dev cyrus-sasl cyrus-sasl-login \
+              cyrus-sasl-crammd5 cyrus-sasl-digestmd5 cyrus-sasl-gssapiv2 cyrus-sasl-scram \
+              krb5-libs openssl zlib zstd-libs libcurl curl-dev && \
+              git config --global --add safe.directory /workspace && \
+              bundle config set --local path vendor/bundle && \
+              bundle install && \
+              apk list --installed | grep -E "(openssl|cyrus-sasl|zstd|zlib)" && \
+              echo "=== Running Integration Specs (Precompiled) ===" && \
+              for file in $(ls spec/integrations/*_spec.rb); do \
+                echo "Running $file with Ruby ${{ matrix.ruby }} (precompiled)"; \
+                bundle exec ruby "$file" || exit 1; \
+              done'