karafka-rdkafka 0.20.0.rc3-x86_64-linux-gnu → 0.20.0.rc5-x86_64-linux-gnu

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 8d2a5aa11f12382418392f01e4ec107949661fec05506f0201346f126525c434
4
- data.tar.gz: b55416183eeed99b3440c9d8f65d13748fce7cb3d315984e5b6fccb0e2b20b0a
3
+ metadata.gz: 89b5b7ab7269112cd5816d40293395df9208e6929eca11140a3bfb17f8a48cc2
4
+ data.tar.gz: 6fea31eaa2c84f6c5d13dd4df7aae1808283a5c82f9fe83a73e46d675625c49c
5
5
  SHA512:
6
- metadata.gz: e91fe7846b57967aeef2bce1cd3bf72e3ee206cfdb4ed50099684cbef1b1a347420cdda5d3002429d6ab0e94b78cc8a28864a6a6e1886f9223b60145706fbac0
7
- data.tar.gz: d22c1226b32a671145323e52a9e22edb94aaa4e9188791912958d104f93e27a15e6b16883ba5f36e3beccaf9132e5155a67db5c2728381ec59b72220ba171ce1
6
+ metadata.gz: 4e1d26e6e3c76f82c9dabc5de8b98bcad8df7b59c4052524ea58325a554b1a8ca83d1b4e3ba2baeb2f35d0fd114287866a1fa71cbcf75d7ca6345cfae8227504
7
+ data.tar.gz: 352839be9a5499a6eda10f98f10c8ef5c147c1cf0fae7646a1e925abd03ad1dbb38b7e21306defdc52b2bfca24d416fb02f1b8f319d253ad6610f67e17cc9ee9
@@ -1,4 +1,5 @@
1
1
  name: CI Linux x86_64 GNU
2
+
2
3
  concurrency:
3
4
  group: ${{ github.workflow }}-${{ github.ref }}
4
5
  cancel-in-progress: true
@@ -0,0 +1,205 @@
1
+ name: CI Linux x86_64 musl
2
+
3
+ concurrency:
4
+ group: ${{ github.workflow }}-${{ github.ref }}
5
+ cancel-in-progress: true
6
+
7
+ on:
8
+ pull_request:
9
+ branches: [ main, master ]
10
+ push:
11
+ branches: [ main, master ]
12
+ schedule:
13
+ - cron: '0 1 * * *'
14
+
15
+ permissions:
16
+ contents: read
17
+
18
+ env:
19
+ BUNDLE_RETRY: 6
20
+ BUNDLE_JOBS: 4
21
+
22
+ jobs:
23
+ build_install:
24
+ timeout-minutes: 30
25
+ runs-on: ubuntu-latest
26
+ container:
27
+ image: alpine:3.22@sha256:8a1f59ffb675680d47db6337b49d22281a139e9d709335b492be023728e11715
28
+ steps:
29
+ - name: Install dependencies
30
+ run: |
31
+ apk add --no-cache git curl ca-certificates build-base linux-headers \
32
+ pkgconf perl autoconf automake libtool bison flex file \
33
+ ruby ruby-dev ruby-bundler bash zstd-dev zlib zlib-dev openssl-dev cyrus-sasl-dev
34
+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
35
+ with:
36
+ fetch-depth: 0
37
+ - name: Configure git safe directory
38
+ run: git config --global --add safe.directory /__w/karafka-rdkafka/karafka-rdkafka
39
+ - name: Build gem with mini_portile
40
+ run: |
41
+ set -e
42
+ bundle config set --local path 'vendor/bundle'
43
+ bundle install
44
+ cd ext && bundle exec rake
45
+ cd ..
46
+ - name: Upload built gem and bundle
47
+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
48
+ with:
49
+ name: rdkafka-built-gem-musl
50
+ path: |
51
+ vendor/bundle/
52
+ .bundle/
53
+ ext/
54
+ lib/
55
+ retention-days: 1
56
+
57
+ specs_install:
58
+ timeout-minutes: 30
59
+ runs-on: ubuntu-latest
60
+ needs: build_install
61
+ strategy:
62
+ fail-fast: false
63
+ matrix:
64
+ ruby:
65
+ - '3.4'
66
+ - '3.3'
67
+ - '3.2'
68
+ - '3.1'
69
+ include:
70
+ - ruby: '3.4'
71
+ coverage: 'true'
72
+ steps:
73
+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
74
+ with:
75
+ fetch-depth: 0
76
+ - name: Download built gem
77
+ uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
78
+ with:
79
+ name: rdkafka-built-gem-musl
80
+ path: ./
81
+ - name: Start Kafka with Docker Compose
82
+ run: |
83
+ docker compose up -d
84
+ echo "Waiting for Kafka to be ready..."
85
+ sleep 10
86
+
87
+ for i in {1..30}; do
88
+ if docker compose exec -T kafka kafka-topics --bootstrap-server localhost:9092 --list >/dev/null 2>&1; then
89
+ echo "Kafka topics command succeeded!"
90
+ break
91
+ fi
92
+ sleep 2
93
+ done
94
+ - name: Run all specs
95
+ env:
96
+ GITHUB_COVERAGE: ${{ matrix.coverage }}
97
+ RDKAFKA_EXT_PATH: ${{ github.workspace }}/ext
98
+ run: |
99
+ docker run --rm \
100
+ --network host \
101
+ -v "${{ github.workspace }}:/workspace" \
102
+ -w /workspace \
103
+ -e "GITHUB_COVERAGE=${{ matrix.coverage }}" \
104
+ -e "RDKAFKA_EXT_PATH=/workspace/ext" \
105
+ ruby:${{ matrix.ruby }}-alpine \
106
+ sh -c 'apk add --no-cache git build-base linux-headers bash \
107
+ cyrus-sasl \
108
+ cyrus-sasl-login \
109
+ cyrus-sasl-crammd5 \
110
+ cyrus-sasl-digestmd5 \
111
+ cyrus-sasl-gssapiv2 \
112
+ cyrus-sasl-scram \
113
+ krb5-libs \
114
+ openssl \
115
+ zlib \
116
+ zlib-dev \
117
+ zstd-libs && \
118
+ git config --global --add safe.directory /workspace && \
119
+ bundle config set --local path vendor/bundle && \
120
+ bundle install && \
121
+ bundle exec ruby -S rspec'
122
+ build_precompiled:
123
+ timeout-minutes: 45
124
+ runs-on: ubuntu-latest
125
+ container:
126
+ image: alpine:3.22@sha256:8a1f59ffb675680d47db6337b49d22281a139e9d709335b492be023728e11715
127
+ steps:
128
+ - name: Install dependencies
129
+ run: |
130
+ apk add --no-cache git curl ca-certificates build-base linux-headers \
131
+ pkgconf perl autoconf automake libtool bison flex file bash wget zstd-dev \
132
+ openssl-dev cyrus-sasl-dev cyrus-sasl cyrus-sasl-login \
133
+ cyrus-sasl-crammd5 cyrus-sasl-digestmd5 cyrus-sasl-gssapiv2 cyrus-sasl-scram \
134
+ krb5-libs openssl zlib zlib-dev zstd-libs
135
+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
136
+ with:
137
+ fetch-depth: 0
138
+ - name: Configure git safe directory
139
+ run: git config --global --add safe.directory /__w/karafka-rdkafka/karafka-rdkafka
140
+ - name: Cache build-tmp directory
141
+ uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
142
+ with:
143
+ path: ext/build-tmp-musl
144
+ key: build-tmp-musl-${{ runner.os }}-${{ hashFiles('ext/*.sh', 'ext/Rakefile') }}-v2
145
+ - name: Build precompiled librdkafka.so
146
+ run: |
147
+ cd ext
148
+ bash ./build_linux_x86_64_musl.sh
149
+ - name: Upload precompiled library
150
+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
151
+ with:
152
+ name: librdkafka-precompiled-musl
153
+ path: ext/
154
+ retention-days: 1
155
+
156
+ specs_precompiled:
157
+ timeout-minutes: 30
158
+ runs-on: ubuntu-latest
159
+ needs: build_precompiled
160
+ strategy:
161
+ fail-fast: false
162
+ matrix:
163
+ ruby:
164
+ - '3.4'
165
+ - '3.3'
166
+ - '3.2'
167
+ - '3.1'
168
+ include:
169
+ - ruby: '3.4'
170
+ coverage: 'true'
171
+ steps:
172
+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
173
+ with:
174
+ fetch-depth: 0
175
+ - name: Download precompiled library
176
+ uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
177
+ with:
178
+ name: librdkafka-precompiled-musl
179
+ path: ext/
180
+ - name: Start Kafka with Docker Compose
181
+ run: |
182
+ docker compose up -d
183
+ echo "Waiting for Kafka to be ready..."
184
+ sleep 10
185
+
186
+ for i in {1..30}; do
187
+ if docker compose exec -T kafka kafka-topics --bootstrap-server localhost:9092 --list >/dev/null 2>&1; then
188
+ echo "Kafka topics command succeeded!"
189
+ break
190
+ fi
191
+ sleep 2
192
+ done
193
+ - name: Run specs with precompiled library
194
+ env:
195
+ GITHUB_COVERAGE: ${{ matrix.coverage }}
196
+ RDKAFKA_EXT_PATH: ${{ github.workspace }}/ext
197
+ run: |
198
+ docker run --rm \
199
+ --network host \
200
+ -v "${{ github.workspace }}:/workspace" \
201
+ -w /workspace \
202
+ -e "GITHUB_COVERAGE=${{ matrix.coverage }}" \
203
+ -e "RDKAFKA_EXT_PATH=/workspace/ext" \
204
+ ruby:${{ matrix.ruby }}-alpine \
205
+ sh -c 'apk add --no-cache git build-base linux-headers bash && git config --global --add safe.directory /workspace && bundle config set --local path vendor/bundle && bundle install && bundle exec ruby -S rspec'
@@ -1,7 +1,9 @@
1
1
  name: CI macOS ARM64
2
+
2
3
  concurrency:
3
4
  group: ${{ github.workflow }}-${{ github.ref }}
4
5
  cancel-in-progress: true
6
+
5
7
  on:
6
8
  pull_request:
7
9
  branches: [ main, master ]
@@ -9,13 +11,16 @@ on:
9
11
  branches: [ main, master ]
10
12
  schedule:
11
13
  - cron: '0 1 * * *'
14
+
12
15
  permissions:
13
16
  contents: read
17
+
14
18
  env:
15
19
  BUNDLE_RETRY: 6
16
20
  BUNDLE_JOBS: 4
17
21
  # Renovate can track and update this version
18
22
  CONFLUENT_VERSION: "8.0.0"
23
+
19
24
  jobs:
20
25
  build_install:
21
26
  timeout-minutes: 30
@@ -1,13 +1,17 @@
1
1
  name: Push Linux x86_64 GNU Platform Gem
2
+
2
3
  on:
3
4
  push:
4
5
  tags:
5
6
  - v*
7
+
6
8
  permissions:
7
9
  contents: read
10
+
8
11
  env:
9
12
  BUNDLE_RETRY: 6
10
13
  BUNDLE_JOBS: 4
14
+
11
15
  jobs:
12
16
  push:
13
17
  if: github.repository_owner == 'karafka'
@@ -51,7 +55,7 @@ jobs:
51
55
  cd ext
52
56
  ./build_linux_x86_64_gnu.sh
53
57
  - name: Configure trusted publishing credentials
54
- uses: rubygems/configure-rubygems-credentials@v1.0.0
58
+ uses: rubygems/configure-rubygems-credentials@bc6dd217f8a4f919d6835fcfefd470ef821f5c44 # v1.0.0
55
59
  - name: Build and push platform-specific gem
56
60
  run: |
57
61
  gem build *.gemspec
@@ -0,0 +1,77 @@
1
+ name: Push Linux x86_64 musl Platform Gem
2
+ on:
3
+ push:
4
+ tags:
5
+ - v*
6
+ permissions:
7
+ contents: read
8
+ env:
9
+ BUNDLE_RETRY: 6
10
+ BUNDLE_JOBS: 4
11
+ jobs:
12
+ build:
13
+ if: github.repository_owner == 'karafka'
14
+ timeout-minutes: 30
15
+ runs-on: ubuntu-latest
16
+ environment: deployment
17
+ container:
18
+ image: alpine:3.22@sha256:8a1f59ffb675680d47db6337b49d22281a139e9d709335b492be023728e11715
19
+ steps:
20
+ - name: Install dependencies
21
+ run: |
22
+ apk add --no-cache git curl ca-certificates build-base linux-headers \
23
+ pkgconf perl autoconf automake libtool bison flex file bash wget zstd-dev \
24
+ openssl-dev cyrus-sasl-dev cyrus-sasl cyrus-sasl-login \
25
+ cyrus-sasl-crammd5 cyrus-sasl-digestmd5 cyrus-sasl-gssapiv2 cyrus-sasl-scram \
26
+ krb5-libs openssl zlib zlib-dev zstd-libs
27
+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
28
+ with:
29
+ fetch-depth: 0
30
+ - name: Configure git safe directory
31
+ run: git config --global --add safe.directory /__w/karafka-rdkafka/karafka-rdkafka
32
+ - name: Cache build-tmp directory
33
+ uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
34
+ with:
35
+ path: ext/build-tmp-musl
36
+ key: build-tmp-musl-${{ runner.os }}-${{ hashFiles('ext/*.sh', 'ext/Rakefile') }}-v2
37
+ - name: Build precompiled librdkafka.so
38
+ run: |
39
+ cd ext
40
+ bash ./build_linux_x86_64_musl.sh
41
+ - name: Upload precompiled library
42
+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
43
+ with:
44
+ name: librdkafka-precompiled-musl
45
+ path: ext/
46
+ retention-days: 1
47
+ push:
48
+ if: github.repository_owner == 'karafka'
49
+ timeout-minutes: 30
50
+ runs-on: ubuntu-latest
51
+ needs: build
52
+ environment: deployment
53
+ permissions:
54
+ contents: write
55
+ id-token: write
56
+ steps:
57
+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
58
+ with:
59
+ fetch-depth: 0
60
+ - name: Download precompiled library
61
+ uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
62
+ with:
63
+ name: librdkafka-precompiled-musl
64
+ path: ext/
65
+ - name: Set up Ruby
66
+ uses: ruby/setup-ruby@a4effe49ee8ee5b8b5091268c473a4628afb5651 # v1.245.0
67
+ with:
68
+ ruby-version: '3.4'
69
+ bundler-cache: false
70
+ - name: Configure trusted publishing credentials
71
+ uses: rubygems/configure-rubygems-credentials@bc6dd217f8a4f919d6835fcfefd470ef821f5c44 # v1.0.0
72
+ - name: Build and push platform-specific gem
73
+ run: |
74
+ gem build *.gemspec
75
+ gem push *.gem
76
+ env:
77
+ RUBY_PLATFORM: 'x86_64-linux-musl'
@@ -0,0 +1,54 @@
1
+ name: Push macOS ARM64 Platform Gem
2
+
3
+ on:
4
+ push:
5
+ tags:
6
+ - v*
7
+
8
+ permissions:
9
+ contents: read
10
+
11
+ env:
12
+ BUNDLE_RETRY: 6
13
+ BUNDLE_JOBS: 4
14
+
15
+ jobs:
16
+ push:
17
+ if: github.repository_owner == 'karafka'
18
+ timeout-minutes: 30
19
+ runs-on: macos-latest
20
+ environment: deployment
21
+ permissions:
22
+ contents: write
23
+ id-token: write
24
+ steps:
25
+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
26
+ with:
27
+ fetch-depth: 0
28
+ - name: Install Bash 4+ and Kerberos
29
+ run: |
30
+ brew install bash
31
+ brew list krb5 &>/dev/null || brew install krb5
32
+ echo "/opt/homebrew/bin" >> $GITHUB_PATH
33
+ - name: Cache build-tmp directory
34
+ uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
35
+ with:
36
+ path: ext/build-tmp-macos
37
+ key: build-tmp-${{ runner.os }}-${{ hashFiles('ext/*.sh', 'ext/Rakefile') }}-v2
38
+ - name: Set up Ruby
39
+ uses: ruby/setup-ruby@a4effe49ee8ee5b8b5091268c473a4628afb5651 # v1.245.0
40
+ with:
41
+ ruby-version: '3.4'
42
+ bundler-cache: false
43
+ - name: Build precompiled librdkafka for macOS ARM64
44
+ run: |
45
+ cd ext
46
+ /opt/homebrew/bin/bash ./build_macos_arm64.sh
47
+ - name: Configure trusted publishing credentials
48
+ uses: rubygems/configure-rubygems-credentials@bc6dd217f8a4f919d6835fcfefd470ef821f5c44 # v1.0.0
49
+ - name: Build and push platform-specific gem
50
+ run: |
51
+ gem build *.gemspec
52
+ gem push *.gem
53
+ env:
54
+ RUBY_PLATFORM: 'arm64-darwin'
data/CHANGELOG.md CHANGED
@@ -2,7 +2,12 @@
2
2
 
3
3
  ## 0.20.0 (Unreleased)
4
4
  - **[Feature]** Add precompiled `x86_64-linux-gnu` setup.
5
+ - **[Feature]** Add precompiled `x86_64-linux-musl` setup.
5
6
  - **[Feature]** Add precompiled `macos_arm64` setup.
7
+ - [Enhancement] Run all specs on each of the platforms with and without precompilation.
8
+ - [Fix] Fix a case where using empty key on the `musl` architecture would cause a segfault.
9
+
10
+ **Note**: Precompiled extensions are a new feature in this release. While they significantly improve installation speed and reduce build dependencies, they should be thoroughly tested in your staging environment before deploying to production. If you encounter any issues with precompiled extensions, you can fall back to building from sources.
6
11
 
7
12
  ## 0.19.5 (2025-05-30)
8
13
  - [Enhancement] Allow for producing to non-existing topics with `key` and `partition_key` present.
@@ -0,0 +1,763 @@
1
+ #!/usr/bin/env bash
2
+ #
3
+ # Build self-contained librdkafka.so for Linux x86_64 musl with checksum verification
4
+ # Usage: ./build-librdkafka-linux-musl.sh
5
+ #
6
+ # Expected directory structure:
7
+ # ext/build_linux_x86_64_musl.sh (this script)
8
+ # ext/build_common.sh (shared functions)
9
+ # dist/librdkafka-*.tar.gz (librdkafka source tarball)
10
+ # dist/patches/*.patch (optional Ruby-specific patches)
11
+ #
12
+ set -euo pipefail
13
+
14
+ # Source common functions and constants
15
+ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
16
+ source "$SCRIPT_DIR/build_common.sh"
17
+
18
+ # Override secure_download for musl/Alpine compatibility
19
+ secure_download() {
20
+ local url="$1"
21
+ local filename="$2"
22
+
23
+ if [ -f "$filename" ]; then
24
+ log "File $filename already exists, verifying checksum..."
25
+ verify_checksum "$filename"
26
+ return 0
27
+ fi
28
+
29
+ log "Downloading $filename from $url..."
30
+
31
+ # Use platform-appropriate download command
32
+ if command -v wget &> /dev/null; then
33
+ # Check if we have GNU wget or BusyBox wget
34
+ if wget --help 2>&1 | grep -q "secure-protocol"; then
35
+ # GNU wget - use security options
36
+ if ! wget --secure-protocol=TLSv1_2 \
37
+ --https-only \
38
+ --timeout=30 \
39
+ --tries=3 \
40
+ --progress=bar \
41
+ "$url" \
42
+ -O "$filename"; then
43
+ error "Failed to download $filename from $url"
44
+ fi
45
+ else
46
+ # BusyBox wget (Alpine) - use basic options
47
+ if ! wget -T 30 \
48
+ -t 3 \
49
+ -q \
50
+ "$url" \
51
+ -O "$filename"; then
52
+ error "Failed to download $filename from $url"
53
+ fi
54
+ fi
55
+ elif command -v curl &> /dev/null; then
56
+ # curl with security options
57
+ if ! curl -L \
58
+ --tlsv1.2 \
59
+ --connect-timeout 30 \
60
+ --max-time 300 \
61
+ --retry 3 \
62
+ --progress-bar \
63
+ "$url" \
64
+ -o "$filename"; then
65
+ error "Failed to download $filename from $url"
66
+ fi
67
+ else
68
+ error "No download utility found (tried wget, curl)"
69
+ fi
70
+
71
+ # Verify checksum immediately after download
72
+ verify_checksum "$filename"
73
+ }
74
+
75
+ # Platform-specific paths
76
+ DIST_DIR="$SCRIPT_DIR/../dist"
77
+ PATCHES_DIR="$DIST_DIR/patches"
78
+ BUILD_DIR="$(pwd)/build-tmp-musl"
79
+ DEPS_PREFIX="/tmp/musl-deps"
80
+
81
+ # musl-specific dependency check
82
+ check_musl_dependencies() {
83
+ log "Checking musl-specific build dependencies..."
84
+
85
+ # Check if we're actually on musl
86
+ if command -v ldd &> /dev/null && ldd --version 2>&1 | grep -q musl; then
87
+ log "✅ musl libc detected"
88
+ elif [ -f /lib/ld-musl-x86_64.so.1 ]; then
89
+ log "✅ musl detected via loader presence"
90
+ else
91
+ warn "Not clearly detected as musl system. This script is optimized for musl-based distributions (Alpine, etc.)"
92
+ warn "Continuing anyway, but glibc-specific optimizations may be used instead..."
93
+ fi
94
+
95
+ # Check for essential build tools
96
+ local missing_tools=()
97
+
98
+ command -v gcc &> /dev/null || missing_tools+=("gcc")
99
+ command -v perl &> /dev/null || missing_tools+=("perl")
100
+ command -v make &> /dev/null || missing_tools+=("make")
101
+ command -v autoconf &> /dev/null || missing_tools+=("autoconf")
102
+ command -v automake &> /dev/null || missing_tools+=("automake")
103
+ command -v libtool &> /dev/null || missing_tools+=("libtool")
104
+ command -v bison &> /dev/null || missing_tools+=("bison")
105
+ command -v flex &> /dev/null || missing_tools+=("flex")
106
+
107
+ # Check for system libraries that dependencies expect
108
+ if [ ! -f /usr/include/sys/types.h ]; then
109
+ missing_tools+=("musl-dev (for system headers)")
110
+ fi
111
+
112
+ # On Alpine, these packages provide the tools:
113
+ # build-base: gcc, make, musl-dev, libc-dev, etc.
114
+ # linux-headers: kernel headers for system calls
115
+ # pkgconf: pkg-config for dependency detection
116
+ # perl: required by OpenSSL and other build systems
117
+ # autoconf, automake, libtool: GNU autotools for configure scripts
118
+ # bison, flex: parser generators (needed by some dependencies)
119
+ # file: for file type detection
120
+
121
+ if [ ${#missing_tools[@]} -gt 0 ]; then
122
+ error "Missing required tools: ${missing_tools[*]}
123
+
124
+ On Alpine Linux, install ALL required packages with:
125
+ apk add build-base linux-headers pkgconf perl \\
126
+ autoconf automake libtool bison flex file
127
+
128
+ Or install individually as needed."
129
+ fi
130
+
131
+ log "✅ All required musl build tools found"
132
+ }
133
+
134
+ # musl-specific compiler setup
135
+ setup_musl_compiler() {
136
+ # musl-specific compiler flags
137
+ export CC="gcc"
138
+ export CFLAGS="-fPIC -O2 -static-libgcc"
139
+ export CXXFLAGS="-fPIC -O2 -static-libgcc"
140
+ export CPPFLAGS=""
141
+ export LDFLAGS="-static-libgcc"
142
+
143
+ log "Applied musl-specific compiler flags"
144
+ }
145
+
146
+ # Build OpenSSL for musl
147
+ build_openssl_musl() {
148
+ local openssl_prefix="$1"
149
+ local openssl_dir="$2"
150
+
151
+ cd "$openssl_dir"
152
+
153
+ # Check both lib and lib64 directories for existing installation
154
+ if [ ! -f "$openssl_prefix/lib/libssl.a" ] && [ ! -f "$openssl_prefix/lib64/libssl.a" ]; then
155
+ log "Configuring and building OpenSSL for musl..."
156
+ make clean 2>/dev/null || true
157
+
158
+ setup_musl_compiler
159
+
160
+ # Create the prefix directory first
161
+ mkdir -p "$openssl_prefix"
162
+
163
+ # Configure OpenSSL for musl (linux-x86_64 works fine)
164
+ ./Configure linux-x86_64 \
165
+ no-shared \
166
+ no-dso \
167
+ no-engine \
168
+ --prefix="$openssl_prefix" \
169
+ --openssldir="$openssl_prefix/ssl"
170
+
171
+ make -j$(get_cpu_count)
172
+
173
+ # Try the install and capture any errors
174
+ log "Installing OpenSSL..."
175
+ if ! make install_sw install_ssldirs 2>&1; then
176
+ warn "install_sw failed, trying full install..."
177
+ make install || {
178
+ # If install fails, check if libraries were actually built
179
+ if [ -f "libssl.a" ] && [ -f "libcrypto.a" ]; then
180
+ log "Install failed but libraries exist, copying manually..."
181
+ mkdir -p "$openssl_prefix/lib" "$openssl_prefix/include"
182
+ cp libssl.a libcrypto.a "$openssl_prefix/lib/"
183
+ cp -r include/openssl "$openssl_prefix/include/" 2>/dev/null || true
184
+ else
185
+ error "OpenSSL build failed - no libraries found"
186
+ fi
187
+ }
188
+ fi
189
+
190
+ # Verify the build - check both possible lib locations
191
+ if [ -f "$openssl_prefix/lib/libssl.a" ] || [ -f "$openssl_prefix/lib64/libssl.a" ]; then
192
+ if [ -f "$openssl_prefix/lib64/libssl.a" ]; then
193
+ log "OpenSSL installed to lib64 directory"
194
+ # Create symlinks in lib for consistency
195
+ mkdir -p "$openssl_prefix/lib"
196
+ ln -sf ../lib64/libssl.a "$openssl_prefix/lib/libssl.a" 2>/dev/null || true
197
+ ln -sf ../lib64/libcrypto.a "$openssl_prefix/lib/libcrypto.a" 2>/dev/null || true
198
+ ln -sf ../lib64/pkgconfig "$openssl_prefix/lib/pkgconfig" 2>/dev/null || true
199
+ fi
200
+ log "✅ Static OpenSSL built successfully"
201
+ else
202
+ error "Failed to build static OpenSSL - libraries not found in lib or lib64"
203
+ fi
204
+ else
205
+ log "Static OpenSSL already built, skipping..."
206
+ fi
207
+ }
208
+
209
+ # Build MIT Kerberos for musl
210
+ build_krb5_musl() {
211
+ local krb5_prefix="$1"
212
+ local krb5_dir="$2"
213
+
214
+ cd "$krb5_dir/src"
215
+
216
+ if [ ! -f "$krb5_prefix/lib/libgssapi_krb5.a" ]; then
217
+ log "Configuring and building MIT Kerberos for musl..."
218
+ make clean 2>/dev/null || true
219
+
220
+ setup_musl_compiler
221
+
222
+ # musl-specific configuration for Kerberos
223
+ # Disable some features that can be problematic on musl
224
+ ./configure \
225
+ --disable-shared \
226
+ --enable-static \
227
+ --prefix="$krb5_prefix" \
228
+ --without-ldap \
229
+ --without-tcl \
230
+ --without-keyutils \
231
+ --disable-rpath \
232
+ --without-system-verto \
233
+ --disable-thread-support \
234
+ --disable-dns-for-realm \
235
+ --without-readline
236
+
237
+ # Build with limited parallelism to avoid musl-specific race conditions
238
+ log "Building Kerberos (will ignore kadmin build failures)..."
239
+ make -j$(( $(get_cpu_count) / 2 )) || {
240
+ log "Full build failed (expected due to kadmin), continuing with libraries..."
241
+ true
242
+ }
243
+
244
+ # Install what was successfully built
245
+ make install || {
246
+ log "Full install failed, installing individual components..."
247
+ make install-mkdirs 2>/dev/null || true
248
+ make -C util install 2>/dev/null || true
249
+ make -C lib install 2>/dev/null || true
250
+ make -C plugins/kdb/db2 install 2>/dev/null || true
251
+ }
252
+
253
+ # Verify we got the essential libraries
254
+ if [ ! -f "$krb5_prefix/lib/libgssapi_krb5.a" ]; then
255
+ error "Failed to build essential Kerberos libraries"
256
+ fi
257
+
258
+ log "✅ Static MIT Kerberos built successfully"
259
+ else
260
+ log "Static MIT Kerberos already built, skipping..."
261
+ fi
262
+ }
263
+
264
+ # Build Cyrus SASL for musl with proper OpenSSL linking
265
+ build_sasl_musl() {
266
+ local sasl_prefix="$1"
267
+ local sasl_dir="$2"
268
+ local krb5_prefix="$3"
269
+ local openssl_prefix="$4" # Add OpenSSL prefix parameter
270
+
271
+ cd "$sasl_dir"
272
+
273
+ if [ ! -f "$sasl_prefix/lib/libsasl2.a" ]; then
274
+ log "Configuring and building Cyrus SASL for musl..."
275
+ make clean 2>/dev/null || true
276
+
277
+ # Apply comprehensive patches for missing time.h includes
278
+ log "Applying comprehensive time.h patches..."
279
+
280
+ # Create backups first
281
+ cp lib/saslutil.c lib/saslutil.c.backup 2>/dev/null || true
282
+ cp plugins/cram.c plugins/cram.c.backup 2>/dev/null || true
283
+
284
+ # Use perl for more reliable in-place editing
285
+ log "Patching lib/saslutil.c..."
286
+ perl -i -pe 's/^#include "saslint\.h"$/#include "saslint.h"\n#include <time.h>/' lib/saslutil.c
287
+
288
+ log "Patching plugins/cram.c..."
289
+ perl -i -pe 's/^#include "plugin_common\.h"$/#include "plugin_common.h"\n#include <time.h>/' plugins/cram.c
290
+
291
+ # Verify patches applied
292
+ if ! grep -q "#include <time.h>" lib/saslutil.c; then
293
+ log "Perl approach failed, using awk fallback for saslutil.c..."
294
+ awk '/^#include "saslint\.h"$/ {print; print "#include <time.h>"; next} {print}' lib/saslutil.c.backup > lib/saslutil.c
295
+ fi
296
+
297
+ if ! grep -q "#include <time.h>" plugins/cram.c; then
298
+ log "Perl approach failed, using awk fallback for cram.c..."
299
+ awk '/^#include "plugin_common\.h"$/ {print; print "#include <time.h>"; next} {print}' plugins/cram.c.backup > plugins/cram.c
300
+ fi
301
+
302
+ # Clean up backup files
303
+ rm -f lib/saslutil.c.backup plugins/cram.c.backup
304
+
305
+ # Verify patches were applied
306
+ if ! grep -q "#include <time.h>" lib/saslutil.c || ! grep -q "#include <time.h>" plugins/cram.c; then
307
+ error "Failed to patch time.h includes - this is required for compilation"
308
+ fi
309
+
310
+ log "✅ time.h patches applied successfully"
311
+
312
+ # Determine correct OpenSSL lib directory
313
+ local openssl_lib_dir
314
+ if [ -f "$openssl_prefix/lib64/libssl.a" ]; then
315
+ openssl_lib_dir="$openssl_prefix/lib64"
316
+ log "Using OpenSSL libraries from lib64"
317
+ else
318
+ openssl_lib_dir="$openssl_prefix/lib"
319
+ log "Using OpenSSL libraries from lib"
320
+ fi
321
+
322
+ # Verify OpenSSL libraries exist
323
+ if [ ! -f "$openssl_lib_dir/libssl.a" ] || [ ! -f "$openssl_lib_dir/libcrypto.a" ]; then
324
+ error "OpenSSL static libraries not found in $openssl_lib_dir"
325
+ fi
326
+
327
+ setup_musl_compiler
328
+
329
+ # Set comprehensive OpenSSL and Kerberos flags
330
+ export CPPFLAGS="$CPPFLAGS -I$krb5_prefix/include -I$openssl_prefix/include"
331
+ export LDFLAGS="$LDFLAGS -L$krb5_prefix/lib -L$openssl_lib_dir"
332
+ export LIBS="-lssl -lcrypto -ldl -lpthread"
333
+
334
+ # Also set PKG_CONFIG_PATH for better library detection
335
+ export PKG_CONFIG_PATH="$openssl_lib_dir/pkgconfig:$krb5_prefix/lib/pkgconfig:${PKG_CONFIG_PATH:-}"
336
+
337
+ log "Environment variables set:"
338
+ log " CPPFLAGS: $CPPFLAGS"
339
+ log " LDFLAGS: $LDFLAGS"
340
+ log " LIBS: $LIBS"
341
+ log " PKG_CONFIG_PATH: $PKG_CONFIG_PATH"
342
+
343
+ # Configure SASL with explicit OpenSSL paths and comprehensive options
344
+ ./configure \
345
+ --disable-shared \
346
+ --enable-static \
347
+ --prefix="$sasl_prefix" \
348
+ --with-openssl="$openssl_prefix" \
349
+ --with-ssl="$openssl_prefix" \
350
+ --enable-gssapi="$krb5_prefix" \
351
+ --without-dblib \
352
+ --disable-gdbm \
353
+ --disable-sample \
354
+ --disable-obsolete_cram_attr \
355
+ --disable-obsolete_digest_attr \
356
+ --without-pam \
357
+ --without-saslauthd \
358
+ --without-pwcheck \
359
+ --without-des \
360
+ --without-authdaemond \
361
+ --disable-java \
362
+ --disable-sql \
363
+ --disable-ldapdb \
364
+ --enable-plain \
365
+ --enable-login \
366
+ --enable-digest \
367
+ --enable-cram \
368
+ --enable-otp \
369
+ --enable-ntlm \
370
+ --enable-scram \
371
+ ac_cv_func_getnameinfo=yes \
372
+ ac_cv_func_getaddrinfo=yes \
373
+ OPENSSL_CFLAGS="-I$openssl_prefix/include" \
374
+ OPENSSL_LIBS="-L$openssl_lib_dir -lssl -lcrypto"
375
+
376
+ log "Configuration completed, starting build..."
377
+
378
+ # Build with reduced parallelism to avoid race conditions
379
+ make -j$(( $(get_cpu_count) / 2 )) || {
380
+ log "Build failed, trying single-threaded build..."
381
+ make clean
382
+ make -j1
383
+ }
384
+
385
+ # Install
386
+ make install
387
+
388
+ # Verify the build
389
+ if [ ! -f "$sasl_prefix/lib/libsasl2.a" ]; then
390
+ error "Failed to build static Cyrus SASL"
391
+ fi
392
+
393
+ # Additional verification - check if OTP plugin was built (this was failing)
394
+ if [ -f "$sasl_prefix/lib/sasl2/libotp.a" ] || [ -f "$sasl_prefix/lib/sasl2/libotp.la" ]; then
395
+ log "✅ OTP plugin built successfully"
396
+ else
397
+ log "⚠️ OTP plugin may not have built, but core SASL library is available"
398
+ fi
399
+
400
+ log "✅ Static Cyrus SASL built successfully"
401
+ else
402
+ log "Static Cyrus SASL already built, skipping..."
403
+ fi
404
+ }
405
+
406
+ # Build static library with musl-specific configuration
407
+ build_static_lib_musl() {
408
+ local lib_name="$1"
409
+ local prefix="$2"
410
+ local source_dir="$3"
411
+ local configure_args="$4"
412
+
413
+ cd "$source_dir"
414
+
415
+ local lib_file="$prefix/lib/lib${lib_name}.a"
416
+ if [ ! -f "$lib_file" ]; then
417
+ log "Configuring and building static $lib_name for musl..."
418
+ make clean 2>/dev/null || true
419
+
420
+ setup_musl_compiler
421
+
422
+ # Run configure with provided arguments
423
+ eval "./configure --prefix=\"$prefix\" $configure_args"
424
+
425
+ make -j$(get_cpu_count)
426
+ make install
427
+
428
+ # Verify the build
429
+ if [ ! -f "$lib_file" ]; then
430
+ error "Failed to build static $lib_name"
431
+ fi
432
+
433
+ log "✅ Static $lib_name built successfully"
434
+ else
435
+ log "Static $lib_name already built, skipping..."
436
+ fi
437
+ }
438
+
439
+ # Check common and musl-specific dependencies
440
+ check_common_dependencies
441
+ check_musl_dependencies
442
+
443
+ # Auto-detect librdkafka tarball
444
+ log "Looking for librdkafka tarball in $DIST_DIR..."
445
+ LIBRDKAFKA_TARBALL=$(find_librdkafka_tarball "$DIST_DIR")
446
+ log "Found librdkafka tarball: $LIBRDKAFKA_TARBALL"
447
+
448
+ # Verify librdkafka tarball checksum if available
449
+ verify_librdkafka_checksum "$LIBRDKAFKA_TARBALL"
450
+
451
+ # Find patches
452
+ PATCHES_FOUND=()
453
+ find_patches "$PATCHES_DIR" PATCHES_FOUND
454
+
455
+ security_log "Starting secure build with checksum verification enabled"
456
+ log "Building self-contained librdkafka.so for Linux x86_64 musl (WITH SASL)"
457
+ log "Dependencies to build:"
458
+ log " - OpenSSL: $OPENSSL_VERSION"
459
+ log " - Cyrus SASL: $CYRUS_SASL_VERSION (with official GCC 14+ patches)"
460
+ log " - MIT Kerberos: $KRB5_VERSION"
461
+ log " - zlib: $ZLIB_VERSION"
462
+ log " - ZStd: $ZSTD_VERSION"
463
+ log "librdkafka source: $LIBRDKAFKA_TARBALL"
464
+ log "Build directory: $BUILD_DIR"
465
+
466
+ # Create build directory
467
+ mkdir -p "$BUILD_DIR"
468
+ cd "$BUILD_DIR"
469
+
470
+ # Build OpenSSL
471
+ log "Building OpenSSL $OPENSSL_VERSION for musl..."
472
+ OPENSSL_PREFIX="$DEPS_PREFIX/static-openssl-$OPENSSL_VERSION"
473
+ OPENSSL_TARBALL="openssl-$OPENSSL_VERSION.tar.gz"
474
+ OPENSSL_DIR="openssl-$OPENSSL_VERSION"
475
+
476
+ secure_download "$(get_openssl_url)" "$OPENSSL_TARBALL"
477
+ extract_if_needed "$OPENSSL_TARBALL" "$OPENSSL_DIR"
478
+ build_openssl_musl "$OPENSSL_PREFIX" "$OPENSSL_DIR"
479
+
480
+ cd "$BUILD_DIR"
481
+
482
+ # Determine OpenSSL lib directory (could be lib or lib64)
483
+ if [ -f "$OPENSSL_PREFIX/lib64/libssl.a" ]; then
484
+ OPENSSL_LIB_DIR="$OPENSSL_PREFIX/lib64"
485
+ log "Using OpenSSL libraries from lib64"
486
+ else
487
+ OPENSSL_LIB_DIR="$OPENSSL_PREFIX/lib"
488
+ log "Using OpenSSL libraries from lib"
489
+ fi
490
+
491
+ # Build MIT Kerberos
492
+ log "Building MIT Kerberos $KRB5_VERSION for musl..."
493
+ KRB5_PREFIX="$DEPS_PREFIX/static-krb5-$KRB5_VERSION"
494
+ KRB5_TARBALL="krb5-$KRB5_VERSION.tar.gz"
495
+ KRB5_DIR="krb5-$KRB5_VERSION"
496
+
497
+ secure_download "$(get_krb5_url)" "$KRB5_TARBALL"
498
+ extract_if_needed "$KRB5_TARBALL" "$KRB5_DIR"
499
+ build_krb5_musl "$KRB5_PREFIX" "$KRB5_DIR"
500
+
501
+ cd "$BUILD_DIR"
502
+
503
+ # Build SASL with official patches (updated call)
504
+ log "Building Cyrus SASL $CYRUS_SASL_VERSION for musl..."
505
+ SASL_PREFIX="$DEPS_PREFIX/static-sasl-$CYRUS_SASL_VERSION"
506
+ SASL_TARBALL="cyrus-sasl-$CYRUS_SASL_VERSION.tar.gz"
507
+ SASL_DIR="cyrus-sasl-$CYRUS_SASL_VERSION"
508
+
509
+ secure_download "$(get_sasl_url)" "$SASL_TARBALL"
510
+ extract_if_needed "$SASL_TARBALL" "$SASL_DIR"
511
+ build_sasl_musl "$SASL_PREFIX" "$SASL_DIR" "$KRB5_PREFIX" "$OPENSSL_PREFIX" # Added OPENSSL_PREFIX parameter
512
+
513
+ cd "$BUILD_DIR"
514
+
515
+ # Build zlib
516
+ log "Building zlib $ZLIB_VERSION for musl..."
517
+ ZLIB_PREFIX="$DEPS_PREFIX/static-zlib-$ZLIB_VERSION"
518
+ ZLIB_TARBALL="zlib-$ZLIB_VERSION.tar.gz"
519
+ ZLIB_DIR="zlib-$ZLIB_VERSION"
520
+
521
+ secure_download "$(get_zlib_url)" "$ZLIB_TARBALL"
522
+ extract_if_needed "$ZLIB_TARBALL" "$ZLIB_DIR"
523
+ build_static_lib_musl "z" "$ZLIB_PREFIX" "$ZLIB_DIR" "--static"
524
+
525
+ cd "$BUILD_DIR"
526
+
527
+ # Build ZStd
528
+ log "Building ZStd $ZSTD_VERSION for musl..."
529
+ ZSTD_PREFIX="$DEPS_PREFIX/static-zstd-$ZSTD_VERSION"
530
+ ZSTD_TARBALL="zstd-$ZSTD_VERSION.tar.gz"
531
+ ZSTD_DIR="zstd-$ZSTD_VERSION"
532
+
533
+ secure_download "$(get_zstd_url)" "$ZSTD_TARBALL"
534
+ extract_if_needed "$ZSTD_TARBALL" "$ZSTD_DIR"
535
+ cd "$ZSTD_DIR"
536
+
537
+ if [ ! -f "$ZSTD_PREFIX/lib/libzstd.a" ]; then
538
+ log "Building ZStd for musl..."
539
+ make clean 2>/dev/null || true
540
+
541
+ setup_musl_compiler
542
+
543
+ # Build static library using ZStd's Makefile
544
+ make lib-mt CFLAGS="$CFLAGS" PREFIX="$ZSTD_PREFIX" -j$(get_cpu_count)
545
+ make install PREFIX="$ZSTD_PREFIX"
546
+
547
+ # Verify the build
548
+ if [ ! -f "$ZSTD_PREFIX/lib/libzstd.a" ]; then
549
+ error "Failed to build static ZStd"
550
+ fi
551
+
552
+ log "✅ Static ZStd built successfully"
553
+ else
554
+ log "Static ZStd already built, skipping..."
555
+ fi
556
+
557
+ cd "$BUILD_DIR"
558
+
559
+ # Extract and patch librdkafka
560
+ log "Extracting librdkafka..."
561
+ tar xzf "$LIBRDKAFKA_TARBALL"
562
+ cd "librdkafka-$LIBRDKAFKA_VERSION"
563
+
564
+ # Fix permissions and apply patches
565
+ fix_configure_permissions
566
+ apply_patches PATCHES_FOUND
567
+
568
+ # Configure librdkafka for musl (with SASL)
569
+ log "Configuring librdkafka for musl (with SASL support)..."
570
+
571
+ setup_musl_compiler
572
+
573
+ # musl-specific configuration with SASL
574
+ export CPPFLAGS="$CPPFLAGS -I$KRB5_PREFIX/include -I$SASL_PREFIX/include"
575
+ export LDFLAGS="$LDFLAGS -L$KRB5_PREFIX/lib -L$SASL_PREFIX/lib"
576
+
577
+ if [ -f configure ]; then
578
+ log "Using mklove configure script"
579
+
580
+ # mklove-specific configure options with SASL
581
+ ./configure \
582
+ --enable-static \
583
+ --disable-shared \
584
+ --disable-curl \
585
+ --enable-gssapi \
586
+ --enable-sasl \
587
+ --disable-c11threads
588
+ else
589
+ error "No configure script found"
590
+ fi
591
+
592
+ # Build librdkafka
593
+ log "Compiling librdkafka..."
594
+ make clean || true
595
+ make -j$(get_cpu_count)
596
+
597
+ # Verify librdkafka.a exists
598
+ if [ ! -f src/librdkafka.a ]; then
599
+ error "librdkafka.a not found after build"
600
+ fi
601
+
602
+ log "librdkafka.a built successfully"
603
+
604
+ # Create self-contained shared library for musl (with SASL)
605
+ log "Creating self-contained librdkafka.so for musl (with SASL support)..."
606
+
607
+ # First, verify all static libraries exist
608
+ log "Verifying static libraries exist..."
609
+ for lib_path in \
610
+ "$SASL_PREFIX/lib/libsasl2.a" \
611
+ "$KRB5_PREFIX/lib/libgssapi_krb5.a" \
612
+ "$KRB5_PREFIX/lib/libkrb5.a" \
613
+ "$KRB5_PREFIX/lib/libk5crypto.a" \
614
+ "$KRB5_PREFIX/lib/libcom_err.a" \
615
+ "$KRB5_PREFIX/lib/libkrb5support.a" \
616
+ "$OPENSSL_LIB_DIR/libssl.a" \
617
+ "$OPENSSL_LIB_DIR/libcrypto.a" \
618
+ "$ZLIB_PREFIX/lib/libz.a" \
619
+ "$ZSTD_PREFIX/lib/libzstd.a"
620
+ do
621
+ if [ ! -f "$lib_path" ]; then
622
+ error "Required static library not found: $lib_path"
623
+ else
624
+ log "✅ Found: $lib_path"
625
+ fi
626
+ done
627
+
628
+ gcc -shared -fPIC \
629
+ -Wl,--whole-archive src/librdkafka.a -Wl,--no-whole-archive \
630
+ -o librdkafka.so \
631
+ -Wl,-Bstatic \
632
+ "$SASL_PREFIX/lib/libsasl2.a" \
633
+ "$KRB5_PREFIX/lib/libgssapi_krb5.a" \
634
+ "$KRB5_PREFIX/lib/libkrb5.a" \
635
+ "$KRB5_PREFIX/lib/libk5crypto.a" \
636
+ "$KRB5_PREFIX/lib/libcom_err.a" \
637
+ "$KRB5_PREFIX/lib/libkrb5support.a" \
638
+ "$OPENSSL_LIB_DIR/libssl.a" \
639
+ "$OPENSSL_LIB_DIR/libcrypto.a" \
640
+ "$ZLIB_PREFIX/lib/libz.a" \
641
+ "$ZSTD_PREFIX/lib/libzstd.a" \
642
+ -Wl,-Bdynamic \
643
+ -lpthread -lm -ldl -lc \
644
+ -static-libgcc \
645
+ -Wl,--as-needed \
646
+ -Wl,--no-undefined
647
+
648
+ if [ ! -f librdkafka.so ]; then
649
+ error "Failed to create librdkafka.so"
650
+ fi
651
+
652
+ log "librdkafka.so created successfully"
653
+
654
+ # Enhanced verification
655
+ log "Verifying static linking of SASL..."
656
+ if command -v ldd &> /dev/null; then
657
+ SASL_DEPS=$(ldd librdkafka.so 2>/dev/null | grep -i sasl || true)
658
+ if [ -n "$SASL_DEPS" ]; then
659
+ error "SASL is still dynamically linked: $SASL_DEPS"
660
+ else
661
+ log "✅ SASL successfully statically linked - no dynamic SASL dependencies found"
662
+ fi
663
+
664
+ # Check for any other problematic dynamic dependencies
665
+ PROBLEMATIC_DEPS=$(ldd librdkafka.so 2>/dev/null | grep -E "(libssl|libcrypto|libz|libzstd|libkrb|libgssapi)" || true)
666
+ if [ -n "$PROBLEMATIC_DEPS" ]; then
667
+ warn "Found other dynamic dependencies that should be static: $PROBLEMATIC_DEPS"
668
+ else
669
+ log "✅ All major dependencies appear to be statically linked"
670
+ fi
671
+ else
672
+ log "ldd not available, skipping dynamic dependency check"
673
+ fi
674
+
675
+ # Additional verification using nm if available
676
+ if command -v nm &> /dev/null; then
677
+ log "Checking for SASL symbols in the library..."
678
+ SASL_SYMBOLS=$(nm -D librdkafka.so 2>/dev/null | grep -i sasl | head -5 || true)
679
+ if [ -n "$SASL_SYMBOLS" ]; then
680
+ log "✅ SASL symbols found in library (first 5):"
681
+ echo "$SASL_SYMBOLS"
682
+ else
683
+ warn "No SASL symbols found - this might indicate a linking issue"
684
+ fi
685
+ fi
686
+
687
+ # Verify the build (musl-compatible)
688
+ log "Verifying musl build..."
689
+ if command -v file &> /dev/null; then
690
+ file librdkafka.so
691
+ else
692
+ log "file command not available, skipping file type check"
693
+ fi
694
+
695
+ log "Checking dependencies with ldd:"
696
+ if command -v ldd &> /dev/null; then
697
+ ldd librdkafka.so
698
+ else
699
+ log "ldd not available, skipping dependency check"
700
+ fi
701
+
702
+ log "Checking for external dependencies (should only show musl and system libraries):"
703
+ if command -v ldd &> /dev/null; then
704
+ EXTERNAL_DEPS=$(ldd librdkafka.so 2>/dev/null | grep -v "musl" | grep -v "ld-musl" | grep -v "librdkafka.so" | grep -v "=>" | grep -v "statically linked" | grep -v "not a dynamic executable" || true)
705
+ if [ -n "$EXTERNAL_DEPS" ]; then
706
+ # Filter out expected system dependencies that are OK on musl
707
+ FILTERED_DEPS=$(echo "$EXTERNAL_DEPS" | grep -v "linux-vdso.so" | grep -v "/lib/ld-musl" || true)
708
+ if [ -n "$FILTERED_DEPS" ]; then
709
+ warn "Found some external dependencies (may be acceptable for musl): $FILTERED_DEPS"
710
+ else
711
+ log "✅ All dependencies are musl system libraries - build is self-contained!"
712
+ fi
713
+ else
714
+ log "✅ No unexpected external dependencies found - library is self-contained!"
715
+ fi
716
+ else
717
+ log "ldd not available, skipping external dependency check"
718
+ fi
719
+
720
+ # Check for musl compatibility
721
+ log "Verifying musl compatibility..."
722
+ if command -v ldd &> /dev/null; then
723
+ if ldd librdkafka.so 2>&1 | grep -q "musl"; then
724
+ log "✅ Library is linked against musl libc"
725
+ elif ldd librdkafka.so 2>&1 | grep -q "statically linked"; then
726
+ log "✅ Library appears to be statically linked (good for musl)"
727
+ else
728
+ warn "Library linking may not be optimal for musl systems"
729
+ fi
730
+ else
731
+ log "ldd not available, skipping musl compatibility check"
732
+ fi
733
+
734
+ # Copy to output directory
735
+ OUTPUT_DIR="$SCRIPT_DIR"
736
+ cp librdkafka.so "$OUTPUT_DIR/"
737
+ log "librdkafka.so copied to: $OUTPUT_DIR/librdkafka.so"
738
+
739
+ # Print summaries
740
+ print_security_summary
741
+
742
+ # Enhanced summary for musl (with SASL)
743
+ log "Build completed successfully!"
744
+ log "📦 Self-contained librdkafka built for Linux x86_64 musl:"
745
+ log " ✅ OpenSSL $OPENSSL_VERSION (SSL/TLS support) - checksum verified"
746
+ log " ✅ Cyrus SASL $CYRUS_SASL_VERSION (authentication for AWS MSK) - checksum verified"
747
+ log " ✅ MIT Kerberos $KRB5_VERSION (GSSAPI/Kerberos authentication) - checksum verified"
748
+ log " ✅ zlib $ZLIB_VERSION (compression) - checksum verified"
749
+ log " ✅ ZStd $ZSTD_VERSION (high-performance compression) - checksum verified"
750
+ log ""
751
+ log "🎯 Ready for deployment on musl-based systems (Alpine Linux, Docker containers)"
752
+ log "☁️ Compatible with AWS MSK and other secured Kafka clusters"
753
+ log "🔐 Supply chain security: All dependencies cryptographically verified"
754
+ log "🐳 Docker-optimized: Perfect for Alpine-based container deployments"
755
+ log "🛠️ GCC 14+ compatible: Official patches applied to Cyrus SASL"
756
+ log ""
757
+ log "Location: $OUTPUT_DIR/librdkafka.so"
758
+
759
+ # Cleanup
760
+ cleanup_build_dir "$BUILD_DIR"
761
+
762
+ # Reset environment variables
763
+ unset CFLAGS CXXFLAGS CPPFLAGS LDFLAGS CC
data/ext/librdkafka.so CHANGED
Binary file
@@ -41,7 +41,9 @@ Gem::Specification.new do |gem|
41
41
  end
42
42
 
43
43
  gem.add_dependency 'ffi', '~> 1.15'
44
+ gem.add_dependency 'logger'
44
45
  gem.add_dependency 'mini_portile2', '~> 2.6'
46
+ gem.add_dependency 'ostruct'
45
47
  gem.add_dependency 'rake', '> 12'
46
48
 
47
49
  gem.add_development_dependency 'pry'
@@ -406,12 +406,15 @@ module Rdkafka
406
406
  def self.partitioner(str, partition_count, partitioner_name = "consistent_random")
407
407
  # Return RD_KAFKA_PARTITION_UA(unassigned partition) when partition count is nil/zero.
408
408
  return -1 unless partition_count&.nonzero?
409
+ # musl architecture crashes with empty string
410
+ return 0 if str.empty?
409
411
 
410
- str_ptr = str.empty? ? FFI::MemoryPointer::NULL : FFI::MemoryPointer.from_string(str)
412
+ str_ptr = FFI::MemoryPointer.from_string(str)
411
413
  method_name = PARTITIONERS.fetch(partitioner_name) do
412
414
  raise Rdkafka::Config::ConfigError.new("Unknown partitioner: #{partitioner_name}")
413
415
  end
414
- public_send(method_name, nil, str_ptr, str.size > 0 ? str.size : 1, partition_count, nil, nil)
416
+
417
+ public_send(method_name, nil, str_ptr, str.size, partition_count, nil, nil)
415
418
  end
416
419
 
417
420
  # Create Topics
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Rdkafka
4
- VERSION = "0.20.0.rc3"
4
+ VERSION = "0.20.0.rc5"
5
5
  LIBRDKAFKA_VERSION = "2.8.0"
6
6
  LIBRDKAFKA_SOURCE_SHA256 = "5bd1c46f63265f31c6bfcedcde78703f77d28238eadf23821c2b43fc30be3e25"
7
7
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: karafka-rdkafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.20.0.rc3
4
+ version: 0.20.0.rc5
5
5
  platform: x86_64-linux-gnu
6
6
  authors:
7
7
  - Thijs Cadier
@@ -24,6 +24,20 @@ dependencies:
24
24
  - - "~>"
25
25
  - !ruby/object:Gem::Version
26
26
  version: '1.15'
27
+ - !ruby/object:Gem::Dependency
28
+ name: logger
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
27
41
  - !ruby/object:Gem::Dependency
28
42
  name: mini_portile2
29
43
  requirement: !ruby/object:Gem::Requirement
@@ -38,6 +52,20 @@ dependencies:
38
52
  - - "~>"
39
53
  - !ruby/object:Gem::Version
40
54
  version: '2.6'
55
+ - !ruby/object:Gem::Dependency
56
+ name: ostruct
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: '0'
41
69
  - !ruby/object:Gem::Dependency
42
70
  name: rake
43
71
  requirement: !ruby/object:Gem::Requirement
@@ -118,8 +146,11 @@ files:
118
146
  - ".github/CODEOWNERS"
119
147
  - ".github/FUNDING.yml"
120
148
  - ".github/workflows/ci_linux_x86_64_gnu.yml"
149
+ - ".github/workflows/ci_linux_x86_64_musl.yml"
121
150
  - ".github/workflows/ci_macos_arm64.yml"
122
151
  - ".github/workflows/push_linux_x86_64_gnu.yml"
152
+ - ".github/workflows/push_linux_x86_64_musl.yml"
153
+ - ".github/workflows/push_macos_arm64.yml"
123
154
  - ".github/workflows/push_ruby.yml"
124
155
  - ".github/workflows/verify-action-pins.yml"
125
156
  - ".gitignore"
@@ -137,6 +168,7 @@ files:
137
168
  - ext/Rakefile
138
169
  - ext/build_common.sh
139
170
  - ext/build_linux_x86_64_gnu.sh
171
+ - ext/build_linux_x86_64_musl.sh
140
172
  - ext/build_macos_arm64.sh
141
173
  - ext/librdkafka.so
142
174
  - karafka-rdkafka.gemspec