karafka-rdkafka 0.20.0.rc3-x86_64-linux-gnu → 0.20.0.rc5-x86_64-linux-gnu
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci_linux_x86_64_gnu.yml +1 -0
- data/.github/workflows/ci_linux_x86_64_musl.yml +205 -0
- data/.github/workflows/ci_macos_arm64.yml +5 -0
- data/.github/workflows/push_linux_x86_64_gnu.yml +5 -1
- data/.github/workflows/push_linux_x86_64_musl.yml +77 -0
- data/.github/workflows/push_macos_arm64.yml +54 -0
- data/CHANGELOG.md +5 -0
- data/ext/build_linux_x86_64_musl.sh +763 -0
- data/ext/librdkafka.so +0 -0
- data/karafka-rdkafka.gemspec +2 -0
- data/lib/rdkafka/bindings.rb +5 -2
- data/lib/rdkafka/version.rb +1 -1
- metadata +33 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 89b5b7ab7269112cd5816d40293395df9208e6929eca11140a3bfb17f8a48cc2
|
4
|
+
data.tar.gz: 6fea31eaa2c84f6c5d13dd4df7aae1808283a5c82f9fe83a73e46d675625c49c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4e1d26e6e3c76f82c9dabc5de8b98bcad8df7b59c4052524ea58325a554b1a8ca83d1b4e3ba2baeb2f35d0fd114287866a1fa71cbcf75d7ca6345cfae8227504
|
7
|
+
data.tar.gz: 352839be9a5499a6eda10f98f10c8ef5c147c1cf0fae7646a1e925abd03ad1dbb38b7e21306defdc52b2bfca24d416fb02f1b8f319d253ad6610f67e17cc9ee9
|
@@ -0,0 +1,205 @@
|
|
1
|
+
name: CI Linux x86_64 musl
|
2
|
+
|
3
|
+
concurrency:
|
4
|
+
group: ${{ github.workflow }}-${{ github.ref }}
|
5
|
+
cancel-in-progress: true
|
6
|
+
|
7
|
+
on:
|
8
|
+
pull_request:
|
9
|
+
branches: [ main, master ]
|
10
|
+
push:
|
11
|
+
branches: [ main, master ]
|
12
|
+
schedule:
|
13
|
+
- cron: '0 1 * * *'
|
14
|
+
|
15
|
+
permissions:
|
16
|
+
contents: read
|
17
|
+
|
18
|
+
env:
|
19
|
+
BUNDLE_RETRY: 6
|
20
|
+
BUNDLE_JOBS: 4
|
21
|
+
|
22
|
+
jobs:
|
23
|
+
build_install:
|
24
|
+
timeout-minutes: 30
|
25
|
+
runs-on: ubuntu-latest
|
26
|
+
container:
|
27
|
+
image: alpine:3.22@sha256:8a1f59ffb675680d47db6337b49d22281a139e9d709335b492be023728e11715
|
28
|
+
steps:
|
29
|
+
- name: Install dependencies
|
30
|
+
run: |
|
31
|
+
apk add --no-cache git curl ca-certificates build-base linux-headers \
|
32
|
+
pkgconf perl autoconf automake libtool bison flex file \
|
33
|
+
ruby ruby-dev ruby-bundler bash zstd-dev zlib zlib-dev openssl-dev cyrus-sasl-dev
|
34
|
+
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
35
|
+
with:
|
36
|
+
fetch-depth: 0
|
37
|
+
- name: Configure git safe directory
|
38
|
+
run: git config --global --add safe.directory /__w/karafka-rdkafka/karafka-rdkafka
|
39
|
+
- name: Build gem with mini_portile
|
40
|
+
run: |
|
41
|
+
set -e
|
42
|
+
bundle config set --local path 'vendor/bundle'
|
43
|
+
bundle install
|
44
|
+
cd ext && bundle exec rake
|
45
|
+
cd ..
|
46
|
+
- name: Upload built gem and bundle
|
47
|
+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
48
|
+
with:
|
49
|
+
name: rdkafka-built-gem-musl
|
50
|
+
path: |
|
51
|
+
vendor/bundle/
|
52
|
+
.bundle/
|
53
|
+
ext/
|
54
|
+
lib/
|
55
|
+
retention-days: 1
|
56
|
+
|
57
|
+
specs_install:
|
58
|
+
timeout-minutes: 30
|
59
|
+
runs-on: ubuntu-latest
|
60
|
+
needs: build_install
|
61
|
+
strategy:
|
62
|
+
fail-fast: false
|
63
|
+
matrix:
|
64
|
+
ruby:
|
65
|
+
- '3.4'
|
66
|
+
- '3.3'
|
67
|
+
- '3.2'
|
68
|
+
- '3.1'
|
69
|
+
include:
|
70
|
+
- ruby: '3.4'
|
71
|
+
coverage: 'true'
|
72
|
+
steps:
|
73
|
+
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
74
|
+
with:
|
75
|
+
fetch-depth: 0
|
76
|
+
- name: Download built gem
|
77
|
+
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
|
78
|
+
with:
|
79
|
+
name: rdkafka-built-gem-musl
|
80
|
+
path: ./
|
81
|
+
- name: Start Kafka with Docker Compose
|
82
|
+
run: |
|
83
|
+
docker compose up -d
|
84
|
+
echo "Waiting for Kafka to be ready..."
|
85
|
+
sleep 10
|
86
|
+
|
87
|
+
for i in {1..30}; do
|
88
|
+
if docker compose exec -T kafka kafka-topics --bootstrap-server localhost:9092 --list >/dev/null 2>&1; then
|
89
|
+
echo "Kafka topics command succeeded!"
|
90
|
+
break
|
91
|
+
fi
|
92
|
+
sleep 2
|
93
|
+
done
|
94
|
+
- name: Run all specs
|
95
|
+
env:
|
96
|
+
GITHUB_COVERAGE: ${{ matrix.coverage }}
|
97
|
+
RDKAFKA_EXT_PATH: ${{ github.workspace }}/ext
|
98
|
+
run: |
|
99
|
+
docker run --rm \
|
100
|
+
--network host \
|
101
|
+
-v "${{ github.workspace }}:/workspace" \
|
102
|
+
-w /workspace \
|
103
|
+
-e "GITHUB_COVERAGE=${{ matrix.coverage }}" \
|
104
|
+
-e "RDKAFKA_EXT_PATH=/workspace/ext" \
|
105
|
+
ruby:${{ matrix.ruby }}-alpine \
|
106
|
+
sh -c 'apk add --no-cache git build-base linux-headers bash \
|
107
|
+
cyrus-sasl \
|
108
|
+
cyrus-sasl-login \
|
109
|
+
cyrus-sasl-crammd5 \
|
110
|
+
cyrus-sasl-digestmd5 \
|
111
|
+
cyrus-sasl-gssapiv2 \
|
112
|
+
cyrus-sasl-scram \
|
113
|
+
krb5-libs \
|
114
|
+
openssl \
|
115
|
+
zlib \
|
116
|
+
zlib-dev \
|
117
|
+
zstd-libs && \
|
118
|
+
git config --global --add safe.directory /workspace && \
|
119
|
+
bundle config set --local path vendor/bundle && \
|
120
|
+
bundle install && \
|
121
|
+
bundle exec ruby -S rspec'
|
122
|
+
build_precompiled:
|
123
|
+
timeout-minutes: 45
|
124
|
+
runs-on: ubuntu-latest
|
125
|
+
container:
|
126
|
+
image: alpine:3.22@sha256:8a1f59ffb675680d47db6337b49d22281a139e9d709335b492be023728e11715
|
127
|
+
steps:
|
128
|
+
- name: Install dependencies
|
129
|
+
run: |
|
130
|
+
apk add --no-cache git curl ca-certificates build-base linux-headers \
|
131
|
+
pkgconf perl autoconf automake libtool bison flex file bash wget zstd-dev \
|
132
|
+
openssl-dev cyrus-sasl-dev cyrus-sasl cyrus-sasl-login \
|
133
|
+
cyrus-sasl-crammd5 cyrus-sasl-digestmd5 cyrus-sasl-gssapiv2 cyrus-sasl-scram \
|
134
|
+
krb5-libs openssl zlib zlib-dev zstd-libs
|
135
|
+
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
136
|
+
with:
|
137
|
+
fetch-depth: 0
|
138
|
+
- name: Configure git safe directory
|
139
|
+
run: git config --global --add safe.directory /__w/karafka-rdkafka/karafka-rdkafka
|
140
|
+
- name: Cache build-tmp directory
|
141
|
+
uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
|
142
|
+
with:
|
143
|
+
path: ext/build-tmp-musl
|
144
|
+
key: build-tmp-musl-${{ runner.os }}-${{ hashFiles('ext/*.sh', 'ext/Rakefile') }}-v2
|
145
|
+
- name: Build precompiled librdkafka.so
|
146
|
+
run: |
|
147
|
+
cd ext
|
148
|
+
bash ./build_linux_x86_64_musl.sh
|
149
|
+
- name: Upload precompiled library
|
150
|
+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
151
|
+
with:
|
152
|
+
name: librdkafka-precompiled-musl
|
153
|
+
path: ext/
|
154
|
+
retention-days: 1
|
155
|
+
|
156
|
+
specs_precompiled:
|
157
|
+
timeout-minutes: 30
|
158
|
+
runs-on: ubuntu-latest
|
159
|
+
needs: build_precompiled
|
160
|
+
strategy:
|
161
|
+
fail-fast: false
|
162
|
+
matrix:
|
163
|
+
ruby:
|
164
|
+
- '3.4'
|
165
|
+
- '3.3'
|
166
|
+
- '3.2'
|
167
|
+
- '3.1'
|
168
|
+
include:
|
169
|
+
- ruby: '3.4'
|
170
|
+
coverage: 'true'
|
171
|
+
steps:
|
172
|
+
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
173
|
+
with:
|
174
|
+
fetch-depth: 0
|
175
|
+
- name: Download precompiled library
|
176
|
+
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
|
177
|
+
with:
|
178
|
+
name: librdkafka-precompiled-musl
|
179
|
+
path: ext/
|
180
|
+
- name: Start Kafka with Docker Compose
|
181
|
+
run: |
|
182
|
+
docker compose up -d
|
183
|
+
echo "Waiting for Kafka to be ready..."
|
184
|
+
sleep 10
|
185
|
+
|
186
|
+
for i in {1..30}; do
|
187
|
+
if docker compose exec -T kafka kafka-topics --bootstrap-server localhost:9092 --list >/dev/null 2>&1; then
|
188
|
+
echo "Kafka topics command succeeded!"
|
189
|
+
break
|
190
|
+
fi
|
191
|
+
sleep 2
|
192
|
+
done
|
193
|
+
- name: Run specs with precompiled library
|
194
|
+
env:
|
195
|
+
GITHUB_COVERAGE: ${{ matrix.coverage }}
|
196
|
+
RDKAFKA_EXT_PATH: ${{ github.workspace }}/ext
|
197
|
+
run: |
|
198
|
+
docker run --rm \
|
199
|
+
--network host \
|
200
|
+
-v "${{ github.workspace }}:/workspace" \
|
201
|
+
-w /workspace \
|
202
|
+
-e "GITHUB_COVERAGE=${{ matrix.coverage }}" \
|
203
|
+
-e "RDKAFKA_EXT_PATH=/workspace/ext" \
|
204
|
+
ruby:${{ matrix.ruby }}-alpine \
|
205
|
+
sh -c 'apk add --no-cache git build-base linux-headers bash && git config --global --add safe.directory /workspace && bundle config set --local path vendor/bundle && bundle install && bundle exec ruby -S rspec'
|
@@ -1,7 +1,9 @@
|
|
1
1
|
name: CI macOS ARM64
|
2
|
+
|
2
3
|
concurrency:
|
3
4
|
group: ${{ github.workflow }}-${{ github.ref }}
|
4
5
|
cancel-in-progress: true
|
6
|
+
|
5
7
|
on:
|
6
8
|
pull_request:
|
7
9
|
branches: [ main, master ]
|
@@ -9,13 +11,16 @@ on:
|
|
9
11
|
branches: [ main, master ]
|
10
12
|
schedule:
|
11
13
|
- cron: '0 1 * * *'
|
14
|
+
|
12
15
|
permissions:
|
13
16
|
contents: read
|
17
|
+
|
14
18
|
env:
|
15
19
|
BUNDLE_RETRY: 6
|
16
20
|
BUNDLE_JOBS: 4
|
17
21
|
# Renovate can track and update this version
|
18
22
|
CONFLUENT_VERSION: "8.0.0"
|
23
|
+
|
19
24
|
jobs:
|
20
25
|
build_install:
|
21
26
|
timeout-minutes: 30
|
@@ -1,13 +1,17 @@
|
|
1
1
|
name: Push Linux x86_64 GNU Platform Gem
|
2
|
+
|
2
3
|
on:
|
3
4
|
push:
|
4
5
|
tags:
|
5
6
|
- v*
|
7
|
+
|
6
8
|
permissions:
|
7
9
|
contents: read
|
10
|
+
|
8
11
|
env:
|
9
12
|
BUNDLE_RETRY: 6
|
10
13
|
BUNDLE_JOBS: 4
|
14
|
+
|
11
15
|
jobs:
|
12
16
|
push:
|
13
17
|
if: github.repository_owner == 'karafka'
|
@@ -51,7 +55,7 @@ jobs:
|
|
51
55
|
cd ext
|
52
56
|
./build_linux_x86_64_gnu.sh
|
53
57
|
- name: Configure trusted publishing credentials
|
54
|
-
uses: rubygems/configure-rubygems-credentials@v1.0.0
|
58
|
+
uses: rubygems/configure-rubygems-credentials@bc6dd217f8a4f919d6835fcfefd470ef821f5c44 # v1.0.0
|
55
59
|
- name: Build and push platform-specific gem
|
56
60
|
run: |
|
57
61
|
gem build *.gemspec
|
@@ -0,0 +1,77 @@
|
|
1
|
+
name: Push Linux x86_64 musl Platform Gem
|
2
|
+
on:
|
3
|
+
push:
|
4
|
+
tags:
|
5
|
+
- v*
|
6
|
+
permissions:
|
7
|
+
contents: read
|
8
|
+
env:
|
9
|
+
BUNDLE_RETRY: 6
|
10
|
+
BUNDLE_JOBS: 4
|
11
|
+
jobs:
|
12
|
+
build:
|
13
|
+
if: github.repository_owner == 'karafka'
|
14
|
+
timeout-minutes: 30
|
15
|
+
runs-on: ubuntu-latest
|
16
|
+
environment: deployment
|
17
|
+
container:
|
18
|
+
image: alpine:3.22@sha256:8a1f59ffb675680d47db6337b49d22281a139e9d709335b492be023728e11715
|
19
|
+
steps:
|
20
|
+
- name: Install dependencies
|
21
|
+
run: |
|
22
|
+
apk add --no-cache git curl ca-certificates build-base linux-headers \
|
23
|
+
pkgconf perl autoconf automake libtool bison flex file bash wget zstd-dev \
|
24
|
+
openssl-dev cyrus-sasl-dev cyrus-sasl cyrus-sasl-login \
|
25
|
+
cyrus-sasl-crammd5 cyrus-sasl-digestmd5 cyrus-sasl-gssapiv2 cyrus-sasl-scram \
|
26
|
+
krb5-libs openssl zlib zlib-dev zstd-libs
|
27
|
+
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
28
|
+
with:
|
29
|
+
fetch-depth: 0
|
30
|
+
- name: Configure git safe directory
|
31
|
+
run: git config --global --add safe.directory /__w/karafka-rdkafka/karafka-rdkafka
|
32
|
+
- name: Cache build-tmp directory
|
33
|
+
uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
|
34
|
+
with:
|
35
|
+
path: ext/build-tmp-musl
|
36
|
+
key: build-tmp-musl-${{ runner.os }}-${{ hashFiles('ext/*.sh', 'ext/Rakefile') }}-v2
|
37
|
+
- name: Build precompiled librdkafka.so
|
38
|
+
run: |
|
39
|
+
cd ext
|
40
|
+
bash ./build_linux_x86_64_musl.sh
|
41
|
+
- name: Upload precompiled library
|
42
|
+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
43
|
+
with:
|
44
|
+
name: librdkafka-precompiled-musl
|
45
|
+
path: ext/
|
46
|
+
retention-days: 1
|
47
|
+
push:
|
48
|
+
if: github.repository_owner == 'karafka'
|
49
|
+
timeout-minutes: 30
|
50
|
+
runs-on: ubuntu-latest
|
51
|
+
needs: build
|
52
|
+
environment: deployment
|
53
|
+
permissions:
|
54
|
+
contents: write
|
55
|
+
id-token: write
|
56
|
+
steps:
|
57
|
+
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
58
|
+
with:
|
59
|
+
fetch-depth: 0
|
60
|
+
- name: Download precompiled library
|
61
|
+
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
|
62
|
+
with:
|
63
|
+
name: librdkafka-precompiled-musl
|
64
|
+
path: ext/
|
65
|
+
- name: Set up Ruby
|
66
|
+
uses: ruby/setup-ruby@a4effe49ee8ee5b8b5091268c473a4628afb5651 # v1.245.0
|
67
|
+
with:
|
68
|
+
ruby-version: '3.4'
|
69
|
+
bundler-cache: false
|
70
|
+
- name: Configure trusted publishing credentials
|
71
|
+
uses: rubygems/configure-rubygems-credentials@bc6dd217f8a4f919d6835fcfefd470ef821f5c44 # v1.0.0
|
72
|
+
- name: Build and push platform-specific gem
|
73
|
+
run: |
|
74
|
+
gem build *.gemspec
|
75
|
+
gem push *.gem
|
76
|
+
env:
|
77
|
+
RUBY_PLATFORM: 'x86_64-linux-musl'
|
@@ -0,0 +1,54 @@
|
|
1
|
+
name: Push macOS ARM64 Platform Gem
|
2
|
+
|
3
|
+
on:
|
4
|
+
push:
|
5
|
+
tags:
|
6
|
+
- v*
|
7
|
+
|
8
|
+
permissions:
|
9
|
+
contents: read
|
10
|
+
|
11
|
+
env:
|
12
|
+
BUNDLE_RETRY: 6
|
13
|
+
BUNDLE_JOBS: 4
|
14
|
+
|
15
|
+
jobs:
|
16
|
+
push:
|
17
|
+
if: github.repository_owner == 'karafka'
|
18
|
+
timeout-minutes: 30
|
19
|
+
runs-on: macos-latest
|
20
|
+
environment: deployment
|
21
|
+
permissions:
|
22
|
+
contents: write
|
23
|
+
id-token: write
|
24
|
+
steps:
|
25
|
+
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
26
|
+
with:
|
27
|
+
fetch-depth: 0
|
28
|
+
- name: Install Bash 4+ and Kerberos
|
29
|
+
run: |
|
30
|
+
brew install bash
|
31
|
+
brew list krb5 &>/dev/null || brew install krb5
|
32
|
+
echo "/opt/homebrew/bin" >> $GITHUB_PATH
|
33
|
+
- name: Cache build-tmp directory
|
34
|
+
uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
|
35
|
+
with:
|
36
|
+
path: ext/build-tmp-macos
|
37
|
+
key: build-tmp-${{ runner.os }}-${{ hashFiles('ext/*.sh', 'ext/Rakefile') }}-v2
|
38
|
+
- name: Set up Ruby
|
39
|
+
uses: ruby/setup-ruby@a4effe49ee8ee5b8b5091268c473a4628afb5651 # v1.245.0
|
40
|
+
with:
|
41
|
+
ruby-version: '3.4'
|
42
|
+
bundler-cache: false
|
43
|
+
- name: Build precompiled librdkafka for macOS ARM64
|
44
|
+
run: |
|
45
|
+
cd ext
|
46
|
+
/opt/homebrew/bin/bash ./build_macos_arm64.sh
|
47
|
+
- name: Configure trusted publishing credentials
|
48
|
+
uses: rubygems/configure-rubygems-credentials@bc6dd217f8a4f919d6835fcfefd470ef821f5c44 # v1.0.0
|
49
|
+
- name: Build and push platform-specific gem
|
50
|
+
run: |
|
51
|
+
gem build *.gemspec
|
52
|
+
gem push *.gem
|
53
|
+
env:
|
54
|
+
RUBY_PLATFORM: 'arm64-darwin'
|
data/CHANGELOG.md
CHANGED
@@ -2,7 +2,12 @@
|
|
2
2
|
|
3
3
|
## 0.20.0 (Unreleased)
|
4
4
|
- **[Feature]** Add precompiled `x86_64-linux-gnu` setup.
|
5
|
+
- **[Feature]** Add precompiled `x86_64-linux-musl` setup.
|
5
6
|
- **[Feature]** Add precompiled `macos_arm64` setup.
|
7
|
+
- [Enhancement] Run all specs on each of the platforms with and without precompilation.
|
8
|
+
- [Fix] Fix a case where using empty key on the `musl` architecture would cause a segfault.
|
9
|
+
|
10
|
+
**Note**: Precompiled extensions are a new feature in this release. While they significantly improve installation speed and reduce build dependencies, they should be thoroughly tested in your staging environment before deploying to production. If you encounter any issues with precompiled extensions, you can fall back to building from sources.
|
6
11
|
|
7
12
|
## 0.19.5 (2025-05-30)
|
8
13
|
- [Enhancement] Allow for producing to non-existing topics with `key` and `partition_key` present.
|
@@ -0,0 +1,763 @@
|
|
1
|
+
#!/usr/bin/env bash
|
2
|
+
#
|
3
|
+
# Build self-contained librdkafka.so for Linux x86_64 musl with checksum verification
|
4
|
+
# Usage: ./build-librdkafka-linux-musl.sh
|
5
|
+
#
|
6
|
+
# Expected directory structure:
|
7
|
+
# ext/build_linux_x86_64_musl.sh (this script)
|
8
|
+
# ext/build_common.sh (shared functions)
|
9
|
+
# dist/librdkafka-*.tar.gz (librdkafka source tarball)
|
10
|
+
# dist/patches/*.patch (optional Ruby-specific patches)
|
11
|
+
#
|
12
|
+
set -euo pipefail
|
13
|
+
|
14
|
+
# Source common functions and constants
|
15
|
+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
16
|
+
source "$SCRIPT_DIR/build_common.sh"
|
17
|
+
|
18
|
+
# Override secure_download for musl/Alpine compatibility
|
19
|
+
secure_download() {
|
20
|
+
local url="$1"
|
21
|
+
local filename="$2"
|
22
|
+
|
23
|
+
if [ -f "$filename" ]; then
|
24
|
+
log "File $filename already exists, verifying checksum..."
|
25
|
+
verify_checksum "$filename"
|
26
|
+
return 0
|
27
|
+
fi
|
28
|
+
|
29
|
+
log "Downloading $filename from $url..."
|
30
|
+
|
31
|
+
# Use platform-appropriate download command
|
32
|
+
if command -v wget &> /dev/null; then
|
33
|
+
# Check if we have GNU wget or BusyBox wget
|
34
|
+
if wget --help 2>&1 | grep -q "secure-protocol"; then
|
35
|
+
# GNU wget - use security options
|
36
|
+
if ! wget --secure-protocol=TLSv1_2 \
|
37
|
+
--https-only \
|
38
|
+
--timeout=30 \
|
39
|
+
--tries=3 \
|
40
|
+
--progress=bar \
|
41
|
+
"$url" \
|
42
|
+
-O "$filename"; then
|
43
|
+
error "Failed to download $filename from $url"
|
44
|
+
fi
|
45
|
+
else
|
46
|
+
# BusyBox wget (Alpine) - use basic options
|
47
|
+
if ! wget -T 30 \
|
48
|
+
-t 3 \
|
49
|
+
-q \
|
50
|
+
"$url" \
|
51
|
+
-O "$filename"; then
|
52
|
+
error "Failed to download $filename from $url"
|
53
|
+
fi
|
54
|
+
fi
|
55
|
+
elif command -v curl &> /dev/null; then
|
56
|
+
# curl with security options
|
57
|
+
if ! curl -L \
|
58
|
+
--tlsv1.2 \
|
59
|
+
--connect-timeout 30 \
|
60
|
+
--max-time 300 \
|
61
|
+
--retry 3 \
|
62
|
+
--progress-bar \
|
63
|
+
"$url" \
|
64
|
+
-o "$filename"; then
|
65
|
+
error "Failed to download $filename from $url"
|
66
|
+
fi
|
67
|
+
else
|
68
|
+
error "No download utility found (tried wget, curl)"
|
69
|
+
fi
|
70
|
+
|
71
|
+
# Verify checksum immediately after download
|
72
|
+
verify_checksum "$filename"
|
73
|
+
}
|
74
|
+
|
75
|
+
# Platform-specific paths
|
76
|
+
DIST_DIR="$SCRIPT_DIR/../dist"
|
77
|
+
PATCHES_DIR="$DIST_DIR/patches"
|
78
|
+
BUILD_DIR="$(pwd)/build-tmp-musl"
|
79
|
+
DEPS_PREFIX="/tmp/musl-deps"
|
80
|
+
|
81
|
+
# musl-specific dependency check
|
82
|
+
check_musl_dependencies() {
|
83
|
+
log "Checking musl-specific build dependencies..."
|
84
|
+
|
85
|
+
# Check if we're actually on musl
|
86
|
+
if command -v ldd &> /dev/null && ldd --version 2>&1 | grep -q musl; then
|
87
|
+
log "✅ musl libc detected"
|
88
|
+
elif [ -f /lib/ld-musl-x86_64.so.1 ]; then
|
89
|
+
log "✅ musl detected via loader presence"
|
90
|
+
else
|
91
|
+
warn "Not clearly detected as musl system. This script is optimized for musl-based distributions (Alpine, etc.)"
|
92
|
+
warn "Continuing anyway, but glibc-specific optimizations may be used instead..."
|
93
|
+
fi
|
94
|
+
|
95
|
+
# Check for essential build tools
|
96
|
+
local missing_tools=()
|
97
|
+
|
98
|
+
command -v gcc &> /dev/null || missing_tools+=("gcc")
|
99
|
+
command -v perl &> /dev/null || missing_tools+=("perl")
|
100
|
+
command -v make &> /dev/null || missing_tools+=("make")
|
101
|
+
command -v autoconf &> /dev/null || missing_tools+=("autoconf")
|
102
|
+
command -v automake &> /dev/null || missing_tools+=("automake")
|
103
|
+
command -v libtool &> /dev/null || missing_tools+=("libtool")
|
104
|
+
command -v bison &> /dev/null || missing_tools+=("bison")
|
105
|
+
command -v flex &> /dev/null || missing_tools+=("flex")
|
106
|
+
|
107
|
+
# Check for system libraries that dependencies expect
|
108
|
+
if [ ! -f /usr/include/sys/types.h ]; then
|
109
|
+
missing_tools+=("musl-dev (for system headers)")
|
110
|
+
fi
|
111
|
+
|
112
|
+
# On Alpine, these packages provide the tools:
|
113
|
+
# build-base: gcc, make, musl-dev, libc-dev, etc.
|
114
|
+
# linux-headers: kernel headers for system calls
|
115
|
+
# pkgconf: pkg-config for dependency detection
|
116
|
+
# perl: required by OpenSSL and other build systems
|
117
|
+
# autoconf, automake, libtool: GNU autotools for configure scripts
|
118
|
+
# bison, flex: parser generators (needed by some dependencies)
|
119
|
+
# file: for file type detection
|
120
|
+
|
121
|
+
if [ ${#missing_tools[@]} -gt 0 ]; then
|
122
|
+
error "Missing required tools: ${missing_tools[*]}
|
123
|
+
|
124
|
+
On Alpine Linux, install ALL required packages with:
|
125
|
+
apk add build-base linux-headers pkgconf perl \\
|
126
|
+
autoconf automake libtool bison flex file
|
127
|
+
|
128
|
+
Or install individually as needed."
|
129
|
+
fi
|
130
|
+
|
131
|
+
log "✅ All required musl build tools found"
|
132
|
+
}
|
133
|
+
|
134
|
+
# musl-specific compiler setup
|
135
|
+
setup_musl_compiler() {
|
136
|
+
# musl-specific compiler flags
|
137
|
+
export CC="gcc"
|
138
|
+
export CFLAGS="-fPIC -O2 -static-libgcc"
|
139
|
+
export CXXFLAGS="-fPIC -O2 -static-libgcc"
|
140
|
+
export CPPFLAGS=""
|
141
|
+
export LDFLAGS="-static-libgcc"
|
142
|
+
|
143
|
+
log "Applied musl-specific compiler flags"
|
144
|
+
}
|
145
|
+
|
146
|
+
# Build OpenSSL for musl
|
147
|
+
build_openssl_musl() {
|
148
|
+
local openssl_prefix="$1"
|
149
|
+
local openssl_dir="$2"
|
150
|
+
|
151
|
+
cd "$openssl_dir"
|
152
|
+
|
153
|
+
# Check both lib and lib64 directories for existing installation
|
154
|
+
if [ ! -f "$openssl_prefix/lib/libssl.a" ] && [ ! -f "$openssl_prefix/lib64/libssl.a" ]; then
|
155
|
+
log "Configuring and building OpenSSL for musl..."
|
156
|
+
make clean 2>/dev/null || true
|
157
|
+
|
158
|
+
setup_musl_compiler
|
159
|
+
|
160
|
+
# Create the prefix directory first
|
161
|
+
mkdir -p "$openssl_prefix"
|
162
|
+
|
163
|
+
# Configure OpenSSL for musl (linux-x86_64 works fine)
|
164
|
+
./Configure linux-x86_64 \
|
165
|
+
no-shared \
|
166
|
+
no-dso \
|
167
|
+
no-engine \
|
168
|
+
--prefix="$openssl_prefix" \
|
169
|
+
--openssldir="$openssl_prefix/ssl"
|
170
|
+
|
171
|
+
make -j$(get_cpu_count)
|
172
|
+
|
173
|
+
# Try the install and capture any errors
|
174
|
+
log "Installing OpenSSL..."
|
175
|
+
if ! make install_sw install_ssldirs 2>&1; then
|
176
|
+
warn "install_sw failed, trying full install..."
|
177
|
+
make install || {
|
178
|
+
# If install fails, check if libraries were actually built
|
179
|
+
if [ -f "libssl.a" ] && [ -f "libcrypto.a" ]; then
|
180
|
+
log "Install failed but libraries exist, copying manually..."
|
181
|
+
mkdir -p "$openssl_prefix/lib" "$openssl_prefix/include"
|
182
|
+
cp libssl.a libcrypto.a "$openssl_prefix/lib/"
|
183
|
+
cp -r include/openssl "$openssl_prefix/include/" 2>/dev/null || true
|
184
|
+
else
|
185
|
+
error "OpenSSL build failed - no libraries found"
|
186
|
+
fi
|
187
|
+
}
|
188
|
+
fi
|
189
|
+
|
190
|
+
# Verify the build - check both possible lib locations
|
191
|
+
if [ -f "$openssl_prefix/lib/libssl.a" ] || [ -f "$openssl_prefix/lib64/libssl.a" ]; then
|
192
|
+
if [ -f "$openssl_prefix/lib64/libssl.a" ]; then
|
193
|
+
log "OpenSSL installed to lib64 directory"
|
194
|
+
# Create symlinks in lib for consistency
|
195
|
+
mkdir -p "$openssl_prefix/lib"
|
196
|
+
ln -sf ../lib64/libssl.a "$openssl_prefix/lib/libssl.a" 2>/dev/null || true
|
197
|
+
ln -sf ../lib64/libcrypto.a "$openssl_prefix/lib/libcrypto.a" 2>/dev/null || true
|
198
|
+
ln -sf ../lib64/pkgconfig "$openssl_prefix/lib/pkgconfig" 2>/dev/null || true
|
199
|
+
fi
|
200
|
+
log "✅ Static OpenSSL built successfully"
|
201
|
+
else
|
202
|
+
error "Failed to build static OpenSSL - libraries not found in lib or lib64"
|
203
|
+
fi
|
204
|
+
else
|
205
|
+
log "Static OpenSSL already built, skipping..."
|
206
|
+
fi
|
207
|
+
}
|
208
|
+
|
209
|
+
# Build MIT Kerberos for musl
|
210
|
+
build_krb5_musl() {
|
211
|
+
local krb5_prefix="$1"
|
212
|
+
local krb5_dir="$2"
|
213
|
+
|
214
|
+
cd "$krb5_dir/src"
|
215
|
+
|
216
|
+
if [ ! -f "$krb5_prefix/lib/libgssapi_krb5.a" ]; then
|
217
|
+
log "Configuring and building MIT Kerberos for musl..."
|
218
|
+
make clean 2>/dev/null || true
|
219
|
+
|
220
|
+
setup_musl_compiler
|
221
|
+
|
222
|
+
# musl-specific configuration for Kerberos
|
223
|
+
# Disable some features that can be problematic on musl
|
224
|
+
./configure \
|
225
|
+
--disable-shared \
|
226
|
+
--enable-static \
|
227
|
+
--prefix="$krb5_prefix" \
|
228
|
+
--without-ldap \
|
229
|
+
--without-tcl \
|
230
|
+
--without-keyutils \
|
231
|
+
--disable-rpath \
|
232
|
+
--without-system-verto \
|
233
|
+
--disable-thread-support \
|
234
|
+
--disable-dns-for-realm \
|
235
|
+
--without-readline
|
236
|
+
|
237
|
+
# Build with limited parallelism to avoid musl-specific race conditions
|
238
|
+
log "Building Kerberos (will ignore kadmin build failures)..."
|
239
|
+
make -j$(( $(get_cpu_count) / 2 )) || {
|
240
|
+
log "Full build failed (expected due to kadmin), continuing with libraries..."
|
241
|
+
true
|
242
|
+
}
|
243
|
+
|
244
|
+
# Install what was successfully built
|
245
|
+
make install || {
|
246
|
+
log "Full install failed, installing individual components..."
|
247
|
+
make install-mkdirs 2>/dev/null || true
|
248
|
+
make -C util install 2>/dev/null || true
|
249
|
+
make -C lib install 2>/dev/null || true
|
250
|
+
make -C plugins/kdb/db2 install 2>/dev/null || true
|
251
|
+
}
|
252
|
+
|
253
|
+
# Verify we got the essential libraries
|
254
|
+
if [ ! -f "$krb5_prefix/lib/libgssapi_krb5.a" ]; then
|
255
|
+
error "Failed to build essential Kerberos libraries"
|
256
|
+
fi
|
257
|
+
|
258
|
+
log "✅ Static MIT Kerberos built successfully"
|
259
|
+
else
|
260
|
+
log "Static MIT Kerberos already built, skipping..."
|
261
|
+
fi
|
262
|
+
}
|
263
|
+
|
264
|
+
# Build Cyrus SASL for musl with proper OpenSSL linking
|
265
|
+
build_sasl_musl() {
|
266
|
+
local sasl_prefix="$1"
|
267
|
+
local sasl_dir="$2"
|
268
|
+
local krb5_prefix="$3"
|
269
|
+
local openssl_prefix="$4" # Add OpenSSL prefix parameter
|
270
|
+
|
271
|
+
cd "$sasl_dir"
|
272
|
+
|
273
|
+
if [ ! -f "$sasl_prefix/lib/libsasl2.a" ]; then
|
274
|
+
log "Configuring and building Cyrus SASL for musl..."
|
275
|
+
make clean 2>/dev/null || true
|
276
|
+
|
277
|
+
# Apply comprehensive patches for missing time.h includes
|
278
|
+
log "Applying comprehensive time.h patches..."
|
279
|
+
|
280
|
+
# Create backups first
|
281
|
+
cp lib/saslutil.c lib/saslutil.c.backup 2>/dev/null || true
|
282
|
+
cp plugins/cram.c plugins/cram.c.backup 2>/dev/null || true
|
283
|
+
|
284
|
+
# Use perl for more reliable in-place editing
|
285
|
+
log "Patching lib/saslutil.c..."
|
286
|
+
perl -i -pe 's/^#include "saslint\.h"$/#include "saslint.h"\n#include <time.h>/' lib/saslutil.c
|
287
|
+
|
288
|
+
log "Patching plugins/cram.c..."
|
289
|
+
perl -i -pe 's/^#include "plugin_common\.h"$/#include "plugin_common.h"\n#include <time.h>/' plugins/cram.c
|
290
|
+
|
291
|
+
# Verify patches applied
|
292
|
+
if ! grep -q "#include <time.h>" lib/saslutil.c; then
|
293
|
+
log "Perl approach failed, using awk fallback for saslutil.c..."
|
294
|
+
awk '/^#include "saslint\.h"$/ {print; print "#include <time.h>"; next} {print}' lib/saslutil.c.backup > lib/saslutil.c
|
295
|
+
fi
|
296
|
+
|
297
|
+
if ! grep -q "#include <time.h>" plugins/cram.c; then
|
298
|
+
log "Perl approach failed, using awk fallback for cram.c..."
|
299
|
+
awk '/^#include "plugin_common\.h"$/ {print; print "#include <time.h>"; next} {print}' plugins/cram.c.backup > plugins/cram.c
|
300
|
+
fi
|
301
|
+
|
302
|
+
# Clean up backup files
|
303
|
+
rm -f lib/saslutil.c.backup plugins/cram.c.backup
|
304
|
+
|
305
|
+
# Verify patches were applied
|
306
|
+
if ! grep -q "#include <time.h>" lib/saslutil.c || ! grep -q "#include <time.h>" plugins/cram.c; then
|
307
|
+
error "Failed to patch time.h includes - this is required for compilation"
|
308
|
+
fi
|
309
|
+
|
310
|
+
log "✅ time.h patches applied successfully"
|
311
|
+
|
312
|
+
# Determine correct OpenSSL lib directory
|
313
|
+
local openssl_lib_dir
|
314
|
+
if [ -f "$openssl_prefix/lib64/libssl.a" ]; then
|
315
|
+
openssl_lib_dir="$openssl_prefix/lib64"
|
316
|
+
log "Using OpenSSL libraries from lib64"
|
317
|
+
else
|
318
|
+
openssl_lib_dir="$openssl_prefix/lib"
|
319
|
+
log "Using OpenSSL libraries from lib"
|
320
|
+
fi
|
321
|
+
|
322
|
+
# Verify OpenSSL libraries exist
|
323
|
+
if [ ! -f "$openssl_lib_dir/libssl.a" ] || [ ! -f "$openssl_lib_dir/libcrypto.a" ]; then
|
324
|
+
error "OpenSSL static libraries not found in $openssl_lib_dir"
|
325
|
+
fi
|
326
|
+
|
327
|
+
setup_musl_compiler
|
328
|
+
|
329
|
+
# Set comprehensive OpenSSL and Kerberos flags
|
330
|
+
export CPPFLAGS="$CPPFLAGS -I$krb5_prefix/include -I$openssl_prefix/include"
|
331
|
+
export LDFLAGS="$LDFLAGS -L$krb5_prefix/lib -L$openssl_lib_dir"
|
332
|
+
export LIBS="-lssl -lcrypto -ldl -lpthread"
|
333
|
+
|
334
|
+
# Also set PKG_CONFIG_PATH for better library detection
|
335
|
+
export PKG_CONFIG_PATH="$openssl_lib_dir/pkgconfig:$krb5_prefix/lib/pkgconfig:${PKG_CONFIG_PATH:-}"
|
336
|
+
|
337
|
+
log "Environment variables set:"
|
338
|
+
log " CPPFLAGS: $CPPFLAGS"
|
339
|
+
log " LDFLAGS: $LDFLAGS"
|
340
|
+
log " LIBS: $LIBS"
|
341
|
+
log " PKG_CONFIG_PATH: $PKG_CONFIG_PATH"
|
342
|
+
|
343
|
+
# Configure SASL with explicit OpenSSL paths and comprehensive options
|
344
|
+
./configure \
|
345
|
+
--disable-shared \
|
346
|
+
--enable-static \
|
347
|
+
--prefix="$sasl_prefix" \
|
348
|
+
--with-openssl="$openssl_prefix" \
|
349
|
+
--with-ssl="$openssl_prefix" \
|
350
|
+
--enable-gssapi="$krb5_prefix" \
|
351
|
+
--without-dblib \
|
352
|
+
--disable-gdbm \
|
353
|
+
--disable-sample \
|
354
|
+
--disable-obsolete_cram_attr \
|
355
|
+
--disable-obsolete_digest_attr \
|
356
|
+
--without-pam \
|
357
|
+
--without-saslauthd \
|
358
|
+
--without-pwcheck \
|
359
|
+
--without-des \
|
360
|
+
--without-authdaemond \
|
361
|
+
--disable-java \
|
362
|
+
--disable-sql \
|
363
|
+
--disable-ldapdb \
|
364
|
+
--enable-plain \
|
365
|
+
--enable-login \
|
366
|
+
--enable-digest \
|
367
|
+
--enable-cram \
|
368
|
+
--enable-otp \
|
369
|
+
--enable-ntlm \
|
370
|
+
--enable-scram \
|
371
|
+
ac_cv_func_getnameinfo=yes \
|
372
|
+
ac_cv_func_getaddrinfo=yes \
|
373
|
+
OPENSSL_CFLAGS="-I$openssl_prefix/include" \
|
374
|
+
OPENSSL_LIBS="-L$openssl_lib_dir -lssl -lcrypto"
|
375
|
+
|
376
|
+
log "Configuration completed, starting build..."
|
377
|
+
|
378
|
+
# Build with reduced parallelism to avoid race conditions
|
379
|
+
make -j$(( $(get_cpu_count) / 2 )) || {
|
380
|
+
log "Build failed, trying single-threaded build..."
|
381
|
+
make clean
|
382
|
+
make -j1
|
383
|
+
}
|
384
|
+
|
385
|
+
# Install
|
386
|
+
make install
|
387
|
+
|
388
|
+
# Verify the build
|
389
|
+
if [ ! -f "$sasl_prefix/lib/libsasl2.a" ]; then
|
390
|
+
error "Failed to build static Cyrus SASL"
|
391
|
+
fi
|
392
|
+
|
393
|
+
# Additional verification - check if OTP plugin was built (this was failing)
|
394
|
+
if [ -f "$sasl_prefix/lib/sasl2/libotp.a" ] || [ -f "$sasl_prefix/lib/sasl2/libotp.la" ]; then
|
395
|
+
log "✅ OTP plugin built successfully"
|
396
|
+
else
|
397
|
+
log "⚠️ OTP plugin may not have built, but core SASL library is available"
|
398
|
+
fi
|
399
|
+
|
400
|
+
log "✅ Static Cyrus SASL built successfully"
|
401
|
+
else
|
402
|
+
log "Static Cyrus SASL already built, skipping..."
|
403
|
+
fi
|
404
|
+
}
|
405
|
+
|
406
|
+
# Build static library with musl-specific configuration
|
407
|
+
build_static_lib_musl() {
|
408
|
+
local lib_name="$1"
|
409
|
+
local prefix="$2"
|
410
|
+
local source_dir="$3"
|
411
|
+
local configure_args="$4"
|
412
|
+
|
413
|
+
cd "$source_dir"
|
414
|
+
|
415
|
+
local lib_file="$prefix/lib/lib${lib_name}.a"
|
416
|
+
if [ ! -f "$lib_file" ]; then
|
417
|
+
log "Configuring and building static $lib_name for musl..."
|
418
|
+
make clean 2>/dev/null || true
|
419
|
+
|
420
|
+
setup_musl_compiler
|
421
|
+
|
422
|
+
# Run configure with provided arguments
|
423
|
+
eval "./configure --prefix=\"$prefix\" $configure_args"
|
424
|
+
|
425
|
+
make -j$(get_cpu_count)
|
426
|
+
make install
|
427
|
+
|
428
|
+
# Verify the build
|
429
|
+
if [ ! -f "$lib_file" ]; then
|
430
|
+
error "Failed to build static $lib_name"
|
431
|
+
fi
|
432
|
+
|
433
|
+
log "✅ Static $lib_name built successfully"
|
434
|
+
else
|
435
|
+
log "Static $lib_name already built, skipping..."
|
436
|
+
fi
|
437
|
+
}
|
438
|
+
|
439
|
+
# Check common and musl-specific dependencies
|
440
|
+
check_common_dependencies
|
441
|
+
check_musl_dependencies
|
442
|
+
|
443
|
+
# Auto-detect librdkafka tarball
|
444
|
+
log "Looking for librdkafka tarball in $DIST_DIR..."
|
445
|
+
LIBRDKAFKA_TARBALL=$(find_librdkafka_tarball "$DIST_DIR")
|
446
|
+
log "Found librdkafka tarball: $LIBRDKAFKA_TARBALL"
|
447
|
+
|
448
|
+
# Verify librdkafka tarball checksum if available
|
449
|
+
verify_librdkafka_checksum "$LIBRDKAFKA_TARBALL"
|
450
|
+
|
451
|
+
# Find patches
|
452
|
+
PATCHES_FOUND=()
|
453
|
+
find_patches "$PATCHES_DIR" PATCHES_FOUND
|
454
|
+
|
455
|
+
security_log "Starting secure build with checksum verification enabled"
|
456
|
+
log "Building self-contained librdkafka.so for Linux x86_64 musl (WITH SASL)"
|
457
|
+
log "Dependencies to build:"
|
458
|
+
log " - OpenSSL: $OPENSSL_VERSION"
|
459
|
+
log " - Cyrus SASL: $CYRUS_SASL_VERSION (with official GCC 14+ patches)"
|
460
|
+
log " - MIT Kerberos: $KRB5_VERSION"
|
461
|
+
log " - zlib: $ZLIB_VERSION"
|
462
|
+
log " - ZStd: $ZSTD_VERSION"
|
463
|
+
log "librdkafka source: $LIBRDKAFKA_TARBALL"
|
464
|
+
log "Build directory: $BUILD_DIR"
|
465
|
+
|
466
|
+
# Create build directory
|
467
|
+
mkdir -p "$BUILD_DIR"
|
468
|
+
cd "$BUILD_DIR"
|
469
|
+
|
470
|
+
# Build OpenSSL
|
471
|
+
log "Building OpenSSL $OPENSSL_VERSION for musl..."
|
472
|
+
OPENSSL_PREFIX="$DEPS_PREFIX/static-openssl-$OPENSSL_VERSION"
|
473
|
+
OPENSSL_TARBALL="openssl-$OPENSSL_VERSION.tar.gz"
|
474
|
+
OPENSSL_DIR="openssl-$OPENSSL_VERSION"
|
475
|
+
|
476
|
+
secure_download "$(get_openssl_url)" "$OPENSSL_TARBALL"
|
477
|
+
extract_if_needed "$OPENSSL_TARBALL" "$OPENSSL_DIR"
|
478
|
+
build_openssl_musl "$OPENSSL_PREFIX" "$OPENSSL_DIR"
|
479
|
+
|
480
|
+
cd "$BUILD_DIR"
|
481
|
+
|
482
|
+
# Determine OpenSSL lib directory (could be lib or lib64)
|
483
|
+
if [ -f "$OPENSSL_PREFIX/lib64/libssl.a" ]; then
|
484
|
+
OPENSSL_LIB_DIR="$OPENSSL_PREFIX/lib64"
|
485
|
+
log "Using OpenSSL libraries from lib64"
|
486
|
+
else
|
487
|
+
OPENSSL_LIB_DIR="$OPENSSL_PREFIX/lib"
|
488
|
+
log "Using OpenSSL libraries from lib"
|
489
|
+
fi
|
490
|
+
|
491
|
+
# Build MIT Kerberos
|
492
|
+
log "Building MIT Kerberos $KRB5_VERSION for musl..."
|
493
|
+
KRB5_PREFIX="$DEPS_PREFIX/static-krb5-$KRB5_VERSION"
|
494
|
+
KRB5_TARBALL="krb5-$KRB5_VERSION.tar.gz"
|
495
|
+
KRB5_DIR="krb5-$KRB5_VERSION"
|
496
|
+
|
497
|
+
secure_download "$(get_krb5_url)" "$KRB5_TARBALL"
|
498
|
+
extract_if_needed "$KRB5_TARBALL" "$KRB5_DIR"
|
499
|
+
build_krb5_musl "$KRB5_PREFIX" "$KRB5_DIR"
|
500
|
+
|
501
|
+
cd "$BUILD_DIR"
|
502
|
+
|
503
|
+
# Build SASL with official patches (updated call)
|
504
|
+
log "Building Cyrus SASL $CYRUS_SASL_VERSION for musl..."
|
505
|
+
SASL_PREFIX="$DEPS_PREFIX/static-sasl-$CYRUS_SASL_VERSION"
|
506
|
+
SASL_TARBALL="cyrus-sasl-$CYRUS_SASL_VERSION.tar.gz"
|
507
|
+
SASL_DIR="cyrus-sasl-$CYRUS_SASL_VERSION"
|
508
|
+
|
509
|
+
secure_download "$(get_sasl_url)" "$SASL_TARBALL"
|
510
|
+
extract_if_needed "$SASL_TARBALL" "$SASL_DIR"
|
511
|
+
build_sasl_musl "$SASL_PREFIX" "$SASL_DIR" "$KRB5_PREFIX" "$OPENSSL_PREFIX" # Added OPENSSL_PREFIX parameter
|
512
|
+
|
513
|
+
cd "$BUILD_DIR"
|
514
|
+
|
515
|
+
# Build zlib
|
516
|
+
log "Building zlib $ZLIB_VERSION for musl..."
|
517
|
+
ZLIB_PREFIX="$DEPS_PREFIX/static-zlib-$ZLIB_VERSION"
|
518
|
+
ZLIB_TARBALL="zlib-$ZLIB_VERSION.tar.gz"
|
519
|
+
ZLIB_DIR="zlib-$ZLIB_VERSION"
|
520
|
+
|
521
|
+
secure_download "$(get_zlib_url)" "$ZLIB_TARBALL"
|
522
|
+
extract_if_needed "$ZLIB_TARBALL" "$ZLIB_DIR"
|
523
|
+
build_static_lib_musl "z" "$ZLIB_PREFIX" "$ZLIB_DIR" "--static"
|
524
|
+
|
525
|
+
cd "$BUILD_DIR"
|
526
|
+
|
527
|
+
# Build ZStd
|
528
|
+
log "Building ZStd $ZSTD_VERSION for musl..."
|
529
|
+
ZSTD_PREFIX="$DEPS_PREFIX/static-zstd-$ZSTD_VERSION"
|
530
|
+
ZSTD_TARBALL="zstd-$ZSTD_VERSION.tar.gz"
|
531
|
+
ZSTD_DIR="zstd-$ZSTD_VERSION"
|
532
|
+
|
533
|
+
secure_download "$(get_zstd_url)" "$ZSTD_TARBALL"
|
534
|
+
extract_if_needed "$ZSTD_TARBALL" "$ZSTD_DIR"
|
535
|
+
cd "$ZSTD_DIR"
|
536
|
+
|
537
|
+
if [ ! -f "$ZSTD_PREFIX/lib/libzstd.a" ]; then
|
538
|
+
log "Building ZStd for musl..."
|
539
|
+
make clean 2>/dev/null || true
|
540
|
+
|
541
|
+
setup_musl_compiler
|
542
|
+
|
543
|
+
# Build static library using ZStd's Makefile
|
544
|
+
make lib-mt CFLAGS="$CFLAGS" PREFIX="$ZSTD_PREFIX" -j$(get_cpu_count)
|
545
|
+
make install PREFIX="$ZSTD_PREFIX"
|
546
|
+
|
547
|
+
# Verify the build
|
548
|
+
if [ ! -f "$ZSTD_PREFIX/lib/libzstd.a" ]; then
|
549
|
+
error "Failed to build static ZStd"
|
550
|
+
fi
|
551
|
+
|
552
|
+
log "✅ Static ZStd built successfully"
|
553
|
+
else
|
554
|
+
log "Static ZStd already built, skipping..."
|
555
|
+
fi
|
556
|
+
|
557
|
+
cd "$BUILD_DIR"
|
558
|
+
|
559
|
+
# Extract and patch librdkafka
|
560
|
+
log "Extracting librdkafka..."
|
561
|
+
tar xzf "$LIBRDKAFKA_TARBALL"
|
562
|
+
cd "librdkafka-$LIBRDKAFKA_VERSION"
|
563
|
+
|
564
|
+
# Fix permissions and apply patches
|
565
|
+
fix_configure_permissions
|
566
|
+
apply_patches PATCHES_FOUND
|
567
|
+
|
568
|
+
# Configure librdkafka for musl (with SASL)
|
569
|
+
log "Configuring librdkafka for musl (with SASL support)..."
|
570
|
+
|
571
|
+
setup_musl_compiler
|
572
|
+
|
573
|
+
# musl-specific configuration with SASL
|
574
|
+
export CPPFLAGS="$CPPFLAGS -I$KRB5_PREFIX/include -I$SASL_PREFIX/include"
|
575
|
+
export LDFLAGS="$LDFLAGS -L$KRB5_PREFIX/lib -L$SASL_PREFIX/lib"
|
576
|
+
|
577
|
+
if [ -f configure ]; then
|
578
|
+
log "Using mklove configure script"
|
579
|
+
|
580
|
+
# mklove-specific configure options with SASL
|
581
|
+
./configure \
|
582
|
+
--enable-static \
|
583
|
+
--disable-shared \
|
584
|
+
--disable-curl \
|
585
|
+
--enable-gssapi \
|
586
|
+
--enable-sasl \
|
587
|
+
--disable-c11threads
|
588
|
+
else
|
589
|
+
error "No configure script found"
|
590
|
+
fi
|
591
|
+
|
592
|
+
# Build librdkafka
|
593
|
+
log "Compiling librdkafka..."
|
594
|
+
make clean || true
|
595
|
+
make -j$(get_cpu_count)
|
596
|
+
|
597
|
+
# Verify librdkafka.a exists
|
598
|
+
if [ ! -f src/librdkafka.a ]; then
|
599
|
+
error "librdkafka.a not found after build"
|
600
|
+
fi
|
601
|
+
|
602
|
+
log "librdkafka.a built successfully"
|
603
|
+
|
604
|
+
# Create self-contained shared library for musl (with SASL)
|
605
|
+
log "Creating self-contained librdkafka.so for musl (with SASL support)..."
|
606
|
+
|
607
|
+
# First, verify all static libraries exist
|
608
|
+
log "Verifying static libraries exist..."
|
609
|
+
for lib_path in \
|
610
|
+
"$SASL_PREFIX/lib/libsasl2.a" \
|
611
|
+
"$KRB5_PREFIX/lib/libgssapi_krb5.a" \
|
612
|
+
"$KRB5_PREFIX/lib/libkrb5.a" \
|
613
|
+
"$KRB5_PREFIX/lib/libk5crypto.a" \
|
614
|
+
"$KRB5_PREFIX/lib/libcom_err.a" \
|
615
|
+
"$KRB5_PREFIX/lib/libkrb5support.a" \
|
616
|
+
"$OPENSSL_LIB_DIR/libssl.a" \
|
617
|
+
"$OPENSSL_LIB_DIR/libcrypto.a" \
|
618
|
+
"$ZLIB_PREFIX/lib/libz.a" \
|
619
|
+
"$ZSTD_PREFIX/lib/libzstd.a"
|
620
|
+
do
|
621
|
+
if [ ! -f "$lib_path" ]; then
|
622
|
+
error "Required static library not found: $lib_path"
|
623
|
+
else
|
624
|
+
log "✅ Found: $lib_path"
|
625
|
+
fi
|
626
|
+
done
|
627
|
+
|
628
|
+
gcc -shared -fPIC \
|
629
|
+
-Wl,--whole-archive src/librdkafka.a -Wl,--no-whole-archive \
|
630
|
+
-o librdkafka.so \
|
631
|
+
-Wl,-Bstatic \
|
632
|
+
"$SASL_PREFIX/lib/libsasl2.a" \
|
633
|
+
"$KRB5_PREFIX/lib/libgssapi_krb5.a" \
|
634
|
+
"$KRB5_PREFIX/lib/libkrb5.a" \
|
635
|
+
"$KRB5_PREFIX/lib/libk5crypto.a" \
|
636
|
+
"$KRB5_PREFIX/lib/libcom_err.a" \
|
637
|
+
"$KRB5_PREFIX/lib/libkrb5support.a" \
|
638
|
+
"$OPENSSL_LIB_DIR/libssl.a" \
|
639
|
+
"$OPENSSL_LIB_DIR/libcrypto.a" \
|
640
|
+
"$ZLIB_PREFIX/lib/libz.a" \
|
641
|
+
"$ZSTD_PREFIX/lib/libzstd.a" \
|
642
|
+
-Wl,-Bdynamic \
|
643
|
+
-lpthread -lm -ldl -lc \
|
644
|
+
-static-libgcc \
|
645
|
+
-Wl,--as-needed \
|
646
|
+
-Wl,--no-undefined
|
647
|
+
|
648
|
+
if [ ! -f librdkafka.so ]; then
|
649
|
+
error "Failed to create librdkafka.so"
|
650
|
+
fi
|
651
|
+
|
652
|
+
log "librdkafka.so created successfully"
|
653
|
+
|
654
|
+
# Enhanced verification
|
655
|
+
log "Verifying static linking of SASL..."
|
656
|
+
if command -v ldd &> /dev/null; then
|
657
|
+
SASL_DEPS=$(ldd librdkafka.so 2>/dev/null | grep -i sasl || true)
|
658
|
+
if [ -n "$SASL_DEPS" ]; then
|
659
|
+
error "SASL is still dynamically linked: $SASL_DEPS"
|
660
|
+
else
|
661
|
+
log "✅ SASL successfully statically linked - no dynamic SASL dependencies found"
|
662
|
+
fi
|
663
|
+
|
664
|
+
# Check for any other problematic dynamic dependencies
|
665
|
+
PROBLEMATIC_DEPS=$(ldd librdkafka.so 2>/dev/null | grep -E "(libssl|libcrypto|libz|libzstd|libkrb|libgssapi)" || true)
|
666
|
+
if [ -n "$PROBLEMATIC_DEPS" ]; then
|
667
|
+
warn "Found other dynamic dependencies that should be static: $PROBLEMATIC_DEPS"
|
668
|
+
else
|
669
|
+
log "✅ All major dependencies appear to be statically linked"
|
670
|
+
fi
|
671
|
+
else
|
672
|
+
log "ldd not available, skipping dynamic dependency check"
|
673
|
+
fi
|
674
|
+
|
675
|
+
# Additional verification using nm if available
|
676
|
+
if command -v nm &> /dev/null; then
|
677
|
+
log "Checking for SASL symbols in the library..."
|
678
|
+
SASL_SYMBOLS=$(nm -D librdkafka.so 2>/dev/null | grep -i sasl | head -5 || true)
|
679
|
+
if [ -n "$SASL_SYMBOLS" ]; then
|
680
|
+
log "✅ SASL symbols found in library (first 5):"
|
681
|
+
echo "$SASL_SYMBOLS"
|
682
|
+
else
|
683
|
+
warn "No SASL symbols found - this might indicate a linking issue"
|
684
|
+
fi
|
685
|
+
fi
|
686
|
+
|
687
|
+
# Verify the build (musl-compatible)
|
688
|
+
log "Verifying musl build..."
|
689
|
+
if command -v file &> /dev/null; then
|
690
|
+
file librdkafka.so
|
691
|
+
else
|
692
|
+
log "file command not available, skipping file type check"
|
693
|
+
fi
|
694
|
+
|
695
|
+
log "Checking dependencies with ldd:"
|
696
|
+
if command -v ldd &> /dev/null; then
|
697
|
+
ldd librdkafka.so
|
698
|
+
else
|
699
|
+
log "ldd not available, skipping dependency check"
|
700
|
+
fi
|
701
|
+
|
702
|
+
log "Checking for external dependencies (should only show musl and system libraries):"
|
703
|
+
if command -v ldd &> /dev/null; then
|
704
|
+
EXTERNAL_DEPS=$(ldd librdkafka.so 2>/dev/null | grep -v "musl" | grep -v "ld-musl" | grep -v "librdkafka.so" | grep -v "=>" | grep -v "statically linked" | grep -v "not a dynamic executable" || true)
|
705
|
+
if [ -n "$EXTERNAL_DEPS" ]; then
|
706
|
+
# Filter out expected system dependencies that are OK on musl
|
707
|
+
FILTERED_DEPS=$(echo "$EXTERNAL_DEPS" | grep -v "linux-vdso.so" | grep -v "/lib/ld-musl" || true)
|
708
|
+
if [ -n "$FILTERED_DEPS" ]; then
|
709
|
+
warn "Found some external dependencies (may be acceptable for musl): $FILTERED_DEPS"
|
710
|
+
else
|
711
|
+
log "✅ All dependencies are musl system libraries - build is self-contained!"
|
712
|
+
fi
|
713
|
+
else
|
714
|
+
log "✅ No unexpected external dependencies found - library is self-contained!"
|
715
|
+
fi
|
716
|
+
else
|
717
|
+
log "ldd not available, skipping external dependency check"
|
718
|
+
fi
|
719
|
+
|
720
|
+
# Check for musl compatibility
|
721
|
+
log "Verifying musl compatibility..."
|
722
|
+
if command -v ldd &> /dev/null; then
|
723
|
+
if ldd librdkafka.so 2>&1 | grep -q "musl"; then
|
724
|
+
log "✅ Library is linked against musl libc"
|
725
|
+
elif ldd librdkafka.so 2>&1 | grep -q "statically linked"; then
|
726
|
+
log "✅ Library appears to be statically linked (good for musl)"
|
727
|
+
else
|
728
|
+
warn "Library linking may not be optimal for musl systems"
|
729
|
+
fi
|
730
|
+
else
|
731
|
+
log "ldd not available, skipping musl compatibility check"
|
732
|
+
fi
|
733
|
+
|
734
|
+
# Copy to output directory
|
735
|
+
OUTPUT_DIR="$SCRIPT_DIR"
|
736
|
+
cp librdkafka.so "$OUTPUT_DIR/"
|
737
|
+
log "librdkafka.so copied to: $OUTPUT_DIR/librdkafka.so"
|
738
|
+
|
739
|
+
# Print summaries
|
740
|
+
print_security_summary
|
741
|
+
|
742
|
+
# Enhanced summary for musl (with SASL)
|
743
|
+
log "Build completed successfully!"
|
744
|
+
log "📦 Self-contained librdkafka built for Linux x86_64 musl:"
|
745
|
+
log " ✅ OpenSSL $OPENSSL_VERSION (SSL/TLS support) - checksum verified"
|
746
|
+
log " ✅ Cyrus SASL $CYRUS_SASL_VERSION (authentication for AWS MSK) - checksum verified"
|
747
|
+
log " ✅ MIT Kerberos $KRB5_VERSION (GSSAPI/Kerberos authentication) - checksum verified"
|
748
|
+
log " ✅ zlib $ZLIB_VERSION (compression) - checksum verified"
|
749
|
+
log " ✅ ZStd $ZSTD_VERSION (high-performance compression) - checksum verified"
|
750
|
+
log ""
|
751
|
+
log "🎯 Ready for deployment on musl-based systems (Alpine Linux, Docker containers)"
|
752
|
+
log "☁️ Compatible with AWS MSK and other secured Kafka clusters"
|
753
|
+
log "🔐 Supply chain security: All dependencies cryptographically verified"
|
754
|
+
log "🐳 Docker-optimized: Perfect for Alpine-based container deployments"
|
755
|
+
log "🛠️ GCC 14+ compatible: Official patches applied to Cyrus SASL"
|
756
|
+
log ""
|
757
|
+
log "Location: $OUTPUT_DIR/librdkafka.so"
|
758
|
+
|
759
|
+
# Cleanup
|
760
|
+
cleanup_build_dir "$BUILD_DIR"
|
761
|
+
|
762
|
+
# Reset environment variables
|
763
|
+
unset CFLAGS CXXFLAGS CPPFLAGS LDFLAGS CC
|
data/ext/librdkafka.so
CHANGED
Binary file
|
data/karafka-rdkafka.gemspec
CHANGED
@@ -41,7 +41,9 @@ Gem::Specification.new do |gem|
|
|
41
41
|
end
|
42
42
|
|
43
43
|
gem.add_dependency 'ffi', '~> 1.15'
|
44
|
+
gem.add_dependency 'logger'
|
44
45
|
gem.add_dependency 'mini_portile2', '~> 2.6'
|
46
|
+
gem.add_dependency 'ostruct'
|
45
47
|
gem.add_dependency 'rake', '> 12'
|
46
48
|
|
47
49
|
gem.add_development_dependency 'pry'
|
data/lib/rdkafka/bindings.rb
CHANGED
@@ -406,12 +406,15 @@ module Rdkafka
|
|
406
406
|
def self.partitioner(str, partition_count, partitioner_name = "consistent_random")
|
407
407
|
# Return RD_KAFKA_PARTITION_UA(unassigned partition) when partition count is nil/zero.
|
408
408
|
return -1 unless partition_count&.nonzero?
|
409
|
+
# musl architecture crashes with empty string
|
410
|
+
return 0 if str.empty?
|
409
411
|
|
410
|
-
str_ptr =
|
412
|
+
str_ptr = FFI::MemoryPointer.from_string(str)
|
411
413
|
method_name = PARTITIONERS.fetch(partitioner_name) do
|
412
414
|
raise Rdkafka::Config::ConfigError.new("Unknown partitioner: #{partitioner_name}")
|
413
415
|
end
|
414
|
-
|
416
|
+
|
417
|
+
public_send(method_name, nil, str_ptr, str.size, partition_count, nil, nil)
|
415
418
|
end
|
416
419
|
|
417
420
|
# Create Topics
|
data/lib/rdkafka/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: karafka-rdkafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.20.0.
|
4
|
+
version: 0.20.0.rc5
|
5
5
|
platform: x86_64-linux-gnu
|
6
6
|
authors:
|
7
7
|
- Thijs Cadier
|
@@ -24,6 +24,20 @@ dependencies:
|
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '1.15'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: logger
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
27
41
|
- !ruby/object:Gem::Dependency
|
28
42
|
name: mini_portile2
|
29
43
|
requirement: !ruby/object:Gem::Requirement
|
@@ -38,6 +52,20 @@ dependencies:
|
|
38
52
|
- - "~>"
|
39
53
|
- !ruby/object:Gem::Version
|
40
54
|
version: '2.6'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: ostruct
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
41
69
|
- !ruby/object:Gem::Dependency
|
42
70
|
name: rake
|
43
71
|
requirement: !ruby/object:Gem::Requirement
|
@@ -118,8 +146,11 @@ files:
|
|
118
146
|
- ".github/CODEOWNERS"
|
119
147
|
- ".github/FUNDING.yml"
|
120
148
|
- ".github/workflows/ci_linux_x86_64_gnu.yml"
|
149
|
+
- ".github/workflows/ci_linux_x86_64_musl.yml"
|
121
150
|
- ".github/workflows/ci_macos_arm64.yml"
|
122
151
|
- ".github/workflows/push_linux_x86_64_gnu.yml"
|
152
|
+
- ".github/workflows/push_linux_x86_64_musl.yml"
|
153
|
+
- ".github/workflows/push_macos_arm64.yml"
|
123
154
|
- ".github/workflows/push_ruby.yml"
|
124
155
|
- ".github/workflows/verify-action-pins.yml"
|
125
156
|
- ".gitignore"
|
@@ -137,6 +168,7 @@ files:
|
|
137
168
|
- ext/Rakefile
|
138
169
|
- ext/build_common.sh
|
139
170
|
- ext/build_linux_x86_64_gnu.sh
|
171
|
+
- ext/build_linux_x86_64_musl.sh
|
140
172
|
- ext/build_macos_arm64.sh
|
141
173
|
- ext/librdkafka.so
|
142
174
|
- karafka-rdkafka.gemspec
|