rdkafka 0.22.0.beta1-x86_64-linux-gnu
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.github/CODEOWNERS +3 -0
- data/.github/FUNDING.yml +1 -0
- data/.github/workflows/ci_linux_x86_64_gnu.yml +249 -0
- data/.github/workflows/ci_linux_x86_64_musl.yml +205 -0
- data/.github/workflows/ci_macos_arm64.yml +306 -0
- data/.github/workflows/push_linux_x86_64_gnu.yml +64 -0
- data/.github/workflows/push_linux_x86_64_musl.yml +77 -0
- data/.github/workflows/push_macos_arm64.yml +54 -0
- data/.github/workflows/push_ruby.yml +37 -0
- data/.github/workflows/verify-action-pins.yml +16 -0
- data/.gitignore +14 -0
- data/.rspec +2 -0
- data/.ruby-gemset +1 -0
- data/.ruby-version +1 -0
- data/.yardopts +2 -0
- data/CHANGELOG.md +247 -0
- data/Gemfile +5 -0
- data/MIT-LICENSE +22 -0
- data/README.md +178 -0
- data/Rakefile +96 -0
- data/docker-compose.yml +25 -0
- data/ext/README.md +19 -0
- data/ext/Rakefile +131 -0
- data/ext/build_common.sh +361 -0
- data/ext/build_linux_x86_64_gnu.sh +306 -0
- data/ext/build_linux_x86_64_musl.sh +763 -0
- data/ext/build_macos_arm64.sh +550 -0
- data/ext/librdkafka.so +0 -0
- data/lib/rdkafka/abstract_handle.rb +116 -0
- data/lib/rdkafka/admin/acl_binding_result.rb +51 -0
- data/lib/rdkafka/admin/config_binding_result.rb +30 -0
- data/lib/rdkafka/admin/config_resource_binding_result.rb +18 -0
- data/lib/rdkafka/admin/create_acl_handle.rb +28 -0
- data/lib/rdkafka/admin/create_acl_report.rb +24 -0
- data/lib/rdkafka/admin/create_partitions_handle.rb +27 -0
- data/lib/rdkafka/admin/create_partitions_report.rb +6 -0
- data/lib/rdkafka/admin/create_topic_handle.rb +29 -0
- data/lib/rdkafka/admin/create_topic_report.rb +24 -0
- data/lib/rdkafka/admin/delete_acl_handle.rb +30 -0
- data/lib/rdkafka/admin/delete_acl_report.rb +23 -0
- data/lib/rdkafka/admin/delete_groups_handle.rb +28 -0
- data/lib/rdkafka/admin/delete_groups_report.rb +24 -0
- data/lib/rdkafka/admin/delete_topic_handle.rb +29 -0
- data/lib/rdkafka/admin/delete_topic_report.rb +24 -0
- data/lib/rdkafka/admin/describe_acl_handle.rb +30 -0
- data/lib/rdkafka/admin/describe_acl_report.rb +24 -0
- data/lib/rdkafka/admin/describe_configs_handle.rb +33 -0
- data/lib/rdkafka/admin/describe_configs_report.rb +54 -0
- data/lib/rdkafka/admin/incremental_alter_configs_handle.rb +33 -0
- data/lib/rdkafka/admin/incremental_alter_configs_report.rb +54 -0
- data/lib/rdkafka/admin.rb +833 -0
- data/lib/rdkafka/bindings.rb +566 -0
- data/lib/rdkafka/callbacks.rb +415 -0
- data/lib/rdkafka/config.rb +398 -0
- data/lib/rdkafka/consumer/headers.rb +79 -0
- data/lib/rdkafka/consumer/message.rb +86 -0
- data/lib/rdkafka/consumer/partition.rb +51 -0
- data/lib/rdkafka/consumer/topic_partition_list.rb +169 -0
- data/lib/rdkafka/consumer.rb +653 -0
- data/lib/rdkafka/error.rb +101 -0
- data/lib/rdkafka/helpers/oauth.rb +58 -0
- data/lib/rdkafka/helpers/time.rb +14 -0
- data/lib/rdkafka/metadata.rb +115 -0
- data/lib/rdkafka/native_kafka.rb +139 -0
- data/lib/rdkafka/producer/delivery_handle.rb +40 -0
- data/lib/rdkafka/producer/delivery_report.rb +46 -0
- data/lib/rdkafka/producer/partitions_count_cache.rb +216 -0
- data/lib/rdkafka/producer.rb +430 -0
- data/lib/rdkafka/version.rb +7 -0
- data/lib/rdkafka.rb +54 -0
- data/rdkafka.gemspec +65 -0
- data/renovate.json +92 -0
- data/spec/rdkafka/abstract_handle_spec.rb +117 -0
- data/spec/rdkafka/admin/create_acl_handle_spec.rb +56 -0
- data/spec/rdkafka/admin/create_acl_report_spec.rb +18 -0
- data/spec/rdkafka/admin/create_topic_handle_spec.rb +52 -0
- data/spec/rdkafka/admin/create_topic_report_spec.rb +16 -0
- data/spec/rdkafka/admin/delete_acl_handle_spec.rb +85 -0
- data/spec/rdkafka/admin/delete_acl_report_spec.rb +72 -0
- data/spec/rdkafka/admin/delete_topic_handle_spec.rb +52 -0
- data/spec/rdkafka/admin/delete_topic_report_spec.rb +16 -0
- data/spec/rdkafka/admin/describe_acl_handle_spec.rb +85 -0
- data/spec/rdkafka/admin/describe_acl_report_spec.rb +73 -0
- data/spec/rdkafka/admin_spec.rb +770 -0
- data/spec/rdkafka/bindings_spec.rb +223 -0
- data/spec/rdkafka/callbacks_spec.rb +20 -0
- data/spec/rdkafka/config_spec.rb +258 -0
- data/spec/rdkafka/consumer/headers_spec.rb +73 -0
- data/spec/rdkafka/consumer/message_spec.rb +139 -0
- data/spec/rdkafka/consumer/partition_spec.rb +57 -0
- data/spec/rdkafka/consumer/topic_partition_list_spec.rb +248 -0
- data/spec/rdkafka/consumer_spec.rb +1274 -0
- data/spec/rdkafka/error_spec.rb +89 -0
- data/spec/rdkafka/metadata_spec.rb +79 -0
- data/spec/rdkafka/native_kafka_spec.rb +130 -0
- data/spec/rdkafka/producer/delivery_handle_spec.rb +45 -0
- data/spec/rdkafka/producer/delivery_report_spec.rb +25 -0
- data/spec/rdkafka/producer/partitions_count_cache_spec.rb +359 -0
- data/spec/rdkafka/producer_spec.rb +1052 -0
- data/spec/spec_helper.rb +195 -0
- metadata +276 -0
@@ -0,0 +1,763 @@
|
|
1
|
+
#!/usr/bin/env bash
|
2
|
+
#
|
3
|
+
# Build self-contained librdkafka.so for Linux x86_64 musl with checksum verification
|
4
|
+
# Usage: ./build-librdkafka-linux-musl.sh
|
5
|
+
#
|
6
|
+
# Expected directory structure:
|
7
|
+
# ext/build_linux_x86_64_musl.sh (this script)
|
8
|
+
# ext/build_common.sh (shared functions)
|
9
|
+
# dist/librdkafka-*.tar.gz (librdkafka source tarball)
|
10
|
+
# dist/patches/*.patch (optional Ruby-specific patches)
|
11
|
+
#
|
12
|
+
set -euo pipefail
|
13
|
+
|
14
|
+
# Source common functions and constants
|
15
|
+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
16
|
+
source "$SCRIPT_DIR/build_common.sh"
|
17
|
+
|
18
|
+
# Override secure_download for musl/Alpine compatibility
|
19
|
+
secure_download() {
|
20
|
+
local url="$1"
|
21
|
+
local filename="$2"
|
22
|
+
|
23
|
+
if [ -f "$filename" ]; then
|
24
|
+
log "File $filename already exists, verifying checksum..."
|
25
|
+
verify_checksum "$filename"
|
26
|
+
return 0
|
27
|
+
fi
|
28
|
+
|
29
|
+
log "Downloading $filename from $url..."
|
30
|
+
|
31
|
+
# Use platform-appropriate download command
|
32
|
+
if command -v wget &> /dev/null; then
|
33
|
+
# Check if we have GNU wget or BusyBox wget
|
34
|
+
if wget --help 2>&1 | grep -q "secure-protocol"; then
|
35
|
+
# GNU wget - use security options
|
36
|
+
if ! wget --secure-protocol=TLSv1_2 \
|
37
|
+
--https-only \
|
38
|
+
--timeout=30 \
|
39
|
+
--tries=3 \
|
40
|
+
--progress=bar \
|
41
|
+
"$url" \
|
42
|
+
-O "$filename"; then
|
43
|
+
error "Failed to download $filename from $url"
|
44
|
+
fi
|
45
|
+
else
|
46
|
+
# BusyBox wget (Alpine) - use basic options
|
47
|
+
if ! wget -T 30 \
|
48
|
+
-t 3 \
|
49
|
+
-q \
|
50
|
+
"$url" \
|
51
|
+
-O "$filename"; then
|
52
|
+
error "Failed to download $filename from $url"
|
53
|
+
fi
|
54
|
+
fi
|
55
|
+
elif command -v curl &> /dev/null; then
|
56
|
+
# curl with security options
|
57
|
+
if ! curl -L \
|
58
|
+
--tlsv1.2 \
|
59
|
+
--connect-timeout 30 \
|
60
|
+
--max-time 300 \
|
61
|
+
--retry 3 \
|
62
|
+
--progress-bar \
|
63
|
+
"$url" \
|
64
|
+
-o "$filename"; then
|
65
|
+
error "Failed to download $filename from $url"
|
66
|
+
fi
|
67
|
+
else
|
68
|
+
error "No download utility found (tried wget, curl)"
|
69
|
+
fi
|
70
|
+
|
71
|
+
# Verify checksum immediately after download
|
72
|
+
verify_checksum "$filename"
|
73
|
+
}
|
74
|
+
|
75
|
+
# Platform-specific paths
|
76
|
+
DIST_DIR="$SCRIPT_DIR/../dist"
|
77
|
+
PATCHES_DIR="$DIST_DIR/patches"
|
78
|
+
BUILD_DIR="$(pwd)/build-tmp-musl"
|
79
|
+
DEPS_PREFIX="/tmp/musl-deps"
|
80
|
+
|
81
|
+
# musl-specific dependency check
|
82
|
+
check_musl_dependencies() {
|
83
|
+
log "Checking musl-specific build dependencies..."
|
84
|
+
|
85
|
+
# Check if we're actually on musl
|
86
|
+
if command -v ldd &> /dev/null && ldd --version 2>&1 | grep -q musl; then
|
87
|
+
log "✅ musl libc detected"
|
88
|
+
elif [ -f /lib/ld-musl-x86_64.so.1 ]; then
|
89
|
+
log "✅ musl detected via loader presence"
|
90
|
+
else
|
91
|
+
warn "Not clearly detected as musl system. This script is optimized for musl-based distributions (Alpine, etc.)"
|
92
|
+
warn "Continuing anyway, but glibc-specific optimizations may be used instead..."
|
93
|
+
fi
|
94
|
+
|
95
|
+
# Check for essential build tools
|
96
|
+
local missing_tools=()
|
97
|
+
|
98
|
+
command -v gcc &> /dev/null || missing_tools+=("gcc")
|
99
|
+
command -v perl &> /dev/null || missing_tools+=("perl")
|
100
|
+
command -v make &> /dev/null || missing_tools+=("make")
|
101
|
+
command -v autoconf &> /dev/null || missing_tools+=("autoconf")
|
102
|
+
command -v automake &> /dev/null || missing_tools+=("automake")
|
103
|
+
command -v libtool &> /dev/null || missing_tools+=("libtool")
|
104
|
+
command -v bison &> /dev/null || missing_tools+=("bison")
|
105
|
+
command -v flex &> /dev/null || missing_tools+=("flex")
|
106
|
+
|
107
|
+
# Check for system libraries that dependencies expect
|
108
|
+
if [ ! -f /usr/include/sys/types.h ]; then
|
109
|
+
missing_tools+=("musl-dev (for system headers)")
|
110
|
+
fi
|
111
|
+
|
112
|
+
# On Alpine, these packages provide the tools:
|
113
|
+
# build-base: gcc, make, musl-dev, libc-dev, etc.
|
114
|
+
# linux-headers: kernel headers for system calls
|
115
|
+
# pkgconf: pkg-config for dependency detection
|
116
|
+
# perl: required by OpenSSL and other build systems
|
117
|
+
# autoconf, automake, libtool: GNU autotools for configure scripts
|
118
|
+
# bison, flex: parser generators (needed by some dependencies)
|
119
|
+
# file: for file type detection
|
120
|
+
|
121
|
+
if [ ${#missing_tools[@]} -gt 0 ]; then
|
122
|
+
error "Missing required tools: ${missing_tools[*]}
|
123
|
+
|
124
|
+
On Alpine Linux, install ALL required packages with:
|
125
|
+
apk add build-base linux-headers pkgconf perl \\
|
126
|
+
autoconf automake libtool bison flex file
|
127
|
+
|
128
|
+
Or install individually as needed."
|
129
|
+
fi
|
130
|
+
|
131
|
+
log "✅ All required musl build tools found"
|
132
|
+
}
|
133
|
+
|
134
|
+
# musl-specific compiler setup
|
135
|
+
setup_musl_compiler() {
|
136
|
+
# musl-specific compiler flags
|
137
|
+
export CC="gcc"
|
138
|
+
export CFLAGS="-fPIC -O2 -static-libgcc"
|
139
|
+
export CXXFLAGS="-fPIC -O2 -static-libgcc"
|
140
|
+
export CPPFLAGS=""
|
141
|
+
export LDFLAGS="-static-libgcc"
|
142
|
+
|
143
|
+
log "Applied musl-specific compiler flags"
|
144
|
+
}
|
145
|
+
|
146
|
+
# Build OpenSSL for musl
|
147
|
+
build_openssl_musl() {
|
148
|
+
local openssl_prefix="$1"
|
149
|
+
local openssl_dir="$2"
|
150
|
+
|
151
|
+
cd "$openssl_dir"
|
152
|
+
|
153
|
+
# Check both lib and lib64 directories for existing installation
|
154
|
+
if [ ! -f "$openssl_prefix/lib/libssl.a" ] && [ ! -f "$openssl_prefix/lib64/libssl.a" ]; then
|
155
|
+
log "Configuring and building OpenSSL for musl..."
|
156
|
+
make clean 2>/dev/null || true
|
157
|
+
|
158
|
+
setup_musl_compiler
|
159
|
+
|
160
|
+
# Create the prefix directory first
|
161
|
+
mkdir -p "$openssl_prefix"
|
162
|
+
|
163
|
+
# Configure OpenSSL for musl (linux-x86_64 works fine)
|
164
|
+
./Configure linux-x86_64 \
|
165
|
+
no-shared \
|
166
|
+
no-dso \
|
167
|
+
no-engine \
|
168
|
+
--prefix="$openssl_prefix" \
|
169
|
+
--openssldir="$openssl_prefix/ssl"
|
170
|
+
|
171
|
+
make -j$(get_cpu_count)
|
172
|
+
|
173
|
+
# Try the install and capture any errors
|
174
|
+
log "Installing OpenSSL..."
|
175
|
+
if ! make install_sw install_ssldirs 2>&1; then
|
176
|
+
warn "install_sw failed, trying full install..."
|
177
|
+
make install || {
|
178
|
+
# If install fails, check if libraries were actually built
|
179
|
+
if [ -f "libssl.a" ] && [ -f "libcrypto.a" ]; then
|
180
|
+
log "Install failed but libraries exist, copying manually..."
|
181
|
+
mkdir -p "$openssl_prefix/lib" "$openssl_prefix/include"
|
182
|
+
cp libssl.a libcrypto.a "$openssl_prefix/lib/"
|
183
|
+
cp -r include/openssl "$openssl_prefix/include/" 2>/dev/null || true
|
184
|
+
else
|
185
|
+
error "OpenSSL build failed - no libraries found"
|
186
|
+
fi
|
187
|
+
}
|
188
|
+
fi
|
189
|
+
|
190
|
+
# Verify the build - check both possible lib locations
|
191
|
+
if [ -f "$openssl_prefix/lib/libssl.a" ] || [ -f "$openssl_prefix/lib64/libssl.a" ]; then
|
192
|
+
if [ -f "$openssl_prefix/lib64/libssl.a" ]; then
|
193
|
+
log "OpenSSL installed to lib64 directory"
|
194
|
+
# Create symlinks in lib for consistency
|
195
|
+
mkdir -p "$openssl_prefix/lib"
|
196
|
+
ln -sf ../lib64/libssl.a "$openssl_prefix/lib/libssl.a" 2>/dev/null || true
|
197
|
+
ln -sf ../lib64/libcrypto.a "$openssl_prefix/lib/libcrypto.a" 2>/dev/null || true
|
198
|
+
ln -sf ../lib64/pkgconfig "$openssl_prefix/lib/pkgconfig" 2>/dev/null || true
|
199
|
+
fi
|
200
|
+
log "✅ Static OpenSSL built successfully"
|
201
|
+
else
|
202
|
+
error "Failed to build static OpenSSL - libraries not found in lib or lib64"
|
203
|
+
fi
|
204
|
+
else
|
205
|
+
log "Static OpenSSL already built, skipping..."
|
206
|
+
fi
|
207
|
+
}
|
208
|
+
|
209
|
+
# Build MIT Kerberos for musl
|
210
|
+
build_krb5_musl() {
|
211
|
+
local krb5_prefix="$1"
|
212
|
+
local krb5_dir="$2"
|
213
|
+
|
214
|
+
cd "$krb5_dir/src"
|
215
|
+
|
216
|
+
if [ ! -f "$krb5_prefix/lib/libgssapi_krb5.a" ]; then
|
217
|
+
log "Configuring and building MIT Kerberos for musl..."
|
218
|
+
make clean 2>/dev/null || true
|
219
|
+
|
220
|
+
setup_musl_compiler
|
221
|
+
|
222
|
+
# musl-specific configuration for Kerberos
|
223
|
+
# Disable some features that can be problematic on musl
|
224
|
+
./configure \
|
225
|
+
--disable-shared \
|
226
|
+
--enable-static \
|
227
|
+
--prefix="$krb5_prefix" \
|
228
|
+
--without-ldap \
|
229
|
+
--without-tcl \
|
230
|
+
--without-keyutils \
|
231
|
+
--disable-rpath \
|
232
|
+
--without-system-verto \
|
233
|
+
--disable-thread-support \
|
234
|
+
--disable-dns-for-realm \
|
235
|
+
--without-readline
|
236
|
+
|
237
|
+
# Build with limited parallelism to avoid musl-specific race conditions
|
238
|
+
log "Building Kerberos (will ignore kadmin build failures)..."
|
239
|
+
make -j$(( $(get_cpu_count) / 2 )) || {
|
240
|
+
log "Full build failed (expected due to kadmin), continuing with libraries..."
|
241
|
+
true
|
242
|
+
}
|
243
|
+
|
244
|
+
# Install what was successfully built
|
245
|
+
make install || {
|
246
|
+
log "Full install failed, installing individual components..."
|
247
|
+
make install-mkdirs 2>/dev/null || true
|
248
|
+
make -C util install 2>/dev/null || true
|
249
|
+
make -C lib install 2>/dev/null || true
|
250
|
+
make -C plugins/kdb/db2 install 2>/dev/null || true
|
251
|
+
}
|
252
|
+
|
253
|
+
# Verify we got the essential libraries
|
254
|
+
if [ ! -f "$krb5_prefix/lib/libgssapi_krb5.a" ]; then
|
255
|
+
error "Failed to build essential Kerberos libraries"
|
256
|
+
fi
|
257
|
+
|
258
|
+
log "✅ Static MIT Kerberos built successfully"
|
259
|
+
else
|
260
|
+
log "Static MIT Kerberos already built, skipping..."
|
261
|
+
fi
|
262
|
+
}
|
263
|
+
|
264
|
+
# Build Cyrus SASL for musl with proper OpenSSL linking
|
265
|
+
build_sasl_musl() {
|
266
|
+
local sasl_prefix="$1"
|
267
|
+
local sasl_dir="$2"
|
268
|
+
local krb5_prefix="$3"
|
269
|
+
local openssl_prefix="$4" # Add OpenSSL prefix parameter
|
270
|
+
|
271
|
+
cd "$sasl_dir"
|
272
|
+
|
273
|
+
if [ ! -f "$sasl_prefix/lib/libsasl2.a" ]; then
|
274
|
+
log "Configuring and building Cyrus SASL for musl..."
|
275
|
+
make clean 2>/dev/null || true
|
276
|
+
|
277
|
+
# Apply comprehensive patches for missing time.h includes
|
278
|
+
log "Applying comprehensive time.h patches..."
|
279
|
+
|
280
|
+
# Create backups first
|
281
|
+
cp lib/saslutil.c lib/saslutil.c.backup 2>/dev/null || true
|
282
|
+
cp plugins/cram.c plugins/cram.c.backup 2>/dev/null || true
|
283
|
+
|
284
|
+
# Use perl for more reliable in-place editing
|
285
|
+
log "Patching lib/saslutil.c..."
|
286
|
+
perl -i -pe 's/^#include "saslint\.h"$/#include "saslint.h"\n#include <time.h>/' lib/saslutil.c
|
287
|
+
|
288
|
+
log "Patching plugins/cram.c..."
|
289
|
+
perl -i -pe 's/^#include "plugin_common\.h"$/#include "plugin_common.h"\n#include <time.h>/' plugins/cram.c
|
290
|
+
|
291
|
+
# Verify patches applied
|
292
|
+
if ! grep -q "#include <time.h>" lib/saslutil.c; then
|
293
|
+
log "Perl approach failed, using awk fallback for saslutil.c..."
|
294
|
+
awk '/^#include "saslint\.h"$/ {print; print "#include <time.h>"; next} {print}' lib/saslutil.c.backup > lib/saslutil.c
|
295
|
+
fi
|
296
|
+
|
297
|
+
if ! grep -q "#include <time.h>" plugins/cram.c; then
|
298
|
+
log "Perl approach failed, using awk fallback for cram.c..."
|
299
|
+
awk '/^#include "plugin_common\.h"$/ {print; print "#include <time.h>"; next} {print}' plugins/cram.c.backup > plugins/cram.c
|
300
|
+
fi
|
301
|
+
|
302
|
+
# Clean up backup files
|
303
|
+
rm -f lib/saslutil.c.backup plugins/cram.c.backup
|
304
|
+
|
305
|
+
# Verify patches were applied
|
306
|
+
if ! grep -q "#include <time.h>" lib/saslutil.c || ! grep -q "#include <time.h>" plugins/cram.c; then
|
307
|
+
error "Failed to patch time.h includes - this is required for compilation"
|
308
|
+
fi
|
309
|
+
|
310
|
+
log "✅ time.h patches applied successfully"
|
311
|
+
|
312
|
+
# Determine correct OpenSSL lib directory
|
313
|
+
local openssl_lib_dir
|
314
|
+
if [ -f "$openssl_prefix/lib64/libssl.a" ]; then
|
315
|
+
openssl_lib_dir="$openssl_prefix/lib64"
|
316
|
+
log "Using OpenSSL libraries from lib64"
|
317
|
+
else
|
318
|
+
openssl_lib_dir="$openssl_prefix/lib"
|
319
|
+
log "Using OpenSSL libraries from lib"
|
320
|
+
fi
|
321
|
+
|
322
|
+
# Verify OpenSSL libraries exist
|
323
|
+
if [ ! -f "$openssl_lib_dir/libssl.a" ] || [ ! -f "$openssl_lib_dir/libcrypto.a" ]; then
|
324
|
+
error "OpenSSL static libraries not found in $openssl_lib_dir"
|
325
|
+
fi
|
326
|
+
|
327
|
+
setup_musl_compiler
|
328
|
+
|
329
|
+
# Set comprehensive OpenSSL and Kerberos flags
|
330
|
+
export CPPFLAGS="$CPPFLAGS -I$krb5_prefix/include -I$openssl_prefix/include"
|
331
|
+
export LDFLAGS="$LDFLAGS -L$krb5_prefix/lib -L$openssl_lib_dir"
|
332
|
+
export LIBS="-lssl -lcrypto -ldl -lpthread"
|
333
|
+
|
334
|
+
# Also set PKG_CONFIG_PATH for better library detection
|
335
|
+
export PKG_CONFIG_PATH="$openssl_lib_dir/pkgconfig:$krb5_prefix/lib/pkgconfig:${PKG_CONFIG_PATH:-}"
|
336
|
+
|
337
|
+
log "Environment variables set:"
|
338
|
+
log " CPPFLAGS: $CPPFLAGS"
|
339
|
+
log " LDFLAGS: $LDFLAGS"
|
340
|
+
log " LIBS: $LIBS"
|
341
|
+
log " PKG_CONFIG_PATH: $PKG_CONFIG_PATH"
|
342
|
+
|
343
|
+
# Configure SASL with explicit OpenSSL paths and comprehensive options
|
344
|
+
./configure \
|
345
|
+
--disable-shared \
|
346
|
+
--enable-static \
|
347
|
+
--prefix="$sasl_prefix" \
|
348
|
+
--with-openssl="$openssl_prefix" \
|
349
|
+
--with-ssl="$openssl_prefix" \
|
350
|
+
--enable-gssapi="$krb5_prefix" \
|
351
|
+
--without-dblib \
|
352
|
+
--disable-gdbm \
|
353
|
+
--disable-sample \
|
354
|
+
--disable-obsolete_cram_attr \
|
355
|
+
--disable-obsolete_digest_attr \
|
356
|
+
--without-pam \
|
357
|
+
--without-saslauthd \
|
358
|
+
--without-pwcheck \
|
359
|
+
--without-des \
|
360
|
+
--without-authdaemond \
|
361
|
+
--disable-java \
|
362
|
+
--disable-sql \
|
363
|
+
--disable-ldapdb \
|
364
|
+
--enable-plain \
|
365
|
+
--enable-login \
|
366
|
+
--enable-digest \
|
367
|
+
--enable-cram \
|
368
|
+
--enable-otp \
|
369
|
+
--enable-ntlm \
|
370
|
+
--enable-scram \
|
371
|
+
ac_cv_func_getnameinfo=yes \
|
372
|
+
ac_cv_func_getaddrinfo=yes \
|
373
|
+
OPENSSL_CFLAGS="-I$openssl_prefix/include" \
|
374
|
+
OPENSSL_LIBS="-L$openssl_lib_dir -lssl -lcrypto"
|
375
|
+
|
376
|
+
log "Configuration completed, starting build..."
|
377
|
+
|
378
|
+
# Build with reduced parallelism to avoid race conditions
|
379
|
+
make -j$(( $(get_cpu_count) / 2 )) || {
|
380
|
+
log "Build failed, trying single-threaded build..."
|
381
|
+
make clean
|
382
|
+
make -j1
|
383
|
+
}
|
384
|
+
|
385
|
+
# Install
|
386
|
+
make install
|
387
|
+
|
388
|
+
# Verify the build
|
389
|
+
if [ ! -f "$sasl_prefix/lib/libsasl2.a" ]; then
|
390
|
+
error "Failed to build static Cyrus SASL"
|
391
|
+
fi
|
392
|
+
|
393
|
+
# Additional verification - check if OTP plugin was built (this was failing)
|
394
|
+
if [ -f "$sasl_prefix/lib/sasl2/libotp.a" ] || [ -f "$sasl_prefix/lib/sasl2/libotp.la" ]; then
|
395
|
+
log "✅ OTP plugin built successfully"
|
396
|
+
else
|
397
|
+
log "⚠️ OTP plugin may not have built, but core SASL library is available"
|
398
|
+
fi
|
399
|
+
|
400
|
+
log "✅ Static Cyrus SASL built successfully"
|
401
|
+
else
|
402
|
+
log "Static Cyrus SASL already built, skipping..."
|
403
|
+
fi
|
404
|
+
}
|
405
|
+
|
406
|
+
# Build static library with musl-specific configuration
|
407
|
+
build_static_lib_musl() {
|
408
|
+
local lib_name="$1"
|
409
|
+
local prefix="$2"
|
410
|
+
local source_dir="$3"
|
411
|
+
local configure_args="$4"
|
412
|
+
|
413
|
+
cd "$source_dir"
|
414
|
+
|
415
|
+
local lib_file="$prefix/lib/lib${lib_name}.a"
|
416
|
+
if [ ! -f "$lib_file" ]; then
|
417
|
+
log "Configuring and building static $lib_name for musl..."
|
418
|
+
make clean 2>/dev/null || true
|
419
|
+
|
420
|
+
setup_musl_compiler
|
421
|
+
|
422
|
+
# Run configure with provided arguments
|
423
|
+
eval "./configure --prefix=\"$prefix\" $configure_args"
|
424
|
+
|
425
|
+
make -j$(get_cpu_count)
|
426
|
+
make install
|
427
|
+
|
428
|
+
# Verify the build
|
429
|
+
if [ ! -f "$lib_file" ]; then
|
430
|
+
error "Failed to build static $lib_name"
|
431
|
+
fi
|
432
|
+
|
433
|
+
log "✅ Static $lib_name built successfully"
|
434
|
+
else
|
435
|
+
log "Static $lib_name already built, skipping..."
|
436
|
+
fi
|
437
|
+
}
|
438
|
+
|
439
|
+
# Check common and musl-specific dependencies
|
440
|
+
check_common_dependencies
|
441
|
+
check_musl_dependencies
|
442
|
+
|
443
|
+
# Auto-detect librdkafka tarball
|
444
|
+
log "Looking for librdkafka tarball in $DIST_DIR..."
|
445
|
+
LIBRDKAFKA_TARBALL=$(find_librdkafka_tarball "$DIST_DIR")
|
446
|
+
log "Found librdkafka tarball: $LIBRDKAFKA_TARBALL"
|
447
|
+
|
448
|
+
# Verify librdkafka tarball checksum if available
|
449
|
+
verify_librdkafka_checksum "$LIBRDKAFKA_TARBALL"
|
450
|
+
|
451
|
+
# Find patches
|
452
|
+
PATCHES_FOUND=()
|
453
|
+
find_patches "$PATCHES_DIR" PATCHES_FOUND
|
454
|
+
|
455
|
+
security_log "Starting secure build with checksum verification enabled"
|
456
|
+
log "Building self-contained librdkafka.so for Linux x86_64 musl (WITH SASL)"
|
457
|
+
log "Dependencies to build:"
|
458
|
+
log " - OpenSSL: $OPENSSL_VERSION"
|
459
|
+
log " - Cyrus SASL: $CYRUS_SASL_VERSION (with official GCC 14+ patches)"
|
460
|
+
log " - MIT Kerberos: $KRB5_VERSION"
|
461
|
+
log " - zlib: $ZLIB_VERSION"
|
462
|
+
log " - ZStd: $ZSTD_VERSION"
|
463
|
+
log "librdkafka source: $LIBRDKAFKA_TARBALL"
|
464
|
+
log "Build directory: $BUILD_DIR"
|
465
|
+
|
466
|
+
# Create build directory
|
467
|
+
mkdir -p "$BUILD_DIR"
|
468
|
+
cd "$BUILD_DIR"
|
469
|
+
|
470
|
+
# Build OpenSSL
|
471
|
+
log "Building OpenSSL $OPENSSL_VERSION for musl..."
|
472
|
+
OPENSSL_PREFIX="$DEPS_PREFIX/static-openssl-$OPENSSL_VERSION"
|
473
|
+
OPENSSL_TARBALL="openssl-$OPENSSL_VERSION.tar.gz"
|
474
|
+
OPENSSL_DIR="openssl-$OPENSSL_VERSION"
|
475
|
+
|
476
|
+
secure_download "$(get_openssl_url)" "$OPENSSL_TARBALL"
|
477
|
+
extract_if_needed "$OPENSSL_TARBALL" "$OPENSSL_DIR"
|
478
|
+
build_openssl_musl "$OPENSSL_PREFIX" "$OPENSSL_DIR"
|
479
|
+
|
480
|
+
cd "$BUILD_DIR"
|
481
|
+
|
482
|
+
# Determine OpenSSL lib directory (could be lib or lib64)
|
483
|
+
if [ -f "$OPENSSL_PREFIX/lib64/libssl.a" ]; then
|
484
|
+
OPENSSL_LIB_DIR="$OPENSSL_PREFIX/lib64"
|
485
|
+
log "Using OpenSSL libraries from lib64"
|
486
|
+
else
|
487
|
+
OPENSSL_LIB_DIR="$OPENSSL_PREFIX/lib"
|
488
|
+
log "Using OpenSSL libraries from lib"
|
489
|
+
fi
|
490
|
+
|
491
|
+
# Build MIT Kerberos
|
492
|
+
log "Building MIT Kerberos $KRB5_VERSION for musl..."
|
493
|
+
KRB5_PREFIX="$DEPS_PREFIX/static-krb5-$KRB5_VERSION"
|
494
|
+
KRB5_TARBALL="krb5-$KRB5_VERSION.tar.gz"
|
495
|
+
KRB5_DIR="krb5-$KRB5_VERSION"
|
496
|
+
|
497
|
+
secure_download "$(get_krb5_url)" "$KRB5_TARBALL"
|
498
|
+
extract_if_needed "$KRB5_TARBALL" "$KRB5_DIR"
|
499
|
+
build_krb5_musl "$KRB5_PREFIX" "$KRB5_DIR"
|
500
|
+
|
501
|
+
cd "$BUILD_DIR"
|
502
|
+
|
503
|
+
# Build SASL with official patches (updated call)
|
504
|
+
log "Building Cyrus SASL $CYRUS_SASL_VERSION for musl..."
|
505
|
+
SASL_PREFIX="$DEPS_PREFIX/static-sasl-$CYRUS_SASL_VERSION"
|
506
|
+
SASL_TARBALL="cyrus-sasl-$CYRUS_SASL_VERSION.tar.gz"
|
507
|
+
SASL_DIR="cyrus-sasl-$CYRUS_SASL_VERSION"
|
508
|
+
|
509
|
+
secure_download "$(get_sasl_url)" "$SASL_TARBALL"
|
510
|
+
extract_if_needed "$SASL_TARBALL" "$SASL_DIR"
|
511
|
+
build_sasl_musl "$SASL_PREFIX" "$SASL_DIR" "$KRB5_PREFIX" "$OPENSSL_PREFIX" # Added OPENSSL_PREFIX parameter
|
512
|
+
|
513
|
+
cd "$BUILD_DIR"
|
514
|
+
|
515
|
+
# Build zlib
|
516
|
+
log "Building zlib $ZLIB_VERSION for musl..."
|
517
|
+
ZLIB_PREFIX="$DEPS_PREFIX/static-zlib-$ZLIB_VERSION"
|
518
|
+
ZLIB_TARBALL="zlib-$ZLIB_VERSION.tar.gz"
|
519
|
+
ZLIB_DIR="zlib-$ZLIB_VERSION"
|
520
|
+
|
521
|
+
secure_download "$(get_zlib_url)" "$ZLIB_TARBALL"
|
522
|
+
extract_if_needed "$ZLIB_TARBALL" "$ZLIB_DIR"
|
523
|
+
build_static_lib_musl "z" "$ZLIB_PREFIX" "$ZLIB_DIR" "--static"
|
524
|
+
|
525
|
+
cd "$BUILD_DIR"
|
526
|
+
|
527
|
+
# Build ZStd
|
528
|
+
log "Building ZStd $ZSTD_VERSION for musl..."
|
529
|
+
ZSTD_PREFIX="$DEPS_PREFIX/static-zstd-$ZSTD_VERSION"
|
530
|
+
ZSTD_TARBALL="zstd-$ZSTD_VERSION.tar.gz"
|
531
|
+
ZSTD_DIR="zstd-$ZSTD_VERSION"
|
532
|
+
|
533
|
+
secure_download "$(get_zstd_url)" "$ZSTD_TARBALL"
|
534
|
+
extract_if_needed "$ZSTD_TARBALL" "$ZSTD_DIR"
|
535
|
+
cd "$ZSTD_DIR"
|
536
|
+
|
537
|
+
if [ ! -f "$ZSTD_PREFIX/lib/libzstd.a" ]; then
|
538
|
+
log "Building ZStd for musl..."
|
539
|
+
make clean 2>/dev/null || true
|
540
|
+
|
541
|
+
setup_musl_compiler
|
542
|
+
|
543
|
+
# Build static library using ZStd's Makefile
|
544
|
+
make lib-mt CFLAGS="$CFLAGS" PREFIX="$ZSTD_PREFIX" -j$(get_cpu_count)
|
545
|
+
make install PREFIX="$ZSTD_PREFIX"
|
546
|
+
|
547
|
+
# Verify the build
|
548
|
+
if [ ! -f "$ZSTD_PREFIX/lib/libzstd.a" ]; then
|
549
|
+
error "Failed to build static ZStd"
|
550
|
+
fi
|
551
|
+
|
552
|
+
log "✅ Static ZStd built successfully"
|
553
|
+
else
|
554
|
+
log "Static ZStd already built, skipping..."
|
555
|
+
fi
|
556
|
+
|
557
|
+
cd "$BUILD_DIR"
|
558
|
+
|
559
|
+
# Extract and patch librdkafka
|
560
|
+
log "Extracting librdkafka..."
|
561
|
+
tar xzf "$LIBRDKAFKA_TARBALL"
|
562
|
+
cd "librdkafka-$LIBRDKAFKA_VERSION"
|
563
|
+
|
564
|
+
# Fix permissions and apply patches
|
565
|
+
fix_configure_permissions
|
566
|
+
apply_patches PATCHES_FOUND
|
567
|
+
|
568
|
+
# Configure librdkafka for musl (with SASL)
|
569
|
+
log "Configuring librdkafka for musl (with SASL support)..."
|
570
|
+
|
571
|
+
setup_musl_compiler
|
572
|
+
|
573
|
+
# musl-specific configuration with SASL
|
574
|
+
export CPPFLAGS="$CPPFLAGS -I$KRB5_PREFIX/include -I$SASL_PREFIX/include"
|
575
|
+
export LDFLAGS="$LDFLAGS -L$KRB5_PREFIX/lib -L$SASL_PREFIX/lib"
|
576
|
+
|
577
|
+
if [ -f configure ]; then
|
578
|
+
log "Using mklove configure script"
|
579
|
+
|
580
|
+
# mklove-specific configure options with SASL
|
581
|
+
./configure \
|
582
|
+
--enable-static \
|
583
|
+
--disable-shared \
|
584
|
+
--disable-curl \
|
585
|
+
--enable-gssapi \
|
586
|
+
--enable-sasl \
|
587
|
+
--disable-c11threads
|
588
|
+
else
|
589
|
+
error "No configure script found"
|
590
|
+
fi
|
591
|
+
|
592
|
+
# Build librdkafka
|
593
|
+
log "Compiling librdkafka..."
|
594
|
+
make clean || true
|
595
|
+
make -j$(get_cpu_count)
|
596
|
+
|
597
|
+
# Verify librdkafka.a exists
|
598
|
+
if [ ! -f src/librdkafka.a ]; then
|
599
|
+
error "librdkafka.a not found after build"
|
600
|
+
fi
|
601
|
+
|
602
|
+
log "librdkafka.a built successfully"
|
603
|
+
|
604
|
+
# Create self-contained shared library for musl (with SASL)
|
605
|
+
log "Creating self-contained librdkafka.so for musl (with SASL support)..."
|
606
|
+
|
607
|
+
# First, verify all static libraries exist
|
608
|
+
log "Verifying static libraries exist..."
|
609
|
+
for lib_path in \
|
610
|
+
"$SASL_PREFIX/lib/libsasl2.a" \
|
611
|
+
"$KRB5_PREFIX/lib/libgssapi_krb5.a" \
|
612
|
+
"$KRB5_PREFIX/lib/libkrb5.a" \
|
613
|
+
"$KRB5_PREFIX/lib/libk5crypto.a" \
|
614
|
+
"$KRB5_PREFIX/lib/libcom_err.a" \
|
615
|
+
"$KRB5_PREFIX/lib/libkrb5support.a" \
|
616
|
+
"$OPENSSL_LIB_DIR/libssl.a" \
|
617
|
+
"$OPENSSL_LIB_DIR/libcrypto.a" \
|
618
|
+
"$ZLIB_PREFIX/lib/libz.a" \
|
619
|
+
"$ZSTD_PREFIX/lib/libzstd.a"
|
620
|
+
do
|
621
|
+
if [ ! -f "$lib_path" ]; then
|
622
|
+
error "Required static library not found: $lib_path"
|
623
|
+
else
|
624
|
+
log "✅ Found: $lib_path"
|
625
|
+
fi
|
626
|
+
done
|
627
|
+
|
628
|
+
gcc -shared -fPIC \
|
629
|
+
-Wl,--whole-archive src/librdkafka.a -Wl,--no-whole-archive \
|
630
|
+
-o librdkafka.so \
|
631
|
+
-Wl,-Bstatic \
|
632
|
+
"$SASL_PREFIX/lib/libsasl2.a" \
|
633
|
+
"$KRB5_PREFIX/lib/libgssapi_krb5.a" \
|
634
|
+
"$KRB5_PREFIX/lib/libkrb5.a" \
|
635
|
+
"$KRB5_PREFIX/lib/libk5crypto.a" \
|
636
|
+
"$KRB5_PREFIX/lib/libcom_err.a" \
|
637
|
+
"$KRB5_PREFIX/lib/libkrb5support.a" \
|
638
|
+
"$OPENSSL_LIB_DIR/libssl.a" \
|
639
|
+
"$OPENSSL_LIB_DIR/libcrypto.a" \
|
640
|
+
"$ZLIB_PREFIX/lib/libz.a" \
|
641
|
+
"$ZSTD_PREFIX/lib/libzstd.a" \
|
642
|
+
-Wl,-Bdynamic \
|
643
|
+
-lpthread -lm -ldl -lc \
|
644
|
+
-static-libgcc \
|
645
|
+
-Wl,--as-needed \
|
646
|
+
-Wl,--no-undefined
|
647
|
+
|
648
|
+
if [ ! -f librdkafka.so ]; then
|
649
|
+
error "Failed to create librdkafka.so"
|
650
|
+
fi
|
651
|
+
|
652
|
+
log "librdkafka.so created successfully"
|
653
|
+
|
654
|
+
# Enhanced verification
|
655
|
+
log "Verifying static linking of SASL..."
|
656
|
+
if command -v ldd &> /dev/null; then
|
657
|
+
SASL_DEPS=$(ldd librdkafka.so 2>/dev/null | grep -i sasl || true)
|
658
|
+
if [ -n "$SASL_DEPS" ]; then
|
659
|
+
error "SASL is still dynamically linked: $SASL_DEPS"
|
660
|
+
else
|
661
|
+
log "✅ SASL successfully statically linked - no dynamic SASL dependencies found"
|
662
|
+
fi
|
663
|
+
|
664
|
+
# Check for any other problematic dynamic dependencies
|
665
|
+
PROBLEMATIC_DEPS=$(ldd librdkafka.so 2>/dev/null | grep -E "(libssl|libcrypto|libz|libzstd|libkrb|libgssapi)" || true)
|
666
|
+
if [ -n "$PROBLEMATIC_DEPS" ]; then
|
667
|
+
warn "Found other dynamic dependencies that should be static: $PROBLEMATIC_DEPS"
|
668
|
+
else
|
669
|
+
log "✅ All major dependencies appear to be statically linked"
|
670
|
+
fi
|
671
|
+
else
|
672
|
+
log "ldd not available, skipping dynamic dependency check"
|
673
|
+
fi
|
674
|
+
|
675
|
+
# Additional verification using nm if available
|
676
|
+
if command -v nm &> /dev/null; then
|
677
|
+
log "Checking for SASL symbols in the library..."
|
678
|
+
SASL_SYMBOLS=$(nm -D librdkafka.so 2>/dev/null | grep -i sasl | head -5 || true)
|
679
|
+
if [ -n "$SASL_SYMBOLS" ]; then
|
680
|
+
log "✅ SASL symbols found in library (first 5):"
|
681
|
+
echo "$SASL_SYMBOLS"
|
682
|
+
else
|
683
|
+
warn "No SASL symbols found - this might indicate a linking issue"
|
684
|
+
fi
|
685
|
+
fi
|
686
|
+
|
687
|
+
# Verify the build (musl-compatible)
|
688
|
+
log "Verifying musl build..."
|
689
|
+
if command -v file &> /dev/null; then
|
690
|
+
file librdkafka.so
|
691
|
+
else
|
692
|
+
log "file command not available, skipping file type check"
|
693
|
+
fi
|
694
|
+
|
695
|
+
log "Checking dependencies with ldd:"
|
696
|
+
if command -v ldd &> /dev/null; then
|
697
|
+
ldd librdkafka.so
|
698
|
+
else
|
699
|
+
log "ldd not available, skipping dependency check"
|
700
|
+
fi
|
701
|
+
|
702
|
+
log "Checking for external dependencies (should only show musl and system libraries):"
|
703
|
+
if command -v ldd &> /dev/null; then
|
704
|
+
EXTERNAL_DEPS=$(ldd librdkafka.so 2>/dev/null | grep -v "musl" | grep -v "ld-musl" | grep -v "librdkafka.so" | grep -v "=>" | grep -v "statically linked" | grep -v "not a dynamic executable" || true)
|
705
|
+
if [ -n "$EXTERNAL_DEPS" ]; then
|
706
|
+
# Filter out expected system dependencies that are OK on musl
|
707
|
+
FILTERED_DEPS=$(echo "$EXTERNAL_DEPS" | grep -v "linux-vdso.so" | grep -v "/lib/ld-musl" || true)
|
708
|
+
if [ -n "$FILTERED_DEPS" ]; then
|
709
|
+
warn "Found some external dependencies (may be acceptable for musl): $FILTERED_DEPS"
|
710
|
+
else
|
711
|
+
log "✅ All dependencies are musl system libraries - build is self-contained!"
|
712
|
+
fi
|
713
|
+
else
|
714
|
+
log "✅ No unexpected external dependencies found - library is self-contained!"
|
715
|
+
fi
|
716
|
+
else
|
717
|
+
log "ldd not available, skipping external dependency check"
|
718
|
+
fi
|
719
|
+
|
720
|
+
# Check for musl compatibility
|
721
|
+
log "Verifying musl compatibility..."
|
722
|
+
if command -v ldd &> /dev/null; then
|
723
|
+
if ldd librdkafka.so 2>&1 | grep -q "musl"; then
|
724
|
+
log "✅ Library is linked against musl libc"
|
725
|
+
elif ldd librdkafka.so 2>&1 | grep -q "statically linked"; then
|
726
|
+
log "✅ Library appears to be statically linked (good for musl)"
|
727
|
+
else
|
728
|
+
warn "Library linking may not be optimal for musl systems"
|
729
|
+
fi
|
730
|
+
else
|
731
|
+
log "ldd not available, skipping musl compatibility check"
|
732
|
+
fi
|
733
|
+
|
734
|
+
# Copy to output directory
|
735
|
+
OUTPUT_DIR="$SCRIPT_DIR"
|
736
|
+
cp librdkafka.so "$OUTPUT_DIR/"
|
737
|
+
log "librdkafka.so copied to: $OUTPUT_DIR/librdkafka.so"
|
738
|
+
|
739
|
+
# Print summaries
|
740
|
+
print_security_summary
|
741
|
+
|
742
|
+
# Enhanced summary for musl (with SASL)
|
743
|
+
log "Build completed successfully!"
|
744
|
+
log "📦 Self-contained librdkafka built for Linux x86_64 musl:"
|
745
|
+
log " ✅ OpenSSL $OPENSSL_VERSION (SSL/TLS support) - checksum verified"
|
746
|
+
log " ✅ Cyrus SASL $CYRUS_SASL_VERSION (authentication for AWS MSK) - checksum verified"
|
747
|
+
log " ✅ MIT Kerberos $KRB5_VERSION (GSSAPI/Kerberos authentication) - checksum verified"
|
748
|
+
log " ✅ zlib $ZLIB_VERSION (compression) - checksum verified"
|
749
|
+
log " ✅ ZStd $ZSTD_VERSION (high-performance compression) - checksum verified"
|
750
|
+
log ""
|
751
|
+
log "🎯 Ready for deployment on musl-based systems (Alpine Linux, Docker containers)"
|
752
|
+
log "☁️ Compatible with AWS MSK and other secured Kafka clusters"
|
753
|
+
log "🔐 Supply chain security: All dependencies cryptographically verified"
|
754
|
+
log "🐳 Docker-optimized: Perfect for Alpine-based container deployments"
|
755
|
+
log "🛠️ GCC 14+ compatible: Official patches applied to Cyrus SASL"
|
756
|
+
log ""
|
757
|
+
log "Location: $OUTPUT_DIR/librdkafka.so"
|
758
|
+
|
759
|
+
# Cleanup
|
760
|
+
cleanup_build_dir "$BUILD_DIR"
|
761
|
+
|
762
|
+
# Reset environment variables
|
763
|
+
unset CFLAGS CXXFLAGS CPPFLAGS LDFLAGS CC
|