rdkafka 0.22.0.beta1-x86_64-linux-gnu

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. checksums.yaml +7 -0
  2. data/.github/CODEOWNERS +3 -0
  3. data/.github/FUNDING.yml +1 -0
  4. data/.github/workflows/ci_linux_x86_64_gnu.yml +249 -0
  5. data/.github/workflows/ci_linux_x86_64_musl.yml +205 -0
  6. data/.github/workflows/ci_macos_arm64.yml +306 -0
  7. data/.github/workflows/push_linux_x86_64_gnu.yml +64 -0
  8. data/.github/workflows/push_linux_x86_64_musl.yml +77 -0
  9. data/.github/workflows/push_macos_arm64.yml +54 -0
  10. data/.github/workflows/push_ruby.yml +37 -0
  11. data/.github/workflows/verify-action-pins.yml +16 -0
  12. data/.gitignore +14 -0
  13. data/.rspec +2 -0
  14. data/.ruby-gemset +1 -0
  15. data/.ruby-version +1 -0
  16. data/.yardopts +2 -0
  17. data/CHANGELOG.md +247 -0
  18. data/Gemfile +5 -0
  19. data/MIT-LICENSE +22 -0
  20. data/README.md +178 -0
  21. data/Rakefile +96 -0
  22. data/docker-compose.yml +25 -0
  23. data/ext/README.md +19 -0
  24. data/ext/Rakefile +131 -0
  25. data/ext/build_common.sh +361 -0
  26. data/ext/build_linux_x86_64_gnu.sh +306 -0
  27. data/ext/build_linux_x86_64_musl.sh +763 -0
  28. data/ext/build_macos_arm64.sh +550 -0
  29. data/ext/librdkafka.so +0 -0
  30. data/lib/rdkafka/abstract_handle.rb +116 -0
  31. data/lib/rdkafka/admin/acl_binding_result.rb +51 -0
  32. data/lib/rdkafka/admin/config_binding_result.rb +30 -0
  33. data/lib/rdkafka/admin/config_resource_binding_result.rb +18 -0
  34. data/lib/rdkafka/admin/create_acl_handle.rb +28 -0
  35. data/lib/rdkafka/admin/create_acl_report.rb +24 -0
  36. data/lib/rdkafka/admin/create_partitions_handle.rb +27 -0
  37. data/lib/rdkafka/admin/create_partitions_report.rb +6 -0
  38. data/lib/rdkafka/admin/create_topic_handle.rb +29 -0
  39. data/lib/rdkafka/admin/create_topic_report.rb +24 -0
  40. data/lib/rdkafka/admin/delete_acl_handle.rb +30 -0
  41. data/lib/rdkafka/admin/delete_acl_report.rb +23 -0
  42. data/lib/rdkafka/admin/delete_groups_handle.rb +28 -0
  43. data/lib/rdkafka/admin/delete_groups_report.rb +24 -0
  44. data/lib/rdkafka/admin/delete_topic_handle.rb +29 -0
  45. data/lib/rdkafka/admin/delete_topic_report.rb +24 -0
  46. data/lib/rdkafka/admin/describe_acl_handle.rb +30 -0
  47. data/lib/rdkafka/admin/describe_acl_report.rb +24 -0
  48. data/lib/rdkafka/admin/describe_configs_handle.rb +33 -0
  49. data/lib/rdkafka/admin/describe_configs_report.rb +54 -0
  50. data/lib/rdkafka/admin/incremental_alter_configs_handle.rb +33 -0
  51. data/lib/rdkafka/admin/incremental_alter_configs_report.rb +54 -0
  52. data/lib/rdkafka/admin.rb +833 -0
  53. data/lib/rdkafka/bindings.rb +566 -0
  54. data/lib/rdkafka/callbacks.rb +415 -0
  55. data/lib/rdkafka/config.rb +398 -0
  56. data/lib/rdkafka/consumer/headers.rb +79 -0
  57. data/lib/rdkafka/consumer/message.rb +86 -0
  58. data/lib/rdkafka/consumer/partition.rb +51 -0
  59. data/lib/rdkafka/consumer/topic_partition_list.rb +169 -0
  60. data/lib/rdkafka/consumer.rb +653 -0
  61. data/lib/rdkafka/error.rb +101 -0
  62. data/lib/rdkafka/helpers/oauth.rb +58 -0
  63. data/lib/rdkafka/helpers/time.rb +14 -0
  64. data/lib/rdkafka/metadata.rb +115 -0
  65. data/lib/rdkafka/native_kafka.rb +139 -0
  66. data/lib/rdkafka/producer/delivery_handle.rb +40 -0
  67. data/lib/rdkafka/producer/delivery_report.rb +46 -0
  68. data/lib/rdkafka/producer/partitions_count_cache.rb +216 -0
  69. data/lib/rdkafka/producer.rb +430 -0
  70. data/lib/rdkafka/version.rb +7 -0
  71. data/lib/rdkafka.rb +54 -0
  72. data/rdkafka.gemspec +65 -0
  73. data/renovate.json +92 -0
  74. data/spec/rdkafka/abstract_handle_spec.rb +117 -0
  75. data/spec/rdkafka/admin/create_acl_handle_spec.rb +56 -0
  76. data/spec/rdkafka/admin/create_acl_report_spec.rb +18 -0
  77. data/spec/rdkafka/admin/create_topic_handle_spec.rb +52 -0
  78. data/spec/rdkafka/admin/create_topic_report_spec.rb +16 -0
  79. data/spec/rdkafka/admin/delete_acl_handle_spec.rb +85 -0
  80. data/spec/rdkafka/admin/delete_acl_report_spec.rb +72 -0
  81. data/spec/rdkafka/admin/delete_topic_handle_spec.rb +52 -0
  82. data/spec/rdkafka/admin/delete_topic_report_spec.rb +16 -0
  83. data/spec/rdkafka/admin/describe_acl_handle_spec.rb +85 -0
  84. data/spec/rdkafka/admin/describe_acl_report_spec.rb +73 -0
  85. data/spec/rdkafka/admin_spec.rb +770 -0
  86. data/spec/rdkafka/bindings_spec.rb +223 -0
  87. data/spec/rdkafka/callbacks_spec.rb +20 -0
  88. data/spec/rdkafka/config_spec.rb +258 -0
  89. data/spec/rdkafka/consumer/headers_spec.rb +73 -0
  90. data/spec/rdkafka/consumer/message_spec.rb +139 -0
  91. data/spec/rdkafka/consumer/partition_spec.rb +57 -0
  92. data/spec/rdkafka/consumer/topic_partition_list_spec.rb +248 -0
  93. data/spec/rdkafka/consumer_spec.rb +1274 -0
  94. data/spec/rdkafka/error_spec.rb +89 -0
  95. data/spec/rdkafka/metadata_spec.rb +79 -0
  96. data/spec/rdkafka/native_kafka_spec.rb +130 -0
  97. data/spec/rdkafka/producer/delivery_handle_spec.rb +45 -0
  98. data/spec/rdkafka/producer/delivery_report_spec.rb +25 -0
  99. data/spec/rdkafka/producer/partitions_count_cache_spec.rb +359 -0
  100. data/spec/rdkafka/producer_spec.rb +1052 -0
  101. data/spec/spec_helper.rb +195 -0
  102. metadata +276 -0
@@ -0,0 +1,763 @@
1
+ #!/usr/bin/env bash
2
+ #
3
+ # Build self-contained librdkafka.so for Linux x86_64 musl with checksum verification
4
+ # Usage: ./build-librdkafka-linux-musl.sh
5
+ #
6
+ # Expected directory structure:
7
+ # ext/build_linux_x86_64_musl.sh (this script)
8
+ # ext/build_common.sh (shared functions)
9
+ # dist/librdkafka-*.tar.gz (librdkafka source tarball)
10
+ # dist/patches/*.patch (optional Ruby-specific patches)
11
+ #
12
+ set -euo pipefail
13
+
14
+ # Source common functions and constants
15
+ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
16
+ source "$SCRIPT_DIR/build_common.sh"
17
+
18
+ # Override secure_download for musl/Alpine compatibility
19
+ secure_download() {
20
+ local url="$1"
21
+ local filename="$2"
22
+
23
+ if [ -f "$filename" ]; then
24
+ log "File $filename already exists, verifying checksum..."
25
+ verify_checksum "$filename"
26
+ return 0
27
+ fi
28
+
29
+ log "Downloading $filename from $url..."
30
+
31
+ # Use platform-appropriate download command
32
+ if command -v wget &> /dev/null; then
33
+ # Check if we have GNU wget or BusyBox wget
34
+ if wget --help 2>&1 | grep -q "secure-protocol"; then
35
+ # GNU wget - use security options
36
+ if ! wget --secure-protocol=TLSv1_2 \
37
+ --https-only \
38
+ --timeout=30 \
39
+ --tries=3 \
40
+ --progress=bar \
41
+ "$url" \
42
+ -O "$filename"; then
43
+ error "Failed to download $filename from $url"
44
+ fi
45
+ else
46
+ # BusyBox wget (Alpine) - use basic options
47
+ if ! wget -T 30 \
48
+ -t 3 \
49
+ -q \
50
+ "$url" \
51
+ -O "$filename"; then
52
+ error "Failed to download $filename from $url"
53
+ fi
54
+ fi
55
+ elif command -v curl &> /dev/null; then
56
+ # curl with security options
57
+ if ! curl -L \
58
+ --tlsv1.2 \
59
+ --connect-timeout 30 \
60
+ --max-time 300 \
61
+ --retry 3 \
62
+ --progress-bar \
63
+ "$url" \
64
+ -o "$filename"; then
65
+ error "Failed to download $filename from $url"
66
+ fi
67
+ else
68
+ error "No download utility found (tried wget, curl)"
69
+ fi
70
+
71
+ # Verify checksum immediately after download
72
+ verify_checksum "$filename"
73
+ }
74
+
75
+ # Platform-specific paths
76
+ DIST_DIR="$SCRIPT_DIR/../dist"
77
+ PATCHES_DIR="$DIST_DIR/patches"
78
+ BUILD_DIR="$(pwd)/build-tmp-musl"
79
+ DEPS_PREFIX="/tmp/musl-deps"
80
+
81
+ # musl-specific dependency check
82
+ check_musl_dependencies() {
83
+ log "Checking musl-specific build dependencies..."
84
+
85
+ # Check if we're actually on musl
86
+ if command -v ldd &> /dev/null && ldd --version 2>&1 | grep -q musl; then
87
+ log "✅ musl libc detected"
88
+ elif [ -f /lib/ld-musl-x86_64.so.1 ]; then
89
+ log "✅ musl detected via loader presence"
90
+ else
91
+ warn "Not clearly detected as musl system. This script is optimized for musl-based distributions (Alpine, etc.)"
92
+ warn "Continuing anyway, but glibc-specific optimizations may be used instead..."
93
+ fi
94
+
95
+ # Check for essential build tools
96
+ local missing_tools=()
97
+
98
+ command -v gcc &> /dev/null || missing_tools+=("gcc")
99
+ command -v perl &> /dev/null || missing_tools+=("perl")
100
+ command -v make &> /dev/null || missing_tools+=("make")
101
+ command -v autoconf &> /dev/null || missing_tools+=("autoconf")
102
+ command -v automake &> /dev/null || missing_tools+=("automake")
103
+ command -v libtool &> /dev/null || missing_tools+=("libtool")
104
+ command -v bison &> /dev/null || missing_tools+=("bison")
105
+ command -v flex &> /dev/null || missing_tools+=("flex")
106
+
107
+ # Check for system libraries that dependencies expect
108
+ if [ ! -f /usr/include/sys/types.h ]; then
109
+ missing_tools+=("musl-dev (for system headers)")
110
+ fi
111
+
112
+ # On Alpine, these packages provide the tools:
113
+ # build-base: gcc, make, musl-dev, libc-dev, etc.
114
+ # linux-headers: kernel headers for system calls
115
+ # pkgconf: pkg-config for dependency detection
116
+ # perl: required by OpenSSL and other build systems
117
+ # autoconf, automake, libtool: GNU autotools for configure scripts
118
+ # bison, flex: parser generators (needed by some dependencies)
119
+ # file: for file type detection
120
+
121
+ if [ ${#missing_tools[@]} -gt 0 ]; then
122
+ error "Missing required tools: ${missing_tools[*]}
123
+
124
+ On Alpine Linux, install ALL required packages with:
125
+ apk add build-base linux-headers pkgconf perl \\
126
+ autoconf automake libtool bison flex file
127
+
128
+ Or install individually as needed."
129
+ fi
130
+
131
+ log "✅ All required musl build tools found"
132
+ }
133
+
134
+ # musl-specific compiler setup
135
+ setup_musl_compiler() {
136
+ # musl-specific compiler flags
137
+ export CC="gcc"
138
+ export CFLAGS="-fPIC -O2 -static-libgcc"
139
+ export CXXFLAGS="-fPIC -O2 -static-libgcc"
140
+ export CPPFLAGS=""
141
+ export LDFLAGS="-static-libgcc"
142
+
143
+ log "Applied musl-specific compiler flags"
144
+ }
145
+
146
+ # Build OpenSSL for musl
147
+ build_openssl_musl() {
148
+ local openssl_prefix="$1"
149
+ local openssl_dir="$2"
150
+
151
+ cd "$openssl_dir"
152
+
153
+ # Check both lib and lib64 directories for existing installation
154
+ if [ ! -f "$openssl_prefix/lib/libssl.a" ] && [ ! -f "$openssl_prefix/lib64/libssl.a" ]; then
155
+ log "Configuring and building OpenSSL for musl..."
156
+ make clean 2>/dev/null || true
157
+
158
+ setup_musl_compiler
159
+
160
+ # Create the prefix directory first
161
+ mkdir -p "$openssl_prefix"
162
+
163
+ # Configure OpenSSL for musl (linux-x86_64 works fine)
164
+ ./Configure linux-x86_64 \
165
+ no-shared \
166
+ no-dso \
167
+ no-engine \
168
+ --prefix="$openssl_prefix" \
169
+ --openssldir="$openssl_prefix/ssl"
170
+
171
+ make -j$(get_cpu_count)
172
+
173
+ # Try the install and capture any errors
174
+ log "Installing OpenSSL..."
175
+ if ! make install_sw install_ssldirs 2>&1; then
176
+ warn "install_sw failed, trying full install..."
177
+ make install || {
178
+ # If install fails, check if libraries were actually built
179
+ if [ -f "libssl.a" ] && [ -f "libcrypto.a" ]; then
180
+ log "Install failed but libraries exist, copying manually..."
181
+ mkdir -p "$openssl_prefix/lib" "$openssl_prefix/include"
182
+ cp libssl.a libcrypto.a "$openssl_prefix/lib/"
183
+ cp -r include/openssl "$openssl_prefix/include/" 2>/dev/null || true
184
+ else
185
+ error "OpenSSL build failed - no libraries found"
186
+ fi
187
+ }
188
+ fi
189
+
190
+ # Verify the build - check both possible lib locations
191
+ if [ -f "$openssl_prefix/lib/libssl.a" ] || [ -f "$openssl_prefix/lib64/libssl.a" ]; then
192
+ if [ -f "$openssl_prefix/lib64/libssl.a" ]; then
193
+ log "OpenSSL installed to lib64 directory"
194
+ # Create symlinks in lib for consistency
195
+ mkdir -p "$openssl_prefix/lib"
196
+ ln -sf ../lib64/libssl.a "$openssl_prefix/lib/libssl.a" 2>/dev/null || true
197
+ ln -sf ../lib64/libcrypto.a "$openssl_prefix/lib/libcrypto.a" 2>/dev/null || true
198
+ ln -sf ../lib64/pkgconfig "$openssl_prefix/lib/pkgconfig" 2>/dev/null || true
199
+ fi
200
+ log "✅ Static OpenSSL built successfully"
201
+ else
202
+ error "Failed to build static OpenSSL - libraries not found in lib or lib64"
203
+ fi
204
+ else
205
+ log "Static OpenSSL already built, skipping..."
206
+ fi
207
+ }
208
+
209
+ # Build MIT Kerberos for musl
210
+ build_krb5_musl() {
211
+ local krb5_prefix="$1"
212
+ local krb5_dir="$2"
213
+
214
+ cd "$krb5_dir/src"
215
+
216
+ if [ ! -f "$krb5_prefix/lib/libgssapi_krb5.a" ]; then
217
+ log "Configuring and building MIT Kerberos for musl..."
218
+ make clean 2>/dev/null || true
219
+
220
+ setup_musl_compiler
221
+
222
+ # musl-specific configuration for Kerberos
223
+ # Disable some features that can be problematic on musl
224
+ ./configure \
225
+ --disable-shared \
226
+ --enable-static \
227
+ --prefix="$krb5_prefix" \
228
+ --without-ldap \
229
+ --without-tcl \
230
+ --without-keyutils \
231
+ --disable-rpath \
232
+ --without-system-verto \
233
+ --disable-thread-support \
234
+ --disable-dns-for-realm \
235
+ --without-readline
236
+
237
+ # Build with limited parallelism to avoid musl-specific race conditions
238
+ log "Building Kerberos (will ignore kadmin build failures)..."
239
+ make -j$(( $(get_cpu_count) / 2 )) || {
240
+ log "Full build failed (expected due to kadmin), continuing with libraries..."
241
+ true
242
+ }
243
+
244
+ # Install what was successfully built
245
+ make install || {
246
+ log "Full install failed, installing individual components..."
247
+ make install-mkdirs 2>/dev/null || true
248
+ make -C util install 2>/dev/null || true
249
+ make -C lib install 2>/dev/null || true
250
+ make -C plugins/kdb/db2 install 2>/dev/null || true
251
+ }
252
+
253
+ # Verify we got the essential libraries
254
+ if [ ! -f "$krb5_prefix/lib/libgssapi_krb5.a" ]; then
255
+ error "Failed to build essential Kerberos libraries"
256
+ fi
257
+
258
+ log "✅ Static MIT Kerberos built successfully"
259
+ else
260
+ log "Static MIT Kerberos already built, skipping..."
261
+ fi
262
+ }
263
+
264
+ # Build Cyrus SASL for musl with proper OpenSSL linking
265
+ build_sasl_musl() {
266
+ local sasl_prefix="$1"
267
+ local sasl_dir="$2"
268
+ local krb5_prefix="$3"
269
+ local openssl_prefix="$4" # Add OpenSSL prefix parameter
270
+
271
+ cd "$sasl_dir"
272
+
273
+ if [ ! -f "$sasl_prefix/lib/libsasl2.a" ]; then
274
+ log "Configuring and building Cyrus SASL for musl..."
275
+ make clean 2>/dev/null || true
276
+
277
+ # Apply comprehensive patches for missing time.h includes
278
+ log "Applying comprehensive time.h patches..."
279
+
280
+ # Create backups first
281
+ cp lib/saslutil.c lib/saslutil.c.backup 2>/dev/null || true
282
+ cp plugins/cram.c plugins/cram.c.backup 2>/dev/null || true
283
+
284
+ # Use perl for more reliable in-place editing
285
+ log "Patching lib/saslutil.c..."
286
+ perl -i -pe 's/^#include "saslint\.h"$/#include "saslint.h"\n#include <time.h>/' lib/saslutil.c
287
+
288
+ log "Patching plugins/cram.c..."
289
+ perl -i -pe 's/^#include "plugin_common\.h"$/#include "plugin_common.h"\n#include <time.h>/' plugins/cram.c
290
+
291
+ # Verify patches applied
292
+ if ! grep -q "#include <time.h>" lib/saslutil.c; then
293
+ log "Perl approach failed, using awk fallback for saslutil.c..."
294
+ awk '/^#include "saslint\.h"$/ {print; print "#include <time.h>"; next} {print}' lib/saslutil.c.backup > lib/saslutil.c
295
+ fi
296
+
297
+ if ! grep -q "#include <time.h>" plugins/cram.c; then
298
+ log "Perl approach failed, using awk fallback for cram.c..."
299
+ awk '/^#include "plugin_common\.h"$/ {print; print "#include <time.h>"; next} {print}' plugins/cram.c.backup > plugins/cram.c
300
+ fi
301
+
302
+ # Clean up backup files
303
+ rm -f lib/saslutil.c.backup plugins/cram.c.backup
304
+
305
+ # Verify patches were applied
306
+ if ! grep -q "#include <time.h>" lib/saslutil.c || ! grep -q "#include <time.h>" plugins/cram.c; then
307
+ error "Failed to patch time.h includes - this is required for compilation"
308
+ fi
309
+
310
+ log "✅ time.h patches applied successfully"
311
+
312
+ # Determine correct OpenSSL lib directory
313
+ local openssl_lib_dir
314
+ if [ -f "$openssl_prefix/lib64/libssl.a" ]; then
315
+ openssl_lib_dir="$openssl_prefix/lib64"
316
+ log "Using OpenSSL libraries from lib64"
317
+ else
318
+ openssl_lib_dir="$openssl_prefix/lib"
319
+ log "Using OpenSSL libraries from lib"
320
+ fi
321
+
322
+ # Verify OpenSSL libraries exist
323
+ if [ ! -f "$openssl_lib_dir/libssl.a" ] || [ ! -f "$openssl_lib_dir/libcrypto.a" ]; then
324
+ error "OpenSSL static libraries not found in $openssl_lib_dir"
325
+ fi
326
+
327
+ setup_musl_compiler
328
+
329
+ # Set comprehensive OpenSSL and Kerberos flags
330
+ export CPPFLAGS="$CPPFLAGS -I$krb5_prefix/include -I$openssl_prefix/include"
331
+ export LDFLAGS="$LDFLAGS -L$krb5_prefix/lib -L$openssl_lib_dir"
332
+ export LIBS="-lssl -lcrypto -ldl -lpthread"
333
+
334
+ # Also set PKG_CONFIG_PATH for better library detection
335
+ export PKG_CONFIG_PATH="$openssl_lib_dir/pkgconfig:$krb5_prefix/lib/pkgconfig:${PKG_CONFIG_PATH:-}"
336
+
337
+ log "Environment variables set:"
338
+ log " CPPFLAGS: $CPPFLAGS"
339
+ log " LDFLAGS: $LDFLAGS"
340
+ log " LIBS: $LIBS"
341
+ log " PKG_CONFIG_PATH: $PKG_CONFIG_PATH"
342
+
343
+ # Configure SASL with explicit OpenSSL paths and comprehensive options
344
+ ./configure \
345
+ --disable-shared \
346
+ --enable-static \
347
+ --prefix="$sasl_prefix" \
348
+ --with-openssl="$openssl_prefix" \
349
+ --with-ssl="$openssl_prefix" \
350
+ --enable-gssapi="$krb5_prefix" \
351
+ --without-dblib \
352
+ --disable-gdbm \
353
+ --disable-sample \
354
+ --disable-obsolete_cram_attr \
355
+ --disable-obsolete_digest_attr \
356
+ --without-pam \
357
+ --without-saslauthd \
358
+ --without-pwcheck \
359
+ --without-des \
360
+ --without-authdaemond \
361
+ --disable-java \
362
+ --disable-sql \
363
+ --disable-ldapdb \
364
+ --enable-plain \
365
+ --enable-login \
366
+ --enable-digest \
367
+ --enable-cram \
368
+ --enable-otp \
369
+ --enable-ntlm \
370
+ --enable-scram \
371
+ ac_cv_func_getnameinfo=yes \
372
+ ac_cv_func_getaddrinfo=yes \
373
+ OPENSSL_CFLAGS="-I$openssl_prefix/include" \
374
+ OPENSSL_LIBS="-L$openssl_lib_dir -lssl -lcrypto"
375
+
376
+ log "Configuration completed, starting build..."
377
+
378
+ # Build with reduced parallelism to avoid race conditions
379
+ make -j$(( $(get_cpu_count) / 2 )) || {
380
+ log "Build failed, trying single-threaded build..."
381
+ make clean
382
+ make -j1
383
+ }
384
+
385
+ # Install
386
+ make install
387
+
388
+ # Verify the build
389
+ if [ ! -f "$sasl_prefix/lib/libsasl2.a" ]; then
390
+ error "Failed to build static Cyrus SASL"
391
+ fi
392
+
393
+ # Additional verification - check if OTP plugin was built (this was failing)
394
+ if [ -f "$sasl_prefix/lib/sasl2/libotp.a" ] || [ -f "$sasl_prefix/lib/sasl2/libotp.la" ]; then
395
+ log "✅ OTP plugin built successfully"
396
+ else
397
+ log "⚠️ OTP plugin may not have built, but core SASL library is available"
398
+ fi
399
+
400
+ log "✅ Static Cyrus SASL built successfully"
401
+ else
402
+ log "Static Cyrus SASL already built, skipping..."
403
+ fi
404
+ }
405
+
406
+ # Build static library with musl-specific configuration
407
+ build_static_lib_musl() {
408
+ local lib_name="$1"
409
+ local prefix="$2"
410
+ local source_dir="$3"
411
+ local configure_args="$4"
412
+
413
+ cd "$source_dir"
414
+
415
+ local lib_file="$prefix/lib/lib${lib_name}.a"
416
+ if [ ! -f "$lib_file" ]; then
417
+ log "Configuring and building static $lib_name for musl..."
418
+ make clean 2>/dev/null || true
419
+
420
+ setup_musl_compiler
421
+
422
+ # Run configure with provided arguments
423
+ eval "./configure --prefix=\"$prefix\" $configure_args"
424
+
425
+ make -j$(get_cpu_count)
426
+ make install
427
+
428
+ # Verify the build
429
+ if [ ! -f "$lib_file" ]; then
430
+ error "Failed to build static $lib_name"
431
+ fi
432
+
433
+ log "✅ Static $lib_name built successfully"
434
+ else
435
+ log "Static $lib_name already built, skipping..."
436
+ fi
437
+ }
438
+
439
+ # Check common and musl-specific dependencies
440
+ check_common_dependencies
441
+ check_musl_dependencies
442
+
443
+ # Auto-detect librdkafka tarball
444
+ log "Looking for librdkafka tarball in $DIST_DIR..."
445
+ LIBRDKAFKA_TARBALL=$(find_librdkafka_tarball "$DIST_DIR")
446
+ log "Found librdkafka tarball: $LIBRDKAFKA_TARBALL"
447
+
448
+ # Verify librdkafka tarball checksum if available
449
+ verify_librdkafka_checksum "$LIBRDKAFKA_TARBALL"
450
+
451
+ # Find patches
452
+ PATCHES_FOUND=()
453
+ find_patches "$PATCHES_DIR" PATCHES_FOUND
454
+
455
+ security_log "Starting secure build with checksum verification enabled"
456
+ log "Building self-contained librdkafka.so for Linux x86_64 musl (WITH SASL)"
457
+ log "Dependencies to build:"
458
+ log " - OpenSSL: $OPENSSL_VERSION"
459
+ log " - Cyrus SASL: $CYRUS_SASL_VERSION (with official GCC 14+ patches)"
460
+ log " - MIT Kerberos: $KRB5_VERSION"
461
+ log " - zlib: $ZLIB_VERSION"
462
+ log " - ZStd: $ZSTD_VERSION"
463
+ log "librdkafka source: $LIBRDKAFKA_TARBALL"
464
+ log "Build directory: $BUILD_DIR"
465
+
466
+ # Create build directory
467
+ mkdir -p "$BUILD_DIR"
468
+ cd "$BUILD_DIR"
469
+
470
+ # Build OpenSSL
471
+ log "Building OpenSSL $OPENSSL_VERSION for musl..."
472
+ OPENSSL_PREFIX="$DEPS_PREFIX/static-openssl-$OPENSSL_VERSION"
473
+ OPENSSL_TARBALL="openssl-$OPENSSL_VERSION.tar.gz"
474
+ OPENSSL_DIR="openssl-$OPENSSL_VERSION"
475
+
476
+ secure_download "$(get_openssl_url)" "$OPENSSL_TARBALL"
477
+ extract_if_needed "$OPENSSL_TARBALL" "$OPENSSL_DIR"
478
+ build_openssl_musl "$OPENSSL_PREFIX" "$OPENSSL_DIR"
479
+
480
+ cd "$BUILD_DIR"
481
+
482
+ # Determine OpenSSL lib directory (could be lib or lib64)
483
+ if [ -f "$OPENSSL_PREFIX/lib64/libssl.a" ]; then
484
+ OPENSSL_LIB_DIR="$OPENSSL_PREFIX/lib64"
485
+ log "Using OpenSSL libraries from lib64"
486
+ else
487
+ OPENSSL_LIB_DIR="$OPENSSL_PREFIX/lib"
488
+ log "Using OpenSSL libraries from lib"
489
+ fi
490
+
491
+ # Build MIT Kerberos
492
+ log "Building MIT Kerberos $KRB5_VERSION for musl..."
493
+ KRB5_PREFIX="$DEPS_PREFIX/static-krb5-$KRB5_VERSION"
494
+ KRB5_TARBALL="krb5-$KRB5_VERSION.tar.gz"
495
+ KRB5_DIR="krb5-$KRB5_VERSION"
496
+
497
+ secure_download "$(get_krb5_url)" "$KRB5_TARBALL"
498
+ extract_if_needed "$KRB5_TARBALL" "$KRB5_DIR"
499
+ build_krb5_musl "$KRB5_PREFIX" "$KRB5_DIR"
500
+
501
+ cd "$BUILD_DIR"
502
+
503
+ # Build SASL with official patches (updated call)
504
+ log "Building Cyrus SASL $CYRUS_SASL_VERSION for musl..."
505
+ SASL_PREFIX="$DEPS_PREFIX/static-sasl-$CYRUS_SASL_VERSION"
506
+ SASL_TARBALL="cyrus-sasl-$CYRUS_SASL_VERSION.tar.gz"
507
+ SASL_DIR="cyrus-sasl-$CYRUS_SASL_VERSION"
508
+
509
+ secure_download "$(get_sasl_url)" "$SASL_TARBALL"
510
+ extract_if_needed "$SASL_TARBALL" "$SASL_DIR"
511
+ build_sasl_musl "$SASL_PREFIX" "$SASL_DIR" "$KRB5_PREFIX" "$OPENSSL_PREFIX" # Added OPENSSL_PREFIX parameter
512
+
513
+ cd "$BUILD_DIR"
514
+
515
+ # Build zlib
516
+ log "Building zlib $ZLIB_VERSION for musl..."
517
+ ZLIB_PREFIX="$DEPS_PREFIX/static-zlib-$ZLIB_VERSION"
518
+ ZLIB_TARBALL="zlib-$ZLIB_VERSION.tar.gz"
519
+ ZLIB_DIR="zlib-$ZLIB_VERSION"
520
+
521
+ secure_download "$(get_zlib_url)" "$ZLIB_TARBALL"
522
+ extract_if_needed "$ZLIB_TARBALL" "$ZLIB_DIR"
523
+ build_static_lib_musl "z" "$ZLIB_PREFIX" "$ZLIB_DIR" "--static"
524
+
525
+ cd "$BUILD_DIR"
526
+
527
+ # Build ZStd
528
+ log "Building ZStd $ZSTD_VERSION for musl..."
529
+ ZSTD_PREFIX="$DEPS_PREFIX/static-zstd-$ZSTD_VERSION"
530
+ ZSTD_TARBALL="zstd-$ZSTD_VERSION.tar.gz"
531
+ ZSTD_DIR="zstd-$ZSTD_VERSION"
532
+
533
+ secure_download "$(get_zstd_url)" "$ZSTD_TARBALL"
534
+ extract_if_needed "$ZSTD_TARBALL" "$ZSTD_DIR"
535
+ cd "$ZSTD_DIR"
536
+
537
+ if [ ! -f "$ZSTD_PREFIX/lib/libzstd.a" ]; then
538
+ log "Building ZStd for musl..."
539
+ make clean 2>/dev/null || true
540
+
541
+ setup_musl_compiler
542
+
543
+ # Build static library using ZStd's Makefile
544
+ make lib-mt CFLAGS="$CFLAGS" PREFIX="$ZSTD_PREFIX" -j$(get_cpu_count)
545
+ make install PREFIX="$ZSTD_PREFIX"
546
+
547
+ # Verify the build
548
+ if [ ! -f "$ZSTD_PREFIX/lib/libzstd.a" ]; then
549
+ error "Failed to build static ZStd"
550
+ fi
551
+
552
+ log "✅ Static ZStd built successfully"
553
+ else
554
+ log "Static ZStd already built, skipping..."
555
+ fi
556
+
557
+ cd "$BUILD_DIR"
558
+
559
+ # Extract and patch librdkafka
560
+ log "Extracting librdkafka..."
561
+ tar xzf "$LIBRDKAFKA_TARBALL"
562
+ cd "librdkafka-$LIBRDKAFKA_VERSION"
563
+
564
+ # Fix permissions and apply patches
565
+ fix_configure_permissions
566
+ apply_patches PATCHES_FOUND
567
+
568
+ # Configure librdkafka for musl (with SASL)
569
+ log "Configuring librdkafka for musl (with SASL support)..."
570
+
571
+ setup_musl_compiler
572
+
573
+ # musl-specific configuration with SASL
574
+ export CPPFLAGS="$CPPFLAGS -I$KRB5_PREFIX/include -I$SASL_PREFIX/include"
575
+ export LDFLAGS="$LDFLAGS -L$KRB5_PREFIX/lib -L$SASL_PREFIX/lib"
576
+
577
+ if [ -f configure ]; then
578
+ log "Using mklove configure script"
579
+
580
+ # mklove-specific configure options with SASL
581
+ ./configure \
582
+ --enable-static \
583
+ --disable-shared \
584
+ --disable-curl \
585
+ --enable-gssapi \
586
+ --enable-sasl \
587
+ --disable-c11threads
588
+ else
589
+ error "No configure script found"
590
+ fi
591
+
592
+ # Build librdkafka
593
+ log "Compiling librdkafka..."
594
+ make clean || true
595
+ make -j$(get_cpu_count)
596
+
597
+ # Verify librdkafka.a exists
598
+ if [ ! -f src/librdkafka.a ]; then
599
+ error "librdkafka.a not found after build"
600
+ fi
601
+
602
+ log "librdkafka.a built successfully"
603
+
604
+ # Create self-contained shared library for musl (with SASL)
605
+ log "Creating self-contained librdkafka.so for musl (with SASL support)..."
606
+
607
+ # First, verify all static libraries exist
608
+ log "Verifying static libraries exist..."
609
+ for lib_path in \
610
+ "$SASL_PREFIX/lib/libsasl2.a" \
611
+ "$KRB5_PREFIX/lib/libgssapi_krb5.a" \
612
+ "$KRB5_PREFIX/lib/libkrb5.a" \
613
+ "$KRB5_PREFIX/lib/libk5crypto.a" \
614
+ "$KRB5_PREFIX/lib/libcom_err.a" \
615
+ "$KRB5_PREFIX/lib/libkrb5support.a" \
616
+ "$OPENSSL_LIB_DIR/libssl.a" \
617
+ "$OPENSSL_LIB_DIR/libcrypto.a" \
618
+ "$ZLIB_PREFIX/lib/libz.a" \
619
+ "$ZSTD_PREFIX/lib/libzstd.a"
620
+ do
621
+ if [ ! -f "$lib_path" ]; then
622
+ error "Required static library not found: $lib_path"
623
+ else
624
+ log "✅ Found: $lib_path"
625
+ fi
626
+ done
627
+
628
+ gcc -shared -fPIC \
629
+ -Wl,--whole-archive src/librdkafka.a -Wl,--no-whole-archive \
630
+ -o librdkafka.so \
631
+ -Wl,-Bstatic \
632
+ "$SASL_PREFIX/lib/libsasl2.a" \
633
+ "$KRB5_PREFIX/lib/libgssapi_krb5.a" \
634
+ "$KRB5_PREFIX/lib/libkrb5.a" \
635
+ "$KRB5_PREFIX/lib/libk5crypto.a" \
636
+ "$KRB5_PREFIX/lib/libcom_err.a" \
637
+ "$KRB5_PREFIX/lib/libkrb5support.a" \
638
+ "$OPENSSL_LIB_DIR/libssl.a" \
639
+ "$OPENSSL_LIB_DIR/libcrypto.a" \
640
+ "$ZLIB_PREFIX/lib/libz.a" \
641
+ "$ZSTD_PREFIX/lib/libzstd.a" \
642
+ -Wl,-Bdynamic \
643
+ -lpthread -lm -ldl -lc \
644
+ -static-libgcc \
645
+ -Wl,--as-needed \
646
+ -Wl,--no-undefined
647
+
648
+ if [ ! -f librdkafka.so ]; then
649
+ error "Failed to create librdkafka.so"
650
+ fi
651
+
652
+ log "librdkafka.so created successfully"
653
+
654
+ # Enhanced verification
655
+ log "Verifying static linking of SASL..."
656
+ if command -v ldd &> /dev/null; then
657
+ SASL_DEPS=$(ldd librdkafka.so 2>/dev/null | grep -i sasl || true)
658
+ if [ -n "$SASL_DEPS" ]; then
659
+ error "SASL is still dynamically linked: $SASL_DEPS"
660
+ else
661
+ log "✅ SASL successfully statically linked - no dynamic SASL dependencies found"
662
+ fi
663
+
664
+ # Check for any other problematic dynamic dependencies
665
+ PROBLEMATIC_DEPS=$(ldd librdkafka.so 2>/dev/null | grep -E "(libssl|libcrypto|libz|libzstd|libkrb|libgssapi)" || true)
666
+ if [ -n "$PROBLEMATIC_DEPS" ]; then
667
+ warn "Found other dynamic dependencies that should be static: $PROBLEMATIC_DEPS"
668
+ else
669
+ log "✅ All major dependencies appear to be statically linked"
670
+ fi
671
+ else
672
+ log "ldd not available, skipping dynamic dependency check"
673
+ fi
674
+
675
+ # Additional verification using nm if available
676
+ if command -v nm &> /dev/null; then
677
+ log "Checking for SASL symbols in the library..."
678
+ SASL_SYMBOLS=$(nm -D librdkafka.so 2>/dev/null | grep -i sasl | head -5 || true)
679
+ if [ -n "$SASL_SYMBOLS" ]; then
680
+ log "✅ SASL symbols found in library (first 5):"
681
+ echo "$SASL_SYMBOLS"
682
+ else
683
+ warn "No SASL symbols found - this might indicate a linking issue"
684
+ fi
685
+ fi
686
+
687
+ # Verify the build (musl-compatible)
688
+ log "Verifying musl build..."
689
+ if command -v file &> /dev/null; then
690
+ file librdkafka.so
691
+ else
692
+ log "file command not available, skipping file type check"
693
+ fi
694
+
695
+ log "Checking dependencies with ldd:"
696
+ if command -v ldd &> /dev/null; then
697
+ ldd librdkafka.so
698
+ else
699
+ log "ldd not available, skipping dependency check"
700
+ fi
701
+
702
+ log "Checking for external dependencies (should only show musl and system libraries):"
703
+ if command -v ldd &> /dev/null; then
704
+ EXTERNAL_DEPS=$(ldd librdkafka.so 2>/dev/null | grep -v "musl" | grep -v "ld-musl" | grep -v "librdkafka.so" | grep -v "=>" | grep -v "statically linked" | grep -v "not a dynamic executable" || true)
705
+ if [ -n "$EXTERNAL_DEPS" ]; then
706
+ # Filter out expected system dependencies that are OK on musl
707
+ FILTERED_DEPS=$(echo "$EXTERNAL_DEPS" | grep -v "linux-vdso.so" | grep -v "/lib/ld-musl" || true)
708
+ if [ -n "$FILTERED_DEPS" ]; then
709
+ warn "Found some external dependencies (may be acceptable for musl): $FILTERED_DEPS"
710
+ else
711
+ log "✅ All dependencies are musl system libraries - build is self-contained!"
712
+ fi
713
+ else
714
+ log "✅ No unexpected external dependencies found - library is self-contained!"
715
+ fi
716
+ else
717
+ log "ldd not available, skipping external dependency check"
718
+ fi
719
+
720
+ # Check for musl compatibility
721
+ log "Verifying musl compatibility..."
722
+ if command -v ldd &> /dev/null; then
723
+ if ldd librdkafka.so 2>&1 | grep -q "musl"; then
724
+ log "✅ Library is linked against musl libc"
725
+ elif ldd librdkafka.so 2>&1 | grep -q "statically linked"; then
726
+ log "✅ Library appears to be statically linked (good for musl)"
727
+ else
728
+ warn "Library linking may not be optimal for musl systems"
729
+ fi
730
+ else
731
+ log "ldd not available, skipping musl compatibility check"
732
+ fi
733
+
734
+ # Copy to output directory
735
+ OUTPUT_DIR="$SCRIPT_DIR"
736
+ cp librdkafka.so "$OUTPUT_DIR/"
737
+ log "librdkafka.so copied to: $OUTPUT_DIR/librdkafka.so"
738
+
739
+ # Print summaries
740
+ print_security_summary
741
+
742
+ # Enhanced summary for musl (with SASL)
743
+ log "Build completed successfully!"
744
+ log "📦 Self-contained librdkafka built for Linux x86_64 musl:"
745
+ log " ✅ OpenSSL $OPENSSL_VERSION (SSL/TLS support) - checksum verified"
746
+ log " ✅ Cyrus SASL $CYRUS_SASL_VERSION (authentication for AWS MSK) - checksum verified"
747
+ log " ✅ MIT Kerberos $KRB5_VERSION (GSSAPI/Kerberos authentication) - checksum verified"
748
+ log " ✅ zlib $ZLIB_VERSION (compression) - checksum verified"
749
+ log " ✅ ZStd $ZSTD_VERSION (high-performance compression) - checksum verified"
750
+ log ""
751
+ log "🎯 Ready for deployment on musl-based systems (Alpine Linux, Docker containers)"
752
+ log "☁️ Compatible with AWS MSK and other secured Kafka clusters"
753
+ log "🔐 Supply chain security: All dependencies cryptographically verified"
754
+ log "🐳 Docker-optimized: Perfect for Alpine-based container deployments"
755
+ log "🛠️ GCC 14+ compatible: Official patches applied to Cyrus SASL"
756
+ log ""
757
+ log "Location: $OUTPUT_DIR/librdkafka.so"
758
+
759
+ # Cleanup
760
+ cleanup_build_dir "$BUILD_DIR"
761
+
762
+ # Reset environment variables
763
+ unset CFLAGS CXXFLAGS CPPFLAGS LDFLAGS CC