karafka-rdkafka 0.20.0.rc2 → 0.20.0.rc5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/ci_linux_x86_64_gnu.yml +249 -0
  3. data/.github/workflows/ci_linux_x86_64_musl.yml +205 -0
  4. data/.github/workflows/ci_macos_arm64.yml +306 -0
  5. data/.github/workflows/push_linux_x86_64_gnu.yml +64 -0
  6. data/.github/workflows/push_linux_x86_64_musl.yml +77 -0
  7. data/.github/workflows/push_macos_arm64.yml +54 -0
  8. data/.github/workflows/push_ruby.yml +37 -0
  9. data/.gitignore +1 -0
  10. data/.ruby-version +1 -1
  11. data/CHANGELOG.md +22 -3
  12. data/README.md +2 -3
  13. data/Rakefile +0 -2
  14. data/dist/{librdkafka-2.10.0.tar.gz → librdkafka-2.8.0.tar.gz} +0 -0
  15. data/docker-compose.yml +1 -1
  16. data/ext/Rakefile +1 -1
  17. data/ext/build_common.sh +361 -0
  18. data/ext/build_linux_x86_64_gnu.sh +306 -0
  19. data/ext/build_linux_x86_64_musl.sh +763 -0
  20. data/ext/build_macos_arm64.sh +550 -0
  21. data/karafka-rdkafka.gemspec +26 -10
  22. data/lib/rdkafka/bindings.rb +31 -4
  23. data/lib/rdkafka/config.rb +4 -1
  24. data/lib/rdkafka/error.rb +8 -1
  25. data/lib/rdkafka/native_kafka.rb +4 -0
  26. data/lib/rdkafka/producer/partitions_count_cache.rb +216 -0
  27. data/lib/rdkafka/producer.rb +40 -28
  28. data/lib/rdkafka/version.rb +3 -3
  29. data/lib/rdkafka.rb +1 -0
  30. data/renovate.json +74 -0
  31. data/spec/rdkafka/admin_spec.rb +15 -2
  32. data/spec/rdkafka/bindings_spec.rb +0 -1
  33. data/spec/rdkafka/config_spec.rb +1 -1
  34. data/spec/rdkafka/consumer_spec.rb +35 -14
  35. data/spec/rdkafka/metadata_spec.rb +2 -2
  36. data/spec/rdkafka/producer/partitions_count_cache_spec.rb +359 -0
  37. data/spec/rdkafka/producer/partitions_count_spec.rb +359 -0
  38. data/spec/rdkafka/producer_spec.rb +198 -7
  39. data/spec/spec_helper.rb +12 -1
  40. metadata +43 -100
  41. checksums.yaml.gz.sig +0 -0
  42. data/.github/workflows/ci.yml +0 -99
  43. data/Guardfile +0 -19
  44. data/certs/cert.pem +0 -26
  45. data.tar.gz.sig +0 -0
  46. metadata.gz.sig +0 -3
@@ -0,0 +1,361 @@
1
+ #!/usr/bin/env bash
2
+ #
3
+ # Common functions and constants for librdkafka builds
4
+ # This file should be sourced by platform-specific build scripts
5
+ #
6
+ # Usage: source "$(dirname "${BASH_SOURCE[0]}")/build_common.sh"
7
+ #
8
+
9
+ # Prevent multiple sourcing
10
+ if [[ "${BUILD_COMMON_SOURCED:-}" == "1" ]]; then
11
+ return 0
12
+ fi
13
+
14
+ BUILD_COMMON_SOURCED=1
15
+
16
+ # Version constants - update these to upgrade dependencies
17
+ readonly OPENSSL_VERSION="3.0.16"
18
+ readonly CYRUS_SASL_VERSION="2.1.28"
19
+ readonly ZLIB_VERSION="1.3.1"
20
+ readonly ZSTD_VERSION="1.5.7"
21
+ readonly KRB5_VERSION="1.21.3"
22
+ readonly LIBRDKAFKA_VERSION="2.8.0"
23
+
24
+ # SHA256 checksums for supply chain security
25
+ # Update these when upgrading versions
26
+ declare -A CHECKSUMS=(
27
+ ["openssl-${OPENSSL_VERSION}.tar.gz"]="57e03c50feab5d31b152af2b764f10379aecd8ee92f16c985983ce4a99f7ef86"
28
+ ["cyrus-sasl-${CYRUS_SASL_VERSION}.tar.gz"]="7ccfc6abd01ed67c1a0924b353e526f1b766b21f42d4562ee635a8ebfc5bb38c"
29
+ ["zlib-1.3.1.tar.gz"]="9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23"
30
+ ["zstd-${ZSTD_VERSION}.tar.gz"]="eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3"
31
+ ["krb5-${KRB5_VERSION}.tar.gz"]="b7a4cd5ead67fb08b980b21abd150ff7217e85ea320c9ed0c6dadd304840ad35"
32
+ ["librdkafka-${LIBRDKAFKA_VERSION}.tar.gz"]="5bd1c46f63265f31c6bfcedcde78703f77d28238eadf23821c2b43fc30be3e25"
33
+ )
34
+
35
+ # Colors for output
36
+ readonly RED='\033[0;31m'
37
+ readonly GREEN='\033[0;32m'
38
+ readonly YELLOW='\033[1;33m'
39
+ readonly BLUE='\033[0;34m'
40
+ readonly NC='\033[0m' # No Color
41
+
42
+ # Logging functions
43
+ log() {
44
+ echo -e "${GREEN}[$(date '+%Y-%m-%d %H:%M:%S')] $1${NC}"
45
+ }
46
+
47
+ warn() {
48
+ echo -e "${YELLOW}[WARNING] $1${NC}"
49
+ }
50
+
51
+ error() {
52
+ echo -e "${RED}[ERROR] $1${NC}"
53
+ exit 1
54
+ }
55
+
56
+ security_log() {
57
+ echo -e "${BLUE}[SECURITY] $1${NC}"
58
+ }
59
+
60
+ # Function to verify checksums
61
+ verify_checksum() {
62
+ local file="$1"
63
+ local expected_checksum="${CHECKSUMS[$file]}"
64
+
65
+ if [ -z "$expected_checksum" ]; then
66
+ error "No checksum defined for $file - this is a security risk!"
67
+ fi
68
+
69
+ security_log "Verifying checksum for $file..."
70
+ local actual_checksum
71
+
72
+ # Use platform-appropriate checksum command
73
+ if command -v sha256sum &> /dev/null; then
74
+ actual_checksum=$(sha256sum "$file" | cut -d' ' -f1)
75
+ elif command -v shasum &> /dev/null; then
76
+ actual_checksum=$(shasum -a 256 "$file" | cut -d' ' -f1)
77
+ else
78
+ error "No SHA256 checksum utility found (tried sha256sum, shasum)"
79
+ fi
80
+
81
+ if [ "$actual_checksum" = "$expected_checksum" ]; then
82
+ security_log "✅ Checksum verified for $file"
83
+ return 0
84
+ else
85
+ error "❌ CHECKSUM MISMATCH for $file!
86
+ Expected: $expected_checksum
87
+ Actual: $actual_checksum
88
+ This could indicate a supply chain attack or corrupted download!"
89
+ fi
90
+ }
91
+
92
+ # Function to securely download and verify files
93
+ secure_download() {
94
+ local url="$1"
95
+ local filename="$2"
96
+
97
+ if [ -f "$filename" ]; then
98
+ log "File $filename already exists, verifying checksum..."
99
+ verify_checksum "$filename"
100
+ return 0
101
+ fi
102
+
103
+ log "Downloading $filename from $url..."
104
+
105
+ # Use platform-appropriate download command
106
+ if command -v wget &> /dev/null; then
107
+ # Linux - use wget with security options
108
+ if ! wget --secure-protocol=TLSv1_2 \
109
+ --https-only \
110
+ --timeout=30 \
111
+ --tries=3 \
112
+ --progress=bar \
113
+ "$url" \
114
+ -O "$filename"; then
115
+ error "Failed to download $filename from $url"
116
+ fi
117
+ elif command -v curl &> /dev/null; then
118
+ # macOS/fallback - use curl with security options
119
+ if ! curl -L \
120
+ --tlsv1.2 \
121
+ --connect-timeout 30 \
122
+ --max-time 300 \
123
+ --retry 3 \
124
+ --progress-bar \
125
+ "$url" \
126
+ -o "$filename"; then
127
+ error "Failed to download $filename from $url"
128
+ fi
129
+ else
130
+ error "No download utility found (tried wget, curl)"
131
+ fi
132
+
133
+ # Verify checksum immediately after download
134
+ verify_checksum "$filename"
135
+ }
136
+
137
+ # Function to detect CPU count for parallel builds
138
+ get_cpu_count() {
139
+ if command -v nproc &> /dev/null; then
140
+ nproc
141
+ elif command -v sysctl &> /dev/null; then
142
+ sysctl -n hw.ncpu
143
+ else
144
+ echo "4" # fallback
145
+ fi
146
+ }
147
+
148
+ # Function to auto-detect librdkafka tarball
149
+ find_librdkafka_tarball() {
150
+ local dist_dir="$1"
151
+ local tarball="$dist_dir/librdkafka-${LIBRDKAFKA_VERSION}.tar.gz"
152
+
153
+ if [ ! -f "$tarball" ]; then
154
+ error "librdkafka-${LIBRDKAFKA_VERSION}.tar.gz not found in $dist_dir"
155
+ fi
156
+
157
+ echo "$tarball"
158
+ }
159
+
160
+ # Function to find and validate patches
161
+ find_patches() {
162
+ local patches_dir="$1"
163
+ local -n patches_array=$2 # nameref to output array
164
+
165
+ patches_array=()
166
+
167
+ if [ -d "$patches_dir" ]; then
168
+ while IFS= read -r -d '' patch; do
169
+ patches_array+=("$patch")
170
+ done < <(find "$patches_dir" -name "*.patch" -type f -print0 | sort -z)
171
+
172
+ if [ ${#patches_array[@]} -gt 0 ]; then
173
+ log "Found ${#patches_array[@]} patches to apply:"
174
+ for patch in "${patches_array[@]}"; do
175
+ log " - $(basename "$patch")"
176
+ done
177
+ else
178
+ log "No patches found in $patches_dir"
179
+ fi
180
+ else
181
+ log "No patches directory found: $patches_dir"
182
+ fi
183
+ }
184
+
185
+ # Function to apply patches
186
+ apply_patches() {
187
+ local -n patches_array=$1 # nameref to patches array
188
+
189
+ if [ ${#patches_array[@]} -gt 0 ]; then
190
+ log "Applying Ruby-specific patches..."
191
+ for patch in "${patches_array[@]}"; do
192
+ log "Applying patch: $(basename "$patch")"
193
+ if patch -p1 < "$patch"; then
194
+ log "✅ Successfully applied $(basename "$patch")"
195
+ else
196
+ error "❌ Failed to apply patch: $(basename "$patch")"
197
+ fi
198
+ done
199
+ log "All patches applied successfully"
200
+ fi
201
+ }
202
+
203
+ # Function to verify librdkafka tarball checksum if available
204
+ verify_librdkafka_checksum() {
205
+ local tarball="$1"
206
+ local filename
207
+ filename=$(basename "$tarball")
208
+
209
+ if [ -n "${CHECKSUMS[$filename]:-}" ]; then
210
+ local current_dir
211
+ current_dir=$(pwd)
212
+ cd "$(dirname "$tarball")"
213
+ verify_checksum "$filename"
214
+ cd "$current_dir"
215
+ else
216
+ warn "No checksum defined for $filename - consider adding one for security"
217
+ fi
218
+ }
219
+
220
+ # Function to set execute permissions on configure scripts
221
+ fix_configure_permissions() {
222
+ log "Setting execute permissions on configure scripts..."
223
+ chmod +x configure* 2>/dev/null || true
224
+ chmod +x mklove/modules/configure.* 2>/dev/null || true
225
+ }
226
+
227
+ # Function to print security summary
228
+ print_security_summary() {
229
+ security_log "🔒 SECURITY VERIFICATION COMPLETE"
230
+ security_log "All dependencies downloaded and verified with SHA256 checksums"
231
+ security_log "Supply chain integrity maintained throughout build process"
232
+ }
233
+
234
+ # Function to print build summary
235
+ print_build_summary() {
236
+ local platform="$1"
237
+ local arch="$2"
238
+ local output_dir="$3"
239
+ local library_name="$4"
240
+
241
+ log "Build completed successfully!"
242
+ log "📦 Self-contained librdkafka built for $platform $arch:"
243
+ log " ✅ OpenSSL $OPENSSL_VERSION (SSL/TLS support) - checksum verified"
244
+ log " ✅ Cyrus SASL $CYRUS_SASL_VERSION (authentication for AWS MSK) - checksum verified"
245
+ log " ✅ MIT Kerberos $KRB5_VERSION (GSSAPI/Kerberos authentication) - checksum verified"
246
+ log " ✅ zlib $ZLIB_VERSION (compression) - checksum verified"
247
+ log " ✅ ZStd $ZSTD_VERSION (high-performance compression) - checksum verified"
248
+ log ""
249
+ log "🎯 Ready for deployment on $platform systems"
250
+ log "☁️ Compatible with AWS MSK and other secured Kafka clusters"
251
+ log "🔐 Supply chain security: All dependencies cryptographically verified"
252
+ log ""
253
+ log "Location: $output_dir/$library_name"
254
+ }
255
+
256
+ # Function to clean up build directory with user prompt (except .tar.gz files in CI)
257
+ cleanup_build_dir() {
258
+ local build_dir="$1"
259
+
260
+ if [ "${CI:-}" = "true" ]; then
261
+ # In CI: remove everything except .tar.gz files without prompting
262
+ echo "CI detected: cleaning up $build_dir (preserving .tar.gz files for caching)"
263
+
264
+ # First, find and move all .tar.gz files to a temp location
265
+ temp_dir=$(mktemp -d)
266
+ find "$build_dir" -name "*.tar.gz" -exec mv {} "$temp_dir/" \; 2>/dev/null || true
267
+
268
+ # Remove everything in build_dir
269
+ rm -rf "$build_dir"/* 2>/dev/null || true
270
+ rm -rf "$build_dir"/.* 2>/dev/null || true
271
+
272
+ # Move .tar.gz files back
273
+ mv "$temp_dir"/* "$build_dir/" 2>/dev/null || true
274
+ rmdir "$temp_dir" 2>/dev/null || true
275
+
276
+ log "Build directory cleaned up (preserved .tar.gz files)"
277
+ else
278
+ # Interactive mode: prompt user
279
+ echo
280
+ read -p "Remove build directory $build_dir? (y/N): " -n 1 -r
281
+ echo
282
+ if [[ $REPLY =~ ^[Yy]$ ]]; then
283
+ rm -rf "$build_dir"
284
+ log "Build directory cleaned up"
285
+ fi
286
+ fi
287
+ }
288
+
289
+ # Function to validate build environment
290
+ check_common_dependencies() {
291
+ log "Checking common build dependencies..."
292
+
293
+ local missing_tools=()
294
+
295
+ command -v tar &> /dev/null || missing_tools+=("tar")
296
+ command -v make &> /dev/null || missing_tools+=("make")
297
+ command -v patch &> /dev/null || missing_tools+=("patch")
298
+
299
+ # Check for download tools
300
+ if ! command -v wget &> /dev/null && ! command -v curl &> /dev/null; then
301
+ missing_tools+=("wget or curl")
302
+ fi
303
+
304
+ # Check for checksum tools
305
+ if ! command -v sha256sum &> /dev/null && ! command -v shasum &> /dev/null; then
306
+ missing_tools+=("sha256sum or shasum")
307
+ fi
308
+
309
+ if [ ${#missing_tools[@]} -gt 0 ]; then
310
+ error "Missing required tools: ${missing_tools[*]}"
311
+ fi
312
+
313
+ log "✅ Common build tools found"
314
+ }
315
+
316
+ # Function to extract tarball if directory doesn't exist
317
+ extract_if_needed() {
318
+ local tarball="$1"
319
+ local expected_dir="$2"
320
+
321
+ if [ ! -d "$expected_dir" ]; then
322
+ log "Extracting $(basename "$tarball")..."
323
+ tar xzf "$tarball"
324
+ else
325
+ log "Directory $expected_dir already exists, skipping extraction"
326
+ fi
327
+ }
328
+
329
+ # Download URLs for dependencies
330
+ get_openssl_url() {
331
+ echo "https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz"
332
+ }
333
+
334
+ get_sasl_url() {
335
+ echo "https://github.com/cyrusimap/cyrus-sasl/releases/download/cyrus-sasl-${CYRUS_SASL_VERSION}/cyrus-sasl-${CYRUS_SASL_VERSION}.tar.gz"
336
+ }
337
+
338
+ get_zlib_url() {
339
+ echo "https://github.com/madler/zlib/releases/download/v${ZLIB_VERSION}/zlib-${ZLIB_VERSION}.tar.gz"
340
+ }
341
+
342
+ get_zstd_url() {
343
+ echo "https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz"
344
+ }
345
+
346
+ get_krb5_url() {
347
+ echo "https://kerberos.org/dist/krb5/${KRB5_VERSION%.*}/krb5-${KRB5_VERSION}.tar.gz"
348
+ }
349
+
350
+ # Export functions and variables that scripts will need
351
+ export -f log warn error security_log
352
+ export -f verify_checksum secure_download get_cpu_count
353
+ export -f find_librdkafka_tarball find_patches apply_patches
354
+ export -f verify_librdkafka_checksum fix_configure_permissions
355
+ export -f print_security_summary print_build_summary cleanup_build_dir
356
+ export -f check_common_dependencies extract_if_needed
357
+ export -f get_openssl_url get_sasl_url get_zlib_url get_zstd_url get_krb5_url
358
+
359
+ # Export constants
360
+ export OPENSSL_VERSION CYRUS_SASL_VERSION ZLIB_VERSION ZSTD_VERSION KRB5_VERSION
361
+ export RED GREEN YELLOW BLUE NC
@@ -0,0 +1,306 @@
1
+ #!/usr/bin/env bash
2
+ #
3
+ # Build self-contained librdkafka.so for Linux x86_64 with checksum verification
4
+ # Usage: ./build-librdkafka-linux.sh
5
+ #
6
+ # Expected directory structure:
7
+ # ext/build_linux_x86_64_gnu.sh (this script)
8
+ # ext/build-common.sh (shared functions)
9
+ # dist/librdkafka-*.tar.gz (librdkafka source tarball)
10
+ # dist/patches/*.patch (optional Ruby-specific patches)
11
+ #
12
+ set -euo pipefail
13
+
14
+ # Source common functions and constants
15
+ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
16
+ source "$SCRIPT_DIR/build_common.sh"
17
+
18
+ # Platform-specific paths
19
+ DIST_DIR="$SCRIPT_DIR/../dist"
20
+ PATCHES_DIR="$DIST_DIR/patches"
21
+ BUILD_DIR="$(pwd)/build-tmp"
22
+ DEPS_PREFIX="/tmp"
23
+
24
+ # Check common dependencies
25
+ check_common_dependencies
26
+
27
+ # Linux-specific dependency check
28
+ log "Checking Linux-specific build dependencies..."
29
+ command -v gcc &> /dev/null || error "gcc not found. Install with: apt-get install build-essential"
30
+
31
+ # Auto-detect librdkafka tarball
32
+ log "Looking for librdkafka tarball in $DIST_DIR..."
33
+ LIBRDKAFKA_TARBALL=$(find_librdkafka_tarball "$DIST_DIR")
34
+ log "Found librdkafka tarball: $LIBRDKAFKA_TARBALL"
35
+
36
+ # Verify librdkafka tarball checksum if available
37
+ verify_librdkafka_checksum "$LIBRDKAFKA_TARBALL"
38
+
39
+ # Find patches
40
+ PATCHES_FOUND=()
41
+ find_patches "$PATCHES_DIR" PATCHES_FOUND
42
+
43
+ security_log "Starting secure build with checksum verification enabled"
44
+ log "Building self-contained librdkafka.so for Linux x86_64 GNU"
45
+ log "Dependencies to build:"
46
+ log " - OpenSSL: $OPENSSL_VERSION"
47
+ log " - Cyrus SASL: $CYRUS_SASL_VERSION"
48
+ log " - MIT Kerberos: $KRB5_VERSION"
49
+ log " - zlib: $ZLIB_VERSION"
50
+ log " - ZStd: $ZSTD_VERSION"
51
+ log "librdkafka source: $LIBRDKAFKA_TARBALL"
52
+ log "Build directory: $BUILD_DIR"
53
+
54
+ # Create build directory
55
+ mkdir -p "$BUILD_DIR"
56
+ cd "$BUILD_DIR"
57
+
58
+ # Build OpenSSL
59
+ log "Building OpenSSL $OPENSSL_VERSION..."
60
+ OPENSSL_PREFIX="$DEPS_PREFIX/static-openssl-$OPENSSL_VERSION"
61
+ OPENSSL_TARBALL="openssl-$OPENSSL_VERSION.tar.gz"
62
+ OPENSSL_DIR="openssl-$OPENSSL_VERSION"
63
+
64
+ secure_download "$(get_openssl_url)" "$OPENSSL_TARBALL"
65
+ extract_if_needed "$OPENSSL_TARBALL" "$OPENSSL_DIR"
66
+ cd "$OPENSSL_DIR"
67
+
68
+ # Check if OpenSSL lib directory exists (lib or lib64)
69
+ if [ ! -f "$OPENSSL_PREFIX/lib/libssl.a" ] && [ ! -f "$OPENSSL_PREFIX/lib64/libssl.a" ]; then
70
+ log "Configuring and building OpenSSL..."
71
+ export CFLAGS="-fPIC"
72
+ ./Configure linux-x86_64 \
73
+ no-shared \
74
+ no-dso \
75
+ --prefix="$OPENSSL_PREFIX"
76
+ make clean || true
77
+ make -j$(get_cpu_count)
78
+ make install
79
+ unset CFLAGS
80
+ log "OpenSSL built successfully"
81
+ else
82
+ log "OpenSSL already built, skipping..."
83
+ fi
84
+
85
+ # Determine OpenSSL lib directory
86
+ if [ -f "$OPENSSL_PREFIX/lib64/libssl.a" ]; then
87
+ OPENSSL_LIB_DIR="$OPENSSL_PREFIX/lib64"
88
+ else
89
+ OPENSSL_LIB_DIR="$OPENSSL_PREFIX/lib"
90
+ fi
91
+ log "OpenSSL libraries in: $OPENSSL_LIB_DIR"
92
+
93
+ cd "$BUILD_DIR"
94
+
95
+ # Build MIT Kerberos (krb5)
96
+ log "Building MIT Kerberos $KRB5_VERSION..."
97
+ KRB5_PREFIX="$DEPS_PREFIX/static-krb5-$KRB5_VERSION"
98
+ KRB5_TARBALL="krb5-$KRB5_VERSION.tar.gz"
99
+ KRB5_DIR="krb5-$KRB5_VERSION"
100
+
101
+ secure_download "$(get_krb5_url)" "$KRB5_TARBALL"
102
+ extract_if_needed "$KRB5_TARBALL" "$KRB5_DIR"
103
+ cd "$KRB5_DIR/src"
104
+
105
+ if [ ! -f "$KRB5_PREFIX/lib/libgssapi_krb5.a" ]; then
106
+ log "Configuring and building MIT Kerberos..."
107
+ make clean 2>/dev/null || true
108
+ ./configure --disable-shared --enable-static --prefix="$KRB5_PREFIX" \
109
+ --without-ldap --without-tcl --without-keyutils \
110
+ --disable-rpath --without-system-verto \
111
+ CFLAGS="-fPIC" CXXFLAGS="-fPIC"
112
+
113
+ # Build everything except the problematic kadmin tools
114
+ log "Building Kerberos (will ignore kadmin build failures)..."
115
+ make -j$(get_cpu_count) || {
116
+ log "Full build failed (expected due to kadmin), continuing with libraries..."
117
+ # The libraries should be built even if kadmin fails
118
+ true
119
+ }
120
+
121
+ # Install what was successfully built
122
+ make install || {
123
+ log "Full install failed, installing individual components..."
124
+ # Try to install the core libraries manually
125
+ make install-mkdirs 2>/dev/null || true
126
+ make -C util install 2>/dev/null || true
127
+ make -C lib install 2>/dev/null || true
128
+ make -C plugins/kdb/db2 install 2>/dev/null || true
129
+ }
130
+
131
+ # Verify we got the essential libraries
132
+ if [ ! -f "$KRB5_PREFIX/lib/libgssapi_krb5.a" ]; then
133
+ error "Failed to build essential Kerberos libraries"
134
+ fi
135
+
136
+ log "MIT Kerberos libraries built successfully"
137
+ else
138
+ log "MIT Kerberos already built, skipping..."
139
+ fi
140
+
141
+ cd "$BUILD_DIR"
142
+
143
+ # Build SASL
144
+ log "Building Cyrus SASL $CYRUS_SASL_VERSION..."
145
+ SASL_PREFIX="$DEPS_PREFIX/static-sasl-$CYRUS_SASL_VERSION"
146
+ SASL_TARBALL="cyrus-sasl-$CYRUS_SASL_VERSION.tar.gz"
147
+ SASL_DIR="cyrus-sasl-$CYRUS_SASL_VERSION"
148
+
149
+ secure_download "$(get_sasl_url)" "$SASL_TARBALL"
150
+ extract_if_needed "$SASL_TARBALL" "$SASL_DIR"
151
+ cd "$SASL_DIR"
152
+
153
+ if [ ! -f "$SASL_PREFIX/lib/libsasl2.a" ]; then
154
+ log "Configuring and building SASL..."
155
+ make clean 2>/dev/null || true
156
+ ./configure --disable-shared --enable-static --prefix="$SASL_PREFIX" \
157
+ --without-dblib --disable-gdbm \
158
+ --enable-gssapi="$KRB5_PREFIX" \
159
+ CFLAGS="-fPIC" CXXFLAGS="-fPIC" \
160
+ CPPFLAGS="-I$KRB5_PREFIX/include" \
161
+ LDFLAGS="-L$KRB5_PREFIX/lib"
162
+ make -j$(get_cpu_count)
163
+ make install
164
+ log "SASL built successfully"
165
+ else
166
+ log "SASL already built, skipping..."
167
+ fi
168
+
169
+ cd "$BUILD_DIR"
170
+
171
+ # Build zlib
172
+ log "Building zlib $ZLIB_VERSION..."
173
+ ZLIB_PREFIX="$DEPS_PREFIX/static-zlib-$ZLIB_VERSION"
174
+ ZLIB_TARBALL="zlib-$ZLIB_VERSION.tar.gz"
175
+ ZLIB_DIR="zlib-$ZLIB_VERSION"
176
+
177
+ secure_download "$(get_zlib_url)" "$ZLIB_TARBALL"
178
+ extract_if_needed "$ZLIB_TARBALL" "$ZLIB_DIR"
179
+ cd "$ZLIB_DIR"
180
+
181
+ if [ ! -f "$ZLIB_PREFIX/lib/libz.a" ]; then
182
+ log "Configuring and building zlib..."
183
+ make clean 2>/dev/null || true
184
+ export CFLAGS="-fPIC"
185
+ ./configure --prefix="$ZLIB_PREFIX" --static
186
+ make -j$(get_cpu_count)
187
+ make install
188
+ unset CFLAGS
189
+ log "zlib built successfully"
190
+ else
191
+ log "zlib already built, skipping..."
192
+ fi
193
+
194
+ cd "$BUILD_DIR"
195
+
196
+ # Build ZStd
197
+ log "Building ZStd $ZSTD_VERSION..."
198
+ ZSTD_PREFIX="$DEPS_PREFIX/static-zstd-$ZSTD_VERSION"
199
+ ZSTD_TARBALL="zstd-$ZSTD_VERSION.tar.gz"
200
+ ZSTD_DIR="zstd-$ZSTD_VERSION"
201
+
202
+ secure_download "$(get_zstd_url)" "$ZSTD_TARBALL"
203
+ extract_if_needed "$ZSTD_TARBALL" "$ZSTD_DIR"
204
+ cd "$ZSTD_DIR"
205
+
206
+ if [ ! -f "$ZSTD_PREFIX/lib/libzstd.a" ]; then
207
+ log "Building ZStd..."
208
+ make clean 2>/dev/null || true
209
+ make lib-mt CFLAGS="-fPIC" PREFIX="$ZSTD_PREFIX" -j$(get_cpu_count)
210
+ # Use standard install target - install-pc may not exist in all versions
211
+ make install PREFIX="$ZSTD_PREFIX"
212
+ log "ZStd built successfully"
213
+ else
214
+ log "ZStd already built, skipping..."
215
+ fi
216
+
217
+ cd "$BUILD_DIR"
218
+
219
+ # Extract and patch librdkafka
220
+ log "Extracting librdkafka..."
221
+ tar xzf "$LIBRDKAFKA_TARBALL"
222
+ cd "librdkafka-$LIBRDKAFKA_VERSION"
223
+
224
+ # Fix permissions and apply patches
225
+ fix_configure_permissions
226
+ apply_patches PATCHES_FOUND
227
+
228
+ # Configure librdkafka
229
+ log "Configuring librdkafka..."
230
+
231
+ if [ -f configure ]; then
232
+ log "Using standard configure (autotools)"
233
+ # Export environment variables for configure to pick up
234
+ export CPPFLAGS="-I$KRB5_PREFIX/include"
235
+ export LDFLAGS="-L$KRB5_PREFIX/lib"
236
+
237
+ ./configure --enable-static --disable-shared --disable-curl \
238
+ --enable-gssapi
239
+
240
+ # Clean up environment variables
241
+ unset CPPFLAGS LDFLAGS
242
+ else
243
+ error "No configure script found (checked: configure.self, configure)"
244
+ fi
245
+
246
+ # Build librdkafka
247
+ log "Compiling librdkafka..."
248
+ make clean || true
249
+ make -j$(get_cpu_count)
250
+
251
+ # Verify librdkafka.a exists
252
+ if [ ! -f src/librdkafka.a ]; then
253
+ error "librdkafka.a not found after build"
254
+ fi
255
+
256
+ log "librdkafka.a built successfully"
257
+
258
+ # Create self-contained shared library
259
+ log "Creating self-contained librdkafka.so..."
260
+
261
+ gcc -shared -fPIC -Wl,--whole-archive src/librdkafka.a -Wl,--no-whole-archive \
262
+ -o librdkafka.so \
263
+ "$SASL_PREFIX/lib/libsasl2.a" \
264
+ "$KRB5_PREFIX/lib/libgssapi_krb5.a" \
265
+ "$KRB5_PREFIX/lib/libkrb5.a" \
266
+ "$KRB5_PREFIX/lib/libk5crypto.a" \
267
+ "$KRB5_PREFIX/lib/libcom_err.a" \
268
+ "$KRB5_PREFIX/lib/libkrb5support.a" \
269
+ "$OPENSSL_LIB_DIR/libssl.a" \
270
+ "$OPENSSL_LIB_DIR/libcrypto.a" \
271
+ "$ZLIB_PREFIX/lib/libz.a" \
272
+ "$ZSTD_PREFIX/lib/libzstd.a" \
273
+ -lpthread -lm -ldl -lresolv
274
+
275
+ if [ ! -f librdkafka.so ]; then
276
+ error "Failed to create librdkafka.so"
277
+ fi
278
+
279
+ log "librdkafka.so created successfully"
280
+
281
+ # Verify the build
282
+ log "Verifying build..."
283
+ file librdkafka.so
284
+
285
+ log "Checking dependencies with ldd:"
286
+ ldd librdkafka.so
287
+
288
+ log "Checking for external dependencies (should only show system libraries):"
289
+ EXTERNAL_DEPS=$(nm -D librdkafka.so | grep " U " | grep -v "@GLIBC" || true)
290
+ if [ -n "$EXTERNAL_DEPS" ]; then
291
+ error "Found external dependencies - library is not self-contained: $EXTERNAL_DEPS"
292
+ else
293
+ log "✅ No external dependencies found - library is self-contained!"
294
+ fi
295
+
296
+ # Copy to output directory
297
+ OUTPUT_DIR="$SCRIPT_DIR"
298
+ cp librdkafka.so "$OUTPUT_DIR/"
299
+ log "librdkafka.so copied to: $OUTPUT_DIR/librdkafka.so"
300
+
301
+ # Print summaries
302
+ print_security_summary
303
+ print_build_summary "Linux" "x86_64" "$OUTPUT_DIR" "librdkafka.so"
304
+
305
+ # Cleanup
306
+ cleanup_build_dir "$BUILD_DIR"