karafka-rdkafka 0.21.0-x86_64-linux-musl → 0.22.0.rc1-x86_64-linux-musl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +7 -0
- data/README.md +1 -0
- data/ext/librdkafka.so +0 -0
- data/karafka-rdkafka.gemspec +33 -6
- data/lib/rdkafka/version.rb +3 -3
- data/renovate.json +5 -17
- metadata +2 -67
- data/.github/CODEOWNERS +0 -3
- data/.github/FUNDING.yml +0 -1
- data/.github/workflows/ci_linux_alpine_x86_64_musl.yml +0 -197
- data/.github/workflows/ci_linux_alpine_x86_64_musl_complementary.yml +0 -264
- data/.github/workflows/ci_linux_debian_x86_64_gnu.yml +0 -271
- data/.github/workflows/ci_linux_debian_x86_64_gnu_complementary.yml +0 -334
- data/.github/workflows/ci_linux_ubuntu_aarch64_gnu.yml +0 -271
- data/.github/workflows/ci_linux_ubuntu_aarch64_gnu_complementary.yml +0 -295
- data/.github/workflows/ci_linux_ubuntu_x86_64_gnu.yml +0 -281
- data/.github/workflows/ci_linux_ubuntu_x86_64_gnu_complementary.yml +0 -294
- data/.github/workflows/ci_macos_arm64.yml +0 -284
- data/.github/workflows/push_linux_aarch64_gnu.yml +0 -65
- data/.github/workflows/push_linux_x86_64_gnu.yml +0 -65
- data/.github/workflows/push_linux_x86_64_musl.yml +0 -79
- data/.github/workflows/push_macos_arm64.yml +0 -54
- data/.github/workflows/push_ruby.yml +0 -37
- data/.github/workflows/trigger-wiki-refresh.yml +0 -30
- data/.github/workflows/verify-action-pins.yml +0 -16
- data/.gitignore +0 -16
- data/.rspec +0 -3
- data/.ruby-gemset +0 -1
- data/.ruby-version +0 -1
- data/.yardopts +0 -2
- data/dist/cyrus-sasl-2.1.28.tar.gz +0 -0
- data/dist/krb5-1.21.3.tar.gz +0 -0
- data/dist/openssl-3.0.16.tar.gz +0 -0
- data/dist/zlib-1.3.1.tar.gz +0 -0
- data/dist/zstd-1.5.7.tar.gz +0 -0
- data/ext/README.md +0 -19
- data/ext/Rakefile +0 -131
- data/ext/build_common.sh +0 -376
- data/ext/build_linux_aarch64_gnu.sh +0 -326
- data/ext/build_linux_x86_64_gnu.sh +0 -317
- data/ext/build_linux_x86_64_musl.sh +0 -773
- data/ext/build_macos_arm64.sh +0 -557
- data/ext/generate-ssl-certs.sh +0 -109
- data/spec/integrations/ssl_stress_spec.rb +0 -121
- data/spec/lib/rdkafka/abstract_handle_spec.rb +0 -117
- data/spec/lib/rdkafka/admin/create_acl_handle_spec.rb +0 -56
- data/spec/lib/rdkafka/admin/create_acl_report_spec.rb +0 -18
- data/spec/lib/rdkafka/admin/create_topic_handle_spec.rb +0 -54
- data/spec/lib/rdkafka/admin/create_topic_report_spec.rb +0 -16
- data/spec/lib/rdkafka/admin/delete_acl_handle_spec.rb +0 -85
- data/spec/lib/rdkafka/admin/delete_acl_report_spec.rb +0 -72
- data/spec/lib/rdkafka/admin/delete_topic_handle_spec.rb +0 -54
- data/spec/lib/rdkafka/admin/delete_topic_report_spec.rb +0 -16
- data/spec/lib/rdkafka/admin/describe_acl_handle_spec.rb +0 -85
- data/spec/lib/rdkafka/admin/describe_acl_report_spec.rb +0 -73
- data/spec/lib/rdkafka/admin_spec.rb +0 -982
- data/spec/lib/rdkafka/bindings_spec.rb +0 -198
- data/spec/lib/rdkafka/callbacks_spec.rb +0 -20
- data/spec/lib/rdkafka/config_spec.rb +0 -258
- data/spec/lib/rdkafka/consumer/headers_spec.rb +0 -73
- data/spec/lib/rdkafka/consumer/message_spec.rb +0 -139
- data/spec/lib/rdkafka/consumer/partition_spec.rb +0 -57
- data/spec/lib/rdkafka/consumer/topic_partition_list_spec.rb +0 -248
- data/spec/lib/rdkafka/consumer_spec.rb +0 -1343
- data/spec/lib/rdkafka/error_spec.rb +0 -95
- data/spec/lib/rdkafka/metadata_spec.rb +0 -79
- data/spec/lib/rdkafka/native_kafka_spec.rb +0 -130
- data/spec/lib/rdkafka/producer/delivery_handle_spec.rb +0 -60
- data/spec/lib/rdkafka/producer/delivery_report_spec.rb +0 -25
- data/spec/lib/rdkafka/producer/partitions_count_cache_spec.rb +0 -359
- data/spec/lib/rdkafka/producer_spec.rb +0 -1527
- data/spec/spec_helper.rb +0 -230
data/ext/build_common.sh
DELETED
@@ -1,376 +0,0 @@
|
|
1
|
-
#!/usr/bin/env bash
|
2
|
-
#
|
3
|
-
# Common functions and constants for librdkafka builds
|
4
|
-
# This file should be sourced by platform-specific build scripts
|
5
|
-
#
|
6
|
-
# Usage: source "$(dirname "${BASH_SOURCE[0]}")/build_common.sh"
|
7
|
-
#
|
8
|
-
|
9
|
-
# Prevent multiple sourcing
|
10
|
-
if [[ "${BUILD_COMMON_SOURCED:-}" == "1" ]]; then
|
11
|
-
return 0
|
12
|
-
fi
|
13
|
-
|
14
|
-
BUILD_COMMON_SOURCED=1
|
15
|
-
|
16
|
-
# Version constants - update these to upgrade dependencies
|
17
|
-
readonly OPENSSL_VERSION="3.0.16"
|
18
|
-
readonly CYRUS_SASL_VERSION="2.1.28"
|
19
|
-
readonly ZLIB_VERSION="1.3.1"
|
20
|
-
readonly ZSTD_VERSION="1.5.7"
|
21
|
-
readonly KRB5_VERSION="1.21.3"
|
22
|
-
readonly LIBRDKAFKA_VERSION="2.11.0"
|
23
|
-
|
24
|
-
# SHA256 checksums for supply chain security
|
25
|
-
# Update these when upgrading versions
|
26
|
-
declare -A CHECKSUMS=(
|
27
|
-
["openssl-${OPENSSL_VERSION}.tar.gz"]="57e03c50feab5d31b152af2b764f10379aecd8ee92f16c985983ce4a99f7ef86"
|
28
|
-
["cyrus-sasl-${CYRUS_SASL_VERSION}.tar.gz"]="7ccfc6abd01ed67c1a0924b353e526f1b766b21f42d4562ee635a8ebfc5bb38c"
|
29
|
-
["zlib-1.3.1.tar.gz"]="9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23"
|
30
|
-
["zstd-${ZSTD_VERSION}.tar.gz"]="eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3"
|
31
|
-
["krb5-${KRB5_VERSION}.tar.gz"]="b7a4cd5ead67fb08b980b21abd150ff7217e85ea320c9ed0c6dadd304840ad35"
|
32
|
-
["librdkafka-${LIBRDKAFKA_VERSION}.tar.gz"]="592a823dc7c09ad4ded1bc8f700da6d4e0c88ffaf267815c6f25e7450b9395ca"
|
33
|
-
)
|
34
|
-
|
35
|
-
# Colors for output
|
36
|
-
readonly RED='\033[0;31m'
|
37
|
-
readonly GREEN='\033[0;32m'
|
38
|
-
readonly YELLOW='\033[1;33m'
|
39
|
-
readonly BLUE='\033[0;34m'
|
40
|
-
readonly NC='\033[0m' # No Color
|
41
|
-
|
42
|
-
# Logging functions
|
43
|
-
log() {
|
44
|
-
echo -e "${GREEN}[$(date '+%Y-%m-%d %H:%M:%S')] $1${NC}"
|
45
|
-
}
|
46
|
-
|
47
|
-
warn() {
|
48
|
-
echo -e "${YELLOW}[WARNING] $1${NC}"
|
49
|
-
}
|
50
|
-
|
51
|
-
error() {
|
52
|
-
echo -e "${RED}[ERROR] $1${NC}"
|
53
|
-
exit 1
|
54
|
-
}
|
55
|
-
|
56
|
-
security_log() {
|
57
|
-
echo -e "${BLUE}[SECURITY] $1${NC}"
|
58
|
-
}
|
59
|
-
|
60
|
-
# Function to verify checksums
|
61
|
-
verify_checksum() {
|
62
|
-
local file="$1"
|
63
|
-
local expected_checksum="${CHECKSUMS[$file]}"
|
64
|
-
|
65
|
-
if [ -z "$expected_checksum" ]; then
|
66
|
-
error "No checksum defined for $file - this is a security risk!"
|
67
|
-
fi
|
68
|
-
|
69
|
-
security_log "Verifying checksum for $file..."
|
70
|
-
local actual_checksum
|
71
|
-
|
72
|
-
# Use platform-appropriate checksum command
|
73
|
-
if command -v sha256sum &> /dev/null; then
|
74
|
-
actual_checksum=$(sha256sum "$file" | cut -d' ' -f1)
|
75
|
-
elif command -v shasum &> /dev/null; then
|
76
|
-
actual_checksum=$(shasum -a 256 "$file" | cut -d' ' -f1)
|
77
|
-
else
|
78
|
-
error "No SHA256 checksum utility found (tried sha256sum, shasum)"
|
79
|
-
fi
|
80
|
-
|
81
|
-
if [ "$actual_checksum" = "$expected_checksum" ]; then
|
82
|
-
security_log "✅ Checksum verified for $file"
|
83
|
-
return 0
|
84
|
-
else
|
85
|
-
error "❌ CHECKSUM MISMATCH for $file!
|
86
|
-
Expected: $expected_checksum
|
87
|
-
Actual: $actual_checksum
|
88
|
-
This could indicate a supply chain attack or corrupted download!"
|
89
|
-
fi
|
90
|
-
}
|
91
|
-
|
92
|
-
# Function to securely download and verify files
|
93
|
-
secure_download() {
|
94
|
-
local url="$1"
|
95
|
-
local filename="$2"
|
96
|
-
|
97
|
-
# Check if file already exists in current directory (may have been already downloaded)
|
98
|
-
if [ -f "$filename" ]; then
|
99
|
-
log "File $filename already exists, verifying checksum..."
|
100
|
-
verify_checksum "$filename"
|
101
|
-
return 0
|
102
|
-
fi
|
103
|
-
|
104
|
-
# Check dist directory relative to script location
|
105
|
-
local script_dir
|
106
|
-
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
107
|
-
local dist_file="$script_dir/../dist/$filename"
|
108
|
-
|
109
|
-
if [ -f "$dist_file" ]; then
|
110
|
-
log "Using distributed $filename from dist/"
|
111
|
-
cp "$dist_file" "$filename"
|
112
|
-
verify_checksum "$filename"
|
113
|
-
return 0
|
114
|
-
fi
|
115
|
-
|
116
|
-
log "Downloading $filename from $url..."
|
117
|
-
|
118
|
-
# Use platform-appropriate download command
|
119
|
-
if command -v wget &> /dev/null; then
|
120
|
-
# Linux - use wget with security options
|
121
|
-
if ! wget --secure-protocol=TLSv1_2 \
|
122
|
-
--https-only \
|
123
|
-
--timeout=30 \
|
124
|
-
--tries=3 \
|
125
|
-
--progress=bar \
|
126
|
-
"$url" \
|
127
|
-
-O "$filename"; then
|
128
|
-
error "Failed to download $filename from $url"
|
129
|
-
fi
|
130
|
-
elif command -v curl &> /dev/null; then
|
131
|
-
# macOS/fallback - use curl with security options
|
132
|
-
if ! curl -L \
|
133
|
-
--tlsv1.2 \
|
134
|
-
--connect-timeout 30 \
|
135
|
-
--max-time 300 \
|
136
|
-
--retry 3 \
|
137
|
-
--progress-bar \
|
138
|
-
"$url" \
|
139
|
-
-o "$filename"; then
|
140
|
-
error "Failed to download $filename from $url"
|
141
|
-
fi
|
142
|
-
else
|
143
|
-
error "No download utility found (tried wget, curl)"
|
144
|
-
fi
|
145
|
-
|
146
|
-
# Verify checksum immediately after download
|
147
|
-
verify_checksum "$filename"
|
148
|
-
}
|
149
|
-
|
150
|
-
# Function to detect CPU count for parallel builds
|
151
|
-
get_cpu_count() {
|
152
|
-
if command -v nproc &> /dev/null; then
|
153
|
-
nproc
|
154
|
-
elif command -v sysctl &> /dev/null; then
|
155
|
-
sysctl -n hw.ncpu
|
156
|
-
else
|
157
|
-
echo "4" # fallback
|
158
|
-
fi
|
159
|
-
}
|
160
|
-
|
161
|
-
# Function to auto-detect librdkafka tarball
|
162
|
-
find_librdkafka_tarball() {
|
163
|
-
local dist_dir="$1"
|
164
|
-
local tarball="$dist_dir/librdkafka-${LIBRDKAFKA_VERSION}.tar.gz"
|
165
|
-
|
166
|
-
if [ ! -f "$tarball" ]; then
|
167
|
-
error "librdkafka-${LIBRDKAFKA_VERSION}.tar.gz not found in $dist_dir"
|
168
|
-
fi
|
169
|
-
|
170
|
-
echo "$tarball"
|
171
|
-
}
|
172
|
-
|
173
|
-
# Function to find and validate patches
|
174
|
-
find_patches() {
|
175
|
-
local patches_dir="$1"
|
176
|
-
local -n patches_array=$2 # nameref to output array
|
177
|
-
|
178
|
-
patches_array=()
|
179
|
-
|
180
|
-
if [ -d "$patches_dir" ]; then
|
181
|
-
while IFS= read -r -d '' patch; do
|
182
|
-
patches_array+=("$patch")
|
183
|
-
done < <(find "$patches_dir" -name "*.patch" -type f -print0 | sort -z)
|
184
|
-
|
185
|
-
if [ ${#patches_array[@]} -gt 0 ]; then
|
186
|
-
log "Found ${#patches_array[@]} patches to apply:"
|
187
|
-
for patch in "${patches_array[@]}"; do
|
188
|
-
log " - $(basename "$patch")"
|
189
|
-
done
|
190
|
-
else
|
191
|
-
log "No patches found in $patches_dir"
|
192
|
-
fi
|
193
|
-
else
|
194
|
-
log "No patches directory found: $patches_dir"
|
195
|
-
fi
|
196
|
-
}
|
197
|
-
|
198
|
-
# Function to apply patches
|
199
|
-
apply_patches() {
|
200
|
-
local -n patches_array=$1 # nameref to patches array
|
201
|
-
|
202
|
-
if [ ${#patches_array[@]} -gt 0 ]; then
|
203
|
-
log "Applying Ruby-specific patches..."
|
204
|
-
for patch in "${patches_array[@]}"; do
|
205
|
-
log "Applying patch: $(basename "$patch")"
|
206
|
-
if patch -p1 < "$patch"; then
|
207
|
-
log "✅ Successfully applied $(basename "$patch")"
|
208
|
-
else
|
209
|
-
error "❌ Failed to apply patch: $(basename "$patch")"
|
210
|
-
fi
|
211
|
-
done
|
212
|
-
log "All patches applied successfully"
|
213
|
-
fi
|
214
|
-
}
|
215
|
-
|
216
|
-
# Function to verify librdkafka tarball checksum if available
|
217
|
-
verify_librdkafka_checksum() {
|
218
|
-
local tarball="$1"
|
219
|
-
local filename
|
220
|
-
filename=$(basename "$tarball")
|
221
|
-
|
222
|
-
if [ -n "${CHECKSUMS[$filename]:-}" ]; then
|
223
|
-
local current_dir
|
224
|
-
current_dir=$(pwd)
|
225
|
-
cd "$(dirname "$tarball")"
|
226
|
-
verify_checksum "$filename"
|
227
|
-
cd "$current_dir"
|
228
|
-
else
|
229
|
-
warn "No checksum defined for $filename - consider adding one for security"
|
230
|
-
fi
|
231
|
-
}
|
232
|
-
|
233
|
-
# Function to set execute permissions on configure scripts
|
234
|
-
fix_configure_permissions() {
|
235
|
-
log "Setting execute permissions on configure scripts..."
|
236
|
-
chmod +x configure* 2>/dev/null || true
|
237
|
-
chmod +x mklove/modules/configure.* 2>/dev/null || true
|
238
|
-
}
|
239
|
-
|
240
|
-
# Function to print security summary
|
241
|
-
print_security_summary() {
|
242
|
-
security_log "🔒 SECURITY VERIFICATION COMPLETE"
|
243
|
-
security_log "All dependencies downloaded and verified with SHA256 checksums"
|
244
|
-
security_log "Supply chain integrity maintained throughout build process"
|
245
|
-
}
|
246
|
-
|
247
|
-
# Function to print build summary
|
248
|
-
print_build_summary() {
|
249
|
-
local platform="$1"
|
250
|
-
local arch="$2"
|
251
|
-
local output_dir="$3"
|
252
|
-
local library_name="$4"
|
253
|
-
|
254
|
-
log "Build completed successfully!"
|
255
|
-
log "📦 Self-contained librdkafka built for $platform $arch:"
|
256
|
-
log " ✅ OpenSSL $OPENSSL_VERSION (SSL/TLS support) - checksum verified"
|
257
|
-
log " ✅ Cyrus SASL $CYRUS_SASL_VERSION (authentication for AWS MSK) - checksum verified"
|
258
|
-
log " ✅ MIT Kerberos $KRB5_VERSION (GSSAPI/Kerberos authentication) - checksum verified"
|
259
|
-
log " ✅ zlib $ZLIB_VERSION (compression) - checksum verified"
|
260
|
-
log " ✅ ZStd $ZSTD_VERSION (high-performance compression) - checksum verified"
|
261
|
-
log ""
|
262
|
-
log "🎯 Ready for deployment on $platform systems"
|
263
|
-
log "☁️ Compatible with AWS MSK and other secured Kafka clusters"
|
264
|
-
log "🔐 Supply chain security: All dependencies cryptographically verified"
|
265
|
-
log ""
|
266
|
-
log "Location: $output_dir/$library_name"
|
267
|
-
}
|
268
|
-
|
269
|
-
# Function to clean up build directory with user prompt (except .tar.gz files in CI)
|
270
|
-
cleanup_build_dir() {
|
271
|
-
local build_dir="$1"
|
272
|
-
|
273
|
-
if [ "${CI:-}" = "true" ]; then
|
274
|
-
# In CI: remove everything except .tar.gz files without prompting
|
275
|
-
echo "CI detected: cleaning up $build_dir (preserving .tar.gz files for caching)"
|
276
|
-
|
277
|
-
# First, find and move all .tar.gz files to a temp location
|
278
|
-
temp_dir=$(mktemp -d)
|
279
|
-
find "$build_dir" -name "*.tar.gz" -exec mv {} "$temp_dir/" \; 2>/dev/null || true
|
280
|
-
|
281
|
-
# Remove everything in build_dir
|
282
|
-
rm -rf "$build_dir"/* 2>/dev/null || true
|
283
|
-
rm -rf "$build_dir"/.* 2>/dev/null || true
|
284
|
-
|
285
|
-
# Move .tar.gz files back
|
286
|
-
mv "$temp_dir"/* "$build_dir/" 2>/dev/null || true
|
287
|
-
rmdir "$temp_dir" 2>/dev/null || true
|
288
|
-
|
289
|
-
log "Build directory cleaned up (preserved .tar.gz files)"
|
290
|
-
else
|
291
|
-
# Interactive mode: prompt user
|
292
|
-
echo
|
293
|
-
read -p "Remove build directory $build_dir? (y/N): " -n 1 -r
|
294
|
-
echo
|
295
|
-
if [[ $REPLY =~ ^[Yy]$ ]]; then
|
296
|
-
rm -rf "$build_dir"
|
297
|
-
log "Build directory cleaned up"
|
298
|
-
fi
|
299
|
-
fi
|
300
|
-
}
|
301
|
-
|
302
|
-
# Function to validate build environment
|
303
|
-
check_common_dependencies() {
|
304
|
-
log "Checking common build dependencies..."
|
305
|
-
|
306
|
-
local missing_tools=()
|
307
|
-
|
308
|
-
command -v tar &> /dev/null || missing_tools+=("tar")
|
309
|
-
command -v make &> /dev/null || missing_tools+=("make")
|
310
|
-
command -v patch &> /dev/null || missing_tools+=("patch")
|
311
|
-
|
312
|
-
# Check for download tools
|
313
|
-
if ! command -v wget &> /dev/null && ! command -v curl &> /dev/null; then
|
314
|
-
missing_tools+=("wget or curl")
|
315
|
-
fi
|
316
|
-
|
317
|
-
# Check for checksum tools
|
318
|
-
if ! command -v sha256sum &> /dev/null && ! command -v shasum &> /dev/null; then
|
319
|
-
missing_tools+=("sha256sum or shasum")
|
320
|
-
fi
|
321
|
-
|
322
|
-
if [ ${#missing_tools[@]} -gt 0 ]; then
|
323
|
-
error "Missing required tools: ${missing_tools[*]}"
|
324
|
-
fi
|
325
|
-
|
326
|
-
log "✅ Common build tools found"
|
327
|
-
}
|
328
|
-
|
329
|
-
# Function to extract tarball if directory doesn't exist
|
330
|
-
extract_if_needed() {
|
331
|
-
local tarball="$1"
|
332
|
-
local expected_dir="$2"
|
333
|
-
|
334
|
-
if [ ! -d "$expected_dir" ]; then
|
335
|
-
log "Extracting $(basename "$tarball")..."
|
336
|
-
tar xzf "$tarball"
|
337
|
-
else
|
338
|
-
log "Directory $expected_dir already exists, skipping extraction"
|
339
|
-
fi
|
340
|
-
}
|
341
|
-
|
342
|
-
# Download URLs for dependencies
|
343
|
-
get_openssl_url() {
|
344
|
-
echo "https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz"
|
345
|
-
}
|
346
|
-
|
347
|
-
get_sasl_url() {
|
348
|
-
echo "https://github.com/cyrusimap/cyrus-sasl/releases/download/cyrus-sasl-${CYRUS_SASL_VERSION}/cyrus-sasl-${CYRUS_SASL_VERSION}.tar.gz"
|
349
|
-
}
|
350
|
-
|
351
|
-
get_zlib_url() {
|
352
|
-
echo "https://github.com/madler/zlib/releases/download/v${ZLIB_VERSION}/zlib-${ZLIB_VERSION}.tar.gz"
|
353
|
-
}
|
354
|
-
|
355
|
-
get_zstd_url() {
|
356
|
-
echo "https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz"
|
357
|
-
}
|
358
|
-
|
359
|
-
get_krb5_url() {
|
360
|
-
# Using MIT mirror since kerberos.org is down
|
361
|
-
# echo "https://kerberos.org/dist/krb5/${KRB5_VERSION%.*}/krb5-${KRB5_VERSION}.tar.gz"
|
362
|
-
echo "https://web.mit.edu/kerberos/dist/krb5/${KRB5_VERSION%.*}/krb5-${KRB5_VERSION}.tar.gz"
|
363
|
-
}
|
364
|
-
|
365
|
-
# Export functions and variables that scripts will need
|
366
|
-
export -f log warn error security_log
|
367
|
-
export -f verify_checksum secure_download get_cpu_count
|
368
|
-
export -f find_librdkafka_tarball find_patches apply_patches
|
369
|
-
export -f verify_librdkafka_checksum fix_configure_permissions
|
370
|
-
export -f print_security_summary print_build_summary cleanup_build_dir
|
371
|
-
export -f check_common_dependencies extract_if_needed
|
372
|
-
export -f get_openssl_url get_sasl_url get_zlib_url get_zstd_url get_krb5_url
|
373
|
-
|
374
|
-
# Export constants
|
375
|
-
export OPENSSL_VERSION CYRUS_SASL_VERSION ZLIB_VERSION ZSTD_VERSION KRB5_VERSION
|
376
|
-
export RED GREEN YELLOW BLUE NC
|
@@ -1,326 +0,0 @@
|
|
1
|
-
#!/usr/bin/env bash
|
2
|
-
#
|
3
|
-
# Build self-contained librdkafka.so for Linux aarch64 GNU with checksum verification
|
4
|
-
# Usage: ./build_linux_aarch64_gnu.sh
|
5
|
-
#
|
6
|
-
# Expected directory structure:
|
7
|
-
# ext/build_linux_aarch64_gnu.sh (this script)
|
8
|
-
# ext/build_common.sh (shared functions)
|
9
|
-
# dist/librdkafka-*.tar.gz (librdkafka source tarball)
|
10
|
-
# dist/patches/*.patch (optional Ruby-specific patches)
|
11
|
-
#
|
12
|
-
set -euo pipefail
|
13
|
-
|
14
|
-
# Source common functions and constants
|
15
|
-
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
16
|
-
source "$SCRIPT_DIR/build_common.sh"
|
17
|
-
|
18
|
-
# Platform-specific paths
|
19
|
-
DIST_DIR="$SCRIPT_DIR/../dist"
|
20
|
-
PATCHES_DIR="$DIST_DIR/patches"
|
21
|
-
BUILD_DIR="$(pwd)/build-tmp"
|
22
|
-
DEPS_PREFIX="/tmp"
|
23
|
-
|
24
|
-
# Check common dependencies
|
25
|
-
check_common_dependencies
|
26
|
-
|
27
|
-
# Linux-specific dependency check
|
28
|
-
log "Checking Linux aarch64 GNU-specific build dependencies..."
|
29
|
-
command -v gcc &> /dev/null || error "gcc not found. Install with: apt-get install build-essential"
|
30
|
-
|
31
|
-
# Verify we're on ARM64 or cross-compiling
|
32
|
-
ARCH=$(uname -m)
|
33
|
-
if [[ "$ARCH" != "aarch64" && "$ARCH" != "arm64" ]]; then
|
34
|
-
warn "Not running on aarch64 ($ARCH detected). Ensure you have aarch64 cross-compilation tools if needed."
|
35
|
-
fi
|
36
|
-
|
37
|
-
# Auto-detect librdkafka tarball
|
38
|
-
log "Looking for librdkafka tarball in $DIST_DIR..."
|
39
|
-
LIBRDKAFKA_TARBALL=$(find_librdkafka_tarball "$DIST_DIR")
|
40
|
-
log "Found librdkafka tarball: $LIBRDKAFKA_TARBALL"
|
41
|
-
|
42
|
-
# Verify librdkafka tarball checksum if available
|
43
|
-
verify_librdkafka_checksum "$LIBRDKAFKA_TARBALL"
|
44
|
-
|
45
|
-
# Find patches
|
46
|
-
PATCHES_FOUND=()
|
47
|
-
find_patches "$PATCHES_DIR" PATCHES_FOUND
|
48
|
-
|
49
|
-
security_log "Starting secure build with checksum verification enabled"
|
50
|
-
log "Building self-contained librdkafka.so for Linux aarch64 GNU"
|
51
|
-
log "Dependencies to build:"
|
52
|
-
log " - OpenSSL: $OPENSSL_VERSION"
|
53
|
-
log " - Cyrus SASL: $CYRUS_SASL_VERSION"
|
54
|
-
log " - MIT Kerberos: $KRB5_VERSION"
|
55
|
-
log " - zlib: $ZLIB_VERSION"
|
56
|
-
log " - ZStd: $ZSTD_VERSION"
|
57
|
-
log "librdkafka source: $LIBRDKAFKA_TARBALL"
|
58
|
-
log "Build directory: $BUILD_DIR"
|
59
|
-
|
60
|
-
# Create build directory
|
61
|
-
mkdir -p "$BUILD_DIR"
|
62
|
-
cd "$BUILD_DIR"
|
63
|
-
|
64
|
-
# Build OpenSSL
|
65
|
-
log "Building OpenSSL $OPENSSL_VERSION for ARM64..."
|
66
|
-
OPENSSL_PREFIX="$DEPS_PREFIX/static-openssl-$OPENSSL_VERSION-arm64"
|
67
|
-
OPENSSL_TARBALL="openssl-$OPENSSL_VERSION.tar.gz"
|
68
|
-
OPENSSL_DIR="openssl-$OPENSSL_VERSION"
|
69
|
-
|
70
|
-
secure_download "$(get_openssl_url)" "$OPENSSL_TARBALL"
|
71
|
-
extract_if_needed "$OPENSSL_TARBALL" "$OPENSSL_DIR"
|
72
|
-
cd "$OPENSSL_DIR"
|
73
|
-
|
74
|
-
# Check if OpenSSL lib directory exists (lib or lib64)
|
75
|
-
if [ ! -f "$OPENSSL_PREFIX/lib/libssl.a" ] && [ ! -f "$OPENSSL_PREFIX/lib64/libssl.a" ]; then
|
76
|
-
log "Configuring and building OpenSSL for ARM64..."
|
77
|
-
export CFLAGS="-fPIC"
|
78
|
-
# Use linux-aarch64 target for ARM64
|
79
|
-
./Configure linux-aarch64 \
|
80
|
-
no-shared \
|
81
|
-
no-dso \
|
82
|
-
--prefix="$OPENSSL_PREFIX"
|
83
|
-
make clean || true
|
84
|
-
make -j$(get_cpu_count)
|
85
|
-
make install
|
86
|
-
unset CFLAGS
|
87
|
-
log "OpenSSL built successfully"
|
88
|
-
else
|
89
|
-
log "OpenSSL already built, skipping..."
|
90
|
-
fi
|
91
|
-
|
92
|
-
# Determine OpenSSL lib directory
|
93
|
-
if [ -f "$OPENSSL_PREFIX/lib64/libssl.a" ]; then
|
94
|
-
OPENSSL_LIB_DIR="$OPENSSL_PREFIX/lib64"
|
95
|
-
else
|
96
|
-
OPENSSL_LIB_DIR="$OPENSSL_PREFIX/lib"
|
97
|
-
fi
|
98
|
-
log "OpenSSL libraries in: $OPENSSL_LIB_DIR"
|
99
|
-
|
100
|
-
cd "$BUILD_DIR"
|
101
|
-
|
102
|
-
# Build MIT Kerberos (krb5)
|
103
|
-
log "Building MIT Kerberos $KRB5_VERSION for ARM64..."
|
104
|
-
KRB5_PREFIX="$DEPS_PREFIX/static-krb5-$KRB5_VERSION-arm64"
|
105
|
-
KRB5_TARBALL="krb5-$KRB5_VERSION.tar.gz"
|
106
|
-
KRB5_DIR="krb5-$KRB5_VERSION"
|
107
|
-
|
108
|
-
secure_download "$(get_krb5_url)" "$KRB5_TARBALL"
|
109
|
-
extract_if_needed "$KRB5_TARBALL" "$KRB5_DIR"
|
110
|
-
cd "$KRB5_DIR/src"
|
111
|
-
|
112
|
-
if [ ! -f "$KRB5_PREFIX/lib/libgssapi_krb5.a" ]; then
|
113
|
-
log "Configuring and building MIT Kerberos for ARM64..."
|
114
|
-
make clean 2>/dev/null || true
|
115
|
-
./configure --disable-shared --enable-static --prefix="$KRB5_PREFIX" \
|
116
|
-
--without-ldap --without-tcl --without-keyutils \
|
117
|
-
--disable-rpath --without-system-verto \
|
118
|
-
CFLAGS="-fPIC" CXXFLAGS="-fPIC"
|
119
|
-
|
120
|
-
# Build everything except the problematic kadmin tools
|
121
|
-
log "Building Kerberos (will ignore kadmin build failures)..."
|
122
|
-
make -j$(get_cpu_count) || {
|
123
|
-
log "Full build failed (expected due to kadmin), continuing with libraries..."
|
124
|
-
# The libraries should be built even if kadmin fails
|
125
|
-
true
|
126
|
-
}
|
127
|
-
|
128
|
-
# Install what was successfully built
|
129
|
-
make install || {
|
130
|
-
log "Full install failed, installing individual components..."
|
131
|
-
# Try to install the core libraries manually
|
132
|
-
make install-mkdirs 2>/dev/null || true
|
133
|
-
make -C util install 2>/dev/null || true
|
134
|
-
make -C lib install 2>/dev/null || true
|
135
|
-
make -C plugins/kdb/db2 install 2>/dev/null || true
|
136
|
-
}
|
137
|
-
|
138
|
-
# Verify we got the essential libraries
|
139
|
-
if [ ! -f "$KRB5_PREFIX/lib/libgssapi_krb5.a" ]; then
|
140
|
-
error "Failed to build essential Kerberos libraries"
|
141
|
-
fi
|
142
|
-
|
143
|
-
log "MIT Kerberos libraries built successfully"
|
144
|
-
else
|
145
|
-
log "MIT Kerberos already built, skipping..."
|
146
|
-
fi
|
147
|
-
|
148
|
-
cd "$BUILD_DIR"
|
149
|
-
|
150
|
-
# Build SASL
|
151
|
-
log "Building Cyrus SASL $CYRUS_SASL_VERSION for ARM64..."
|
152
|
-
SASL_PREFIX="$DEPS_PREFIX/static-sasl-$CYRUS_SASL_VERSION-arm64"
|
153
|
-
SASL_TARBALL="cyrus-sasl-$CYRUS_SASL_VERSION.tar.gz"
|
154
|
-
SASL_DIR="cyrus-sasl-$CYRUS_SASL_VERSION"
|
155
|
-
|
156
|
-
secure_download "$(get_sasl_url)" "$SASL_TARBALL"
|
157
|
-
extract_if_needed "$SASL_TARBALL" "$SASL_DIR"
|
158
|
-
cd "$SASL_DIR"
|
159
|
-
|
160
|
-
if [ ! -f "$SASL_PREFIX/lib/libsasl2.a" ]; then
|
161
|
-
log "Configuring and building SASL for ARM64..."
|
162
|
-
make clean 2>/dev/null || true
|
163
|
-
./configure --disable-shared --enable-static --prefix="$SASL_PREFIX" \
|
164
|
-
--without-dblib --disable-gdbm \
|
165
|
-
--enable-gssapi="$KRB5_PREFIX" \
|
166
|
-
CFLAGS="-fPIC" CXXFLAGS="-fPIC" \
|
167
|
-
CPPFLAGS="-I$KRB5_PREFIX/include" \
|
168
|
-
LDFLAGS="-L$KRB5_PREFIX/lib"
|
169
|
-
make -j$(get_cpu_count)
|
170
|
-
make install
|
171
|
-
log "SASL built successfully"
|
172
|
-
else
|
173
|
-
log "SASL already built, skipping..."
|
174
|
-
fi
|
175
|
-
|
176
|
-
cd "$BUILD_DIR"
|
177
|
-
|
178
|
-
# Build zlib
|
179
|
-
log "Building zlib $ZLIB_VERSION for ARM64..."
|
180
|
-
ZLIB_PREFIX="$DEPS_PREFIX/static-zlib-$ZLIB_VERSION-arm64"
|
181
|
-
ZLIB_TARBALL="zlib-$ZLIB_VERSION.tar.gz"
|
182
|
-
ZLIB_DIR="zlib-$ZLIB_VERSION"
|
183
|
-
|
184
|
-
secure_download "$(get_zlib_url)" "$ZLIB_TARBALL"
|
185
|
-
extract_if_needed "$ZLIB_TARBALL" "$ZLIB_DIR"
|
186
|
-
cd "$ZLIB_DIR"
|
187
|
-
|
188
|
-
if [ ! -f "$ZLIB_PREFIX/lib/libz.a" ]; then
|
189
|
-
log "Configuring and building zlib for ARM64..."
|
190
|
-
make clean 2>/dev/null || true
|
191
|
-
export CFLAGS="-fPIC"
|
192
|
-
./configure --prefix="$ZLIB_PREFIX" --static
|
193
|
-
make -j$(get_cpu_count)
|
194
|
-
make install
|
195
|
-
unset CFLAGS
|
196
|
-
log "zlib built successfully"
|
197
|
-
else
|
198
|
-
log "zlib already built, skipping..."
|
199
|
-
fi
|
200
|
-
|
201
|
-
cd "$BUILD_DIR"
|
202
|
-
|
203
|
-
# Build ZStd
|
204
|
-
log "Building ZStd $ZSTD_VERSION for ARM64..."
|
205
|
-
ZSTD_PREFIX="$DEPS_PREFIX/static-zstd-$ZSTD_VERSION-arm64"
|
206
|
-
ZSTD_TARBALL="zstd-$ZSTD_VERSION.tar.gz"
|
207
|
-
ZSTD_DIR="zstd-$ZSTD_VERSION"
|
208
|
-
|
209
|
-
secure_download "$(get_zstd_url)" "$ZSTD_TARBALL"
|
210
|
-
extract_if_needed "$ZSTD_TARBALL" "$ZSTD_DIR"
|
211
|
-
cd "$ZSTD_DIR"
|
212
|
-
|
213
|
-
if [ ! -f "$ZSTD_PREFIX/lib/libzstd.a" ]; then
|
214
|
-
log "Building ZStd for ARM64..."
|
215
|
-
make clean 2>/dev/null || true
|
216
|
-
make lib-mt CFLAGS="-fPIC" PREFIX="$ZSTD_PREFIX" -j$(get_cpu_count)
|
217
|
-
# Use standard install target - install-pc may not exist in all versions
|
218
|
-
make install PREFIX="$ZSTD_PREFIX"
|
219
|
-
log "ZStd built successfully"
|
220
|
-
else
|
221
|
-
log "ZStd already built, skipping..."
|
222
|
-
fi
|
223
|
-
|
224
|
-
cd "$BUILD_DIR"
|
225
|
-
|
226
|
-
# Extract and patch librdkafka
|
227
|
-
log "Extracting librdkafka..."
|
228
|
-
tar xzf "$LIBRDKAFKA_TARBALL"
|
229
|
-
cd "librdkafka-$LIBRDKAFKA_VERSION"
|
230
|
-
|
231
|
-
# Fix permissions and apply patches
|
232
|
-
fix_configure_permissions
|
233
|
-
apply_patches PATCHES_FOUND
|
234
|
-
|
235
|
-
# Configure librdkafka
|
236
|
-
log "Configuring librdkafka for ARM64..."
|
237
|
-
|
238
|
-
if [ -f configure ]; then
|
239
|
-
log "Using standard configure (autotools)"
|
240
|
-
# Export environment variables for configure to pick up
|
241
|
-
export CPPFLAGS="-I$KRB5_PREFIX/include"
|
242
|
-
export LDFLAGS="-L$KRB5_PREFIX/lib"
|
243
|
-
|
244
|
-
./configure --enable-static --disable-shared --disable-curl \
|
245
|
-
--enable-gssapi
|
246
|
-
|
247
|
-
# Clean up environment variables
|
248
|
-
unset CPPFLAGS LDFLAGS
|
249
|
-
else
|
250
|
-
error "No configure script found (checked: configure.self, configure)"
|
251
|
-
fi
|
252
|
-
|
253
|
-
# Build librdkafka
|
254
|
-
log "Compiling librdkafka for ARM64..."
|
255
|
-
make clean || true
|
256
|
-
make -j$(get_cpu_count)
|
257
|
-
|
258
|
-
# Verify librdkafka.a exists
|
259
|
-
if [ ! -f src/librdkafka.a ]; then
|
260
|
-
error "librdkafka.a not found after build"
|
261
|
-
fi
|
262
|
-
|
263
|
-
log "librdkafka.a built successfully"
|
264
|
-
|
265
|
-
# Create self-contained shared library
|
266
|
-
log "Creating self-contained librdkafka.so for ARM64..."
|
267
|
-
|
268
|
-
# Write the export map
|
269
|
-
cat > export.map <<'EOF'
|
270
|
-
{
|
271
|
-
global:
|
272
|
-
rd_kafka_*;
|
273
|
-
local:
|
274
|
-
*;
|
275
|
-
};
|
276
|
-
EOF
|
277
|
-
|
278
|
-
# Link everything statically, expose only rd_kafka_* symbols
|
279
|
-
aarch64-linux-gnu-gcc -shared -fPIC \
|
280
|
-
-Wl,--version-script=export.map \
|
281
|
-
-Wl,--whole-archive src/librdkafka.a -Wl,--no-whole-archive \
|
282
|
-
-o librdkafka.so \
|
283
|
-
"$SASL_PREFIX/lib/libsasl2.a" \
|
284
|
-
"$KRB5_PREFIX/lib/libgssapi_krb5.a" \
|
285
|
-
"$KRB5_PREFIX/lib/libkrb5.a" \
|
286
|
-
"$KRB5_PREFIX/lib/libk5crypto.a" \
|
287
|
-
"$KRB5_PREFIX/lib/libcom_err.a" \
|
288
|
-
"$KRB5_PREFIX/lib/libkrb5support.a" \
|
289
|
-
"$OPENSSL_LIB_DIR/libssl.a" \
|
290
|
-
"$OPENSSL_LIB_DIR/libcrypto.a" \
|
291
|
-
"$ZLIB_PREFIX/lib/libz.a" \
|
292
|
-
"$ZSTD_PREFIX/lib/libzstd.a" \
|
293
|
-
-lpthread -lm -ldl -lresolv
|
294
|
-
|
295
|
-
if [ ! -f librdkafka.so ]; then
|
296
|
-
error "Failed to create librdkafka.so"
|
297
|
-
fi
|
298
|
-
|
299
|
-
log "librdkafka.so created successfully"
|
300
|
-
|
301
|
-
# Verify the build
|
302
|
-
log "Verifying build..."
|
303
|
-
file librdkafka.so
|
304
|
-
|
305
|
-
log "Checking dependencies with ldd:"
|
306
|
-
ldd librdkafka.so
|
307
|
-
|
308
|
-
log "Checking for external dependencies (should only show system libraries):"
|
309
|
-
EXTERNAL_DEPS=$(nm -D librdkafka.so | grep " U " | grep -v "@GLIBC" || true)
|
310
|
-
if [ -n "$EXTERNAL_DEPS" ]; then
|
311
|
-
error "Found external dependencies - library is not self-contained: $EXTERNAL_DEPS"
|
312
|
-
else
|
313
|
-
log "✅ No external dependencies found - library is self-contained!"
|
314
|
-
fi
|
315
|
-
|
316
|
-
# Copy to output directory
|
317
|
-
OUTPUT_DIR="$SCRIPT_DIR"
|
318
|
-
cp librdkafka.so "$OUTPUT_DIR/"
|
319
|
-
log "librdkafka.so copied to: $OUTPUT_DIR/librdkafka.so"
|
320
|
-
|
321
|
-
# Print summaries
|
322
|
-
print_security_summary
|
323
|
-
print_build_summary "Linux" "aarch64 GNU" "$OUTPUT_DIR" "librdkafka.so"
|
324
|
-
|
325
|
-
# Cleanup
|
326
|
-
cleanup_build_dir "$BUILD_DIR"
|