karafka-rdkafka 0.21.0-aarch64-linux-gnu

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. checksums.yaml +7 -0
  2. data/.github/CODEOWNERS +3 -0
  3. data/.github/FUNDING.yml +1 -0
  4. data/.github/workflows/ci_linux_alpine_x86_64_musl.yml +197 -0
  5. data/.github/workflows/ci_linux_alpine_x86_64_musl_complementary.yml +264 -0
  6. data/.github/workflows/ci_linux_debian_x86_64_gnu.yml +271 -0
  7. data/.github/workflows/ci_linux_debian_x86_64_gnu_complementary.yml +334 -0
  8. data/.github/workflows/ci_linux_ubuntu_aarch64_gnu.yml +271 -0
  9. data/.github/workflows/ci_linux_ubuntu_aarch64_gnu_complementary.yml +295 -0
  10. data/.github/workflows/ci_linux_ubuntu_x86_64_gnu.yml +281 -0
  11. data/.github/workflows/ci_linux_ubuntu_x86_64_gnu_complementary.yml +294 -0
  12. data/.github/workflows/ci_macos_arm64.yml +284 -0
  13. data/.github/workflows/push_linux_aarch64_gnu.yml +65 -0
  14. data/.github/workflows/push_linux_x86_64_gnu.yml +65 -0
  15. data/.github/workflows/push_linux_x86_64_musl.yml +79 -0
  16. data/.github/workflows/push_macos_arm64.yml +54 -0
  17. data/.github/workflows/push_ruby.yml +37 -0
  18. data/.github/workflows/trigger-wiki-refresh.yml +30 -0
  19. data/.github/workflows/verify-action-pins.yml +16 -0
  20. data/.gitignore +16 -0
  21. data/.rspec +3 -0
  22. data/.ruby-gemset +1 -0
  23. data/.ruby-version +1 -0
  24. data/.yardopts +2 -0
  25. data/CHANGELOG.md +344 -0
  26. data/Gemfile +5 -0
  27. data/MIT-LICENSE +22 -0
  28. data/README.md +78 -0
  29. data/Rakefile +96 -0
  30. data/dist/cyrus-sasl-2.1.28.tar.gz +0 -0
  31. data/dist/krb5-1.21.3.tar.gz +0 -0
  32. data/dist/openssl-3.0.16.tar.gz +0 -0
  33. data/dist/zlib-1.3.1.tar.gz +0 -0
  34. data/dist/zstd-1.5.7.tar.gz +0 -0
  35. data/docker-compose-ssl.yml +35 -0
  36. data/docker-compose.yml +25 -0
  37. data/ext/README.md +19 -0
  38. data/ext/Rakefile +131 -0
  39. data/ext/build_common.sh +376 -0
  40. data/ext/build_linux_aarch64_gnu.sh +326 -0
  41. data/ext/build_linux_x86_64_gnu.sh +317 -0
  42. data/ext/build_linux_x86_64_musl.sh +773 -0
  43. data/ext/build_macos_arm64.sh +557 -0
  44. data/ext/generate-ssl-certs.sh +109 -0
  45. data/ext/librdkafka.so +0 -0
  46. data/karafka-rdkafka.gemspec +65 -0
  47. data/lib/rdkafka/abstract_handle.rb +116 -0
  48. data/lib/rdkafka/admin/acl_binding_result.rb +51 -0
  49. data/lib/rdkafka/admin/config_binding_result.rb +30 -0
  50. data/lib/rdkafka/admin/config_resource_binding_result.rb +18 -0
  51. data/lib/rdkafka/admin/create_acl_handle.rb +28 -0
  52. data/lib/rdkafka/admin/create_acl_report.rb +24 -0
  53. data/lib/rdkafka/admin/create_partitions_handle.rb +30 -0
  54. data/lib/rdkafka/admin/create_partitions_report.rb +6 -0
  55. data/lib/rdkafka/admin/create_topic_handle.rb +32 -0
  56. data/lib/rdkafka/admin/create_topic_report.rb +24 -0
  57. data/lib/rdkafka/admin/delete_acl_handle.rb +30 -0
  58. data/lib/rdkafka/admin/delete_acl_report.rb +23 -0
  59. data/lib/rdkafka/admin/delete_groups_handle.rb +28 -0
  60. data/lib/rdkafka/admin/delete_groups_report.rb +24 -0
  61. data/lib/rdkafka/admin/delete_topic_handle.rb +32 -0
  62. data/lib/rdkafka/admin/delete_topic_report.rb +24 -0
  63. data/lib/rdkafka/admin/describe_acl_handle.rb +30 -0
  64. data/lib/rdkafka/admin/describe_acl_report.rb +24 -0
  65. data/lib/rdkafka/admin/describe_configs_handle.rb +33 -0
  66. data/lib/rdkafka/admin/describe_configs_report.rb +48 -0
  67. data/lib/rdkafka/admin/incremental_alter_configs_handle.rb +33 -0
  68. data/lib/rdkafka/admin/incremental_alter_configs_report.rb +48 -0
  69. data/lib/rdkafka/admin.rb +832 -0
  70. data/lib/rdkafka/bindings.rb +583 -0
  71. data/lib/rdkafka/callbacks.rb +415 -0
  72. data/lib/rdkafka/config.rb +395 -0
  73. data/lib/rdkafka/consumer/headers.rb +79 -0
  74. data/lib/rdkafka/consumer/message.rb +86 -0
  75. data/lib/rdkafka/consumer/partition.rb +57 -0
  76. data/lib/rdkafka/consumer/topic_partition_list.rb +190 -0
  77. data/lib/rdkafka/consumer.rb +663 -0
  78. data/lib/rdkafka/error.rb +201 -0
  79. data/lib/rdkafka/helpers/oauth.rb +58 -0
  80. data/lib/rdkafka/helpers/time.rb +14 -0
  81. data/lib/rdkafka/metadata.rb +115 -0
  82. data/lib/rdkafka/native_kafka.rb +139 -0
  83. data/lib/rdkafka/producer/delivery_handle.rb +48 -0
  84. data/lib/rdkafka/producer/delivery_report.rb +45 -0
  85. data/lib/rdkafka/producer/partitions_count_cache.rb +216 -0
  86. data/lib/rdkafka/producer.rb +497 -0
  87. data/lib/rdkafka/version.rb +7 -0
  88. data/lib/rdkafka.rb +54 -0
  89. data/renovate.json +92 -0
  90. data/spec/integrations/ssl_stress_spec.rb +121 -0
  91. data/spec/lib/rdkafka/abstract_handle_spec.rb +117 -0
  92. data/spec/lib/rdkafka/admin/create_acl_handle_spec.rb +56 -0
  93. data/spec/lib/rdkafka/admin/create_acl_report_spec.rb +18 -0
  94. data/spec/lib/rdkafka/admin/create_topic_handle_spec.rb +54 -0
  95. data/spec/lib/rdkafka/admin/create_topic_report_spec.rb +16 -0
  96. data/spec/lib/rdkafka/admin/delete_acl_handle_spec.rb +85 -0
  97. data/spec/lib/rdkafka/admin/delete_acl_report_spec.rb +72 -0
  98. data/spec/lib/rdkafka/admin/delete_topic_handle_spec.rb +54 -0
  99. data/spec/lib/rdkafka/admin/delete_topic_report_spec.rb +16 -0
  100. data/spec/lib/rdkafka/admin/describe_acl_handle_spec.rb +85 -0
  101. data/spec/lib/rdkafka/admin/describe_acl_report_spec.rb +73 -0
  102. data/spec/lib/rdkafka/admin_spec.rb +982 -0
  103. data/spec/lib/rdkafka/bindings_spec.rb +198 -0
  104. data/spec/lib/rdkafka/callbacks_spec.rb +20 -0
  105. data/spec/lib/rdkafka/config_spec.rb +258 -0
  106. data/spec/lib/rdkafka/consumer/headers_spec.rb +73 -0
  107. data/spec/lib/rdkafka/consumer/message_spec.rb +139 -0
  108. data/spec/lib/rdkafka/consumer/partition_spec.rb +57 -0
  109. data/spec/lib/rdkafka/consumer/topic_partition_list_spec.rb +248 -0
  110. data/spec/lib/rdkafka/consumer_spec.rb +1343 -0
  111. data/spec/lib/rdkafka/error_spec.rb +95 -0
  112. data/spec/lib/rdkafka/metadata_spec.rb +79 -0
  113. data/spec/lib/rdkafka/native_kafka_spec.rb +130 -0
  114. data/spec/lib/rdkafka/producer/delivery_handle_spec.rb +60 -0
  115. data/spec/lib/rdkafka/producer/delivery_report_spec.rb +25 -0
  116. data/spec/lib/rdkafka/producer/partitions_count_cache_spec.rb +359 -0
  117. data/spec/lib/rdkafka/producer_spec.rb +1527 -0
  118. data/spec/spec_helper.rb +230 -0
  119. metadata +320 -0
@@ -0,0 +1,326 @@
1
+ #!/usr/bin/env bash
2
+ #
3
+ # Build self-contained librdkafka.so for Linux aarch64 GNU with checksum verification
4
+ # Usage: ./build_linux_aarch64_gnu.sh
5
+ #
6
+ # Expected directory structure:
7
+ # ext/build_linux_aarch64_gnu.sh (this script)
8
+ # ext/build_common.sh (shared functions)
9
+ # dist/librdkafka-*.tar.gz (librdkafka source tarball)
10
+ # dist/patches/*.patch (optional Ruby-specific patches)
11
+ #
12
+ set -euo pipefail
13
+
14
+ # Source common functions and constants
15
+ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
16
+ source "$SCRIPT_DIR/build_common.sh"
17
+
18
+ # Platform-specific paths
19
+ DIST_DIR="$SCRIPT_DIR/../dist"
20
+ PATCHES_DIR="$DIST_DIR/patches"
21
+ BUILD_DIR="$(pwd)/build-tmp"
22
+ DEPS_PREFIX="/tmp"
23
+
24
+ # Check common dependencies
25
+ check_common_dependencies
26
+
27
+ # Linux-specific dependency check
28
+ log "Checking Linux aarch64 GNU-specific build dependencies..."
29
+ command -v gcc &> /dev/null || error "gcc not found. Install with: apt-get install build-essential"
30
+
31
+ # Verify we're on ARM64 or cross-compiling
32
+ ARCH=$(uname -m)
33
+ if [[ "$ARCH" != "aarch64" && "$ARCH" != "arm64" ]]; then
34
+ warn "Not running on aarch64 ($ARCH detected). Ensure you have aarch64 cross-compilation tools if needed."
35
+ fi
36
+
37
+ # Auto-detect librdkafka tarball
38
+ log "Looking for librdkafka tarball in $DIST_DIR..."
39
+ LIBRDKAFKA_TARBALL=$(find_librdkafka_tarball "$DIST_DIR")
40
+ log "Found librdkafka tarball: $LIBRDKAFKA_TARBALL"
41
+
42
+ # Verify librdkafka tarball checksum if available
43
+ verify_librdkafka_checksum "$LIBRDKAFKA_TARBALL"
44
+
45
+ # Find patches
46
+ PATCHES_FOUND=()
47
+ find_patches "$PATCHES_DIR" PATCHES_FOUND
48
+
49
+ security_log "Starting secure build with checksum verification enabled"
50
+ log "Building self-contained librdkafka.so for Linux aarch64 GNU"
51
+ log "Dependencies to build:"
52
+ log " - OpenSSL: $OPENSSL_VERSION"
53
+ log " - Cyrus SASL: $CYRUS_SASL_VERSION"
54
+ log " - MIT Kerberos: $KRB5_VERSION"
55
+ log " - zlib: $ZLIB_VERSION"
56
+ log " - ZStd: $ZSTD_VERSION"
57
+ log "librdkafka source: $LIBRDKAFKA_TARBALL"
58
+ log "Build directory: $BUILD_DIR"
59
+
60
+ # Create build directory
61
+ mkdir -p "$BUILD_DIR"
62
+ cd "$BUILD_DIR"
63
+
64
+ # Build OpenSSL
65
+ log "Building OpenSSL $OPENSSL_VERSION for ARM64..."
66
+ OPENSSL_PREFIX="$DEPS_PREFIX/static-openssl-$OPENSSL_VERSION-arm64"
67
+ OPENSSL_TARBALL="openssl-$OPENSSL_VERSION.tar.gz"
68
+ OPENSSL_DIR="openssl-$OPENSSL_VERSION"
69
+
70
+ secure_download "$(get_openssl_url)" "$OPENSSL_TARBALL"
71
+ extract_if_needed "$OPENSSL_TARBALL" "$OPENSSL_DIR"
72
+ cd "$OPENSSL_DIR"
73
+
74
+ # Check if OpenSSL lib directory exists (lib or lib64)
75
+ if [ ! -f "$OPENSSL_PREFIX/lib/libssl.a" ] && [ ! -f "$OPENSSL_PREFIX/lib64/libssl.a" ]; then
76
+ log "Configuring and building OpenSSL for ARM64..."
77
+ export CFLAGS="-fPIC"
78
+ # Use linux-aarch64 target for ARM64
79
+ ./Configure linux-aarch64 \
80
+ no-shared \
81
+ no-dso \
82
+ --prefix="$OPENSSL_PREFIX"
83
+ make clean || true
84
+ make -j$(get_cpu_count)
85
+ make install
86
+ unset CFLAGS
87
+ log "OpenSSL built successfully"
88
+ else
89
+ log "OpenSSL already built, skipping..."
90
+ fi
91
+
92
+ # Determine OpenSSL lib directory
93
+ if [ -f "$OPENSSL_PREFIX/lib64/libssl.a" ]; then
94
+ OPENSSL_LIB_DIR="$OPENSSL_PREFIX/lib64"
95
+ else
96
+ OPENSSL_LIB_DIR="$OPENSSL_PREFIX/lib"
97
+ fi
98
+ log "OpenSSL libraries in: $OPENSSL_LIB_DIR"
99
+
100
+ cd "$BUILD_DIR"
101
+
102
+ # Build MIT Kerberos (krb5)
103
+ log "Building MIT Kerberos $KRB5_VERSION for ARM64..."
104
+ KRB5_PREFIX="$DEPS_PREFIX/static-krb5-$KRB5_VERSION-arm64"
105
+ KRB5_TARBALL="krb5-$KRB5_VERSION.tar.gz"
106
+ KRB5_DIR="krb5-$KRB5_VERSION"
107
+
108
+ secure_download "$(get_krb5_url)" "$KRB5_TARBALL"
109
+ extract_if_needed "$KRB5_TARBALL" "$KRB5_DIR"
110
+ cd "$KRB5_DIR/src"
111
+
112
+ if [ ! -f "$KRB5_PREFIX/lib/libgssapi_krb5.a" ]; then
113
+ log "Configuring and building MIT Kerberos for ARM64..."
114
+ make clean 2>/dev/null || true
115
+ ./configure --disable-shared --enable-static --prefix="$KRB5_PREFIX" \
116
+ --without-ldap --without-tcl --without-keyutils \
117
+ --disable-rpath --without-system-verto \
118
+ CFLAGS="-fPIC" CXXFLAGS="-fPIC"
119
+
120
+ # Build everything except the problematic kadmin tools
121
+ log "Building Kerberos (will ignore kadmin build failures)..."
122
+ make -j$(get_cpu_count) || {
123
+ log "Full build failed (expected due to kadmin), continuing with libraries..."
124
+ # The libraries should be built even if kadmin fails
125
+ true
126
+ }
127
+
128
+ # Install what was successfully built
129
+ make install || {
130
+ log "Full install failed, installing individual components..."
131
+ # Try to install the core libraries manually
132
+ make install-mkdirs 2>/dev/null || true
133
+ make -C util install 2>/dev/null || true
134
+ make -C lib install 2>/dev/null || true
135
+ make -C plugins/kdb/db2 install 2>/dev/null || true
136
+ }
137
+
138
+ # Verify we got the essential libraries
139
+ if [ ! -f "$KRB5_PREFIX/lib/libgssapi_krb5.a" ]; then
140
+ error "Failed to build essential Kerberos libraries"
141
+ fi
142
+
143
+ log "MIT Kerberos libraries built successfully"
144
+ else
145
+ log "MIT Kerberos already built, skipping..."
146
+ fi
147
+
148
+ cd "$BUILD_DIR"
149
+
150
+ # Build SASL
151
+ log "Building Cyrus SASL $CYRUS_SASL_VERSION for ARM64..."
152
+ SASL_PREFIX="$DEPS_PREFIX/static-sasl-$CYRUS_SASL_VERSION-arm64"
153
+ SASL_TARBALL="cyrus-sasl-$CYRUS_SASL_VERSION.tar.gz"
154
+ SASL_DIR="cyrus-sasl-$CYRUS_SASL_VERSION"
155
+
156
+ secure_download "$(get_sasl_url)" "$SASL_TARBALL"
157
+ extract_if_needed "$SASL_TARBALL" "$SASL_DIR"
158
+ cd "$SASL_DIR"
159
+
160
+ if [ ! -f "$SASL_PREFIX/lib/libsasl2.a" ]; then
161
+ log "Configuring and building SASL for ARM64..."
162
+ make clean 2>/dev/null || true
163
+ ./configure --disable-shared --enable-static --prefix="$SASL_PREFIX" \
164
+ --without-dblib --disable-gdbm \
165
+ --enable-gssapi="$KRB5_PREFIX" \
166
+ CFLAGS="-fPIC" CXXFLAGS="-fPIC" \
167
+ CPPFLAGS="-I$KRB5_PREFIX/include" \
168
+ LDFLAGS="-L$KRB5_PREFIX/lib"
169
+ make -j$(get_cpu_count)
170
+ make install
171
+ log "SASL built successfully"
172
+ else
173
+ log "SASL already built, skipping..."
174
+ fi
175
+
176
+ cd "$BUILD_DIR"
177
+
178
+ # Build zlib
179
+ log "Building zlib $ZLIB_VERSION for ARM64..."
180
+ ZLIB_PREFIX="$DEPS_PREFIX/static-zlib-$ZLIB_VERSION-arm64"
181
+ ZLIB_TARBALL="zlib-$ZLIB_VERSION.tar.gz"
182
+ ZLIB_DIR="zlib-$ZLIB_VERSION"
183
+
184
+ secure_download "$(get_zlib_url)" "$ZLIB_TARBALL"
185
+ extract_if_needed "$ZLIB_TARBALL" "$ZLIB_DIR"
186
+ cd "$ZLIB_DIR"
187
+
188
+ if [ ! -f "$ZLIB_PREFIX/lib/libz.a" ]; then
189
+ log "Configuring and building zlib for ARM64..."
190
+ make clean 2>/dev/null || true
191
+ export CFLAGS="-fPIC"
192
+ ./configure --prefix="$ZLIB_PREFIX" --static
193
+ make -j$(get_cpu_count)
194
+ make install
195
+ unset CFLAGS
196
+ log "zlib built successfully"
197
+ else
198
+ log "zlib already built, skipping..."
199
+ fi
200
+
201
+ cd "$BUILD_DIR"
202
+
203
+ # Build ZStd
204
+ log "Building ZStd $ZSTD_VERSION for ARM64..."
205
+ ZSTD_PREFIX="$DEPS_PREFIX/static-zstd-$ZSTD_VERSION-arm64"
206
+ ZSTD_TARBALL="zstd-$ZSTD_VERSION.tar.gz"
207
+ ZSTD_DIR="zstd-$ZSTD_VERSION"
208
+
209
+ secure_download "$(get_zstd_url)" "$ZSTD_TARBALL"
210
+ extract_if_needed "$ZSTD_TARBALL" "$ZSTD_DIR"
211
+ cd "$ZSTD_DIR"
212
+
213
+ if [ ! -f "$ZSTD_PREFIX/lib/libzstd.a" ]; then
214
+ log "Building ZStd for ARM64..."
215
+ make clean 2>/dev/null || true
216
+ make lib-mt CFLAGS="-fPIC" PREFIX="$ZSTD_PREFIX" -j$(get_cpu_count)
217
+ # Use standard install target - install-pc may not exist in all versions
218
+ make install PREFIX="$ZSTD_PREFIX"
219
+ log "ZStd built successfully"
220
+ else
221
+ log "ZStd already built, skipping..."
222
+ fi
223
+
224
+ cd "$BUILD_DIR"
225
+
226
+ # Extract and patch librdkafka
227
+ log "Extracting librdkafka..."
228
+ tar xzf "$LIBRDKAFKA_TARBALL"
229
+ cd "librdkafka-$LIBRDKAFKA_VERSION"
230
+
231
+ # Fix permissions and apply patches
232
+ fix_configure_permissions
233
+ apply_patches PATCHES_FOUND
234
+
235
+ # Configure librdkafka
236
+ log "Configuring librdkafka for ARM64..."
237
+
238
+ if [ -f configure ]; then
239
+ log "Using standard configure (autotools)"
240
+ # Export environment variables for configure to pick up
241
+ export CPPFLAGS="-I$KRB5_PREFIX/include"
242
+ export LDFLAGS="-L$KRB5_PREFIX/lib"
243
+
244
+ ./configure --enable-static --disable-shared --disable-curl \
245
+ --enable-gssapi
246
+
247
+ # Clean up environment variables
248
+ unset CPPFLAGS LDFLAGS
249
+ else
250
+ error "No configure script found (checked: configure.self, configure)"
251
+ fi
252
+
253
+ # Build librdkafka
254
+ log "Compiling librdkafka for ARM64..."
255
+ make clean || true
256
+ make -j$(get_cpu_count)
257
+
258
+ # Verify librdkafka.a exists
259
+ if [ ! -f src/librdkafka.a ]; then
260
+ error "librdkafka.a not found after build"
261
+ fi
262
+
263
+ log "librdkafka.a built successfully"
264
+
265
+ # Create self-contained shared library
266
+ log "Creating self-contained librdkafka.so for ARM64..."
267
+
268
+ # Write the export map
269
+ cat > export.map <<'EOF'
270
+ {
271
+ global:
272
+ rd_kafka_*;
273
+ local:
274
+ *;
275
+ };
276
+ EOF
277
+
278
+ # Link everything statically, expose only rd_kafka_* symbols
279
+ aarch64-linux-gnu-gcc -shared -fPIC \
280
+ -Wl,--version-script=export.map \
281
+ -Wl,--whole-archive src/librdkafka.a -Wl,--no-whole-archive \
282
+ -o librdkafka.so \
283
+ "$SASL_PREFIX/lib/libsasl2.a" \
284
+ "$KRB5_PREFIX/lib/libgssapi_krb5.a" \
285
+ "$KRB5_PREFIX/lib/libkrb5.a" \
286
+ "$KRB5_PREFIX/lib/libk5crypto.a" \
287
+ "$KRB5_PREFIX/lib/libcom_err.a" \
288
+ "$KRB5_PREFIX/lib/libkrb5support.a" \
289
+ "$OPENSSL_LIB_DIR/libssl.a" \
290
+ "$OPENSSL_LIB_DIR/libcrypto.a" \
291
+ "$ZLIB_PREFIX/lib/libz.a" \
292
+ "$ZSTD_PREFIX/lib/libzstd.a" \
293
+ -lpthread -lm -ldl -lresolv
294
+
295
+ if [ ! -f librdkafka.so ]; then
296
+ error "Failed to create librdkafka.so"
297
+ fi
298
+
299
+ log "librdkafka.so created successfully"
300
+
301
+ # Verify the build
302
+ log "Verifying build..."
303
+ file librdkafka.so
304
+
305
+ log "Checking dependencies with ldd:"
306
+ ldd librdkafka.so
307
+
308
+ log "Checking for external dependencies (should only show system libraries):"
309
+ EXTERNAL_DEPS=$(nm -D librdkafka.so | grep " U " | grep -v "@GLIBC" || true)
310
+ if [ -n "$EXTERNAL_DEPS" ]; then
311
+ error "Found external dependencies - library is not self-contained: $EXTERNAL_DEPS"
312
+ else
313
+ log "✅ No external dependencies found - library is self-contained!"
314
+ fi
315
+
316
+ # Copy to output directory
317
+ OUTPUT_DIR="$SCRIPT_DIR"
318
+ cp librdkafka.so "$OUTPUT_DIR/"
319
+ log "librdkafka.so copied to: $OUTPUT_DIR/librdkafka.so"
320
+
321
+ # Print summaries
322
+ print_security_summary
323
+ print_build_summary "Linux" "aarch64 GNU" "$OUTPUT_DIR" "librdkafka.so"
324
+
325
+ # Cleanup
326
+ cleanup_build_dir "$BUILD_DIR"
@@ -0,0 +1,317 @@
1
+ #!/usr/bin/env bash
2
+ #
3
+ # Build self-contained librdkafka.so for Linux x86_64 with checksum verification
4
+ # Usage: ./build-librdkafka-linux.sh
5
+ #
6
+ # Expected directory structure:
7
+ # ext/build_linux_x86_64_gnu.sh (this script)
8
+ # ext/build-common.sh (shared functions)
9
+ # dist/librdkafka-*.tar.gz (librdkafka source tarball)
10
+ # dist/patches/*.patch (optional Ruby-specific patches)
11
+ #
12
+ set -euo pipefail
13
+
14
+ # Source common functions and constants
15
+ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
16
+ source "$SCRIPT_DIR/build_common.sh"
17
+
18
+ # Platform-specific paths
19
+ DIST_DIR="$SCRIPT_DIR/../dist"
20
+ PATCHES_DIR="$DIST_DIR/patches"
21
+ BUILD_DIR="$(pwd)/build-tmp"
22
+ DEPS_PREFIX="/tmp"
23
+
24
+ # Check common dependencies
25
+ check_common_dependencies
26
+
27
+ # Linux-specific dependency check
28
+ log "Checking Linux-specific build dependencies..."
29
+ command -v gcc &> /dev/null || error "gcc not found. Install with: apt-get install build-essential"
30
+
31
+ # Auto-detect librdkafka tarball
32
+ log "Looking for librdkafka tarball in $DIST_DIR..."
33
+ LIBRDKAFKA_TARBALL=$(find_librdkafka_tarball "$DIST_DIR")
34
+ log "Found librdkafka tarball: $LIBRDKAFKA_TARBALL"
35
+
36
+ # Verify librdkafka tarball checksum if available
37
+ verify_librdkafka_checksum "$LIBRDKAFKA_TARBALL"
38
+
39
+ # Find patches
40
+ PATCHES_FOUND=()
41
+ find_patches "$PATCHES_DIR" PATCHES_FOUND
42
+
43
+ security_log "Starting secure build with checksum verification enabled"
44
+ log "Building self-contained librdkafka.so for Linux x86_64 GNU"
45
+ log "Dependencies to build:"
46
+ log " - OpenSSL: $OPENSSL_VERSION"
47
+ log " - Cyrus SASL: $CYRUS_SASL_VERSION"
48
+ log " - MIT Kerberos: $KRB5_VERSION"
49
+ log " - zlib: $ZLIB_VERSION"
50
+ log " - ZStd: $ZSTD_VERSION"
51
+ log "librdkafka source: $LIBRDKAFKA_TARBALL"
52
+ log "Build directory: $BUILD_DIR"
53
+
54
+ # Create build directory
55
+ mkdir -p "$BUILD_DIR"
56
+ cd "$BUILD_DIR"
57
+
58
+ # Build OpenSSL
59
+ log "Building OpenSSL $OPENSSL_VERSION..."
60
+ OPENSSL_PREFIX="$DEPS_PREFIX/static-openssl-$OPENSSL_VERSION"
61
+ OPENSSL_TARBALL="openssl-$OPENSSL_VERSION.tar.gz"
62
+ OPENSSL_DIR="openssl-$OPENSSL_VERSION"
63
+
64
+ secure_download "$(get_openssl_url)" "$OPENSSL_TARBALL"
65
+ extract_if_needed "$OPENSSL_TARBALL" "$OPENSSL_DIR"
66
+ cd "$OPENSSL_DIR"
67
+
68
+ # Check if OpenSSL lib directory exists (lib or lib64)
69
+ if [ ! -f "$OPENSSL_PREFIX/lib/libssl.a" ] && [ ! -f "$OPENSSL_PREFIX/lib64/libssl.a" ]; then
70
+ log "Configuring and building OpenSSL..."
71
+ export CFLAGS="-fPIC"
72
+ ./Configure linux-x86_64 \
73
+ no-shared \
74
+ no-dso \
75
+ --prefix="$OPENSSL_PREFIX"
76
+ make clean || true
77
+ make -j$(get_cpu_count)
78
+ make install
79
+ unset CFLAGS
80
+ log "OpenSSL built successfully"
81
+ else
82
+ log "OpenSSL already built, skipping..."
83
+ fi
84
+
85
+ # Determine OpenSSL lib directory
86
+ if [ -f "$OPENSSL_PREFIX/lib64/libssl.a" ]; then
87
+ OPENSSL_LIB_DIR="$OPENSSL_PREFIX/lib64"
88
+ else
89
+ OPENSSL_LIB_DIR="$OPENSSL_PREFIX/lib"
90
+ fi
91
+ log "OpenSSL libraries in: $OPENSSL_LIB_DIR"
92
+
93
+ cd "$BUILD_DIR"
94
+
95
+ # Build MIT Kerberos (krb5)
96
+ log "Building MIT Kerberos $KRB5_VERSION..."
97
+ KRB5_PREFIX="$DEPS_PREFIX/static-krb5-$KRB5_VERSION"
98
+ KRB5_TARBALL="krb5-$KRB5_VERSION.tar.gz"
99
+ KRB5_DIR="krb5-$KRB5_VERSION"
100
+
101
+ secure_download "$(get_krb5_url)" "$KRB5_TARBALL"
102
+ extract_if_needed "$KRB5_TARBALL" "$KRB5_DIR"
103
+ cd "$KRB5_DIR/src"
104
+
105
+ if [ ! -f "$KRB5_PREFIX/lib/libgssapi_krb5.a" ]; then
106
+ log "Configuring and building MIT Kerberos..."
107
+ make clean 2>/dev/null || true
108
+ ./configure --disable-shared --enable-static --prefix="$KRB5_PREFIX" \
109
+ --without-ldap --without-tcl --without-keyutils \
110
+ --disable-rpath --without-system-verto \
111
+ CFLAGS="-fPIC" CXXFLAGS="-fPIC"
112
+
113
+ # Build everything except the problematic kadmin tools
114
+ log "Building Kerberos (will ignore kadmin build failures)..."
115
+ make -j$(get_cpu_count) || {
116
+ log "Full build failed (expected due to kadmin), continuing with libraries..."
117
+ # The libraries should be built even if kadmin fails
118
+ true
119
+ }
120
+
121
+ # Install what was successfully built
122
+ make install || {
123
+ log "Full install failed, installing individual components..."
124
+ # Try to install the core libraries manually
125
+ make install-mkdirs 2>/dev/null || true
126
+ make -C util install 2>/dev/null || true
127
+ make -C lib install 2>/dev/null || true
128
+ make -C plugins/kdb/db2 install 2>/dev/null || true
129
+ }
130
+
131
+ # Verify we got the essential libraries
132
+ if [ ! -f "$KRB5_PREFIX/lib/libgssapi_krb5.a" ]; then
133
+ error "Failed to build essential Kerberos libraries"
134
+ fi
135
+
136
+ log "MIT Kerberos libraries built successfully"
137
+ else
138
+ log "MIT Kerberos already built, skipping..."
139
+ fi
140
+
141
+ cd "$BUILD_DIR"
142
+
143
+ # Build SASL
144
+ log "Building Cyrus SASL $CYRUS_SASL_VERSION..."
145
+ SASL_PREFIX="$DEPS_PREFIX/static-sasl-$CYRUS_SASL_VERSION"
146
+ SASL_TARBALL="cyrus-sasl-$CYRUS_SASL_VERSION.tar.gz"
147
+ SASL_DIR="cyrus-sasl-$CYRUS_SASL_VERSION"
148
+
149
+ secure_download "$(get_sasl_url)" "$SASL_TARBALL"
150
+ extract_if_needed "$SASL_TARBALL" "$SASL_DIR"
151
+ cd "$SASL_DIR"
152
+
153
+ if [ ! -f "$SASL_PREFIX/lib/libsasl2.a" ]; then
154
+ log "Configuring and building SASL..."
155
+ make clean 2>/dev/null || true
156
+ ./configure --disable-shared --enable-static --prefix="$SASL_PREFIX" \
157
+ --without-dblib --disable-gdbm \
158
+ --enable-gssapi="$KRB5_PREFIX" \
159
+ CFLAGS="-fPIC" CXXFLAGS="-fPIC" \
160
+ CPPFLAGS="-I$KRB5_PREFIX/include" \
161
+ LDFLAGS="-L$KRB5_PREFIX/lib"
162
+ make -j$(get_cpu_count)
163
+ make install
164
+ log "SASL built successfully"
165
+ else
166
+ log "SASL already built, skipping..."
167
+ fi
168
+
169
+ cd "$BUILD_DIR"
170
+
171
+ # Build zlib
172
+ log "Building zlib $ZLIB_VERSION..."
173
+ ZLIB_PREFIX="$DEPS_PREFIX/static-zlib-$ZLIB_VERSION"
174
+ ZLIB_TARBALL="zlib-$ZLIB_VERSION.tar.gz"
175
+ ZLIB_DIR="zlib-$ZLIB_VERSION"
176
+
177
+ secure_download "$(get_zlib_url)" "$ZLIB_TARBALL"
178
+ extract_if_needed "$ZLIB_TARBALL" "$ZLIB_DIR"
179
+ cd "$ZLIB_DIR"
180
+
181
+ if [ ! -f "$ZLIB_PREFIX/lib/libz.a" ]; then
182
+ log "Configuring and building zlib..."
183
+ make clean 2>/dev/null || true
184
+ export CFLAGS="-fPIC"
185
+ ./configure --prefix="$ZLIB_PREFIX" --static
186
+ make -j$(get_cpu_count)
187
+ make install
188
+ unset CFLAGS
189
+ log "zlib built successfully"
190
+ else
191
+ log "zlib already built, skipping..."
192
+ fi
193
+
194
+ cd "$BUILD_DIR"
195
+
196
+ # Build ZStd
197
+ log "Building ZStd $ZSTD_VERSION..."
198
+ ZSTD_PREFIX="$DEPS_PREFIX/static-zstd-$ZSTD_VERSION"
199
+ ZSTD_TARBALL="zstd-$ZSTD_VERSION.tar.gz"
200
+ ZSTD_DIR="zstd-$ZSTD_VERSION"
201
+
202
+ secure_download "$(get_zstd_url)" "$ZSTD_TARBALL"
203
+ extract_if_needed "$ZSTD_TARBALL" "$ZSTD_DIR"
204
+ cd "$ZSTD_DIR"
205
+
206
+ if [ ! -f "$ZSTD_PREFIX/lib/libzstd.a" ]; then
207
+ log "Building ZStd..."
208
+ make clean 2>/dev/null || true
209
+ make lib-mt CFLAGS="-fPIC" PREFIX="$ZSTD_PREFIX" -j$(get_cpu_count)
210
+ # Use standard install target - install-pc may not exist in all versions
211
+ make install PREFIX="$ZSTD_PREFIX"
212
+ log "ZStd built successfully"
213
+ else
214
+ log "ZStd already built, skipping..."
215
+ fi
216
+
217
+ cd "$BUILD_DIR"
218
+
219
+ # Extract and patch librdkafka
220
+ log "Extracting librdkafka..."
221
+ tar xzf "$LIBRDKAFKA_TARBALL"
222
+ cd "librdkafka-$LIBRDKAFKA_VERSION"
223
+
224
+ # Fix permissions and apply patches
225
+ fix_configure_permissions
226
+ apply_patches PATCHES_FOUND
227
+
228
+ # Configure librdkafka
229
+ log "Configuring librdkafka..."
230
+
231
+ if [ -f configure ]; then
232
+ log "Using standard configure (autotools)"
233
+ # Export environment variables for configure to pick up
234
+ export CPPFLAGS="-I$KRB5_PREFIX/include"
235
+ export LDFLAGS="-L$KRB5_PREFIX/lib"
236
+
237
+ ./configure --enable-static --disable-shared --disable-curl \
238
+ --enable-gssapi
239
+
240
+ # Clean up environment variables
241
+ unset CPPFLAGS LDFLAGS
242
+ else
243
+ error "No configure script found (checked: configure.self, configure)"
244
+ fi
245
+
246
+ # Build librdkafka
247
+ log "Compiling librdkafka..."
248
+ make clean || true
249
+ make -j$(get_cpu_count)
250
+
251
+ # Verify librdkafka.a exists
252
+ if [ ! -f src/librdkafka.a ]; then
253
+ error "librdkafka.a not found after build"
254
+ fi
255
+
256
+ log "librdkafka.a built successfully"
257
+
258
+ # Create self-contained shared library
259
+ log "Creating self-contained librdkafka.so..."
260
+
261
+ echo '
262
+ {
263
+ global:
264
+ rd_kafka_*;
265
+ local:
266
+ *;
267
+ };
268
+ ' > export.map
269
+
270
+ gcc -shared -fPIC \
271
+ -Wl,--version-script=export.map \
272
+ -Wl,--whole-archive src/librdkafka.a -Wl,--no-whole-archive \
273
+ -o librdkafka.so \
274
+ "$SASL_PREFIX/lib/libsasl2.a" \
275
+ "$KRB5_PREFIX/lib/libgssapi_krb5.a" \
276
+ "$KRB5_PREFIX/lib/libkrb5.a" \
277
+ "$KRB5_PREFIX/lib/libk5crypto.a" \
278
+ "$KRB5_PREFIX/lib/libcom_err.a" \
279
+ "$KRB5_PREFIX/lib/libkrb5support.a" \
280
+ "$OPENSSL_LIB_DIR/libssl.a" \
281
+ "$OPENSSL_LIB_DIR/libcrypto.a" \
282
+ "$ZLIB_PREFIX/lib/libz.a" \
283
+ "$ZSTD_PREFIX/lib/libzstd.a" \
284
+ -lpthread -lm -ldl -lresolv
285
+
286
+ if [ ! -f librdkafka.so ]; then
287
+ error "Failed to create librdkafka.so"
288
+ fi
289
+
290
+ log "librdkafka.so created successfully"
291
+
292
+ # Verify the build
293
+ log "Verifying build..."
294
+ file librdkafka.so
295
+
296
+ log "Checking dependencies with ldd:"
297
+ ldd librdkafka.so
298
+
299
+ log "Checking for non-system library dependencies:"
300
+ NON_SYSTEM_DEPS=$(ldd librdkafka.so | grep -v -E "(linux-vdso|ld-linux|libc\.so|libpthread\.so|libm\.so|libdl\.so)" || true)
301
+ if [ -n "$NON_SYSTEM_DEPS" ]; then
302
+ error "Found non-system dependencies: $NON_SYSTEM_DEPS"
303
+ else
304
+ log "✅ Only system dependencies found - library is portable!"
305
+ fi
306
+
307
+ # Copy to output directory
308
+ OUTPUT_DIR="$SCRIPT_DIR"
309
+ cp librdkafka.so "$OUTPUT_DIR/"
310
+ log "librdkafka.so copied to: $OUTPUT_DIR/librdkafka.so"
311
+
312
+ # Print summaries
313
+ print_security_summary
314
+ print_build_summary "Linux" "x86_64" "$OUTPUT_DIR" "librdkafka.so"
315
+
316
+ # Cleanup
317
+ cleanup_build_dir "$BUILD_DIR"