karafka-rdkafka 0.21.0.rc1 → 0.21.0.rc2

data/ext/build_macos_arm64.sh

@@ -448,6 +448,12 @@ log "Creating self-contained librdkafka.dylib with Kerberos support..."
 
 # Create self-contained shared library by linking all static dependencies (NOW INCLUDING KERBEROS)
 # This is the macOS equivalent of your Linux gcc -shared command
+
+# Write symbol export file (macOS equivalent of export.map)
+cat > export_symbols.txt <<'EOF'
+_rd_kafka_*
+EOF
+
 clang -dynamiclib -fPIC \
     -Wl,-force_load,src/librdkafka.a \
     -Wl,-force_load,"$SASL_PREFIX/lib/libsasl2.a" \
@@ -462,6 +468,7 @@ clang -dynamiclib -fPIC \
     -Wl,-force_load,"$ZSTD_PREFIX/lib/libzstd.a" \
     -o librdkafka.dylib \
     -lpthread -lc -arch $ARCH -lresolv \
+    -framework GSS -framework Kerberos \
     -install_name @rpath/librdkafka.dylib \
     -Wl,-undefined,dynamic_lookup
 

data/ext/generate-ssl-certs.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+
+#==============================================================================
+# Kafka SSL Certificate Generator
+#==============================================================================
+#
+# DESCRIPTION:
+#   Generates SSL certificates for testing Kafka with SSL/TLS encryption.
+#   Creates both Java KeyStore (JKS) files for Kafka server and PEM files
+#   for client applications like rdkafka.
+#
+# PURPOSE:
+#   - Test SSL connectivity between Kafka clients and brokers
+#   - Validate rdkafka SSL integration
+#   - Enable encrypted communication for development/testing environments
+#
+# USAGE:
+#   ./ext/generate-ssl-certs.sh
+#   docker compose -f docker-compose-ssl.yml up
+#
+# REQUIREMENTS:
+#   - OpenSSL (for certificate generation)
+#   - Java keytool (usually included with JDK/JRE)
+#   - Write permissions in current directory
+#
+# OUTPUT FILES (created in ./ssl/ directory):
+#   ├── kafka.server.keystore.jks    # Kafka server's private key and certificate
+#   ├── kafka.server.truststore.jks  # Trusted CA certificates for Kafka
+#   ├── kafka_keystore_creds         # Password file for keystore
+#   ├── kafka_truststore_creds       # Password file for truststore
+#   ├── kafka_ssl_key_creds          # Password file for SSL keys
+#   ├── ca-cert                      # CA certificate (for rdkafka clients)
+#   └── ca-cert.pem                  # CA certificate in PEM format
+#
+# CONFIGURATION:
+#   - Certificate validity: 365 days
+#   - Password: "confluent" (all certificates use same password for simplicity)
+#   - Subject: CN=localhost (suitable for local testing)
+#   - CA Subject: CN=localhost-ca
+#
+# DOCKER COMPOSE INTEGRATION:
+#   Use with docker-compose-ssl.yml that mounts ./ssl directory to
+#   /etc/kafka/secrets inside the Kafka container.
+#
+# RDKAFKA CLIENT CONFIGURATION:
+#   security.protocol=SSL
+#   ssl.ca.location=./ssl/ca-cert
+#   ssl.endpoint.identification.algorithm=none  # For localhost testing
+#
+# NOTES:
+#   - Safe to run multiple times (cleans up existing files)
+#   - Certificates are self-signed and suitable for testing only
+#   - For production, use certificates signed by a trusted CA
+#   - All passwords are set to "confluent" for simplicity
+#
+#==============================================================================
+
+# Create ssl directory and clean up any existing files
+mkdir -p ssl
+cd ssl
+
+# Clean up existing files
+rm -f kafka.server.keystore.jks kafka.server.truststore.jks
+rm -f kafka_keystore_creds kafka_truststore_creds kafka_ssl_key_creds
+rm -f ca-key ca-cert cert-file cert-signed ca-cert.srl ca-cert.pem
+
+echo "Cleaned up existing SSL files..."
+
+# Set variables
+VALIDITY_DAYS=365
+PASSWORD="confluent"  # Use a simpler, well-known password
+DNAME="CN=localhost,OU=Test,O=Test,L=Test,ST=Test,C=US"
+
+# Create password files (all same password for simplicity)
+echo "$PASSWORD" > kafka_keystore_creds
+echo "$PASSWORD" > kafka_truststore_creds
+echo "$PASSWORD" > kafka_ssl_key_creds
+
+# Step 1: Generate CA key and certificate
+openssl req -new -x509 -keyout ca-key -out ca-cert -days $VALIDITY_DAYS -subj "/CN=localhost-ca/OU=Test/O=Test/L=Test/S=Test/C=US" -passin pass:$PASSWORD -passout pass:$PASSWORD
+
+# Step 2: Create truststore and import the CA certificate
+keytool -keystore kafka.server.truststore.jks -alias CARoot -import -file ca-cert -storepass $PASSWORD -keypass $PASSWORD -noprompt
+
+# Step 3: Create keystore
+keytool -keystore kafka.server.keystore.jks -alias localhost -validity $VALIDITY_DAYS -genkey -keyalg RSA -dname "$DNAME" -storepass $PASSWORD -keypass $PASSWORD
+
+# Step 4: Create certificate signing request
+keytool -keystore kafka.server.keystore.jks -alias localhost -certreq -file cert-file -storepass $PASSWORD -keypass $PASSWORD
+
+# Step 5: Sign the certificate with the CA
+openssl x509 -req -CA ca-cert -CAkey ca-key -in cert-file -out cert-signed -days $VALIDITY_DAYS -CAcreateserial -passin pass:$PASSWORD
+
+# Step 6: Import CA certificate into keystore
+keytool -keystore kafka.server.keystore.jks -alias CARoot -import -file ca-cert -storepass $PASSWORD -keypass $PASSWORD -noprompt
+
+# Step 7: Import signed certificate into keystore
+keytool -keystore kafka.server.keystore.jks -alias localhost -import -file cert-signed -storepass $PASSWORD -keypass $PASSWORD -noprompt
+
+# Export CA certificate to PEM format for rdkafka
+cp ca-cert ca-cert.pem
+
+# Clean up intermediate files (but keep ca-cert.pem for rdkafka)
+rm ca-key cert-file cert-signed
+
+echo "SSL certificates generated successfully!"
+echo "Password: $PASSWORD"
+echo ""
+echo "For rdkafka, use ca-cert.pem or ca-cert files"

data/karafka-rdkafka.gemspec

@@ -41,6 +41,7 @@ Gem::Specification.new do |gem|
   end
 
   gem.add_dependency 'ffi', '~> 1.15'
+  gem.add_dependency 'json', '> 2.0'
   gem.add_dependency 'logger'
   gem.add_dependency 'mini_portile2', '~> 2.6'
   gem.add_dependency 'rake', '> 12'
@@ -50,6 +51,7 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency 'rspec', '~> 3.5'
   gem.add_development_dependency 'rake'
   gem.add_development_dependency 'simplecov'
+  gem.add_development_dependency 'warning'
 
   gem.metadata = {
     'funding_uri' => 'https://karafka.io/#become-pro',

data/lib/rdkafka/bindings.rb

@@ -160,7 +160,6 @@ module Rdkafka
     attach_function :rd_kafka_error_is_retriable, [:pointer], :int
     attach_function :rd_kafka_error_txn_requires_abort, [:pointer], :int
     attach_function :rd_kafka_error_destroy, [:pointer], :void
-    attach_function :rd_kafka_error_code, [:pointer], :int
     attach_function :rd_kafka_get_err_descs, [:pointer, :pointer], :void
 
     # Configuration

data/lib/rdkafka/consumer.rb

@@ -344,7 +344,7 @@ module Rdkafka
         topic_out = {}
         partitions.each do |p|
           next if p.offset.nil?
-          _, high = query_watermark_offsets(
+          _low, high = query_watermark_offsets(
             topic,
             p.partition,
             watermark_timeout_ms

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.21.0.rc1"
+  VERSION = "0.21.0.rc2"
   LIBRDKAFKA_VERSION = "2.11.0"
   LIBRDKAFKA_SOURCE_SHA256 = "592a823dc7c09ad4ded1bc8f700da6d4e0c88ffaf267815c6f25e7450b9395ca"
 end

data/spec/integrations/ssl_stress_spec.rb

@@ -0,0 +1,121 @@
+# ssl_stress_test.rb
+#
+# This script is designed to stress-test the OpenSSL SSL/TLS layer under high concurrency
+# to help detect regressions like the one described in OpenSSL issue #28171:
+# https://github.com/openssl/openssl/issues/28171
+#
+# Issue summary:
+#   - OpenSSL 3.0.17 introduced a concurrency-related regression.
+#   - Multiple threads sharing the same SSL_CTX and making parallel TLS connections
+#     (often with certificate verification enabled) can cause segmentation faults
+#     due to race conditions in X509 store handling.
+#   - Affected users include Python (httpx), Rust (reqwest, native-tls), and C applications.
+#
+# Script details:
+#   - Starts 100 SSL servers using self-signed, in-memory certs on sequential localhost ports.
+#   - Uses `rdkafka-ruby` to spin up 100 consumer threads that continuously create and destroy
+#     SSL connections to these servers for a given duration.
+#   - This mimics high TLS connection churn and aims to trigger latent SSL_CTX or X509_STORE
+#     threading bugs like double-frees, memory corruption, or segmentation faults.
+#
+# Goal:
+#   - Catch regressions early by validating that heavy concurrent SSL use does not lead to crashes.
+#   - Provide a minimal and repeatable reproducer when diagnosing OpenSSL-level SSL instability.
+#
+# In case of a failure, segfault will happen
+
+require 'rdkafka'
+require 'socket'
+require 'openssl'
+
+$stdout.sync = true
+
+STARTING_PORT = 19093
+NUM_PORTS = 150
+BATCHES = 100
+PORTS = STARTING_PORT...(STARTING_PORT + NUM_PORTS)
+
+CONFIG = {
+  'bootstrap.servers': Array.new(NUM_PORTS) { |i| "127.0.0.1:#{19093+i}" }.join(','),
+  'security.protocol': 'SSL',
+  'enable.ssl.certificate.verification': false
+}
+
+# Generate in-memory self-signed cert
+key = OpenSSL::PKey::RSA.new(2048)
+
+name = OpenSSL::X509::Name.parse("/CN=127.0.0.1")
+cert = OpenSSL::X509::Certificate.new
+cert.version = 2
+cert.serial = 1
+cert.subject = name
+cert.issuer = name
+cert.public_key = key.public_key
+cert.not_before = Time.now
+cert.not_after = Time.now + 3600
+cert.sign(key, OpenSSL::Digest::SHA256.new)
+
+# Start servers on multiple ports
+PORTS.map do |port|
+  Thread.new do
+    # Prepare SSL context
+    # We do not use a shared context for the server because the goal is to stress librdkafka layer
+    # and not the Ruby SSL layer
+    ssl_context = OpenSSL::SSL::SSLContext.new
+    ssl_context.cert = cert
+    ssl_context.key  = key
+
+    tcp_server = TCPServer.new('127.0.0.1', port)
+    ssl_server = OpenSSL::SSL::SSLServer.new(tcp_server, ssl_context)
+
+    loop do
+      begin
+        ssl_socket = ssl_server.accept
+        ssl_socket.close
+      rescue => e
+        # Some errors are expected and irrelevant
+        next if e.message.include?('unexpected eof while reading')
+
+        puts "Port #{port} SSL error: #{e}"
+      end
+    end
+  end
+end
+
+puts "Starting #{NUM_PORTS} on ports from #{PORTS.first} to #{PORTS.last} SSL servers"
+
+timeout = 30
+start = Time.now
+
+# Wait for the servers to be available
+# We want to make sure that they are available so we are sure that librdkafka actually hammers
+# them
+loop do
+  all_up = PORTS.all? do |port|
+             TCPSocket.new('127.0.0.1', port).close
+             true
+           rescue
+             false
+           end
+
+  break if all_up
+
+  raise "Timeout waiting for SSL servers" if Time.now - start > timeout
+
+  sleep 0.1
+end
+
+puts "SSL servers ready"
+
+start_time = Time.now
+duration = 60 * 10 # 10 minutes - it should crash faster than that if SSL vulnerable
+attempts = 0
+
+while Time.now - start_time < duration do
+  css = Array.new(BATCHES) { Rdkafka::Config.new(CONFIG) }
+  csss = css.map(&:consumer)
+  # This print is needed. No idea why but it increases the chances of segfault
+  p attempts += 1
+  sleep(1)
+  csss.each(&:close)
+end

data/spec/{rdkafka → lib/rdkafka}/admin_spec.rb

@@ -518,17 +518,17 @@ describe Rdkafka::Admin do
     before do
       #create topic for testing acl
       create_topic_handle = admin.create_topic(resource_name, topic_partition_count, topic_replication_factor)
-      create_topic_report = create_topic_handle.wait(max_wait_timeout: 15.0)
+      create_topic_handle.wait(max_wait_timeout: 15.0)
     end
 
     after do
       #delete acl
       delete_acl_handle = admin.delete_acl(resource_type: resource_type, resource_name: resource_name, resource_pattern_type: resource_pattern_type, principal: principal, host: host, operation: operation, permission_type: permission_type)
-      delete_acl_report = delete_acl_handle.wait(max_wait_timeout: 15.0)
+      delete_acl_handle.wait(max_wait_timeout: 15.0)
 
       #delete topic that was created for testing acl
       delete_topic_handle = admin.delete_topic(resource_name)
-      delete_topic_report = delete_topic_handle.wait(max_wait_timeout: 15.0)
+      delete_topic_handle.wait(max_wait_timeout: 15.0)
     end
 
     describe "#create_acl" do
@@ -876,7 +876,17 @@ describe Rdkafka::Admin do
   end
 
   describe '#create_partitions' do
-    let(:metadata) { admin.metadata(topic_name).topics.first }
+    let(:metadata) do
+      begin
+        admin.metadata(topic_name).topics.first
+      rescue Rdkafka::RdkafkaError
+        # We have to wait because if we query too fast after topic creation request, it may not
+        # yet be available throwing an error.
+        # This occurs mostly on slow CIs
+        sleep(1)
+        admin.metadata(topic_name).topics.first
+      end
+    end
 
     context 'when topic does not exist' do
       it 'expect to fail due to unknown partition' do
@@ -898,6 +908,8 @@ describe Rdkafka::Admin do
 
       it 'expect not to change number of partitions' do
         expect { admin.create_partitions(topic_name, 2).wait }.to raise_error(Rdkafka::RdkafkaError, /invalid_partitions/)
+        # On slow CI this may propagate, thus we wait a bit
+        sleep(1)
         expect(metadata[:partition_count]).to eq(5)
       end
     end

data/spec/{rdkafka → lib/rdkafka}/consumer_spec.rb

@@ -175,6 +175,7 @@ describe Rdkafka::Consumer do
     before do
       admin = rdkafka_producer_config.admin
       admin.create_topic(topic, 1, 1).wait
+      wait_for_topic(admin, topic)
       admin.close
     end
 
@@ -278,6 +279,7 @@ describe Rdkafka::Consumer do
     before do
       admin = rdkafka_producer_config.admin
       admin.create_topic(topic, 1, 1).wait
+      wait_for_topic(admin, topic)
       admin.close
     end
 
@@ -426,7 +428,7 @@ describe Rdkafka::Consumer do
   describe '#assignment_lost?' do
     it "should not return true as we do have an assignment" do
       consumer.subscribe("consume_test_topic")
-      expected_subscription = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+      Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
         list.add_topic("consume_test_topic")
       end
 
@@ -436,7 +438,7 @@ describe Rdkafka::Consumer do
 
     it "should not return true after voluntary unsubscribing" do
       consumer.subscribe("consume_test_topic")
-      expected_subscription = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+      Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
         list.add_topic("consume_test_topic")
       end
 
@@ -1276,7 +1278,6 @@ describe Rdkafka::Consumer do
 
       # Consume to the end
       consumer.subscribe("consume_test_topic")
-      eof_count = 0
       eof_error = nil
 
       loop do
@@ -1293,4 +1294,50 @@ describe Rdkafka::Consumer do
       expect(eof_error.code).to eq(:partition_eof)
     end
   end
+
+  describe "long running consumption" do
+    let(:consumer) { rdkafka_consumer_config.consumer }
+    let(:producer) { rdkafka_producer_config.producer }
+
+    after { consumer.close }
+    after { producer.close }
+
+    it "should consume messages continuously for 60 seconds" do
+      consumer.subscribe("consume_test_topic")
+      wait_for_assignment(consumer)
+
+      messages_consumed = 0
+      start_time = Time.now
+
+      # Producer thread - sends message every second
+      producer_thread = Thread.new do
+        counter = 0
+        while Time.now - start_time < 60
+          producer.produce(
+            topic: "consume_test_topic",
+            payload: "payload #{counter}",
+            key: "key #{counter}",
+            partition: 0
+          ).wait
+          counter += 1
+          sleep(1)
+        end
+      end
+
+      # Consumer loop
+      while Time.now - start_time < 60
+        message = consumer.poll(1000)
+        if message
+          expect(message).to be_a Rdkafka::Consumer::Message
+          expect(message.topic).to eq("consume_test_topic")
+          messages_consumed += 1
+          consumer.commit if messages_consumed % 10 == 0
+        end
+      end
+
+      producer_thread.join
+
+      expect(messages_consumed).to be > 50 # Should consume most messages
+    end
+  end
 end

data/spec/{rdkafka → lib/rdkafka}/metadata_spec.rb

@@ -31,7 +31,7 @@ describe Rdkafka::Metadata do
         expect(subject.brokers.length).to eq(1)
         expect(subject.brokers[0][:broker_id]).to eq(1)
         expect(%w[127.0.0.1 localhost]).to include(subject.brokers[0][:broker_name])
-        expect(subject.brokers[0][:broker_port]).to eq(9092)
+        expect(subject.brokers[0][:broker_port]).to eq(rdkafka_base_config[:'bootstrap.servers'].split(':').last.to_i)
       end
 
       it "#topics returns data on our test topic" do
@@ -54,7 +54,7 @@ describe Rdkafka::Metadata do
       expect(subject.brokers.length).to eq(1)
       expect(subject.brokers[0][:broker_id]).to eq(1)
       expect(%w[127.0.0.1 localhost]).to include(subject.brokers[0][:broker_name])
-      expect(subject.brokers[0][:broker_port]).to eq(9092)
+      expect(subject.brokers[0][:broker_port]).to eq(rdkafka_base_config[:'bootstrap.servers'].split(':').last.to_i)
     end
 
     it "#topics returns data about all of our test topics" do

data/spec/{rdkafka → lib/rdkafka}/producer/delivery_report_spec.rb

@@ -20,6 +20,6 @@ describe Rdkafka::Producer::DeliveryReport do
   end
 
   it "should get the error" do
-    expect(subject.error).to eq -1
+    expect(subject.error).to eq(-1)
   end
 end

data/spec/{rdkafka → lib/rdkafka}/producer_spec.rb

@@ -663,7 +663,7 @@ describe Rdkafka::Producer do
   context "when not being able to deliver the message" do
     let(:producer) do
       rdkafka_producer_config(
-        "bootstrap.servers": "127.0.0.1:9093",
+        "bootstrap.servers": "127.0.0.1:9095",
         "message.timeout.ms": 100
       ).producer
     end
@@ -779,14 +779,15 @@ describe Rdkafka::Producer do
     context 'when it cannot flush due to a timeout' do
       let(:producer) do
         rdkafka_producer_config(
-          "bootstrap.servers": "127.0.0.1:9093",
+          "bootstrap.servers": "127.0.0.1:9095",
           "message.timeout.ms": 2_000
         ).producer
       end
 
       after do
         # Allow rdkafka to evict message preventing memory-leak
-        sleep(2)
+        # We give it a bit more time as on slow CIs things take time
+        sleep(5)
       end
 
       it "should return false on flush when cannot deliver and beyond timeout" do
@@ -826,7 +827,7 @@ describe Rdkafka::Producer do
     context 'when there are outgoing things in the queue' do
       let(:producer) do
         rdkafka_producer_config(
-          "bootstrap.servers": "127.0.0.1:9093",
+          "bootstrap.servers": "127.0.0.1:9095",
           "message.timeout.ms": 2_000
         ).producer
       end
@@ -862,7 +863,7 @@ describe Rdkafka::Producer do
         before { producer.delivery_callback = delivery_callback }
 
         it "should run the callback" do
-          handle = producer.produce(
+          producer.produce(
             topic:     "produce_test_topic",
             payload:   "payload headers"
           )

data/spec/spec_helper.rb

@@ -1,5 +1,19 @@
 # frozen_string_literal: true
 
+Warning[:performance] = true if RUBY_VERSION >= '3.3'
+Warning[:deprecated] = true
+$VERBOSE = true
+
+require 'warning'
+
+Warning.process do |warning|
+  next unless warning.include?(Dir.pwd)
+  # Allow OpenStruct usage only in specs
+  next if warning.include?('OpenStruct use') && warning.include?('_spec')
+
+  raise "Warning in your code: #{warning}"
+end
+
 unless ENV["CI"] == "true"
   require "simplecov"
   SimpleCov.start do
@@ -14,12 +28,25 @@ require "timeout"
 require "securerandom"
 
 def rdkafka_base_config
-  {
-    :"bootstrap.servers" => "localhost:9092",
-    # Display statistics and refresh often just to cover those in specs
-    :'statistics.interval.ms' => 1_000,
-    :'topic.metadata.refresh.interval.ms' => 1_000
-  }
+  if ENV['KAFKA_SSL_ENABLED'] == 'true'
+    {
+      :"bootstrap.servers" => "localhost:9093",
+      # Display statistics and refresh often just to cover those in specs
+      :'statistics.interval.ms' => 1_000,
+      :'topic.metadata.refresh.interval.ms' => 1_000,
+      # SSL Configuration
+      :'security.protocol' => 'SSL',
+      :'ssl.ca.location' => './ssl/ca-cert',
+      :'ssl.endpoint.identification.algorithm' => 'none'
+    }
+  else
+    {
+      :"bootstrap.servers" => "localhost:9092",
+      # Display statistics and refresh often just to cover those in specs
+      :'statistics.interval.ms' => 1_000,
+      :'topic.metadata.refresh.interval.ms' => 1_000
+    }
+  end
 end
 
 def rdkafka_config(config_overrides={})
@@ -122,6 +149,16 @@ def wait_for_unassignment(consumer)
   end
 end
 
+def wait_for_topic(admin, topic)
+  admin.metadata(topic)
+rescue Rdkafka::RdkafkaError => e
+  raise unless e.code == :unknown_topic_or_part
+
+  sleep(0.5)
+
+  retry
+end
+
 def notify_listener(listener, &block)
   # 1. subscribe and poll
   consumer.subscribe("consume_test_topic")
@@ -169,9 +206,9 @@ RSpec.configure do |config|
   end
 
   config.around(:each) do |example|
-    # Timeout specs after a minute. If they take longer
+    # Timeout specs after 1.5 minute. If they take longer
     # they are probably stuck
-    Timeout::timeout(60) do
+    Timeout::timeout(90) do
       example.run
     end
   end