karafka-rdkafka 0.20.0.rc5-arm64-darwin → 0.21.0-arm64-darwin

data/lib/rdkafka/bindings.rb

@@ -160,7 +160,6 @@ module Rdkafka
     attach_function :rd_kafka_error_is_retriable, [:pointer], :int
     attach_function :rd_kafka_error_txn_requires_abort, [:pointer], :int
     attach_function :rd_kafka_error_destroy, [:pointer], :void
-    attach_function :rd_kafka_error_code, [:pointer], :int
     attach_function :rd_kafka_get_err_descs, [:pointer, :pointer], :void
 
     # Configuration
@@ -403,18 +402,16 @@ module Rdkafka
       hsh[name] = method_name
     end
 
-    def self.partitioner(str, partition_count, partitioner_name = "consistent_random")
+    def self.partitioner(topic_ptr, str, partition_count, partitioner = "consistent_random")
       # Return RD_KAFKA_PARTITION_UA(unassigned partition) when partition count is nil/zero.
       return -1 unless partition_count&.nonzero?
-      # musl architecture crashes with empty string
-      return 0 if str.empty?
 
-      str_ptr = FFI::MemoryPointer.from_string(str)
-      method_name = PARTITIONERS.fetch(partitioner_name) do
-        raise Rdkafka::Config::ConfigError.new("Unknown partitioner: #{partitioner_name}")
+      str_ptr = str.empty? ? FFI::MemoryPointer::NULL : FFI::MemoryPointer.from_string(str)
+      method_name = PARTITIONERS.fetch(partitioner) do
+        raise Rdkafka::Config::ConfigError.new("Unknown partitioner: #{partitioner}")
       end
 
-      public_send(method_name, nil, str_ptr, str.size, partition_count, nil, nil)
+      public_send(method_name, topic_ptr, str_ptr, str.size, partition_count, nil, nil)
     end
 
     # Create Topics
@@ -532,6 +529,7 @@ module Rdkafka
     RD_KAFKA_RESOURCE_TOPIC = 2
     RD_KAFKA_RESOURCE_GROUP = 3
     RD_KAFKA_RESOURCE_BROKER = 4
+    RD_KAFKA_RESOURCE_TRANSACTIONAL_ID = 5
 
     # rd_kafka_ResourcePatternType_t - https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7320
 

data/lib/rdkafka/config.rb

@@ -129,10 +129,7 @@ module Rdkafka
     end
 
     # Default config that can be overwritten.
-    DEFAULT_CONFIG = {
-      # Request api version so advanced features work
-      :"api.version.request" => true
-    }.freeze
+    DEFAULT_CONFIG = {}.freeze
 
     # Required config that cannot be overwritten.
     REQUIRED_CONFIG = {

data/lib/rdkafka/consumer.rb

@@ -344,7 +344,7 @@ module Rdkafka
         topic_out = {}
         partitions.each do |p|
           next if p.offset.nil?
-          _, high = query_watermark_offsets(
+          _low, high = query_watermark_offsets(
             topic,
             p.partition,
             watermark_timeout_ms

data/lib/rdkafka/producer.rb

@@ -51,13 +51,13 @@ module Rdkafka
 
     # @private
     # @param native_kafka [NativeKafka]
-    # @param partitioner_name [String, nil] name of the partitioner we want to use or nil to use
+    # @param partitioner [String, nil] name of the partitioner we want to use or nil to use
     #   the "consistent_random" default
-    def initialize(native_kafka, partitioner_name)
+    def initialize(native_kafka, partitioner)
       @topics_refs_map = {}
       @topics_configs = {}
       @native_kafka = native_kafka
-      @partitioner_name = partitioner_name || "consistent_random"
+      @partitioner = partitioner || "consistent_random"
 
       # Makes sure, that native kafka gets closed before it gets GCed by Ruby
       ObjectSpace.define_finalizer(self, native_kafka.finalizer)
@@ -337,7 +337,8 @@ module Rdkafka
       timestamp: nil,
       headers: nil,
       label: nil,
-      topic_config: EMPTY_HASH
+      topic_config: EMPTY_HASH,
+      partitioner: @partitioner
     )
       closed_producer_check(__method__)
 
@@ -369,10 +370,14 @@ module Rdkafka
 
         # Check if there are no overrides for the partitioner and use the default one only when
         # no per-topic is present.
-        partitioner_name = @topics_configs.dig(topic, topic_config_hash, :partitioner) || @partitioner_name
+        selected_partitioner = @topics_configs.dig(topic, topic_config_hash, :partitioner) || partitioner
 
         # If the topic is not present, set to -1
-        partition = Rdkafka::Bindings.partitioner(partition_key, partition_count, partitioner_name) if partition_count.positive?
+        partition = Rdkafka::Bindings.partitioner(
+          topic_ref,
+          partition_key,
+          partition_count,
+          selected_partitioner) if partition_count.positive?
       end
 
       # If partition is nil, use -1 to let librdafka set the partition randomly or

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.20.0.rc5"
-  LIBRDKAFKA_VERSION = "2.8.0"
-  LIBRDKAFKA_SOURCE_SHA256 = "5bd1c46f63265f31c6bfcedcde78703f77d28238eadf23821c2b43fc30be3e25"
+  VERSION = "0.21.0"
+  LIBRDKAFKA_VERSION = "2.11.0"
+  LIBRDKAFKA_SOURCE_SHA256 = "592a823dc7c09ad4ded1bc8f700da6d4e0c88ffaf267815c6f25e7450b9395ca"
 end

data/spec/integrations/ssl_stress_spec.rb

@@ -0,0 +1,121 @@
+# ssl_stress_test.rb
+#
+# This script is designed to stress-test the OpenSSL SSL/TLS layer under high concurrency
+# to help detect regressions like the one described in OpenSSL issue #28171:
+# https://github.com/openssl/openssl/issues/28171
+#
+# Issue summary:
+#   - OpenSSL 3.0.17 introduced a concurrency-related regression.
+#   - Multiple threads sharing the same SSL_CTX and making parallel TLS connections
+#     (often with certificate verification enabled) can cause segmentation faults
+#     due to race conditions in X509 store handling.
+#   - Affected users include Python (httpx), Rust (reqwest, native-tls), and C applications.
+#
+# Script details:
+#   - Starts 100 SSL servers using self-signed, in-memory certs on sequential localhost ports.
+#   - Uses `rdkafka-ruby` to spin up 100 consumer threads that continuously create and destroy
+#     SSL connections to these servers for a given duration.
+#   - This mimics high TLS connection churn and aims to trigger latent SSL_CTX or X509_STORE
+#     threading bugs like double-frees, memory corruption, or segmentation faults.
+#
+# Goal:
+#   - Catch regressions early by validating that heavy concurrent SSL use does not lead to crashes.
+#   - Provide a minimal and repeatable reproducer when diagnosing OpenSSL-level SSL instability.
+#
+# In case of a failure, segfault will happen
+
+require 'rdkafka'
+require 'socket'
+require 'openssl'
+
+$stdout.sync = true
+
+STARTING_PORT = 19093
+NUM_PORTS = 150
+BATCHES = 100
+PORTS = STARTING_PORT...(STARTING_PORT + NUM_PORTS)
+
+CONFIG = {
+  'bootstrap.servers': Array.new(NUM_PORTS) { |i| "127.0.0.1:#{19093+i}" }.join(','),
+  'security.protocol': 'SSL',
+  'enable.ssl.certificate.verification': false
+}
+
+# Generate in-memory self-signed cert
+key = OpenSSL::PKey::RSA.new(2048)
+
+name = OpenSSL::X509::Name.parse("/CN=127.0.0.1")
+cert = OpenSSL::X509::Certificate.new
+cert.version = 2
+cert.serial = 1
+cert.subject = name
+cert.issuer = name
+cert.public_key = key.public_key
+cert.not_before = Time.now
+cert.not_after = Time.now + 3600
+cert.sign(key, OpenSSL::Digest::SHA256.new)
+
+# Start servers on multiple ports
+PORTS.map do |port|
+  Thread.new do
+    # Prepare SSL context
+    # We do not use a shared context for the server because the goal is to stress librdkafka layer
+    # and not the Ruby SSL layer
+    ssl_context = OpenSSL::SSL::SSLContext.new
+    ssl_context.cert = cert
+    ssl_context.key  = key
+
+    tcp_server = TCPServer.new('127.0.0.1', port)
+    ssl_server = OpenSSL::SSL::SSLServer.new(tcp_server, ssl_context)
+
+    loop do
+      begin
+        ssl_socket = ssl_server.accept
+        ssl_socket.close
+      rescue => e
+        # Some errors are expected and irrelevant
+        next if e.message.include?('unexpected eof while reading')
+
+        puts "Port #{port} SSL error: #{e}"
+      end
+    end
+  end
+end
+
+puts "Starting #{NUM_PORTS} on ports from #{PORTS.first} to #{PORTS.last} SSL servers"
+
+timeout = 30
+start = Time.now
+
+# Wait for the servers to be available
+# We want to make sure that they are available so we are sure that librdkafka actually hammers
+# them
+loop do
+  all_up = PORTS.all? do |port|
+             TCPSocket.new('127.0.0.1', port).close
+             true
+           rescue
+             false
+           end
+
+  break if all_up
+
+  raise "Timeout waiting for SSL servers" if Time.now - start > timeout
+
+  sleep 0.1
+end
+
+puts "SSL servers ready"
+
+start_time = Time.now
+duration = 60 * 10 # 10 minutes - it should crash faster than that if SSL vulnerable
+attempts = 0
+
+while Time.now - start_time < duration do
+  css = Array.new(BATCHES) { Rdkafka::Config.new(CONFIG) }
+  csss = css.map(&:consumer)
+  # This print is needed. No idea why but it increases the chances of segfault
+  p attempts += 1
+  sleep(1)
+  csss.each(&:close)
+end

data/spec/{rdkafka → lib/rdkafka}/admin_spec.rb

@@ -34,7 +34,7 @@ describe Rdkafka::Admin do
   describe '#describe_errors' do
     let(:errors) { admin.class.describe_errors }
 
-    it { expect(errors.size).to eq(170) }
+    it { expect(errors.size).to eq(172) }
     it { expect(errors[-184]).to eq(code: -184, description: 'Local: Queue full', name: '_QUEUE_FULL') }
     it { expect(errors[21]).to eq(code: 21, description: 'Broker: Invalid required acks value', name: 'INVALID_REQUIRED_ACKS') }
   end
@@ -513,22 +513,22 @@ describe Rdkafka::Admin do
     end
   end
 
-  describe "#ACL tests" do
+  describe "#ACL tests for topic resource" do
     let(:non_existing_resource_name) {"non-existing-topic"}
     before do
       #create topic for testing acl
       create_topic_handle = admin.create_topic(resource_name, topic_partition_count, topic_replication_factor)
-      create_topic_report = create_topic_handle.wait(max_wait_timeout: 15.0)
+      create_topic_handle.wait(max_wait_timeout: 15.0)
     end
 
     after do
       #delete acl
       delete_acl_handle = admin.delete_acl(resource_type: resource_type, resource_name: resource_name, resource_pattern_type: resource_pattern_type, principal: principal, host: host, operation: operation, permission_type: permission_type)
-      delete_acl_report = delete_acl_handle.wait(max_wait_timeout: 15.0)
+      delete_acl_handle.wait(max_wait_timeout: 15.0)
 
       #delete topic that was created for testing acl
       delete_topic_handle = admin.delete_topic(resource_name)
-      delete_topic_report = delete_topic_handle.wait(max_wait_timeout: 15.0)
+      delete_topic_handle.wait(max_wait_timeout: 15.0)
     end
 
     describe "#create_acl" do
@@ -615,6 +615,207 @@ describe Rdkafka::Admin do
     end
   end
 
+  describe "#ACL tests for transactional_id" do
+    let(:transactional_id_resource_name) {"test-transactional-id"}
+    let(:non_existing_transactional_id) {"non-existing-transactional-id"}
+    let(:transactional_id_resource_type) { Rdkafka::Bindings::RD_KAFKA_RESOURCE_TRANSACTIONAL_ID }
+    let(:transactional_id_resource_pattern_type) { Rdkafka::Bindings::RD_KAFKA_RESOURCE_PATTERN_LITERAL }
+    let(:transactional_id_principal) { "User:test-user" }
+    let(:transactional_id_host) { "*" }
+    let(:transactional_id_operation) { Rdkafka::Bindings::RD_KAFKA_ACL_OPERATION_WRITE }
+    let(:transactional_id_permission_type) { Rdkafka::Bindings::RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW }
+
+    after do
+      # Clean up any ACLs that might have been created during tests
+      begin
+        delete_acl_handle = admin.delete_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: nil,
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        delete_acl_handle.wait(max_wait_timeout: 15.0)
+      rescue
+        # Ignore cleanup errors
+      end
+    end
+
+    describe "#create_acl" do
+      it "creates acl for a transactional_id" do
+        create_acl_handle = admin.create_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: transactional_id_resource_name,
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        create_acl_report = create_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(create_acl_report.rdkafka_response).to eq(0)
+        expect(create_acl_report.rdkafka_response_string).to eq("")
+      end
+
+      it "creates acl for a non-existing transactional_id" do
+        # ACL creation for transactional_ids that don't exist will still get created successfully
+        create_acl_handle = admin.create_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: non_existing_transactional_id,
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        create_acl_report = create_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(create_acl_report.rdkafka_response).to eq(0)
+        expect(create_acl_report.rdkafka_response_string).to eq("")
+
+        # Clean up the ACL that was created for the non-existing transactional_id
+        delete_acl_handle = admin.delete_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: non_existing_transactional_id,
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        delete_acl_report = delete_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(delete_acl_handle[:response]).to eq(0)
+        expect(delete_acl_report.deleted_acls.size).to eq(1)
+      end
+    end
+
+    describe "#describe_acl" do
+      it "describes acl of a transactional_id that does not exist" do
+        describe_acl_handle = admin.describe_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: non_existing_transactional_id,
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        describe_acl_report = describe_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(describe_acl_handle[:response]).to eq(0)
+        expect(describe_acl_report.acls.size).to eq(0)
+      end
+
+      it "creates acls and describes the newly created transactional_id acls" do
+        # Create first ACL
+        create_acl_handle = admin.create_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: "test_transactional_id_1",
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        create_acl_report = create_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(create_acl_report.rdkafka_response).to eq(0)
+        expect(create_acl_report.rdkafka_response_string).to eq("")
+
+        # Create second ACL
+        create_acl_handle = admin.create_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: "test_transactional_id_2",
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        create_acl_report = create_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(create_acl_report.rdkafka_response).to eq(0)
+        expect(create_acl_report.rdkafka_response_string).to eq("")
+
+        # Since we create and immediately check, this is slow on loaded CIs, hence we wait
+        sleep(2)
+
+        # Describe ACLs - filter by transactional_id resource type
+        describe_acl_handle = admin.describe_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: nil,
+          resource_pattern_type: Rdkafka::Bindings::RD_KAFKA_RESOURCE_PATTERN_ANY,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        describe_acl_report = describe_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(describe_acl_handle[:response]).to eq(0)
+        expect(describe_acl_report.acls.length).to eq(2)
+      end
+    end
+
+    describe "#delete_acl" do
+      it "deletes acl of a transactional_id that does not exist" do
+        delete_acl_handle = admin.delete_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: non_existing_transactional_id,
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        delete_acl_report = delete_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(delete_acl_handle[:response]).to eq(0)
+        expect(delete_acl_report.deleted_acls.size).to eq(0)
+      end
+
+      it "creates transactional_id acls and deletes the newly created acls" do
+        # Create first ACL
+        create_acl_handle = admin.create_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: "test_transactional_id_1",
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        create_acl_report = create_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(create_acl_report.rdkafka_response).to eq(0)
+        expect(create_acl_report.rdkafka_response_string).to eq("")
+
+        # Create second ACL
+        create_acl_handle = admin.create_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: "test_transactional_id_2",
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        create_acl_report = create_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(create_acl_report.rdkafka_response).to eq(0)
+        expect(create_acl_report.rdkafka_response_string).to eq("")
+
+        # Delete ACLs - resource_name nil to delete all ACLs with any resource name and matching all other filters
+        delete_acl_handle = admin.delete_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: nil,
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        delete_acl_report = delete_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(delete_acl_handle[:response]).to eq(0)
+        expect(delete_acl_report.deleted_acls.length).to eq(2)
+      end
+    end
+  end
+
   describe('Group tests') do
     describe "#delete_group" do
       describe("with an existing group") do
@@ -675,7 +876,17 @@ describe Rdkafka::Admin do
   end
 
   describe '#create_partitions' do
-    let(:metadata) { admin.metadata(topic_name).topics.first }
+    let(:metadata) do
+      begin
+        admin.metadata(topic_name).topics.first
+      rescue Rdkafka::RdkafkaError
+        # We have to wait because if we query too fast after topic creation request, it may not
+        # yet be available throwing an error.
+        # This occurs mostly on slow CIs
+        sleep(1)
+        admin.metadata(topic_name).topics.first
+      end
+    end
 
     context 'when topic does not exist' do
       it 'expect to fail due to unknown partition' do
@@ -697,6 +908,8 @@ describe Rdkafka::Admin do
 
       it 'expect not to change number of partitions' do
         expect { admin.create_partitions(topic_name, 2).wait }.to raise_error(Rdkafka::RdkafkaError, /invalid_partitions/)
+        # On slow CI this may propagate, thus we wait a bit
+        sleep(1)
         expect(metadata[:partition_count]).to eq(5)
       end
     end

data/spec/{rdkafka → lib/rdkafka}/bindings_spec.rb

@@ -77,30 +77,6 @@ describe Rdkafka::Bindings do
     end
   end
 
-  describe "partitioner" do
-    let(:partition_key) { ('a'..'z').to_a.shuffle.take(15).join('') }
-    let(:partition_count) { rand(50) + 1 }
-
-    it "should return the same partition for a similar string and the same partition count" do
-      result_1 = Rdkafka::Bindings.partitioner(partition_key, partition_count)
-      result_2 = Rdkafka::Bindings.partitioner(partition_key, partition_count)
-      expect(result_1).to eq(result_2)
-    end
-
-    it "should match the old partitioner" do
-      result_1 = Rdkafka::Bindings.partitioner(partition_key, partition_count)
-      result_2 = (Zlib.crc32(partition_key) % partition_count)
-      expect(result_1).to eq(result_2)
-    end
-
-    it "should return the partition calculated by the specified partitioner" do
-      result_1 = Rdkafka::Bindings.partitioner(partition_key, partition_count, "murmur2")
-      ptr = FFI::MemoryPointer.from_string(partition_key)
-      result_2 = Rdkafka::Bindings.rd_kafka_msg_partitioner_murmur2(nil, ptr, partition_key.size, partition_count, nil, nil)
-      expect(result_1).to eq(result_2)
-    end
-  end
-
   describe "stats callback" do
     context "without a stats callback" do
       it "should do nothing" do

data/spec/{rdkafka → lib/rdkafka}/config_spec.rb

@@ -159,7 +159,7 @@ describe Rdkafka::Config do
 
     it "should use default configuration" do
       config = Rdkafka::Config.new
-      expect(config[:"api.version.request"]).to eq true
+      expect(config[:"api.version.request"]).to eq nil
     end
 
     it "should create a consumer with valid config" do

data/spec/{rdkafka → lib/rdkafka}/consumer_spec.rb

@@ -175,6 +175,7 @@ describe Rdkafka::Consumer do
     before do
       admin = rdkafka_producer_config.admin
       admin.create_topic(topic, 1, 1).wait
+      wait_for_topic(admin, topic)
       admin.close
     end
 
@@ -278,6 +279,7 @@ describe Rdkafka::Consumer do
     before do
       admin = rdkafka_producer_config.admin
       admin.create_topic(topic, 1, 1).wait
+      wait_for_topic(admin, topic)
       admin.close
     end
 
@@ -426,7 +428,7 @@ describe Rdkafka::Consumer do
   describe '#assignment_lost?' do
     it "should not return true as we do have an assignment" do
       consumer.subscribe("consume_test_topic")
-      expected_subscription = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+      Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
         list.add_topic("consume_test_topic")
       end
 
@@ -436,7 +438,7 @@ describe Rdkafka::Consumer do
 
     it "should not return true after voluntary unsubscribing" do
       consumer.subscribe("consume_test_topic")
-      expected_subscription = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+      Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
         list.add_topic("consume_test_topic")
       end
 
@@ -1276,7 +1278,6 @@ describe Rdkafka::Consumer do
 
       # Consume to the end
       consumer.subscribe("consume_test_topic")
-      eof_count = 0
       eof_error = nil
 
       loop do
@@ -1291,9 +1292,52 @@ describe Rdkafka::Consumer do
       end
 
       expect(eof_error.code).to eq(:partition_eof)
-      expect(eof_error.details[:topic]).to eq('consume_test_topic')
-      expect(eof_error.details[:partition]).to be_a(Integer)
-      expect(eof_error.details[:offset]).to be_a(Integer)
+    end
+  end
+
+  describe "long running consumption" do
+    let(:consumer) { rdkafka_consumer_config.consumer }
+    let(:producer) { rdkafka_producer_config.producer }
+
+    after { consumer.close }
+    after { producer.close }
+
+    it "should consume messages continuously for 60 seconds" do
+      consumer.subscribe("consume_test_topic")
+      wait_for_assignment(consumer)
+
+      messages_consumed = 0
+      start_time = Time.now
+
+      # Producer thread - sends message every second
+      producer_thread = Thread.new do
+        counter = 0
+        while Time.now - start_time < 60
+          producer.produce(
+            topic: "consume_test_topic",
+            payload: "payload #{counter}",
+            key: "key #{counter}",
+            partition: 0
+          ).wait
+          counter += 1
+          sleep(1)
+        end
+      end
+
+      # Consumer loop
+      while Time.now - start_time < 60
+        message = consumer.poll(1000)
+        if message
+          expect(message).to be_a Rdkafka::Consumer::Message
+          expect(message.topic).to eq("consume_test_topic")
+          messages_consumed += 1
+          consumer.commit if messages_consumed % 10 == 0
+        end
+      end
+
+      producer_thread.join
+
+      expect(messages_consumed).to be > 50 # Should consume most messages
     end
   end
 end

data/spec/{rdkafka → lib/rdkafka}/metadata_spec.rb

@@ -31,7 +31,7 @@ describe Rdkafka::Metadata do
         expect(subject.brokers.length).to eq(1)
         expect(subject.brokers[0][:broker_id]).to eq(1)
         expect(%w[127.0.0.1 localhost]).to include(subject.brokers[0][:broker_name])
-        expect(subject.brokers[0][:broker_port]).to eq(9092)
+        expect(subject.brokers[0][:broker_port]).to eq(rdkafka_base_config[:'bootstrap.servers'].split(':').last.to_i)
       end
 
       it "#topics returns data on our test topic" do
@@ -54,7 +54,7 @@ describe Rdkafka::Metadata do
       expect(subject.brokers.length).to eq(1)
       expect(subject.brokers[0][:broker_id]).to eq(1)
       expect(%w[127.0.0.1 localhost]).to include(subject.brokers[0][:broker_name])
-      expect(subject.brokers[0][:broker_port]).to eq(9092)
+      expect(subject.brokers[0][:broker_port]).to eq(rdkafka_base_config[:'bootstrap.servers'].split(':').last.to_i)
     end
 
     it "#topics returns data about all of our test topics" do

data/spec/{rdkafka → lib/rdkafka}/producer/delivery_report_spec.rb

@@ -20,6 +20,6 @@ describe Rdkafka::Producer::DeliveryReport do
   end
 
   it "should get the error" do
-    expect(subject.error).to eq -1
+    expect(subject.error).to eq(-1)
   end
 end