karafka-rdkafka 0.21.0-aarch64-linux-gnu

data/spec/integrations/ssl_stress_spec.rb

@@ -0,0 +1,121 @@
+# ssl_stress_test.rb
+#
+# This script is designed to stress-test the OpenSSL SSL/TLS layer under high concurrency
+# to help detect regressions like the one described in OpenSSL issue #28171:
+# https://github.com/openssl/openssl/issues/28171
+#
+# Issue summary:
+#   - OpenSSL 3.0.17 introduced a concurrency-related regression.
+#   - Multiple threads sharing the same SSL_CTX and making parallel TLS connections
+#     (often with certificate verification enabled) can cause segmentation faults
+#     due to race conditions in X509 store handling.
+#   - Affected users include Python (httpx), Rust (reqwest, native-tls), and C applications.
+#
+# Script details:
+#   - Starts 100 SSL servers using self-signed, in-memory certs on sequential localhost ports.
+#   - Uses `rdkafka-ruby` to spin up 100 consumer threads that continuously create and destroy
+#     SSL connections to these servers for a given duration.
+#   - This mimics high TLS connection churn and aims to trigger latent SSL_CTX or X509_STORE
+#     threading bugs like double-frees, memory corruption, or segmentation faults.
+#
+# Goal:
+#   - Catch regressions early by validating that heavy concurrent SSL use does not lead to crashes.
+#   - Provide a minimal and repeatable reproducer when diagnosing OpenSSL-level SSL instability.
+#
+# In case of a failure, segfault will happen
+
+require 'rdkafka'
+require 'socket'
+require 'openssl'
+
+$stdout.sync = true
+
+STARTING_PORT = 19093
+NUM_PORTS = 150
+BATCHES = 100
+PORTS = STARTING_PORT...(STARTING_PORT + NUM_PORTS)
+
+CONFIG = {
+  'bootstrap.servers': Array.new(NUM_PORTS) { |i| "127.0.0.1:#{19093+i}" }.join(','),
+  'security.protocol': 'SSL',
+  'enable.ssl.certificate.verification': false
+}
+
+# Generate in-memory self-signed cert
+key = OpenSSL::PKey::RSA.new(2048)
+
+name = OpenSSL::X509::Name.parse("/CN=127.0.0.1")
+cert = OpenSSL::X509::Certificate.new
+cert.version = 2
+cert.serial = 1
+cert.subject = name
+cert.issuer = name
+cert.public_key = key.public_key
+cert.not_before = Time.now
+cert.not_after = Time.now + 3600
+cert.sign(key, OpenSSL::Digest::SHA256.new)
+
+# Start servers on multiple ports
+PORTS.map do |port|
+  Thread.new do
+    # Prepare SSL context
+    # We do not use a shared context for the server because the goal is to stress librdkafka layer
+    # and not the Ruby SSL layer
+    ssl_context = OpenSSL::SSL::SSLContext.new
+    ssl_context.cert = cert
+    ssl_context.key  = key
+
+    tcp_server = TCPServer.new('127.0.0.1', port)
+    ssl_server = OpenSSL::SSL::SSLServer.new(tcp_server, ssl_context)
+
+    loop do
+      begin
+        ssl_socket = ssl_server.accept
+        ssl_socket.close
+      rescue => e
+        # Some errors are expected and irrelevant
+        next if e.message.include?('unexpected eof while reading')
+
+        puts "Port #{port} SSL error: #{e}"
+      end
+    end
+  end
+end
+
+puts "Starting #{NUM_PORTS} on ports from #{PORTS.first} to #{PORTS.last} SSL servers"
+
+timeout = 30
+start = Time.now
+
+# Wait for the servers to be available
+# We want to make sure that they are available so we are sure that librdkafka actually hammers
+# them
+loop do
+  all_up = PORTS.all? do |port|
+             TCPSocket.new('127.0.0.1', port).close
+             true
+           rescue
+             false
+           end
+
+  break if all_up
+
+  raise "Timeout waiting for SSL servers" if Time.now - start > timeout
+
+  sleep 0.1
+end
+
+puts "SSL servers ready"
+
+start_time = Time.now
+duration = 60 * 10 # 10 minutes - it should crash faster than that if SSL vulnerable
+attempts = 0
+
+while Time.now - start_time < duration do
+  css = Array.new(BATCHES) { Rdkafka::Config.new(CONFIG) }
+  csss = css.map(&:consumer)
+  # This print is needed. No idea why but it increases the chances of segfault
+  p attempts += 1
+  sleep(1)
+  csss.each(&:close)
+end

data/spec/lib/rdkafka/abstract_handle_spec.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+describe Rdkafka::AbstractHandle do
+  let(:response) { 0 }
+  let(:result) { -1 }
+
+  context "A subclass that does not implement the required methods" do
+
+    class BadTestHandle < Rdkafka::AbstractHandle
+      layout :pending, :bool,
+             :response, :int
+    end
+
+    it "raises an exception if operation_name is called" do
+      expect {
+        BadTestHandle.new.operation_name
+      }.to raise_exception(RuntimeError, /Must be implemented by subclass!/)
+    end
+
+    it "raises an exception if create_result is called" do
+      expect {
+        BadTestHandle.new.create_result
+      }.to raise_exception(RuntimeError, /Must be implemented by subclass!/)
+    end
+  end
+
+  class TestHandle < Rdkafka::AbstractHandle
+    layout :pending, :bool,
+           :response, :int,
+           :result, :int
+
+    def operation_name
+      "test_operation"
+    end
+
+    def create_result
+      self[:result]
+    end
+  end
+
+  subject do
+    TestHandle.new.tap do |handle|
+      handle[:pending] = pending_handle
+      handle[:response] = response
+      handle[:result] = result
+    end
+  end
+
+  describe ".register and .remove" do
+    let(:pending_handle) { true }
+
+    it "should register and remove a delivery handle" do
+      Rdkafka::AbstractHandle.register(subject)
+      removed = Rdkafka::AbstractHandle.remove(subject.to_ptr.address)
+      expect(removed).to eq subject
+      expect(Rdkafka::AbstractHandle::REGISTRY).to be_empty
+    end
+  end
+
+  describe "#pending?" do
+    context "when true" do
+      let(:pending_handle) { true }
+
+      it "should be true" do
+        expect(subject.pending?).to be true
+      end
+    end
+
+    context "when not true" do
+      let(:pending_handle) { false }
+
+      it "should be false" do
+        expect(subject.pending?).to be false
+      end
+    end
+  end
+
+  describe "#wait" do
+    context 'when pending_handle true' do
+      let(:pending_handle) { true }
+
+      it "should wait until the timeout and then raise an error" do
+        expect {
+          subject.wait(max_wait_timeout: 0.1)
+        }.to raise_error Rdkafka::AbstractHandle::WaitTimeoutError, /test_operation/
+      end
+    end
+
+    context 'when pending_handle false' do
+      let(:pending_handle) { false }
+
+      context "without error" do
+        let(:result) { 1 }
+
+        it "should return a result" do
+          wait_result = subject.wait
+          expect(wait_result).to eq(result)
+        end
+
+        it "should wait without a timeout" do
+          wait_result = subject.wait(max_wait_timeout: nil)
+          expect(wait_result).to eq(result)
+        end
+      end
+
+      context "with error" do
+        let(:response) { 20 }
+
+        it "should raise an rdkafka error" do
+          expect {
+            subject.wait
+          }.to raise_error Rdkafka::RdkafkaError
+        end
+      end
+    end
+  end
+end

data/spec/lib/rdkafka/admin/create_acl_handle_spec.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+
+describe Rdkafka::Admin::CreateAclHandle do
+  # If create acl was successful there is no error object
+  # the error code is set to RD_KAFKA_RESP_ERR_NO_ERRORa
+  # https://github.com/confluentinc/librdkafka/blob/1f9f245ac409f50f724695c628c7a0d54a763b9a/src/rdkafka_error.c#L169
+  let(:response) { Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR }
+
+  subject do
+    Rdkafka::Admin::CreateAclHandle.new.tap do |handle|
+      handle[:pending] = pending_handle
+      handle[:response] = response
+      # If create acl was successful there is no error object and the error_string is set to ""
+      # https://github.com/confluentinc/librdkafka/blob/1f9f245ac409f50f724695c628c7a0d54a763b9a/src/rdkafka_error.c#L178
+      handle[:response_string] = FFI::MemoryPointer.from_string("")
+    end
+  end
+
+  describe "#wait" do
+    let(:pending_handle) { true }
+
+    it "should wait until the timeout and then raise an error" do
+      expect {
+        subject.wait(max_wait_timeout: 0.1)
+      }.to raise_error Rdkafka::Admin::CreateAclHandle::WaitTimeoutError, /create acl/
+    end
+
+    context "when not pending anymore and no error" do
+      let(:pending_handle) { false }
+
+      it "should return a create acl report" do
+        report = subject.wait
+
+        expect(report.rdkafka_response_string).to eq("")
+      end
+
+      it "should wait without a timeout" do
+        report = subject.wait(max_wait_timeout: nil)
+
+        expect(report.rdkafka_response_string).to eq("")
+      end
+    end
+  end
+
+  describe "#raise_error" do
+    let(:pending_handle) { false }
+
+    it "should raise the appropriate error" do
+      expect {
+        subject.raise_error
+      }.to raise_exception(Rdkafka::RdkafkaError, /Success \(no_error\)/)
+    end
+  end
+end

data/spec/lib/rdkafka/admin/create_acl_report_spec.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+
+describe Rdkafka::Admin::CreateAclReport do
+  subject { Rdkafka::Admin::CreateAclReport.new(
+      rdkafka_response: Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR,
+      rdkafka_response_string: FFI::MemoryPointer.from_string("")
+  )}
+
+  it "should get RD_KAFKA_RESP_ERR_NO_ERROR " do
+    expect(subject.rdkafka_response).to eq(0)
+  end
+
+  it "should get empty string" do
+    expect(subject.rdkafka_response_string).to eq("")
+  end
+end

data/spec/lib/rdkafka/admin/create_topic_handle_spec.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+describe Rdkafka::Admin::CreateTopicHandle do
+  let(:response) { 0 }
+
+  subject do
+    Rdkafka::Admin::CreateTopicHandle.new.tap do |handle|
+      handle[:pending] = pending_handle
+      handle[:response] = response
+      handle[:error_string] = FFI::Pointer::NULL
+      handle[:result_name] = FFI::MemoryPointer.from_string("my-test-topic")
+    end
+  end
+
+  describe "#wait" do
+    let(:pending_handle) { true }
+
+    it "should wait until the timeout and then raise an error" do
+      expect {
+        subject.wait(max_wait_timeout: 0.1)
+      }.to raise_error Rdkafka::Admin::CreateTopicHandle::WaitTimeoutError, /create topic/
+    end
+
+    context "when not pending anymore and no error" do
+      let(:pending_handle) { false }
+
+      it "should return a create topic report" do
+        report = subject.wait
+
+        expect(report.error_string).to eq(nil)
+        expect(report.result_name).to eq("my-test-topic")
+      end
+
+      it "should wait without a timeout" do
+        report = subject.wait(max_wait_timeout: nil)
+
+        expect(report.error_string).to eq(nil)
+        expect(report.result_name).to eq("my-test-topic")
+      end
+    end
+  end
+
+  describe "#raise_error" do
+    let(:pending_handle) { false }
+
+    before { subject[:response] = -1 }
+
+    it "should raise the appropriate error when there is an error" do
+      expect {
+        subject.raise_error
+      }.to raise_exception(Rdkafka::RdkafkaError, /Unknown broker error \(unknown\)/)
+    end
+  end
+end

data/spec/lib/rdkafka/admin/create_topic_report_spec.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+describe Rdkafka::Admin::CreateTopicReport do
+  subject { Rdkafka::Admin::CreateTopicReport.new(
+      FFI::MemoryPointer.from_string("error string"),
+      FFI::MemoryPointer.from_string("result name")
+  )}
+
+  it "should get the error string" do
+    expect(subject.error_string).to eq("error string")
+  end
+
+  it "should get the result name" do
+    expect(subject.result_name).to eq("result name")
+  end
+end

data/spec/lib/rdkafka/admin/delete_acl_handle_spec.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+
+describe Rdkafka::Admin::DeleteAclHandle do
+  let(:response) { Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR }
+  let(:resource_name)         {"acl-test-topic"}
+  let(:resource_type)         {Rdkafka::Bindings::RD_KAFKA_RESOURCE_TOPIC}
+  let(:resource_pattern_type) {Rdkafka::Bindings::RD_KAFKA_RESOURCE_PATTERN_LITERAL}
+  let(:principal)             {"User:anonymous"}
+  let(:host)                  {"*"}
+  let(:operation)             {Rdkafka::Bindings::RD_KAFKA_ACL_OPERATION_READ}
+  let(:permission_type)       {Rdkafka::Bindings::RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW}
+  let(:delete_acl_ptr)        {FFI::Pointer::NULL}
+
+  subject do
+    error_buffer = FFI::MemoryPointer.from_string(" " * 256)
+    delete_acl_ptr = Rdkafka::Bindings.rd_kafka_AclBinding_new(
+      resource_type,
+      FFI::MemoryPointer.from_string(resource_name),
+      resource_pattern_type,
+      FFI::MemoryPointer.from_string(principal),
+      FFI::MemoryPointer.from_string(host),
+      operation,
+      permission_type,
+      error_buffer,
+      256
+    )
+    if delete_acl_ptr.null?
+      raise Rdkafka::Config::ConfigError.new(error_buffer.read_string)
+    end
+    pointer_array = [delete_acl_ptr]
+    delete_acls_array_ptr = FFI::MemoryPointer.new(:pointer)
+    delete_acls_array_ptr.write_array_of_pointer(pointer_array)
+    Rdkafka::Admin::DeleteAclHandle.new.tap do |handle|
+      handle[:pending] = pending_handle
+      handle[:response] = response
+      handle[:response_string] = FFI::MemoryPointer.from_string("")
+      handle[:matching_acls] = delete_acls_array_ptr
+      handle[:matching_acls_count] = 1
+    end
+  end
+
+  after do
+    if delete_acl_ptr != FFI::Pointer::NULL
+      Rdkafka::Bindings.rd_kafka_AclBinding_destroy(delete_acl_ptr)
+    end
+  end
+
+  describe "#wait" do
+    let(:pending_handle) { true }
+
+    it "should wait until the timeout and then raise an error" do
+      expect {
+        subject.wait(max_wait_timeout: 0.1)
+      }.to raise_error Rdkafka::Admin::DeleteAclHandle::WaitTimeoutError, /delete acl/
+    end
+
+    context "when not pending anymore and no error" do
+      let(:pending_handle) { false }
+
+      it "should return a delete acl report" do
+        report = subject.wait
+
+        expect(report.deleted_acls.length).to eq(1)
+      end
+
+      it "should wait without a timeout" do
+        report = subject.wait(max_wait_timeout: nil)
+
+        expect(report.deleted_acls[0].matching_acl_resource_name).to eq(resource_name)
+      end
+    end
+  end
+
+  describe "#raise_error" do
+    let(:pending_handle) { false }
+
+    it "should raise the appropriate error" do
+      expect {
+        subject.raise_error
+      }.to raise_exception(Rdkafka::RdkafkaError, /Success \(no_error\)/)
+    end
+  end
+end

data/spec/lib/rdkafka/admin/delete_acl_report_spec.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+
+describe Rdkafka::Admin::DeleteAclReport do
+
+  let(:resource_name)         {"acl-test-topic"}
+  let(:resource_type)         {Rdkafka::Bindings::RD_KAFKA_RESOURCE_TOPIC}
+  let(:resource_pattern_type) {Rdkafka::Bindings::RD_KAFKA_RESOURCE_PATTERN_LITERAL}
+  let(:principal)             {"User:anonymous"}
+  let(:host)                  {"*"}
+  let(:operation)             {Rdkafka::Bindings::RD_KAFKA_ACL_OPERATION_READ}
+  let(:permission_type)       {Rdkafka::Bindings::RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW}
+  let(:delete_acl_ptr)        {FFI::Pointer::NULL}
+
+  subject do
+    error_buffer = FFI::MemoryPointer.from_string(" " * 256)
+    delete_acl_ptr = Rdkafka::Bindings.rd_kafka_AclBinding_new(
+      resource_type,
+      FFI::MemoryPointer.from_string(resource_name),
+      resource_pattern_type,
+      FFI::MemoryPointer.from_string(principal),
+      FFI::MemoryPointer.from_string(host),
+      operation,
+      permission_type,
+      error_buffer,
+      256
+    )
+    if delete_acl_ptr.null?
+      raise Rdkafka::Config::ConfigError.new(error_buffer.read_string)
+    end
+    pointer_array = [delete_acl_ptr]
+    delete_acls_array_ptr = FFI::MemoryPointer.new(:pointer)
+    delete_acls_array_ptr.write_array_of_pointer(pointer_array)
+    Rdkafka::Admin::DeleteAclReport.new(matching_acls: delete_acls_array_ptr, matching_acls_count: 1)
+  end
+
+  after do
+    if delete_acl_ptr != FFI::Pointer::NULL
+      Rdkafka::Bindings.rd_kafka_AclBinding_destroy(delete_acl_ptr)
+    end
+  end
+
+  it "should get deleted acl resource type as Rdkafka::Bindings::RD_KAFKA_RESOURCE_TOPIC" do
+    expect(subject.deleted_acls[0].matching_acl_resource_type).to eq(Rdkafka::Bindings::RD_KAFKA_RESOURCE_TOPIC)
+  end
+
+  it "should get deleted acl resource name as acl-test-topic" do
+    expect(subject.deleted_acls[0].matching_acl_resource_name).to eq(resource_name)
+  end
+
+  it "should get deleted acl resource pattern type as Rdkafka::Bindings::RD_KAFKA_RESOURCE_PATTERN_LITERAL" do
+    expect(subject.deleted_acls[0].matching_acl_resource_pattern_type).to eq(Rdkafka::Bindings::RD_KAFKA_RESOURCE_PATTERN_LITERAL)
+    expect(subject.deleted_acls[0].matching_acl_pattern_type).to eq(Rdkafka::Bindings::RD_KAFKA_RESOURCE_PATTERN_LITERAL)
+  end
+
+  it "should get deleted acl principal as User:anonymous" do
+    expect(subject.deleted_acls[0].matching_acl_principal).to eq("User:anonymous")
+  end
+
+  it "should get deleted  acl host as * " do
+    expect(subject.deleted_acls[0].matching_acl_host).to eq("*")
+  end
+
+  it "should get deleted acl operation as Rdkafka::Bindings::RD_KAFKA_ACL_OPERATION_READ" do
+    expect(subject.deleted_acls[0].matching_acl_operation).to eq(Rdkafka::Bindings::RD_KAFKA_ACL_OPERATION_READ)
+  end
+
+  it "should get deleted acl permission_type as Rdkafka::Bindings::RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW" do
+    expect(subject.deleted_acls[0].matching_acl_permission_type).to eq(Rdkafka::Bindings::RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW)
+  end
+end

data/spec/lib/rdkafka/admin/delete_topic_handle_spec.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+describe Rdkafka::Admin::DeleteTopicHandle do
+  let(:response) { 0 }
+
+  subject do
+    Rdkafka::Admin::DeleteTopicHandle.new.tap do |handle|
+      handle[:pending] = pending_handle
+      handle[:response] = response
+      handle[:error_string] = FFI::Pointer::NULL
+      handle[:result_name] = FFI::MemoryPointer.from_string("my-test-topic")
+    end
+  end
+
+  describe "#wait" do
+    let(:pending_handle) { true }
+
+    it "should wait until the timeout and then raise an error" do
+      expect {
+        subject.wait(max_wait_timeout: 0.1)
+      }.to raise_error Rdkafka::Admin::DeleteTopicHandle::WaitTimeoutError, /delete topic/
+    end
+
+    context "when not pending anymore and no error" do
+      let(:pending_handle) { false }
+
+      it "should return a delete topic report" do
+        report = subject.wait
+
+        expect(report.error_string).to eq(nil)
+        expect(report.result_name).to eq("my-test-topic")
+      end
+
+      it "should wait without a timeout" do
+        report = subject.wait(max_wait_timeout: nil)
+
+        expect(report.error_string).to eq(nil)
+        expect(report.result_name).to eq("my-test-topic")
+      end
+    end
+  end
+
+  describe "#raise_error" do
+    let(:pending_handle) { false }
+
+    before { subject[:response] = -1 }
+
+    it "should raise the appropriate error" do
+      expect {
+        subject.raise_error
+      }.to raise_exception(Rdkafka::RdkafkaError, /Unknown broker error \(unknown\)/)
+    end
+  end
+end

data/spec/lib/rdkafka/admin/delete_topic_report_spec.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+describe Rdkafka::Admin::DeleteTopicReport do
+  subject { Rdkafka::Admin::DeleteTopicReport.new(
+      FFI::MemoryPointer.from_string("error string"),
+      FFI::MemoryPointer.from_string("result name")
+  )}
+
+  it "should get the error string" do
+    expect(subject.error_string).to eq("error string")
+  end
+
+  it "should get the result name" do
+    expect(subject.result_name).to eq("result name")
+  end
+end