karafka-rdkafka 0.20.0.rc5-arm64-darwin → 0.21.0-arm64-darwin

data/spec/{rdkafka → lib/rdkafka}/producer_spec.rb

@@ -263,8 +263,6 @@ describe Rdkafka::Producer do
     expect(message.partition).to eq 1
     expect(message.payload).to eq "payload"
     expect(message.key).to eq "key"
-    # Since api.version.request is on by default we will get
-    # the message creation timestamp if it's not set.
     expect(message.timestamp).to be_within(10).of(Time.now)
   end
 
@@ -340,7 +338,7 @@ describe Rdkafka::Producer do
       )
     end
 
-    expect(messages[0].partition).to eq 0
+    expect(messages[0].partition).to be >= 0
     expect(messages[0].key).to eq 'a'
   end
 
@@ -665,7 +663,7 @@ describe Rdkafka::Producer do
   context "when not being able to deliver the message" do
     let(:producer) do
       rdkafka_producer_config(
-        "bootstrap.servers": "127.0.0.1:9093",
+        "bootstrap.servers": "127.0.0.1:9095",
         "message.timeout.ms": 100
       ).producer
     end
@@ -781,14 +779,15 @@ describe Rdkafka::Producer do
     context 'when it cannot flush due to a timeout' do
       let(:producer) do
         rdkafka_producer_config(
-          "bootstrap.servers": "127.0.0.1:9093",
+          "bootstrap.servers": "127.0.0.1:9095",
           "message.timeout.ms": 2_000
         ).producer
       end
 
       after do
         # Allow rdkafka to evict message preventing memory-leak
-        sleep(2)
+        # We give it a bit more time as on slow CIs things take time
+        sleep(5)
       end
 
       it "should return false on flush when cannot deliver and beyond timeout" do
@@ -828,7 +827,7 @@ describe Rdkafka::Producer do
     context 'when there are outgoing things in the queue' do
       let(:producer) do
         rdkafka_producer_config(
-          "bootstrap.servers": "127.0.0.1:9093",
+          "bootstrap.servers": "127.0.0.1:9095",
           "message.timeout.ms": 2_000
         ).producer
       end
@@ -864,7 +863,7 @@ describe Rdkafka::Producer do
         before { producer.delivery_callback = delivery_callback }
 
         it "should run the callback" do
-          handle = producer.produce(
+          producer.produce(
             topic:     "produce_test_topic",
             payload:   "payload headers"
           )
@@ -1231,4 +1230,298 @@ describe Rdkafka::Producer do
       end
     end
   end
+
+  let(:producer) { rdkafka_producer_config.producer }
+  let(:all_partitioners) { %w(random consistent consistent_random murmur2 murmur2_random fnv1a fnv1a_random) }
+
+  describe "partitioner behavior through producer API" do
+    context "testing all partitioners with same key" do
+      it "should not return partition 0 for all partitioners" do
+        test_key = "test-key-123"
+        results = {}
+
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "test payload",
+            partition_key: test_key,
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          results[partitioner] = report.partition
+        end
+
+        # Should not all be the same partition (especially not all 0)
+        unique_partitions = results.values.uniq
+        expect(unique_partitions.size).to be > 1
+      end
+    end
+
+    context "empty string partition key" do
+      it "should produce message with empty partition key without crashing and go to partition 0 for all partitioners" do
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "test payload",
+            key: "test-key",
+            partition_key: "",
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          expect(report.partition).to be >= 0
+        end
+      end
+    end
+
+    context "nil partition key" do
+      it "should handle nil partition key gracefully" do
+        handle = producer.produce(
+          topic: "partitioner_test_topic",
+          payload: "test payload",
+          key: "test-key",
+          partition_key: nil
+        )
+
+        report = handle.wait(max_wait_timeout: 5)
+        expect(report.partition).to be >= 0
+        expect(report.partition).to be < producer.partition_count("partitioner_test_topic")
+      end
+    end
+
+    context "various key types and lengths with different partitioners" do
+      it "should handle very short keys with all partitioners" do
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "test payload",
+            partition_key: "a",
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          expect(report.partition).to be >= 0
+          expect(report.partition).to be < producer.partition_count("partitioner_test_topic")
+        end
+      end
+
+      it "should handle very long keys with all partitioners" do
+        long_key = "a" * 1000
+
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "test payload",
+            partition_key: long_key,
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          expect(report.partition).to be >= 0
+          expect(report.partition).to be < producer.partition_count("partitioner_test_topic")
+        end
+      end
+
+      it "should handle unicode keys with all partitioners" do
+        unicode_key = "测试键值🚀"
+
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "test payload",
+            partition_key: unicode_key,
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          expect(report.partition).to be >= 0
+          expect(report.partition).to be < producer.partition_count("partitioner_test_topic")
+        end
+      end
+    end
+
+    context "consistency testing for deterministic partitioners" do
+      %w(consistent murmur2 fnv1a).each do |partitioner|
+        it "should consistently route same partition key to same partition with #{partitioner}" do
+          partition_key = "consistent-test-key"
+
+          # Produce multiple messages with same partition key
+          reports = 5.times.map do
+            handle = producer.produce(
+              topic: "partitioner_test_topic",
+              payload: "test payload #{Time.now.to_f}",
+              partition_key: partition_key,
+              partitioner: partitioner
+            )
+            handle.wait(max_wait_timeout: 5)
+          end
+
+          # All should go to same partition
+          partitions = reports.map(&:partition).uniq
+          expect(partitions.size).to eq(1)
+        end
+      end
+    end
+
+    context "randomness testing for random partitioners" do
+      %w(random consistent_random murmur2_random fnv1a_random).each do |partitioner|
+        it "should potentially distribute across partitions with #{partitioner}" do
+          # Note: random partitioners might still return same value by chance
+          partition_key = "random-test-key"
+
+          reports = 10.times.map do
+            handle = producer.produce(
+              topic: "partitioner_test_topic",
+              payload: "test payload #{Time.now.to_f}",
+              partition_key: partition_key,
+              partitioner: partitioner
+            )
+            handle.wait(max_wait_timeout: 5)
+          end
+
+          partitions = reports.map(&:partition)
+
+          # Just ensure they're valid partitions
+          partitions.each do |partition|
+            expect(partition).to be >= 0
+            expect(partition).to be < producer.partition_count("partitioner_test_topic")
+          end
+        end
+      end
+    end
+
+    context "comparing different partitioners with same key" do
+      it "should route different partition keys to potentially different partitions" do
+        keys = ["key1", "key2", "key3", "key4", "key5"]
+
+        all_partitioners.each do |partitioner|
+          reports = keys.map do |key|
+            handle = producer.produce(
+              topic: "partitioner_test_topic",
+              payload: "test payload",
+              partition_key: key,
+              partitioner: partitioner
+            )
+            handle.wait(max_wait_timeout: 5)
+          end
+
+          partitions = reports.map(&:partition).uniq
+
+          # Should distribute across multiple partitions for most partitioners
+          # (though some might hash all keys to same partition by chance)
+          expect(partitions.all? { |p| p >= 0 && p < producer.partition_count("partitioner_test_topic") }).to be true
+        end
+      end
+    end
+
+    context "partition key vs regular key behavior" do
+      it "should use partition key for partitioning when both key and partition_key are provided" do
+        # Use keys that would hash to different partitions
+        regular_key = "regular-key-123"
+        partition_key = "partition-key-456"
+
+        # Message with both keys
+        handle1 = producer.produce(
+          topic: "partitioner_test_topic",
+          payload: "test payload 1",
+          key: regular_key,
+          partition_key: partition_key
+        )
+
+        # Message with only partition key (should go to same partition)
+        handle2 = producer.produce(
+          topic: "partitioner_test_topic",
+          payload: "test payload 2",
+          partition_key: partition_key
+        )
+
+        # Message with only regular key (should go to different partition)
+        handle3 = producer.produce(
+          topic: "partitioner_test_topic",
+          payload: "test payload 3",
+          key: regular_key
+        )
+
+        report1 = handle1.wait(max_wait_timeout: 5)
+        report2 = handle2.wait(max_wait_timeout: 5)
+        report3 = handle3.wait(max_wait_timeout: 5)
+
+        # Messages 1 and 2 should go to same partition (both use partition_key)
+        expect(report1.partition).to eq(report2.partition)
+
+        # Message 3 should potentially go to different partition (uses regular key)
+        expect(report3.partition).not_to eq(report1.partition)
+      end
+    end
+
+    context "edge case combinations with different partitioners" do
+      it "should handle nil partition key with all partitioners" do
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "test payload",
+            key: "test-key",
+            partition_key: nil,
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          expect(report.partition).to be >= 0
+          expect(report.partition).to be < producer.partition_count("partitioner_test_topic")
+        end
+      end
+
+      it "should handle whitespace-only partition key with all partitioners" do
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "test payload",
+            partition_key: "   ",
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          expect(report.partition).to be >= 0
+          expect(report.partition).to be < producer.partition_count("partitioner_test_topic")
+        end
+      end
+
+      it "should handle newline characters in partition key with all partitioners" do
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "test payload",
+            partition_key: "key\nwith\nnewlines",
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          expect(report.partition).to be >= 0
+          expect(report.partition).to be < producer.partition_count("partitioner_test_topic")
+        end
+      end
+    end
+
+    context "debugging partitioner issues" do
+      it "should show if all partitioners return 0 (indicating a problem)" do
+        test_key = "debug-test-key"
+        zero_count = 0
+
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "debug payload",
+            partition_key: test_key,
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          zero_count += 1 if report.partition == 0
+        end
+
+        expect(zero_count).to be < all_partitioners.size
+      end
+    end
+  end
 end

data/spec/spec_helper.rb

@@ -1,5 +1,19 @@
 # frozen_string_literal: true
 
+Warning[:performance] = true if RUBY_VERSION >= '3.3'
+Warning[:deprecated] = true
+$VERBOSE = true
+
+require 'warning'
+
+Warning.process do |warning|
+  next unless warning.include?(Dir.pwd)
+  # Allow OpenStruct usage only in specs
+  next if warning.include?('OpenStruct use') && warning.include?('_spec')
+
+  raise "Warning in your code: #{warning}"
+end
+
 unless ENV["CI"] == "true"
   require "simplecov"
   SimpleCov.start do
@@ -14,14 +28,25 @@ require "timeout"
 require "securerandom"
 
 def rdkafka_base_config
-  {
-    :"api.version.request" => false,
-    :"broker.version.fallback" => "1.0",
-    :"bootstrap.servers" => "127.0.0.1:9092",
-    # Display statistics and refresh often just to cover those in specs
-    :'statistics.interval.ms' => 1_000,
-    :'topic.metadata.refresh.interval.ms' => 1_000
-  }
+  if ENV['KAFKA_SSL_ENABLED'] == 'true'
+    {
+      :"bootstrap.servers" => "localhost:9093",
+      # Display statistics and refresh often just to cover those in specs
+      :'statistics.interval.ms' => 1_000,
+      :'topic.metadata.refresh.interval.ms' => 1_000,
+      # SSL Configuration
+      :'security.protocol' => 'SSL',
+      :'ssl.ca.location' => './ssl/ca-cert',
+      :'ssl.endpoint.identification.algorithm' => 'none'
+    }
+  else
+    {
+      :"bootstrap.servers" => "localhost:9092",
+      # Display statistics and refresh often just to cover those in specs
+      :'statistics.interval.ms' => 1_000,
+      :'topic.metadata.refresh.interval.ms' => 1_000
+    }
+  end
 end
 
 def rdkafka_config(config_overrides={})
@@ -78,18 +103,32 @@ end
 
 def wait_for_message(topic:, delivery_report:, timeout_in_seconds: 30, consumer: nil)
   new_consumer = consumer.nil?
-  consumer ||= rdkafka_consumer_config.consumer
+  consumer ||= rdkafka_consumer_config('allow.auto.create.topics': true).consumer
   consumer.subscribe(topic)
   timeout = Time.now.to_i + timeout_in_seconds
+  retry_count = 0
+  max_retries = 10
+
   loop do
     if timeout <= Time.now.to_i
       raise "Timeout of #{timeout_in_seconds} seconds reached in wait_for_message"
     end
-    message = consumer.poll(100)
-    if message &&
-        message.partition == delivery_report.partition &&
-        message.offset == delivery_report.offset
-      return message
+
+    begin
+      message = consumer.poll(100)
+      if message &&
+          message.partition == delivery_report.partition &&
+          message.offset == delivery_report.offset
+        return message
+      end
+    rescue Rdkafka::RdkafkaError => e
+      if e.code == :unknown_topic_or_part && retry_count < max_retries
+        retry_count += 1
+        sleep(0.1) # Small delay before retry
+        next
+      else
+        raise
+      end
     end
   end
 ensure
@@ -110,6 +149,16 @@ def wait_for_unassignment(consumer)
   end
 end
 
+def wait_for_topic(admin, topic)
+  admin.metadata(topic)
+rescue Rdkafka::RdkafkaError => e
+  raise unless e.code == :unknown_topic_or_part
+
+  sleep(0.5)
+
+  retry
+end
+
 def notify_listener(listener, &block)
   # 1. subscribe and poll
   consumer.subscribe("consume_test_topic")
@@ -157,9 +206,9 @@ RSpec.configure do |config|
   end
 
   config.around(:each) do |example|
-    # Timeout specs after a minute. If they take longer
+    # Timeout specs after 1.5 minute. If they take longer
     # they are probably stuck
-    Timeout::timeout(60) do
+    Timeout::timeout(90) do
       example.run
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.20.0.rc5
+  version: 0.21.0
 platform: arm64-darwin
 authors:
 - Thijs Cadier
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.15'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: logger
   requirement: !ruby/object:Gem::Requirement
@@ -53,33 +67,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.6'
 - !ruby/object:Gem::Dependency
-  name: ostruct
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ">"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '12'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ">"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '12'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: ostruct
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '12'
-  type: :runtime
+        version: '0'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '12'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -136,6 +150,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: warning
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Modern Kafka client library for Ruby based on librdkafka
 email:
 - contact@karafka.io
@@ -145,13 +173,21 @@ extra_rdoc_files: []
 files:
 - ".github/CODEOWNERS"
 - ".github/FUNDING.yml"
-- ".github/workflows/ci_linux_x86_64_gnu.yml"
-- ".github/workflows/ci_linux_x86_64_musl.yml"
+- ".github/workflows/ci_linux_alpine_x86_64_musl.yml"
+- ".github/workflows/ci_linux_alpine_x86_64_musl_complementary.yml"
+- ".github/workflows/ci_linux_debian_x86_64_gnu.yml"
+- ".github/workflows/ci_linux_debian_x86_64_gnu_complementary.yml"
+- ".github/workflows/ci_linux_ubuntu_aarch64_gnu.yml"
+- ".github/workflows/ci_linux_ubuntu_aarch64_gnu_complementary.yml"
+- ".github/workflows/ci_linux_ubuntu_x86_64_gnu.yml"
+- ".github/workflows/ci_linux_ubuntu_x86_64_gnu_complementary.yml"
 - ".github/workflows/ci_macos_arm64.yml"
+- ".github/workflows/push_linux_aarch64_gnu.yml"
 - ".github/workflows/push_linux_x86_64_gnu.yml"
 - ".github/workflows/push_linux_x86_64_musl.yml"
 - ".github/workflows/push_macos_arm64.yml"
 - ".github/workflows/push_ruby.yml"
+- ".github/workflows/trigger-wiki-refresh.yml"
 - ".github/workflows/verify-action-pins.yml"
 - ".gitignore"
 - ".rspec"
@@ -163,13 +199,21 @@ files:
 - MIT-LICENSE
 - README.md
 - Rakefile
+- dist/cyrus-sasl-2.1.28.tar.gz
+- dist/krb5-1.21.3.tar.gz
+- dist/openssl-3.0.16.tar.gz
+- dist/zlib-1.3.1.tar.gz
+- dist/zstd-1.5.7.tar.gz
+- docker-compose-ssl.yml
 - docker-compose.yml
 - ext/README.md
 - ext/Rakefile
 - ext/build_common.sh
+- ext/build_linux_aarch64_gnu.sh
 - ext/build_linux_x86_64_gnu.sh
 - ext/build_linux_x86_64_musl.sh
 - ext/build_macos_arm64.sh
+- ext/generate-ssl-certs.sh
 - ext/librdkafka.dylib
 - karafka-rdkafka.gemspec
 - lib/rdkafka.rb
@@ -215,34 +259,34 @@ files:
 - lib/rdkafka/producer/partitions_count_cache.rb
 - lib/rdkafka/version.rb
 - renovate.json
-- spec/rdkafka/abstract_handle_spec.rb
-- spec/rdkafka/admin/create_acl_handle_spec.rb
-- spec/rdkafka/admin/create_acl_report_spec.rb
-- spec/rdkafka/admin/create_topic_handle_spec.rb
-- spec/rdkafka/admin/create_topic_report_spec.rb
-- spec/rdkafka/admin/delete_acl_handle_spec.rb
-- spec/rdkafka/admin/delete_acl_report_spec.rb
-- spec/rdkafka/admin/delete_topic_handle_spec.rb
-- spec/rdkafka/admin/delete_topic_report_spec.rb
-- spec/rdkafka/admin/describe_acl_handle_spec.rb
-- spec/rdkafka/admin/describe_acl_report_spec.rb
-- spec/rdkafka/admin_spec.rb
-- spec/rdkafka/bindings_spec.rb
-- spec/rdkafka/callbacks_spec.rb
-- spec/rdkafka/config_spec.rb
-- spec/rdkafka/consumer/headers_spec.rb
-- spec/rdkafka/consumer/message_spec.rb
-- spec/rdkafka/consumer/partition_spec.rb
-- spec/rdkafka/consumer/topic_partition_list_spec.rb
-- spec/rdkafka/consumer_spec.rb
-- spec/rdkafka/error_spec.rb
-- spec/rdkafka/metadata_spec.rb
-- spec/rdkafka/native_kafka_spec.rb
-- spec/rdkafka/producer/delivery_handle_spec.rb
-- spec/rdkafka/producer/delivery_report_spec.rb
-- spec/rdkafka/producer/partitions_count_cache_spec.rb
-- spec/rdkafka/producer/partitions_count_spec.rb
-- spec/rdkafka/producer_spec.rb
+- spec/integrations/ssl_stress_spec.rb
+- spec/lib/rdkafka/abstract_handle_spec.rb
+- spec/lib/rdkafka/admin/create_acl_handle_spec.rb
+- spec/lib/rdkafka/admin/create_acl_report_spec.rb
+- spec/lib/rdkafka/admin/create_topic_handle_spec.rb
+- spec/lib/rdkafka/admin/create_topic_report_spec.rb
+- spec/lib/rdkafka/admin/delete_acl_handle_spec.rb
+- spec/lib/rdkafka/admin/delete_acl_report_spec.rb
+- spec/lib/rdkafka/admin/delete_topic_handle_spec.rb
+- spec/lib/rdkafka/admin/delete_topic_report_spec.rb
+- spec/lib/rdkafka/admin/describe_acl_handle_spec.rb
+- spec/lib/rdkafka/admin/describe_acl_report_spec.rb
+- spec/lib/rdkafka/admin_spec.rb
+- spec/lib/rdkafka/bindings_spec.rb
+- spec/lib/rdkafka/callbacks_spec.rb
+- spec/lib/rdkafka/config_spec.rb
+- spec/lib/rdkafka/consumer/headers_spec.rb
+- spec/lib/rdkafka/consumer/message_spec.rb
+- spec/lib/rdkafka/consumer/partition_spec.rb
+- spec/lib/rdkafka/consumer/topic_partition_list_spec.rb
+- spec/lib/rdkafka/consumer_spec.rb
+- spec/lib/rdkafka/error_spec.rb
+- spec/lib/rdkafka/metadata_spec.rb
+- spec/lib/rdkafka/native_kafka_spec.rb
+- spec/lib/rdkafka/producer/delivery_handle_spec.rb
+- spec/lib/rdkafka/producer/delivery_report_spec.rb
+- spec/lib/rdkafka/producer/partitions_count_cache_spec.rb
+- spec/lib/rdkafka/producer_spec.rb
 - spec/spec_helper.rb
 licenses:
 - MIT
@@ -268,7 +312,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.7
+rubygems_version: 3.6.9
 specification_version: 4
 summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.
   It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+