rdkafka 0.12.0.beta.0 → 0.12.0.beta.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a58b7e8b03dec32027bf3d9ec513a87e6bb0a8abf2f6598cd34a965bebce2dbb
-  data.tar.gz: 603c985bd19a2213671b600680a748279e21d7b4aeb3f23a7f1efa16a5e1229c
+  metadata.gz: 1a6f8dfde957dbdc31ad2f229a6da70ef3023705bfc68e4414e896118b7df3c6
+  data.tar.gz: ed07422acb0268bb6483bda6eb0a93a8d75194c06589918e2d8a1cde5bf7ea5b
 SHA512:
-  metadata.gz: 9e5a5bd4c911f93572d0f8be2e5f576b08ee0ff86ab773e9e7d9db1182002e62c1c57b03cfe6e7b686c9890c35f6fca811fe10daad3a0d4d57bd67a5340c7fa1
-  data.tar.gz: 5fd3c57d9797f3b1f7c636f8190cee0b8cfc56f14daa03a2be21da6ff11fff1b7e00e233ab10a37b280b06ac2a197c155ebf1fdd6dfc8a97af9ce8bd38a341e1
+  metadata.gz: 8e51b05b0aa38d8d31ee0e8394f4d5a637ca8ef591fa2f0c76699ced276101902816f89c1c200ab45f02a13f68ed2fe35c961070c851c840e46986526ac1422a
+  data.tar.gz: f7414245c4ce729abe326a7fac4be7617a68615c3d4339888e22424bb0a49faa5077c2caf725453ab361f1f88afff7d1a7ff4928afcb433792983f2851484c12

data/lib/rdkafka/admin.rb

@@ -90,7 +90,7 @@ module Rdkafka
             admin_options_ptr,
             queue_ptr
         )
-      rescue Exception => err
+      rescue Exception
         CreateTopicHandle.remove(create_topic_handle.to_ptr.address)
         raise
       ensure
@@ -140,7 +140,7 @@ module Rdkafka
             admin_options_ptr,
             queue_ptr
         )
-      rescue Exception => err
+      rescue Exception
         DeleteTopicHandle.remove(delete_topic_handle.to_ptr.address)
         raise
       ensure

data/lib/rdkafka/bindings.rb

@@ -246,14 +246,21 @@ module Rdkafka
     attach_function :rd_kafka_conf_set_dr_msg_cb, [:pointer, :delivery_cb], :void
 
     # Partitioner
-    attach_function :rd_kafka_msg_partitioner_consistent_random, [:pointer, :pointer, :size_t, :int32, :pointer, :pointer], :int32
+    PARTITIONERS = %w(random consistent consistent_random murmur2 murmur2_random fnv1a fnv1a_random).each_with_object({}) do |name, hsh|
+      method_name = "rd_kafka_msg_partitioner_#{name}".to_sym
+      attach_function method_name, [:pointer, :pointer, :size_t, :int32, :pointer, :pointer], :int32
+      hsh[name] = method_name
+    end
 
-    def self.partitioner(str, partition_count)
+    def self.partitioner(str, partition_count, partitioner_name = "consistent_random")
       # Return RD_KAFKA_PARTITION_UA(unassigned partition) when partition count is nil/zero.
       return -1 unless partition_count&.nonzero?
 
       str_ptr = FFI::MemoryPointer.from_string(str)
-      rd_kafka_msg_partitioner_consistent_random(nil, str_ptr, str.size, partition_count, nil, nil)
+      method_name = PARTITIONERS.fetch(partitioner_name) do
+        raise Rdkafka::Config::ConfigError.new("Unknown partitioner: #{partitioner_name}")
+      end
+      public_send(method_name, nil, str_ptr, str.size, partition_count, nil, nil)
     end
 
     # Create Topics

data/lib/rdkafka/config.rb

@@ -179,7 +179,7 @@ module Rdkafka
       # Set callback to receive delivery reports on config
       Rdkafka::Bindings.rd_kafka_conf_set_dr_msg_cb(config, Rdkafka::Callbacks::DeliveryCallbackFunction)
       # Return producer with Kafka client
-      Rdkafka::Producer.new(Rdkafka::Producer::Client.new(native_kafka(config, :rd_kafka_producer))).tap do |producer|
+      Rdkafka::Producer.new(Rdkafka::Producer::Client.new(native_kafka(config, :rd_kafka_producer)), self[:partitioner]).tap do |producer|
         opaque.producer = producer
       end
     end

data/lib/rdkafka/producer.rb

@@ -10,8 +10,9 @@ module Rdkafka
     attr_reader :delivery_callback
 
     # @private
-    def initialize(client)
+    def initialize(client, partitioner_name)
       @client = client
+      @partitioner_name = partitioner_name || "consistent_random"
 
       # Makes sure, that the producer gets closed before it gets GCed by Ruby
       ObjectSpace.define_finalizer(self, client.finalizer)
@@ -85,7 +86,7 @@ module Rdkafka
       if partition_key
         partition_count = partition_count(topic)
         # If the topic is not present, set to -1
-        partition = Rdkafka::Bindings.partitioner(partition_key, partition_count) if partition_count
+        partition = Rdkafka::Bindings.partitioner(partition_key, partition_count, @partitioner_name) if partition_count
       end
 
       # If partition is nil, use -1 to let librdafka set the partition randomly or

data/lib/rdkafka/version.rb

@@ -1,5 +1,5 @@
 module Rdkafka
-  VERSION = "0.12.0.beta.0"
-  LIBRDKAFKA_VERSION = "1.8.2"
-  LIBRDKAFKA_SOURCE_SHA256 = "6a747d293a7a4613bd2897e28e8791476fbe1ae7361f2530a876e0fd483482a6"
+  VERSION = "0.12.0.beta.3"
+  LIBRDKAFKA_VERSION = "1.9.0-RC2"
+  LIBRDKAFKA_SOURCE_SHA256 = "f38c7007d01c489c7b743d200a6760678f43b171b36886c4dda39d4a8d5aab59"
 end

data/spec/rdkafka/bindings_spec.rb

@@ -76,6 +76,13 @@ describe Rdkafka::Bindings do
       result_2 = (Zlib.crc32(partition_key) % partition_count)
       expect(result_1).to eq(result_2)
     end
+
+    it "should return the partition calculated by the specified partitioner" do
+      result_1 = Rdkafka::Bindings.partitioner(partition_key, partition_count, "murmur2")
+      ptr = FFI::MemoryPointer.from_string(partition_key)
+      result_2 = Rdkafka::Bindings.rd_kafka_msg_partitioner_murmur2(nil, ptr, partition_key.size, partition_count, nil, nil)
+      expect(result_1).to eq(result_2)
+    end
   end
 
   describe "stats callback" do

data/spec/rdkafka/consumer_spec.rb

@@ -241,7 +241,7 @@ describe Rdkafka::Consumer do
 
     it "should return the assignment when subscribed" do
       # Make sure there's a message
-      report = producer.produce(
+      producer.produce(
         topic:     "consume_test_topic",
         payload:   "payload 1",
         key:       "key 1",
@@ -272,7 +272,7 @@ describe Rdkafka::Consumer do
     it "should close a consumer" do
       consumer.subscribe("consume_test_topic")
       100.times do |i|
-        report = producer.produce(
+        producer.produce(
           topic:     "consume_test_topic",
           payload:   "payload #{i}",
           key:       "key #{i}",
@@ -289,7 +289,7 @@ describe Rdkafka::Consumer do
   describe "#commit, #committed and #store_offset" do
     # Make sure there's a stored offset
     let!(:report) do
-      report = producer.produce(
+      producer.produce(
         topic:     "consume_test_topic",
         payload:   "payload 1",
         key:       "key 1",
@@ -831,7 +831,6 @@ describe Rdkafka::Consumer do
         )
         consumer = config.consumer
         consumer.subscribe(topic_name)
-        loop_count = 0
         batches_yielded = []
         exceptions_yielded = []
         each_batch_iterations = 0
@@ -875,7 +874,6 @@ describe Rdkafka::Consumer do
         )
         consumer = config.consumer
         consumer.subscribe(topic_name)
-        loop_count = 0
         batches_yielded = []
         exceptions_yielded = []
         each_batch_iterations = 0

data/spec/rdkafka/producer/client_spec.rb

@@ -1,15 +1,16 @@
 require "spec_helper"
 
 describe Rdkafka::Producer::Client do
-  let(:native) { double }
+  let(:config) { rdkafka_producer_config }
+  let(:native) { config.send(:native_kafka, config.send(:native_config), :rd_kafka_producer) }
   let(:closing) { false }
   let(:thread) { double(Thread) }
 
   subject(:client) { described_class.new(native) }
 
   before do
-    allow(Rdkafka::Bindings).to receive(:rd_kafka_poll).with(native, 250)
-    allow(Rdkafka::Bindings).to receive(:rd_kafka_outq_len).with(native).and_return(0)
+    allow(Rdkafka::Bindings).to receive(:rd_kafka_poll).with(instance_of(FFI::Pointer), 250).and_call_original
+    allow(Rdkafka::Bindings).to receive(:rd_kafka_outq_len).with(instance_of(FFI::Pointer)).and_return(0).and_call_original
     allow(Rdkafka::Bindings).to receive(:rd_kafka_destroy)
     allow(Thread).to receive(:new).and_return(thread)
 
@@ -41,13 +42,13 @@ describe Rdkafka::Producer::Client do
 
     it "polls the native with default 250ms timeout" do
       polling_loop_expects do
-        expect(Rdkafka::Bindings).to receive(:rd_kafka_poll).with(native, 250)
+        expect(Rdkafka::Bindings).to receive(:rd_kafka_poll).with(instance_of(FFI::Pointer), 250).at_least(:once)
       end
     end
 
     it "check the out queue of native client" do
       polling_loop_expects do
-        expect(Rdkafka::Bindings).to receive(:rd_kafka_outq_len).with(native)
+        expect(Rdkafka::Bindings).to receive(:rd_kafka_outq_len).with(native).at_least(:once)
       end
     end
   end

data/spec/rdkafka/producer_spec.rb

@@ -376,13 +376,9 @@ describe Rdkafka::Producer do
     end
   end
 
-  it "should produce a message in a forked process" do
+  it "should produce a message in a forked process", skip: defined?(JRUBY_VERSION) && "Kernel#fork is not available" do
     # Fork, produce a message, send the report over a pipe and
     # wait for and check the message in the main process.
-
-    # Kernel#fork is not available in JRuby
-    skip if defined?(JRUBY_VERSION)
-
     reader, writer = IO.pipe
 
     fork do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.12.0.beta.0
+  version: 0.12.0.beta.3
 platform: ruby
 authors:
 - Thijs Cadier
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-03-03 00:00:00.000000000 Z
+date: 2022-04-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -206,7 +206,7 @@ homepage: https://github.com/thijsc/rdkafka-ruby
 licenses:
 - MIT
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -221,8 +221,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
-rubygems_version: 3.1.4
-signing_key:
+rubygems_version: 3.0.3
+signing_key: 
 specification_version: 4
 summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.
   It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+