rdkafka 0.12.0.beta.1 → 0.12.0.beta.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b7009092099e6d6f23e2e4c3ed22799e8e8473ac4265a7c72a81b55d7f4d3df6
-  data.tar.gz: 847c47119591237cca59f178e7a0c438f098ed26039bf96de04d01df7d03362c
+  metadata.gz: a52f754380304d2a9bef30ecd2cbf0f33347cdb02254758f5452d9b9880a5bae
+  data.tar.gz: ed3f56449ad95f679bb5e686117ff6590533bd70965679cedbf952a3d30e0cfb
 SHA512:
-  metadata.gz: 49b59362a354331726158d9d4f3c34b882ac87495d688861089a316873d7a25d4cdea16f37d347cd59db04f2b79410b4885d15edf336d75c4b8b9a4675326af6
-  data.tar.gz: f85edc6bc9937a65169314ce9c068caac55dbd9ddd9d6b2e25c31f97323ac3b2584e9ff8f2d65342cd296278cb524802814e06c806c88b01b940bc13153f5418
+  metadata.gz: 8e5d72b357d60352cabcac36e0257fc72d1ffcfd5f13bcca5652add92f3cac421941af3b2ae19015575490021287218a06755268727f3e07de5826c30d7e9f98
+  data.tar.gz: 84a475f9167d598ad8499412a44c16019beaf77ec297f6d9c0879f27d0bcae079b0006bc880334c7e9d7310d9110250db69bdface85937cb1ec0abd5b56c4b2b

data/ext/Rakefile

@@ -28,16 +28,25 @@ task :default => :clean do
     from_extension = 'so.1'
     to_extension = 'so'
   end
+
+  # Get path to the compiled library
   lib_path = File.join(File.dirname(__FILE__), "ports/#{recipe.host}/librdkafka/#{Rdkafka::LIBRDKAFKA_VERSION}/lib/librdkafka.#{from_extension}")
-  FileUtils.mv(lib_path, File.join(File.dirname(__FILE__), "librdkafka.#{to_extension}"))
+
+  # Get target dir
+  target_dir = ENV["RUBYARCHDIR"] || File.dirname(__FILE__)
+
+  # Move the compliled library there
+  FileUtils.mv(lib_path, File.join(target_dir, "librdkafka.#{to_extension}"))
+
   # Cleanup files created by miniportile we don't need in the gem
-  FileUtils.rm_rf File.join(File.dirname(__FILE__), "tmp")
-  FileUtils.rm_rf File.join(File.dirname(__FILE__), "ports")
+  FileUtils.rm_rf File.join(target_dir, "tmp")
+  FileUtils.rm_rf File.join(target_dir, "ports")
 end
 
 task :clean do
-  FileUtils.rm_f File.join(File.dirname(__FILE__), "librdkafka.dylib")
-  FileUtils.rm_f File.join(File.dirname(__FILE__), "librdkafka.so")
+  target_dir = ENV["RUBYARCHDIR"] || File.dirname(__FILE__)
+  FileUtils.rm_f File.join(target_dir, "librdkafka.dylib")
+  FileUtils.rm_f File.join(target_dir, "librdkafka.so")
   FileUtils.rm_rf File.join(File.dirname(__FILE__), "ports")
   FileUtils.rm_rf File.join(File.dirname(__FILE__), "tmp")
 end

data/lib/rdkafka/bindings.rb

@@ -246,14 +246,21 @@ module Rdkafka
     attach_function :rd_kafka_conf_set_dr_msg_cb, [:pointer, :delivery_cb], :void
 
     # Partitioner
-    attach_function :rd_kafka_msg_partitioner_consistent_random, [:pointer, :pointer, :size_t, :int32, :pointer, :pointer], :int32
+    PARTITIONERS = %w(random consistent consistent_random murmur2 murmur2_random fnv1a fnv1a_random).each_with_object({}) do |name, hsh|
+      method_name = "rd_kafka_msg_partitioner_#{name}".to_sym
+      attach_function method_name, [:pointer, :pointer, :size_t, :int32, :pointer, :pointer], :int32
+      hsh[name] = method_name
+    end
 
-    def self.partitioner(str, partition_count)
+    def self.partitioner(str, partition_count, partitioner_name = "consistent_random")
       # Return RD_KAFKA_PARTITION_UA(unassigned partition) when partition count is nil/zero.
       return -1 unless partition_count&.nonzero?
 
       str_ptr = FFI::MemoryPointer.from_string(str)
-      rd_kafka_msg_partitioner_consistent_random(nil, str_ptr, str.size, partition_count, nil, nil)
+      method_name = PARTITIONERS.fetch(partitioner_name) do
+        raise Rdkafka::Config::ConfigError.new("Unknown partitioner: #{partitioner_name}")
+      end
+      public_send(method_name, nil, str_ptr, str.size, partition_count, nil, nil)
     end
 
     # Create Topics

data/lib/rdkafka/config.rb

@@ -179,7 +179,7 @@ module Rdkafka
       # Set callback to receive delivery reports on config
       Rdkafka::Bindings.rd_kafka_conf_set_dr_msg_cb(config, Rdkafka::Callbacks::DeliveryCallbackFunction)
       # Return producer with Kafka client
-      Rdkafka::Producer.new(Rdkafka::Producer::Client.new(native_kafka(config, :rd_kafka_producer))).tap do |producer|
+      Rdkafka::Producer.new(Rdkafka::Producer::Client.new(native_kafka(config, :rd_kafka_producer)), self[:partitioner]).tap do |producer|
         opaque.producer = producer
       end
     end

data/lib/rdkafka/producer.rb

@@ -10,8 +10,9 @@ module Rdkafka
     attr_reader :delivery_callback
 
     # @private
-    def initialize(client)
+    def initialize(client, partitioner_name)
       @client = client
+      @partitioner_name = partitioner_name || "consistent_random"
 
       # Makes sure, that the producer gets closed before it gets GCed by Ruby
       ObjectSpace.define_finalizer(self, client.finalizer)
@@ -85,7 +86,7 @@ module Rdkafka
       if partition_key
         partition_count = partition_count(topic)
         # If the topic is not present, set to -1
-        partition = Rdkafka::Bindings.partitioner(partition_key, partition_count) if partition_count
+        partition = Rdkafka::Bindings.partitioner(partition_key, partition_count, @partitioner_name) if partition_count
       end
 
       # If partition is nil, use -1 to let librdafka set the partition randomly or

data/lib/rdkafka/version.rb

@@ -1,5 +1,5 @@
 module Rdkafka
-  VERSION = "0.12.0.beta.1"
-  LIBRDKAFKA_VERSION = "1.9.0-RC1"
-  LIBRDKAFKA_SOURCE_SHA256 = "c6aefe22ee14e8d036304bd61a5e5a61e100f0554df911fe107af1375c9f01f5"
+  VERSION = "0.12.0.beta.2"
+  LIBRDKAFKA_VERSION = "1.9.0-RC2"
+  LIBRDKAFKA_SOURCE_SHA256 = "f38c7007d01c489c7b743d200a6760678f43b171b36886c4dda39d4a8d5aab59"
 end

data/spec/rdkafka/bindings_spec.rb

@@ -76,6 +76,13 @@ describe Rdkafka::Bindings do
       result_2 = (Zlib.crc32(partition_key) % partition_count)
       expect(result_1).to eq(result_2)
     end
+
+    it "should return the partition calculated by the specified partitioner" do
+      result_1 = Rdkafka::Bindings.partitioner(partition_key, partition_count, "murmur2")
+      ptr = FFI::MemoryPointer.from_string(partition_key)
+      result_2 = Rdkafka::Bindings.rd_kafka_msg_partitioner_murmur2(nil, ptr, partition_key.size, partition_count, nil, nil)
+      expect(result_1).to eq(result_2)
+    end
   end
 
   describe "stats callback" do

data/spec/rdkafka/producer/client_spec.rb

@@ -42,13 +42,13 @@ describe Rdkafka::Producer::Client do
 
     it "polls the native with default 250ms timeout" do
       polling_loop_expects do
-        expect(Rdkafka::Bindings).to receive(:rd_kafka_poll).with(instance_of(FFI::Pointer), 250)
+        expect(Rdkafka::Bindings).to receive(:rd_kafka_poll).with(instance_of(FFI::Pointer), 250).at_least(:once)
       end
     end
 
     it "check the out queue of native client" do
       polling_loop_expects do
-        expect(Rdkafka::Bindings).to receive(:rd_kafka_outq_len).with(native)
+        expect(Rdkafka::Bindings).to receive(:rd_kafka_outq_len).with(native).at_least(:once)
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.12.0.beta.1
+  version: 0.12.0.beta.2
 platform: ruby
 authors:
 - Thijs Cadier
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-03-24 00:00:00.000000000 Z
+date: 2022-04-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi