rdkafka 0.7.0 → 0.8.0.beta.1

data/lib/rdkafka/consumer/headers.rb

@@ -19,7 +19,7 @@ module Rdkafka
           raise Rdkafka::RdkafkaError.new(err, "Error reading message headers")
         end
 
-        headers_ptr = headers_ptrptr.read(:pointer).tap { |it| it.autorelease = false }
+        headers_ptr = headers_ptrptr.read_pointer
 
         name_ptrptr = FFI::MemoryPointer.new(:pointer)
         value_ptrptr = FFI::MemoryPointer.new(:pointer)
@@ -42,12 +42,14 @@ module Rdkafka
             raise Rdkafka::RdkafkaError.new(err, "Error reading a message header at index #{idx}")
           end
 
-          name = name_ptrptr.read(:pointer).tap { |it| it.autorelease = false }
-          name = name.read_string_to_null
+          name_ptr = name_ptrptr.read_pointer
+          name = name_ptr.respond_to?(:read_string_to_null) ? name_ptr.read_string_to_null : name_ptr.read_string
 
           size = size_ptr[:value]
-          value = value_ptrptr.read(:pointer).tap { |it| it.autorelease = false }
-          value = value.read_string(size)
+
+          value_ptr = value_ptrptr.read_pointer
+
+          value = value_ptr.read_string(size)
 
           headers[name.to_sym] = value
 

data/lib/rdkafka/consumer/topic_partition_list.rb

@@ -106,7 +106,7 @@ module Rdkafka
             data[elem[:topic]] = nil
           else
             partitions = data[elem[:topic]] || []
-            offset = if elem[:offset] == -1001
+            offset = if elem[:offset] == Rdkafka::Bindings::RD_KAFKA_OFFSET_INVALID
                        nil
                      else
                        elem[:offset]
@@ -125,10 +125,10 @@ module Rdkafka
       #
       # The pointer will be cleaned by `rd_kafka_topic_partition_list_destroy` when GC releases it.
       #
-      # @return [FFI::AutoPointer]
+      # @return [FFI::Pointer]
       # @private
       def to_native_tpl
-        tpl = TopicPartitionList.new_native_tpl(count)
+        tpl = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(count)
 
         @data.each do |topic, partitions|
           if partitions
@@ -138,6 +138,7 @@ module Rdkafka
                 topic,
                 p.partition
               )
+
               if p.offset
                 Rdkafka::Bindings.rd_kafka_topic_partition_list_set_offset(
                   tpl,
@@ -158,17 +159,6 @@ module Rdkafka
 
         tpl
       end
-
-      # Creates a new native tpl and wraps it into FFI::AutoPointer which in turn calls
-      # `rd_kafka_topic_partition_list_destroy` when a pointer will be cleaned by GC
-      #
-      # @param count [Integer] an initial capacity of partitions list
-      # @return [FFI::AutoPointer]
-      # @private
-      def self.new_native_tpl(count)
-        tpl = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(count)
-        FFI::AutoPointer.new(tpl, Rdkafka::Bindings.method(:rd_kafka_topic_partition_list_destroy))
-      end
     end
   end
 end

data/lib/rdkafka/error.rb

@@ -39,6 +39,11 @@ module Rdkafka
     def is_partition_eof?
       code == :partition_eof
     end
+
+    # Error comparison
+    def ==(another_error)
+       another_error.is_a?(self.class) && (self.to_s == another_error.to_s)
+    end
   end
 
   # Error with topic partition list returned by the underlying rdkafka library.

data/lib/rdkafka/metadata.rb

@@ -0,0 +1,91 @@
+module Rdkafka
+  class Metadata
+    attr_reader :brokers, :topics
+
+    def initialize(native_client, topic_name = nil)
+      native_topic = if topic_name
+        Rdkafka::Bindings.rd_kafka_topic_new(native_client, topic_name, nil)
+      end
+
+      ptr = FFI::MemoryPointer.new(:pointer)
+
+      # Retrieve metadata flag is 0/1 for single/multiple topics.
+      topic_flag = topic_name ? 1 : 0
+
+      # Retrieve the Metadata
+      result = Rdkafka::Bindings.rd_kafka_metadata(native_client, topic_flag, native_topic, ptr, 250)
+
+      # Error Handling
+      Rdkafka::Error.new(result) unless result.zero?
+
+      metadata_from_native(ptr.read_pointer)
+    ensure
+      Rdkafka::Bindings.rd_kafka_topic_destroy(native_topic) if topic_name
+      Rdkafka::Bindings.rd_kafka_metadata_destroy(ptr.read_pointer)
+    end
+
+    private
+
+    def metadata_from_native(ptr)
+      metadata = Metadata.new(ptr)
+      @brokers = Array.new(metadata[:brokers_count]) do |i|
+        BrokerMetadata.new(metadata[:brokers_metadata] + (i * BrokerMetadata.size)).to_h
+      end
+
+      @topics = Array.new(metadata[:topics_count]) do |i|
+        topic = TopicMetadata.new(metadata[:topics_metadata] + (i * TopicMetadata.size))
+        Rdkafka::Error.new(topic[:rd_kafka_resp_err]) unless topic[:rd_kafka_resp_err].zero?
+
+        partitions = Array.new(topic[:partition_count]) do |j|
+          partition = PartitionMetadata.new(topic[:partitions_metadata] + (j * PartitionMetadata.size))
+          Rdkafka::Error.new(partition[:rd_kafka_resp_err]) unless partition[:rd_kafka_resp_err].zero?
+          partition.to_h
+        end
+        topic.to_h.merge!(partitions: partitions)
+      end
+    end
+
+    class CustomFFIStruct < FFI::Struct
+      def to_h
+        members.each_with_object({}) do |mem, hsh|
+          val = self.[](mem)
+          next if val.is_a?(FFI::Pointer) || mem == :rd_kafka_resp_err
+
+          hsh[mem] = self.[](mem)
+        end
+      end
+    end
+
+    class Metadata < CustomFFIStruct
+      layout :brokers_count, :int,
+             :brokers_metadata, :pointer,
+             :topics_count, :int,
+             :topics_metadata, :pointer,
+             :broker_id, :int32,
+             :broker_name, :string
+    end
+
+    class BrokerMetadata < CustomFFIStruct
+      layout :broker_id, :int32,
+             :broker_name, :string,
+             :broker_port, :int
+    end
+
+    class TopicMetadata < CustomFFIStruct
+      layout :topic_name, :string,
+             :partition_count, :int,
+             :partitions_metadata, :pointer,
+             :rd_kafka_resp_err, :int
+    end
+
+    class PartitionMetadata < CustomFFIStruct
+      layout :partition_id, :int32,
+             :rd_kafka_resp_err, :int,
+             :leader, :int32,
+             :replica_count, :int,
+             :replicas, :pointer,
+             :in_sync_replica_brokers, :int,
+             :isrs, :pointer
+    end
+  end
+end

data/lib/rdkafka/producer.rb

@@ -37,10 +37,25 @@ module Rdkafka
 
     # Close this producer and wait for the internal poll queue to empty.
     def close
+      return unless @native_kafka
+
       # Indicate to polling thread that we're closing
       @closing = true
       # Wait for the polling thread to finish up
       @polling_thread.join
+      Rdkafka::Bindings.rd_kafka_destroy(@native_kafka)
+      @native_kafka = nil
+    end
+
+    # Partition count for a given topic.
+    # NOTE: If 'allow.auto.create.topics' is set to true in the broker, the topic will be auto-created after returning nil.
+    #
+    # @param topic [String] The topic name.
+    #
+    # @return partition count [Integer,nil]
+    #
+    def partition_count(topic)
+      Rdkafka::Metadata.new(@native_kafka, topic).topics&.select { |x| x[:topic_name] == topic }&.dig(0, :partition_count)
     end
 
     # Produces a message to a Kafka topic. The message is added to rdkafka's queue, call {DeliveryHandle#wait wait} on the returned delivery handle to make sure it is delivered.
@@ -58,7 +73,7 @@ module Rdkafka
     # @raise [RdkafkaError] When adding the message to rdkafka's queue failed
     #
     # @return [DeliveryHandle] Delivery handle that can be used to wait for the result of producing this message
-    def produce(topic:, payload: nil, key: nil, partition: nil, timestamp: nil, headers: nil)
+    def produce(topic:, payload: nil, key: nil, partition: nil, partition_key: nil, timestamp: nil, headers: nil)
       # Start by checking and converting the input
 
       # Get payload length
@@ -75,9 +90,15 @@ module Rdkafka
                    key.bytesize
                  end
 
-      # If partition is nil use -1 to let Kafka set the partition based
-      # on the key/randomly if there is no key
-      partition = -1 if partition.nil?
+      if partition_key
+        partition_count = partition_count(topic)
+        # If the topic is not present, set to -1
+        partition = Rdkafka::Bindings.partitioner(partition_key, partition_count) if partition_count
+      end
+
+      # If partition is nil, use -1 to let librdafka set the partition randomly or
+      # based on the key when present.
+      partition ||= -1
 
       # If timestamp is nil use 0 and let Kafka set one. If an integer or time
       # use it.

data/lib/rdkafka/producer/delivery_report.rb

@@ -10,11 +10,16 @@ module Rdkafka
       # @return [Integer]
       attr_reader :offset
 
+      # Error in case happen during produce.
+      # @return [string]
+      attr_reader :error
+
       private
 
-      def initialize(partition, offset)
+      def initialize(partition, offset, error = nil)
         @partition = partition
         @offset = offset
+        @error = error
       end
     end
   end

data/lib/rdkafka/version.rb

@@ -1,5 +1,5 @@
 module Rdkafka
-  VERSION = "0.7.0"
-  LIBRDKAFKA_VERSION = "1.2.0"
-  LIBRDKAFKA_SOURCE_SHA256 = "eedde1c96104e4ac2d22a4230e34f35dd60d53976ae2563e3dd7c27190a96859"
+  VERSION = "0.8.0.beta.1"
+  LIBRDKAFKA_VERSION = "1.4.0"
+  LIBRDKAFKA_SOURCE_SHA256 = "ae27ea3f3d0d32d29004e7f709efbba2666c5383a107cc45b3a1949486b2eb84"
 end

data/rdkafka.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |gem|
 
   gem.add_dependency 'ffi', '~> 1.9'
   gem.add_dependency 'mini_portile2', '~> 2.1'
-  gem.add_dependency 'rake', '~> 12.3'
+  gem.add_dependency 'rake', '>= 12.3'
 
   gem.add_development_dependency 'pry', '~> 0.10'
   gem.add_development_dependency 'rspec', '~> 3.5'

data/spec/rdkafka/bindings_spec.rb

@@ -1,4 +1,5 @@
 require "spec_helper"
+require 'zlib'
 
 describe Rdkafka::Bindings do
   it "should load librdkafka" do
@@ -7,12 +8,12 @@ describe Rdkafka::Bindings do
 
   describe ".lib_extension" do
     it "should know the lib extension for darwin" do
-      expect(Gem::Platform.local).to receive(:os).and_return("darwin-aaa")
+      stub_const('RbConfig::CONFIG', 'host_os' =>'darwin')
       expect(Rdkafka::Bindings.lib_extension).to eq "dylib"
     end
 
     it "should know the lib extension for linux" do
-      expect(Gem::Platform.local).to receive(:os).and_return("linux")
+      stub_const('RbConfig::CONFIG', 'host_os' =>'linux')
       expect(Rdkafka::Bindings.lib_extension).to eq "so"
     end
   end
@@ -60,6 +61,23 @@ describe Rdkafka::Bindings do
     end
   end
 
+  describe "partitioner" do
+    let(:partition_key) { ('a'..'z').to_a.shuffle.take(15).join('') }
+    let(:partition_count) { rand(50) + 1 }
+
+    it "should return the same partition for a similar string and the same partition count" do
+      result_1 = Rdkafka::Bindings.partitioner(partition_key, partition_count)
+      result_2 = Rdkafka::Bindings.partitioner(partition_key, partition_count)
+      expect(result_1).to eq(result_2)
+    end
+
+    it "should match the old partitioner" do
+      result_1 = Rdkafka::Bindings.partitioner(partition_key, partition_count)
+      result_2 = (Zlib.crc32(partition_key) % partition_count)
+      expect(result_1).to eq(result_2)
+    end
+  end
+
   describe "stats callback" do
     context "without a stats callback" do
       it "should do nothing" do

data/spec/rdkafka/config_spec.rb

@@ -50,7 +50,9 @@ describe Rdkafka::Config do
     end
 
     it "should create a consumer with valid config" do
-      expect(rdkafka_config.consumer).to be_a Rdkafka::Consumer
+      consumer = rdkafka_config.consumer
+      expect(consumer).to be_a Rdkafka::Consumer
+      consumer.close
     end
 
     it "should raise an error when creating a consumer with invalid config" do
@@ -76,7 +78,9 @@ describe Rdkafka::Config do
     end
 
     it "should create a producer with valid config" do
-      expect(rdkafka_config.producer).to be_a Rdkafka::Producer
+      producer = rdkafka_config.producer
+      expect(producer).to be_a Rdkafka::Producer
+      producer.close
     end
 
     it "should raise an error when creating a producer with invalid config" do

data/spec/rdkafka/consumer/message_spec.rb

@@ -1,7 +1,8 @@
 require "spec_helper"
 
 describe Rdkafka::Consumer::Message do
-  let(:native_topic) { new_native_topic }
+  let(:native_client) { new_native_client }
+  let(:native_topic) { new_native_topic(native_client: native_client) }
   let(:payload) { nil }
   let(:key) { nil }
   let(:native_message) do
@@ -24,6 +25,10 @@ describe Rdkafka::Consumer::Message do
     end
   end
 
+  after(:each) do
+    Rdkafka::Bindings.rd_kafka_destroy(native_client)
+  end
+
   subject { Rdkafka::Consumer::Message.new(native_message) }
 
   before do

data/spec/rdkafka/consumer_spec.rb

@@ -6,7 +6,10 @@ describe Rdkafka::Consumer do
   let(:consumer) { config.consumer }
   let(:producer) { config.producer }
 
-  describe "#subscripe, #unsubscribe and #subscription" do
+  after { consumer.close }
+  after { producer.close }
+
+  describe "#subscribe, #unsubscribe and #subscription" do
     it "should subscribe, unsubscribe and return the subscription" do
       expect(consumer.subscription).to be_empty
 
@@ -88,7 +91,6 @@ describe Rdkafka::Consumer do
         # 8. ensure that message is successfully consumed
         records = consumer.poll(timeout)
         expect(records).not_to be_nil
-        consumer.commit
       end
     end
 
@@ -205,8 +207,6 @@ describe Rdkafka::Consumer do
         expect(records&.payload).to eq "payload c"
         records = consumer.poll(timeout)
         expect(records).to be_nil
-
-        consumer.commit
       end
     end
   end
@@ -313,11 +313,11 @@ describe Rdkafka::Consumer do
       }.to raise_error TypeError
     end
 
-    context "with a commited consumer" do
+    context "with a committed consumer" do
       before :all do
-        # Make sure there are some message
-        producer = rdkafka_config.producer
+        # Make sure there are some messages.
         handles = []
+        producer = rdkafka_config.producer
         10.times do
           (0..2).each do |i|
             handles << producer.produce(
@@ -329,6 +329,7 @@ describe Rdkafka::Consumer do
           end
         end
         handles.each(&:wait)
+        producer.close
       end
 
       before do
@@ -389,20 +390,26 @@ describe Rdkafka::Consumer do
 
       describe "#store_offset" do
         before do
+          config = {}
           config[:'enable.auto.offset.store'] = false
           config[:'enable.auto.commit'] = false
-          consumer.subscribe("consume_test_topic")
-          wait_for_assignment(consumer)
+          @new_consumer = rdkafka_config(config).consumer
+          @new_consumer.subscribe("consume_test_topic")
+          wait_for_assignment(@new_consumer)
+        end
+
+        after do
+          @new_consumer.close
         end
 
         it "should store the offset for a message" do
-          consumer.store_offset(message)
-          consumer.commit
+          @new_consumer.store_offset(message)
+          @new_consumer.commit
 
           list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
             list.add_topic("consume_test_topic", [0, 1, 2])
           end
-          partitions = consumer.committed(list).to_h["consume_test_topic"]
+          partitions = @new_consumer.committed(list).to_h["consume_test_topic"]
           expect(partitions).not_to be_nil
           expect(partitions[message.partition].offset).to eq(message.offset + 1)
         end
@@ -410,7 +417,7 @@ describe Rdkafka::Consumer do
         it "should raise an error with invalid input" do
           allow(message).to receive(:partition).and_return(9999)
           expect {
-            consumer.store_offset(message)
+            @new_consumer.store_offset(message)
           }.to raise_error Rdkafka::RdkafkaError
         end
       end
@@ -554,12 +561,12 @@ describe Rdkafka::Consumer do
         payload:   "payload 1",
         key:       "key 1"
       ).wait
-
       consumer.subscribe("consume_test_topic")
-      message = consumer.poll(5000)
-      expect(message).to be_a Rdkafka::Consumer::Message
+      message = consumer.each {|m| break m}
 
-      # Message content is tested in producer spec
+      expect(message).to be_a Rdkafka::Consumer::Message
+      expect(message.payload).to eq('payload 1')
+      expect(message.key).to eq('key 1')
     end
 
     it "should raise an error when polling fails" do