rdkafka 0.4.2 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 277ff64f82622eddd6c31265d1b4b91b6b60bed802590bf9b4d015ea7a32ce0a
-  data.tar.gz: 027fc22c22349729bb04288f9010eda156a9d04cf9a616bb237b1fc56eb9aed5
+  metadata.gz: c01eae203c0d0a12be82edacae9fb3ecde7c73dc96e6a95d456434a2d0a2e5d9
+  data.tar.gz: 50bced86691e02ca0f1af3df2140751f530b667aa5ddf6f6822110991e3f4b43
 SHA512:
-  metadata.gz: 0cb6c97fa7ac62ead42bcd8da1d22f56b02f56e54a7671e7e2722d029839d24c7390c0fe10890530afb4e9a002e9f5fcc7e213777cf956e762d443edf466715e
-  data.tar.gz: 54865cb4b0772cffa1c91fd4e971a3e23f9f9b24959c83402460bca8b2d268e15bef8df8ea011848a57d75c4088198d02f77713ff9123647f62b7b5ece38a787
+  metadata.gz: af1930db8fea0a9ebec2ab63c404be88a805349badbc575e351e90d694d989a962dcfd4db32a5e344fddc83e8f97ca7600188ccc6d8e46ccd9a73afa99affbed
+  data.tar.gz: c8a629fe6ea2a9e1a0d6c38e6d2eb46f1ea8d6791e7d2bdd2751c83ab5a6ed6fc90d7783eaeef3e2f324f01d50e3bb3d308c12048001519dea42c3810c399240

data/.travis.yml

@@ -11,11 +11,10 @@ env:
     - KAFKA_HEAP_OPTS="-Xmx512m -Xms512m"
 
 rvm:
-  - 2.1
-  - 2.2
   - 2.3
   - 2.4
   - 2.5
+  - 2.6
 
 before_install:
   - gem update --system

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 0.5.0
+* Bump librdkafka to 1.0.0 (by breunigs)
+* Add cluster and member information (by dmexe)
+* Support message headers for consumer & producer (by dmexe)
+* Add consumer rebalance listener (by dmexe)
+
 # 0.4.2
 * Delivery callback for producer
 * Document list param of commit method

data/README.md

@@ -8,7 +8,7 @@
 The `rdkafka` gem is a modern Kafka client library for Ruby based on
 [librdkafka](https://github.com/edenhill/librdkafka/).
 It wraps the production-ready C client using the [ffi](https://github.com/ffi/ffi)
-gem and targets Kafka 1.0+ and Ruby 2.1+.
+gem and targets Kafka 1.0+ and Ruby 2.3+.
 
 This gem only provides a high-level Kafka consumer. If you are running
 an older version of Kafka and/or need the legacy simple consumer we

data/docker-compose.yml

@@ -13,6 +13,6 @@ services:
       KAFKA_ADVERTISED_PORT: 9092
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
-      KAFKA_CREATE_TOPICS: "consume_test_topic:3:1,empty_test_topic:3:1,load_test_topic:3:1,produce_test_topic:3:1,rake_test_topic:3:1,empty_test_topic:3:1"
+      KAFKA_CREATE_TOPICS: "consume_test_topic:3:1,empty_test_topic:3:1,load_test_topic:3:1,produce_test_topic:3:1,rake_test_topic:3:1,watermarks_test_topic:3:1"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock

data/lib/rdkafka.rb

@@ -3,6 +3,7 @@ require "rdkafka/version"
 require "rdkafka/bindings"
 require "rdkafka/config"
 require "rdkafka/consumer"
+require "rdkafka/consumer/headers"
 require "rdkafka/consumer/message"
 require "rdkafka/consumer/partition"
 require "rdkafka/consumer/topic_partition_list"

data/lib/rdkafka/bindings.rb

@@ -17,11 +17,25 @@ module Rdkafka
 
     ffi_lib File.join(File.dirname(__FILE__), "../../ext/librdkafka.#{lib_extension}")
 
+    RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS = -175
+    RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS = -174
+    RD_KAFKA_RESP_ERR__NOENT = -156
+    RD_KAFKA_RESP_ERR_NO_ERROR = 0
+
+    class SizePtr < FFI::Struct
+      layout :value, :size_t
+    end
+
     # Polling
 
     attach_function :rd_kafka_poll, [:pointer, :int], :void, blocking: true
     attach_function :rd_kafka_outq_len, [:pointer], :int, blocking: true
 
+    # Metadata
+
+    attach_function :rd_kafka_memberid, [:pointer], :string
+    attach_function :rd_kafka_clusterid, [:pointer], :string
+
     # Message struct
 
     class Message < FFI::Struct
@@ -148,6 +162,45 @@ module Rdkafka
     attach_function :rd_kafka_consumer_poll, [:pointer, :int], :pointer, blocking: true
     attach_function :rd_kafka_consumer_close, [:pointer], :void, blocking: true
     attach_function :rd_kafka_offset_store, [:pointer, :int32, :int64], :int
+    attach_function :rd_kafka_pause_partitions, [:pointer, :pointer], :int
+    attach_function :rd_kafka_resume_partitions, [:pointer, :pointer], :int
+
+    # Headers
+    attach_function :rd_kafka_header_get_all, [:pointer, :size_t, :pointer, :pointer, SizePtr], :int
+    attach_function :rd_kafka_message_headers, [:pointer, :pointer], :int
+
+    # Rebalance
+
+    callback :rebalance_cb_function, [:pointer, :int, :pointer, :pointer], :void
+    attach_function :rd_kafka_conf_set_rebalance_cb, [:pointer, :rebalance_cb_function], :void
+
+    RebalanceCallback = FFI::Function.new(
+      :void, [:pointer, :int, :pointer, :pointer]
+    ) do |client_ptr, code, partitions_ptr, opaque_ptr|
+      case code
+      when RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS
+        Rdkafka::Bindings.rd_kafka_assign(client_ptr, partitions_ptr)
+      else # RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS or errors
+        Rdkafka::Bindings.rd_kafka_assign(client_ptr, FFI::Pointer::NULL)
+      end
+
+      opaque = Rdkafka::Config.opaques[opaque_ptr.to_i]
+      return unless opaque
+
+      tpl = Rdkafka::Consumer::TopicPartitionList.from_native_tpl(partitions_ptr).freeze
+      consumer = Rdkafka::Consumer.new(client_ptr)
+
+      begin
+        case code
+        when RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS
+          opaque.call_on_partitions_assigned(consumer, tpl)
+        when RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS
+          opaque.call_on_partitions_revoked(consumer, tpl)
+        end
+      rescue Exception => err
+        Rdkafka::Config.logger.error("Unhandled exception: #{err.class} - #{err.message}")
+      end
+    end
 
     # Stats
 
@@ -164,6 +217,8 @@ module Rdkafka
     RD_KAFKA_VTYPE_OPAQUE = 6
     RD_KAFKA_VTYPE_MSGFLAGS = 7
     RD_KAFKA_VTYPE_TIMESTAMP = 8
+    RD_KAFKA_VTYPE_HEADER = 9
+    RD_KAFKA_VTYPE_HEADERS = 10
 
     RD_KAFKA_MSG_F_COPY = 0x2
 

data/lib/rdkafka/config.rb

@@ -72,6 +72,7 @@ module Rdkafka
     # @return [Config]
     def initialize(config_hash = {})
       @config_hash = DEFAULT_CONFIG.merge(config_hash)
+      @consumer_rebalance_listener = nil
     end
 
     # Set a config option.
@@ -93,6 +94,13 @@ module Rdkafka
       @config_hash[key]
     end
 
+    # Get notifications on partition assignment/revocation for the subscribed topics
+    #
+    # @param listener [Object, #on_partitions_assigned, #on_partitions_revoked] listener instance
+    def consumer_rebalance_listener=(listener)
+      @consumer_rebalance_listener = listener
+    end
+
     # Create a consumer with this configuration.
     #
     # @raise [ConfigError] When the configuration contains invalid options
@@ -100,9 +108,19 @@ module Rdkafka
     #
     # @return [Consumer] The created consumer
     def consumer
-      kafka = native_kafka(native_config, :rd_kafka_consumer)
+      opaque = Opaque.new
+      config = native_config(opaque)
+
+      if @consumer_rebalance_listener
+        opaque.consumer_rebalance_listener = @consumer_rebalance_listener
+        Rdkafka::Bindings.rd_kafka_conf_set_rebalance_cb(config, Rdkafka::Bindings::RebalanceCallback)
+      end
+
+      kafka = native_kafka(config, :rd_kafka_consumer)
+
       # Redirect the main queue to the consumer
       Rdkafka::Bindings.rd_kafka_poll_set_consumer(kafka)
+
       # Return consumer with Kafka client
       Rdkafka::Consumer.new(kafka)
     end
@@ -204,9 +222,24 @@ module Rdkafka
   # @private
   class Opaque
     attr_accessor :producer
+    attr_accessor :consumer_rebalance_listener
 
     def call_delivery_callback(delivery_handle)
       producer.call_delivery_callback(delivery_handle) if producer
     end
+
+    def call_on_partitions_assigned(consumer, list)
+      return unless consumer_rebalance_listener
+      return unless consumer_rebalance_listener.respond_to?(:on_partitions_assigned)
+
+      consumer_rebalance_listener.on_partitions_assigned(consumer, list)
+    end
+
+    def call_on_partitions_revoked(consumer, list)
+      return unless consumer_rebalance_listener
+      return unless consumer_rebalance_listener.respond_to?(:on_partitions_revoked)
+
+      consumer_rebalance_listener.on_partitions_revoked(consumer, list)
+    end
   end
 end

data/lib/rdkafka/consumer.rb

@@ -30,7 +30,8 @@ module Rdkafka
     # @return [nil]
     def subscribe(*topics)
       # Create topic partition list with topics and no partition set
-      tpl = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(topics.length)
+      tpl = TopicPartitionList.new_native_tpl(topics.length)
+
       topics.each do |topic|
         Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
           tpl,
@@ -43,9 +44,6 @@ module Rdkafka
       if response != 0
         raise Rdkafka::RdkafkaError.new(response, "Error subscribing to '#{topics.join(', ')}'")
       end
-    ensure
-      # Clean up the topic partition list
-      Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
     end
 
     # Unsubscribe from all subscribed topics.
@@ -60,6 +58,44 @@ module Rdkafka
       end
     end
 
+    # Pause producing or consumption for the provided list of partitions
+    #
+    # @param list [TopicPartitionList] The topic with partitions to pause
+    #
+    # @raise [RdkafkaTopicPartitionListError] When pausing subscription fails.
+    #
+    # @return [nil]
+    def pause(list)
+      unless list.is_a?(TopicPartitionList)
+        raise TypeError.new("list has to be a TopicPartitionList")
+      end
+      tpl = list.to_native_tpl
+      response = Rdkafka::Bindings.rd_kafka_pause_partitions(@native_kafka, tpl)
+
+      if response != 0
+        list = TopicPartitionList.from_native_tpl(tpl)
+        raise Rdkafka::RdkafkaTopicPartitionListError.new(response, list, "Error pausing '#{list.to_h}'")
+      end
+    end
+
+    # Resume producing consumption for the provided list of partitions
+    #
+    # @param list [TopicPartitionList] The topic with partitions to pause
+    #
+    # @raise [RdkafkaError] When resume subscription fails.
+    #
+    # @return [nil]
+    def resume(list)
+      unless list.is_a?(TopicPartitionList)
+        raise TypeError.new("list has to be a TopicPartitionList")
+      end
+      tpl = list.to_native_tpl
+      response = Rdkafka::Bindings.rd_kafka_resume_partitions(@native_kafka, tpl)
+      if response != 0
+        raise Rdkafka::RdkafkaError.new(response, "Error resume '#{list.to_h}'")
+      end
+    end
+
     # Return the current subscription to topics and partitions
     #
     # @raise [RdkafkaError] When getting the subscription fails.
@@ -67,12 +103,17 @@ module Rdkafka
     # @return [TopicPartitionList]
     def subscription
       tpl = FFI::MemoryPointer.new(:pointer)
-      tpl.autorelease = false
       response = Rdkafka::Bindings.rd_kafka_subscription(@native_kafka, tpl)
       if response != 0
         raise Rdkafka::RdkafkaError.new(response)
       end
-      Rdkafka::Consumer::TopicPartitionList.from_native_tpl(tpl.get_pointer(0))
+      tpl = tpl.read(:pointer).tap { |it| it.autorelease = false }
+
+      begin
+        Rdkafka::Consumer::TopicPartitionList.from_native_tpl(tpl)
+      ensure
+        Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
+      end
     end
 
     # Atomic assignment of partitions to consume
@@ -89,8 +130,6 @@ module Rdkafka
       if response != 0
         raise Rdkafka::RdkafkaError.new(response, "Error assigning '#{list.to_h}'")
       end
-    ensure
-      Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl) if tpl
     end
 
     # Returns the current partition assignment.
@@ -100,12 +139,18 @@ module Rdkafka
     # @return [TopicPartitionList]
     def assignment
       tpl = FFI::MemoryPointer.new(:pointer)
-      tpl.autorelease = false
       response = Rdkafka::Bindings.rd_kafka_assignment(@native_kafka, tpl)
       if response != 0
         raise Rdkafka::RdkafkaError.new(response)
       end
-      Rdkafka::Consumer::TopicPartitionList.from_native_tpl(tpl.get_pointer(0))
+
+      tpl = tpl.read(:pointer).tap { |it| it.autorelease = false  }
+
+      begin
+        Rdkafka::Consumer::TopicPartitionList.from_native_tpl(tpl)
+      ensure
+        Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy tpl
+      end
     end
 
     # Return the current committed offset per partition for this consumer group.
@@ -117,7 +162,7 @@ module Rdkafka
     # @raise [RdkafkaError] When getting the committed positions fails.
     #
     # @return [TopicPartitionList]
-    def committed(list=nil, timeout_ms=200)
+    def committed(list=nil, timeout_ms=1200)
       if list.nil?
         list = assignment
       elsif !list.is_a?(TopicPartitionList)
@@ -190,6 +235,22 @@ module Rdkafka
       out
     end
 
+    # Returns the ClusterId as reported in broker metadata.
+    #
+    # @return [String, nil]
+    def cluster_id
+      Rdkafka::Bindings.rd_kafka_clusterid(@native_kafka)
+    end
+
+    # Returns this client's broker-assigned group member id
+    #
+    # This currently requires the high-level KafkaConsumer
+    #
+    # @return [String, nil]
+    def member_id
+      Rdkafka::Bindings.rd_kafka_memberid(@native_kafka)
+    end
+
     # Store offset of a message to be used in the next commit of this consumer
     #
     # When using this `enable.auto.offset.store` should be set to `false` in the config.
@@ -242,8 +303,6 @@ module Rdkafka
       if response != 0
         raise Rdkafka::RdkafkaError.new(response)
       end
-    ensure
-      Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl) if tpl
     end
 
     # Poll for the next message on one of the subscribed topics

data/lib/rdkafka/consumer/headers.rb

@@ -0,0 +1,61 @@
+module Rdkafka
+  class Consumer
+    # A message headers
+    class Headers
+      # Reads a native kafka's message header into ruby's hash
+      #
+      # @return [Hash<String, String>] a message headers
+      #
+      # @raise [Rdkafka::RdkafkaError] when fail to read headers
+      #
+      # @private
+      def self.from_native(native_message)
+        headers_ptrptr = FFI::MemoryPointer.new(:pointer)
+        err = Rdkafka::Bindings.rd_kafka_message_headers(native_message, headers_ptrptr)
+
+        if err == Rdkafka::Bindings::RD_KAFKA_RESP_ERR__NOENT
+          return {}
+        elsif err != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+          raise Rdkafka::RdkafkaError.new(err, "Error reading message headers")
+        end
+
+        headers_ptr = headers_ptrptr.read(:pointer).tap { |it| it.autorelease = false }
+
+        name_ptrptr = FFI::MemoryPointer.new(:pointer)
+        value_ptrptr = FFI::MemoryPointer.new(:pointer)
+        size_ptr = Rdkafka::Bindings::SizePtr.new
+        headers = {}
+
+        idx = 0
+        loop do
+          err = Rdkafka::Bindings.rd_kafka_header_get_all(
+            headers_ptr,
+            idx,
+            name_ptrptr,
+            value_ptrptr,
+            size_ptr
+          )
+
+          if err == Rdkafka::Bindings::RD_KAFKA_RESP_ERR__NOENT
+            break
+          elsif err != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+            raise Rdkafka::RdkafkaError.new(err, "Error reading a message header at index #{idx}")
+          end
+
+          name = name_ptrptr.read(:pointer).tap { |it| it.autorelease = false }
+          name = name.read_string_to_null
+
+          size = size_ptr[:value]
+          value = value_ptrptr.read(:pointer).tap { |it| it.autorelease = false }
+          value = value.read_string(size)
+
+          headers[name.to_sym] = value
+
+          idx += 1
+        end
+
+        headers
+      end
+    end
+  end
+end

data/lib/rdkafka/consumer/message.rb

@@ -26,6 +26,9 @@ module Rdkafka
       # @return [Time, nil]
       attr_reader :timestamp
 
+      # @return [Hash<String, String>] a message headers
+      attr_reader :headers
+
       # @private
       def initialize(native_message)
         # Set topic
@@ -54,12 +57,16 @@ module Rdkafka
                      else
                        nil
                      end
+
+        @headers = Headers.from_native(native_message)
       end
 
       # Human readable representation of this message.
       # @return [String]
       def to_s
-        "<Message in '#{topic}' with key '#{truncate(key)}', payload '#{truncate(payload)}', partition #{partition}, offset #{offset}, timestamp #{timestamp}>"
+        is_headers = @headers.empty? ? "" : ", headers #{headers.size}"
+
+        "<Message in '#{topic}' with key '#{truncate(key)}', payload '#{truncate(payload)}', partition #{partition}, offset #{offset}, timestamp #{timestamp}#{is_headers}>"
       end
 
       def truncate(string)
@@ -69,6 +76,9 @@ module Rdkafka
           string
         end
       end
+
+      private
+
     end
   end
 end

data/lib/rdkafka/consumer/partition.rb

@@ -10,20 +10,25 @@ module Rdkafka
       # @return [Integer, nil]
       attr_reader :offset
 
+      # Partition's error code
+      # @retuen [Integer]
+      attr_reader :err
+
       # @private
-      def initialize(partition, offset)
+      def initialize(partition, offset, err = 0)
         @partition = partition
         @offset = offset
+        @err = err
       end
 
       # Human readable representation of this partition.
       # @return [String]
       def to_s
-        if offset.nil?
-          "<Partition #{partition} without offset>"
-        else
-          "<Partition #{partition} with offset #{offset}>"
-        end
+        message = "<Partition #{partition}"
+        message += " offset=#{offset}" if offset
+        message += " err=#{err}" if err != 0
+        message += ">"
+        message
       end
 
       # Human readable representation of this partition.

data/lib/rdkafka/consumer/topic_partition_list.rb

@@ -54,7 +54,7 @@ module Rdkafka
           if partitions.is_a? Integer
             partitions = (0..partitions - 1)
           end
-          @data[topic.to_s] = partitions.map { |p| Partition.new(p, nil) }
+          @data[topic.to_s] = partitions.map { |p| Partition.new(p, nil, 0) }
         end
       end
 
@@ -88,7 +88,7 @@ module Rdkafka
 
       # Create a new topic partition list based of a native one.
       #
-      # @param pointer [FFI::Pointer] Optional pointer to an existing native list. Its contents will be copied and afterwards it will be destroyed.
+      # @param pointer [FFI::Pointer] Optional pointer to an existing native list. Its contents will be copied.
       #
       # @return [TopicPartitionList]
       #
@@ -111,7 +111,7 @@ module Rdkafka
                      else
                        elem[:offset]
                      end
-            partition = Partition.new(elem[:partition], offset)
+            partition = Partition.new(elem[:partition], offset, elem[:err])
             partitions.push(partition)
             data[elem[:topic]] = partitions
           end
@@ -119,42 +119,55 @@ module Rdkafka
 
         # Return the created object
         TopicPartitionList.new(data)
-      ensure
-        # Destroy the tpl
-        Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(pointer)
       end
 
-      # Create a native tpl with the contents of this object added
+      # Create a native tpl with the contents of this object added.
       #
+      # The pointer will be cleaned by `rd_kafka_topic_partition_list_destroy` when GC releases it.
+      #
+      # @return [FFI::AutoPointer]
       # @private
       def to_native_tpl
-        Rdkafka::Bindings.rd_kafka_topic_partition_list_new(count).tap do |tpl|
-          @data.each do |topic, partitions|
-            if partitions
-              partitions.each do |p|
-                Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
-                  tpl,
-                  topic,
-                  p.partition
-                )
-                if p.offset
-                  Rdkafka::Bindings.rd_kafka_topic_partition_list_set_offset(
-                    tpl,
-                    topic,
-                    p.partition,
-                    p.offset
-                  )
-                end
-              end
-            else
+        tpl = TopicPartitionList.new_native_tpl(count)
+
+        @data.each do |topic, partitions|
+          if partitions
+            partitions.each do |p|
               Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
                 tpl,
                 topic,
-                -1
+                p.partition
               )
+              if p.offset
+                Rdkafka::Bindings.rd_kafka_topic_partition_list_set_offset(
+                  tpl,
+                  topic,
+                  p.partition,
+                  p.offset
+                )
+              end
             end
+          else
+            Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
+              tpl,
+              topic,
+              -1
+            )
           end
         end
+
+        tpl
+      end
+
+      # Creates a new native tpl and wraps it into FFI::AutoPointer which in turn calls
+      # `rd_kafka_topic_partition_list_destroy` when a pointer will be cleaned by GC
+      #
+      # @param count [Integer] an initial capacity of partitions list
+      # @return [FFI::AutoPointer]
+      # @private
+      def self.new_native_tpl(count)
+        tpl = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(count)
+        FFI::AutoPointer.new(tpl, Rdkafka::Bindings.method(:rd_kafka_topic_partition_list_destroy))
       end
     end
   end

data/lib/rdkafka/error.rb

@@ -40,4 +40,16 @@ module Rdkafka
       code == :partition_eof
     end
   end
+
+  # Error with potic partition list returned by the underlying rdkafka library.
+  class RdkafkaTopicPartitionListError < RdkafkaError
+    # @return [TopicPartitionList]
+    attr_reader :topic_partition_list
+
+    # @private
+    def initialize(response, topic_partition_list, message_prefix=nil)
+      super(response, message_prefix)
+      @topic_partition_list = topic_partition_list
+    end
+  end
 end

data/lib/rdkafka/producer.rb

@@ -57,11 +57,12 @@ module Rdkafka
     # @param key [String] The message's key
     # @param partition [Integer,nil] Optional partition to produce to
     # @param timestamp [Time,Integer,nil] Optional timestamp of this message. Integer timestamp is in milliseconds since Jan 1 1970.
+    # @param headers [Hash<String,String>] Optional message headers
     #
     # @raise [RdkafkaError] When adding the message to rdkafka's queue failed
     #
     # @return [DeliveryHandle] Delivery handle that can be used to wait for the result of producing this message
-    def produce(topic:, payload: nil, key: nil, partition: nil, timestamp: nil)
+    def produce(topic:, payload: nil, key: nil, partition: nil, timestamp: nil, headers: nil)
       # Start by checking and converting the input
 
       # Get payload length
@@ -101,9 +102,7 @@ module Rdkafka
       delivery_handle[:offset] = -1
       DeliveryHandle.register(delivery_handle.to_ptr.address, delivery_handle)
 
-      # Produce the message
-      response = Rdkafka::Bindings.rd_kafka_producev(
-        @native_kafka,
+      args = [
         :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_TOPIC, :string, topic,
         :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_MSGFLAGS, :int, Rdkafka::Bindings::RD_KAFKA_MSG_F_COPY,
         :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_VALUE, :buffer_in, payload, :size_t, payload_size,
@@ -111,7 +110,27 @@ module Rdkafka
         :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_PARTITION, :int32, partition,
         :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_TIMESTAMP, :int64, raw_timestamp,
         :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_OPAQUE, :pointer, delivery_handle,
-        :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_END
+      ]
+
+      if headers
+        headers.each do |key0, value0|
+          key = key0.to_s
+          value = value0.to_s
+          args += [
+            :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_HEADER,
+            :string, key,
+            :pointer, value,
+            :size_t, value.bytes.size
+          ]
+        end
+      end
+
+      args += [:int, Rdkafka::Bindings::RD_KAFKA_VTYPE_END]
+
+      # Produce the message
+      response = Rdkafka::Bindings.rd_kafka_producev(
+        @native_kafka,
+        *args
       )
 
       # Raise error if the produce call was not successfull

data/lib/rdkafka/version.rb

@@ -1,5 +1,5 @@
 module Rdkafka
-  VERSION = "0.4.2"
-  LIBRDKAFKA_VERSION = "0.11.6"
-  LIBRDKAFKA_SOURCE_SHA256 = '9c0afb8b53779d968225edf1e79da48a162895ad557900f75e7978f65e642032'
+  VERSION = "0.5.0"
+  LIBRDKAFKA_VERSION = "1.0.0"
+  LIBRDKAFKA_SOURCE_SHA256 = 'b00a0d9f0e8c7ceb67b93b4ee67f3c68279a843a15bf4a6742eb64897519aa09'
 end

data/spec/rdkafka/consumer/message_spec.rb

@@ -23,8 +23,25 @@ describe Rdkafka::Consumer::Message do
       end
     end
   end
+
   subject { Rdkafka::Consumer::Message.new(native_message) }
 
+  before do
+    # mock headers, because it produces 'segmentation fault' while settings or reading headers for
+    # a message which is created from scratch
+    #
+    # Code dump example:
+    #
+    # ```
+    # frame #7: 0x000000010dacf5ab librdkafka.dylib`rd_list_destroy + 11
+    # frame #8: 0x000000010dae5a7e librdkafka.dylib`rd_kafka_headers_destroy + 14
+    # frame #9: 0x000000010da9ab40 librdkafka.dylib`rd_kafka_message_set_headers + 32
+    # ```
+    expect( Rdkafka::Bindings).to receive(:rd_kafka_message_headers).with(any_args) do
+      Rdkafka::Bindings::RD_KAFKA_RESP_ERR__NOENT
+    end
+  end
+
   it "should have a topic" do
     expect(subject.topic).to eq "topic_name"
   end

data/spec/rdkafka/consumer/partition_spec.rb

@@ -2,7 +2,8 @@ require "spec_helper"
 
 describe Rdkafka::Consumer::Partition do
   let(:offset) { 100 }
-  subject { Rdkafka::Consumer::Partition.new(1, offset) }
+  let(:err) { 0 }
+  subject { Rdkafka::Consumer::Partition.new(1, offset, err) }
 
   it "should have a partition" do
     expect(subject.partition).to eq 1
@@ -12,22 +13,34 @@ describe Rdkafka::Consumer::Partition do
     expect(subject.offset).to eq 100
   end
 
+  it "should have an err code" do
+    expect(subject.err).to eq 0
+  end
+
   describe "#to_s" do
     it "should return a human readable representation" do
-      expect(subject.to_s).to eq "<Partition 1 with offset 100>"
+      expect(subject.to_s).to eq "<Partition 1 offset=100>"
     end
   end
 
   describe "#inspect" do
     it "should return a human readable representation" do
-      expect(subject.to_s).to eq "<Partition 1 with offset 100>"
+      expect(subject.to_s).to eq "<Partition 1 offset=100>"
     end
 
     context "without offset" do
       let(:offset) { nil }
 
       it "should return a human readable representation" do
-        expect(subject.to_s).to eq "<Partition 1 without offset>"
+        expect(subject.to_s).to eq "<Partition 1>"
+      end
+    end
+
+    context "with err code" do
+      let(:err) { 1 }
+
+      it "should return a human readable representation" do
+        expect(subject.to_s).to eq "<Partition 1 offset=100 err=1>"
       end
     end
   end

data/spec/rdkafka/consumer/topic_partition_list_spec.rb

@@ -118,7 +118,7 @@ describe Rdkafka::Consumer::TopicPartitionList do
       list = Rdkafka::Consumer::TopicPartitionList.new
       list.add_topic("topic1", [0, 1])
 
-      expected = "<TopicPartitionList: {\"topic1\"=>[<Partition 0 without offset>, <Partition 1 without offset>]}>"
+      expected = "<TopicPartitionList: {\"topic1\"=>[<Partition 0>, <Partition 1>]}>"
 
       expect(list.to_s).to eq expected
     end

data/spec/rdkafka/consumer_spec.rb

@@ -47,6 +47,83 @@ describe Rdkafka::Consumer do
     end
   end
 
+  describe "#pause and #resume" do
+    context "subscription" do
+      let(:timeout) { 1000 }
+
+      before { consumer.subscribe("consume_test_topic") }
+      after { consumer.unsubscribe }
+
+      it "should pause and then resume" do
+        # 1. partitions are assigned
+        wait_for_assignment(consumer)
+        expect(consumer.assignment).not_to be_empty
+
+        # 2. send a first message
+        send_one_message
+
+        # 3. ensure that message is successfully consumed
+        records = consumer.poll(timeout)
+        expect(records).not_to be_nil
+        consumer.commit
+
+        # 4. send a second message
+        send_one_message
+
+        # 5. pause the subscription
+        tpl = Rdkafka::Consumer::TopicPartitionList.new
+        tpl.add_topic("consume_test_topic", (0..2))
+        consumer.pause(tpl)
+
+        # 6. unsure that messages are not available
+        records = consumer.poll(timeout)
+        expect(records).to be_nil
+
+        # 7. resume the subscription
+        tpl = Rdkafka::Consumer::TopicPartitionList.new
+        tpl.add_topic("consume_test_topic", (0..2))
+        consumer.resume(tpl)
+
+        # 8. ensure that message is successfuly consumed
+        records = consumer.poll(timeout)
+        expect(records).not_to be_nil
+        consumer.commit
+      end
+    end
+
+    it "should raise when not TopicPartitionList" do
+      expect { consumer.pause(true) }.to raise_error(TypeError)
+      expect { consumer.resume(true) }.to raise_error(TypeError)
+    end
+
+    it "should raise an error when pausing fails" do
+      list = Rdkafka::Consumer::TopicPartitionList.new.tap { |tpl| tpl.add_topic('topic', (0..1)) }
+
+      expect(Rdkafka::Bindings).to receive(:rd_kafka_pause_partitions).and_return(20)
+      expect {
+        consumer.pause(list)
+      }.to raise_error do |err|
+        expect(err).to be_instance_of(Rdkafka::RdkafkaTopicPartitionListError)
+        expect(err.topic_partition_list).to be
+      end
+    end
+
+    it "should raise an error when resume fails" do
+      expect(Rdkafka::Bindings).to receive(:rd_kafka_resume_partitions).and_return(20)
+      expect {
+        consumer.resume(Rdkafka::Consumer::TopicPartitionList.new)
+      }.to raise_error Rdkafka::RdkafkaError
+    end
+
+    def send_one_message
+      producer.produce(
+        topic:     "consume_test_topic",
+        payload:   "payload 1",
+        key:       "key 1"
+      ).wait
+    end
+  end
+
   describe "#assign and #assignment" do
     it "should return an empty assignment if nothing is assigned" do
       expect(consumer.assignment).to be_empty
@@ -257,13 +334,13 @@ describe Rdkafka::Consumer do
     it "should return the watermark offsets" do
       # Make sure there's a message
       producer.produce(
-        topic:     "consume_test_topic",
+        topic:     "watermarks_test_topic",
         payload:   "payload 1",
         key:       "key 1",
         partition: 0
       ).wait
 
-      low, high = consumer.query_watermark_offsets("consume_test_topic", 0, 5000)
+      low, high = consumer.query_watermark_offsets("watermarks_test_topic", 0, 5000)
       expect(low).to eq 0
       expect(high).to be > 0
     end
@@ -358,6 +435,22 @@ describe Rdkafka::Consumer do
     end
   end
 
+  describe "#cluster_id" do
+    it 'should return the current ClusterId' do
+      consumer.subscribe("consume_test_topic")
+      wait_for_assignment(consumer)
+      expect(consumer.cluster_id).not_to be_empty
+    end
+  end
+
+  describe "#member_id" do
+    it 'should return the current MemberId' do
+      consumer.subscribe("consume_test_topic")
+      wait_for_assignment(consumer)
+      expect(consumer.member_id).to start_with('rdkafka-')
+    end
+  end
+
   describe "#poll" do
     it "should return nil if there is no subscription" do
       expect(consumer.poll(1000)).to be_nil
@@ -395,6 +488,68 @@ describe Rdkafka::Consumer do
     end
   end
 
+  describe "#poll with headers" do
+    it "should return message with headers" do
+      report = producer.produce(
+        topic:     "consume_test_topic",
+        key:       "key headers",
+        headers:   { foo: 'bar' }
+      ).wait
+
+      message = wait_for_message(topic: "consume_test_topic", consumer: consumer, delivery_report: report)
+      expect(message).to be
+      expect(message.key).to eq('key headers')
+      expect(message.headers).to include(foo: 'bar')
+    end
+
+    it "should return message with no headers" do
+      report = producer.produce(
+        topic:     "consume_test_topic",
+        key:       "key no headers",
+        headers:   nil
+      ).wait
+
+      message = wait_for_message(topic: "consume_test_topic", consumer: consumer, delivery_report: report)
+      expect(message).to be
+      expect(message.key).to eq('key no headers')
+      expect(message.headers).to be_empty
+    end
+
+    it "should raise an error when message headers aren't readable" do
+      expect(Rdkafka::Bindings).to receive(:rd_kafka_message_headers).with(any_args) { 1 }
+
+      report = producer.produce(
+        topic:     "consume_test_topic",
+        key:       "key err headers",
+        headers:   nil
+      ).wait
+
+      expect {
+        wait_for_message(topic: "consume_test_topic", consumer: consumer, delivery_report: report)
+      }.to raise_error do |err|
+        expect(err).to be_instance_of(Rdkafka::RdkafkaError)
+        expect(err.message).to start_with("Error reading message headers")
+      end
+    end
+
+    it "should raise an error when the first message header aren't readable" do
+      expect(Rdkafka::Bindings).to receive(:rd_kafka_header_get_all).with(any_args) { 1 }
+
+      report = producer.produce(
+        topic:     "consume_test_topic",
+        key:       "key err headers",
+        headers:   { foo: 'bar' }
+      ).wait
+
+      expect {
+        wait_for_message(topic: "consume_test_topic", consumer: consumer, delivery_report: report)
+      }.to raise_error do |err|
+        expect(err).to be_instance_of(Rdkafka::RdkafkaError)
+        expect(err.message).to start_with("Error reading a message header at index 0")
+      end
+    end
+  end
+
   describe "#each" do
     it "should yield messages" do
       handles = []
@@ -417,4 +572,61 @@ describe Rdkafka::Consumer do
       end
     end
   end
+
+  describe "a rebalance listener" do
+    it "should get notifications" do
+      listener = Struct.new(:queue) do
+        def on_partitions_assigned(consumer, list)
+          collect(:assign, list)
+        end
+
+        def on_partitions_revoked(consumer, list)
+          collect(:revoke, list)
+        end
+
+        def collect(name, list)
+          partitions = list.to_h.map { |key, values| [key, values.map(&:partition)] }.flatten
+          queue << ([name] + partitions)
+        end
+      end.new([])
+
+      notify_listener(listener)
+
+      expect(listener.queue).to eq([
+        [:assign, "consume_test_topic", 0, 1, 2],
+        [:revoke, "consume_test_topic", 0, 1, 2]
+      ])
+    end
+
+    it 'should handle callback exceptions' do
+      listener = Struct.new(:queue) do
+        def on_partitions_assigned(consumer, list)
+          queue << :assigned
+          raise 'boom'
+        end
+
+        def on_partitions_revoked(consumer, list)
+          queue << :revoked
+          raise 'boom'
+        end
+      end.new([])
+
+      notify_listener(listener)
+
+      expect(listener.queue).to eq([:assigned, :revoked])
+    end
+
+    def notify_listener(listener)
+      # 1. subscribe and poll
+      config.consumer_rebalance_listener = listener
+      consumer.subscribe("consume_test_topic")
+      wait_for_assignment(consumer)
+      consumer.poll(100)
+
+      # 2. unsubscribe
+      consumer.unsubscribe
+      wait_for_unassignment(consumer)
+      consumer.close
+    end
+  end
 end

data/spec/rdkafka/producer_spec.rb

@@ -217,6 +217,48 @@ describe Rdkafka::Producer do
     expect(message.payload).to be_nil
   end
 
+  it "should produce a message with headers" do
+    handle = producer.produce(
+      topic:     "produce_test_topic",
+      payload:   "payload headers",
+      key:       "key headers",
+      headers:   { foo: :bar, baz: :foobar }
+    )
+    report = handle.wait(5)
+
+    # Consume message and verify it's content
+    message = wait_for_message(
+      topic: "produce_test_topic",
+      delivery_report: report
+    )
+
+    expect(message.payload).to eq "payload headers"
+    expect(message.key).to eq "key headers"
+    expect(message.headers[:foo]).to eq "bar"
+    expect(message.headers[:baz]).to eq "foobar"
+    expect(message.headers[:foobar]).to be_nil
+  end
+
+  it "should produce a message with empty headers" do
+    handle = producer.produce(
+      topic:     "produce_test_topic",
+      payload:   "payload headers",
+      key:       "key headers",
+      headers:   {}
+    )
+    report = handle.wait(5)
+
+    # Consume message and verify it's content
+    message = wait_for_message(
+      topic: "produce_test_topic",
+      delivery_report: report
+    )
+
+    expect(message.payload).to eq "payload headers"
+    expect(message.key).to eq "key headers"
+    expect(message.headers).to be_empty
+  end
+
   it "should produce message that aren't waited for and not crash" do
     5.times do
       200.times do

data/spec/spec_helper.rb

@@ -61,3 +61,10 @@ def wait_for_assignment(consumer)
     sleep 1
   end
 end
+
+def wait_for_unassignment(consumer)
+  10.times do
+    break if consumer.assignment.empty?
+    sleep 1
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.5.0
 platform: ruby
 authors:
 - Thijs Cadier
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-01-12 00:00:00.000000000 Z
+date: 2019-04-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -131,6 +131,7 @@ files:
 - lib/rdkafka/bindings.rb
 - lib/rdkafka/config.rb
 - lib/rdkafka/consumer.rb
+- lib/rdkafka/consumer/headers.rb
 - lib/rdkafka/consumer/message.rb
 - lib/rdkafka/consumer/partition.rb
 - lib/rdkafka/consumer/topic_partition_list.rb