rdkafka 0.6.0 → 0.9.0

data/lib/rdkafka/admin/create_topic_handle.rb

@@ -0,0 +1,27 @@
+module Rdkafka
+  class Admin
+    class CreateTopicHandle < AbstractHandle
+      layout :pending, :bool,
+             :response, :int,
+             :error_string, :pointer,
+             :result_name, :pointer
+
+      # @return [String] the name of the operation
+      def operation_name
+        "create topic"
+      end
+
+      # @return [Boolean] whether the create topic was successful
+      def create_result
+        CreateTopicReport.new(self[:error_string], self[:result_name])
+      end
+
+      def raise_error
+        raise RdkafkaError.new(
+            self[:response],
+            broker_message: CreateTopicReport.new(self[:error_string], self[:result_name]).error_string
+        )
+      end
+    end
+  end
+end

data/lib/rdkafka/admin/create_topic_report.rb

@@ -0,0 +1,22 @@
+module Rdkafka
+  class Admin
+    class CreateTopicReport
+      # Any error message generated from the CreateTopic
+      # @return [String]
+      attr_reader :error_string
+
+      # The name of the topic created
+      # @return [String]
+      attr_reader :result_name
+
+      def initialize(error_string, result_name)
+        if error_string != FFI::Pointer::NULL
+          @error_string = error_string.read_string
+        end
+        if result_name != FFI::Pointer::NULL
+          @result_name = @result_name = result_name.read_string
+        end
+      end
+    end
+  end
+end

data/lib/rdkafka/admin/delete_topic_handle.rb

@@ -0,0 +1,27 @@
+module Rdkafka
+  class Admin
+    class DeleteTopicHandle < AbstractHandle
+      layout :pending, :bool,
+             :response, :int,
+             :error_string, :pointer,
+             :result_name, :pointer
+
+      # @return [String] the name of the operation
+      def operation_name
+        "delete topic"
+      end
+
+      # @return [Boolean] whether the delete topic was successful
+      def create_result
+        DeleteTopicReport.new(self[:error_string], self[:result_name])
+      end
+
+      def raise_error
+        raise RdkafkaError.new(
+            self[:response],
+            broker_message: DeleteTopicReport.new(self[:error_string], self[:result_name]).error_string
+        )
+      end
+    end
+  end
+end

data/lib/rdkafka/admin/delete_topic_report.rb

@@ -0,0 +1,22 @@
+module Rdkafka
+  class Admin
+    class DeleteTopicReport
+      # Any error message generated from the DeleteTopic
+      # @return [String]
+      attr_reader :error_string
+
+      # The name of the topic deleted
+      # @return [String]
+      attr_reader :result_name
+
+      def initialize(error_string, result_name)
+        if error_string != FFI::Pointer::NULL
+          @error_string = error_string.read_string
+        end
+        if result_name != FFI::Pointer::NULL
+          @result_name = @result_name = result_name.read_string
+        end
+      end
+    end
+  end
+end

data/lib/rdkafka/bindings.rb

@@ -8,7 +8,7 @@ module Rdkafka
     extend FFI::Library
 
     def self.lib_extension
-      if Gem::Platform.local.os.include?("darwin")
+      if RbConfig::CONFIG['host_os'] =~ /darwin/
         'dylib'
       else
         'so'
@@ -22,6 +22,11 @@ module Rdkafka
     RD_KAFKA_RESP_ERR__NOENT = -156
     RD_KAFKA_RESP_ERR_NO_ERROR = 0
 
+    RD_KAFKA_OFFSET_END       = -1
+    RD_KAFKA_OFFSET_BEGINNING = -2
+    RD_KAFKA_OFFSET_STORED    = -1000
+    RD_KAFKA_OFFSET_INVALID   = -1001
+
     class SizePtr < FFI::Struct
       layout :value, :size_t
     end
@@ -35,6 +40,8 @@ module Rdkafka
 
     attach_function :rd_kafka_memberid, [:pointer], :string
     attach_function :rd_kafka_clusterid, [:pointer], :string
+    attach_function :rd_kafka_metadata, [:pointer, :int, :pointer, :pointer, :int], :int
+    attach_function :rd_kafka_metadata_destroy, [:pointer], :void
 
     # Message struct
 
@@ -123,7 +130,7 @@ module Rdkafka
                  else
                    Logger::UNKNOWN
                  end
-      Rdkafka::Config.logger.add(severity) { "rdkafka: #{line}" }
+      Rdkafka::Config.log_queue << [severity, "rdkafka: #{line}"]
     end
 
     StatsCallback = FFI::Function.new(
@@ -227,22 +234,61 @@ module Rdkafka
     callback :delivery_cb, [:pointer, :pointer, :pointer], :void
     attach_function :rd_kafka_conf_set_dr_msg_cb, [:pointer, :delivery_cb], :void
 
-    DeliveryCallback = FFI::Function.new(
-      :void, [:pointer, :pointer, :pointer]
-    ) do |client_ptr, message_ptr, opaque_ptr|
-      message = Message.new(message_ptr)
-      delivery_handle_ptr_address = message[:_private].address
-      if delivery_handle = Rdkafka::Producer::DeliveryHandle.remove(delivery_handle_ptr_address)
-        # Update delivery handle
-        delivery_handle[:pending] = false
-        delivery_handle[:response] = message[:err]
-        delivery_handle[:partition] = message[:partition]
-        delivery_handle[:offset] = message[:offset]
-        # Call delivery callback on opaque
-        if opaque = Rdkafka::Config.opaques[opaque_ptr.to_i]
-          opaque.call_delivery_callback(Rdkafka::Producer::DeliveryReport.new(message[:partition], message[:offset]))
-        end
-      end
+    # Partitioner
+    attach_function :rd_kafka_msg_partitioner_consistent_random, [:pointer, :pointer, :size_t, :int32, :pointer, :pointer], :int32
+
+    def self.partitioner(str, partition_count)
+      # Return RD_KAFKA_PARTITION_UA(unassigned partition) when partition count is nil/zero.
+      return -1 unless partition_count&.nonzero?
+
+      str_ptr = FFI::MemoryPointer.from_string(str)
+      rd_kafka_msg_partitioner_consistent_random(nil, str_ptr, str.size, partition_count, nil, nil)
     end
+
+    # Create Topics
+
+    RD_KAFKA_ADMIN_OP_CREATETOPICS     = 1   # rd_kafka_admin_op_t
+    RD_KAFKA_EVENT_CREATETOPICS_RESULT = 100 # rd_kafka_event_type_t
+
+    attach_function :rd_kafka_CreateTopics, [:pointer, :pointer, :size_t, :pointer, :pointer], :void
+    attach_function :rd_kafka_NewTopic_new, [:pointer, :size_t, :size_t, :pointer, :size_t], :pointer
+    attach_function :rd_kafka_NewTopic_set_config, [:pointer, :string, :string], :int32
+    attach_function :rd_kafka_NewTopic_destroy, [:pointer], :void
+    attach_function :rd_kafka_event_CreateTopics_result, [:pointer], :pointer
+    attach_function :rd_kafka_CreateTopics_result_topics, [:pointer, :pointer], :pointer
+
+    # Delete Topics
+
+    RD_KAFKA_ADMIN_OP_DELETETOPICS     = 2   # rd_kafka_admin_op_t
+    RD_KAFKA_EVENT_DELETETOPICS_RESULT = 101 # rd_kafka_event_type_t
+
+    attach_function :rd_kafka_DeleteTopics, [:pointer, :pointer, :size_t, :pointer, :pointer], :int32
+    attach_function :rd_kafka_DeleteTopic_new, [:pointer], :pointer
+    attach_function :rd_kafka_DeleteTopic_destroy, [:pointer], :void
+    attach_function :rd_kafka_event_DeleteTopics_result, [:pointer], :pointer
+    attach_function :rd_kafka_DeleteTopics_result_topics, [:pointer, :pointer], :pointer
+
+    # Background Queue and Callback
+
+    attach_function :rd_kafka_queue_get_background, [:pointer], :pointer
+    attach_function :rd_kafka_conf_set_background_event_cb, [:pointer, :pointer], :void
+    attach_function :rd_kafka_queue_destroy, [:pointer], :void
+
+    # Admin Options
+
+    attach_function :rd_kafka_AdminOptions_new, [:pointer, :int32], :pointer
+    attach_function :rd_kafka_AdminOptions_set_opaque, [:pointer, :pointer], :void
+    attach_function :rd_kafka_AdminOptions_destroy, [:pointer], :void
+
+    # Extracting data from event types
+
+    attach_function :rd_kafka_event_type, [:pointer], :int32
+    attach_function :rd_kafka_event_opaque, [:pointer], :pointer
+
+    # Extracting data from topic results
+
+    attach_function :rd_kafka_topic_result_error, [:pointer], :int32
+    attach_function :rd_kafka_topic_result_error_string, [:pointer], :pointer
+    attach_function :rd_kafka_topic_result_name, [:pointer], :pointer
   end
 end

data/lib/rdkafka/callbacks.rb

@@ -0,0 +1,106 @@
+module Rdkafka
+  module Callbacks
+
+    # Extracts attributes of a rd_kafka_topic_result_t
+    #
+    # @private
+    class TopicResult
+      attr_reader :result_error, :error_string, :result_name
+
+      def initialize(topic_result_pointer)
+        @result_error = Rdkafka::Bindings.rd_kafka_topic_result_error(topic_result_pointer)
+        @error_string = Rdkafka::Bindings.rd_kafka_topic_result_error_string(topic_result_pointer)
+        @result_name = Rdkafka::Bindings.rd_kafka_topic_result_name(topic_result_pointer)
+      end
+
+      def self.create_topic_results_from_array(count, array_pointer)
+        (1..count).map do |index|
+          result_pointer = (array_pointer + (index - 1)).read_pointer
+          new(result_pointer)
+        end
+      end
+    end
+
+    # FFI Function used for Create Topic and Delete Topic callbacks
+    BackgroundEventCallbackFunction = FFI::Function.new(
+        :void, [:pointer, :pointer, :pointer]
+    ) do |client_ptr, event_ptr, opaque_ptr|
+      BackgroundEventCallback.call(client_ptr, event_ptr, opaque_ptr)
+    end
+
+    # @private
+    class BackgroundEventCallback
+      def self.call(_, event_ptr, _)
+        event_type = Rdkafka::Bindings.rd_kafka_event_type(event_ptr)
+        if event_type == Rdkafka::Bindings::RD_KAFKA_EVENT_CREATETOPICS_RESULT
+          process_create_topic(event_ptr)
+        elsif event_type == Rdkafka::Bindings::RD_KAFKA_EVENT_DELETETOPICS_RESULT
+          process_delete_topic(event_ptr)
+        end
+      end
+
+      private
+
+      def self.process_create_topic(event_ptr)
+        create_topics_result = Rdkafka::Bindings.rd_kafka_event_CreateTopics_result(event_ptr)
+
+        # Get the number of create topic results
+        pointer_to_size_t = FFI::MemoryPointer.new(:int32)
+        create_topic_result_array = Rdkafka::Bindings.rd_kafka_CreateTopics_result_topics(create_topics_result, pointer_to_size_t)
+        create_topic_results = TopicResult.create_topic_results_from_array(pointer_to_size_t.read_int, create_topic_result_array)
+        create_topic_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
+
+        if create_topic_handle = Rdkafka::Admin::CreateTopicHandle.remove(create_topic_handle_ptr.address)
+          create_topic_handle[:response] = create_topic_results[0].result_error
+          create_topic_handle[:error_string] = create_topic_results[0].error_string
+          create_topic_handle[:result_name] = create_topic_results[0].result_name
+          create_topic_handle[:pending] = false
+        end
+      end
+
+      def self.process_delete_topic(event_ptr)
+        delete_topics_result = Rdkafka::Bindings.rd_kafka_event_DeleteTopics_result(event_ptr)
+
+        # Get the number of topic results
+        pointer_to_size_t = FFI::MemoryPointer.new(:int32)
+        delete_topic_result_array = Rdkafka::Bindings.rd_kafka_DeleteTopics_result_topics(delete_topics_result, pointer_to_size_t)
+        delete_topic_results = TopicResult.create_topic_results_from_array(pointer_to_size_t.read_int, delete_topic_result_array)
+        delete_topic_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
+
+        if delete_topic_handle = Rdkafka::Admin::DeleteTopicHandle.remove(delete_topic_handle_ptr.address)
+          delete_topic_handle[:response] = delete_topic_results[0].result_error
+          delete_topic_handle[:error_string] = delete_topic_results[0].error_string
+          delete_topic_handle[:result_name] = delete_topic_results[0].result_name
+          delete_topic_handle[:pending] = false
+        end
+      end
+    end
+
+    # FFI Function used for Message Delivery callbacks
+
+    DeliveryCallbackFunction = FFI::Function.new(
+        :void, [:pointer, :pointer, :pointer]
+    ) do |client_ptr, message_ptr, opaque_ptr|
+      DeliveryCallback.call(client_ptr, message_ptr, opaque_ptr)
+    end
+
+    # @private
+    class DeliveryCallback
+      def self.call(_, message_ptr, opaque_ptr)
+        message = Rdkafka::Bindings::Message.new(message_ptr)
+        delivery_handle_ptr_address = message[:_private].address
+        if delivery_handle = Rdkafka::Producer::DeliveryHandle.remove(delivery_handle_ptr_address)
+          # Update delivery handle
+          delivery_handle[:response] = message[:err]
+          delivery_handle[:partition] = message[:partition]
+          delivery_handle[:offset] = message[:offset]
+          delivery_handle[:pending] = false
+          # Call delivery callback on opaque
+          if opaque = Rdkafka::Config.opaques[opaque_ptr.to_i]
+            opaque.call_delivery_callback(Rdkafka::Producer::DeliveryReport.new(message[:partition], message[:offset], message[:err]))
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rdkafka/config.rb

@@ -11,6 +11,15 @@ module Rdkafka
     @@statistics_callback = nil
     # @private
     @@opaques = {}
+    # @private
+    @@log_queue = Queue.new
+
+    Thread.start do
+      loop do
+        severity, msg = @@log_queue.pop
+        @@logger.add(severity, msg)
+      end
+    end
 
     # Returns the current logger, by default this is a logger to stdout.
     #
@@ -19,6 +28,15 @@ module Rdkafka
       @@logger
     end
 
+    # Returns a queue whose contents will be passed to the configured logger. Each entry
+    # should follow the format [Logger::Severity, String]. The benefit over calling the
+    # logger directly is that this is safe to use from trap contexts.
+    #
+    # @return [Queue]
+    def self.log_queue
+      @@log_queue
+    end
+
     # Set the logger that will be used for all logging output by this library.
     #
     # @param logger [Logger] The logger to be used
@@ -33,11 +51,11 @@ module Rdkafka
     # You can configure if and how often this happens using `statistics.interval.ms`.
     # The callback is called with a hash that's documented here: https://github.com/edenhill/librdkafka/blob/master/STATISTICS.md
     #
-    # @param callback [Proc] The callback
+    # @param callback [Proc, #call] The callback
     #
     # @return [nil]
     def self.statistics_callback=(callback)
-      raise TypeError.new("Callback has to be a proc or lambda") unless callback.is_a? Proc
+      raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call)
       @@statistics_callback = callback
     end
 
@@ -67,7 +85,7 @@ module Rdkafka
 
     # Returns a new config with the provided options which are merged with {DEFAULT_CONFIG}.
     #
-    # @param config_hash [Hash<String,Symbol => String>] The config options for rdkafka
+    # @param config_hash [Hash{String,Symbol => String}] The config options for rdkafka
     #
     # @return [Config]
     def initialize(config_hash = {})
@@ -137,13 +155,26 @@ module Rdkafka
       # Create Kafka config
       config = native_config(opaque)
       # Set callback to receive delivery reports on config
-      Rdkafka::Bindings.rd_kafka_conf_set_dr_msg_cb(config, Rdkafka::Bindings::DeliveryCallback)
+      Rdkafka::Bindings.rd_kafka_conf_set_dr_msg_cb(config, Rdkafka::Callbacks::DeliveryCallbackFunction)
       # Return producer with Kafka client
       Rdkafka::Producer.new(native_kafka(config, :rd_kafka_producer)).tap do |producer|
         opaque.producer = producer
       end
     end
 
+    # Create an admin instance with this configuration.
+    #
+    # @raise [ConfigError] When the configuration contains invalid options
+    # @raise [ClientCreationError] When the native client cannot be created
+    #
+    # @return [Admin] The created admin instance
+    def admin
+      opaque = Opaque.new
+      config = native_config(opaque)
+      Rdkafka::Bindings.rd_kafka_conf_set_background_event_cb(config, Rdkafka::Callbacks::BackgroundEventCallbackFunction)
+      Rdkafka::Admin.new(native_kafka(config, :rd_kafka_producer))
+    end
+
     # Error that is returned by the underlying rdkafka error if an invalid configuration option is present.
     class ConfigError < RuntimeError; end
 
@@ -155,7 +186,7 @@ module Rdkafka
 
     private
 
-    # This method is only intented to be used to create a client,
+    # This method is only intended to be used to create a client,
     # using it in another way will leak memory.
     def native_config(opaque=nil)
       Rdkafka::Bindings.rd_kafka_conf_new.tap do |config|
@@ -212,10 +243,8 @@ module Rdkafka
         Rdkafka::Bindings.rd_kafka_queue_get_main(handle)
       )
 
-      FFI::AutoPointer.new(
-        handle,
-        Rdkafka::Bindings.method(:rd_kafka_destroy)
-      )
+      # Return handle which should be closed using rd_kafka_destroy after usage.
+      handle
     end
   end
 

data/lib/rdkafka/consumer.rb

@@ -5,6 +5,9 @@ module Rdkafka
   #
   # To create a consumer set up a {Config} and call {Config#consumer consumer} on that. It is
   # mandatory to set `:"group.id"` in the configuration.
+  #
+  # Consumer implements `Enumerable`, so you can use `each` to consume messages, or for example
+  # `each_slice` to consume batches of messages.
   class Consumer
     include Enumerable
 
@@ -17,8 +20,12 @@ module Rdkafka
     # Close this consumer
     # @return [nil]
     def close
+      return unless @native_kafka
+
       @closing = true
       Rdkafka::Bindings.rd_kafka_consumer_close(@native_kafka)
+      Rdkafka::Bindings.rd_kafka_destroy(@native_kafka)
+      @native_kafka = nil
     end
 
     # Subscribe to one or more topics letting Kafka handle partition assignments.
@@ -29,21 +36,22 @@ module Rdkafka
     #
     # @return [nil]
     def subscribe(*topics)
+      closed_consumer_check(__method__)
+
       # Create topic partition list with topics and no partition set
-      tpl = TopicPartitionList.new_native_tpl(topics.length)
+      tpl = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(topics.length)
 
       topics.each do |topic|
-        Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
-          tpl,
-          topic,
-          -1
-        )
+        Rdkafka::Bindings.rd_kafka_topic_partition_list_add(tpl, topic, -1)
       end
+
       # Subscribe to topic partition list and check this was successful
       response = Rdkafka::Bindings.rd_kafka_subscribe(@native_kafka, tpl)
       if response != 0
         raise Rdkafka::RdkafkaError.new(response, "Error subscribing to '#{topics.join(', ')}'")
       end
+    ensure
+      Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl) unless tpl.nil?
     end
 
     # Unsubscribe from all subscribed topics.
@@ -52,6 +60,8 @@ module Rdkafka
     #
     # @return [nil]
     def unsubscribe
+      closed_consumer_check(__method__)
+
       response = Rdkafka::Bindings.rd_kafka_unsubscribe(@native_kafka)
       if response != 0
         raise Rdkafka::RdkafkaError.new(response)
@@ -66,15 +76,23 @@ module Rdkafka
     #
     # @return [nil]
     def pause(list)
+      closed_consumer_check(__method__)
+
       unless list.is_a?(TopicPartitionList)
         raise TypeError.new("list has to be a TopicPartitionList")
       end
+
       tpl = list.to_native_tpl
-      response = Rdkafka::Bindings.rd_kafka_pause_partitions(@native_kafka, tpl)
 
-      if response != 0
-        list = TopicPartitionList.from_native_tpl(tpl)
-        raise Rdkafka::RdkafkaTopicPartitionListError.new(response, list, "Error pausing '#{list.to_h}'")
+      begin
+        response = Rdkafka::Bindings.rd_kafka_pause_partitions(@native_kafka, tpl)
+
+        if response != 0
+          list = TopicPartitionList.from_native_tpl(tpl)
+          raise Rdkafka::RdkafkaTopicPartitionListError.new(response, list, "Error pausing '#{list.to_h}'")
+        end
+      ensure
+        Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
       end
     end
 
@@ -86,13 +104,21 @@ module Rdkafka
     #
     # @return [nil]
     def resume(list)
+      closed_consumer_check(__method__)
+
       unless list.is_a?(TopicPartitionList)
         raise TypeError.new("list has to be a TopicPartitionList")
       end
+
       tpl = list.to_native_tpl
-      response = Rdkafka::Bindings.rd_kafka_resume_partitions(@native_kafka, tpl)
-      if response != 0
-        raise Rdkafka::RdkafkaError.new(response, "Error resume '#{list.to_h}'")
+
+      begin
+        response = Rdkafka::Bindings.rd_kafka_resume_partitions(@native_kafka, tpl)
+        if response != 0
+          raise Rdkafka::RdkafkaError.new(response, "Error resume '#{list.to_h}'")
+        end
+      ensure
+        Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
       end
     end
 
@@ -102,17 +128,21 @@ module Rdkafka
     #
     # @return [TopicPartitionList]
     def subscription
-      tpl = FFI::MemoryPointer.new(:pointer)
-      response = Rdkafka::Bindings.rd_kafka_subscription(@native_kafka, tpl)
+      closed_consumer_check(__method__)
+
+      ptr = FFI::MemoryPointer.new(:pointer)
+      response = Rdkafka::Bindings.rd_kafka_subscription(@native_kafka, ptr)
+
       if response != 0
         raise Rdkafka::RdkafkaError.new(response)
       end
-      tpl = tpl.read(:pointer).tap { |it| it.autorelease = false }
+
+      native = ptr.read_pointer
 
       begin
-        Rdkafka::Consumer::TopicPartitionList.from_native_tpl(tpl)
+        Rdkafka::Consumer::TopicPartitionList.from_native_tpl(native)
       ensure
-        Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
+        Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(native)
       end
     end
 
@@ -122,13 +152,21 @@ module Rdkafka
     #
     # @raise [RdkafkaError] When assigning fails
     def assign(list)
+      closed_consumer_check(__method__)
+
       unless list.is_a?(TopicPartitionList)
         raise TypeError.new("list has to be a TopicPartitionList")
       end
+
       tpl = list.to_native_tpl
-      response = Rdkafka::Bindings.rd_kafka_assign(@native_kafka, tpl)
-      if response != 0
-        raise Rdkafka::RdkafkaError.new(response, "Error assigning '#{list.to_h}'")
+
+      begin
+        response = Rdkafka::Bindings.rd_kafka_assign(@native_kafka, tpl)
+        if response != 0
+          raise Rdkafka::RdkafkaError.new(response, "Error assigning '#{list.to_h}'")
+        end
+      ensure
+        Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
       end
     end
 
@@ -138,19 +176,25 @@ module Rdkafka
     #
     # @return [TopicPartitionList]
     def assignment
-      tpl = FFI::MemoryPointer.new(:pointer)
-      response = Rdkafka::Bindings.rd_kafka_assignment(@native_kafka, tpl)
+      closed_consumer_check(__method__)
+
+      ptr = FFI::MemoryPointer.new(:pointer)
+      response = Rdkafka::Bindings.rd_kafka_assignment(@native_kafka, ptr)
       if response != 0
         raise Rdkafka::RdkafkaError.new(response)
       end
 
-      tpl = tpl.read(:pointer).tap { |it| it.autorelease = false  }
+      tpl = ptr.read_pointer
 
-      begin
-        Rdkafka::Consumer::TopicPartitionList.from_native_tpl(tpl)
-      ensure
-        Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy tpl
+      if !tpl.null?
+        begin
+          Rdkafka::Consumer::TopicPartitionList.from_native_tpl(tpl)
+        ensure
+          Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy tpl
+        end
       end
+    ensure
+      ptr.free unless ptr.nil?
     end
 
     # Return the current committed offset per partition for this consumer group.
@@ -163,17 +207,25 @@ module Rdkafka
     #
     # @return [TopicPartitionList]
     def committed(list=nil, timeout_ms=1200)
+      closed_consumer_check(__method__)
+
       if list.nil?
         list = assignment
       elsif !list.is_a?(TopicPartitionList)
         raise TypeError.new("list has to be nil or a TopicPartitionList")
       end
+
       tpl = list.to_native_tpl
-      response = Rdkafka::Bindings.rd_kafka_committed(@native_kafka, tpl, timeout_ms)
-      if response != 0
-        raise Rdkafka::RdkafkaError.new(response)
+
+      begin
+        response = Rdkafka::Bindings.rd_kafka_committed(@native_kafka, tpl, timeout_ms)
+        if response != 0
+          raise Rdkafka::RdkafkaError.new(response)
+        end
+        TopicPartitionList.from_native_tpl(tpl)
+      ensure
+        Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
       end
-      TopicPartitionList.from_native_tpl(tpl)
     end
 
     # Query broker for low (oldest/beginning) and high (newest/end) offsets for a partition.
@@ -186,6 +238,8 @@ module Rdkafka
     #
     # @return [Integer] The low and high watermark
     def query_watermark_offsets(topic, partition, timeout_ms=200)
+      closed_consumer_check(__method__)
+
       low = FFI::MemoryPointer.new(:int64, 1)
       high = FFI::MemoryPointer.new(:int64, 1)
 
@@ -195,13 +249,16 @@ module Rdkafka
         partition,
         low,
         high,
-        timeout_ms
+        timeout_ms,
       )
       if response != 0
         raise Rdkafka::RdkafkaError.new(response, "Error querying watermark offsets for partition #{partition} of #{topic}")
       end
 
-      return low.read_int64, high.read_int64
+      return low.read_array_of_int64(1).first, high.read_array_of_int64(1).first
+    ensure
+      low.free   unless low.nil?
+      high.free  unless high.nil?
     end
 
     # Calculate the consumer lag per partition for the provided topic partition list.
@@ -217,6 +274,7 @@ module Rdkafka
     # @return [Hash<String, Hash<Integer, Integer>>] A hash containing all topics with the lag per partition
     def lag(topic_partition_list, watermark_timeout_ms=100)
       out = {}
+
       topic_partition_list.to_h.each do |topic, partitions|
         # Query high watermarks for this topic's partitions
         # and compare to the offset in the list.
@@ -239,6 +297,7 @@ module Rdkafka
     #
     # @return [String, nil]
     def cluster_id
+      closed_consumer_check(__method__)
       Rdkafka::Bindings.rd_kafka_clusterid(@native_kafka)
     end
 
@@ -248,6 +307,7 @@ module Rdkafka
     #
     # @return [String, nil]
     def member_id
+      closed_consumer_check(__method__)
       Rdkafka::Bindings.rd_kafka_memberid(@native_kafka)
     end
 
@@ -261,6 +321,8 @@ module Rdkafka
     #
     # @return [nil]
     def store_offset(message)
+      closed_consumer_check(__method__)
+
       # rd_kafka_offset_store is one of the few calls that does not support
       # a string as the topic, so create a native topic for it.
       native_topic = Rdkafka::Bindings.rd_kafka_topic_new(
@@ -291,6 +353,8 @@ module Rdkafka
     #
     # @return [nil]
     def seek(message)
+      closed_consumer_check(__method__)
+
       # rd_kafka_offset_store is one of the few calls that does not support
       # a string as the topic, so create a native topic for it.
       native_topic = Rdkafka::Bindings.rd_kafka_topic_new(
@@ -313,26 +377,37 @@ module Rdkafka
       end
     end
 
-    # Commit the current offsets of this consumer
+    # Manually commit the current offsets of this consumer.
+    #
+    # To use this set `enable.auto.commit`to `false` to disable automatic triggering
+    # of commits.
+    #
+    # If `enable.auto.offset.store` is set to `true` the offset of the last consumed
+    # message for every partition is used. If set to `false` you can use {store_offset} to
+    # indicate when a message has been fully processed.
     #
     # @param list [TopicPartitionList,nil] The topic with partitions to commit
     # @param async [Boolean] Whether to commit async or wait for the commit to finish
     #
-    # @raise [RdkafkaError] When comitting fails
+    # @raise [RdkafkaError] When committing fails
     #
     # @return [nil]
     def commit(list=nil, async=false)
+      closed_consumer_check(__method__)
+
       if !list.nil? && !list.is_a?(TopicPartitionList)
         raise TypeError.new("list has to be nil or a TopicPartitionList")
       end
-      tpl = if list
-              list.to_native_tpl
-            else
-              nil
-            end
-      response = Rdkafka::Bindings.rd_kafka_commit(@native_kafka, tpl, async)
-      if response != 0
-        raise Rdkafka::RdkafkaError.new(response)
+
+      tpl = list ? list.to_native_tpl : nil
+
+      begin
+        response = Rdkafka::Bindings.rd_kafka_commit(@native_kafka, tpl, async)
+        if response != 0
+          raise Rdkafka::RdkafkaError.new(response)
+        end
+      ensure
+        Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl) if tpl
       end
     end
 
@@ -344,6 +419,8 @@ module Rdkafka
     #
     # @return [Message, nil] A message or nil if there was no new message within the timeout
     def poll(timeout_ms)
+      closed_consumer_check(__method__)
+
       message_ptr = Rdkafka::Bindings.rd_kafka_consumer_poll(@native_kafka, timeout_ms)
       if message_ptr.null?
         nil
@@ -367,16 +444,20 @@ module Rdkafka
     # Poll for new messages and yield for each received one. Iteration
     # will end when the consumer is closed.
     #
+    # If `enable.partition.eof` is turned on in the config this will raise an
+    # error when an eof is reached, so you probably want to disable that when
+    # using this method of iteration.
+    #
     # @raise [RdkafkaError] When polling fails
     #
     # @yieldparam message [Message] Received message
     #
     # @return [nil]
-    def each(&block)
+    def each
       loop do
         message = poll(250)
         if message
-          block.call(message)
+          yield(message)
         else
           if @closing
             break
@@ -386,5 +467,99 @@ module Rdkafka
         end
       end
     end
+
+    def closed_consumer_check(method)
+      raise Rdkafka::ClosedConsumerError.new(method) if @native_kafka.nil?
+    end
+
+    # Poll for new messages and yield them in batches that may contain
+    # messages from more than one partition.
+    #
+    # Rather than yield each message immediately as soon as it is received,
+    # each_batch will attempt to wait for as long as `timeout_ms` in order
+    # to create a batch of up to but no more than `max_items` in size.
+    #
+    # Said differently, if more than `max_items` are available within
+    # `timeout_ms`, then `each_batch` will yield early with `max_items` in the
+    # array, but if `timeout_ms` passes by with fewer messages arriving, it
+    # will yield an array of fewer messages, quite possibly zero.
+    #
+    # In order to prevent wrongly auto committing many messages at once across
+    # possibly many partitions, callers must explicitly indicate which messages
+    # have been successfully processed as some consumed messages may not have
+    # been yielded yet. To do this, the caller should set
+    # `enable.auto.offset.store` to false and pass processed messages to
+    # {store_offset}. It is also possible, though more complex, to set
+    # 'enable.auto.commit' to false and then pass a manually assembled
+    # TopicPartitionList to {commit}.
+    #
+    # As with `each`, iteration will end when the consumer is closed.
+    #
+    # Exception behavior is more complicated than with `each`, in that if
+    # :yield_on_error is true, and an exception is raised during the
+    # poll, and messages have already been received, they will be yielded to
+    # the caller before the exception is allowed to propogate.
+    #
+    # If you are setting either auto.commit or auto.offset.store to false in
+    # the consumer configuration, then you should let yield_on_error keep its
+    # default value of false because you are gauranteed to see these messages
+    # again. However, if both auto.commit and auto.offset.store are set to
+    # true, you should set yield_on_error to true so you can process messages
+    # that you may or may not see again.
+    #
+    # @param max_items [Integer] Maximum size of the yielded array of messages
+    #
+    # @param bytes_threshold [Integer] Threshold number of total message bytes in the yielded array of messages
+    #
+    # @param timeout_ms [Integer] max time to wait for up to max_items
+    #
+    # @raise [RdkafkaError] When polling fails
+    #
+    # @yield [messages, pending_exception]
+    # @yieldparam messages [Array] An array of received Message
+    # @yieldparam pending_exception [Exception] normally nil, or an exception
+    # which will be propogated after processing of the partial batch is complete.
+    #
+    # @return [nil]
+    def each_batch(max_items: 100, bytes_threshold: Float::INFINITY, timeout_ms: 250, yield_on_error: false, &block)
+      closed_consumer_check(__method__)
+      slice = []
+      bytes = 0
+      end_time = monotonic_now + timeout_ms / 1000.0
+      loop do
+        break if @closing
+        max_wait = end_time - monotonic_now
+        max_wait_ms = if max_wait <= 0
+                        0  # should not block, but may retrieve a message
+                      else
+                        (max_wait * 1000).floor
+                      end
+        message = nil
+        begin
+          message = poll max_wait_ms
+        rescue Rdkafka::RdkafkaError => error
+          raise unless yield_on_error
+          raise if slice.empty?
+          yield slice.dup, error
+          raise
+        end
+        if message
+          slice << message
+          bytes += message.payload.bytesize
+        end
+        if slice.size == max_items || bytes >= bytes_threshold || monotonic_now >= end_time - 0.001
+          yield slice.dup, nil
+          slice.clear
+          bytes = 0
+          end_time = monotonic_now + timeout_ms / 1000.0
+        end
+      end
+    end
+
+    private
+    def monotonic_now
+      # needed because Time.now can go backwards
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
   end
 end