rdkafka 0.4.2 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 277ff64f82622eddd6c31265d1b4b91b6b60bed802590bf9b4d015ea7a32ce0a
-  data.tar.gz: 027fc22c22349729bb04288f9010eda156a9d04cf9a616bb237b1fc56eb9aed5
+  metadata.gz: 8bf97d10412b4f3c0801f657796fd05c94ceede3caafe6a45719c840b534752a
+  data.tar.gz: 8cc93bffb8119cf9c97aa60cf1f6d634e66a99686e0785150ae51283a6514e8c
 SHA512:
-  metadata.gz: 0cb6c97fa7ac62ead42bcd8da1d22f56b02f56e54a7671e7e2722d029839d24c7390c0fe10890530afb4e9a002e9f5fcc7e213777cf956e762d443edf466715e
-  data.tar.gz: 54865cb4b0772cffa1c91fd4e971a3e23f9f9b24959c83402460bca8b2d268e15bef8df8ea011848a57d75c4088198d02f77713ff9123647f62b7b5ece38a787
+  metadata.gz: e8e085b3dd9d80d77003d2c7f0fbda69371c233b121eac2594556d7d62da99a4bed4aace93e9489783c789e64bafd732faaad3de2ffb5d3a5af8568c5b43676b
+  data.tar.gz: 5a267a9814e87e876544e04e6657829c298ff01b03cb5ff41a0205795ea2fb4f6a2147ee1a0949e52f7a069f5dc73d5ed2072867194032e8ea913ac5d6871e23

data/.travis.yml

@@ -11,14 +11,27 @@ env:
     - KAFKA_HEAP_OPTS="-Xmx512m -Xms512m"
 
 rvm:
-  - 2.1
-  - 2.2
-  - 2.3
   - 2.4
   - 2.5
+  - 2.6
+  - 2.7.1
+  - jruby-9.2.9.0
+  - jruby-9.2.10.0
+  - jruby-9.2.11.1
 
 before_install:
   - gem update --system
+  - |
+    r_eng="$(ruby -e 'STDOUT.write RUBY_ENGINE')";
+    if [ "$r_eng" == "jruby" ]; then
+      sudo apt-get update && \
+      sudo apt-get install -y git && \
+      sudo apt-get install -y libpthread-stubs0-dev && \
+      sudo apt-get install -y build-essential && \
+      sudo apt-get install -y zlib1g-dev && \
+      sudo apt-get install -y libssl-dev && \
+      sudo apt-get install -y libsasl2-dev
+    fi
 
 before_script:
   - docker-compose up -d
@@ -28,7 +41,7 @@ before_script:
   - ./cc-test-reporter before-build
 
 script:
-  - bundle exec rspec
+  - bundle exec rspec --format documentation
 
 after_script:
   - docker-compose stop

data/CHANGELOG.md

@@ -1,3 +1,26 @@
+# 0.8.0
+* Upgrade librdkafka to 1.4.0
+* Integrate librdkafka metadata API and add partition_key (by Adithya-copart)
+* Ruby 2.7 compatibility fix (by Geoff Thé)A
+* Add error to delivery report (by Alex Stanovsky)
+* Don't override CPPFLAGS and LDFLAGS if already set on Mac (by Hiroshi Hatake)
+* Allow use of Rake 13.x and up (by Tomasz Pajor)
+
+# 0.7.0
+* Bump librdkafka to 1.2.0 (by rob-as)
+* Allow customizing the wait time for delivery report availability (by mensfeld)
+
+# 0.6.0
+* Bump librdkafka to 1.1.0 (by Chris Gaffney)
+* Implement seek (by breunigs)
+
+# 0.5.0
+* Bump librdkafka to 1.0.0 (by breunigs)
+* Add cluster and member information (by dmexe)
+* Support message headers for consumer & producer (by dmexe)
+* Add consumer rebalance listener (by dmexe)
+* Implement pause/resume partitions (by dmexe)
+
 # 0.4.2
 * Delivery callback for producer
 * Document list param of commit method

data/README.md

@@ -8,7 +8,11 @@
 The `rdkafka` gem is a modern Kafka client library for Ruby based on
 [librdkafka](https://github.com/edenhill/librdkafka/).
 It wraps the production-ready C client using the [ffi](https://github.com/ffi/ffi)
-gem and targets Kafka 1.0+ and Ruby 2.1+.
+gem and targets Kafka 1.0+ and Ruby 2.4+.
+
+`rdkafka` was written because we needed a reliable Ruby client for
+Kafka that supports modern Kafka at [AppSignal](https://appsignal.com).
+We run it in production on very high traffic systems.
 
 This gem only provides a high-level Kafka consumer. If you are running
 an older version of Kafka and/or need the legacy simple consumer we
@@ -68,13 +72,6 @@ end
 delivery_handles.each(&:wait)
 ```
 
-## Known issues
-
-When using forked process such as when using Unicorn you currently need
-to make sure that you create rdkafka instances after forking. Otherwise
-they will not work and crash your Ruby process when they are garbage
-collected. See https://github.com/appsignal/rdkafka-ruby/issues/19
-
 ## Development
 
 A Docker Compose file is included to run Kafka and Zookeeper. To run

data/docker-compose.yml

@@ -1,18 +1,22 @@
+
 version: '2'
 services:
   zookeeper:
-    image: wurstmeister/zookeeper
-    ports:
-      - "2181:2181"
+    image: confluentinc/cp-zookeeper:latest
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_TICK_TIME: 2000
+
   kafka:
-    image: wurstmeister/kafka:1.0.1
+    image: confluentinc/cp-kafka:latest
+    depends_on:
+      - zookeeper
     ports:
-      - "9092:9092"
+      - 9092:9092
     environment:
-      KAFKA_ADVERTISED_HOST_NAME: localhost
-      KAFKA_ADVERTISED_PORT: 9092
+      KAFKA_BROKER_ID: 1
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
-      KAFKA_CREATE_TOPICS: "consume_test_topic:3:1,empty_test_topic:3:1,load_test_topic:3:1,produce_test_topic:3:1,rake_test_topic:3:1,empty_test_topic:3:1"
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1

data/ext/README.md

@@ -5,19 +5,7 @@ this gem is installed.
 
 To update the `librdkafka` version follow the following steps:
 
-* Download the new version `tar.gz` from  https://github.com/edenhill/librdkafka/
-* Generate a `sha256` with (if using MacOS) `shasum -a 256 <file>`
-* Change the `sha256` in `lib/rdkafka/version.rb`
+* Go to https://github.com/edenhill/librdkafka/releases to get the new
+  version number and asset checksum for `tar.gz`.
 * Change the version in `lib/rdkafka/version.rb`
-
-## Disclaimer
-
-Currently the `librdkafka` project does not provide
-checksums of releases. The checksum provided here is generated on a best
-effort basis. If the CDN would be compromised at the time of download the
-checksum could be incorrect.
-
-Do your own verification if you rely on this behaviour.
-
-Once https://github.com/appsignal/rdkafka-ruby/issues/44 is implemented
-we will change this process.
+* Change the `sha256` in `lib/rdkafka/version.rb`

data/ext/Rakefile

@@ -23,9 +23,10 @@ task :default => :clean do
   recipe = MiniPortile.new("librdkafka", Rdkafka::LIBRDKAFKA_VERSION)
 
   # Use default homebrew openssl if we're on mac and the directory exists
-  if recipe.host.include?("darwin") && Dir.exists?("/usr/local/opt/openssl")
-    ENV["CPPFLAGS"] = "-I/usr/local/opt/openssl/include"
-    ENV["LDFLAGS"] = "-L/usr/local/opt/openssl/lib"
+  # and each of flags is not empty
+  if recipe.host&.include?("darwin") && Dir.exist?("/usr/local/opt/openssl")
+    ENV["CPPFLAGS"] = "-I/usr/local/opt/openssl/include" unless ENV["CPPFLAGS"]
+    ENV["LDFLAGS"] = "-L/usr/local/opt/openssl/lib" unless ENV["LDFLAGS"]
   end
 
   recipe.files << {
@@ -55,3 +56,22 @@ task :clean do
   FileUtils.rm_rf File.join(File.dirname(__FILE__), "ports")
   FileUtils.rm_rf File.join(File.dirname(__FILE__), "tmp")
 end
+
+namespace :build do
+  desc "Build librdkafka at the given git sha or tag"
+  task :git, [:ref] do |task, args|
+    ref = args[:ref]
+    version = "git-#{ref}"
+
+    recipe = MiniPortile.new("librdkafka", version)
+    recipe.files << "https://github.com/edenhill/librdkafka/archive/#{ref}.tar.gz"
+    recipe.configure_options = ["--host=#{recipe.host}"]
+    recipe.cook
+
+    ext = recipe.host.include?("darwin") ? "dylib" : "so"
+    lib = File.expand_path("ports/#{recipe.host}/librdkafka/#{version}/lib/librdkafka.#{ext}", __dir__)
+
+    # Copy will copy the content, following any symlinks
+    FileUtils.cp(lib, __dir__)
+  end
+end

data/lib/rdkafka.rb

@@ -3,10 +3,12 @@ require "rdkafka/version"
 require "rdkafka/bindings"
 require "rdkafka/config"
 require "rdkafka/consumer"
+require "rdkafka/consumer/headers"
 require "rdkafka/consumer/message"
 require "rdkafka/consumer/partition"
 require "rdkafka/consumer/topic_partition_list"
 require "rdkafka/error"
+require "rdkafka/metadata"
 require "rdkafka/producer"
 require "rdkafka/producer/delivery_handle"
 require "rdkafka/producer/delivery_report"

data/lib/rdkafka/bindings.rb

@@ -8,7 +8,7 @@ module Rdkafka
     extend FFI::Library
 
     def self.lib_extension
-      if Gem::Platform.local.os.include?("darwin")
+      if RbConfig::CONFIG['host_os'] =~ /darwin/
         'dylib'
       else
         'so'
@@ -17,11 +17,32 @@ module Rdkafka
 
     ffi_lib File.join(File.dirname(__FILE__), "../../ext/librdkafka.#{lib_extension}")
 
+    RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS = -175
+    RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS = -174
+    RD_KAFKA_RESP_ERR__NOENT = -156
+    RD_KAFKA_RESP_ERR_NO_ERROR = 0
+
+    RD_KAFKA_OFFSET_END       = -1
+    RD_KAFKA_OFFSET_BEGINNING = -2
+    RD_KAFKA_OFFSET_STORED    = -1000
+    RD_KAFKA_OFFSET_INVALID   = -1001
+
+    class SizePtr < FFI::Struct
+      layout :value, :size_t
+    end
+
     # Polling
 
     attach_function :rd_kafka_poll, [:pointer, :int], :void, blocking: true
     attach_function :rd_kafka_outq_len, [:pointer], :int, blocking: true
 
+    # Metadata
+
+    attach_function :rd_kafka_memberid, [:pointer], :string
+    attach_function :rd_kafka_clusterid, [:pointer], :string
+    attach_function :rd_kafka_metadata, [:pointer, :int, :pointer, :pointer, :int], :int
+    attach_function :rd_kafka_metadata_destroy, [:pointer], :void
+
     # Message struct
 
     class Message < FFI::Struct
@@ -148,6 +169,46 @@ module Rdkafka
     attach_function :rd_kafka_consumer_poll, [:pointer, :int], :pointer, blocking: true
     attach_function :rd_kafka_consumer_close, [:pointer], :void, blocking: true
     attach_function :rd_kafka_offset_store, [:pointer, :int32, :int64], :int
+    attach_function :rd_kafka_pause_partitions, [:pointer, :pointer], :int
+    attach_function :rd_kafka_resume_partitions, [:pointer, :pointer], :int
+    attach_function :rd_kafka_seek, [:pointer, :int32, :int64, :int], :int
+
+    # Headers
+    attach_function :rd_kafka_header_get_all, [:pointer, :size_t, :pointer, :pointer, SizePtr], :int
+    attach_function :rd_kafka_message_headers, [:pointer, :pointer], :int
+
+    # Rebalance
+
+    callback :rebalance_cb_function, [:pointer, :int, :pointer, :pointer], :void
+    attach_function :rd_kafka_conf_set_rebalance_cb, [:pointer, :rebalance_cb_function], :void
+
+    RebalanceCallback = FFI::Function.new(
+      :void, [:pointer, :int, :pointer, :pointer]
+    ) do |client_ptr, code, partitions_ptr, opaque_ptr|
+      case code
+      when RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS
+        Rdkafka::Bindings.rd_kafka_assign(client_ptr, partitions_ptr)
+      else # RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS or errors
+        Rdkafka::Bindings.rd_kafka_assign(client_ptr, FFI::Pointer::NULL)
+      end
+
+      opaque = Rdkafka::Config.opaques[opaque_ptr.to_i]
+      return unless opaque
+
+      tpl = Rdkafka::Consumer::TopicPartitionList.from_native_tpl(partitions_ptr).freeze
+      consumer = Rdkafka::Consumer.new(client_ptr)
+
+      begin
+        case code
+        when RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS
+          opaque.call_on_partitions_assigned(consumer, tpl)
+        when RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS
+          opaque.call_on_partitions_revoked(consumer, tpl)
+        end
+      rescue Exception => err
+        Rdkafka::Config.logger.error("Unhandled exception: #{err.class} - #{err.message}")
+      end
+    end
 
     # Stats
 
@@ -164,6 +225,8 @@ module Rdkafka
     RD_KAFKA_VTYPE_OPAQUE = 6
     RD_KAFKA_VTYPE_MSGFLAGS = 7
     RD_KAFKA_VTYPE_TIMESTAMP = 8
+    RD_KAFKA_VTYPE_HEADER = 9
+    RD_KAFKA_VTYPE_HEADERS = 10
 
     RD_KAFKA_MSG_F_COPY = 0x2
 
@@ -171,6 +234,17 @@ module Rdkafka
     callback :delivery_cb, [:pointer, :pointer, :pointer], :void
     attach_function :rd_kafka_conf_set_dr_msg_cb, [:pointer, :delivery_cb], :void
 
+    # Partitioner
+    attach_function :rd_kafka_msg_partitioner_consistent_random, [:pointer, :pointer, :size_t, :int32, :pointer, :pointer], :int32
+
+    def self.partitioner(str, partition_count)
+      # Return RD_KAFKA_PARTITION_UA(unassigned partition) when partition count is nil/zero.
+      return -1 unless partition_count&.nonzero?
+
+      str_ptr = FFI::MemoryPointer.from_string(str)
+      rd_kafka_msg_partitioner_consistent_random(nil, str_ptr, str.size, partition_count, nil, nil)
+    end
+
     DeliveryCallback = FFI::Function.new(
       :void, [:pointer, :pointer, :pointer]
     ) do |client_ptr, message_ptr, opaque_ptr|
@@ -184,7 +258,7 @@ module Rdkafka
         delivery_handle[:offset] = message[:offset]
         # Call delivery callback on opaque
         if opaque = Rdkafka::Config.opaques[opaque_ptr.to_i]
-          opaque.call_delivery_callback(Rdkafka::Producer::DeliveryReport.new(message[:partition], message[:offset]))
+          opaque.call_delivery_callback(Rdkafka::Producer::DeliveryReport.new(message[:partition], message[:offset], message[:err]))
         end
       end
     end

data/lib/rdkafka/config.rb

@@ -67,11 +67,12 @@ module Rdkafka
 
     # Returns a new config with the provided options which are merged with {DEFAULT_CONFIG}.
     #
-    # @param config_hash [Hash<String,Symbol => String>] The config options for rdkafka
+    # @param config_hash [Hash{String,Symbol => String}] The config options for rdkafka
     #
     # @return [Config]
     def initialize(config_hash = {})
       @config_hash = DEFAULT_CONFIG.merge(config_hash)
+      @consumer_rebalance_listener = nil
     end
 
     # Set a config option.
@@ -93,6 +94,13 @@ module Rdkafka
       @config_hash[key]
     end
 
+    # Get notifications on partition assignment/revocation for the subscribed topics
+    #
+    # @param listener [Object, #on_partitions_assigned, #on_partitions_revoked] listener instance
+    def consumer_rebalance_listener=(listener)
+      @consumer_rebalance_listener = listener
+    end
+
     # Create a consumer with this configuration.
     #
     # @raise [ConfigError] When the configuration contains invalid options
@@ -100,9 +108,19 @@ module Rdkafka
     #
     # @return [Consumer] The created consumer
     def consumer
-      kafka = native_kafka(native_config, :rd_kafka_consumer)
+      opaque = Opaque.new
+      config = native_config(opaque)
+
+      if @consumer_rebalance_listener
+        opaque.consumer_rebalance_listener = @consumer_rebalance_listener
+        Rdkafka::Bindings.rd_kafka_conf_set_rebalance_cb(config, Rdkafka::Bindings::RebalanceCallback)
+      end
+
+      kafka = native_kafka(config, :rd_kafka_consumer)
+
       # Redirect the main queue to the consumer
       Rdkafka::Bindings.rd_kafka_poll_set_consumer(kafka)
+
       # Return consumer with Kafka client
       Rdkafka::Consumer.new(kafka)
     end
@@ -137,7 +155,7 @@ module Rdkafka
 
     private
 
-    # This method is only intented to be used to create a client,
+    # This method is only intended to be used to create a client,
     # using it in another way will leak memory.
     def native_config(opaque=nil)
       Rdkafka::Bindings.rd_kafka_conf_new.tap do |config|
@@ -194,19 +212,32 @@ module Rdkafka
         Rdkafka::Bindings.rd_kafka_queue_get_main(handle)
       )
 
-      FFI::AutoPointer.new(
-        handle,
-        Rdkafka::Bindings.method(:rd_kafka_destroy)
-      )
+      # Return handle which should be closed using rd_kafka_destroy after usage.
+      handle
     end
   end
 
   # @private
   class Opaque
     attr_accessor :producer
+    attr_accessor :consumer_rebalance_listener
 
     def call_delivery_callback(delivery_handle)
       producer.call_delivery_callback(delivery_handle) if producer
     end
+
+    def call_on_partitions_assigned(consumer, list)
+      return unless consumer_rebalance_listener
+      return unless consumer_rebalance_listener.respond_to?(:on_partitions_assigned)
+
+      consumer_rebalance_listener.on_partitions_assigned(consumer, list)
+    end
+
+    def call_on_partitions_revoked(consumer, list)
+      return unless consumer_rebalance_listener
+      return unless consumer_rebalance_listener.respond_to?(:on_partitions_revoked)
+
+      consumer_rebalance_listener.on_partitions_revoked(consumer, list)
+    end
   end
 end

data/lib/rdkafka/consumer.rb

@@ -5,6 +5,9 @@ module Rdkafka
   #
   # To create a consumer set up a {Config} and call {Config#consumer consumer} on that. It is
   # mandatory to set `:"group.id"` in the configuration.
+  #
+  # Consumer implements `Enumerable`, so you can use `each` to consume messages, or for example
+  # `each_slice` to consume batches of messages.
   class Consumer
     include Enumerable
 
@@ -17,8 +20,12 @@ module Rdkafka
     # Close this consumer
     # @return [nil]
     def close
+      return unless @native_kafka
+
       @closing = true
       Rdkafka::Bindings.rd_kafka_consumer_close(@native_kafka)
+      Rdkafka::Bindings.rd_kafka_destroy(@native_kafka)
+      @native_kafka = nil
     end
 
     # Subscribe to one or more topics letting Kafka handle partition assignments.
@@ -31,20 +38,17 @@ module Rdkafka
     def subscribe(*topics)
       # Create topic partition list with topics and no partition set
       tpl = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(topics.length)
+
       topics.each do |topic|
-        Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
-          tpl,
-          topic,
-          -1
-        )
+        Rdkafka::Bindings.rd_kafka_topic_partition_list_add(tpl, topic, -1)
       end
+
       # Subscribe to topic partition list and check this was successful
       response = Rdkafka::Bindings.rd_kafka_subscribe(@native_kafka, tpl)
       if response != 0
         raise Rdkafka::RdkafkaError.new(response, "Error subscribing to '#{topics.join(', ')}'")
       end
     ensure
-      # Clean up the topic partition list
       Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
     end
 
@@ -60,19 +64,76 @@ module Rdkafka
       end
     end
 
+    # Pause producing or consumption for the provided list of partitions
+    #
+    # @param list [TopicPartitionList] The topic with partitions to pause
+    #
+    # @raise [RdkafkaTopicPartitionListError] When pausing subscription fails.
+    #
+    # @return [nil]
+    def pause(list)
+      unless list.is_a?(TopicPartitionList)
+        raise TypeError.new("list has to be a TopicPartitionList")
+      end
+
+      tpl = list.to_native_tpl
+
+      begin
+        response = Rdkafka::Bindings.rd_kafka_pause_partitions(@native_kafka, tpl)
+
+        if response != 0
+          list = TopicPartitionList.from_native_tpl(tpl)
+          raise Rdkafka::RdkafkaTopicPartitionListError.new(response, list, "Error pausing '#{list.to_h}'")
+        end
+      ensure
+        Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
+      end
+    end
+
+    # Resume producing consumption for the provided list of partitions
+    #
+    # @param list [TopicPartitionList] The topic with partitions to pause
+    #
+    # @raise [RdkafkaError] When resume subscription fails.
+    #
+    # @return [nil]
+    def resume(list)
+      unless list.is_a?(TopicPartitionList)
+        raise TypeError.new("list has to be a TopicPartitionList")
+      end
+
+      tpl = list.to_native_tpl
+
+      begin
+        response = Rdkafka::Bindings.rd_kafka_resume_partitions(@native_kafka, tpl)
+        if response != 0
+          raise Rdkafka::RdkafkaError.new(response, "Error resume '#{list.to_h}'")
+        end
+      ensure
+        Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
+      end
+    end
+
     # Return the current subscription to topics and partitions
     #
     # @raise [RdkafkaError] When getting the subscription fails.
     #
     # @return [TopicPartitionList]
     def subscription
-      tpl = FFI::MemoryPointer.new(:pointer)
-      tpl.autorelease = false
-      response = Rdkafka::Bindings.rd_kafka_subscription(@native_kafka, tpl)
+      ptr = FFI::MemoryPointer.new(:pointer)
+      response = Rdkafka::Bindings.rd_kafka_subscription(@native_kafka, ptr)
+
       if response != 0
         raise Rdkafka::RdkafkaError.new(response)
       end
-      Rdkafka::Consumer::TopicPartitionList.from_native_tpl(tpl.get_pointer(0))
+
+      native = ptr.read_pointer
+
+      begin
+        Rdkafka::Consumer::TopicPartitionList.from_native_tpl(native)
+      ensure
+        Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(native)
+      end
     end
 
     # Atomic assignment of partitions to consume
@@ -84,13 +145,17 @@ module Rdkafka
       unless list.is_a?(TopicPartitionList)
         raise TypeError.new("list has to be a TopicPartitionList")
       end
+
       tpl = list.to_native_tpl
-      response = Rdkafka::Bindings.rd_kafka_assign(@native_kafka, tpl)
-      if response != 0
-        raise Rdkafka::RdkafkaError.new(response, "Error assigning '#{list.to_h}'")
+
+      begin
+        response = Rdkafka::Bindings.rd_kafka_assign(@native_kafka, tpl)
+        if response != 0
+          raise Rdkafka::RdkafkaError.new(response, "Error assigning '#{list.to_h}'")
+        end
+      ensure
+        Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
       end
-    ensure
-      Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl) if tpl
     end
 
     # Returns the current partition assignment.
@@ -99,13 +164,23 @@ module Rdkafka
     #
     # @return [TopicPartitionList]
     def assignment
-      tpl = FFI::MemoryPointer.new(:pointer)
-      tpl.autorelease = false
-      response = Rdkafka::Bindings.rd_kafka_assignment(@native_kafka, tpl)
+      ptr = FFI::MemoryPointer.new(:pointer)
+      response = Rdkafka::Bindings.rd_kafka_assignment(@native_kafka, ptr)
       if response != 0
         raise Rdkafka::RdkafkaError.new(response)
       end
-      Rdkafka::Consumer::TopicPartitionList.from_native_tpl(tpl.get_pointer(0))
+
+      tpl = ptr.read_pointer
+
+      if !tpl.null?
+        begin
+          Rdkafka::Consumer::TopicPartitionList.from_native_tpl(tpl)
+        ensure
+          Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy tpl
+        end
+      end
+    ensure
+      ptr.free
     end
 
     # Return the current committed offset per partition for this consumer group.
@@ -117,18 +192,24 @@ module Rdkafka
     # @raise [RdkafkaError] When getting the committed positions fails.
     #
     # @return [TopicPartitionList]
-    def committed(list=nil, timeout_ms=200)
+    def committed(list=nil, timeout_ms=1200)
       if list.nil?
         list = assignment
       elsif !list.is_a?(TopicPartitionList)
         raise TypeError.new("list has to be nil or a TopicPartitionList")
       end
+
       tpl = list.to_native_tpl
-      response = Rdkafka::Bindings.rd_kafka_committed(@native_kafka, tpl, timeout_ms)
-      if response != 0
-        raise Rdkafka::RdkafkaError.new(response)
+
+      begin
+        response = Rdkafka::Bindings.rd_kafka_committed(@native_kafka, tpl, timeout_ms)
+        if response != 0
+          raise Rdkafka::RdkafkaError.new(response)
+        end
+        TopicPartitionList.from_native_tpl(tpl)
+      ensure
+        Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
       end
-      TopicPartitionList.from_native_tpl(tpl)
     end
 
     # Query broker for low (oldest/beginning) and high (newest/end) offsets for a partition.
@@ -150,13 +231,16 @@ module Rdkafka
         partition,
         low,
         high,
-        timeout_ms
+        timeout_ms,
       )
       if response != 0
         raise Rdkafka::RdkafkaError.new(response, "Error querying watermark offsets for partition #{partition} of #{topic}")
       end
 
-      return low.read_int64, high.read_int64
+      return low.read_array_of_int64(1).first, high.read_array_of_int64(1).first
+    ensure
+      low.free
+      high.free
     end
 
     # Calculate the consumer lag per partition for the provided topic partition list.
@@ -172,6 +256,7 @@ module Rdkafka
     # @return [Hash<String, Hash<Integer, Integer>>] A hash containing all topics with the lag per partition
     def lag(topic_partition_list, watermark_timeout_ms=100)
       out = {}
+
       topic_partition_list.to_h.each do |topic, partitions|
         # Query high watermarks for this topic's partitions
         # and compare to the offset in the list.
@@ -190,6 +275,22 @@ module Rdkafka
       out
     end
 
+    # Returns the ClusterId as reported in broker metadata.
+    #
+    # @return [String, nil]
+    def cluster_id
+      Rdkafka::Bindings.rd_kafka_clusterid(@native_kafka)
+    end
+
+    # Returns this client's broker-assigned group member id
+    #
+    # This currently requires the high-level KafkaConsumer
+    #
+    # @return [String, nil]
+    def member_id
+      Rdkafka::Bindings.rd_kafka_memberid(@native_kafka)
+    end
+
     # Store offset of a message to be used in the next commit of this consumer
     #
     # When using this `enable.auto.offset.store` should be set to `false` in the config.
@@ -221,29 +322,67 @@ module Rdkafka
       end
     end
 
-    # Commit the current offsets of this consumer
+    # Seek to a particular message. The next poll on the topic/partition will return the
+    # message at the given offset.
+    #
+    # @param message [Rdkafka::Consumer::Message] The message to which to seek
+    #
+    # @raise [RdkafkaError] When seeking fails
+    #
+    # @return [nil]
+    def seek(message)
+      # rd_kafka_offset_store is one of the few calls that does not support
+      # a string as the topic, so create a native topic for it.
+      native_topic = Rdkafka::Bindings.rd_kafka_topic_new(
+        @native_kafka,
+        message.topic,
+        nil
+      )
+      response = Rdkafka::Bindings.rd_kafka_seek(
+        native_topic,
+        message.partition,
+        message.offset,
+        0 # timeout
+      )
+      if response != 0
+        raise Rdkafka::RdkafkaError.new(response)
+      end
+    ensure
+      if native_topic && !native_topic.null?
+        Rdkafka::Bindings.rd_kafka_topic_destroy(native_topic)
+      end
+    end
+
+    # Manually commit the current offsets of this consumer.
+    #
+    # To use this set `enable.auto.commit`to `false` to disable automatic triggering
+    # of commits.
+    #
+    # If `enable.auto.offset.store` is set to `true` the offset of the last consumed
+    # message for every partition is used. If set to `false` you can use {store_offset} to
+    # indicate when a message has been fully processed.
     #
     # @param list [TopicPartitionList,nil] The topic with partitions to commit
     # @param async [Boolean] Whether to commit async or wait for the commit to finish
     #
-    # @raise [RdkafkaError] When comitting fails
+    # @raise [RdkafkaError] When committing fails
     #
     # @return [nil]
     def commit(list=nil, async=false)
       if !list.nil? && !list.is_a?(TopicPartitionList)
         raise TypeError.new("list has to be nil or a TopicPartitionList")
       end
-      tpl = if list
-              list.to_native_tpl
-            else
-              nil
-            end
-      response = Rdkafka::Bindings.rd_kafka_commit(@native_kafka, tpl, async)
-      if response != 0
-        raise Rdkafka::RdkafkaError.new(response)
+
+      tpl = list ? list.to_native_tpl : nil
+
+      begin
+        response = Rdkafka::Bindings.rd_kafka_commit(@native_kafka, tpl, async)
+        if response != 0
+          raise Rdkafka::RdkafkaError.new(response)
+        end
+      ensure
+        Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl) if tpl
       end
-    ensure
-      Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl) if tpl
     end
 
     # Poll for the next message on one of the subscribed topics
@@ -254,6 +393,8 @@ module Rdkafka
     #
     # @return [Message, nil] A message or nil if there was no new message within the timeout
     def poll(timeout_ms)
+      return unless @native_kafka
+
       message_ptr = Rdkafka::Bindings.rd_kafka_consumer_poll(@native_kafka, timeout_ms)
       if message_ptr.null?
         nil
@@ -277,16 +418,20 @@ module Rdkafka
     # Poll for new messages and yield for each received one. Iteration
     # will end when the consumer is closed.
     #
+    # If `enable.partition.eof` is turned on in the config this will raise an
+    # error when an eof is reached, so you probably want to disable that when
+    # using this method of iteration.
+    #
     # @raise [RdkafkaError] When polling fails
     #
     # @yieldparam message [Message] Received message
     #
     # @return [nil]
-    def each(&block)
+    def each
       loop do
         message = poll(250)
         if message
-          block.call(message)
+          yield(message)
         else
           if @closing
             break