karafka-rdkafka 0.12.4 → 0.13.0.beta1

data/lib/rdkafka/bindings.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "ffi"
 require "json"
 require "logger"
@@ -15,7 +17,7 @@ module Rdkafka
       end
     end
 
-    ffi_lib File.join(File.dirname(__FILE__), "../../ext/librdkafka.#{lib_extension}")
+    ffi_lib File.join(__dir__, "../../ext/librdkafka.#{lib_extension}")
 
     RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS = -175
     RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS = -174
@@ -33,15 +35,17 @@ module Rdkafka
 
     # Polling
 
+    attach_function :rd_kafka_flush, [:pointer, :int], :void, blocking: true
     attach_function :rd_kafka_poll, [:pointer, :int], :void, blocking: true
     attach_function :rd_kafka_outq_len, [:pointer], :int, blocking: true
 
     # Metadata
 
-    attach_function :rd_kafka_memberid, [:pointer], :string
-    attach_function :rd_kafka_clusterid, [:pointer], :string
-    attach_function :rd_kafka_metadata, [:pointer, :int, :pointer, :pointer, :int], :int
-    attach_function :rd_kafka_metadata_destroy, [:pointer], :void
+    attach_function :rd_kafka_name, [:pointer], :string, blocking: true
+    attach_function :rd_kafka_memberid, [:pointer], :string, blocking: true
+    attach_function :rd_kafka_clusterid, [:pointer], :string, blocking: true
+    attach_function :rd_kafka_metadata, [:pointer, :int, :pointer, :pointer, :int], :int, blocking: true
+    attach_function :rd_kafka_metadata_destroy, [:pointer], :void, blocking: true
 
     # Message struct
 
@@ -110,6 +114,7 @@ module Rdkafka
     attach_function :rd_kafka_conf_set_stats_cb, [:pointer, :stats_cb], :void
     callback :error_cb, [:pointer, :int, :string, :pointer], :void
     attach_function :rd_kafka_conf_set_error_cb, [:pointer, :error_cb], :void
+    attach_function :rd_kafka_rebalance_protocol, [:pointer], :string
 
     # Log queue
     attach_function :rd_kafka_set_log_queue, [:pointer, :pointer], :void
@@ -166,24 +171,28 @@ module Rdkafka
     ]
 
     attach_function :rd_kafka_new, [:kafka_type, :pointer, :pointer, :int], :pointer
+
     attach_function :rd_kafka_destroy, [:pointer], :void
 
     # Consumer
 
-    attach_function :rd_kafka_subscribe, [:pointer, :pointer], :int
-    attach_function :rd_kafka_unsubscribe, [:pointer], :int
-    attach_function :rd_kafka_subscription, [:pointer, :pointer], :int
-    attach_function :rd_kafka_assign, [:pointer, :pointer], :int
-    attach_function :rd_kafka_assignment, [:pointer, :pointer], :int
-    attach_function :rd_kafka_committed, [:pointer, :pointer, :int], :int
+    attach_function :rd_kafka_subscribe, [:pointer, :pointer], :int, blocking: true
+    attach_function :rd_kafka_unsubscribe, [:pointer], :int, blocking: true
+    attach_function :rd_kafka_subscription, [:pointer, :pointer], :int, blocking: true
+    attach_function :rd_kafka_assign, [:pointer, :pointer], :int, blocking: true
+    attach_function :rd_kafka_incremental_assign, [:pointer, :pointer], :int, blocking: true
+    attach_function :rd_kafka_incremental_unassign, [:pointer, :pointer], :int, blocking: true
+    attach_function :rd_kafka_assignment, [:pointer, :pointer], :int, blocking: true
+    attach_function :rd_kafka_assignment_lost, [:pointer], :int, blocking: true
+    attach_function :rd_kafka_committed, [:pointer, :pointer, :int], :int, blocking: true
     attach_function :rd_kafka_commit, [:pointer, :pointer, :bool], :int, blocking: true
-    attach_function :rd_kafka_poll_set_consumer, [:pointer], :void
+    attach_function :rd_kafka_poll_set_consumer, [:pointer], :void, blocking: true
     attach_function :rd_kafka_consumer_poll, [:pointer, :int], :pointer, blocking: true
     attach_function :rd_kafka_consumer_close, [:pointer], :void, blocking: true
-    attach_function :rd_kafka_offset_store, [:pointer, :int32, :int64], :int
-    attach_function :rd_kafka_pause_partitions, [:pointer, :pointer], :int
-    attach_function :rd_kafka_resume_partitions, [:pointer, :pointer], :int
-    attach_function :rd_kafka_seek, [:pointer, :int32, :int64, :int], :int
+    attach_function :rd_kafka_offset_store, [:pointer, :int32, :int64], :int, blocking: true
+    attach_function :rd_kafka_pause_partitions, [:pointer, :pointer], :int, blocking: true
+    attach_function :rd_kafka_resume_partitions, [:pointer, :pointer], :int, blocking: true
+    attach_function :rd_kafka_seek, [:pointer, :int32, :int64, :int], :int, blocking: true
 
     # Headers
     attach_function :rd_kafka_header_get_all, [:pointer, :size_t, :pointer, :pointer, SizePtr], :int
@@ -192,30 +201,36 @@ module Rdkafka
     # Rebalance
 
     callback :rebalance_cb_function, [:pointer, :int, :pointer, :pointer], :void
-    attach_function :rd_kafka_conf_set_rebalance_cb, [:pointer, :rebalance_cb_function], :void
+    attach_function :rd_kafka_conf_set_rebalance_cb, [:pointer, :rebalance_cb_function], :void, blocking: true
 
     RebalanceCallback = FFI::Function.new(
       :void, [:pointer, :int, :pointer, :pointer]
     ) do |client_ptr, code, partitions_ptr, opaque_ptr|
       case code
       when RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS
-        Rdkafka::Bindings.rd_kafka_assign(client_ptr, partitions_ptr)
+        if Rdkafka::Bindings.rd_kafka_rebalance_protocol(client_ptr) == "COOPERATIVE"
+          Rdkafka::Bindings.rd_kafka_incremental_assign(client_ptr, partitions_ptr)
+        else
+          Rdkafka::Bindings.rd_kafka_assign(client_ptr, partitions_ptr)
+        end
       else # RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS or errors
-        Rdkafka::Bindings.rd_kafka_assign(client_ptr, FFI::Pointer::NULL)
+        if Rdkafka::Bindings.rd_kafka_rebalance_protocol(client_ptr) == "COOPERATIVE"
+          Rdkafka::Bindings.rd_kafka_incremental_unassign(client_ptr, partitions_ptr)
+        else
+          Rdkafka::Bindings.rd_kafka_assign(client_ptr, FFI::Pointer::NULL)
+        end
       end
 
       opaque = Rdkafka::Config.opaques[opaque_ptr.to_i]
       return unless opaque
 
       tpl = Rdkafka::Consumer::TopicPartitionList.from_native_tpl(partitions_ptr).freeze
-      consumer = Rdkafka::Consumer.new(client_ptr)
-
       begin
         case code
         when RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS
-          opaque.call_on_partitions_assigned(consumer, tpl)
+          opaque.call_on_partitions_assigned(tpl)
         when RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS
-          opaque.call_on_partitions_revoked(consumer, tpl)
+          opaque.call_on_partitions_revoked(tpl)
         end
       rescue Exception => err
         Rdkafka::Config.logger.error("Unhandled exception: #{err.class} - #{err.message}")
@@ -242,7 +257,7 @@ module Rdkafka
 
     RD_KAFKA_MSG_F_COPY = 0x2
 
-    attach_function :rd_kafka_producev, [:pointer, :varargs], :int
+    attach_function :rd_kafka_producev, [:pointer, :varargs], :int, blocking: true
     callback :delivery_cb, [:pointer, :pointer, :pointer], :void
     attach_function :rd_kafka_conf_set_dr_msg_cb, [:pointer, :delivery_cb], :void
 
@@ -257,11 +272,11 @@ module Rdkafka
       # Return RD_KAFKA_PARTITION_UA(unassigned partition) when partition count is nil/zero.
       return -1 unless partition_count&.nonzero?
 
-      str_ptr = FFI::MemoryPointer.from_string(str)
+      str_ptr = str.empty? ? FFI::MemoryPointer::NULL : FFI::MemoryPointer.from_string(str)
       method_name = PARTITIONERS.fetch(partitioner_name) do
         raise Rdkafka::Config::ConfigError.new("Unknown partitioner: #{partitioner_name}")
       end
-      public_send(method_name, nil, str_ptr, str.size, partition_count, nil, nil)
+      public_send(method_name, nil, str_ptr, str.size > 0 ? str.size : 1, partition_count, nil, nil)
     end
 
     # Create Topics
@@ -269,23 +284,23 @@ module Rdkafka
     RD_KAFKA_ADMIN_OP_CREATETOPICS     = 1   # rd_kafka_admin_op_t
     RD_KAFKA_EVENT_CREATETOPICS_RESULT = 100 # rd_kafka_event_type_t
 
-    attach_function :rd_kafka_CreateTopics, [:pointer, :pointer, :size_t, :pointer, :pointer], :void
-    attach_function :rd_kafka_NewTopic_new, [:pointer, :size_t, :size_t, :pointer, :size_t], :pointer
-    attach_function :rd_kafka_NewTopic_set_config, [:pointer, :string, :string], :int32
-    attach_function :rd_kafka_NewTopic_destroy, [:pointer], :void
-    attach_function :rd_kafka_event_CreateTopics_result, [:pointer], :pointer
-    attach_function :rd_kafka_CreateTopics_result_topics, [:pointer, :pointer], :pointer
+    attach_function :rd_kafka_CreateTopics, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
+    attach_function :rd_kafka_NewTopic_new, [:pointer, :size_t, :size_t, :pointer, :size_t], :pointer, blocking: true
+    attach_function :rd_kafka_NewTopic_set_config, [:pointer, :string, :string], :int32, blocking: true
+    attach_function :rd_kafka_NewTopic_destroy, [:pointer], :void, blocking: true
+    attach_function :rd_kafka_event_CreateTopics_result, [:pointer], :pointer, blocking: true
+    attach_function :rd_kafka_CreateTopics_result_topics, [:pointer, :pointer], :pointer, blocking: true
 
     # Delete Topics
 
     RD_KAFKA_ADMIN_OP_DELETETOPICS     = 2   # rd_kafka_admin_op_t
     RD_KAFKA_EVENT_DELETETOPICS_RESULT = 101 # rd_kafka_event_type_t
 
-    attach_function :rd_kafka_DeleteTopics, [:pointer, :pointer, :size_t, :pointer, :pointer], :int32
-    attach_function :rd_kafka_DeleteTopic_new, [:pointer], :pointer
-    attach_function :rd_kafka_DeleteTopic_destroy, [:pointer], :void
-    attach_function :rd_kafka_event_DeleteTopics_result, [:pointer], :pointer
-    attach_function :rd_kafka_DeleteTopics_result_topics, [:pointer, :pointer], :pointer
+    attach_function :rd_kafka_DeleteTopics, [:pointer, :pointer, :size_t, :pointer, :pointer], :int32, blocking: true
+    attach_function :rd_kafka_DeleteTopic_new, [:pointer], :pointer, blocking: true
+    attach_function :rd_kafka_DeleteTopic_destroy, [:pointer], :void, blocking: true
+    attach_function :rd_kafka_event_DeleteTopics_result, [:pointer], :pointer, blocking: true
+    attach_function :rd_kafka_DeleteTopics_result_topics, [:pointer, :pointer], :pointer, blocking: true
 
     # Create partitions
     RD_KAFKA_ADMIN_OP_CREATEPARTITIONS = 3

data/lib/rdkafka/callbacks.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rdkafka
   module Callbacks
 

data/lib/rdkafka/config.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "logger"
 
 module Rdkafka
@@ -30,7 +32,6 @@ module Rdkafka
       @@logger
     end
 
-
     # Returns a queue whose contents will be passed to the configured logger. Each entry
     # should follow the format [Logger::Severity, String]. The benefit over calling the
     # logger directly is that this is safe to use from trap contexts.
@@ -47,7 +48,7 @@ module Rdkafka
     # @return [nil]
     def self.logger=(logger)
       raise NoLoggerError if logger.nil?
-      @@logger=logger
+      @@logger = logger
     end
 
     # Set a callback that will be called every time the underlying client emits statistics.
@@ -156,13 +157,14 @@ module Rdkafka
         Rdkafka::Bindings.rd_kafka_conf_set_rebalance_cb(config, Rdkafka::Bindings::RebalanceCallback)
       end
 
+      # Create native client
       kafka = native_kafka(config, :rd_kafka_consumer)
 
       # Redirect the main queue to the consumer
       Rdkafka::Bindings.rd_kafka_poll_set_consumer(kafka)
 
       # Return consumer with Kafka client
-      Rdkafka::Consumer.new(kafka)
+      Rdkafka::Consumer.new(Rdkafka::NativeKafka.new(kafka, run_polling_thread: false))
     end
 
     # Create a producer with this configuration.
@@ -179,7 +181,8 @@ module Rdkafka
       # Set callback to receive delivery reports on config
       Rdkafka::Bindings.rd_kafka_conf_set_dr_msg_cb(config, Rdkafka::Callbacks::DeliveryCallbackFunction)
       # Return producer with Kafka client
-      Rdkafka::Producer.new(Rdkafka::Producer::Client.new(native_kafka(config, :rd_kafka_producer)), self[:partitioner]).tap do |producer|
+      partitioner_name = self[:partitioner] || self["partitioner"]
+      Rdkafka::Producer.new(Rdkafka::NativeKafka.new(native_kafka(config, :rd_kafka_producer), run_polling_thread: true), partitioner_name).tap do |producer|
         opaque.producer = producer
       end
     end
@@ -194,7 +197,7 @@ module Rdkafka
       opaque = Opaque.new
       config = native_config(opaque)
       Rdkafka::Bindings.rd_kafka_conf_set_background_event_cb(config, Rdkafka::Callbacks::BackgroundEventCallbackFunction)
-      Rdkafka::Admin.new(native_kafka(config, :rd_kafka_producer))
+      Rdkafka::Admin.new(Rdkafka::NativeKafka.new(native_kafka(config, :rd_kafka_producer), run_polling_thread: true))
     end
 
     # Error that is returned by the underlying rdkafka error if an invalid configuration option is present.
@@ -210,7 +213,7 @@ module Rdkafka
 
     # This method is only intended to be used to create a client,
     # using it in another way will leak memory.
-    def native_config(opaque=nil)
+    def native_config(opaque = nil)
       Rdkafka::Bindings.rd_kafka_conf_new.tap do |config|
         # Create config
         @config_hash.merge(REQUIRED_CONFIG).each do |key, value|
@@ -282,18 +285,18 @@ module Rdkafka
       producer.call_delivery_callback(delivery_report, delivery_handle) if producer
     end
 
-    def call_on_partitions_assigned(consumer, list)
+    def call_on_partitions_assigned(list)
       return unless consumer_rebalance_listener
       return unless consumer_rebalance_listener.respond_to?(:on_partitions_assigned)
 
-      consumer_rebalance_listener.on_partitions_assigned(consumer, list)
+      consumer_rebalance_listener.on_partitions_assigned(list)
     end
 
-    def call_on_partitions_revoked(consumer, list)
+    def call_on_partitions_revoked(list)
       return unless consumer_rebalance_listener
       return unless consumer_rebalance_listener.respond_to?(:on_partitions_revoked)
 
-      consumer_rebalance_listener.on_partitions_revoked(consumer, list)
+      consumer_rebalance_listener.on_partitions_revoked(list)
     end
   end
 end

data/lib/rdkafka/consumer/headers.rb

@@ -1,10 +1,26 @@
+# frozen_string_literal: true
+
 module Rdkafka
   class Consumer
-    # A message headers
-    class Headers
-      # Reads a native kafka's message header into ruby's hash
+    # Interface to return headers for a consumer message
+    module Headers
+      class HashWithSymbolKeysTreatedLikeStrings < Hash
+        def [](key)
+          if key.is_a?(Symbol)
+            Kernel.warn("rdkafka deprecation warning: header access with Symbol key #{key.inspect} treated as a String. " \
+                        "Please change your code to use String keys to avoid this warning. Symbol keys will break in version 1.")
+            super(key.to_s)
+          else
+            super
+          end
+        end
+      end
+
+      # Reads a librdkafka native message's headers and returns them as a Ruby Hash
+      #
+      # @param [librdkakfa message] native_message
       #
-      # @return [Hash<String, String>] a message headers
+      # @return [Hash<String, String>] headers Hash for the native_message
       #
       # @raise [Rdkafka::RdkafkaError] when fail to read headers
       #
@@ -24,7 +40,8 @@ module Rdkafka
         name_ptrptr = FFI::MemoryPointer.new(:pointer)
         value_ptrptr = FFI::MemoryPointer.new(:pointer)
         size_ptr = Rdkafka::Bindings::SizePtr.new
-        headers = {}
+
+        headers = HashWithSymbolKeysTreatedLikeStrings.new
 
         idx = 0
         loop do
@@ -51,12 +68,12 @@ module Rdkafka
 
           value = value_ptr.read_string(size)
 
-          headers[name.to_sym] = value
+          headers[name] = value
 
           idx += 1
         end
 
-        headers
+        headers.freeze
       end
     end
   end

data/lib/rdkafka/consumer/message.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rdkafka
   class Consumer
     # A message that was consumed from a topic.
@@ -18,7 +20,7 @@ module Rdkafka
       # @return [String, nil]
       attr_reader :key
 
-      # This message's offset in it's partition
+      # This message's offset in its partition
       # @return [Integer]
       attr_reader :offset
 

data/lib/rdkafka/consumer/partition.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rdkafka
   class Consumer
     # Information about a partition, used in {TopicPartitionList}.

data/lib/rdkafka/consumer/topic_partition_list.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rdkafka
   class Consumer
     # A list of topics with their partition information

data/lib/rdkafka/consumer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rdkafka
   # A consumer of Kafka messages. It uses the high-level consumer approach where the Kafka
   # brokers automatically assign partitions and load balance partitions over consumers that
@@ -14,18 +16,33 @@ module Rdkafka
     # @private
     def initialize(native_kafka)
       @native_kafka = native_kafka
-      @closing = false
+    end
+
+    # @return [String] consumer name
+    def name
+      @name ||= @native_kafka.with_inner do |inner|
+        ::Rdkafka::Bindings.rd_kafka_name(inner)
+      end
+    end
+
+    def finalizer
+      ->(_) { close }
     end
 
     # Close this consumer
     # @return [nil]
     def close
-      return unless @native_kafka
+      return if closed?
+      ObjectSpace.undefine_finalizer(self)
+      @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_consumer_close(inner)
+      end
+      @native_kafka.close
+    end
 
-      @closing = true
-      Rdkafka::Bindings.rd_kafka_consumer_close(@native_kafka)
-      Rdkafka::Bindings.rd_kafka_destroy(@native_kafka)
-      @native_kafka = nil
+    # Whether this consumer has closed
+    def closed?
+      @native_kafka.closed?
     end
 
     # Subscribe to one or more topics letting Kafka handle partition assignments.
@@ -46,7 +63,9 @@ module Rdkafka
       end
 
       # Subscribe to topic partition list and check this was successful
-      response = Rdkafka::Bindings.rd_kafka_subscribe(@native_kafka, tpl)
+      response = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_subscribe(inner, tpl)
+      end
       if response != 0
         raise Rdkafka::RdkafkaError.new(response, "Error subscribing to '#{topics.join(', ')}'")
       end
@@ -62,7 +81,9 @@ module Rdkafka
     def unsubscribe
       closed_consumer_check(__method__)
 
-      response = Rdkafka::Bindings.rd_kafka_unsubscribe(@native_kafka)
+      response = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_unsubscribe(inner)
+      end
       if response != 0
         raise Rdkafka::RdkafkaError.new(response)
       end
@@ -85,7 +106,9 @@ module Rdkafka
       tpl = list.to_native_tpl
 
       begin
-        response = Rdkafka::Bindings.rd_kafka_pause_partitions(@native_kafka, tpl)
+        response = @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_pause_partitions(inner, tpl)
+        end
 
         if response != 0
           list = TopicPartitionList.from_native_tpl(tpl)
@@ -113,7 +136,9 @@ module Rdkafka
       tpl = list.to_native_tpl
 
       begin
-        response = Rdkafka::Bindings.rd_kafka_resume_partitions(@native_kafka, tpl)
+        response = @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_resume_partitions(inner, tpl)
+        end
         if response != 0
           raise Rdkafka::RdkafkaError.new(response, "Error resume '#{list.to_h}'")
         end
@@ -131,7 +156,9 @@ module Rdkafka
       closed_consumer_check(__method__)
 
       ptr = FFI::MemoryPointer.new(:pointer)
-      response = Rdkafka::Bindings.rd_kafka_subscription(@native_kafka, ptr)
+      response = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_subscription(inner, ptr)
+      end
 
       if response != 0
         raise Rdkafka::RdkafkaError.new(response)
@@ -161,7 +188,9 @@ module Rdkafka
       tpl = list.to_native_tpl
 
       begin
-        response = Rdkafka::Bindings.rd_kafka_assign(@native_kafka, tpl)
+        response = @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_assign(inner, tpl)
+        end
         if response != 0
           raise Rdkafka::RdkafkaError.new(response, "Error assigning '#{list.to_h}'")
         end
@@ -179,7 +208,9 @@ module Rdkafka
       closed_consumer_check(__method__)
 
       ptr = FFI::MemoryPointer.new(:pointer)
-      response = Rdkafka::Bindings.rd_kafka_assignment(@native_kafka, ptr)
+      response = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_assignment(inner, ptr)
+      end
       if response != 0
         raise Rdkafka::RdkafkaError.new(response)
       end
@@ -197,6 +228,15 @@ module Rdkafka
       ptr.free unless ptr.nil?
     end
 
+    # @return [Boolean] true if our current assignment has been lost involuntarily.
+    def assignment_lost?
+      closed_consumer_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        !Rdkafka::Bindings.rd_kafka_assignment_lost(inner).zero?
+      end
+    end
+
     # Return the current committed offset per partition for this consumer group.
     # The offset field of each requested partition will either be set to stored offset or to -1001 in case there was no stored offset for that partition.
     #
@@ -218,7 +258,9 @@ module Rdkafka
       tpl = list.to_native_tpl
 
       begin
-        response = Rdkafka::Bindings.rd_kafka_committed(@native_kafka, tpl, timeout_ms)
+        response = @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_committed(inner, tpl, timeout_ms)
+        end
         if response != 0
           raise Rdkafka::RdkafkaError.new(response)
         end
@@ -243,14 +285,16 @@ module Rdkafka
       low = FFI::MemoryPointer.new(:int64, 1)
       high = FFI::MemoryPointer.new(:int64, 1)
 
-      response = Rdkafka::Bindings.rd_kafka_query_watermark_offsets(
-        @native_kafka,
-        topic,
-        partition,
-        low,
-        high,
-        timeout_ms,
-      )
+      response = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_query_watermark_offsets(
+          inner,
+          topic,
+          partition,
+          low,
+          high,
+          timeout_ms,
+        )
+      end
       if response != 0
         raise Rdkafka::RdkafkaError.new(response, "Error querying watermark offsets for partition #{partition} of #{topic}")
       end
@@ -298,7 +342,9 @@ module Rdkafka
     # @return [String, nil]
     def cluster_id
       closed_consumer_check(__method__)
-      Rdkafka::Bindings.rd_kafka_clusterid(@native_kafka)
+      @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_clusterid(inner)
+      end
     end
 
     # Returns this client's broker-assigned group member id
@@ -308,7 +354,9 @@ module Rdkafka
     # @return [String, nil]
     def member_id
       closed_consumer_check(__method__)
-      Rdkafka::Bindings.rd_kafka_memberid(@native_kafka)
+      @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_memberid(inner)
+      end
     end
 
     # Store offset of a message to be used in the next commit of this consumer
@@ -325,11 +373,13 @@ module Rdkafka
 
       # rd_kafka_offset_store is one of the few calls that does not support
       # a string as the topic, so create a native topic for it.
-      native_topic = Rdkafka::Bindings.rd_kafka_topic_new(
-        @native_kafka,
-        message.topic,
-        nil
-      )
+      native_topic = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_topic_new(
+          inner,
+          message.topic,
+          nil
+        )
+      end
       response = Rdkafka::Bindings.rd_kafka_offset_store(
         native_topic,
         message.partition,
@@ -357,11 +407,13 @@ module Rdkafka
 
       # rd_kafka_offset_store is one of the few calls that does not support
       # a string as the topic, so create a native topic for it.
-      native_topic = Rdkafka::Bindings.rd_kafka_topic_new(
-        @native_kafka,
-        message.topic,
-        nil
-      )
+      native_topic = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_topic_new(
+          inner,
+          message.topic,
+          nil
+        )
+      end
       response = Rdkafka::Bindings.rd_kafka_seek(
         native_topic,
         message.partition,
@@ -402,7 +454,9 @@ module Rdkafka
       tpl = list ? list.to_native_tpl : nil
 
       begin
-        response = Rdkafka::Bindings.rd_kafka_commit(@native_kafka, tpl, async)
+        response = @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_commit(inner, tpl, async)
+        end
         if response != 0
           raise Rdkafka::RdkafkaError.new(response)
         end
@@ -421,7 +475,9 @@ module Rdkafka
     def poll(timeout_ms)
       closed_consumer_check(__method__)
 
-      message_ptr = Rdkafka::Bindings.rd_kafka_consumer_poll(@native_kafka, timeout_ms)
+      message_ptr = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_consumer_poll(inner, timeout_ms)
+      end
       if message_ptr.null?
         nil
       else
@@ -436,7 +492,7 @@ module Rdkafka
       end
     ensure
       # Clean up rdkafka message if there is one
-      if !message_ptr.nil? && !message_ptr.null?
+      if message_ptr && !message_ptr.null?
         Rdkafka::Bindings.rd_kafka_message_destroy(message_ptr)
       end
     end
@@ -459,7 +515,7 @@ module Rdkafka
         if message
           yield(message)
         else
-          if @closing
+          if closed?
             break
           else
             next
@@ -468,10 +524,6 @@ module Rdkafka
       end
     end
 
-    def closed_consumer_check(method)
-      raise Rdkafka::ClosedConsumerError.new(method) if @native_kafka.nil?
-    end
-
     # Poll for new messages and yield them in batches that may contain
     # messages from more than one partition.
     #
@@ -527,7 +579,7 @@ module Rdkafka
       bytes = 0
       end_time = monotonic_now + timeout_ms / 1000.0
       loop do
-        break if @closing
+        break if closed?
         max_wait = end_time - monotonic_now
         max_wait_ms = if max_wait <= 0
                         0  # should not block, but may retrieve a message
@@ -545,7 +597,7 @@ module Rdkafka
         end
         if message
           slice << message
-          bytes += message.payload.bytesize
+          bytes += message.payload.bytesize if message.payload
         end
         if slice.size == max_items || bytes >= bytes_threshold || monotonic_now >= end_time - 0.001
           yield slice.dup, nil
@@ -561,5 +613,9 @@ module Rdkafka
       # needed because Time.now can go backwards
       Process.clock_gettime(Process::CLOCK_MONOTONIC)
     end
+
+    def closed_consumer_check(method)
+      raise Rdkafka::ClosedConsumerError.new(method) if closed?
+    end
   end
 end