rdkafka 0.12.0 → 0.15.0

data/lib/rdkafka/config.rb

@@ -1,9 +1,9 @@
-require "logger"
+# frozen_string_literal: true
 
 module Rdkafka
   # Configuration for a Kafka consumer or producer. You can create an instance and use
   # the consumer and producer methods to create a client. Documentation of the available
-  # configuration options is available on https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md.
+  # configuration options is available on https://github.com/confluentinc/librdkafka/blob/master/CONFIGURATION.md.
   class Config
     # @private
     @@logger = Logger.new(STDOUT)
@@ -12,7 +12,7 @@ module Rdkafka
     # @private
     @@error_callback = nil
     # @private
-    @@opaques = {}
+    @@opaques = ObjectSpace::WeakMap.new
     # @private
     @@log_queue = Queue.new
 
@@ -30,7 +30,6 @@ module Rdkafka
       @@logger
     end
 
-
     # Returns a queue whose contents will be passed to the configured logger. Each entry
     # should follow the format [Logger::Severity, String]. The benefit over calling the
     # logger directly is that this is safe to use from trap contexts.
@@ -47,18 +46,18 @@ module Rdkafka
     # @return [nil]
     def self.logger=(logger)
       raise NoLoggerError if logger.nil?
-      @@logger=logger
+      @@logger = logger
     end
 
     # Set a callback that will be called every time the underlying client emits statistics.
     # You can configure if and how often this happens using `statistics.interval.ms`.
-    # The callback is called with a hash that's documented here: https://github.com/edenhill/librdkafka/blob/master/STATISTICS.md
+    # The callback is called with a hash that's documented here: https://github.com/confluentinc/librdkafka/blob/master/STATISTICS.md
     #
     # @param callback [Proc, #call] The callback
     #
     # @return [nil]
     def self.statistics_callback=(callback)
-      raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call)
+      raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call) || callback == nil
       @@statistics_callback = callback
     end
 
@@ -113,6 +112,7 @@ module Rdkafka
     def initialize(config_hash = {})
       @config_hash = DEFAULT_CONFIG.merge(config_hash)
       @consumer_rebalance_listener = nil
+      @consumer_poll_set = true
     end
 
     # Set a config option.
@@ -141,12 +141,28 @@ module Rdkafka
       @consumer_rebalance_listener = listener
     end
 
-    # Create a consumer with this configuration.
+    # Should we use a single queue for the underlying consumer and events.
     #
-    # @raise [ConfigError] When the configuration contains invalid options
-    # @raise [ClientCreationError] When the native client cannot be created
+    # This is an advanced API that allows for more granular control of the polling process.
+    # When this value is set to `false` (`true` by defualt), there will be two queues that need to
+    # be polled:
+    #   - main librdkafka queue for events
+    #   - consumer queue with messages and rebalances
+    #
+    # It is recommended to use the defaults and only set it to `false` in advance multi-threaded
+    # and complex cases where granular events handling control is needed.
+    #
+    # @param poll_set [Boolean]
+    def consumer_poll_set=(poll_set)
+      @consumer_poll_set = poll_set
+    end
+
+    # Creates a consumer with this configuration.
     #
     # @return [Consumer] The created consumer
+    #
+    # @raise [ConfigError] When the configuration contains invalid options
+    # @raise [ClientCreationError] When the native client cannot be created
     def consumer
       opaque = Opaque.new
       config = native_config(opaque)
@@ -156,21 +172,28 @@ module Rdkafka
         Rdkafka::Bindings.rd_kafka_conf_set_rebalance_cb(config, Rdkafka::Bindings::RebalanceCallback)
       end
 
+      # Create native client
       kafka = native_kafka(config, :rd_kafka_consumer)
 
-      # Redirect the main queue to the consumer
-      Rdkafka::Bindings.rd_kafka_poll_set_consumer(kafka)
+      # Redirect the main queue to the consumer queue
+      Rdkafka::Bindings.rd_kafka_poll_set_consumer(kafka) if @consumer_poll_set
 
       # Return consumer with Kafka client
-      Rdkafka::Consumer.new(kafka)
+      Rdkafka::Consumer.new(
+        Rdkafka::NativeKafka.new(
+          kafka,
+          run_polling_thread: false,
+          opaque: opaque
+        )
+      )
     end
 
     # Create a producer with this configuration.
     #
+    # @return [Producer] The created producer
+    #
     # @raise [ConfigError] When the configuration contains invalid options
     # @raise [ClientCreationError] When the native client cannot be created
-    #
-    # @return [Producer] The created producer
     def producer
       # Create opaque
       opaque = Opaque.new
@@ -179,22 +202,36 @@ module Rdkafka
       # Set callback to receive delivery reports on config
       Rdkafka::Bindings.rd_kafka_conf_set_dr_msg_cb(config, Rdkafka::Callbacks::DeliveryCallbackFunction)
       # Return producer with Kafka client
-      Rdkafka::Producer.new(Rdkafka::Producer::Client.new(native_kafka(config, :rd_kafka_producer)), self[:partitioner]).tap do |producer|
+      partitioner_name = self[:partitioner] || self["partitioner"]
+      Rdkafka::Producer.new(
+        Rdkafka::NativeKafka.new(
+          native_kafka(config, :rd_kafka_producer),
+          run_polling_thread: true,
+          opaque: opaque
+        ),
+        partitioner_name
+      ).tap do |producer|
         opaque.producer = producer
       end
     end
 
-    # Create an admin instance with this configuration.
+    # Creates an admin instance with this configuration.
+    #
+    # @return [Admin] The created admin instance
     #
     # @raise [ConfigError] When the configuration contains invalid options
     # @raise [ClientCreationError] When the native client cannot be created
-    #
-    # @return [Admin] The created admin instance
     def admin
       opaque = Opaque.new
       config = native_config(opaque)
       Rdkafka::Bindings.rd_kafka_conf_set_background_event_cb(config, Rdkafka::Callbacks::BackgroundEventCallbackFunction)
-      Rdkafka::Admin.new(native_kafka(config, :rd_kafka_producer))
+      Rdkafka::Admin.new(
+        Rdkafka::NativeKafka.new(
+          native_kafka(config, :rd_kafka_producer),
+          run_polling_thread: true,
+          opaque: opaque
+        )
+      )
     end
 
     # Error that is returned by the underlying rdkafka error if an invalid configuration option is present.
@@ -210,7 +247,7 @@ module Rdkafka
 
     # This method is only intended to be used to create a client,
     # using it in another way will leak memory.
-    def native_config(opaque=nil)
+    def native_config(opaque = nil)
       Rdkafka::Bindings.rd_kafka_conf_new.tap do |config|
         # Create config
         @config_hash.merge(REQUIRED_CONFIG).each do |key, value|
@@ -282,18 +319,18 @@ module Rdkafka
       producer.call_delivery_callback(delivery_report, delivery_handle) if producer
     end
 
-    def call_on_partitions_assigned(consumer, list)
+    def call_on_partitions_assigned(list)
       return unless consumer_rebalance_listener
       return unless consumer_rebalance_listener.respond_to?(:on_partitions_assigned)
 
-      consumer_rebalance_listener.on_partitions_assigned(consumer, list)
+      consumer_rebalance_listener.on_partitions_assigned(list)
     end
 
-    def call_on_partitions_revoked(consumer, list)
+    def call_on_partitions_revoked(list)
       return unless consumer_rebalance_listener
       return unless consumer_rebalance_listener.respond_to?(:on_partitions_revoked)
 
-      consumer_rebalance_listener.on_partitions_revoked(consumer, list)
+      consumer_rebalance_listener.on_partitions_revoked(list)
     end
   end
 end

data/lib/rdkafka/consumer/headers.rb

@@ -1,14 +1,28 @@
+# frozen_string_literal: true
+
 module Rdkafka
   class Consumer
-    # A message headers
-    class Headers
-      # Reads a native kafka's message header into ruby's hash
+    # Interface to return headers for a consumer message
+    module Headers
+      class HashWithSymbolKeysTreatedLikeStrings < Hash
+        def [](key)
+          if key.is_a?(Symbol)
+            Kernel.warn("rdkafka deprecation warning: header access with Symbol key #{key.inspect} treated as a String. " \
+                        "Please change your code to use String keys to avoid this warning. Symbol keys will break in version 1.")
+            super(key.to_s)
+          else
+            super
+          end
+        end
+      end
+
+      # Reads a librdkafka native message's headers and returns them as a Ruby Hash
       #
-      # @return [Hash<String, String>] a message headers
+      # @private
       #
+      # @param [Rdkafka::Bindings::Message] native_message
+      # @return [Hash<String, String>] headers Hash for the native_message
       # @raise [Rdkafka::RdkafkaError] when fail to read headers
-      #
-      # @private
       def self.from_native(native_message)
         headers_ptrptr = FFI::MemoryPointer.new(:pointer)
         err = Rdkafka::Bindings.rd_kafka_message_headers(native_message, headers_ptrptr)
@@ -24,7 +38,8 @@ module Rdkafka
         name_ptrptr = FFI::MemoryPointer.new(:pointer)
         value_ptrptr = FFI::MemoryPointer.new(:pointer)
         size_ptr = Rdkafka::Bindings::SizePtr.new
-        headers = {}
+
+        headers = HashWithSymbolKeysTreatedLikeStrings.new
 
         idx = 0
         loop do
@@ -51,12 +66,12 @@ module Rdkafka
 
           value = value_ptr.read_string(size)
 
-          headers[name.to_sym] = value
+          headers[name] = value
 
           idx += 1
         end
 
-        headers
+        headers.freeze
       end
     end
   end

data/lib/rdkafka/consumer/message.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rdkafka
   class Consumer
     # A message that was consumed from a topic.
@@ -18,7 +20,7 @@ module Rdkafka
       # @return [String, nil]
       attr_reader :key
 
-      # This message's offset in it's partition
+      # This message's offset in its partition
       # @return [Integer]
       attr_reader :offset
 

data/lib/rdkafka/consumer/partition.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rdkafka
   class Consumer
     # Information about a partition, used in {TopicPartitionList}.

data/lib/rdkafka/consumer/topic_partition_list.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rdkafka
   class Consumer
     # A list of topics with their partition information
@@ -34,6 +36,11 @@ module Rdkafka
       # Add a topic with optionally partitions to the list.
       # Calling this method multiple times for the same topic will overwrite the previous configuraton.
       #
+      # @param topic [String] The topic's name
+      # @param partitions [Array<Integer>, Range<Integer>, Integer] The topic's partitions or partition count
+      #
+      # @return [nil]
+      #
       # @example Add a topic with unassigned partitions
       #   tpl.add_topic("topic")
       #
@@ -43,10 +50,6 @@ module Rdkafka
       # @example Add a topic with all topics up to a count
       #   tpl.add_topic("topic", 9)
       #
-      # @param topic [String] The topic's name
-      # @param partitions [Array<Integer>, Range<Integer>, Integer] The topic's partitions or partition count
-      #
-      # @return [nil]
       def add_topic(topic, partitions=nil)
         if partitions.nil?
           @data[topic.to_s] = nil
@@ -88,11 +91,11 @@ module Rdkafka
 
       # Create a new topic partition list based of a native one.
       #
+      # @private
+      #
       # @param pointer [FFI::Pointer] Optional pointer to an existing native list. Its contents will be copied.
       #
       # @return [TopicPartitionList]
-      #
-      # @private
       def self.from_native_tpl(pointer)
         # Data to be moved into the tpl
         data = {}
@@ -125,8 +128,8 @@ module Rdkafka
       #
       # The pointer will be cleaned by `rd_kafka_topic_partition_list_destroy` when GC releases it.
       #
-      # @return [FFI::Pointer]
       # @private
+      # @return [FFI::Pointer]
       def to_native_tpl
         tpl = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(count)
 
@@ -140,11 +143,13 @@ module Rdkafka
               )
 
               if p.offset
+                offset = p.offset.is_a?(Time) ? p.offset.to_f * 1_000 : p.offset
+
                 Rdkafka::Bindings.rd_kafka_topic_partition_list_set_offset(
                   tpl,
                   topic,
                   p.partition,
-                  p.offset
+                  offset
                 )
               end
             end