rdkafka 0.13.0 → 0.15.1

data/lib/rdkafka/callbacks.rb

@@ -23,6 +23,96 @@ module Rdkafka
       end
     end
 
+    class GroupResult
+      attr_reader :result_error, :error_string, :result_name
+      def initialize(group_result_pointer)
+        native_error = Rdkafka::Bindings.rd_kafka_group_result_error(group_result_pointer)
+
+        if native_error.null?
+          @result_error = 0
+          @error_string = FFI::Pointer::NULL
+        else
+          @result_error = native_error[:code]
+          @error_string = native_error[:errstr]
+        end
+
+        @result_name = Rdkafka::Bindings.rd_kafka_group_result_name(group_result_pointer)
+      end
+      def self.create_group_results_from_array(count, array_pointer)
+        (1..count).map do |index|
+          result_pointer = (array_pointer + (index - 1)).read_pointer
+          new(result_pointer)
+        end
+      end
+    end
+
+    # Extracts attributes of rd_kafka_acl_result_t
+    #
+    # @private
+    class CreateAclResult
+      attr_reader :result_error, :error_string
+
+      def initialize(acl_result_pointer)
+        rd_kafka_error_pointer = Bindings.rd_kafka_acl_result_error(acl_result_pointer)
+        @result_error = Rdkafka::Bindings.rd_kafka_error_code(rd_kafka_error_pointer)
+        @error_string = Rdkafka::Bindings.rd_kafka_error_string(rd_kafka_error_pointer)
+      end
+
+      def self.create_acl_results_from_array(count, array_pointer)
+        (1..count).map do |index|
+          result_pointer = (array_pointer + (index - 1)).read_pointer
+          new(result_pointer)
+        end
+      end
+    end
+
+    # Extracts attributes of rd_kafka_DeleteAcls_result_response_t
+    #
+    # @private
+    class DeleteAclResult
+      attr_reader :result_error, :error_string, :matching_acls, :matching_acls_count
+
+      def initialize(acl_result_pointer)
+        @matching_acls=[]
+        rd_kafka_error_pointer = Rdkafka::Bindings.rd_kafka_DeleteAcls_result_response_error(acl_result_pointer)
+        @result_error = Rdkafka::Bindings.rd_kafka_error_code(rd_kafka_error_pointer)
+        @error_string = Rdkafka::Bindings.rd_kafka_error_string(rd_kafka_error_pointer)
+        if @result_error == 0
+           # Get the number of matching acls
+           pointer_to_size_t = FFI::MemoryPointer.new(:int32)
+           @matching_acls = Rdkafka::Bindings.rd_kafka_DeleteAcls_result_response_matching_acls(acl_result_pointer, pointer_to_size_t)
+           @matching_acls_count = pointer_to_size_t.read_int
+        end
+      end
+
+      def self.delete_acl_results_from_array(count, array_pointer)
+        (1..count).map do |index|
+          result_pointer = (array_pointer + (index - 1)).read_pointer
+          new(result_pointer)
+        end
+      end
+    end
+
+    # Extracts attributes of rd_kafka_DeleteAcls_result_response_t
+    #
+    # @private
+    class DescribeAclResult
+      attr_reader :result_error, :error_string, :matching_acls, :matching_acls_count
+
+      def initialize(event_ptr)
+        @matching_acls=[]
+        @result_error = Rdkafka::Bindings.rd_kafka_event_error(event_ptr)
+        @error_string = Rdkafka::Bindings.rd_kafka_event_error_string(event_ptr)
+        if @result_error == 0
+          acl_describe_result = Rdkafka::Bindings.rd_kafka_event_DescribeAcls_result(event_ptr)
+          # Get the number of matching acls
+          pointer_to_size_t = FFI::MemoryPointer.new(:int32)
+          @matching_acls = Rdkafka::Bindings.rd_kafka_DescribeAcls_result_acls(acl_describe_result, pointer_to_size_t)
+          @matching_acls_count = pointer_to_size_t.read_int
+        end
+      end
+    end
+
     # FFI Function used for Create Topic and Delete Topic callbacks
     BackgroundEventCallbackFunction = FFI::Function.new(
         :void, [:pointer, :pointer, :pointer]
@@ -38,6 +128,16 @@ module Rdkafka
           process_create_topic(event_ptr)
         elsif event_type == Rdkafka::Bindings::RD_KAFKA_EVENT_DELETETOPICS_RESULT
           process_delete_topic(event_ptr)
+        elsif event_type == Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATEPARTITIONS_RESULT
+          process_create_partitions(event_ptr)
+        elsif event_type == Rdkafka::Bindings::RD_KAFKA_EVENT_CREATEACLS_RESULT
+          process_create_acl(event_ptr)
+        elsif event_type == Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEACLS_RESULT
+          process_delete_acl(event_ptr)
+        elsif event_type == Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBEACLS_RESULT
+          process_describe_acl(event_ptr)
+        elsif event_type == Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEGROUPS_RESULT
+          process_delete_groups(event_ptr)
         end
       end
 
@@ -60,6 +160,23 @@ module Rdkafka
         end
       end
 
+      def self.process_delete_groups(event_ptr)
+        delete_groups_result = Rdkafka::Bindings.rd_kafka_event_DeleteGroups_result(event_ptr)
+
+        # Get the number of delete group results
+        pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
+        delete_group_result_array = Rdkafka::Bindings.rd_kafka_DeleteGroups_result_groups(delete_groups_result, pointer_to_size_t)
+        delete_group_results = GroupResult.create_group_results_from_array(pointer_to_size_t.read_int, delete_group_result_array) # TODO fix this
+        delete_group_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
+
+        if (delete_group_handle = Rdkafka::Admin::DeleteGroupsHandle.remove(delete_group_handle_ptr.address))
+          delete_group_handle[:response] = delete_group_results[0].result_error
+          delete_group_handle[:error_string] = delete_group_results[0].error_string
+          delete_group_handle[:result_name] = delete_group_results[0].result_name
+          delete_group_handle[:pending] = false
+        end
+      end
+
       def self.process_delete_topic(event_ptr)
         delete_topics_result = Rdkafka::Bindings.rd_kafka_event_DeleteTopics_result(event_ptr)
 
@@ -76,6 +193,74 @@ module Rdkafka
           delete_topic_handle[:pending] = false
         end
       end
+
+      def self.process_create_partitions(event_ptr)
+        create_partitionss_result = Rdkafka::Bindings.rd_kafka_event_CreatePartitions_result(event_ptr)
+
+        # Get the number of create topic results
+        pointer_to_size_t = FFI::MemoryPointer.new(:int32)
+        create_partitions_result_array = Rdkafka::Bindings.rd_kafka_CreatePartitions_result_topics(create_partitionss_result, pointer_to_size_t)
+        create_partitions_results = TopicResult.create_topic_results_from_array(pointer_to_size_t.read_int, create_partitions_result_array)
+        create_partitions_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
+
+        if create_partitions_handle = Rdkafka::Admin::CreatePartitionsHandle.remove(create_partitions_handle_ptr.address)
+          create_partitions_handle[:response] = create_partitions_results[0].result_error
+          create_partitions_handle[:error_string] = create_partitions_results[0].error_string
+          create_partitions_handle[:result_name] = create_partitions_results[0].result_name
+          create_partitions_handle[:pending] = false
+        end
+      end
+
+      def self.process_create_acl(event_ptr)
+        create_acls_result = Rdkafka::Bindings.rd_kafka_event_CreateAcls_result(event_ptr)
+
+        # Get the number of acl results
+        pointer_to_size_t = FFI::MemoryPointer.new(:int32)
+        create_acl_result_array = Rdkafka::Bindings.rd_kafka_CreateAcls_result_acls(create_acls_result, pointer_to_size_t)
+        create_acl_results = CreateAclResult.create_acl_results_from_array(pointer_to_size_t.read_int, create_acl_result_array)
+        create_acl_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
+
+        if create_acl_handle = Rdkafka::Admin::CreateAclHandle.remove(create_acl_handle_ptr.address)
+          create_acl_handle[:response] = create_acl_results[0].result_error
+          create_acl_handle[:response_string] = create_acl_results[0].error_string
+          create_acl_handle[:pending] = false
+        end
+      end
+
+      def self.process_delete_acl(event_ptr)
+        delete_acls_result = Rdkafka::Bindings.rd_kafka_event_DeleteAcls_result(event_ptr)
+
+        # Get the number of acl results
+        pointer_to_size_t = FFI::MemoryPointer.new(:int32)
+        delete_acl_result_responses = Rdkafka::Bindings.rd_kafka_DeleteAcls_result_responses(delete_acls_result, pointer_to_size_t)
+        delete_acl_results = DeleteAclResult.delete_acl_results_from_array(pointer_to_size_t.read_int, delete_acl_result_responses)
+        delete_acl_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
+
+        if delete_acl_handle = Rdkafka::Admin::DeleteAclHandle.remove(delete_acl_handle_ptr.address)
+          delete_acl_handle[:response] = delete_acl_results[0].result_error
+          delete_acl_handle[:response_string] = delete_acl_results[0].error_string
+          delete_acl_handle[:pending] = false
+          if delete_acl_results[0].result_error == 0
+            delete_acl_handle[:matching_acls] = delete_acl_results[0].matching_acls
+            delete_acl_handle[:matching_acls_count] = delete_acl_results[0].matching_acls_count
+          end
+        end
+      end
+
+      def self.process_describe_acl(event_ptr)
+        describe_acl = DescribeAclResult.new(event_ptr)
+        describe_acl_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
+
+        if describe_acl_handle = Rdkafka::Admin::DescribeAclHandle.remove(describe_acl_handle_ptr.address)
+          describe_acl_handle[:response] = describe_acl.result_error
+          describe_acl_handle[:response_string] = describe_acl.error_string
+          describe_acl_handle[:pending] = false
+          if describe_acl.result_error == 0
+            describe_acl_handle[:acls]       = describe_acl.matching_acls
+            describe_acl_handle[:acls_count] = describe_acl.matching_acls_count
+          end
+        end
+      end
     end
 
     # FFI Function used for Message Delivery callbacks
@@ -103,10 +288,20 @@ module Rdkafka
 
           # Call delivery callback on opaque
           if opaque = Rdkafka::Config.opaques[opaque_ptr.to_i]
-            opaque.call_delivery_callback(Rdkafka::Producer::DeliveryReport.new(message[:partition], message[:offset], topic_name, message[:err]), delivery_handle)
+            opaque.call_delivery_callback(
+              Rdkafka::Producer::DeliveryReport.new(
+                message[:partition],
+                message[:offset],
+                topic_name,
+                message[:err],
+                delivery_handle.label
+              ),
+              delivery_handle
+            )
           end
         end
       end
     end
+
   end
 end

data/lib/rdkafka/config.rb

@@ -1,11 +1,9 @@
 # frozen_string_literal: true
 
-require "logger"
-
 module Rdkafka
   # Configuration for a Kafka consumer or producer. You can create an instance and use
   # the consumer and producer methods to create a client. Documentation of the available
-  # configuration options is available on https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md.
+  # configuration options is available on https://github.com/confluentinc/librdkafka/blob/master/CONFIGURATION.md.
   class Config
     # @private
     @@logger = Logger.new(STDOUT)
@@ -14,7 +12,7 @@ module Rdkafka
     # @private
     @@error_callback = nil
     # @private
-    @@opaques = {}
+    @@opaques = ObjectSpace::WeakMap.new
     # @private
     @@log_queue = Queue.new
 
@@ -53,13 +51,13 @@ module Rdkafka
 
     # Set a callback that will be called every time the underlying client emits statistics.
     # You can configure if and how often this happens using `statistics.interval.ms`.
-    # The callback is called with a hash that's documented here: https://github.com/edenhill/librdkafka/blob/master/STATISTICS.md
+    # The callback is called with a hash that's documented here: https://github.com/confluentinc/librdkafka/blob/master/STATISTICS.md
     #
     # @param callback [Proc, #call] The callback
     #
     # @return [nil]
     def self.statistics_callback=(callback)
-      raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call)
+      raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call) || callback == nil
       @@statistics_callback = callback
     end
 
@@ -114,6 +112,7 @@ module Rdkafka
     def initialize(config_hash = {})
       @config_hash = DEFAULT_CONFIG.merge(config_hash)
       @consumer_rebalance_listener = nil
+      @consumer_poll_set = true
     end
 
     # Set a config option.
@@ -142,12 +141,28 @@ module Rdkafka
       @consumer_rebalance_listener = listener
     end
 
-    # Create a consumer with this configuration.
+    # Should we use a single queue for the underlying consumer and events.
     #
-    # @raise [ConfigError] When the configuration contains invalid options
-    # @raise [ClientCreationError] When the native client cannot be created
+    # This is an advanced API that allows for more granular control of the polling process.
+    # When this value is set to `false` (`true` by defualt), there will be two queues that need to
+    # be polled:
+    #   - main librdkafka queue for events
+    #   - consumer queue with messages and rebalances
+    #
+    # It is recommended to use the defaults and only set it to `false` in advance multi-threaded
+    # and complex cases where granular events handling control is needed.
+    #
+    # @param poll_set [Boolean]
+    def consumer_poll_set=(poll_set)
+      @consumer_poll_set = poll_set
+    end
+
+    # Creates a consumer with this configuration.
     #
     # @return [Consumer] The created consumer
+    #
+    # @raise [ConfigError] When the configuration contains invalid options
+    # @raise [ClientCreationError] When the native client cannot be created
     def consumer
       opaque = Opaque.new
       config = native_config(opaque)
@@ -160,19 +175,25 @@ module Rdkafka
       # Create native client
       kafka = native_kafka(config, :rd_kafka_consumer)
 
-      # Redirect the main queue to the consumer
-      Rdkafka::Bindings.rd_kafka_poll_set_consumer(kafka)
+      # Redirect the main queue to the consumer queue
+      Rdkafka::Bindings.rd_kafka_poll_set_consumer(kafka) if @consumer_poll_set
 
       # Return consumer with Kafka client
-      Rdkafka::Consumer.new(Rdkafka::NativeKafka.new(kafka, run_polling_thread: false))
+      Rdkafka::Consumer.new(
+        Rdkafka::NativeKafka.new(
+          kafka,
+          run_polling_thread: false,
+          opaque: opaque
+        )
+      )
     end
 
     # Create a producer with this configuration.
     #
+    # @return [Producer] The created producer
+    #
     # @raise [ConfigError] When the configuration contains invalid options
     # @raise [ClientCreationError] When the native client cannot be created
-    #
-    # @return [Producer] The created producer
     def producer
       # Create opaque
       opaque = Opaque.new
@@ -182,22 +203,35 @@ module Rdkafka
       Rdkafka::Bindings.rd_kafka_conf_set_dr_msg_cb(config, Rdkafka::Callbacks::DeliveryCallbackFunction)
       # Return producer with Kafka client
       partitioner_name = self[:partitioner] || self["partitioner"]
-      Rdkafka::Producer.new(Rdkafka::NativeKafka.new(native_kafka(config, :rd_kafka_producer), run_polling_thread: true), partitioner_name).tap do |producer|
+      Rdkafka::Producer.new(
+        Rdkafka::NativeKafka.new(
+          native_kafka(config, :rd_kafka_producer),
+          run_polling_thread: true,
+          opaque: opaque
+        ),
+        partitioner_name
+      ).tap do |producer|
         opaque.producer = producer
       end
     end
 
-    # Create an admin instance with this configuration.
+    # Creates an admin instance with this configuration.
+    #
+    # @return [Admin] The created admin instance
     #
     # @raise [ConfigError] When the configuration contains invalid options
     # @raise [ClientCreationError] When the native client cannot be created
-    #
-    # @return [Admin] The created admin instance
     def admin
       opaque = Opaque.new
       config = native_config(opaque)
       Rdkafka::Bindings.rd_kafka_conf_set_background_event_cb(config, Rdkafka::Callbacks::BackgroundEventCallbackFunction)
-      Rdkafka::Admin.new(Rdkafka::NativeKafka.new(native_kafka(config, :rd_kafka_producer), run_polling_thread: true))
+      Rdkafka::Admin.new(
+        Rdkafka::NativeKafka.new(
+          native_kafka(config, :rd_kafka_producer),
+          run_polling_thread: true,
+          opaque: opaque
+        )
+      )
     end
 
     # Error that is returned by the underlying rdkafka error if an invalid configuration option is present.

data/lib/rdkafka/consumer/headers.rb

@@ -18,13 +18,11 @@ module Rdkafka
 
       # Reads a librdkafka native message's headers and returns them as a Ruby Hash
       #
-      # @param [librdkakfa message] native_message
+      # @private
       #
+      # @param [Rdkafka::Bindings::Message] native_message
       # @return [Hash<String, String>] headers Hash for the native_message
-      #
       # @raise [Rdkafka::RdkafkaError] when fail to read headers
-      #
-      # @private
       def self.from_native(native_message)
         headers_ptrptr = FFI::MemoryPointer.new(:pointer)
         err = Rdkafka::Bindings.rd_kafka_message_headers(native_message, headers_ptrptr)

data/lib/rdkafka/consumer/topic_partition_list.rb

@@ -36,6 +36,11 @@ module Rdkafka
       # Add a topic with optionally partitions to the list.
       # Calling this method multiple times for the same topic will overwrite the previous configuraton.
       #
+      # @param topic [String] The topic's name
+      # @param partitions [Array<Integer>, Range<Integer>, Integer] The topic's partitions or partition count
+      #
+      # @return [nil]
+      #
       # @example Add a topic with unassigned partitions
       #   tpl.add_topic("topic")
       #
@@ -45,10 +50,6 @@ module Rdkafka
       # @example Add a topic with all topics up to a count
       #   tpl.add_topic("topic", 9)
       #
-      # @param topic [String] The topic's name
-      # @param partitions [Array<Integer>, Range<Integer>, Integer] The topic's partitions or partition count
-      #
-      # @return [nil]
       def add_topic(topic, partitions=nil)
         if partitions.nil?
           @data[topic.to_s] = nil
@@ -90,11 +91,11 @@ module Rdkafka
 
       # Create a new topic partition list based of a native one.
       #
+      # @private
+      #
       # @param pointer [FFI::Pointer] Optional pointer to an existing native list. Its contents will be copied.
       #
       # @return [TopicPartitionList]
-      #
-      # @private
       def self.from_native_tpl(pointer)
         # Data to be moved into the tpl
         data = {}
@@ -127,8 +128,8 @@ module Rdkafka
       #
       # The pointer will be cleaned by `rd_kafka_topic_partition_list_destroy` when GC releases it.
       #
-      # @return [FFI::Pointer]
       # @private
+      # @return [FFI::Pointer]
       def to_native_tpl
         tpl = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(count)
 
@@ -142,11 +143,13 @@ module Rdkafka
               )
 
               if p.offset
+                offset = p.offset.is_a?(Time) ? p.offset.to_f * 1_000 : p.offset
+
                 Rdkafka::Bindings.rd_kafka_topic_partition_list_set_offset(
                   tpl,
                   topic,
                   p.partition,
-                  p.offset
+                  offset
                 )
               end
             end