rdkafka 0.22.2 → 0.29.0

data/lib/rdkafka/abstract_handle.rb

@@ -14,9 +14,16 @@ module Rdkafka
 
     # Registry for registering all the handles.
     REGISTRY = {}
+    # Guards REGISTRY mutations. The registry is shared across producing/consuming threads and the
+    # background polling thread (which removes handles from FFI callbacks), so the Hash must not be
+    # mutated concurrently. This is effectively safe on MRI under the GVL but not on JRuby, where a
+    # lost write could leave a handle unregistered (never unlocked -> `wait` times out for a
+    # delivered message) or never removed (permanent leak).
+    REGISTRY_MUTEX = Mutex.new
     # Default wait timeout is 31 years
     MAX_WAIT_TIMEOUT_FOREVER = 10_000_000_000
 
+    private_constant :REGISTRY_MUTEX
     private_constant :MAX_WAIT_TIMEOUT_FOREVER
 
     class << self
@@ -25,17 +32,28 @@ module Rdkafka
       # @param handle [AbstractHandle] any handle we want to register
       def register(handle)
         address = handle.to_ptr.address
-        REGISTRY[address] = handle
+        REGISTRY_MUTEX.synchronize { REGISTRY[address] = handle }
       end
 
       # Removes handle from the register based on the handle address
       #
       # @param address [Integer] address of the registered handle we want to remove
       def remove(address)
-        REGISTRY.delete(address)
+        REGISTRY_MUTEX.synchronize { REGISTRY.delete(address) }
       end
     end
 
+    # Operation result prepared by the background event callback. Background events are
+    # destroyed as soon as the callback returns, so anything the waiting thread needs has to be
+    # copied out of event-owned memory into Ruby objects and stored here before `#unlock` runs.
+    # When result parsing fails, holds the captured exception so it can be re-raised on the
+    # waiting thread.
+    attr_accessor :result
+
+    # Broker-provided error message copied out of event-owned memory by the background event
+    # callback, used by `#raise_error`.
+    attr_accessor :broker_message
+
     def initialize
       @mutex = Thread::Mutex.new
       @resource = Thread::ConditionVariable.new
@@ -54,16 +72,34 @@ module Rdkafka
     # If there is a timeout this does not mean the operation failed, rdkafka might still be working
     # on the operation. In this case it is possible to call wait again.
     #
-    # @param max_wait_timeout [Numeric, nil] Amount of time to wait before timing out.
-    #   If this is nil we will wait forever
+    # @param max_wait_timeout [Numeric, nil] DEPRECATED: Use max_wait_timeout_ms instead.
+    #   Amount of time in seconds to wait before timing out. Will be removed in v1.0.0.
+    # @param max_wait_timeout_ms [Numeric, nil] Amount of time in milliseconds to wait before
+    #   timing out. If this is nil we will wait forever. Defaults to 60,000ms (60 seconds).
     # @param raise_response_error [Boolean] should we raise error when waiting finishes
     #
     # @return [Object] Operation-specific result
     #
     # @raise [RdkafkaError] When the operation failed
     # @raise [WaitTimeoutError] When the timeout has been reached and the handle is still pending
-    def wait(max_wait_timeout: 60, raise_response_error: true)
-      timeout = max_wait_timeout ? monotonic_now + max_wait_timeout : MAX_WAIT_TIMEOUT_FOREVER
+    def wait(max_wait_timeout: :not_provided, max_wait_timeout_ms: :not_provided, raise_response_error: true)
+      # Determine which timeout value to use
+      if max_wait_timeout != :not_provided && max_wait_timeout_ms != :not_provided
+        warn "DEPRECATION WARNING: Both max_wait_timeout and max_wait_timeout_ms were provided. " \
+             "Using max_wait_timeout_ms. The max_wait_timeout parameter is deprecated and will be removed in v1.0.0."
+        timeout_ms = max_wait_timeout_ms
+      elsif max_wait_timeout != :not_provided
+        warn "DEPRECATION WARNING: max_wait_timeout (seconds) is deprecated. " \
+             "Use max_wait_timeout_ms (milliseconds) instead. This parameter will be removed in v1.0.0."
+        timeout_ms = max_wait_timeout ? (max_wait_timeout * 1000).to_i : nil
+      elsif max_wait_timeout_ms == :not_provided
+        timeout_ms = Defaults::HANDLE_WAIT_TIMEOUT_MS
+      else
+        timeout_ms = max_wait_timeout_ms
+      end
+
+      timeout_s = timeout_ms ? timeout_ms / 1000.0 : nil
+      timeout = timeout_s ? monotonic_now + timeout_s : MAX_WAIT_TIMEOUT_FOREVER
 
       @mutex.synchronize do
         loop do
@@ -74,7 +110,7 @@ module Rdkafka
               @resource.wait(@mutex, to_wait)
             else
               raise WaitTimeoutError.new(
-                "Waiting for #{operation_name} timed out after #{max_wait_timeout} seconds"
+                "Waiting for #{operation_name} timed out after #{timeout_ms} ms"
               )
             end
           elsif self[:response] != 0 && raise_response_error
@@ -104,6 +140,17 @@ module Rdkafka
       raise "Must be implemented by subclass!"
     end
 
+    # Returns the operation result prepared by the background event callback, re-raising an
+    # exception that was captured while the event memory was still readable.
+    #
+    # @return [Object] operation-specific result
+    # @raise [RdkafkaError] when result parsing in the event callback failed
+    def prepared_result
+      raise(result) if result.is_a?(Exception)
+
+      result
+    end
+
     # Allow subclasses to override
     def raise_error
       raise RdkafkaError.new(self[:response])

data/lib/rdkafka/admin/acl_binding_result.rb

@@ -3,17 +3,17 @@
 module Rdkafka
   class Admin
     # Extracts attributes of rd_kafka_AclBinding_t
-    #
     class AclBindingResult
       attr_reader :result_error, :error_string, :matching_acl_resource_type,
-                  :matching_acl_resource_name, :matching_acl_resource_pattern_type,
-                  :matching_acl_principal, :matching_acl_host, :matching_acl_operation,
-                  :matching_acl_permission_type
+        :matching_acl_resource_name, :matching_acl_resource_pattern_type,
+        :matching_acl_principal, :matching_acl_host, :matching_acl_operation,
+        :matching_acl_permission_type
 
       # This attribute was initially released under the name that is now an alias
       # We keep it for backwards compatibility but it was changed for the consistency
-      alias matching_acl_pattern_type matching_acl_resource_pattern_type
+      alias_method :matching_acl_pattern_type, :matching_acl_resource_pattern_type
 
+      # @param matching_acl [FFI::Pointer] pointer to the ACL binding struct
       def initialize(matching_acl)
         rd_kafka_error_pointer = Rdkafka::Bindings.rd_kafka_AclBinding_error(matching_acl)
         @result_error = Rdkafka::Bindings.rd_kafka_error_code(rd_kafka_error_pointer)

data/lib/rdkafka/admin/config_binding_result.rb

@@ -17,12 +17,12 @@ module Rdkafka
         @synonyms = []
 
         # The code below builds up the config synonyms using same config binding
-        pointer_to_size_t = FFI::MemoryPointer.new(:int32)
+        pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
         synonym_ptr = Bindings.rd_kafka_ConfigEntry_synonyms(config_ptr, pointer_to_size_t)
         synonyms_ptr = synonym_ptr.read_array_of_pointer(pointer_to_size_t.read_int)
 
         (1..pointer_to_size_t.read_int).map do |ar|
-          self.class.new synonyms_ptr[ar - 1]
+          @synonyms << self.class.new(synonyms_ptr[ar - 1])
         end
       end
     end

data/lib/rdkafka/admin/config_resource_binding_result.rb

@@ -6,6 +6,7 @@ module Rdkafka
     class ConfigResourceBindingResult
       attr_reader :name, :type, :configs, :configs_count
 
+      # @param config_resource_ptr [FFI::Pointer] pointer to the config resource struct
       def initialize(config_resource_ptr)
         ffi_binding = Bindings::ConfigResource.new(config_resource_ptr)
 

data/lib/rdkafka/admin/create_acl_handle.rb

@@ -2,25 +2,29 @@
 
 module Rdkafka
   class Admin
+    # Handle for create ACL operation
     class CreateAclHandle < AbstractHandle
       layout :pending, :bool,
-             :response, :int,
-             :response_string, :pointer
+        :response, :int
 
       # @return [String] the name of the operation
       def operation_name
         "create acl"
       end
 
-      # @return [CreateAclReport] instance with rdkafka_response value as 0 and rdkafka_response_string value as empty string if the acl creation was successful
+      # @return [CreateAclReport] report prepared by the background event callback, with
+      #   rdkafka_response value as 0 and rdkafka_response_string value as empty string if the
+      #   acl creation was successful
       def create_result
-        CreateAclReport.new(rdkafka_response: self[:response], rdkafka_response_string: self[:response_string])
+        prepared_result
       end
 
+      # Raises an error if the operation failed
+      # @raise [RdkafkaError]
       def raise_error
         raise RdkafkaError.new(
-            self[:response],
-            broker_message: self[:response_string].read_string
+          self[:response],
+          broker_message: broker_message
         )
       end
     end

data/lib/rdkafka/admin/create_acl_report.rb

@@ -2,17 +2,18 @@
 
 module Rdkafka
   class Admin
+    # Report for create ACL operation result
     class CreateAclReport
-
       # Upon successful creation of Acl RD_KAFKA_RESP_ERR_NO_ERROR - 0 is returned as rdkafka_response
       # @return [Integer]
       attr_reader :rdkafka_response
 
-
       # Upon successful creation of Acl empty string will be returned as rdkafka_response_string
       # @return [String]
       attr_reader :rdkafka_response_string
 
+      # @param rdkafka_response [Integer] response code from librdkafka
+      # @param rdkafka_response_string [FFI::Pointer] pointer to response string
       def initialize(rdkafka_response:, rdkafka_response_string:)
         @rdkafka_response = rdkafka_response
         if rdkafka_response_string != FFI::Pointer::NULL

data/lib/rdkafka/admin/create_partitions_handle.rb

@@ -1,25 +1,26 @@
 module Rdkafka
   class Admin
+    # Handle for create partitions operation
     class CreatePartitionsHandle < AbstractHandle
       layout :pending, :bool,
-             :response, :int,
-             :error_string, :pointer,
-             :result_name, :pointer
+        :response, :int
 
       # @return [String] the name of the operation
       def operation_name
         "create partitions"
       end
 
-      # @return [Boolean] whether the create topic was successful
+      # @return [CreatePartitionsReport] report prepared by the background event callback
       def create_result
-        CreatePartitionsReport.new(self[:error_string], self[:result_name])
+        prepared_result
       end
 
+      # Raises an error if the operation failed
+      # @raise [RdkafkaError]
       def raise_error
         raise RdkafkaError.new(
-            self[:response],
-            broker_message: CreateTopicReport.new(self[:error_string], self[:result_name]).error_string
+          self[:response],
+          broker_message: broker_message
         )
       end
     end

data/lib/rdkafka/admin/create_partitions_report.rb

@@ -1,5 +1,6 @@
 module Rdkafka
   class Admin
+    # Report for create partitions operation result
     class CreatePartitionsReport < CreateTopicReport
     end
   end

data/lib/rdkafka/admin/create_topic_handle.rb

@@ -2,26 +2,27 @@
 
 module Rdkafka
   class Admin
+    # Handle for create topic operation
     class CreateTopicHandle < AbstractHandle
       layout :pending, :bool,
-             :response, :int,
-             :error_string, :pointer,
-             :result_name, :pointer
+        :response, :int
 
       # @return [String] the name of the operation
       def operation_name
         "create topic"
       end
 
-      # @return [Boolean] whether the create topic was successful
+      # @return [CreateTopicReport] report prepared by the background event callback
       def create_result
-        CreateTopicReport.new(self[:error_string], self[:result_name])
+        prepared_result
       end
 
+      # Raises an error if the operation failed
+      # @raise [RdkafkaError]
       def raise_error
         raise RdkafkaError.new(
-            self[:response],
-            broker_message: CreateTopicReport.new(self[:error_string], self[:result_name]).error_string
+          self[:response],
+          broker_message: broker_message
         )
       end
     end

data/lib/rdkafka/admin/create_topic_report.rb

@@ -2,6 +2,7 @@
 
 module Rdkafka
   class Admin
+    # Report for create topic operation result
     class CreateTopicReport
       # Any error message generated from the CreateTopic
       # @return [String]
@@ -11,6 +12,8 @@ module Rdkafka
       # @return [String]
       attr_reader :result_name
 
+      # @param error_string [FFI::Pointer] pointer to error string
+      # @param result_name [FFI::Pointer] pointer to topic name
       def initialize(error_string, result_name)
         if error_string != FFI::Pointer::NULL
           @error_string = error_string.read_string

data/lib/rdkafka/admin/delete_acl_handle.rb

@@ -2,27 +2,28 @@
 
 module Rdkafka
   class Admin
+    # Handle for delete ACL operation
     class DeleteAclHandle < AbstractHandle
       layout :pending, :bool,
-             :response, :int,
-             :response_string, :pointer,
-             :matching_acls, :pointer,
-             :matching_acls_count, :int
+        :response, :int
 
       # @return [String] the name of the operation
       def operation_name
         "delete acl"
       end
 
-      # @return [DeleteAclReport] instance with an array of matching_acls
+      # @return [DeleteAclReport] report prepared by the background event callback, with an
+      #   array of matching_acls
       def create_result
-        DeleteAclReport.new(matching_acls: self[:matching_acls], matching_acls_count: self[:matching_acls_count])
+        prepared_result
       end
 
+      # Raises an error if the operation failed
+      # @raise [RdkafkaError]
       def raise_error
         raise RdkafkaError.new(
-            self[:response],
-            broker_message: self[:response_string].read_string
+          self[:response],
+          broker_message: broker_message
         )
       end
     end

data/lib/rdkafka/admin/delete_acl_report.rb

@@ -2,16 +2,18 @@
 
 module Rdkafka
   class Admin
+    # Report for delete ACL operation result
     class DeleteAclReport
-
       # deleted acls
       # @return [Rdkafka::Bindings::AclBindingResult]
       attr_reader :deleted_acls
 
+      # @param matching_acls [FFI::Pointer] pointer to matching ACLs array
+      # @param matching_acls_count [Integer] number of matching ACLs
       def initialize(matching_acls:, matching_acls_count:)
-        @deleted_acls=[]
+        @deleted_acls = []
         if matching_acls != FFI::Pointer::NULL
-          acl_binding_result_pointers  = matching_acls.read_array_of_pointer(matching_acls_count)
+          acl_binding_result_pointers = matching_acls.read_array_of_pointer(matching_acls_count)
           (1..matching_acls_count).map do |matching_acl_index|
             acl_binding_result = AclBindingResult.new(acl_binding_result_pointers[matching_acl_index - 1])
             @deleted_acls << acl_binding_result

data/lib/rdkafka/admin/delete_groups_handle.rb

@@ -2,25 +2,28 @@
 
 module Rdkafka
   class Admin
+    # Handle for delete groups operation
     class DeleteGroupsHandle < AbstractHandle
-      layout :pending, :bool, # TODO: ???
-             :response, :int,
-             :error_string, :pointer,
-             :result_name, :pointer
+      layout :pending, :bool,
+        :response, :int
 
       # @return [String] the name of the operation
       def operation_name
         "delete groups"
       end
 
+      # Creates the result report
+      # @return [DeleteGroupsReport]
       def create_result
-        DeleteGroupsReport.new(self[:error_string], self[:result_name])
+        prepared_result
       end
 
+      # Raises an error if the operation failed
+      # @raise [RdkafkaError]
       def raise_error
         raise RdkafkaError.new(
-            self[:response],
-            broker_message: create_result.error_string
+          self[:response],
+          broker_message: broker_message
         )
       end
     end

data/lib/rdkafka/admin/delete_groups_report.rb

@@ -2,6 +2,7 @@
 
 module Rdkafka
   class Admin
+    # Report for delete groups operation result
     class DeleteGroupsReport
       # Any error message generated from the DeleteTopic
       # @return [String]
@@ -11,6 +12,8 @@ module Rdkafka
       # @return [String]
       attr_reader :result_name
 
+      # @param error_string [FFI::Pointer] pointer to error string
+      # @param result_name [FFI::Pointer] pointer to group name
       def initialize(error_string, result_name)
         if error_string != FFI::Pointer::NULL
           @error_string = error_string.read_string

data/lib/rdkafka/admin/delete_topic_handle.rb

@@ -2,26 +2,27 @@
 
 module Rdkafka
   class Admin
+    # Handle for delete topic operation
     class DeleteTopicHandle < AbstractHandle
       layout :pending, :bool,
-             :response, :int,
-             :error_string, :pointer,
-             :result_name, :pointer
+        :response, :int
 
       # @return [String] the name of the operation
       def operation_name
         "delete topic"
       end
 
-      # @return [Boolean] whether the delete topic was successful
+      # @return [DeleteTopicReport] report prepared by the background event callback
       def create_result
-        DeleteTopicReport.new(self[:error_string], self[:result_name])
+        prepared_result
       end
 
+      # Raises an error if the operation failed
+      # @raise [RdkafkaError]
       def raise_error
         raise RdkafkaError.new(
-            self[:response],
-            broker_message: DeleteTopicReport.new(self[:error_string], self[:result_name]).error_string
+          self[:response],
+          broker_message: broker_message
         )
       end
     end

data/lib/rdkafka/admin/delete_topic_report.rb

@@ -2,6 +2,7 @@
 
 module Rdkafka
   class Admin
+    # Report for delete topic operation result
     class DeleteTopicReport
       # Any error message generated from the DeleteTopic
       # @return [String]
@@ -11,6 +12,8 @@ module Rdkafka
       # @return [String]
       attr_reader :result_name
 
+      # @param error_string [FFI::Pointer] pointer to error string
+      # @param result_name [FFI::Pointer] pointer to topic name
       def initialize(error_string, result_name)
         if error_string != FFI::Pointer::NULL
           @error_string = error_string.read_string

data/lib/rdkafka/admin/describe_acl_handle.rb

@@ -2,27 +2,28 @@
 
 module Rdkafka
   class Admin
+    # Handle for describe ACL operation
     class DescribeAclHandle < AbstractHandle
       layout :pending, :bool,
-             :response, :int,
-             :response_string, :pointer,
-             :acls, :pointer,
-             :acls_count, :int
+        :response, :int
 
       # @return [String] the name of the operation.
       def operation_name
         "describe acl"
       end
 
-      # @return [DescribeAclReport] instance with an array of acls that matches the request filters.
+      # @return [DescribeAclReport] report prepared by the background event callback, with an
+      #   array of acls that matches the request filters.
       def create_result
-        DescribeAclReport.new(acls: self[:acls], acls_count: self[:acls_count])
+        prepared_result
       end
 
+      # Raises an error if the operation failed
+      # @raise [RdkafkaError]
       def raise_error
         raise RdkafkaError.new(
-            self[:response],
-            broker_message: self[:response_string].read_string
+          self[:response],
+          broker_message: broker_message
         )
       end
     end

data/lib/rdkafka/admin/describe_acl_report.rb

@@ -2,17 +2,19 @@
 
 module Rdkafka
   class Admin
+    # Report for describe ACL operation result
     class DescribeAclReport
-
       # acls that exists in the cluster for the resource_type, resource_name and pattern_type filters provided in the request.
       # @return [Rdkafka::Bindings::AclBindingResult] array of matching acls.
       attr_reader :acls
 
+      # @param acls [FFI::Pointer] pointer to ACLs array
+      # @param acls_count [Integer] number of ACLs
       def initialize(acls:, acls_count:)
-        @acls=[]
+        @acls = []
 
         if acls != FFI::Pointer::NULL
-          acl_binding_result_pointers  = acls.read_array_of_pointer(acls_count)
+          acl_binding_result_pointers = acls.read_array_of_pointer(acls_count)
           (1..acls_count).map do |acl_index|
             acl_binding_result = AclBindingResult.new(acl_binding_result_pointers[acl_index - 1])
             @acls << acl_binding_result

data/lib/rdkafka/admin/describe_configs_handle.rb

@@ -2,30 +2,27 @@
 
 module Rdkafka
   class Admin
+    # Handle for describe configs operation
     class DescribeConfigsHandle < AbstractHandle
       layout :pending, :bool,
-             :response, :int,
-             :response_string, :pointer,
-             :config_entries, :pointer,
-             :entry_count, :int
+        :response, :int
 
       # @return [String] the name of the operation.
       def operation_name
         "describe configs"
       end
 
-      # @return [DescribeAclReport] instance with an array of acls that matches the request filters.
+      # @return [DescribeConfigsReport] report prepared by the background event callback
       def create_result
-        DescribeConfigsReport.new(
-          config_entries: self[:config_entries],
-          entry_count: self[:entry_count]
-        )
+        prepared_result
       end
 
+      # Raises an error if the operation failed
+      # @raise [RdkafkaError]
       def raise_error
         raise RdkafkaError.new(
           self[:response],
-          broker_message: self[:response_string].read_string
+          broker_message: broker_message
         )
       end
     end

data/lib/rdkafka/admin/describe_configs_report.rb

@@ -2,11 +2,14 @@
 
 module Rdkafka
   class Admin
+    # Report for describe configs operation result
     class DescribeConfigsReport
       attr_reader :resources
 
+      # @param config_entries [FFI::Pointer] pointer to config entries array
+      # @param entry_count [Integer] number of config entries
       def initialize(config_entries:, entry_count:)
-        @resources=[]
+        @resources = []
 
         return if config_entries == FFI::Pointer::NULL
 
@@ -16,7 +19,7 @@ module Rdkafka
           .each do |config_resource_result_ptr|
             config_resource_result = ConfigResourceBindingResult.new(config_resource_result_ptr)
 
-            pointer_to_size_t = FFI::MemoryPointer.new(:int32)
+            pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
             configs_ptr = Bindings.rd_kafka_ConfigResource_configs(
               config_resource_result_ptr,
               pointer_to_size_t
@@ -29,14 +32,13 @@ module Rdkafka
 
             @resources << config_resource_result
           end
-      ensure
-        return if config_entries == FFI::Pointer::NULL
-
-        Bindings.rd_kafka_ConfigResource_destroy_array(config_entries, entry_count)
       end
 
       private
 
+      # Validates the config resource result and raises an error if invalid
+      # @param config_resource_result_ptr [FFI::Pointer] pointer to the config resource result
+      # @raise [RdkafkaError] when the config resource has an error
       def validate!(config_resource_result_ptr)
         code = Bindings.rd_kafka_ConfigResource_error(config_resource_result_ptr)
 

data/lib/rdkafka/admin/incremental_alter_configs_handle.rb

@@ -2,30 +2,27 @@
 
 module Rdkafka
   class Admin
+    # Handle for incremental alter configs operation
     class IncrementalAlterConfigsHandle < AbstractHandle
       layout :pending, :bool,
-             :response, :int,
-             :response_string, :pointer,
-             :config_entries, :pointer,
-             :entry_count, :int
+        :response, :int
 
       # @return [String] the name of the operation.
       def operation_name
         "incremental alter configs"
       end
 
-      # @return [DescribeAclReport] instance with an array of acls that matches the request filters.
+      # @return [IncrementalAlterConfigsReport] report prepared by the background event callback
       def create_result
-        IncrementalAlterConfigsReport.new(
-          config_entries: self[:config_entries],
-          entry_count: self[:entry_count]
-        )
+        prepared_result
       end
 
+      # Raises an error if the operation failed
+      # @raise [RdkafkaError]
       def raise_error
         raise RdkafkaError.new(
           self[:response],
-          broker_message: self[:response_string].read_string
+          broker_message: broker_message
         )
       end
     end