rdkafka 0.22.2 → 0.29.0

data/lib/rdkafka/bindings.rb

@@ -2,18 +2,61 @@
 
 module Rdkafka
   # @private
+  #
+  # @note
+  #   There are two types of responses related to errors:
+  #     - rd_kafka_error_t - a C object that we need to remap into an error or null when no error
+  #     - rd_kafka_resp_err_t - response error code (numeric) that we can use directly
+  #
+  #   It is critical to ensure, that we handle them correctly. The result type should be:
+  #     - rd_kafka_error_t - :pointer
+  #     - rd_kafka_resp_err_t - :int
   module Bindings
     extend FFI::Library
 
+    # Returns the library extension based on the host OS
+    # @return [String] 'dylib' on macOS, 'so' on other systems
     def self.lib_extension
-      if RbConfig::CONFIG['host_os'] =~ /darwin/
-        'dylib'
+      if /darwin/.match?(RbConfig::CONFIG["host_os"])
+        "dylib"
       else
-        'so'
+        "so"
       end
     end
 
-    ffi_lib File.join(__dir__, "../../ext/librdkafka.#{lib_extension}")
+    # Wrap ffi_lib to provide better error messages for glibc compatibility issues
+    begin
+      ffi_lib File.join(__dir__, "../../ext/librdkafka.#{lib_extension}")
+    rescue LoadError => e
+      error_message = e.message
+
+      # Check if this is a glibc version mismatch error
+      if /GLIBC_[\d.]+['"` ]?\s*not found/i.match?(error_message)
+        glibc_version = error_message[/GLIBC_([\d.]+)/, 1] || "unknown"
+
+        raise Rdkafka::LibraryLoadError, <<~ERROR_MSG.strip
+          Failed to load librdkafka due to glibc compatibility issue.
+
+          The precompiled librdkafka binary requires glibc version #{glibc_version} or higher,
+          but your system has an older version installed.
+
+          To resolve this issue, you have two options:
+
+          1. Upgrade your system to a supported platform (recommended)
+
+          2. Force compilation from source by reinstalling without the precompiled binary:
+             gem install rdkafka --platform=ruby
+
+             Or if using Bundler, add to your Gemfile:
+             gem 'rdkafka', force_ruby_platform: true
+
+          Original error: #{error_message}
+        ERROR_MSG
+      else
+        # Re-raise the original error if it's not a glibc issue
+        raise
+      end
+    end
 
     RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS = -175
     RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS = -174
@@ -21,13 +64,17 @@ module Rdkafka
     RD_KAFKA_RESP_ERR__NOENT = -156
     RD_KAFKA_RESP_ERR_NO_ERROR = 0
 
-    RD_KAFKA_OFFSET_END       = -1
+    RD_KAFKA_OFFSET_END = -1
     RD_KAFKA_OFFSET_BEGINNING = -2
-    RD_KAFKA_OFFSET_STORED    = -1000
-    RD_KAFKA_OFFSET_INVALID   = -1001
+    RD_KAFKA_OFFSET_STORED = -1000
+    RD_KAFKA_OFFSET_INVALID = -1001
+
+    RD_KAFKA_PARTITION_UA = -1
+    RD_KAFKA_PARTITION_UA_STR = RD_KAFKA_PARTITION_UA.to_s.freeze
 
     EMPTY_HASH = {}.freeze
 
+    # FFI struct for size_t pointer wrapper
     class SizePtr < FFI::Struct
       layout :value, :size_t
     end
@@ -43,26 +90,32 @@ module Rdkafka
     attach_function :rd_kafka_poll, [:pointer, :int], :int, blocking: true
     attach_function :rd_kafka_outq_len, [:pointer], :int, blocking: true
 
+    # Non-blocking poll variants (do not release GVL)
+    # These are more efficient for poll(0) calls in fiber schedulers where GVL
+    # release/reacquire overhead is wasteful since we don't expect to wait.
+    # Uses the same underlying C function but with blocking: false to skip GVL release.
+    attach_function :rd_kafka_poll_nb, :rd_kafka_poll, [:pointer, :int], :int, blocking: false
+
     # Metadata
 
     attach_function :rd_kafka_name, [:pointer], :string
-    attach_function :rd_kafka_memberid, [:pointer], :string, blocking: true
-    attach_function :rd_kafka_clusterid, [:pointer], :string, blocking: true
+    attach_function :rd_kafka_memberid, [:pointer], :pointer, blocking: true
+    attach_function :rd_kafka_clusterid, [:pointer, :int], :pointer, blocking: true
+    attach_function :rd_kafka_mem_free, [:pointer, :pointer], :void
     attach_function :rd_kafka_metadata, [:pointer, :int, :pointer, :pointer, :int], :int, blocking: true
     attach_function :rd_kafka_metadata_destroy, [:pointer], :void, blocking: true
 
-    # Message struct
-
+    # FFI struct representing a Kafka message (rd_kafka_message_t)
     class Message < FFI::Struct
       layout :err, :int,
-             :rkt, :pointer,
-             :partition, :int32,
-             :payload, :pointer,
-             :len, :size_t,
-             :key, :pointer,
-             :key_len, :size_t,
-             :offset, :int64,
-             :_private, :pointer
+        :rkt, :pointer,
+        :partition, :int32,
+        :payload, :pointer,
+        :len, :size_t,
+        :key, :pointer,
+        :key_len, :size_t,
+        :offset, :int64,
+        :_private, :pointer
     end
 
     attach_function :rd_kafka_message_destroy, [:pointer], :void
@@ -71,40 +124,38 @@ module Rdkafka
     attach_function :rd_kafka_topic_destroy, [:pointer], :pointer
     attach_function :rd_kafka_topic_name, [:pointer], :string
 
-    # TopicPartition ad TopicPartitionList structs
-
+    # FFI struct representing a topic partition (rd_kafka_topic_partition_t)
     class TopicPartition < FFI::Struct
       layout :topic, :string,
-             :partition, :int32,
-             :offset, :int64,
-             :metadata, :pointer,
-             :metadata_size, :size_t,
-             :opaque, :pointer,
-             :err, :int,
-             :_private, :pointer
+        :partition, :int32,
+        :offset, :int64,
+        :metadata, :pointer,
+        :metadata_size, :size_t,
+        :opaque, :pointer,
+        :err, :int,
+        :_private, :pointer
     end
 
+    # FFI struct representing a topic partition list (rd_kafka_topic_partition_list_t)
     class TopicPartitionList < FFI::Struct
       layout :cnt, :int,
-             :size, :int,
-             :elems, :pointer
+        :size, :int,
+        :elems, :pointer
     end
 
     attach_function :rd_kafka_topic_partition_list_new, [:int32], :pointer
-    attach_function :rd_kafka_topic_partition_list_add, [:pointer, :string, :int32], :void
+    attach_function :rd_kafka_topic_partition_list_add, [:pointer, :string, :int32], :pointer
     attach_function :rd_kafka_topic_partition_list_set_offset, [:pointer, :string, :int32, :int64], :void
     attach_function :rd_kafka_topic_partition_list_destroy, [:pointer], :void
     attach_function :rd_kafka_topic_partition_list_copy, [:pointer], :pointer
 
-    # Configs management
-    #
-    # Structs for management of configurations
-    # Each configuration is attached to a resource and one resource can have many configuration
-    # details. Each resource will also have separate errors results if obtaining configuration
-    # was not possible for any reason
+    # FFI struct representing a config resource (rd_kafka_ConfigResource_t)
+    # Structs for management of configurations. Each configuration is attached to a resource
+    # and one resource can have many configuration details. Each resource will also have
+    # separate errors results if obtaining configuration was not possible for any reason
     class ConfigResource < FFI::Struct
       layout :type, :int,
-             :name, :string
+        :name, :string
     end
 
     attach_function :rd_kafka_DescribeConfigs, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
@@ -133,20 +184,45 @@ module Rdkafka
     RD_KAFKA_ADMIN_OP_INCREMENTALALTERCONFIGS = 16
     RD_KAFKA_EVENT_INCREMENTALALTERCONFIGS_RESULT = 131072
 
-    RD_KAFKA_ALTER_CONFIG_OP_TYPE_SET      = 0
-    RD_KAFKA_ALTER_CONFIG_OP_TYPE_DELETE   = 1
-    RD_KAFKA_ALTER_CONFIG_OP_TYPE_APPEND   = 2
+    RD_KAFKA_ALTER_CONFIG_OP_TYPE_SET = 0
+    RD_KAFKA_ALTER_CONFIG_OP_TYPE_DELETE = 1
+    RD_KAFKA_ALTER_CONFIG_OP_TYPE_APPEND = 2
     RD_KAFKA_ALTER_CONFIG_OP_TYPE_SUBTRACT = 3
 
-    # Errors
+    # List Offsets
+    RD_KAFKA_ADMIN_OP_LISTOFFSETS = 20
+    RD_KAFKA_EVENT_LISTOFFSETS_RESULT = 0x400000
+
+    # rd_kafka_IsolationLevel_t
+    RD_KAFKA_ISOLATION_LEVEL_READ_UNCOMMITTED = 0
+    RD_KAFKA_ISOLATION_LEVEL_READ_COMMITTED = 1
+
+    # rd_kafka_OffsetSpec_t
+    RD_KAFKA_OFFSET_SPEC_MAX_TIMESTAMP = -3
+    RD_KAFKA_OFFSET_SPEC_EARLIEST = -2
+    RD_KAFKA_OFFSET_SPEC_LATEST = -1
+
+    attach_function :rd_kafka_ListOffsets, [:pointer, :pointer, :pointer, :pointer], :void, blocking: true
+    attach_function :rd_kafka_event_ListOffsets_result, [:pointer], :pointer
+    attach_function :rd_kafka_ListOffsets_result_infos, [:pointer, :pointer], :pointer
+    attach_function :rd_kafka_ListOffsetsResultInfo_topic_partition, [:pointer], :pointer
+    attach_function :rd_kafka_ListOffsetsResultInfo_timestamp, [:pointer], :int64
+    attach_function :rd_kafka_AdminOptions_set_isolation_level, [:pointer, :int], :pointer
+    attach_function :rd_kafka_topic_partition_get_leader_epoch, [:pointer], :int32
+
+    # FFI struct for error description (rd_kafka_err_desc)
     class NativeErrorDesc < FFI::Struct
       layout :code, :int,
-             :name, :pointer,
-             :desc, :pointer
+        :name, :pointer,
+        :desc, :pointer
     end
 
     attach_function :rd_kafka_err2name, [:int], :string
     attach_function :rd_kafka_err2str, [:int], :string
+    attach_function :rd_kafka_error_is_fatal, [:pointer], :int
+    attach_function :rd_kafka_error_is_retriable, [:pointer], :int
+    attach_function :rd_kafka_error_txn_requires_abort, [:pointer], :int
+    attach_function :rd_kafka_error_destroy, [:pointer], :void
     attach_function :rd_kafka_get_err_descs, [:pointer, :pointer], :void
 
     # Configuration
@@ -159,6 +235,9 @@ module Rdkafka
 
     attach_function :rd_kafka_conf_new, [], :pointer
     attach_function :rd_kafka_conf_set, [:pointer, :string, :string, :pointer, :int], :kafka_config_response
+    attach_function :rd_kafka_conf_dump, [:pointer, :pointer], :pointer
+    attach_function :rd_kafka_conf_dump_free, [:pointer, :size_t], :void
+    attach_function :rd_kafka_conf_destroy, [:pointer], :void
     callback :log_cb, [:pointer, :int, :string, :string], :void
     attach_function :rd_kafka_conf_set_log_cb, [:pointer, :log_cb], :void
     attach_function :rd_kafka_conf_set_opaque, [:pointer, :pointer], :void
@@ -174,27 +253,37 @@ module Rdkafka
     # Log queue
     attach_function :rd_kafka_set_log_queue, [:pointer, :pointer], :void
     attach_function :rd_kafka_queue_get_main, [:pointer], :pointer
+    attach_function :rd_kafka_queue_get_background, [:pointer], :pointer
+
+    # Queue IO Event Support - for fiber scheduler integration
+    # Enables notifications to a custom FD when queue transitions from empty to non-empty
+    # @param queue rd_kafka_queue_t* - the queue to monitor
+    # @param fd int - file descriptor to write to (provide your own pipe/eventfd)
+    # @param payload const void* - data to write to fd
+    # @param size size_t - size of payload
+    attach_function :rd_kafka_queue_io_event_enable, [:pointer, :int, :pointer, :size_t], :void
     # Per topic configs
     attach_function :rd_kafka_topic_conf_new, [], :pointer
+    attach_function :rd_kafka_topic_conf_destroy, [:pointer], :void
     attach_function :rd_kafka_topic_conf_set, [:pointer, :string, :string, :pointer, :int], :kafka_config_response
 
     LogCallback = FFI::Function.new(
       :void, [:pointer, :int, :string, :string]
     ) do |_client_ptr, level, _level_string, line|
       severity = case level
-                 when 0, 1, 2
-                   Logger::FATAL
-                 when 3
-                   Logger::ERROR
-                 when 4
-                   Logger::WARN
-                 when 5, 6
-                   Logger::INFO
-                 when 7
-                   Logger::DEBUG
-                 else
-                   Logger::UNKNOWN
-                 end
+      when 0, 1, 2
+        Logger::FATAL
+      when 3
+        Logger::ERROR
+      when 4
+        Logger::WARN
+      when 5, 6
+        Logger::INFO
+      when 7
+        Logger::DEBUG
+      else
+        Logger::UNKNOWN
+      end
 
       Rdkafka::Config.ensure_log_thread
       Rdkafka::Config.log_queue << [severity, "rdkafka: #{line}"]
@@ -220,8 +309,8 @@ module Rdkafka
         #
         # Since this cache is shared, having few consumers and/or producers in one process will
         # automatically improve the querying times even with low refresh times.
-        (stats['topics'] || EMPTY_HASH).each do |topic_name, details|
-          partitions_count = details['partitions'].keys.reject { |k| k == '-1' }.size
+        (stats["topics"] || EMPTY_HASH).each do |topic_name, details|
+          partitions_count = details["partitions"].keys.count { |k| !(k == RD_KAFKA_PARTITION_UA_STR) }
 
           next unless partitions_count.positive?
 
@@ -232,14 +321,15 @@ module Rdkafka
       end
 
       # Return 0 so librdkafka frees the json string
-      0
+      RD_KAFKA_RESP_ERR_NO_ERROR
     end
 
     ErrorCallback = FFI::Function.new(
       :void, [:pointer, :int, :string, :pointer]
-    ) do |_client_prr, err_code, reason, _opaque|
+    ) do |client_ptr, err_code, reason, _opaque|
       if Rdkafka::Config.error_callback
-        error = Rdkafka::RdkafkaError.new(err_code, broker_message: reason)
+        instance_name = client_ptr.null? ? nil : Rdkafka::Bindings.rd_kafka_name(client_ptr)
+        error = Rdkafka::RdkafkaError.new(err_code, broker_message: reason, instance_name: instance_name)
         error.set_backtrace(caller)
         Rdkafka::Config.error_callback.call(error)
       end
@@ -266,7 +356,7 @@ module Rdkafka
     OAuthbearerTokenRefreshCallback = FFI::Function.new(
       :void, [:pointer, :string, :pointer]
     ) do |client_ptr, config, _opaque|
-      if Rdkafka::Config.oauthbearer_token_refresh_callback
+      if Rdkafka::Config.oauthbearer_token_refresh_callback && !client_ptr.null?
         Rdkafka::Config.oauthbearer_token_refresh_callback.call(config, Rdkafka::Bindings.rd_kafka_name(client_ptr))
       end
     end
@@ -296,7 +386,13 @@ module Rdkafka
     attach_function :rd_kafka_commit, [:pointer, :pointer, :bool], :int, blocking: true
     attach_function :rd_kafka_poll_set_consumer, [:pointer], :void, blocking: true
     attach_function :rd_kafka_consumer_poll, [:pointer, :int], :pointer, blocking: true
+    # Non-blocking consumer poll variant (does not release GVL)
+    # More efficient for poll(0) calls in fiber schedulers.
+    attach_function :rd_kafka_consumer_poll_nb, :rd_kafka_consumer_poll, [:pointer, :int], :pointer, blocking: false
     attach_function :rd_kafka_consumer_close, [:pointer], :void, blocking: true
+    attach_function :rd_kafka_queue_get_consumer, [:pointer], :pointer
+    attach_function :rd_kafka_consume_batch_queue, [:pointer, :int, :pointer, :size_t], :ssize_t, blocking: true
+    attach_function :rd_kafka_consume_batch_queue_nb, :rd_kafka_consume_batch_queue, [:pointer, :int, :pointer, :size_t], :ssize_t, blocking: false
     attach_function :rd_kafka_offsets_store, [:pointer, :pointer], :int, blocking: true
     attach_function :rd_kafka_pause_partitions, [:pointer, :pointer], :int, blocking: true
     attach_function :rd_kafka_resume_partitions, [:pointer, :pointer], :int, blocking: true
@@ -352,10 +448,9 @@ module Rdkafka
 
     # Stats
 
-    attach_function :rd_kafka_query_watermark_offsets, [:pointer, :string, :int, :pointer, :pointer, :int], :int
+    attach_function :rd_kafka_query_watermark_offsets, [:pointer, :string, :int, :pointer, :pointer, :int], :int, blocking: true
 
     # Producer
-
     RD_KAFKA_VTYPE_END = 0
     RD_KAFKA_VTYPE_TOPIC = 1
     RD_KAFKA_VTYPE_RKT = 2
@@ -377,16 +472,25 @@ module Rdkafka
     callback :delivery_cb, [:pointer, :pointer, :pointer], :void
     attach_function :rd_kafka_conf_set_dr_msg_cb, [:pointer, :delivery_cb], :void
 
-    # Partitioner
-    PARTITIONERS = %w(random consistent consistent_random murmur2 murmur2_random fnv1a fnv1a_random).each_with_object({}) do |name, hsh|
-      method_name = "rd_kafka_msg_partitioner_#{name}".to_sym
+    # Hash mapping partitioner names to their FFI function symbols
+    # @return [Hash{String => Symbol}]
+    PARTITIONERS = %w[random consistent consistent_random murmur2 murmur2_random fnv1a fnv1a_random].each_with_object({}) do |name, hsh|
+      method_name = :"rd_kafka_msg_partitioner_#{name}"
       attach_function method_name, [:pointer, :pointer, :size_t, :int32, :pointer, :pointer], :int32
       hsh[name] = method_name
     end
 
+    # Calculates the partition for a message based on the partitioner
+    #
+    # @param topic_ptr [FFI::Pointer] pointer to the topic handle
+    # @param str [String] the partition key string
+    # @param partition_count [Integer, nil] number of partitions
+    # @param partitioner [String] name of the partitioner to use
+    # @return [Integer] partition number or RD_KAFKA_PARTITION_UA if unassigned
+    # @raise [Rdkafka::Config::ConfigError] when an unknown partitioner is specified
     def self.partitioner(topic_ptr, str, partition_count, partitioner = "consistent_random")
       # Return RD_KAFKA_PARTITION_UA(unassigned partition) when partition count is nil/zero.
-      return -1 unless partition_count&.nonzero?
+      return RD_KAFKA_PARTITION_UA unless partition_count&.nonzero?
 
       str_ptr = str.empty? ? FFI::MemoryPointer::NULL : FFI::MemoryPointer.from_string(str)
       method_name = PARTITIONERS.fetch(partitioner) do
@@ -397,8 +501,7 @@ module Rdkafka
     end
 
     # Create Topics
-
-    RD_KAFKA_ADMIN_OP_CREATETOPICS     = 1   # rd_kafka_admin_op_t
+    RD_KAFKA_ADMIN_OP_CREATETOPICS = 1   # rd_kafka_admin_op_t
     RD_KAFKA_EVENT_CREATETOPICS_RESULT = 100 # rd_kafka_event_type_t
 
     attach_function :rd_kafka_CreateTopics, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
@@ -409,8 +512,7 @@ module Rdkafka
     attach_function :rd_kafka_CreateTopics_result_topics, [:pointer, :pointer], :pointer, blocking: true
 
     # Delete Topics
-
-    RD_KAFKA_ADMIN_OP_DELETETOPICS     = 2   # rd_kafka_admin_op_t
+    RD_KAFKA_ADMIN_OP_DELETETOPICS = 2   # rd_kafka_admin_op_t
     RD_KAFKA_EVENT_DELETETOPICS_RESULT = 101 # rd_kafka_event_type_t
 
     attach_function :rd_kafka_DeleteTopics, [:pointer, :pointer, :size_t, :pointer, :pointer], :int32, blocking: true
@@ -430,7 +532,6 @@ module Rdkafka
     attach_function :rd_kafka_CreatePartitions_result_topics, [:pointer, :pointer], :pointer
 
     # Delete Group
-
     RD_KAFKA_ADMIN_OP_DELETEGROUPS = 7   # rd_kafka_admin_op_t
     RD_KAFKA_EVENT_DELETEGROUPS_RESULT = 106 # rd_kafka_event_type_t
 
@@ -442,7 +543,6 @@ module Rdkafka
 
     # Background Queue and Callback
 
-    attach_function :rd_kafka_queue_get_background, [:pointer], :pointer
     attach_function :rd_kafka_conf_set_background_event_cb, [:pointer, :pointer], :void
     attach_function :rd_kafka_queue_destroy, [:pointer], :void
 
@@ -456,6 +556,7 @@ module Rdkafka
 
     attach_function :rd_kafka_event_type, [:pointer], :int32
     attach_function :rd_kafka_event_opaque, [:pointer], :pointer
+    attach_function :rd_kafka_event_destroy, [:pointer], :void
 
     # Extracting data from topic results
 
@@ -464,8 +565,7 @@ module Rdkafka
     attach_function :rd_kafka_topic_result_name, [:pointer], :pointer
 
     # Create Acls
-
-    RD_KAFKA_ADMIN_OP_CREATEACLS     = 9
+    RD_KAFKA_ADMIN_OP_CREATEACLS = 9
     RD_KAFKA_EVENT_CREATEACLS_RESULT = 1024
 
     attach_function :rd_kafka_CreateAcls, [:pointer, :pointer, :size_t, :pointer, :pointer], :void
@@ -473,8 +573,7 @@ module Rdkafka
     attach_function :rd_kafka_CreateAcls_result_acls, [:pointer, :pointer], :pointer
 
     # Delete Acls
-
-    RD_KAFKA_ADMIN_OP_DELETEACLS     = 11
+    RD_KAFKA_ADMIN_OP_DELETEACLS = 11
     RD_KAFKA_EVENT_DELETEACLS_RESULT = 4096
 
     attach_function :rd_kafka_DeleteAcls, [:pointer, :pointer, :size_t, :pointer, :pointer], :void
@@ -484,8 +583,7 @@ module Rdkafka
     attach_function :rd_kafka_DeleteAcls_result_response_matching_acls, [:pointer, :pointer], :pointer
 
     # Describe Acls
-
-    RD_KAFKA_ADMIN_OP_DESCRIBEACLS     = 10
+    RD_KAFKA_ADMIN_OP_DESCRIBEACLS = 10
     RD_KAFKA_EVENT_DESCRIBEACLS_RESULT = 2048
 
     attach_function :rd_kafka_DescribeAcls, [:pointer, :pointer, :pointer, :pointer], :void
@@ -501,45 +599,42 @@ module Rdkafka
     attach_function :rd_kafka_AclBinding_host, [:pointer], :pointer
     attach_function :rd_kafka_AclBinding_operation, [:pointer], :int32
     attach_function :rd_kafka_AclBinding_permission_type, [:pointer], :int32
-    attach_function :rd_kafka_AclBinding_new, [:int32, :pointer, :int32, :pointer, :pointer, :int32, :int32, :pointer, :size_t ], :pointer
-    attach_function :rd_kafka_AclBindingFilter_new, [:int32, :pointer, :int32, :pointer, :pointer, :int32, :int32, :pointer, :size_t ], :pointer
+    attach_function :rd_kafka_AclBinding_new, [:int32, :pointer, :int32, :pointer, :pointer, :int32, :int32, :pointer, :size_t], :pointer
+    attach_function :rd_kafka_AclBindingFilter_new, [:int32, :pointer, :int32, :pointer, :pointer, :int32, :int32, :pointer, :size_t], :pointer
     attach_function :rd_kafka_AclBinding_destroy, [:pointer], :void
 
     # rd_kafka_ResourceType_t - https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7307
-
-    RD_KAFKA_RESOURCE_ANY   = 1
+    RD_KAFKA_RESOURCE_ANY = 1
     RD_KAFKA_RESOURCE_TOPIC = 2
     RD_KAFKA_RESOURCE_GROUP = 3
     RD_KAFKA_RESOURCE_BROKER = 4
     RD_KAFKA_RESOURCE_TRANSACTIONAL_ID = 5
 
     # rd_kafka_ResourcePatternType_t - https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7320
-
-    RD_KAFKA_RESOURCE_PATTERN_ANY      = 1
-    RD_KAFKA_RESOURCE_PATTERN_MATCH    = 2
-    RD_KAFKA_RESOURCE_PATTERN_LITERAL  = 3
+    RD_KAFKA_RESOURCE_PATTERN_UNKNOWN = 0
+    RD_KAFKA_RESOURCE_PATTERN_ANY = 1
+    RD_KAFKA_RESOURCE_PATTERN_MATCH = 2
+    RD_KAFKA_RESOURCE_PATTERN_LITERAL = 3
     RD_KAFKA_RESOURCE_PATTERN_PREFIXED = 4
 
     # rd_kafka_AclOperation_t - https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8403
-
-    RD_KAFKA_ACL_OPERATION_ANY              = 1
-    RD_KAFKA_ACL_OPERATION_ALL              = 2
-    RD_KAFKA_ACL_OPERATION_READ             = 3
-    RD_KAFKA_ACL_OPERATION_WRITE            = 4
-    RD_KAFKA_ACL_OPERATION_CREATE           = 5
-    RD_KAFKA_ACL_OPERATION_DELETE           = 6
-    RD_KAFKA_ACL_OPERATION_ALTER            = 7
-    RD_KAFKA_ACL_OPERATION_DESCRIBE         = 8
-    RD_KAFKA_ACL_OPERATION_CLUSTER_ACTION   = 9
+    RD_KAFKA_ACL_OPERATION_ANY = 1
+    RD_KAFKA_ACL_OPERATION_ALL = 2
+    RD_KAFKA_ACL_OPERATION_READ = 3
+    RD_KAFKA_ACL_OPERATION_WRITE = 4
+    RD_KAFKA_ACL_OPERATION_CREATE = 5
+    RD_KAFKA_ACL_OPERATION_DELETE = 6
+    RD_KAFKA_ACL_OPERATION_ALTER = 7
+    RD_KAFKA_ACL_OPERATION_DESCRIBE = 8
+    RD_KAFKA_ACL_OPERATION_CLUSTER_ACTION = 9
     RD_KAFKA_ACL_OPERATION_DESCRIBE_CONFIGS = 10
-    RD_KAFKA_ACL_OPERATION_ALTER_CONFIGS    = 11
+    RD_KAFKA_ACL_OPERATION_ALTER_CONFIGS = 11
     RD_KAFKA_ACL_OPERATION_IDEMPOTENT_WRITE = 12
 
     # rd_kafka_AclPermissionType_t - https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8435
-
-    RD_KAFKA_ACL_PERMISSION_TYPE_ANY     = 1
-    RD_KAFKA_ACL_PERMISSION_TYPE_DENY    = 2
-    RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW   = 3
+    RD_KAFKA_ACL_PERMISSION_TYPE_ANY = 1
+    RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
+    RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
 
     # Extracting error details from Acl results
     attach_function :rd_kafka_acl_result_error, [:pointer], :pointer
@@ -549,14 +644,13 @@ module Rdkafka
     attach_function :rd_kafka_event_error_string, [:pointer], :pointer
     attach_function :rd_kafka_AclBinding_error, [:pointer], :pointer
 
-
-    # Extracting data from group results
-    class NativeError < FFI::Struct # rd_kafka_error_t
+    # FFI struct for native error (rd_kafka_error_t)
+    class NativeError < FFI::Struct
       layout :code, :int32,
-             :errstr, :pointer,
-             :fatal, :u_int8_t,
-             :retriable, :u_int8_t,
-             :txn_requires_abort, :u_int8_t
+        :errstr, :pointer,
+        :fatal, :u_int8_t,
+        :retriable, :u_int8_t,
+        :txn_requires_abort, :u_int8_t
     end
 
     attach_function :rd_kafka_group_result_error, [:pointer], NativeError.by_ref # rd_kafka_group_result_t* => rd_kafka_error_t*

data/lib/rdkafka/callbacks/base_handler.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  module Callbacks
+    # Base class for admin background-event result handlers.
+    #
+    # Each subclass handles one librdkafka admin result event type: it extracts the result from
+    # the event, looks up the matching {AbstractHandle} in the registry and resolves it (sets the
+    # response, builds the report, copies the broker message, then unlocks it).
+    #
+    # The event is owned by the application and destroyed by {BackgroundEventCallback} right after
+    # dispatch, so handlers must copy everything they need out of event-owned memory into Ruby
+    # objects before returning.
+    #
+    # @private
+    class BaseHandler
+      class << self
+        # Processes a background event for the operation this handler is responsible for.
+        # @param event_ptr [FFI::Pointer] pointer to the event
+        # @return [void]
+        def call(event_ptr)
+          raise NotImplementedError, "#{name} must implement .call"
+        end
+
+        private
+
+        # Copies an event-owned C string into a Ruby string
+        # @param ptr [FFI::Pointer] pointer to the C string
+        # @return [String, nil] copied string or nil when NULL
+        def read_event_string(ptr)
+          ptr.null? ? nil : ptr.read_string
+        end
+
+        # Resolves a handle directly from an operation-level event error and unlocks it.
+        #
+        # librdkafka signals a failure of the whole admin operation (e.g. `_TIMED_OUT` when the
+        # brokers are unreachable, or `_DESTROY` when the client is closed with the request in
+        # flight) by setting the event error and delivering an *empty* results array. The
+        # per-element result handlers must not index into that empty array; they call this first
+        # and skip results parsing when it returns true. Without this, indexing `results[0]`
+        # raises inside the FFI callback, the handle is never unlocked, and the caller's `wait`
+        # blocks until its own timeout and raises `WaitTimeoutError`, losing the real error code.
+        #
+        # @param event_ptr [FFI::Pointer] pointer to the event
+        # @param handle [Rdkafka::AbstractHandle] the handle to resolve
+        # @return [Boolean] true when the event carried an operation-level error and the handle
+        #   was resolved here
+        def resolve_operation_error(event_ptr, handle)
+          error_code = Rdkafka::Bindings.rd_kafka_event_error(event_ptr)
+
+          return false if error_code == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+
+          handle[:response] = error_code
+          handle.broker_message = read_event_string(Rdkafka::Bindings.rd_kafka_event_error_string(event_ptr))
+          handle.unlock
+
+          true
+        end
+      end
+    end
+  end
+end

data/lib/rdkafka/callbacks/create_acl_handler.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  module Callbacks
+    # Handles `RD_KAFKA_EVENT_CREATEACLS_RESULT` events
+    # @private
+    class CreateAclHandler < BaseHandler
+      class << self
+        # Resolves the create-ACL handle from its result event
+        # @param event_ptr [FFI::Pointer] pointer to the event
+        # @return [void]
+        def call(event_ptr)
+          create_acls_result = Rdkafka::Bindings.rd_kafka_event_CreateAcls_result(event_ptr)
+
+          # Get the number of acl results
+          pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
+          create_acl_result_array = Rdkafka::Bindings.rd_kafka_CreateAcls_result_acls(create_acls_result, pointer_to_size_t)
+          create_acl_results = CreateAclResult.create_acl_results_from_array(pointer_to_size_t.read_int, create_acl_result_array)
+          create_acl_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
+
+          if create_acl_handle = Rdkafka::Admin::CreateAclHandle.remove(create_acl_handle_ptr.address)
+            unless resolve_operation_error(event_ptr, create_acl_handle)
+              create_acl_handle[:response] = create_acl_results[0].result_error
+              create_acl_handle.result = Rdkafka::Admin::CreateAclReport.new(
+                rdkafka_response: create_acl_results[0].result_error,
+                rdkafka_response_string: create_acl_results[0].error_string
+              )
+              create_acl_handle.broker_message = create_acl_handle.result.rdkafka_response_string
+
+              create_acl_handle.unlock
+            end
+          end
+        end
+      end
+    end
+  end
+end