karafka-rdkafka 0.30.2-aarch64-linux-gnu → 0.32.0-aarch64-linux-gnu

data/lib/rdkafka/bindings/admin/list_offsets.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  module Bindings
+    # List Offsets
+    RD_KAFKA_ADMIN_OP_LISTOFFSETS = 21
+    RD_KAFKA_EVENT_LISTOFFSETS_RESULT = 0x400000
+
+    # rd_kafka_IsolationLevel_t
+    RD_KAFKA_ISOLATION_LEVEL_READ_UNCOMMITTED = 0
+    RD_KAFKA_ISOLATION_LEVEL_READ_COMMITTED = 1
+
+    # rd_kafka_OffsetSpec_t
+    RD_KAFKA_OFFSET_SPEC_MAX_TIMESTAMP = -3
+    RD_KAFKA_OFFSET_SPEC_EARLIEST = -2
+    RD_KAFKA_OFFSET_SPEC_LATEST = -1
+
+    attach_function :rd_kafka_ListOffsets, [:pointer, :pointer, :pointer, :pointer], :void, blocking: true
+    attach_function :rd_kafka_event_ListOffsets_result, [:pointer], :pointer
+    attach_function :rd_kafka_ListOffsets_result_infos, [:pointer, :pointer], :pointer
+    attach_function :rd_kafka_ListOffsetsResultInfo_topic_partition, [:pointer], :pointer
+    attach_function :rd_kafka_ListOffsetsResultInfo_timestamp, [:pointer], :int64
+    attach_function :rd_kafka_AdminOptions_set_isolation_level, [:pointer, :int], :pointer
+    attach_function :rd_kafka_topic_partition_get_leader_epoch, [:pointer], :int32
+  end
+end

data/lib/rdkafka/bindings/admin.rb

@@ -0,0 +1,211 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  module Bindings
+    # Create Topics
+    RD_KAFKA_ADMIN_OP_CREATETOPICS = 1   # rd_kafka_admin_op_t
+    RD_KAFKA_EVENT_CREATETOPICS_RESULT = 100 # rd_kafka_event_type_t
+
+    attach_function :rd_kafka_CreateTopics, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
+    attach_function :rd_kafka_NewTopic_new, [:pointer, :size_t, :size_t, :pointer, :size_t], :pointer, blocking: true
+    attach_function :rd_kafka_NewTopic_set_config, [:pointer, :string, :string], :int32, blocking: true
+    attach_function :rd_kafka_NewTopic_destroy, [:pointer], :void, blocking: true
+    attach_function :rd_kafka_event_CreateTopics_result, [:pointer], :pointer, blocking: true
+    attach_function :rd_kafka_CreateTopics_result_topics, [:pointer, :pointer], :pointer, blocking: true
+
+    # Delete Topics
+    RD_KAFKA_ADMIN_OP_DELETETOPICS = 2   # rd_kafka_admin_op_t
+    RD_KAFKA_EVENT_DELETETOPICS_RESULT = 101 # rd_kafka_event_type_t
+
+    attach_function :rd_kafka_DeleteTopics, [:pointer, :pointer, :size_t, :pointer, :pointer], :int32, blocking: true
+    attach_function :rd_kafka_DeleteTopic_new, [:pointer], :pointer, blocking: true
+    attach_function :rd_kafka_DeleteTopic_destroy, [:pointer], :void, blocking: true
+    attach_function :rd_kafka_event_DeleteTopics_result, [:pointer], :pointer, blocking: true
+    attach_function :rd_kafka_DeleteTopics_result_topics, [:pointer, :pointer], :pointer, blocking: true
+
+    # Create partitions
+    RD_KAFKA_ADMIN_OP_CREATEPARTITIONS = 3
+    RD_KAFKA_ADMIN_OP_CREATEPARTITIONS_RESULT = 102
+
+    attach_function :rd_kafka_CreatePartitions, [:pointer, :pointer, :size_t, :pointer, :pointer], :void
+    attach_function :rd_kafka_NewPartitions_new, %i[pointer size_t pointer size_t], :pointer
+    attach_function :rd_kafka_NewPartitions_destroy, [:pointer], :void
+    attach_function :rd_kafka_event_CreatePartitions_result, [:pointer], :pointer
+    attach_function :rd_kafka_CreatePartitions_result_topics, [:pointer, :pointer], :pointer
+
+    # Delete Group
+    RD_KAFKA_ADMIN_OP_DELETEGROUPS = 7   # rd_kafka_admin_op_t
+    RD_KAFKA_EVENT_DELETEGROUPS_RESULT = 106 # rd_kafka_event_type_t
+
+    attach_function :rd_kafka_DeleteGroups, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
+    attach_function :rd_kafka_DeleteGroup_new, [:pointer], :pointer, blocking: true
+    attach_function :rd_kafka_DeleteGroup_destroy, [:pointer], :void, blocking: true
+    attach_function :rd_kafka_event_DeleteGroups_result, [:pointer], :pointer, blocking: true # rd_kafka_event_t* => rd_kafka_DeleteGroups_result_t*
+    attach_function :rd_kafka_DeleteGroups_result_groups, [:pointer, :pointer], :pointer, blocking: true # rd_kafka_DeleteGroups_result_t*, size_t* => rd_kafka_group_result_t**
+
+    # Alter Consumer Group Offsets
+    RD_KAFKA_ADMIN_OP_ALTERCONSUMERGROUPOFFSETS = 15   # rd_kafka_admin_op_t
+    RD_KAFKA_EVENT_ALTERCONSUMERGROUPOFFSETS_RESULT = 0x10000 # rd_kafka_event_type_t
+
+    attach_function :rd_kafka_AlterConsumerGroupOffsets, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
+    attach_function :rd_kafka_AlterConsumerGroupOffsets_new, [:pointer, :pointer], :pointer, blocking: true # const char*, const rd_kafka_topic_partition_list_t* => rd_kafka_AlterConsumerGroupOffsets_t*
+    attach_function :rd_kafka_AlterConsumerGroupOffsets_destroy, [:pointer], :void, blocking: true
+    attach_function :rd_kafka_event_AlterConsumerGroupOffsets_result, [:pointer], :pointer, blocking: true # rd_kafka_event_t* => rd_kafka_AlterConsumerGroupOffsets_result_t*
+    attach_function :rd_kafka_AlterConsumerGroupOffsets_result_groups, [:pointer, :pointer], :pointer, blocking: true # rd_kafka_AlterConsumerGroupOffsets_result_t*, size_t* => rd_kafka_group_result_t**
+
+    # List Consumer Group Offsets
+    RD_KAFKA_ADMIN_OP_LISTCONSUMERGROUPOFFSETS = 14   # rd_kafka_admin_op_t
+    RD_KAFKA_EVENT_LISTCONSUMERGROUPOFFSETS_RESULT = 0x8000 # rd_kafka_event_type_t
+
+    attach_function :rd_kafka_ListConsumerGroupOffsets, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
+    attach_function :rd_kafka_ListConsumerGroupOffsets_new, [:pointer, :pointer], :pointer, blocking: true # const char*, const rd_kafka_topic_partition_list_t* => rd_kafka_ListConsumerGroupOffsets_t*
+    attach_function :rd_kafka_ListConsumerGroupOffsets_destroy, [:pointer], :void, blocking: true
+    attach_function :rd_kafka_event_ListConsumerGroupOffsets_result, [:pointer], :pointer, blocking: true # rd_kafka_event_t* => rd_kafka_ListConsumerGroupOffsets_result_t*
+    attach_function :rd_kafka_ListConsumerGroupOffsets_result_groups, [:pointer, :pointer], :pointer, blocking: true # rd_kafka_ListConsumerGroupOffsets_result_t*, size_t* => rd_kafka_group_result_t**
+    attach_function :rd_kafka_AdminOptions_set_require_stable_offsets, [:pointer, :int], :pointer, blocking: true # => rd_kafka_error_t*
+
+    # Delete Consumer Group Offsets
+    RD_KAFKA_ADMIN_OP_DELETECONSUMERGROUPOFFSETS = 8   # rd_kafka_admin_op_t
+    RD_KAFKA_EVENT_DELETECONSUMERGROUPOFFSETS_RESULT = 107 # rd_kafka_event_type_t
+
+    attach_function :rd_kafka_DeleteConsumerGroupOffsets, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
+    attach_function :rd_kafka_DeleteConsumerGroupOffsets_new, [:pointer, :pointer], :pointer, blocking: true # const char*, const rd_kafka_topic_partition_list_t* => rd_kafka_DeleteConsumerGroupOffsets_t*
+    attach_function :rd_kafka_DeleteConsumerGroupOffsets_destroy, [:pointer], :void, blocking: true
+    attach_function :rd_kafka_event_DeleteConsumerGroupOffsets_result, [:pointer], :pointer, blocking: true # rd_kafka_event_t* => rd_kafka_DeleteConsumerGroupOffsets_result_t*
+    attach_function :rd_kafka_DeleteConsumerGroupOffsets_result_groups, [:pointer, :pointer], :pointer, blocking: true # rd_kafka_DeleteConsumerGroupOffsets_result_t*, size_t* => rd_kafka_group_result_t**
+
+    # Delete Records
+    RD_KAFKA_ADMIN_OP_DELETERECORDS = 6   # rd_kafka_admin_op_t
+    RD_KAFKA_EVENT_DELETERECORDS_RESULT = 105 # rd_kafka_event_type_t
+
+    attach_function :rd_kafka_DeleteRecords, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
+    attach_function :rd_kafka_DeleteRecords_new, [:pointer], :pointer, blocking: true
+    attach_function :rd_kafka_DeleteRecords_destroy, [:pointer], :void, blocking: true
+    attach_function :rd_kafka_event_DeleteRecords_result, [:pointer], :pointer, blocking: true # rd_kafka_event_t* => rd_kafka_DeleteRecords_result_t*
+    attach_function :rd_kafka_DeleteRecords_result_offsets, [:pointer], :pointer, blocking: true # rd_kafka_DeleteRecords_result_t* => rd_kafka_topic_partition_list_t*
+
+    # List Consumer Groups
+    RD_KAFKA_ADMIN_OP_LISTCONSUMERGROUPS = 12   # rd_kafka_admin_op_t
+    RD_KAFKA_EVENT_LISTCONSUMERGROUPS_RESULT = 0x2000 # rd_kafka_event_type_t
+
+    # Consumer group states (rd_kafka_consumer_group_state_t)
+    RD_KAFKA_CONSUMER_GROUP_STATE_UNKNOWN = 0
+    RD_KAFKA_CONSUMER_GROUP_STATE_PREPARING_REBALANCE = 1
+    RD_KAFKA_CONSUMER_GROUP_STATE_COMPLETING_REBALANCE = 2
+    RD_KAFKA_CONSUMER_GROUP_STATE_STABLE = 3
+    RD_KAFKA_CONSUMER_GROUP_STATE_DEAD = 4
+    RD_KAFKA_CONSUMER_GROUP_STATE_EMPTY = 5
+
+    attach_function :rd_kafka_ListConsumerGroups, [:pointer, :pointer, :pointer], :void, blocking: true
+    attach_function :rd_kafka_event_ListConsumerGroups_result, [:pointer], :pointer, blocking: true # rd_kafka_event_t* => rd_kafka_ListConsumerGroups_result_t*
+    attach_function :rd_kafka_ListConsumerGroups_result_valid, [:pointer, :pointer], :pointer, blocking: true # result*, size_t* => rd_kafka_ConsumerGroupListing_t**
+    attach_function :rd_kafka_ListConsumerGroups_result_errors, [:pointer, :pointer], :pointer, blocking: true # result*, size_t* => rd_kafka_error_t**
+    attach_function :rd_kafka_ConsumerGroupListing_group_id, [:pointer], :pointer, blocking: true # => const char*
+    attach_function :rd_kafka_ConsumerGroupListing_is_simple_consumer_group, [:pointer], :int, blocking: true
+    attach_function :rd_kafka_ConsumerGroupListing_state, [:pointer], :int, blocking: true
+    attach_function :rd_kafka_consumer_group_state_name, [:int], :pointer, blocking: true # => const char*
+
+    # Describe Consumer Groups
+    RD_KAFKA_ADMIN_OP_DESCRIBECONSUMERGROUPS = 13   # rd_kafka_admin_op_t
+    RD_KAFKA_EVENT_DESCRIBECONSUMERGROUPS_RESULT = 0x4000 # rd_kafka_event_type_t
+
+    # Consumer group types (rd_kafka_consumer_group_type_t)
+    RD_KAFKA_CONSUMER_GROUP_TYPE_UNKNOWN = 0
+    RD_KAFKA_CONSUMER_GROUP_TYPE_CONSUMER = 1
+    RD_KAFKA_CONSUMER_GROUP_TYPE_CLASSIC = 2
+
+    attach_function :rd_kafka_DescribeConsumerGroups, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
+    attach_function :rd_kafka_event_DescribeConsumerGroups_result, [:pointer], :pointer, blocking: true # rd_kafka_event_t* => rd_kafka_DescribeConsumerGroups_result_t*
+    attach_function :rd_kafka_DescribeConsumerGroups_result_groups, [:pointer, :pointer], :pointer, blocking: true # result*, size_t* => rd_kafka_ConsumerGroupDescription_t**
+    attach_function :rd_kafka_ConsumerGroupDescription_group_id, [:pointer], :pointer, blocking: true # => const char*
+    attach_function :rd_kafka_ConsumerGroupDescription_error, [:pointer], :pointer, blocking: true # => const rd_kafka_error_t*
+    attach_function :rd_kafka_ConsumerGroupDescription_is_simple_consumer_group, [:pointer], :int, blocking: true
+    attach_function :rd_kafka_ConsumerGroupDescription_partition_assignor, [:pointer], :pointer, blocking: true # => const char*
+    attach_function :rd_kafka_ConsumerGroupDescription_state, [:pointer], :int, blocking: true
+    attach_function :rd_kafka_ConsumerGroupDescription_type, [:pointer], :int, blocking: true
+    attach_function :rd_kafka_ConsumerGroupDescription_authorized_operations, [:pointer, :pointer], :pointer, blocking: true # grpdesc*, size_t* => const rd_kafka_AclOperation_t*
+    attach_function :rd_kafka_ConsumerGroupDescription_coordinator, [:pointer], :pointer, blocking: true # => const rd_kafka_Node_t*
+    attach_function :rd_kafka_ConsumerGroupDescription_member_count, [:pointer], :size_t, blocking: true
+    attach_function :rd_kafka_ConsumerGroupDescription_member, [:pointer, :size_t], :pointer, blocking: true # => const rd_kafka_MemberDescription_t*
+    attach_function :rd_kafka_MemberDescription_client_id, [:pointer], :pointer, blocking: true # => const char*
+    attach_function :rd_kafka_MemberDescription_group_instance_id, [:pointer], :pointer, blocking: true # => const char*
+    attach_function :rd_kafka_MemberDescription_consumer_id, [:pointer], :pointer, blocking: true # => const char*
+    attach_function :rd_kafka_MemberDescription_host, [:pointer], :pointer, blocking: true # => const char*
+    attach_function :rd_kafka_MemberDescription_assignment, [:pointer], :pointer, blocking: true # => const rd_kafka_MemberAssignment_t*
+    attach_function :rd_kafka_MemberDescription_target_assignment, [:pointer], :pointer, blocking: true # => const rd_kafka_MemberAssignment_t*
+    attach_function :rd_kafka_MemberAssignment_partitions, [:pointer], :pointer, blocking: true # => const rd_kafka_topic_partition_list_t*
+    attach_function :rd_kafka_consumer_group_type_name, [:int], :pointer, blocking: true # => const char*
+
+    # Describe Topics and Describe Cluster
+    RD_KAFKA_ADMIN_OP_DESCRIBETOPICS = 19   # rd_kafka_admin_op_t
+    RD_KAFKA_EVENT_DESCRIBETOPICS_RESULT = 0x100000 # rd_kafka_event_type_t
+    RD_KAFKA_ADMIN_OP_DESCRIBECLUSTER = 20   # rd_kafka_admin_op_t
+    RD_KAFKA_EVENT_DESCRIBECLUSTER_RESULT = 0x200000 # rd_kafka_event_type_t
+
+    attach_function :rd_kafka_AdminOptions_set_include_authorized_operations, [:pointer, :int], :pointer, blocking: true # => rd_kafka_error_t*
+    attach_function :rd_kafka_TopicCollection_of_topic_names, [:pointer, :size_t], :pointer, blocking: true # const char**, size_t => rd_kafka_TopicCollection_t*
+    attach_function :rd_kafka_TopicCollection_destroy, [:pointer], :void, blocking: true
+    attach_function :rd_kafka_DescribeTopics, [:pointer, :pointer, :pointer, :pointer], :void, blocking: true
+    attach_function :rd_kafka_event_DescribeTopics_result, [:pointer], :pointer, blocking: true # rd_kafka_event_t* => rd_kafka_DescribeTopics_result_t*
+    attach_function :rd_kafka_DescribeTopics_result_topics, [:pointer, :pointer], :pointer, blocking: true # result*, size_t* => rd_kafka_TopicDescription_t**
+    attach_function :rd_kafka_TopicDescription_name, [:pointer], :pointer, blocking: true # => const char*
+    attach_function :rd_kafka_TopicDescription_topic_id, [:pointer], :pointer, blocking: true # => rd_kafka_Uuid_t*
+    attach_function :rd_kafka_TopicDescription_is_internal, [:pointer], :int, blocking: true
+    attach_function :rd_kafka_TopicDescription_partitions, [:pointer, :pointer], :pointer, blocking: true # desc*, size_t* => rd_kafka_TopicPartitionInfo_t**
+    attach_function :rd_kafka_TopicDescription_authorized_operations, [:pointer, :pointer], :pointer, blocking: true # desc*, size_t* => rd_kafka_AclOperation_t*
+    attach_function :rd_kafka_TopicDescription_error, [:pointer], :pointer, blocking: true # => rd_kafka_error_t*
+    attach_function :rd_kafka_TopicPartitionInfo_partition, [:pointer], :int, blocking: true
+    attach_function :rd_kafka_TopicPartitionInfo_leader, [:pointer], :pointer, blocking: true # => rd_kafka_Node_t*
+    attach_function :rd_kafka_TopicPartitionInfo_replicas, [:pointer, :pointer], :pointer, blocking: true # info*, size_t* => rd_kafka_Node_t**
+    attach_function :rd_kafka_TopicPartitionInfo_isr, [:pointer, :pointer], :pointer, blocking: true # info*, size_t* => rd_kafka_Node_t**
+    attach_function :rd_kafka_Uuid_base64str, [:pointer], :pointer, blocking: true # => const char*
+    attach_function :rd_kafka_Uuid_most_significant_bits, [:pointer], :int64, blocking: true
+    attach_function :rd_kafka_Uuid_least_significant_bits, [:pointer], :int64, blocking: true
+    attach_function :rd_kafka_DescribeCluster, [:pointer, :pointer, :pointer], :void, blocking: true
+    attach_function :rd_kafka_event_DescribeCluster_result, [:pointer], :pointer, blocking: true # rd_kafka_event_t* => rd_kafka_DescribeCluster_result_t*
+    attach_function :rd_kafka_DescribeCluster_result_nodes, [:pointer, :pointer], :pointer, blocking: true # result*, size_t* => rd_kafka_Node_t**
+    attach_function :rd_kafka_DescribeCluster_result_controller, [:pointer], :pointer, blocking: true # => rd_kafka_Node_t*
+    attach_function :rd_kafka_DescribeCluster_result_cluster_id, [:pointer], :pointer, blocking: true # => const char*
+    attach_function :rd_kafka_DescribeCluster_result_authorized_operations, [:pointer, :pointer], :pointer, blocking: true # result*, size_t* => rd_kafka_AclOperation_t*
+    attach_function :rd_kafka_Node_id, [:pointer], :int, blocking: true
+    attach_function :rd_kafka_Node_host, [:pointer], :pointer, blocking: true # => const char*
+    attach_function :rd_kafka_Node_port, [:pointer], :uint16, blocking: true
+    attach_function :rd_kafka_Node_rack, [:pointer], :pointer, blocking: true # => const char*
+
+    # Background Queue and Callback
+
+    attach_function :rd_kafka_conf_set_background_event_cb, [:pointer, :pointer], :void
+    attach_function :rd_kafka_queue_destroy, [:pointer], :void
+
+    # Admin Options
+
+    attach_function :rd_kafka_AdminOptions_new, [:pointer, :int32], :pointer
+    attach_function :rd_kafka_AdminOptions_set_opaque, [:pointer, :pointer], :void
+    attach_function :rd_kafka_AdminOptions_destroy, [:pointer], :void
+    attach_function :rd_kafka_AdminOptions_set_request_timeout, [:pointer, :int, :pointer, :size_t], :int32
+    attach_function :rd_kafka_AdminOptions_set_operation_timeout, [:pointer, :int, :pointer, :size_t], :int32
+    attach_function :rd_kafka_AdminOptions_set_validate_only, [:pointer, :int, :pointer, :size_t], :int32
+
+    # Extracting data from event types
+
+    attach_function :rd_kafka_event_type, [:pointer], :int32
+    attach_function :rd_kafka_event_opaque, [:pointer], :pointer
+    attach_function :rd_kafka_event_destroy, [:pointer], :void
+    attach_function :rd_kafka_event_error, [:pointer], :int32
+    attach_function :rd_kafka_event_error_string, [:pointer], :pointer
+
+    # Extracting data from topic results
+
+    attach_function :rd_kafka_topic_result_error, [:pointer], :int32
+    attach_function :rd_kafka_topic_result_error_string, [:pointer], :pointer
+    attach_function :rd_kafka_topic_result_name, [:pointer], :pointer
+
+    attach_function :rd_kafka_group_result_error, [:pointer], NativeError.by_ref # rd_kafka_group_result_t* => rd_kafka_error_t*
+    attach_function :rd_kafka_group_result_name, [:pointer], :pointer
+    attach_function :rd_kafka_group_result_partitions, [:pointer], :pointer # rd_kafka_group_result_t* => rd_kafka_topic_partition_list_t*
+  end
+end
+
+require_relative "admin/configs"
+require_relative "admin/list_offsets"
+require_relative "admin/acls"

data/lib/rdkafka/bindings/callbacks.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  module Bindings
+    LogCallback = FFI::Function.new(
+      :void, [:pointer, :int, :string, :string]
+    ) do |_client_ptr, level, _level_string, line|
+      severity = case level
+      when 0, 1, 2
+        Logger::FATAL
+      when 3
+        Logger::ERROR
+      when 4
+        Logger::WARN
+      when 5, 6
+        Logger::INFO
+      when 7
+        Logger::DEBUG
+      else
+        Logger::UNKNOWN
+      end
+
+      Rdkafka::Config.ensure_log_thread
+      Rdkafka::Config.log_queue << [severity, "rdkafka: #{line}"]
+    end
+
+    StatsCallback = FFI::Function.new(
+      :int, [:pointer, :string, :int, :pointer]
+    ) do |_client_ptr, json, _json_len, _opaque|
+      # Read the callback once: another thread may clear it while this one runs
+      statistics_callback = Rdkafka::Config.statistics_callback
+
+      if statistics_callback
+        stats = JSON.parse(json)
+
+        # If user requested statistics callbacks, we can use the statistics data to get the
+        # partitions count for each topic when this data is published. That way we do not have
+        # to query this information when user is using `partition_key`. This takes around 0.02ms
+        # every statistics interval period (most likely every 5 seconds) and saves us from making
+        # any queries to the cluster for the partition count.
+        #
+        # One edge case is if user would set the `statistics.interval.ms` much higher than the
+        # default current partition count refresh (30 seconds). This is taken care of as the lack
+        # of reporting to the partitions cache will cause cache expire and blocking refresh.
+        #
+        # If user sets `topic.metadata.refresh.interval.ms` too high this is on the user.
+        #
+        # Since this cache is shared, having few consumers and/or producers in one process will
+        # automatically improve the querying times even with low refresh times.
+        (stats["topics"] || EMPTY_HASH).each do |topic_name, details|
+          partitions_count = details["partitions"].keys.count { |k| !(k == RD_KAFKA_PARTITION_UA_STR) }
+
+          next unless partitions_count.positive?
+
+          Rdkafka::Producer.partitions_count_cache.set(topic_name, partitions_count)
+        end
+
+        statistics_callback.call(stats)
+      end
+
+      # Return 0 so librdkafka frees the json string
+      RD_KAFKA_RESP_ERR_NO_ERROR
+    end
+
+    # Retrieves fatal error details from a kafka client handle.
+    # This is a helper method to extract fatal error information consistently
+    # across different parts of the codebase (callbacks, testing utilities, etc.).
+    #
+    # @param client_ptr [FFI::Pointer] Native kafka client pointer
+    # @return [Hash, nil] Hash with :error_code and :error_string if fatal error occurred,
+    #   nil otherwise
+    #
+    # @example
+    #   details = Rdkafka::Bindings.extract_fatal_error(client_ptr)
+    #   if details
+    #     puts "Fatal error #{details[:error_code]}: #{details[:error_string]}"
+    #   end
+    def self.extract_fatal_error(client_ptr)
+      error_buffer = FFI::MemoryPointer.new(:char, FATAL_ERROR_BUFFER_SIZE)
+
+      error_code = rd_kafka_fatal_error(client_ptr, error_buffer, FATAL_ERROR_BUFFER_SIZE)
+
+      return nil if error_code == RD_KAFKA_RESP_ERR_NO_ERROR
+
+      {
+        error_code: error_code,
+        error_string: error_buffer.read_string
+      }
+    end
+
+    ErrorCallback = FFI::Function.new(
+      :void, [:pointer, :int, :string, :pointer]
+    ) do |client_ptr, err_code, reason, _opaque|
+      error_callback = Rdkafka::Config.error_callback
+
+      if error_callback
+        instance_name = client_ptr.null? ? nil : Rdkafka::Bindings.rd_kafka_name(client_ptr)
+
+        # Handle fatal errors according to librdkafka documentation:
+        # When ERR__FATAL is received, we must call rd_kafka_fatal_error()
+        # to get the actual underlying fatal error code and description.
+        error = if err_code == RD_KAFKA_RESP_ERR__FATAL
+          Rdkafka::RdkafkaError.build_fatal(
+            client_ptr,
+            fallback_error_code: err_code,
+            fallback_message: reason,
+            instance_name: instance_name
+          )
+        else
+          Rdkafka::RdkafkaError.build(err_code, broker_message: reason, instance_name: instance_name)
+        end
+
+        error.set_backtrace(caller)
+        error_callback.call(error)
+      end
+    end
+
+    # The OAuth callback is currently global and contextless. This means that the callback will be
+    # called for all instances, and the callback must be able to determine to which instance it is
+    # associated. The instance name will be provided in the callback, allowing the callback to
+    # reference the correct instance.
+    #
+    # An example of how to use the instance name in the callback is given below.
+    # The `refresh_token` is configured as the `oauthbearer_token_refresh_callback`.
+    # `instances` is a map of client names to client instances, maintained by the user.
+    #
+    # ```
+    #   def refresh_token(config, client_name)
+    #     client = instances[client_name]
+    #     client.oauthbearer_set_token(
+    #       token: 'new-token-value',
+    #       lifetime_ms: token-lifetime-ms,
+    #       principal_name: 'principal-name'
+    #     )
+    #   end
+    # ```
+    OAuthbearerTokenRefreshCallback = FFI::Function.new(
+      :void, [:pointer, :string, :pointer]
+    ) do |client_ptr, config, opaque_ptr|
+      refresh_callback = Rdkafka::Config.oauthbearer_token_refresh_callback
+
+      if refresh_callback && !client_ptr.null?
+        client_name = Rdkafka::Bindings.rd_kafka_name(client_ptr)
+
+        # Share consumers have no way to query their librdkafka client name (the share handle
+        # exposes no name accessor), so hand it over here - where librdkafka provides it -
+        # letting applications correlate this callback's client name with a ShareConsumer via
+        # ShareConsumer#name.
+        Rdkafka::Config.opaques[opaque_ptr.to_i]&.capture_client_name(client_name)
+
+        refresh_callback.call(config, client_name)
+      end
+    end
+  end
+end

data/lib/rdkafka/bindings/configuration.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  module Bindings
+    # Configuration
+
+    enum :kafka_config_response, [
+      :config_unknown, -2,
+      :config_invalid, -1,
+      :config_ok, 0
+    ]
+
+    attach_function :rd_kafka_conf_new, [], :pointer
+    attach_function :rd_kafka_conf_set, [:pointer, :string, :string, :pointer, :size_t], :kafka_config_response
+    attach_function :rd_kafka_conf_get, [:pointer, :string, :pointer, :pointer], :kafka_config_response
+    attach_function :rd_kafka_conf, [:pointer], :pointer
+    attach_function :rd_kafka_conf_dump, [:pointer, :pointer], :pointer
+    attach_function :rd_kafka_conf_dump_free, [:pointer, :size_t], :void
+    attach_function :rd_kafka_conf_destroy, [:pointer], :void
+    callback :log_cb, [:pointer, :int, :string, :string], :void
+    attach_function :rd_kafka_conf_set_log_cb, [:pointer, :log_cb], :void
+    attach_function :rd_kafka_conf_set_opaque, [:pointer, :pointer], :void
+    callback :stats_cb, [:pointer, :string, :int, :pointer], :int
+    attach_function :rd_kafka_conf_set_stats_cb, [:pointer, :stats_cb], :void
+    callback :error_cb, [:pointer, :int, :string, :pointer], :void
+    attach_function :rd_kafka_conf_set_error_cb, [:pointer, :error_cb], :void
+    attach_function :rd_kafka_rebalance_protocol, [:pointer], :string
+    callback :oauthbearer_token_refresh_cb, [:pointer, :string, :pointer], :void
+    attach_function :rd_kafka_conf_set_oauthbearer_token_refresh_cb, [:pointer, :oauthbearer_token_refresh_cb], :void
+    attach_function :rd_kafka_oauthbearer_set_token, [:pointer, :string, :int64, :pointer, :pointer, :size_t, :pointer, :size_t], :int
+    attach_function :rd_kafka_oauthbearer_set_token_failure, [:pointer, :string], :int
+    # Log queue
+    attach_function :rd_kafka_set_log_queue, [:pointer, :pointer], :void
+    attach_function :rd_kafka_queue_get_main, [:pointer], :pointer
+    attach_function :rd_kafka_queue_get_background, [:pointer], :pointer
+
+    # Queue IO Event Support - for fiber scheduler integration
+    # Enables notifications to a custom FD when queue transitions from empty to non-empty
+    # Arguments:
+    # - queue (rd_kafka_queue_t*) - the queue to monitor
+    # - fd (int) - file descriptor to write to (provide your own pipe/eventfd)
+    # - payload (const void*) - data to write to fd
+    # - size (size_t) - size of payload
+    attach_function :rd_kafka_queue_io_event_enable, [:pointer, :int, :pointer, :size_t], :void
+    # Per topic configs
+    attach_function :rd_kafka_topic_conf_new, [], :pointer
+    attach_function :rd_kafka_topic_conf_destroy, [:pointer], :void
+    attach_function :rd_kafka_topic_conf_set, [:pointer, :string, :string, :pointer, :size_t], :kafka_config_response
+  end
+end

data/lib/rdkafka/bindings/consumer.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  module Bindings
+    # Consumer
+
+    attach_function :rd_kafka_subscribe, [:pointer, :pointer], :int, blocking: true
+    attach_function :rd_kafka_unsubscribe, [:pointer], :int, blocking: true
+    attach_function :rd_kafka_subscription, [:pointer, :pointer], :int, blocking: true
+    attach_function :rd_kafka_assign, [:pointer, :pointer], :int, blocking: true
+    attach_function :rd_kafka_incremental_assign, [:pointer, :pointer], :int, blocking: true
+    attach_function :rd_kafka_incremental_unassign, [:pointer, :pointer], :int, blocking: true
+    attach_function :rd_kafka_assignment, [:pointer, :pointer], :int, blocking: true
+    attach_function :rd_kafka_assignment_lost, [:pointer], :int, blocking: true
+    attach_function :rd_kafka_committed, [:pointer, :pointer, :int], :int, blocking: true
+    attach_function :rd_kafka_commit, [:pointer, :pointer, :bool], :int, blocking: true
+    attach_function :rd_kafka_poll_set_consumer, [:pointer], :void, blocking: true
+    attach_function :rd_kafka_consumer_poll, [:pointer, :int], :pointer, blocking: true
+    # Non-blocking consumer poll variant (does not release GVL)
+    # More efficient for poll(0) calls in fiber schedulers.
+    attach_function :rd_kafka_consumer_poll_nb, :rd_kafka_consumer_poll, [:pointer, :int], :pointer, blocking: false
+    attach_function :rd_kafka_consumer_close, [:pointer], :int, blocking: true
+    attach_function :rd_kafka_queue_get_consumer, [:pointer], :pointer
+    attach_function :rd_kafka_consume_batch_queue, [:pointer, :int, :pointer, :size_t], :ssize_t, blocking: true
+    attach_function :rd_kafka_consume_batch_queue_nb, :rd_kafka_consume_batch_queue, [:pointer, :int, :pointer, :size_t], :ssize_t, blocking: false
+    attach_function :rd_kafka_offsets_store, [:pointer, :pointer], :int, blocking: true
+    attach_function :rd_kafka_pause_partitions, [:pointer, :pointer], :int, blocking: true
+    attach_function :rd_kafka_resume_partitions, [:pointer, :pointer], :int, blocking: true
+    attach_function :rd_kafka_seek, [:pointer, :int32, :int64, :int], :int, blocking: true
+    attach_function :rd_kafka_offsets_for_times, [:pointer, :pointer, :int], :int, blocking: true
+    attach_function :rd_kafka_position, [:pointer, :pointer], :int, blocking: true
+    # those two are used for eos support
+    attach_function :rd_kafka_consumer_group_metadata, [:pointer], :pointer, blocking: true
+    attach_function :rd_kafka_consumer_group_metadata_destroy, [:pointer], :void, blocking: true
+
+    # Headers
+    attach_function :rd_kafka_header_get_all, [:pointer, :size_t, :pointer, :pointer, SizePtr], :int
+    attach_function :rd_kafka_message_headers, [:pointer, :pointer], :int
+
+    # Rebalance
+
+    callback :rebalance_cb_function, [:pointer, :int, :pointer, :pointer], :void
+    attach_function :rd_kafka_conf_set_rebalance_cb, [:pointer, :rebalance_cb_function], :void, blocking: true
+
+    RebalanceCallback = FFI::Function.new(
+      :void, [:pointer, :int, :pointer, :pointer]
+    ) do |client_ptr, code, partitions_ptr, opaque_ptr|
+      case code
+      when RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS
+        if Rdkafka::Bindings.rd_kafka_rebalance_protocol(client_ptr) == "COOPERATIVE"
+          Rdkafka::Bindings.rd_kafka_incremental_assign(client_ptr, partitions_ptr)
+        else
+          Rdkafka::Bindings.rd_kafka_assign(client_ptr, partitions_ptr)
+        end
+      else # RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS or errors
+        if Rdkafka::Bindings.rd_kafka_rebalance_protocol(client_ptr) == "COOPERATIVE"
+          Rdkafka::Bindings.rd_kafka_incremental_unassign(client_ptr, partitions_ptr)
+        else
+          Rdkafka::Bindings.rd_kafka_assign(client_ptr, FFI::Pointer::NULL)
+        end
+      end
+
+      opaque = Rdkafka::Config.opaques[opaque_ptr.to_i]
+      next unless opaque
+
+      tpl = Rdkafka::Consumer::TopicPartitionList.from_native_tpl(partitions_ptr).freeze
+      begin
+        case code
+        when RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS
+          opaque.call_on_partitions_assigned(tpl)
+        when RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS
+          opaque.call_on_partitions_revoked(tpl)
+        end
+      rescue Exception => err
+        Rdkafka::Config.logger.error("Unhandled exception: #{err.class} - #{err.message}")
+      end
+    end
+
+    # Offset commit
+
+    callback :offset_commit_cb_function, [:pointer, :int, :pointer, :pointer], :void
+    attach_function :rd_kafka_conf_set_offset_commit_cb, [:pointer, :offset_commit_cb_function], :void, blocking: true
+
+    # Served from the consumer poll (or the events poll when `consumer_poll_set` is false) and
+    # from the consumer close, on the thread that calls them
+    OffsetCommitCallback = FFI::Function.new(
+      :void, [:pointer, :int, :pointer, :pointer]
+    ) do |_client_ptr, code, offsets_ptr, opaque_ptr|
+      opaque = Rdkafka::Config.opaques[opaque_ptr.to_i]
+      next unless opaque
+
+      # Success and `no_offset` (a commit with nothing new to commit) are not failures
+      failed = code != RD_KAFKA_RESP_ERR_NO_ERROR && code != RD_KAFKA_RESP_ERR__NO_OFFSET
+      every = opaque.offsets_committed_listened?
+      on_failure = failed && opaque.offsets_commit_failed_listened?
+
+      # The list has an entry for every assigned partition, so build it only when it is used
+      if every || on_failure
+        error = code.zero? ? nil : Rdkafka::RdkafkaError.new(code)
+        tpl = if offsets_ptr.null?
+          Rdkafka::Consumer::TopicPartitionList.new
+        else
+          Rdkafka::Consumer::TopicPartitionList.from_native_tpl(offsets_ptr)
+        end
+        tpl.freeze
+
+        begin
+          opaque.call_on_offsets_committed(error, tpl) if every
+        rescue Exception => err
+          Rdkafka::Config.logger.error("Unhandled exception: #{err.class} - #{err.message}")
+        end
+
+        begin
+          opaque.call_on_offsets_commit_failed(error, tpl) if on_failure
+        rescue Exception => err
+          Rdkafka::Config.logger.error("Unhandled exception: #{err.class} - #{err.message}")
+        end
+      end
+    end
+
+    # Watermark offsets
+
+    attach_function :rd_kafka_query_watermark_offsets, [:pointer, :string, :int, :pointer, :pointer, :int], :int, blocking: true
+  end
+end

data/lib/rdkafka/bindings/errors.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  module Bindings
+    attach_function :rd_kafka_err2name, [:int], :string
+    attach_function :rd_kafka_err2str, [:int], :string
+    attach_function :rd_kafka_error_is_fatal, [:pointer], :int
+    attach_function :rd_kafka_error_is_retriable, [:pointer], :int
+    attach_function :rd_kafka_error_txn_requires_abort, [:pointer], :int
+    attach_function :rd_kafka_error_destroy, [:pointer], :void
+    attach_function :rd_kafka_error_code, [:pointer], :int32
+    attach_function :rd_kafka_error_string, [:pointer], :pointer
+    attach_function :rd_kafka_get_err_descs, [:pointer, :pointer], :void
+    attach_function :rd_kafka_fatal_error, [:pointer, :pointer, :size_t], :int
+    attach_function :rd_kafka_test_fatal_error, [:pointer, :int, :string], :int
+  end
+end

data/lib/rdkafka/bindings/handle.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  module Bindings
+    # Handle
+
+    enum :kafka_type, [
+      :rd_kafka_producer,
+      :rd_kafka_consumer
+    ]
+
+    attach_function :rd_kafka_new, [:kafka_type, :pointer, :pointer, :size_t], :pointer
+
+    attach_function :rd_kafka_destroy, [:pointer], :void
+  end
+end

data/lib/rdkafka/bindings/messages.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  module Bindings
+    # Messages, topics and topic partition lists
+
+    attach_function :rd_kafka_message_destroy, [:pointer], :void
+    attach_function :rd_kafka_message_timestamp, [:pointer, :pointer], :int64
+    attach_function :rd_kafka_message_latency, [:pointer], :int64
+    attach_function :rd_kafka_message_broker_id, [:pointer], :int32
+    attach_function :rd_kafka_message_status, [:pointer], :int
+
+    # Message persistence status (rd_kafka_msg_status_t)
+    RD_KAFKA_MSG_STATUS_NOT_PERSISTED = 0
+    RD_KAFKA_MSG_STATUS_POSSIBLY_PERSISTED = 1
+    RD_KAFKA_MSG_STATUS_PERSISTED = 2
+
+    attach_function :rd_kafka_topic_new, [:pointer, :string, :pointer], :pointer
+    attach_function :rd_kafka_topic_destroy, [:pointer], :pointer
+    attach_function :rd_kafka_topic_name, [:pointer], :string
+
+    attach_function :rd_kafka_topic_partition_list_new, [:int32], :pointer
+    attach_function :rd_kafka_topic_partition_list_add, [:pointer, :string, :int32], :pointer
+    attach_function :rd_kafka_topic_partition_list_set_offset, [:pointer, :string, :int32, :int64], :void
+    attach_function :rd_kafka_topic_partition_list_destroy, [:pointer], :void
+    attach_function :rd_kafka_topic_partition_list_copy, [:pointer], :pointer
+  end
+end

data/lib/rdkafka/bindings/metadata.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  module Bindings
+    # Metadata
+
+    attach_function :rd_kafka_name, [:pointer], :string
+    # Both return a newly allocated string the caller must release with rd_kafka_mem_free; they
+    # are attached as :pointer (not :string) so we can free them instead of leaking. clusterid
+    # also takes a timeout_ms (the previous arity was wrong, leaving that argument as garbage).
+    attach_function :rd_kafka_memberid, [:pointer], :pointer, blocking: true
+    attach_function :rd_kafka_clusterid, [:pointer, :int], :pointer, blocking: true
+    attach_function :rd_kafka_mem_free, [:pointer, :pointer], :void
+    attach_function :rd_kafka_metadata, [:pointer, :int, :pointer, :pointer, :int], :int, blocking: true
+    attach_function :rd_kafka_metadata_destroy, [:pointer], :void, blocking: true
+  end
+end