rdkafka 0.13.0 → 0.15.1

data/lib/rdkafka/admin.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "objspace"
-
 module Rdkafka
   class Admin
     # @private
@@ -16,6 +14,19 @@ module Rdkafka
       ->(_) { close }
     end
 
+    # Performs the metadata request using admin
+    #
+    # @param topic_name [String, nil] metadat about particular topic or all if nil
+    # @param timeout_ms [Integer] metadata request timeout
+    # @return [Metadata] requested metadata
+    def metadata(topic_name = nil, timeout_ms = 2_000)
+      closed_admin_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        Metadata.new(inner, topic_name, timeout_ms)
+      end
+    end
+
     # Close this admin instance
     def close
       return if closed?
@@ -30,11 +41,12 @@ module Rdkafka
 
     # Create a topic with the given partition count and replication factor
     #
+    # @return [CreateTopicHandle] Create topic handle that can be used to wait for the result of
+    #   creating the topic
+    #
     # @raise [ConfigError] When the partition count or replication factor are out of valid range
     # @raise [RdkafkaError] When the topic name is invalid or the topic already exists
     # @raise [RdkafkaError] When the topic configuration is invalid
-    #
-    # @return [CreateTopicHandle] Create topic handle that can be used to wait for the result of creating the topic
     def create_topic(topic_name, partition_count, replication_factor, topic_config={})
       closed_admin_check(__method__)
 
@@ -107,11 +119,64 @@ module Rdkafka
       create_topic_handle
     end
 
-    # Delete the named topic
+    def delete_group(group_id)
+      closed_admin_check(__method__)
+
+      # Create a rd_kafka_DeleteGroup_t representing the new topic
+      delete_groups_ptr = Rdkafka::Bindings.rd_kafka_DeleteGroup_new(
+        FFI::MemoryPointer.from_string(group_id)
+      )
+
+      pointer_array = [delete_groups_ptr]
+      groups_array_ptr = FFI::MemoryPointer.new(:pointer)
+      groups_array_ptr.write_array_of_pointer(pointer_array)
+
+      # Get a pointer to the queue that our request will be enqueued on
+      queue_ptr = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
+      end
+      if queue_ptr.null?
+        Rdkafka::Bindings.rd_kafka_DeleteTopic_destroy(delete_topic_ptr)
+        raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
+      end
+
+      # Create and register the handle we will return to the caller
+      delete_groups_handle = DeleteGroupsHandle.new
+      delete_groups_handle[:pending] = true
+      delete_groups_handle[:response] = -1
+      DeleteGroupsHandle.register(delete_groups_handle)
+      admin_options_ptr = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_AdminOptions_new(inner, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_DELETETOPICS)
+      end
+      Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, delete_groups_handle.to_ptr)
+
+      begin
+        @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_DeleteGroups(
+            inner,
+            groups_array_ptr,
+            1,
+            admin_options_ptr,
+            queue_ptr
+          )
+        end
+      rescue Exception
+        DeleteGroupsHandle.remove(delete_groups_handle.to_ptr.address)
+        raise
+      ensure
+        Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
+        Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
+        Rdkafka::Bindings.rd_kafka_DeleteGroup_destroy(delete_groups_ptr)
+      end
+
+      delete_groups_handle
+    end
+
+    # Deletes the named topic
     #
+    # @return [DeleteTopicHandle] Delete topic handle that can be used to wait for the result of
+    #   deleting the topic
     # @raise [RdkafkaError] When the topic name is invalid or the topic does not exist
-    #
-    # @return [DeleteTopicHandle] Delete topic handle that can be used to wait for the result of deleting the topic
     def delete_topic(topic_name)
       closed_admin_check(__method__)
 
@@ -164,7 +229,384 @@ module Rdkafka
       delete_topic_handle
     end
 
+    # Creates extra partitions for a given topic
+    #
+    # @param topic_name [String]
+    # @param partition_count [Integer] how many partitions we want to end up with for given topic
+    #
+    # @raise [ConfigError] When the partition count or replication factor are out of valid range
+    # @raise [RdkafkaError] When the topic name is invalid or the topic already exists
+    # @raise [RdkafkaError] When the topic configuration is invalid
+    #
+    # @return [CreateTopicHandle] Create topic handle that can be used to wait for the result of creating the topic
+    def create_partitions(topic_name, partition_count)
+      closed_admin_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        error_buffer = FFI::MemoryPointer.from_string(" " * 256)
+        new_partitions_ptr = Rdkafka::Bindings.rd_kafka_NewPartitions_new(
+          FFI::MemoryPointer.from_string(topic_name),
+          partition_count,
+          error_buffer,
+          256
+        )
+        if new_partitions_ptr.null?
+          raise Rdkafka::Config::ConfigError.new(error_buffer.read_string)
+        end
+
+        pointer_array = [new_partitions_ptr]
+        topics_array_ptr = FFI::MemoryPointer.new(:pointer)
+        topics_array_ptr.write_array_of_pointer(pointer_array)
+
+        # Get a pointer to the queue that our request will be enqueued on
+        queue_ptr = Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
+        if queue_ptr.null?
+          Rdkafka::Bindings.rd_kafka_NewPartitions_destroy(new_partitions_ptr)
+          raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
+        end
+
+        # Create and register the handle we will return to the caller
+        create_partitions_handle = CreatePartitionsHandle.new
+        create_partitions_handle[:pending] = true
+        create_partitions_handle[:response] = -1
+        CreatePartitionsHandle.register(create_partitions_handle)
+        admin_options_ptr = Rdkafka::Bindings.rd_kafka_AdminOptions_new(inner, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATEPARTITIONS)
+        Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, create_partitions_handle.to_ptr)
+
+        begin
+          Rdkafka::Bindings.rd_kafka_CreatePartitions(
+            inner,
+            topics_array_ptr,
+            1,
+            admin_options_ptr,
+            queue_ptr
+          )
+        rescue Exception
+          CreatePartitionsHandle.remove(create_partitions_handle.to_ptr.address)
+          raise
+        ensure
+          Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
+          Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
+          Rdkafka::Bindings.rd_kafka_NewPartitions_destroy(new_partitions_ptr)
+        end
+
+        create_partitions_handle
+      end
+    end
+
+    # Create acl
+    # @param resource_type - values of type rd_kafka_ResourceType_t
+    #        https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7307
+    #        valid values are:
+    #           RD_KAFKA_RESOURCE_TOPIC   = 2
+    #           RD_KAFKA_RESOURCE_GROUP   = 3
+    #           RD_KAFKA_RESOURCE_BROKER  = 4
+    # @param resource_pattern_type - values of type rd_kafka_ResourcePatternType_t
+    #        https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7320
+    #        valid values are:
+    #           RD_KAFKA_RESOURCE_PATTERN_MATCH    = 2
+    #           RD_KAFKA_RESOURCE_PATTERN_LITERAL  = 3
+    #           RD_KAFKA_RESOURCE_PATTERN_PREFIXED = 4
+    # @param operation - values of type rd_kafka_AclOperation_t
+    #        https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8403
+    #        valid values are:
+    #           RD_KAFKA_ACL_OPERATION_ALL              = 2
+    #           RD_KAFKA_ACL_OPERATION_READ             = 3
+    #           RD_KAFKA_ACL_OPERATION_WRITE            = 4
+    #           RD_KAFKA_ACL_OPERATION_CREATE           = 5
+    #           RD_KAFKA_ACL_OPERATION_DELETE           = 6
+    #           RD_KAFKA_ACL_OPERATION_ALTER            = 7
+    #           RD_KAFKA_ACL_OPERATION_DESCRIBE         = 8
+    #           RD_KAFKA_ACL_OPERATION_CLUSTER_ACTION   = 9
+    #           RD_KAFKA_ACL_OPERATION_DESCRIBE_CONFIGS = 10
+    #           RD_KAFKA_ACL_OPERATION_ALTER_CONFIGS    = 11
+    #           RD_KAFKA_ACL_OPERATION_IDEMPOTENT_WRITE = 12
+    # @param permission_type - values of type rd_kafka_AclPermissionType_t
+    #        https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8435
+    #        valid values are:
+    #           RD_KAFKA_ACL_PERMISSION_TYPE_DENY  = 2
+    #           RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
+    #
+    # @return [CreateAclHandle] Create acl handle that can be used to wait for the result of creating the acl
+    #
+    # @raise [RdkafkaError]
+    def create_acl(resource_type:, resource_name:, resource_pattern_type:, principal:, host:, operation:, permission_type:)
+      closed_admin_check(__method__)
+
+      # Create a rd_kafka_AclBinding_t representing the new acl
+      error_buffer = FFI::MemoryPointer.from_string(" " * 256)
+      new_acl_ptr = Rdkafka::Bindings.rd_kafka_AclBinding_new(
+        resource_type,
+        FFI::MemoryPointer.from_string(resource_name),
+        resource_pattern_type,
+        FFI::MemoryPointer.from_string(principal),
+        FFI::MemoryPointer.from_string(host),
+        operation,
+        permission_type,
+        error_buffer,
+        256
+      )
+      if new_acl_ptr.null?
+        raise Rdkafka::Config::ConfigError.new(error_buffer.read_string)
+      end
+
+      # Note that rd_kafka_CreateAcls can create more than one acl at a time
+      pointer_array = [new_acl_ptr]
+      acls_array_ptr = FFI::MemoryPointer.new(:pointer)
+      acls_array_ptr.write_array_of_pointer(pointer_array)
+
+      # Get a pointer to the queue that our request will be enqueued on
+      queue_ptr = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
+      end
+
+      if queue_ptr.null?
+        Rdkafka::Bindings.rd_kafka_AclBinding_destroy(new_acl_ptr)
+        raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
+      end
+
+      # Create and register the handle that we will return to the caller
+      create_acl_handle = CreateAclHandle.new
+      create_acl_handle[:pending] = true
+      create_acl_handle[:response] = -1
+      CreateAclHandle.register(create_acl_handle)
+
+      admin_options_ptr = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_AdminOptions_new(inner, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATEACLS)
+      end
+
+      Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, create_acl_handle.to_ptr)
+
+      begin
+        @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_CreateAcls(
+            inner,
+            acls_array_ptr,
+            1,
+            admin_options_ptr,
+            queue_ptr
+          )
+        end
+      rescue Exception
+        CreateAclHandle.remove(create_acl_handle.to_ptr.address)
+        raise
+      ensure
+        Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
+        Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
+        Rdkafka::Bindings.rd_kafka_AclBinding_destroy(new_acl_ptr)
+      end
+
+      create_acl_handle
+    end
+
+    # Delete acl
+    #
+    # @param resource_type - values of type rd_kafka_ResourceType_t
+    #        https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7307
+    #        valid values are:
+    #           RD_KAFKA_RESOURCE_TOPIC   = 2
+    #           RD_KAFKA_RESOURCE_GROUP   = 3
+    #           RD_KAFKA_RESOURCE_BROKER  = 4
+    # @param resource_pattern_type - values of type rd_kafka_ResourcePatternType_t
+    #        https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7320
+    #        valid values are:
+    #           RD_KAFKA_RESOURCE_PATTERN_MATCH    = 2
+    #           RD_KAFKA_RESOURCE_PATTERN_LITERAL  = 3
+    #           RD_KAFKA_RESOURCE_PATTERN_PREFIXED = 4
+    # @param operation - values of type rd_kafka_AclOperation_t
+    #        https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8403
+    #        valid values are:
+    #           RD_KAFKA_ACL_OPERATION_ALL              = 2
+    #           RD_KAFKA_ACL_OPERATION_READ             = 3
+    #           RD_KAFKA_ACL_OPERATION_WRITE            = 4
+    #           RD_KAFKA_ACL_OPERATION_CREATE           = 5
+    #           RD_KAFKA_ACL_OPERATION_DELETE           = 6
+    #           RD_KAFKA_ACL_OPERATION_ALTER            = 7
+    #           RD_KAFKA_ACL_OPERATION_DESCRIBE         = 8
+    #           RD_KAFKA_ACL_OPERATION_CLUSTER_ACTION   = 9
+    #           RD_KAFKA_ACL_OPERATION_DESCRIBE_CONFIGS = 10
+    #           RD_KAFKA_ACL_OPERATION_ALTER_CONFIGS    = 11
+    #           RD_KAFKA_ACL_OPERATION_IDEMPOTENT_WRITE = 12
+    # @param permission_type - values of type rd_kafka_AclPermissionType_t
+    #        https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8435
+    #        valid values are:
+    #           RD_KAFKA_ACL_PERMISSION_TYPE_DENY  = 2
+    #           RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
+    # @return [DeleteAclHandle] Delete acl handle that can be used to wait for the result of deleting the acl
+    #
+    # @raise [RdkafkaError]
+    def delete_acl(resource_type:, resource_name:, resource_pattern_type:, principal:, host:, operation:, permission_type:)
+      closed_admin_check(__method__)
+
+      # Create a rd_kafka_AclBinding_t representing the acl to be deleted
+      error_buffer = FFI::MemoryPointer.from_string(" " * 256)
+
+      delete_acl_ptr = Rdkafka::Bindings.rd_kafka_AclBindingFilter_new(
+        resource_type,
+        resource_name ? FFI::MemoryPointer.from_string(resource_name) : nil,
+        resource_pattern_type,
+        principal ? FFI::MemoryPointer.from_string(principal) : nil,
+        host ? FFI::MemoryPointer.from_string(host) : nil,
+        operation,
+        permission_type,
+        error_buffer,
+        256
+      )
+
+      if delete_acl_ptr.null?
+        raise Rdkafka::Config::ConfigError.new(error_buffer.read_string)
+      end
+
+      # Note that rd_kafka_DeleteAcls can delete more than one acl at a time
+      pointer_array = [delete_acl_ptr]
+      acls_array_ptr = FFI::MemoryPointer.new(:pointer)
+      acls_array_ptr.write_array_of_pointer(pointer_array)
+
+      # Get a pointer to the queue that our request will be enqueued on
+      queue_ptr = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
+      end
+
+      if queue_ptr.null?
+        Rdkafka::Bindings.rd_kafka_AclBinding_destroy(new_acl_ptr)
+        raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
+      end
+
+      # Create and register the handle that we will return to the caller
+      delete_acl_handle = DeleteAclHandle.new
+      delete_acl_handle[:pending] = true
+      delete_acl_handle[:response] = -1
+      DeleteAclHandle.register(delete_acl_handle)
+
+      admin_options_ptr = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_AdminOptions_new(inner, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_DELETEACLS)
+      end
+
+      Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, delete_acl_handle.to_ptr)
+
+      begin
+        @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_DeleteAcls(
+            inner,
+            acls_array_ptr,
+            1,
+            admin_options_ptr,
+            queue_ptr
+          )
+        end
+      rescue Exception
+        DeleteAclHandle.remove(delete_acl_handle.to_ptr.address)
+        raise
+      ensure
+        Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
+        Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
+        Rdkafka::Bindings.rd_kafka_AclBinding_destroy(delete_acl_ptr)
+      end
+
+      delete_acl_handle
+    end
+
+    # Describe acls
+    #
+    # @param resource_type - values of type rd_kafka_ResourceType_t
+    #        https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7307
+    #        valid values are:
+    #           RD_KAFKA_RESOURCE_TOPIC   = 2
+    #           RD_KAFKA_RESOURCE_GROUP   = 3
+    #           RD_KAFKA_RESOURCE_BROKER  = 4
+    # @param resource_pattern_type - values of type rd_kafka_ResourcePatternType_t
+    #        https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7320
+    #        valid values are:
+    #           RD_KAFKA_RESOURCE_PATTERN_MATCH    = 2
+    #           RD_KAFKA_RESOURCE_PATTERN_LITERAL  = 3
+    #           RD_KAFKA_RESOURCE_PATTERN_PREFIXED = 4
+    # @param operation - values of type rd_kafka_AclOperation_t
+    #        https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8403
+    #        valid values are:
+    #           RD_KAFKA_ACL_OPERATION_ALL              = 2
+    #           RD_KAFKA_ACL_OPERATION_READ             = 3
+    #           RD_KAFKA_ACL_OPERATION_WRITE            = 4
+    #           RD_KAFKA_ACL_OPERATION_CREATE           = 5
+    #           RD_KAFKA_ACL_OPERATION_DELETE           = 6
+    #           RD_KAFKA_ACL_OPERATION_ALTER            = 7
+    #           RD_KAFKA_ACL_OPERATION_DESCRIBE         = 8
+    #           RD_KAFKA_ACL_OPERATION_CLUSTER_ACTION   = 9
+    #           RD_KAFKA_ACL_OPERATION_DESCRIBE_CONFIGS = 10
+    #           RD_KAFKA_ACL_OPERATION_ALTER_CONFIGS    = 11
+    #           RD_KAFKA_ACL_OPERATION_IDEMPOTENT_WRITE = 12
+    # @param permission_type - values of type rd_kafka_AclPermissionType_t
+    #        https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8435
+    #        valid values are:
+    #           RD_KAFKA_ACL_PERMISSION_TYPE_DENY  = 2
+    #           RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
+    # @return [DescribeAclHandle] Describe acl handle that can be used to wait for the result of fetching acls
+    #
+    # @raise [RdkafkaError]
+    def describe_acl(resource_type:, resource_name:, resource_pattern_type:, principal:, host:, operation:, permission_type:)
+      closed_admin_check(__method__)
+
+      # Create a rd_kafka_AclBinding_t with the filters to fetch existing acls
+      error_buffer = FFI::MemoryPointer.from_string(" " * 256)
+      describe_acl_ptr = Rdkafka::Bindings.rd_kafka_AclBindingFilter_new(
+        resource_type,
+        resource_name ? FFI::MemoryPointer.from_string(resource_name) : nil,
+        resource_pattern_type,
+        principal ? FFI::MemoryPointer.from_string(principal) : nil,
+        host ? FFI::MemoryPointer.from_string(host) : nil,
+        operation,
+        permission_type,
+        error_buffer,
+        256
+      )
+      if describe_acl_ptr.null?
+        raise Rdkafka::Config::ConfigError.new(error_buffer.read_string)
+      end
+
+      # Get a pointer to the queue that our request will be enqueued on
+      queue_ptr = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
+      end
+
+      if queue_ptr.null?
+        Rdkafka::Bindings.rd_kafka_AclBinding_destroy(new_acl_ptr)
+        raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
+      end
+
+      # Create and register the handle that we will return to the caller
+      describe_acl_handle = DescribeAclHandle.new
+      describe_acl_handle[:pending] = true
+      describe_acl_handle[:response] = -1
+      DescribeAclHandle.register(describe_acl_handle)
+
+      admin_options_ptr = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_AdminOptions_new(inner, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_DESCRIBEACLS)
+      end
+
+      Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, describe_acl_handle.to_ptr)
+
+      begin
+        @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_DescribeAcls(
+            inner,
+            describe_acl_ptr,
+            admin_options_ptr,
+            queue_ptr
+          )
+        end
+      rescue Exception
+        DescribeAclHandle.remove(describe_acl_handle.to_ptr.address)
+        raise
+      ensure
+        Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
+        Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
+        Rdkafka::Bindings.rd_kafka_AclBinding_destroy(describe_acl_ptr)
+      end
+
+      describe_acl_handle
+    end
+
     private
+
     def closed_admin_check(method)
       raise Rdkafka::ClosedAdminError.new(method) if closed?
     end

data/lib/rdkafka/bindings.rb

@@ -1,9 +1,5 @@
 # frozen_string_literal: true
 
-require "ffi"
-require "json"
-require "logger"
-
 module Rdkafka
   # @private
   module Bindings
@@ -35,12 +31,13 @@ module Rdkafka
 
     # Polling
 
-    attach_function :rd_kafka_flush, [:pointer, :int], :void, blocking: true
-    attach_function :rd_kafka_poll, [:pointer, :int], :void, blocking: true
+    attach_function :rd_kafka_flush, [:pointer, :int], :int, blocking: true
+    attach_function :rd_kafka_poll, [:pointer, :int], :int, blocking: true
     attach_function :rd_kafka_outq_len, [:pointer], :int, blocking: true
 
     # Metadata
 
+    attach_function :rd_kafka_name, [:pointer], :string, blocking: true
     attach_function :rd_kafka_memberid, [:pointer], :string, blocking: true
     attach_function :rd_kafka_clusterid, [:pointer], :string, blocking: true
     attach_function :rd_kafka_metadata, [:pointer, :int, :pointer, :pointer, :int], :int, blocking: true
@@ -157,6 +154,7 @@ module Rdkafka
     ) do |_client_prr, err_code, reason, _opaque|
       if Rdkafka::Config.error_callback
         error = Rdkafka::RdkafkaError.new(err_code, broker_message: reason)
+        error.set_backtrace(caller)
         Rdkafka::Config.error_callback.call(error)
       end
     end
@@ -181,15 +179,21 @@ module Rdkafka
     attach_function :rd_kafka_incremental_assign, [:pointer, :pointer], :int, blocking: true
     attach_function :rd_kafka_incremental_unassign, [:pointer, :pointer], :int, blocking: true
     attach_function :rd_kafka_assignment, [:pointer, :pointer], :int, blocking: true
+    attach_function :rd_kafka_assignment_lost, [:pointer], :int, blocking: true
     attach_function :rd_kafka_committed, [:pointer, :pointer, :int], :int, blocking: true
     attach_function :rd_kafka_commit, [:pointer, :pointer, :bool], :int, blocking: true
     attach_function :rd_kafka_poll_set_consumer, [:pointer], :void, blocking: true
     attach_function :rd_kafka_consumer_poll, [:pointer, :int], :pointer, blocking: true
     attach_function :rd_kafka_consumer_close, [:pointer], :void, blocking: true
-    attach_function :rd_kafka_offset_store, [:pointer, :int32, :int64], :int, blocking: true
+    attach_function :rd_kafka_offsets_store, [:pointer, :pointer], :int, blocking: true
     attach_function :rd_kafka_pause_partitions, [:pointer, :pointer], :int, blocking: true
     attach_function :rd_kafka_resume_partitions, [:pointer, :pointer], :int, blocking: true
     attach_function :rd_kafka_seek, [:pointer, :int32, :int64, :int], :int, blocking: true
+    attach_function :rd_kafka_offsets_for_times, [:pointer, :pointer, :int], :int, blocking: true
+    attach_function :rd_kafka_position, [:pointer, :pointer], :int, blocking: true
+    # those two are used for eos support
+    attach_function :rd_kafka_consumer_group_metadata, [:pointer], :pointer, blocking: true
+    attach_function :rd_kafka_consumer_group_metadata_destroy, [:pointer], :void, blocking: true
 
     # Headers
     attach_function :rd_kafka_header_get_all, [:pointer, :size_t, :pointer, :pointer, SizePtr], :int
@@ -251,10 +255,13 @@ module Rdkafka
     RD_KAFKA_VTYPE_TIMESTAMP = 8
     RD_KAFKA_VTYPE_HEADER = 9
     RD_KAFKA_VTYPE_HEADERS = 10
+    RD_KAFKA_PURGE_F_QUEUE = 1
+    RD_KAFKA_PURGE_F_INFLIGHT = 2
 
     RD_KAFKA_MSG_F_COPY = 0x2
 
     attach_function :rd_kafka_producev, [:pointer, :varargs], :int, blocking: true
+    attach_function :rd_kafka_purge, [:pointer, :int], :int, blocking: true
     callback :delivery_cb, [:pointer, :pointer, :pointer], :void
     attach_function :rd_kafka_conf_set_dr_msg_cb, [:pointer, :delivery_cb], :void
 
@@ -299,6 +306,27 @@ module Rdkafka
     attach_function :rd_kafka_event_DeleteTopics_result, [:pointer], :pointer, blocking: true
     attach_function :rd_kafka_DeleteTopics_result_topics, [:pointer, :pointer], :pointer, blocking: true
 
+    # Create partitions
+    RD_KAFKA_ADMIN_OP_CREATEPARTITIONS = 3
+    RD_KAFKA_ADMIN_OP_CREATEPARTITIONS_RESULT = 102
+
+    attach_function :rd_kafka_CreatePartitions, [:pointer, :pointer, :size_t, :pointer, :pointer], :void
+    attach_function :rd_kafka_NewPartitions_new, %i[pointer size_t pointer size_t], :pointer
+    attach_function :rd_kafka_NewPartitions_destroy, [:pointer], :void
+    attach_function :rd_kafka_event_CreatePartitions_result, [:pointer], :pointer
+    attach_function :rd_kafka_CreatePartitions_result_topics, [:pointer, :pointer], :pointer
+
+    # Delete Group
+
+    RD_KAFKA_ADMIN_OP_DELETEGROUPS = 7   # rd_kafka_admin_op_t
+    RD_KAFKA_EVENT_DELETEGROUPS_RESULT = 106 # rd_kafka_event_type_t
+
+    attach_function :rd_kafka_DeleteGroups, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
+    attach_function :rd_kafka_DeleteGroup_new, [:pointer], :pointer, blocking: true
+    attach_function :rd_kafka_DeleteGroup_destroy, [:pointer], :void, blocking: true
+    attach_function :rd_kafka_event_DeleteGroups_result, [:pointer], :pointer, blocking: true # rd_kafka_event_t* => rd_kafka_DeleteGroups_result_t*
+    attach_function :rd_kafka_DeleteGroups_result_groups, [:pointer, :pointer], :pointer, blocking: true # rd_kafka_DeleteGroups_result_t*, size_t* => rd_kafka_group_result_t**
+
     # Background Queue and Callback
 
     attach_function :rd_kafka_queue_get_background, [:pointer], :pointer
@@ -321,5 +349,103 @@ module Rdkafka
     attach_function :rd_kafka_topic_result_error, [:pointer], :int32
     attach_function :rd_kafka_topic_result_error_string, [:pointer], :pointer
     attach_function :rd_kafka_topic_result_name, [:pointer], :pointer
+
+    # Create Acls
+
+    RD_KAFKA_ADMIN_OP_CREATEACLS     = 9
+    RD_KAFKA_EVENT_CREATEACLS_RESULT = 1024
+
+    attach_function :rd_kafka_CreateAcls, [:pointer, :pointer, :size_t, :pointer, :pointer], :void
+    attach_function :rd_kafka_event_CreateAcls_result, [:pointer], :pointer
+    attach_function :rd_kafka_CreateAcls_result_acls, [:pointer, :pointer], :pointer
+
+    # Delete Acls
+
+    RD_KAFKA_ADMIN_OP_DELETEACLS     = 11
+    RD_KAFKA_EVENT_DELETEACLS_RESULT = 4096
+
+    attach_function :rd_kafka_DeleteAcls, [:pointer, :pointer, :size_t, :pointer, :pointer], :void
+    attach_function :rd_kafka_event_DeleteAcls_result, [:pointer], :pointer
+    attach_function :rd_kafka_DeleteAcls_result_responses, [:pointer, :pointer], :pointer
+    attach_function :rd_kafka_DeleteAcls_result_response_error, [:pointer], :pointer
+    attach_function :rd_kafka_DeleteAcls_result_response_matching_acls, [:pointer, :pointer], :pointer
+
+    # Describe Acls
+
+    RD_KAFKA_ADMIN_OP_DESCRIBEACLS     = 10
+    RD_KAFKA_EVENT_DESCRIBEACLS_RESULT = 2048
+
+    attach_function :rd_kafka_DescribeAcls, [:pointer, :pointer, :pointer, :pointer], :void
+    attach_function :rd_kafka_event_DescribeAcls_result, [:pointer], :pointer
+    attach_function :rd_kafka_DescribeAcls_result_acls, [:pointer, :pointer], :pointer
+
+    # Acl Bindings
+
+    attach_function :rd_kafka_AclBinding_restype, [:pointer], :int32
+    attach_function :rd_kafka_AclBinding_name, [:pointer], :pointer
+    attach_function :rd_kafka_AclBinding_resource_pattern_type, [:pointer], :int32
+    attach_function :rd_kafka_AclBinding_principal, [:pointer], :pointer
+    attach_function :rd_kafka_AclBinding_host, [:pointer], :pointer
+    attach_function :rd_kafka_AclBinding_operation, [:pointer], :int32
+    attach_function :rd_kafka_AclBinding_permission_type, [:pointer], :int32
+    attach_function :rd_kafka_AclBinding_new, [:int32, :pointer, :int32, :pointer, :pointer, :int32, :int32, :pointer, :size_t ], :pointer
+    attach_function :rd_kafka_AclBindingFilter_new, [:int32, :pointer, :int32, :pointer, :pointer, :int32, :int32, :pointer, :size_t ], :pointer
+    attach_function :rd_kafka_AclBinding_destroy, [:pointer], :void
+
+    # rd_kafka_ResourceType_t - https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7307
+
+    RD_KAFKA_RESOURCE_ANY   = 1
+    RD_KAFKA_RESOURCE_TOPIC = 2
+    RD_KAFKA_RESOURCE_GROUP = 3
+    RD_KAFKA_RESOURCE_BROKER = 4
+
+    # rd_kafka_ResourcePatternType_t - https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7320
+
+    RD_KAFKA_RESOURCE_PATTERN_ANY      = 1
+    RD_KAFKA_RESOURCE_PATTERN_MATCH    = 2
+    RD_KAFKA_RESOURCE_PATTERN_LITERAL  = 3
+    RD_KAFKA_RESOURCE_PATTERN_PREFIXED = 4
+
+    # rd_kafka_AclOperation_t - https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8403
+
+    RD_KAFKA_ACL_OPERATION_ANY              = 1
+    RD_KAFKA_ACL_OPERATION_ALL              = 2
+    RD_KAFKA_ACL_OPERATION_READ             = 3
+    RD_KAFKA_ACL_OPERATION_WRITE            = 4
+    RD_KAFKA_ACL_OPERATION_CREATE           = 5
+    RD_KAFKA_ACL_OPERATION_DELETE           = 6
+    RD_KAFKA_ACL_OPERATION_ALTER            = 7
+    RD_KAFKA_ACL_OPERATION_DESCRIBE         = 8
+    RD_KAFKA_ACL_OPERATION_CLUSTER_ACTION   = 9
+    RD_KAFKA_ACL_OPERATION_DESCRIBE_CONFIGS = 10
+    RD_KAFKA_ACL_OPERATION_ALTER_CONFIGS    = 11
+    RD_KAFKA_ACL_OPERATION_IDEMPOTENT_WRITE = 12
+
+    # rd_kafka_AclPermissionType_t - https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8435
+
+    RD_KAFKA_ACL_PERMISSION_TYPE_ANY     = 1
+    RD_KAFKA_ACL_PERMISSION_TYPE_DENY    = 2
+    RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW   = 3
+
+    # Extracting error details from Acl results
+    attach_function :rd_kafka_acl_result_error, [:pointer], :pointer
+    attach_function :rd_kafka_error_code, [:pointer], :int32
+    attach_function :rd_kafka_error_string, [:pointer], :pointer
+    attach_function :rd_kafka_event_error, [:pointer], :int32
+    attach_function :rd_kafka_event_error_string, [:pointer], :pointer
+    attach_function :rd_kafka_AclBinding_error, [:pointer], :pointer
+
+
+    # Extracting data from group results
+    class NativeError < FFI::Struct # rd_kafka_error_t
+      layout :code, :int32,
+             :errstr, :pointer,
+             :fatal, :u_int8_t,
+             :retriable, :u_int8_t,
+             :txn_requires_abort, :u_int8_t
+    end
+
+    attach_function :rd_kafka_group_result_error, [:pointer], NativeError.by_ref # rd_kafka_group_result_t* => rd_kafka_error_t*
+    attach_function :rd_kafka_group_result_name, [:pointer], :pointer
   end
 end