karafka-rdkafka 0.14.10 → 0.15.0

data/lib/rdkafka/admin.rb

@@ -2,6 +2,8 @@
 
 module Rdkafka
   class Admin
+    include Helpers::OAuth
+
     # @private
     def initialize(native_kafka)
       @native_kafka = native_kafka
@@ -10,6 +12,19 @@ module Rdkafka
       ObjectSpace.define_finalizer(self, native_kafka.finalizer)
     end
 
+    # Starts the native Kafka polling thread and kicks off the init polling
+    # @note Not needed to run unless explicit start was disabled
+    def start
+      @native_kafka.start
+    end
+
+    # @return [String] admin name
+    def name
+      @name ||= @native_kafka.with_inner do |inner|
+        ::Rdkafka::Bindings.rd_kafka_name(inner)
+      end
+    end
+
     def finalizer
       ->(_) { close }
     end
@@ -605,6 +620,165 @@ module Rdkafka
       describe_acl_handle
     end
 
+    # Describe configs
+    #
+    # @param resources [Array<Hash>]  Array where elements are hashes with two keys:
+    #   - `:resource_type` - numerical resource type based on Kafka API
+    #   - `:resource_name` - string with resource name
+    # @return [DescribeConfigsHandle] Describe config handle that can be used to wait for the
+    #   result of fetching resources with their appropriate configs
+    #
+    # @raise [RdkafkaError]
+    #
+    # @note Several resources can be requested at one go, but only one broker at a time
+    def describe_configs(resources)
+      closed_admin_check(__method__)
+
+      handle = DescribeConfigsHandle.new
+      handle[:pending] = true
+      handle[:response] = -1
+
+      queue_ptr = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
+      end
+
+      if queue_ptr.null?
+        raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
+      end
+
+      admin_options_ptr = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_AdminOptions_new(
+          inner,
+          Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_DESCRIBECONFIGS
+        )
+      end
+
+      DescribeConfigsHandle.register(handle)
+      Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
+
+      pointer_array = resources.map do |resource_details|
+        Rdkafka::Bindings.rd_kafka_ConfigResource_new(
+          resource_details.fetch(:resource_type),
+          FFI::MemoryPointer.from_string(
+            resource_details.fetch(:resource_name)
+          )
+        )
+      end
+
+      configs_array_ptr = FFI::MemoryPointer.new(:pointer, pointer_array.size)
+      configs_array_ptr.write_array_of_pointer(pointer_array)
+
+      begin
+        @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_DescribeConfigs(
+            inner,
+            configs_array_ptr,
+            pointer_array.size,
+            admin_options_ptr,
+            queue_ptr
+          )
+        end
+      rescue Exception
+        DescribeConfigsHandle.remove(handle.to_ptr.address)
+
+        raise
+      ensure
+        Rdkafka::Bindings.rd_kafka_ConfigResource_destroy_array(
+          configs_array_ptr,
+          pointer_array.size
+        ) if configs_array_ptr
+      end
+
+      handle
+    end
+
+    # Alters in an incremental way all the configs provided for given resources
+    #
+    # @param resources_with_configs [Array<Hash>] resources with the configs key that contains
+    # name, value and the proper op_type to perform on this value.
+    #
+    # @return [IncrementalAlterConfigsHandle] Incremental alter configs handle that can be used to
+    #   wait for the result of altering resources with their appropriate configs
+    #
+    # @raise [RdkafkaError]
+    #
+    # @note Several resources can be requested at one go, but only one broker at a time
+    # @note The results won't contain altered values but only the altered resources
+    def incremental_alter_configs(resources_with_configs)
+      closed_admin_check(__method__)
+
+      handle = IncrementalAlterConfigsHandle.new
+      handle[:pending] = true
+      handle[:response] = -1
+
+      queue_ptr = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
+      end
+
+      if queue_ptr.null?
+        raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
+      end
+
+      admin_options_ptr = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_AdminOptions_new(
+          inner,
+          Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_INCREMENTALALTERCONFIGS
+        )
+      end
+
+      IncrementalAlterConfigsHandle.register(handle)
+      Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
+
+      # Tu poprawnie tworzyc
+      pointer_array = resources_with_configs.map do |resource_details|
+        # First build the appropriate resource representation
+        resource_ptr = Rdkafka::Bindings.rd_kafka_ConfigResource_new(
+          resource_details.fetch(:resource_type),
+          FFI::MemoryPointer.from_string(
+            resource_details.fetch(:resource_name)
+          )
+        )
+
+        resource_details.fetch(:configs).each do |config|
+          Bindings.rd_kafka_ConfigResource_add_incremental_config(
+            resource_ptr,
+            config.fetch(:name),
+            config.fetch(:op_type),
+            config.fetch(:value)
+          )
+        end
+
+        resource_ptr
+      end
+
+      configs_array_ptr = FFI::MemoryPointer.new(:pointer, pointer_array.size)
+      configs_array_ptr.write_array_of_pointer(pointer_array)
+
+
+      begin
+        @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_IncrementalAlterConfigs(
+            inner,
+            configs_array_ptr,
+            pointer_array.size,
+            admin_options_ptr,
+            queue_ptr
+          )
+        end
+      rescue Exception
+        IncrementalAlterConfigsHandle.remove(handle.to_ptr.address)
+
+        raise
+      ensure
+        Rdkafka::Bindings.rd_kafka_ConfigResource_destroy_array(
+          configs_array_ptr,
+          pointer_array.size
+        ) if configs_array_ptr
+      end
+
+      handle
+    end
+
     private
 
     def closed_admin_check(method)

data/lib/rdkafka/bindings.rb

@@ -26,6 +26,7 @@ module Rdkafka
 
     RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS = -175
     RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS = -174
+    RD_KAFKA_RESP_ERR__STATE = -172
     RD_KAFKA_RESP_ERR__NOENT = -156
     RD_KAFKA_RESP_ERR_NO_ERROR = 0
 
@@ -97,6 +98,48 @@ module Rdkafka
     attach_function :rd_kafka_topic_partition_list_destroy, [:pointer], :void
     attach_function :rd_kafka_topic_partition_list_copy, [:pointer], :pointer
 
+    # Configs management
+    #
+    # Structs for management of configurations
+    # Each configuration is attached to a resource and one resource can have many configuration
+    # details. Each resource will also have separate errors results if obtaining configuration
+    # was not possible for any reason
+    class ConfigResource < FFI::Struct
+      layout :type, :int,
+             :name, :string
+    end
+
+    attach_function :rd_kafka_DescribeConfigs, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
+    attach_function :rd_kafka_ConfigResource_new, [:int32, :pointer], :pointer
+    attach_function :rd_kafka_ConfigResource_destroy_array, [:pointer, :int32], :void
+    attach_function :rd_kafka_event_DescribeConfigs_result, [:pointer], :pointer
+    attach_function :rd_kafka_DescribeConfigs_result_resources, [:pointer, :pointer], :pointer
+    attach_function :rd_kafka_ConfigResource_configs, [:pointer, :pointer], :pointer
+    attach_function :rd_kafka_ConfigEntry_name, [:pointer], :string
+    attach_function :rd_kafka_ConfigEntry_value, [:pointer], :string
+    attach_function :rd_kafka_ConfigEntry_is_read_only, [:pointer], :int
+    attach_function :rd_kafka_ConfigEntry_is_default, [:pointer], :int
+    attach_function :rd_kafka_ConfigEntry_is_sensitive, [:pointer], :int
+    attach_function :rd_kafka_ConfigEntry_is_synonym, [:pointer], :int
+    attach_function :rd_kafka_ConfigEntry_synonyms, [:pointer, :pointer], :pointer
+    attach_function :rd_kafka_ConfigResource_error, [:pointer], :int
+    attach_function :rd_kafka_ConfigResource_error_string, [:pointer], :string
+    attach_function :rd_kafka_IncrementalAlterConfigs, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
+    attach_function :rd_kafka_IncrementalAlterConfigs_result_resources, [:pointer, :pointer], :pointer
+    attach_function :rd_kafka_ConfigResource_add_incremental_config, [:pointer, :string, :int32, :string], :pointer
+    attach_function :rd_kafka_event_IncrementalAlterConfigs_result, [:pointer], :pointer
+
+    RD_KAFKA_ADMIN_OP_DESCRIBECONFIGS = 5
+    RD_KAFKA_EVENT_DESCRIBECONFIGS_RESULT = 104
+
+    RD_KAFKA_ADMIN_OP_INCREMENTALALTERCONFIGS = 16
+    RD_KAFKA_EVENT_INCREMENTALALTERCONFIGS_RESULT = 131072
+
+    RD_KAFKA_ALTER_CONFIG_OP_TYPE_SET      = 0
+    RD_KAFKA_ALTER_CONFIG_OP_TYPE_DELETE   = 1
+    RD_KAFKA_ALTER_CONFIG_OP_TYPE_APPEND   = 2
+    RD_KAFKA_ALTER_CONFIG_OP_TYPE_SUBTRACT = 3
+
     # Errors
 
     attach_function :rd_kafka_err2name, [:int], :string
@@ -125,7 +168,10 @@ module Rdkafka
     callback :error_cb, [:pointer, :int, :string, :pointer], :void
     attach_function :rd_kafka_conf_set_error_cb, [:pointer, :error_cb], :void
     attach_function :rd_kafka_rebalance_protocol, [:pointer], :string
-
+    callback :oauthbearer_token_refresh_cb, [:pointer, :string, :pointer], :void
+    attach_function :rd_kafka_conf_set_oauthbearer_token_refresh_cb, [:pointer, :oauthbearer_token_refresh_cb], :void
+    attach_function :rd_kafka_oauthbearer_set_token, [:pointer, :string, :int64, :pointer, :pointer, :int, :pointer, :int], :int
+    attach_function :rd_kafka_oauthbearer_set_token_failure, [:pointer, :string], :int
     # Log queue
     attach_function :rd_kafka_set_log_queue, [:pointer, :pointer], :void
     attach_function :rd_kafka_queue_get_main, [:pointer], :pointer
@@ -134,13 +180,13 @@ module Rdkafka
       :void, [:pointer, :int, :string, :string]
     ) do |_client_ptr, level, _level_string, line|
       severity = case level
-                 when 0 || 1 || 2
+                 when 0, 1, 2
                    Logger::FATAL
                  when 3
                    Logger::ERROR
                  when 4
                    Logger::WARN
-                 when 5 || 6
+                 when 5, 6
                    Logger::INFO
                  when 7
                    Logger::DEBUG
@@ -175,6 +221,32 @@ module Rdkafka
       end
     end
 
+    # The OAuth callback is currently global and contextless.
+    # This means that the callback will be called for all instances, and the callback must be able to determine to which instance it is associated.
+    # The instance name will be provided in the callback, allowing the callback to reference the correct instance.
+    #
+    # An example of how to use the instance name in the callback is given below.
+    # The `refresh_token` is configured as the `oauthbearer_token_refresh_callback`.
+    # `instances` is a map of client names to client instances, maintained by the user.
+    #
+    # ```
+    #   def refresh_token(config, client_name)
+    #     client = instances[client_name]
+    #     client.oauthbearer_set_token(
+    #       token: 'new-token-value',
+    #       lifetime_ms: token-lifetime-ms,
+    #       principal_name: 'principal-name'
+    #     )
+    #   end
+    # ```
+    OAuthbearerTokenRefreshCallback = FFI::Function.new(
+      :void, [:pointer, :string, :pointer]
+    ) do |client_ptr, config, _opaque|
+      if Rdkafka::Config.oauthbearer_token_refresh_callback
+        Rdkafka::Config.oauthbearer_token_refresh_callback.call(config, Rdkafka::Bindings.rd_kafka_name(client_ptr))
+      end
+    end
+
     # Handle
 
     enum :kafka_type, [

data/lib/rdkafka/callbacks.rb

@@ -113,6 +113,42 @@ module Rdkafka
       end
     end
 
+    class DescribeConfigsResult
+      attr_reader :result_error, :error_string, :results, :results_count
+
+      def initialize(event_ptr)
+        @results=[]
+        @result_error = Rdkafka::Bindings.rd_kafka_event_error(event_ptr)
+        @error_string = Rdkafka::Bindings.rd_kafka_event_error_string(event_ptr)
+
+        if @result_error == 0
+          configs_describe_result = Rdkafka::Bindings.rd_kafka_event_DescribeConfigs_result(event_ptr)
+          # Get the number of matching acls
+          pointer_to_size_t = FFI::MemoryPointer.new(:int32)
+          @results = Rdkafka::Bindings.rd_kafka_DescribeConfigs_result_resources(configs_describe_result, pointer_to_size_t)
+          @results_count = pointer_to_size_t.read_int
+        end
+      end
+    end
+
+    class IncrementalAlterConfigsResult
+      attr_reader :result_error, :error_string, :results, :results_count
+
+      def initialize(event_ptr)
+        @results=[]
+        @result_error = Rdkafka::Bindings.rd_kafka_event_error(event_ptr)
+        @error_string = Rdkafka::Bindings.rd_kafka_event_error_string(event_ptr)
+
+        if @result_error == 0
+          incremental_alter_result = Rdkafka::Bindings.rd_kafka_event_IncrementalAlterConfigs_result(event_ptr)
+          # Get the number of matching acls
+          pointer_to_size_t = FFI::MemoryPointer.new(:int32)
+          @results = Rdkafka::Bindings.rd_kafka_IncrementalAlterConfigs_result_resources(incremental_alter_result, pointer_to_size_t)
+          @results_count = pointer_to_size_t.read_int
+        end
+      end
+    end
+
     # FFI Function used for Create Topic and Delete Topic callbacks
     BackgroundEventCallbackFunction = FFI::Function.new(
         :void, [:pointer, :pointer, :pointer]
@@ -123,20 +159,24 @@ module Rdkafka
     # @private
     class BackgroundEventCallback
       def self.call(_, event_ptr, _)
-        event_type = Rdkafka::Bindings.rd_kafka_event_type(event_ptr)
-        if event_type == Rdkafka::Bindings::RD_KAFKA_EVENT_CREATETOPICS_RESULT
+        case Rdkafka::Bindings.rd_kafka_event_type(event_ptr)
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_CREATETOPICS_RESULT
           process_create_topic(event_ptr)
-        elsif event_type == Rdkafka::Bindings::RD_KAFKA_EVENT_DELETETOPICS_RESULT
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBECONFIGS_RESULT
+          process_describe_configs(event_ptr)
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_INCREMENTALALTERCONFIGS_RESULT
+          process_incremental_alter_configs(event_ptr)
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETETOPICS_RESULT
           process_delete_topic(event_ptr)
-        elsif event_type == Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATEPARTITIONS_RESULT
+        when Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATEPARTITIONS_RESULT
           process_create_partitions(event_ptr)
-        elsif event_type == Rdkafka::Bindings::RD_KAFKA_EVENT_CREATEACLS_RESULT
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_CREATEACLS_RESULT
           process_create_acl(event_ptr)
-        elsif event_type == Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEACLS_RESULT
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEACLS_RESULT
           process_delete_acl(event_ptr)
-        elsif event_type == Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBEACLS_RESULT
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBEACLS_RESULT
           process_describe_acl(event_ptr)
-        elsif event_type == Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEGROUPS_RESULT
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEGROUPS_RESULT
           process_delete_groups(event_ptr)
         end
       end
@@ -156,7 +196,44 @@ module Rdkafka
           create_topic_handle[:response] = create_topic_results[0].result_error
           create_topic_handle[:error_string] = create_topic_results[0].error_string
           create_topic_handle[:result_name] = create_topic_results[0].result_name
-          create_topic_handle[:pending] = false
+
+          create_topic_handle.unlock
+        end
+      end
+
+      def self.process_describe_configs(event_ptr)
+        describe_configs = DescribeConfigsResult.new(event_ptr)
+        describe_configs_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
+
+        if describe_configs_handle = Rdkafka::Admin::DescribeConfigsHandle.remove(describe_configs_handle_ptr.address)
+          describe_configs_handle[:response] = describe_configs.result_error
+          describe_configs_handle[:response_string] = describe_configs.error_string
+          describe_configs_handle[:pending] = false
+
+          if describe_configs.result_error == 0
+            describe_configs_handle[:config_entries] = describe_configs.results
+            describe_configs_handle[:entry_count] = describe_configs.results_count
+          end
+
+          describe_configs_handle.unlock
+        end
+      end
+
+      def self.process_incremental_alter_configs(event_ptr)
+        incremental_alter = IncrementalAlterConfigsResult.new(event_ptr)
+        incremental_alter_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
+
+        if incremental_alter_handle = Rdkafka::Admin::IncrementalAlterConfigsHandle.remove(incremental_alter_handle_ptr.address)
+          incremental_alter_handle[:response] = incremental_alter.result_error
+          incremental_alter_handle[:response_string] = incremental_alter.error_string
+          incremental_alter_handle[:pending] = false
+
+          if incremental_alter.result_error == 0
+            incremental_alter_handle[:config_entries] = incremental_alter.results
+            incremental_alter_handle[:entry_count] = incremental_alter.results_count
+          end
+
+          incremental_alter_handle.unlock
         end
       end
 
@@ -173,7 +250,8 @@ module Rdkafka
           delete_group_handle[:response] = delete_group_results[0].result_error
           delete_group_handle[:error_string] = delete_group_results[0].error_string
           delete_group_handle[:result_name] = delete_group_results[0].result_name
-          delete_group_handle[:pending] = false
+
+          delete_group_handle.unlock
         end
       end
 
@@ -190,7 +268,8 @@ module Rdkafka
           delete_topic_handle[:response] = delete_topic_results[0].result_error
           delete_topic_handle[:error_string] = delete_topic_results[0].error_string
           delete_topic_handle[:result_name] = delete_topic_results[0].result_name
-          delete_topic_handle[:pending] = false
+
+          delete_topic_handle.unlock
         end
       end
 
@@ -207,7 +286,8 @@ module Rdkafka
           create_partitions_handle[:response] = create_partitions_results[0].result_error
           create_partitions_handle[:error_string] = create_partitions_results[0].error_string
           create_partitions_handle[:result_name] = create_partitions_results[0].result_name
-          create_partitions_handle[:pending] = false
+
+          create_partitions_handle.unlock
         end
       end
 
@@ -223,7 +303,8 @@ module Rdkafka
         if create_acl_handle = Rdkafka::Admin::CreateAclHandle.remove(create_acl_handle_ptr.address)
           create_acl_handle[:response] = create_acl_results[0].result_error
           create_acl_handle[:response_string] = create_acl_results[0].error_string
-          create_acl_handle[:pending] = false
+
+          create_acl_handle.unlock
         end
       end
 
@@ -239,11 +320,13 @@ module Rdkafka
         if delete_acl_handle = Rdkafka::Admin::DeleteAclHandle.remove(delete_acl_handle_ptr.address)
           delete_acl_handle[:response] = delete_acl_results[0].result_error
           delete_acl_handle[:response_string] = delete_acl_results[0].error_string
-          delete_acl_handle[:pending] = false
+
           if delete_acl_results[0].result_error == 0
             delete_acl_handle[:matching_acls] = delete_acl_results[0].matching_acls
             delete_acl_handle[:matching_acls_count] = delete_acl_results[0].matching_acls_count
           end
+
+          delete_acl_handle.unlock
         end
       end
 
@@ -254,17 +337,18 @@ module Rdkafka
         if describe_acl_handle = Rdkafka::Admin::DescribeAclHandle.remove(describe_acl_handle_ptr.address)
           describe_acl_handle[:response] = describe_acl.result_error
           describe_acl_handle[:response_string] = describe_acl.error_string
-          describe_acl_handle[:pending] = false
+
           if describe_acl.result_error == 0
-            describe_acl_handle[:acls]       = describe_acl.matching_acls
+            describe_acl_handle[:acls] = describe_acl.matching_acls
             describe_acl_handle[:acls_count] = describe_acl.matching_acls_count
           end
+
+          describe_acl_handle.unlock
         end
       end
     end
 
     # FFI Function used for Message Delivery callbacks
-
     DeliveryCallbackFunction = FFI::Function.new(
         :void, [:pointer, :pointer, :pointer]
     ) do |client_ptr, message_ptr, opaque_ptr|
@@ -284,7 +368,6 @@ module Rdkafka
           delivery_handle[:partition] = message[:partition]
           delivery_handle[:offset] = message[:offset]
           delivery_handle[:topic_name] = FFI::MemoryPointer.from_string(topic_name)
-          delivery_handle[:pending] = false
 
           # Call delivery callback on opaque
           if opaque = Rdkafka::Config.opaques[opaque_ptr.to_i]
@@ -299,9 +382,10 @@ module Rdkafka
               delivery_handle
             )
           end
+
+          delivery_handle.unlock
         end
       end
     end
-
   end
 end

data/lib/rdkafka/config.rb

@@ -15,12 +15,13 @@ module Rdkafka
     @@opaques = ObjectSpace::WeakMap.new
     # @private
     @@log_queue = Queue.new
-    # @private
     # We memoize thread on the first log flush
     # This allows us also to restart logger thread on forks
     @@log_thread = nil
     # @private
     @@log_mutex = Mutex.new
+    # @private
+    @@oauthbearer_token_refresh_callback = nil
 
     # Returns the current logger, by default this is a logger to stdout.
     #
@@ -104,6 +105,24 @@ module Rdkafka
       @@error_callback
     end
 
+    # Sets the SASL/OAUTHBEARER token refresh callback.
+    # This callback will be triggered when it is time to refresh the client's OAUTHBEARER token
+    #
+    # @param callback [Proc, #call] The callback
+    #
+    # @return [nil]
+    def self.oauthbearer_token_refresh_callback=(callback)
+      raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call) || callback == nil
+      @@oauthbearer_token_refresh_callback = callback
+    end
+
+    # Returns the current oauthbearer_token_refresh_callback callback, by default this is nil.
+    #
+    # @return [Proc, nil]
+    def self.oauthbearer_token_refresh_callback
+      @@oauthbearer_token_refresh_callback
+    end
+
     # @private
     def self.opaques
       @@opaques
@@ -176,11 +195,13 @@ module Rdkafka
 
     # Creates a consumer with this configuration.
     #
+    # @param native_kafka_auto_start [Boolean] should the native kafka operations be started
+    #   automatically. Defaults to true. Set to false only when doing complex initialization.
     # @return [Consumer] The created consumer
     #
     # @raise [ConfigError] When the configuration contains invalid options
     # @raise [ClientCreationError] When the native client cannot be created
-    def consumer
+    def consumer(native_kafka_auto_start: true)
       opaque = Opaque.new
       config = native_config(opaque)
 
@@ -200,18 +221,21 @@ module Rdkafka
         Rdkafka::NativeKafka.new(
           kafka,
           run_polling_thread: false,
-          opaque: opaque
+          opaque: opaque,
+          auto_start: native_kafka_auto_start
         )
       )
     end
 
     # Create a producer with this configuration.
     #
+    # @param native_kafka_auto_start [Boolean] should the native kafka operations be started
+    #   automatically. Defaults to true. Set to false only when doing complex initialization.
     # @return [Producer] The created producer
     #
     # @raise [ConfigError] When the configuration contains invalid options
     # @raise [ClientCreationError] When the native client cannot be created
-    def producer
+    def producer(native_kafka_auto_start: true)
       # Create opaque
       opaque = Opaque.new
       # Create Kafka config
@@ -220,11 +244,15 @@ module Rdkafka
       Rdkafka::Bindings.rd_kafka_conf_set_dr_msg_cb(config, Rdkafka::Callbacks::DeliveryCallbackFunction)
       # Return producer with Kafka client
       partitioner_name = self[:partitioner] || self["partitioner"]
+
+      kafka = native_kafka(config, :rd_kafka_producer)
+
       Rdkafka::Producer.new(
         Rdkafka::NativeKafka.new(
-          native_kafka(config, :rd_kafka_producer),
+          kafka,
           run_polling_thread: true,
-          opaque: opaque
+          opaque: opaque,
+          auto_start: native_kafka_auto_start
         ),
         partitioner_name
       ).tap do |producer|
@@ -234,19 +262,25 @@ module Rdkafka
 
     # Creates an admin instance with this configuration.
     #
+    # @param native_kafka_auto_start [Boolean] should the native kafka operations be started
+    #   automatically. Defaults to true. Set to false only when doing complex initialization.
     # @return [Admin] The created admin instance
     #
     # @raise [ConfigError] When the configuration contains invalid options
     # @raise [ClientCreationError] When the native client cannot be created
-    def admin
+    def admin(native_kafka_auto_start: true)
       opaque = Opaque.new
       config = native_config(opaque)
       Rdkafka::Bindings.rd_kafka_conf_set_background_event_cb(config, Rdkafka::Callbacks::BackgroundEventCallbackFunction)
+
+      kafka = native_kafka(config, :rd_kafka_producer)
+
       Rdkafka::Admin.new(
         Rdkafka::NativeKafka.new(
-          native_kafka(config, :rd_kafka_producer),
+          kafka,
           run_polling_thread: true,
-          opaque: opaque
+          opaque: opaque,
+          auto_start: native_kafka_auto_start
         )
       )
     end
@@ -300,6 +334,9 @@ module Rdkafka
 
         # Set error callback
         Rdkafka::Bindings.rd_kafka_conf_set_error_cb(config, Rdkafka::Bindings::ErrorCallback)
+
+        # Set oauth callback
+        Rdkafka::Bindings.rd_kafka_conf_set_oauthbearer_token_refresh_cb(config, Rdkafka::Bindings::OAuthbearerTokenRefreshCallback)
       end
     end
 

data/lib/rdkafka/consumer.rb

@@ -13,12 +13,19 @@ module Rdkafka
   class Consumer
     include Enumerable
     include Helpers::Time
+    include Helpers::OAuth
 
     # @private
     def initialize(native_kafka)
       @native_kafka = native_kafka
     end
 
+    # Starts the native Kafka polling thread and kicks off the init polling
+    # @note Not needed to run unless explicit start was disabled
+    def start
+      @native_kafka.start
+    end
+
     # @return [String] consumer name
     def name
       @name ||= @native_kafka.with_inner do |inner|