rdkafka 0.15.1 → 0.16.0.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8636c80e1798cf24b34cf25a20ca24f35e2951fb179843a3a85a94fe0274ca76
-  data.tar.gz: f115aa7fff4961d42280a7ad6fd78fed40568b936139d588ae2362a2f0f45c25
+  metadata.gz: 3135d4f2663517517330d165948e9761ffc0ecd20942f911a6ee9541c437ee7e
+  data.tar.gz: 9a489c2400c4054e9cec0d6c8d24f75bce407d370f8c393b7b31dcd0dbf7c361
 SHA512:
-  metadata.gz: e5b5368a732e42b1c57aff93a7172c95b0bad93ac646dcba495c0509c5b6c29cf8753601d1f04f028c98579846e5db7e7119ae9a6a4cca2f41441316b60b5c9c
-  data.tar.gz: 8596d6944d5151df3ad875d93dbd7cf2aee00c1ded85e053e96880b8c5420ca6ba18a72f57cc867a7bfedf38060be3c9ea4334d79d7a21b2503a078bce1d266a
+  metadata.gz: be8eba2aec012af189d893ebf6ddcd4e4dec117aecd7f36afa76bc6ee4c2a3fa93f4ac4160efa16e8c91efd7011ee41cda00a5e2146e18044a166035850cd490
+  data.tar.gz: d1761a6ab9c7d9ee539d79679dc730cc392fc2d7369d3f78b9c8d8f1c800855ab15ae7c472259b43e95139aa30d4777026c0529671fd85f98f3e5fceeaf53e62

data/.github/workflows/ci.yml

@@ -25,9 +25,7 @@ jobs:
           - '3.3'
           - '3.2'
           - '3.1'
-          - '3.1.0'
           - '3.0'
-          - '3.0.0'
           - '2.7'
         include:
           - ruby: '3.3'
@@ -37,9 +35,9 @@ jobs:
       - name: Install package dependencies
         run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
 
-      - name: Start Kafka with docker-compose
+      - name: Start Kafka with docker compose
         run: |
-          docker-compose up -d || (sleep 5 && docker-compose up -d)
+          docker compose up -d || (sleep 5 && docker compose up -d)
 
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1

data/.gitignore

@@ -10,3 +10,5 @@ ext/librdkafka.*
 doc
 coverage
 vendor
+.idea/
+out/

data/.ruby-version

@@ -1 +1 @@
-3.3.0
+3.3.1

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Rdkafka Changelog
 
+## 0.16.0 (Unreleased)
+- **[Feature]** Oauthbearer token refresh callback (bruce-szalwinski-he)
+- [Enhancement] Replace time poll based wait engine with an event based to improve response times on blocking operations and wait (nijikon + mensfeld)
+- [Enhancement] Allow for usage of the second regex engine of librdkafka by setting `RDKAFKA_DISABLE_REGEX_EXT` during build (mensfeld)
+- [Enhancement] name polling Thread as `rdkafka.native_kafka#<name>` (nijikon)
+- [Change] Allow for native kafka thread operations deferring and manual start for consumer, producer and admin.
+- [Change] The `wait_timeout` argument in `AbstractHandle.wait` method is deprecated and will be removed in future versions without replacement. We don't rely on it's value anymore (nijikon)
+- [Fix] Background logger stops working after forking causing memory leaks (mensfeld)
+- [Fix] Fix bogus case/when syntax. Levels 1, 2, and 6 previously defaulted to UNKNOWN (jjowdy)
+
 ## 0.15.1 (2024-01-30)
 - [Enhancement] Provide support for Nix OS (alexandriainfantino)
 - [Enhancement] Replace `rd_kafka_offset_store` with `rd_kafka_offsets_store` (mensfeld)
@@ -20,7 +30,7 @@
 - **[Feature]** Add `Admin#delete_group` utility (piotaixr)
 - **[Feature]** Add Create and Delete ACL Feature To Admin Functions (vgnanasekaran)
 - **[Feature]** Support `#assignment_lost?` on a consumer to check for involuntary assignment revocation (mensfeld)
-- [Enhancement] Expose alternative way of managing consumer events via a separate queue (mensfeld) 
+- [Enhancement] Expose alternative way of managing consumer events via a separate queue (mensfeld)
 - [Enhancement] **Bump** librdkafka to 2.3.0 (mensfeld)
 - [Enhancement] Increase the `#lag` and `#query_watermark_offsets` default timeouts from 100ms to 1000ms. This will compensate for network glitches and remote clusters operations (mensfeld)
 - [Change] Use `SecureRandom.uuid` instead of `random` for test consumer groups (mensfeld)

data/README.md

@@ -18,7 +18,7 @@ become EOL.
 
 `rdkafka` was written because of the need for a reliable Ruby client for Kafka that supports modern Kafka at [AppSignal](https://appsignal.com). AppSignal runs it in production on very high-traffic systems.
 
-The most important pieces of a Kafka client are implemented, and we aim to provide all relevant consumer, producer, and admin APIs.
+The most essential pieces of a Kafka client are implemented, and we aim to provide all relevant consumer, producer, and admin APIs.
 
 ## Table of content
 
@@ -30,6 +30,7 @@ The most important pieces of a Kafka client are implemented, and we aim to provi
 - [Higher Level Libraries](#higher-level-libraries)
   * [Message Processing Frameworks](#message-processing-frameworks)
   * [Message Publishing Libraries](#message-publishing-libraries)
+- [Forking](#forking)
 - [Development](#development)
 - [Example](#example)
 - [Versions](#versions)
@@ -47,12 +48,13 @@ While rdkafka-ruby aims to simplify the use of librdkafka in Ruby applications,
 
 ## Installation
 
-This gem downloads and compiles librdkafka when it is installed. If you
-If you have any problems installing the gem, please open an issue.
+When installed, this gem downloads and compiles librdkafka. If you have any problems installing the gem, please open an issue.
 
 ## Usage
 
-See the [documentation](https://karafka.io/docs/code/rdkafka-ruby/) for full details on how to use this gem. Two quick examples:
+Please see the [documentation](https://karafka.io/docs/code/rdkafka-ruby/) for full details on how to use this gem. Below are two quick examples.
+
+Unless you are seeking specific low-level capabilities, we **strongly** recommend using [Karafka](https://github.com/karafka/karafka) and [WaterDrop](https://github.com/karafka/waterdrop) when working with Kafka. These are higher-level libraries also maintained by us based on rdkafka-ruby.
 
 ### Consuming Messages
 
@@ -74,7 +76,7 @@ end
 
 ### Producing Messages
 
-Produce a number of messages, put the delivery handles in an array, and
+Produce several messages, put the delivery handles in an array, and
 wait for them before exiting. This way the messages will be batched and
 efficiently sent to Kafka.
 
@@ -95,13 +97,11 @@ end
 delivery_handles.each(&:wait)
 ```
 
-Note that creating a producer consumes some resources that will not be
-released until it `#close` is explicitly called, so be sure to call
-`Config#producer` only as necessary.
+Note that creating a producer consumes some resources that will not be released until it `#close` is explicitly called, so be sure to call `Config#producer` only as necessary.
 
 ## Higher Level Libraries
 
-Currently, there are two actively developed frameworks based on rdkafka-ruby, that provide higher-level API that can be used to work with Kafka messages and one library for publishing messages.
+Currently, there are two actively developed frameworks based on `rdkafka-ruby`, that provide higher-level API that can be used to work with Kafka messages and one library for publishing messages.
 
 ### Message Processing Frameworks
 
@@ -112,6 +112,16 @@ Currently, there are two actively developed frameworks based on rdkafka-ruby, th
 
 * [WaterDrop](https://github.com/karafka/waterdrop) – Standalone Karafka library for producing Kafka messages.
 
+## Forking
+
+When working with `rdkafka-ruby`, it's essential to know that the underlying `librdkafka` library does not support fork-safe operations, even though it is thread-safe. Forking a process after initializing librdkafka clients can lead to unpredictable behavior due to inherited file descriptors and memory states. This limitation requires careful handling, especially in Ruby applications that rely on forking.
+
+To address this, it's highly recommended to:
+
+- Never initialize any `rdkafka-ruby` producers or consumers before forking to avoid state corruption.
+- Before forking, always close any open producers or consumers if you've opened any.
+- Use high-level libraries like [WaterDrop](https://github.com/karafka/waterdrop) and [Karafka](https://github.com/karafka/karafka/), which provide abstractions for handling librdkafka's intricacies.
+
 ## Development
 
 Contributors are encouraged to focus on enhancements that align with the core goal of the library. We appreciate contributions but will likely not accept pull requests for features that:

data/docker-compose.yml

@@ -3,7 +3,7 @@ version: '2'
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:7.5.3
+    image: confluentinc/cp-kafka:7.6.1
 
     ports:
       - 9092:9092

data/ext/Rakefile

@@ -27,6 +27,14 @@ task :default => :clean do
       :sha256 => Rdkafka::LIBRDKAFKA_SOURCE_SHA256
     }
     recipe.configure_options = ["--host=#{recipe.host}"]
+
+    # Disable using libc regex engine in favor of the embedded one
+    # The default regex engine of librdkafka does not always work exactly as most of the users
+    # would expect, hence this flag allows for changing it to the other one
+    if ENV.key?('RDKAFKA_DISABLE_REGEX_EXT')
+      recipe.configure_options << '--disable-regex-ext'
+    end
+
     recipe.cook
     # Move dynamic library we're interested in
     if recipe.host.include?('darwin')

data/lib/rdkafka/abstract_handle.rb

@@ -14,6 +14,13 @@ module Rdkafka
 
     # Registry for registering all the handles.
     REGISTRY = {}
+    # Default wait timeout is 31 years
+    MAX_WAIT_TIMEOUT_FOREVER = 10_000_000_000
+    # Deprecation message for wait_timeout argument in wait method
+    WAIT_TIMEOUT_DEPRECATION_MESSAGE = "The 'wait_timeout' argument is deprecated and will be removed in future versions without replacement. " \
+      "We don't rely on it's value anymore. Please refactor your code to remove references to it."
+
+    private_constant :MAX_WAIT_TIMEOUT_FOREVER
 
     class << self
       # Adds handle to the register
@@ -32,6 +39,12 @@ module Rdkafka
       end
     end
 
+    def initialize
+      @mutex = Thread::Mutex.new
+      @resource = Thread::ConditionVariable.new
+
+      super
+    end
 
     # Whether the handle is still pending.
     #
@@ -45,37 +58,48 @@ module Rdkafka
     # on the operation. In this case it is possible to call wait again.
     #
     # @param max_wait_timeout [Numeric, nil] Amount of time to wait before timing out.
-    #   If this is nil it does not time out.
-    # @param wait_timeout [Numeric] Amount of time we should wait before we recheck if the
-    #   operation has completed
+    #   If this is nil we will wait forever
+    # @param wait_timeout [nil] deprecated
     # @param raise_response_error [Boolean] should we raise error when waiting finishes
     #
     # @return [Object] Operation-specific result
     #
     # @raise [RdkafkaError] When the operation failed
     # @raise [WaitTimeoutError] When the timeout has been reached and the handle is still pending
-    def wait(max_wait_timeout: 60, wait_timeout: 0.1, raise_response_error: true)
-      timeout = if max_wait_timeout
-                  monotonic_now + max_wait_timeout
-                else
-                  nil
-                end
-      loop do
-        if pending?
-          if timeout && timeout <= monotonic_now
-            raise WaitTimeoutError.new(
-              "Waiting for #{operation_name} timed out after #{max_wait_timeout} seconds"
-            )
+    def wait(max_wait_timeout: 60, wait_timeout: nil, raise_response_error: true)
+      Kernel.warn(WAIT_TIMEOUT_DEPRECATION_MESSAGE) unless wait_timeout.nil?
+
+      timeout = max_wait_timeout ? monotonic_now + max_wait_timeout : MAX_WAIT_TIMEOUT_FOREVER
+
+      @mutex.synchronize do
+        loop do
+          if pending?
+            to_wait = (timeout - monotonic_now)
+
+            if to_wait.positive?
+              @resource.wait(@mutex, to_wait)
+            else
+              raise WaitTimeoutError.new(
+                "Waiting for #{operation_name} timed out after #{max_wait_timeout} seconds"
+              )
+            end
+          elsif self[:response] != 0 && raise_response_error
+            raise_error
+          else
+            return create_result
           end
-          sleep wait_timeout
-        elsif self[:response] != 0 && raise_response_error
-          raise_error
-        else
-          return create_result
         end
       end
     end
 
+    # Unlock the resources
+    def unlock
+      @mutex.synchronize do
+        self[:pending] = false
+        @resource.broadcast
+      end
+    end
+
     # @return [String] the name of the operation (e.g. "delivery")
     def operation_name
       raise "Must be implemented by subclass!"

data/lib/rdkafka/admin/create_topic_report.rb

@@ -16,7 +16,7 @@ module Rdkafka
           @error_string = error_string.read_string
         end
         if result_name != FFI::Pointer::NULL
-          @result_name = @result_name = result_name.read_string
+          @result_name = result_name.read_string
         end
       end
     end

data/lib/rdkafka/admin/delete_groups_report.rb

@@ -16,7 +16,7 @@ module Rdkafka
           @error_string = error_string.read_string
         end
         if result_name != FFI::Pointer::NULL
-          @result_name = @result_name = result_name.read_string
+          @result_name = result_name.read_string
         end
       end
     end

data/lib/rdkafka/admin/delete_topic_report.rb

@@ -16,7 +16,7 @@ module Rdkafka
           @error_string = error_string.read_string
         end
         if result_name != FFI::Pointer::NULL
-          @result_name = @result_name = result_name.read_string
+          @result_name = result_name.read_string
         end
       end
     end

data/lib/rdkafka/admin.rb

@@ -2,6 +2,8 @@
 
 module Rdkafka
   class Admin
+    include Helpers::OAuth
+
     # @private
     def initialize(native_kafka)
       @native_kafka = native_kafka
@@ -10,6 +12,19 @@ module Rdkafka
       ObjectSpace.define_finalizer(self, native_kafka.finalizer)
     end
 
+    # Starts the native Kafka polling thread and kicks off the init polling
+    # @note Not needed to run unless explicit start was disabled
+    def start
+      @native_kafka.start
+    end
+
+    # @return [String] admin name
+    def name
+      @name ||= @native_kafka.with_inner do |inner|
+        ::Rdkafka::Bindings.rd_kafka_name(inner)
+      end
+    end
+
     def finalizer
       ->(_) { close }
     end

data/lib/rdkafka/bindings.rb

@@ -17,6 +17,7 @@ module Rdkafka
 
     RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS = -175
     RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS = -174
+    RD_KAFKA_RESP_ERR__STATE = -172
     RD_KAFKA_RESP_ERR__NOENT = -156
     RD_KAFKA_RESP_ERR_NO_ERROR = 0
 
@@ -111,7 +112,10 @@ module Rdkafka
     callback :error_cb, [:pointer, :int, :string, :pointer], :void
     attach_function :rd_kafka_conf_set_error_cb, [:pointer, :error_cb], :void
     attach_function :rd_kafka_rebalance_protocol, [:pointer], :string
-
+    callback :oauthbearer_token_refresh_cb, [:pointer, :string, :pointer], :void
+    attach_function :rd_kafka_conf_set_oauthbearer_token_refresh_cb, [:pointer, :oauthbearer_token_refresh_cb], :void
+    attach_function :rd_kafka_oauthbearer_set_token, [:pointer, :string, :int64, :pointer, :pointer, :int, :pointer, :int], :int
+    attach_function :rd_kafka_oauthbearer_set_token_failure, [:pointer, :string], :int
     # Log queue
     attach_function :rd_kafka_set_log_queue, [:pointer, :pointer], :void
     attach_function :rd_kafka_queue_get_main, [:pointer], :pointer
@@ -120,19 +124,21 @@ module Rdkafka
       :void, [:pointer, :int, :string, :string]
     ) do |_client_ptr, level, _level_string, line|
       severity = case level
-                 when 0 || 1 || 2
+                 when 0, 1, 2
                    Logger::FATAL
                  when 3
                    Logger::ERROR
                  when 4
                    Logger::WARN
-                 when 5 || 6
+                 when 5, 6
                    Logger::INFO
                  when 7
                    Logger::DEBUG
                  else
                    Logger::UNKNOWN
                  end
+
+      Rdkafka::Config.ensure_log_thread
       Rdkafka::Config.log_queue << [severity, "rdkafka: #{line}"]
     end
 
@@ -159,6 +165,32 @@ module Rdkafka
       end
     end
 
+    # The OAuth callback is currently global and contextless.
+    # This means that the callback will be called for all instances, and the callback must be able to determine to which instance it is associated.
+    # The instance name will be provided in the callback, allowing the callback to reference the correct instance.
+    #
+    # An example of how to use the instance name in the callback is given below.
+    # The `refresh_token` is configured as the `oauthbearer_token_refresh_callback`.
+    # `instances` is a map of client names to client instances, maintained by the user.
+    #
+    # ```
+    #   def refresh_token(config, client_name)
+    #     client = instances[client_name]
+    #     client.oauthbearer_set_token(
+    #       token: 'new-token-value',
+    #       lifetime_ms: token-lifetime-ms,
+    #       principal_name: 'principal-name'
+    #     )
+    #   end
+    # ```
+    OAuthbearerTokenRefreshCallback = FFI::Function.new(
+      :void, [:pointer, :string, :pointer]
+    ) do |client_ptr, config, _opaque|
+      if Rdkafka::Config.oauthbearer_token_refresh_callback
+        Rdkafka::Config.oauthbearer_token_refresh_callback.call(config, Rdkafka::Bindings.rd_kafka_name(client_ptr))
+      end
+    end
+
     # Handle
 
     enum :kafka_type, [

data/lib/rdkafka/callbacks.rb

@@ -156,7 +156,8 @@ module Rdkafka
           create_topic_handle[:response] = create_topic_results[0].result_error
           create_topic_handle[:error_string] = create_topic_results[0].error_string
           create_topic_handle[:result_name] = create_topic_results[0].result_name
-          create_topic_handle[:pending] = false
+
+          create_topic_handle.unlock
         end
       end
 
@@ -173,7 +174,8 @@ module Rdkafka
           delete_group_handle[:response] = delete_group_results[0].result_error
           delete_group_handle[:error_string] = delete_group_results[0].error_string
           delete_group_handle[:result_name] = delete_group_results[0].result_name
-          delete_group_handle[:pending] = false
+
+          delete_group_handle.unlock
         end
       end
 
@@ -190,7 +192,8 @@ module Rdkafka
           delete_topic_handle[:response] = delete_topic_results[0].result_error
           delete_topic_handle[:error_string] = delete_topic_results[0].error_string
           delete_topic_handle[:result_name] = delete_topic_results[0].result_name
-          delete_topic_handle[:pending] = false
+
+          delete_topic_handle.unlock
         end
       end
 
@@ -207,7 +210,8 @@ module Rdkafka
           create_partitions_handle[:response] = create_partitions_results[0].result_error
           create_partitions_handle[:error_string] = create_partitions_results[0].error_string
           create_partitions_handle[:result_name] = create_partitions_results[0].result_name
-          create_partitions_handle[:pending] = false
+
+          create_partitions_handle.unlock
         end
       end
 
@@ -223,7 +227,8 @@ module Rdkafka
         if create_acl_handle = Rdkafka::Admin::CreateAclHandle.remove(create_acl_handle_ptr.address)
           create_acl_handle[:response] = create_acl_results[0].result_error
           create_acl_handle[:response_string] = create_acl_results[0].error_string
-          create_acl_handle[:pending] = false
+
+          create_acl_handle.unlock
         end
       end
 
@@ -239,11 +244,13 @@ module Rdkafka
         if delete_acl_handle = Rdkafka::Admin::DeleteAclHandle.remove(delete_acl_handle_ptr.address)
           delete_acl_handle[:response] = delete_acl_results[0].result_error
           delete_acl_handle[:response_string] = delete_acl_results[0].error_string
-          delete_acl_handle[:pending] = false
+
           if delete_acl_results[0].result_error == 0
             delete_acl_handle[:matching_acls] = delete_acl_results[0].matching_acls
             delete_acl_handle[:matching_acls_count] = delete_acl_results[0].matching_acls_count
           end
+
+          delete_acl_handle.unlock
         end
       end
 
@@ -254,17 +261,18 @@ module Rdkafka
         if describe_acl_handle = Rdkafka::Admin::DescribeAclHandle.remove(describe_acl_handle_ptr.address)
           describe_acl_handle[:response] = describe_acl.result_error
           describe_acl_handle[:response_string] = describe_acl.error_string
-          describe_acl_handle[:pending] = false
+
           if describe_acl.result_error == 0
             describe_acl_handle[:acls]       = describe_acl.matching_acls
             describe_acl_handle[:acls_count] = describe_acl.matching_acls_count
           end
+
+          describe_acl_handle.unlock
         end
       end
     end
 
     # FFI Function used for Message Delivery callbacks
-
     DeliveryCallbackFunction = FFI::Function.new(
         :void, [:pointer, :pointer, :pointer]
     ) do |client_ptr, message_ptr, opaque_ptr|
@@ -284,7 +292,6 @@ module Rdkafka
           delivery_handle[:partition] = message[:partition]
           delivery_handle[:offset] = message[:offset]
           delivery_handle[:topic_name] = FFI::MemoryPointer.from_string(topic_name)
-          delivery_handle[:pending] = false
 
           # Call delivery callback on opaque
           if opaque = Rdkafka::Config.opaques[opaque_ptr.to_i]
@@ -299,9 +306,10 @@ module Rdkafka
               delivery_handle
             )
           end
+
+          delivery_handle.unlock
         end
       end
     end
-
   end
 end