karafka-rdkafka 0.13.8 → 0.13.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4082c381a9d131273cc005b61b06f0e4c73b27044213d730fa5c7faeec606e07
-  data.tar.gz: 06754bdba16fc3feaf648670e766cdbbc60342b1a43902ef08e42a5acfd4b2ac
+  metadata.gz: c954a06b4461885d7648e8081d0426ebb895a82f1a27607f2224fc9e60843574
+  data.tar.gz: a25bf01b430920c7fd2cf22b0dd23ea1de9dd15d1b45f614025e97a3b725f8f3
 SHA512:
-  metadata.gz: 89c0b5078f97c31d0e209ebbf13279a2d079aef35f7b4c2a4bf92cac17a69ccfed1769f84d1d06ca734c9a95174b0b5fa4fa9b6933f63e30fe71c8493bda4ed1
-  data.tar.gz: 490a160689be7c4e261b2b8ff1ffdbcb5aec53cfe1dd0acabd0da33eff708cb7a1d09afe3c3a6752fc9a6345c005f1f0919604035a97240c5f9ed120870e802c
+  metadata.gz: afe480fedbbe5dc6f055709aaf3601acb17aaaa890f1c6e7423b90263eb8fa893de0e8b9bb571e6abf1c5fc0b8df2a6be2e5de4af1de1b68414204275981246e
+  data.tar.gz: 7bb81ff6dcbc95b018ede8bd986eda701f1f5f3bb3985684d436a38a2173f8753b01e1642780eb9ad52a65ad02e2b3b7d656c4259758c8571ef2858822384875

data/.gitignore

@@ -1,3 +1,6 @@
+# Ignore bundler config.
+/.bundle
+
 Gemfile.lock
 ext/ports
 ext/tmp
@@ -6,3 +9,4 @@ ext/librdkafka.*
 .yardoc
 doc
 coverage
+vendor

data/.rspec

@@ -1 +1,2 @@
+--require spec_helper
 --format documentation

data/.ruby-gemset

@@ -0,0 +1 @@
+rdkafka-ruby

data/.ruby-version

@@ -0,0 +1 @@
+3.2.2

data/CHANGELOG.md

@@ -1,33 +1,39 @@
-# 0.13.8 (2023-10-31)
+# Rdkafka Changelog
+
+## 0.13.9 (2023-11-07)
+- [Enhancement] Expose alternative way of managing consumer events via a separate queue.
+- [Enhancement] Allow for setting `statistics_callback` as nil to reset predefined settings configured by a different gem.
+
+## 0.13.8 (2023-10-31)
 - [Enhancement] Get consumer position (thijsc & mensfeld)
 
-# 0.13.7 (2023-10-31)
+## 0.13.7 (2023-10-31)
 - [Change] Drop support for Ruby 2.6 due to incompatibilities in usage of `ObjectSpace::WeakMap`
 - [Fix] Fix dangling Opaque references.
 
-# 0.13.6 (2023-10-17)
+## 0.13.6 (2023-10-17)
 * **[Feature]** Support transactions API in the producer
 * [Enhancement] Add `raise_response_error` flag to the `Rdkafka::AbstractHandle`.
 * [Enhancement] Provide `#purge` to remove any outstanding requests from the producer.
 * [Enhancement] Fix `#flush` does not handle the timeouts errors by making it return true if all flushed or false if failed. We do **not** raise an exception here to keep it backwards compatible.
 
-# 0.13.5
+## 0.13.5
 * Fix DeliveryReport `create_result#error` being nil despite an error being associated with it
 
-# 0.13.4
+## 0.13.4
 * Always call initial poll on librdkafka to make sure oauth bearer cb is handled pre-operations.
 
-# 0.13.3
+## 0.13.3
 * Bump librdkafka to 2.2.0
 
-# 0.13.2
+## 0.13.2
 * Ensure operations counter decrement is fully thread-safe
 * Bump librdkafka to 2.1.1
 
-# 0.13.1
+## 0.13.1
 * Add offsets_for_times method on consumer (timflapper)
 
-# 0.13.0
+## 0.13.0 (2023-07-24)
 * Support cooperative sticky partition assignment in the rebalance callback (methodmissing)
 * Support both string and symbol header keys (ColinDKelley)
 * Handle tombstone messages properly (kgalieva)
@@ -48,32 +54,32 @@
 * Retry metadta fetches on certain errors with a backoff (mensfeld)
 * Do not lock access to underlying native kafka client and rely on Karafka granular locking (mensfeld)
 
-# 0.12.3
+## 0.12.3
 - Include backtrace in non-raised binded errors.
 - Include topic name in the delivery reports
 
-# 0.12.2
+## 0.12.2
 * Increase the metadata default timeout from 250ms to 2 seconds. This should allow for working with remote clusters.
 
-# 0.12.1
+## 0.12.1
 * Bumps librdkafka to 2.0.2 (lmaia)
 * Add support for adding more partitions via Admin API
 
-# 0.12.0
+## 0.12.0 (2022-06-17)
 * Bumps librdkafka to 1.9.0
 * Fix crash on empty partition key (mensfeld)
 * Pass the delivery handle to the callback (gvisokinskas)
 
-# 0.11.0
+## 0.11.0 (2021-11-17)
 * Upgrade librdkafka to 1.8.2
 * Bump supported minimum Ruby version to 2.6
 * Better homebrew path detection
 
-# 0.10.0
+## 0.10.0 (2021-09-07)
 * Upgrade librdkafka to 1.5.0
 * Add error callback config
 
-# 0.9.0
+## 0.9.0 (2021-06-23)
 * Fixes for Ruby 3.0
 * Allow any callable object for callbacks (gremerritt)
 * Reduce memory allocations in Rdkafka::Producer#produce (jturkel)
@@ -81,13 +87,13 @@
 * Allow passing in topic configuration on create_topic (dezka)
 * Add each_batch method to consumer (mgrosso)
 
-# 0.8.1
+## 0.8.1 (2020-12-07)
 * Fix topic_flag behaviour and add tests for Metadata (geoff2k)
 * Add topic admin interface (geoff2k)
 * Raise an exception if @native_kafka is nil (geoff2k)
 * Option to use zstd compression (jasonmartens)
 
-# 0.8.0
+## 0.8.0 (2020-06-02)
 * Upgrade librdkafka to 1.4.0
 * Integrate librdkafka metadata API and add partition_key (by Adithya-copart)
 * Ruby 2.7 compatibility fix (by Geoff Thé)A
@@ -95,22 +101,22 @@
 * Don't override CPPFLAGS and LDFLAGS if already set on Mac (by Hiroshi Hatake)
 * Allow use of Rake 13.x and up (by Tomasz Pajor)
 
-# 0.7.0
+## 0.7.0 (2019-09-21)
 * Bump librdkafka to 1.2.0 (by rob-as)
 * Allow customizing the wait time for delivery report availability (by mensfeld)
 
-# 0.6.0
+## 0.6.0 (2019-07-23)
 * Bump librdkafka to 1.1.0 (by Chris Gaffney)
 * Implement seek (by breunigs)
 
-# 0.5.0
+## 0.5.0 (2019-04-11)
 * Bump librdkafka to 1.0.0 (by breunigs)
 * Add cluster and member information (by dmexe)
 * Support message headers for consumer & producer (by dmexe)
 * Add consumer rebalance listener (by dmexe)
 * Implement pause/resume partitions (by dmexe)
 
-# 0.4.2
+## 0.4.2 (2019-01-12)
 * Delivery callback for producer
 * Document list param of commit method
 * Use default Homebrew openssl location if present
@@ -119,10 +125,10 @@
 * Add support for storing message offsets
 * Add missing runtime dependency to rake
 
-# 0.4.1
+## 0.4.1 (2018-10-19)
 * Bump librdkafka to 0.11.6
 
-# 0.4.0
+## 0.4.0 (2018-09-24)
 * Improvements in librdkafka archive download
 * Add global statistics callback
 * Use Time for timestamps, potentially breaking change if you
@@ -134,34 +140,34 @@
 * Support committing a topic partition list
 * Add consumer assignment method
 
-# 0.3.5
+## 0.3.5 (2018-01-17)
 * Fix crash when not waiting for delivery handles
 * Run specs on Ruby 2.5
 
-# 0.3.4
+## 0.3.4 (2017-12-05)
 * Bump librdkafka to 0.11.3
 
-# 0.3.3
+## 0.3.3 (2017-10-27)
 * Fix bug that prevent display of `RdkafkaError` message
 
-# 0.3.2
+## 0.3.2 (2017-10-25)
 * `add_topic` now supports using a partition count
 * Add way to make errors clearer with an extra message
 * Show topics in subscribe error message
 * Show partition and topic in query watermark offsets error message
 
-# 0.3.1
+## 0.3.1 (2017-10-23)
 * Bump librdkafka to 0.11.1
 * Officially support ranges in `add_topic` for topic partition list.
 * Add consumer lag calculator
 
-# 0.3.0
+## 0.3.0 (2017-10-17)
 * Move both add topic methods to one `add_topic` in `TopicPartitionList`
 * Add committed offsets to consumer
 * Add query watermark offset to consumer
 
-# 0.2.0
+## 0.2.0 (2017-10-13)
 * Some refactoring and add inline documentation
 
-# 0.1.x
+## 0.1.x (2017-09-10)
 * Initial working version including producing and consuming

data/{LICENSE → MIT-LICENSE}

@@ -1,6 +1,7 @@
 The MIT License (MIT)
 
-Copyright (c) 2017 Thijs Cadier
+Copyright (c) 2017-2023 Thijs Cadier
+              2023, Maciej Mensfeld
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -10,16 +10,16 @@
 ---
 
 The `rdkafka` gem is a modern Kafka client library for Ruby based on
-[librdkafka](https://github.com/edenhill/librdkafka/).
+[librdkafka](https://github.com/confluentinc/librdkafka/).
 It wraps the production-ready C client using the [ffi](https://github.com/ffi/ffi)
-gem and targets Kafka 1.0+ and Ruby versions that are under security or
-active maintenance. We remove Ruby version from our CI builds if they 
+gem and targets Kafka 1.0+ and Ruby versions under security or
+active maintenance. We remove a Ruby version from our CI builds when they 
 become EOL.
 
 `rdkafka` was written because of the need for a reliable Ruby client for Kafka that supports modern Kafka at [AppSignal](https://appsignal.com). AppSignal runs it in production on very high-traffic systems.
 
 The most important pieces of a Kafka client are implemented. We're
-working towards feature completeness, you can track that here:
+working towards feature completeness. You can track that here:
 https://github.com/appsignal/rdkafka-ruby/milestone/1
 
 ## Table of content
@@ -38,7 +38,7 @@ https://github.com/appsignal/rdkafka-ruby/milestone/1
 ## Installation
 
 This gem downloads and compiles librdkafka when it is installed. If you
-have any problems installing the gem please open an issue.
+If you have any problems installing the gem, please open an issue.
 
 ## Usage
 
@@ -64,9 +64,9 @@ end
 
 ### Producing messages
 
-Produce a number of messages, put the delivery handles in an array and
+Produce a number of messages, put the delivery handles in an array, and
 wait for them before exiting. This way the messages will be batched and
-sent to Kafka in an efficient way.
+efficiently sent to Kafka.
 
 ```ruby
 config = {:"bootstrap.servers" => "localhost:9092"}
@@ -91,7 +91,7 @@ released until it `#close` is explicitly called, so be sure to call
 
 ## Higher level libraries
 
-Currently, there are two actively developed frameworks based on rdkafka-ruby, that provide higher level API that can be used to work with Kafka messages and one library for publishing messages.
+Currently, there are two actively developed frameworks based on rdkafka-ruby, that provide higher-level API that can be used to work with Kafka messages and one library for publishing messages.
 
 ### Message processing frameworks
 
@@ -104,7 +104,7 @@ Currently, there are two actively developed frameworks based on rdkafka-ruby, th
 
 ## Development
 
-A Docker Compose file is included to run Kafka and Zookeeper. To run
+A Docker Compose file is included to run Kafka. To run
 that:
 
 ```
@@ -122,7 +122,7 @@ DEBUG_PRODUCER=true bundle exec rspec
 DEBUG_CONSUMER=true bundle exec rspec
 ```
 
-After running the tests you can bring the cluster down to start with a
+After running the tests, you can bring the cluster down to start with a
 clean slate:
 
 ```
@@ -131,7 +131,7 @@ docker-compose down
 
 ## Example
 
-To see everything working run these in separate tabs:
+To see everything working, run these in separate tabs:
 
 ```
 bundle exec rake consume_messages

data/ext/README.md

@@ -5,7 +5,7 @@ this gem is installed.
 
 To update the `librdkafka` version follow the following steps:
 
-* Go to https://github.com/edenhill/librdkafka/releases to get the new
+* Go to https://github.com/confluentinc/librdkafka/releases to get the new
   version number and asset checksum for `tar.gz`.
 * Change the version in `lib/rdkafka/version.rb`
 * Change the `sha256` in `lib/rdkafka/version.rb`

data/ext/Rakefile

@@ -17,7 +17,7 @@ task :default => :clean do
   end
 
   recipe.files << {
-    :url => "https://codeload.github.com/edenhill/librdkafka/tar.gz/v#{Rdkafka::LIBRDKAFKA_VERSION}",
+    :url => "https://codeload.github.com/confluentinc/librdkafka/tar.gz/v#{Rdkafka::LIBRDKAFKA_VERSION}",
     :sha256 => Rdkafka::LIBRDKAFKA_SOURCE_SHA256
   }
   recipe.configure_options = ["--host=#{recipe.host}"]

data/lib/rdkafka/abstract_handle.rb

@@ -1,28 +1,37 @@
 # frozen_string_literal: true
 
-require "ffi"
-
 module Rdkafka
+  # This class serves as an abstract base class to represent handles within the Rdkafka module.
+  # As a subclass of `FFI::Struct`, this class provides a blueprint for other specific handle
+  # classes to inherit from, ensuring they adhere to a particular structure and behavior.
+  #
+  # Subclasses must define their own layout, and the layout must start with:
+  #
+  # layout :pending, :bool,
+  #        :response, :int
   class AbstractHandle < FFI::Struct
-    # Subclasses must define their own layout, and the layout must start with:
-    #
-    # layout :pending, :bool,
-    #        :response, :int
+    include Helpers::Time
 
+    # Registry for registering all the handles.
     REGISTRY = {}
 
-    CURRENT_TIME = -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }.freeze
-
-    private_constant :CURRENT_TIME
+    class << self
+      # Adds handle to the register
+      #
+      # @param handle [AbstractHandle] any handle we want to register
+      def register(handle)
+        address = handle.to_ptr.address
+        REGISTRY[address] = handle
+      end
 
-    def self.register(handle)
-      address = handle.to_ptr.address
-      REGISTRY[address] = handle
+      # Removes handle from the register based on the handle address
+      #
+      # @param address [Integer] address of the registered handle we want to remove
+      def remove(address)
+        REGISTRY.delete(address)
+      end
     end
 
-    def self.remove(address)
-      REGISTRY.delete(address)
-    end
 
     # Whether the handle is still pending.
     #
@@ -32,27 +41,31 @@ module Rdkafka
     end
 
     # Wait for the operation to complete or raise an error if this takes longer than the timeout.
-    # If there is a timeout this does not mean the operation failed, rdkafka might still be working on the operation.
-    # In this case it is possible to call wait again.
+    # If there is a timeout this does not mean the operation failed, rdkafka might still be working
+    # on the operation. In this case it is possible to call wait again.
     #
-    # @param max_wait_timeout [Numeric, nil] Amount of time to wait before timing out. If this is nil it does not time out.
-    # @param wait_timeout [Numeric] Amount of time we should wait before we recheck if the operation has completed
+    # @param max_wait_timeout [Numeric, nil] Amount of time to wait before timing out.
+    #   If this is nil it does not time out.
+    # @param wait_timeout [Numeric] Amount of time we should wait before we recheck if the
+    #   operation has completed
     # @param raise_response_error [Boolean] should we raise error when waiting finishes
     #
+    # @return [Object] Operation-specific result
+    #
     # @raise [RdkafkaError] When the operation failed
     # @raise [WaitTimeoutError] When the timeout has been reached and the handle is still pending
-    #
-    # @return [Object] Operation-specific result
     def wait(max_wait_timeout: 60, wait_timeout: 0.1, raise_response_error: true)
       timeout = if max_wait_timeout
-                  CURRENT_TIME.call + max_wait_timeout
+                  monotonic_now + max_wait_timeout
                 else
                   nil
                 end
       loop do
         if pending?
-          if timeout && timeout <= CURRENT_TIME.call
-            raise WaitTimeoutError.new("Waiting for #{operation_name} timed out after #{max_wait_timeout} seconds")
+          if timeout && timeout <= monotonic_now
+            raise WaitTimeoutError.new(
+              "Waiting for #{operation_name} timed out after #{max_wait_timeout} seconds"
+            )
           end
           sleep wait_timeout
         elsif self[:response] != 0 && raise_response_error

data/lib/rdkafka/admin.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "objspace"
-
 module Rdkafka
   class Admin
     # @private
@@ -30,11 +28,12 @@ module Rdkafka
 
     # Create a topic with the given partition count and replication factor
     #
+    # @return [CreateTopicHandle] Create topic handle that can be used to wait for the result of
+    #   creating the topic
+    #
     # @raise [ConfigError] When the partition count or replication factor are out of valid range
     # @raise [RdkafkaError] When the topic name is invalid or the topic already exists
     # @raise [RdkafkaError] When the topic configuration is invalid
-    #
-    # @return [CreateTopicHandle] Create topic handle that can be used to wait for the result of creating the topic
     def create_topic(topic_name, partition_count, replication_factor, topic_config={})
       closed_admin_check(__method__)
 
@@ -107,11 +106,11 @@ module Rdkafka
       create_topic_handle
     end
 
-    # Delete the named topic
+    # Deletes the named topic
     #
+    # @return [DeleteTopicHandle] Delete topic handle that can be used to wait for the result of
+    #   deleting the topic
     # @raise [RdkafkaError] When the topic name is invalid or the topic does not exist
-    #
-    # @return [DeleteTopicHandle] Delete topic handle that can be used to wait for the result of deleting the topic
     def delete_topic(topic_name)
       closed_admin_check(__method__)
 

data/lib/rdkafka/bindings.rb

@@ -1,9 +1,5 @@
 # frozen_string_literal: true
 
-require "ffi"
-require "json"
-require "logger"
-
 module Rdkafka
   # @private
   #

data/lib/rdkafka/config.rb

@@ -1,11 +1,9 @@
 # frozen_string_literal: true
 
-require "logger"
-
 module Rdkafka
   # Configuration for a Kafka consumer or producer. You can create an instance and use
   # the consumer and producer methods to create a client. Documentation of the available
-  # configuration options is available on https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md.
+  # configuration options is available on https://github.com/confluentinc/librdkafka/blob/master/CONFIGURATION.md.
   class Config
     # @private
     @@logger = Logger.new(STDOUT)
@@ -53,13 +51,13 @@ module Rdkafka
 
     # Set a callback that will be called every time the underlying client emits statistics.
     # You can configure if and how often this happens using `statistics.interval.ms`.
-    # The callback is called with a hash that's documented here: https://github.com/edenhill/librdkafka/blob/master/STATISTICS.md
+    # The callback is called with a hash that's documented here: https://github.com/confluentinc/librdkafka/blob/master/STATISTICS.md
     #
     # @param callback [Proc, #call] The callback
     #
     # @return [nil]
     def self.statistics_callback=(callback)
-      raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call)
+      raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call) || callback == nil
       @@statistics_callback = callback
     end
 
@@ -114,6 +112,7 @@ module Rdkafka
     def initialize(config_hash = {})
       @config_hash = DEFAULT_CONFIG.merge(config_hash)
       @consumer_rebalance_listener = nil
+      @consumer_poll_set = true
     end
 
     # Set a config option.
@@ -142,12 +141,28 @@ module Rdkafka
       @consumer_rebalance_listener = listener
     end
 
-    # Create a consumer with this configuration.
+    # Should we use a single queue for the underlying consumer and events.
     #
-    # @raise [ConfigError] When the configuration contains invalid options
-    # @raise [ClientCreationError] When the native client cannot be created
+    # This is an advanced API that allows for more granular control of the polling process.
+    # When this value is set to `false` (`true` by defualt), there will be two queues that need to
+    # be polled:
+    #   - main librdkafka queue for events
+    #   - consumer queue with messages and rebalances
+    #
+    # It is recommended to use the defaults and only set it to `false` in advance multi-threaded
+    # and complex cases where granular events handling control is needed.
+    #
+    # @param poll_set [Boolean]
+    def consumer_poll_set=(poll_set)
+      @consumer_poll_set = poll_set
+    end
+
+    # Creates a consumer with this configuration.
     #
     # @return [Consumer] The created consumer
+    #
+    # @raise [ConfigError] When the configuration contains invalid options
+    # @raise [ClientCreationError] When the native client cannot be created
     def consumer
       opaque = Opaque.new
       config = native_config(opaque)
@@ -160,8 +175,8 @@ module Rdkafka
       # Create native client
       kafka = native_kafka(config, :rd_kafka_consumer)
 
-      # Redirect the main queue to the consumer
-      Rdkafka::Bindings.rd_kafka_poll_set_consumer(kafka)
+      # Redirect the main queue to the consumer queue
+      Rdkafka::Bindings.rd_kafka_poll_set_consumer(kafka) if @consumer_poll_set
 
       # Return consumer with Kafka client
       Rdkafka::Consumer.new(
@@ -175,10 +190,10 @@ module Rdkafka
 
     # Create a producer with this configuration.
     #
+    # @return [Producer] The created producer
+    #
     # @raise [ConfigError] When the configuration contains invalid options
     # @raise [ClientCreationError] When the native client cannot be created
-    #
-    # @return [Producer] The created producer
     def producer
       # Create opaque
       opaque = Opaque.new
@@ -200,12 +215,12 @@ module Rdkafka
       end
     end
 
-    # Create an admin instance with this configuration.
+    # Creates an admin instance with this configuration.
+    #
+    # @return [Admin] The created admin instance
     #
     # @raise [ConfigError] When the configuration contains invalid options
     # @raise [ClientCreationError] When the native client cannot be created
-    #
-    # @return [Admin] The created admin instance
     def admin
       opaque = Opaque.new
       config = native_config(opaque)

data/lib/rdkafka/consumer/headers.rb

@@ -18,13 +18,11 @@ module Rdkafka
 
       # Reads a librdkafka native message's headers and returns them as a Ruby Hash
       #
-      # @param [librdkakfa message] native_message
+      # @private
       #
+      # @param [librdkakfa message] native_message
       # @return [Hash<String, String>] headers Hash for the native_message
-      #
       # @raise [Rdkafka::RdkafkaError] when fail to read headers
-      #
-      # @private
       def self.from_native(native_message)
         headers_ptrptr = FFI::MemoryPointer.new(:pointer)
         err = Rdkafka::Bindings.rd_kafka_message_headers(native_message, headers_ptrptr)

data/lib/rdkafka/consumer.rb

@@ -12,6 +12,7 @@ module Rdkafka
   # `each_slice` to consume batches of messages.
   class Consumer
     include Enumerable
+    include Helpers::Time
 
     # @private
     def initialize(native_kafka)
@@ -54,13 +55,11 @@ module Rdkafka
       @native_kafka.closed?
     end
 
-    # Subscribe to one or more topics letting Kafka handle partition assignments.
+    # Subscribes to one or more topics letting Kafka handle partition assignments.
     #
     # @param topics [Array<String>] One or more topic names
-    #
-    # @raise [RdkafkaError] When subscribing fails
-    #
     # @return [nil]
+    # @raise [RdkafkaError] When subscribing fails
     def subscribe(*topics)
       closed_consumer_check(__method__)
 
@@ -83,9 +82,8 @@ module Rdkafka
 
     # Unsubscribe from all subscribed topics.
     #
-    # @raise [RdkafkaError] When unsubscribing fails
-    #
     # @return [nil]
+    # @raise [RdkafkaError] When unsubscribing fails
     def unsubscribe
       closed_consumer_check(__method__)
 
@@ -101,10 +99,8 @@ module Rdkafka
     # Pause producing or consumption for the provided list of partitions
     #
     # @param list [TopicPartitionList] The topic with partitions to pause
-    #
-    # @raise [RdkafkaTopicPartitionListError] When pausing subscription fails.
-    #
     # @return [nil]
+    # @raise [RdkafkaTopicPartitionListError] When pausing subscription fails.
     def pause(list)
       closed_consumer_check(__method__)
 
@@ -128,13 +124,11 @@ module Rdkafka
       end
     end
 
-    # Resume producing consumption for the provided list of partitions
+    # Resumes producing consumption for the provided list of partitions
     #
     # @param list [TopicPartitionList] The topic with partitions to pause
-    #
-    # @raise [RdkafkaError] When resume subscription fails.
-    #
     # @return [nil]
+    # @raise [RdkafkaError] When resume subscription fails.
     def resume(list)
       closed_consumer_check(__method__)
 
@@ -157,11 +151,10 @@ module Rdkafka
       end
     end
 
-    # Return the current subscription to topics and partitions
-    #
-    # @raise [RdkafkaError] When getting the subscription fails.
+    # Returns the current subscription to topics and partitions
     #
     # @return [TopicPartitionList]
+    # @raise [RdkafkaError] When getting the subscription fails.
     def subscription
       closed_consumer_check(__method__)
 
@@ -184,7 +177,6 @@ module Rdkafka
     # Atomic assignment of partitions to consume
     #
     # @param list [TopicPartitionList] The topic with partitions to assign
-    #
     # @raise [RdkafkaError] When assigning fails
     def assign(list)
       closed_consumer_check(__method__)
@@ -208,9 +200,8 @@ module Rdkafka
 
     # Returns the current partition assignment.
     #
-    # @raise [RdkafkaError] When getting the assignment fails.
-    #
     # @return [TopicPartitionList]
+    # @raise [RdkafkaError] When getting the assignment fails.
     def assignment
       closed_consumer_check(__method__)
 
@@ -244,14 +235,14 @@ module Rdkafka
     end
 
     # Return the current committed offset per partition for this consumer group.
-    # The offset field of each requested partition will either be set to stored offset or to -1001 in case there was no stored offset for that partition.
+    # The offset field of each requested partition will either be set to stored offset or to -1001
+    # in case there was no stored offset for that partition.
     #
-    # @param list [TopicPartitionList, nil] The topic with partitions to get the offsets for or nil to use the current subscription.
+    # @param list [TopicPartitionList, nil] The topic with partitions to get the offsets for or nil
+    #   to use the current subscription.
     # @param timeout_ms [Integer] The timeout for fetching this information.
-    #
-    # @raise [RdkafkaError] When getting the committed positions fails.
-    #
     # @return [TopicPartitionList]
+    # @raise [RdkafkaError] When getting the committed positions fails.
     def committed(list=nil, timeout_ms=1200)
       closed_consumer_check(__method__)
 
@@ -307,10 +298,8 @@ module Rdkafka
     # @param topic [String] The topic to query
     # @param partition [Integer] The partition to query
     # @param timeout_ms [Integer] The timeout for querying the broker
-    #
-    # @raise [RdkafkaError] When querying the broker fails.
-    #
     # @return [Integer] The low and high watermark
+    # @raise [RdkafkaError] When querying the broker fails.
     def query_watermark_offsets(topic, partition, timeout_ms=200)
       closed_consumer_check(__method__)
 
@@ -343,10 +332,9 @@ module Rdkafka
     #
     # @param topic_partition_list [TopicPartitionList] The list to calculate lag for.
     # @param watermark_timeout_ms [Integer] The timeout for each query watermark call.
-    #
+    # @return [Hash<String, Hash<Integer, Integer>>] A hash containing all topics with the lag
+    #   per partition
     # @raise [RdkafkaError] When querying the broker fails.
-    #
-    # @return [Hash<String, Hash<Integer, Integer>>] A hash containing all topics with the lag per partition
     def lag(topic_partition_list, watermark_timeout_ms=100)
       out = {}
 
@@ -395,10 +383,8 @@ module Rdkafka
     # When using this `enable.auto.offset.store` should be set to `false` in the config.
     #
     # @param message [Rdkafka::Consumer::Message] The message which offset will be stored
-    #
-    # @raise [RdkafkaError] When storing the offset fails
-    #
     # @return [nil]
+    # @raise [RdkafkaError] When storing the offset fails
     def store_offset(message)
       closed_consumer_check(__method__)
 
@@ -430,10 +416,8 @@ module Rdkafka
     # message at the given offset.
     #
     # @param message [Rdkafka::Consumer::Message] The message to which to seek
-    #
-    # @raise [RdkafkaError] When seeking fails
-    #
     # @return [nil]
+    # @raise [RdkafkaError] When seeking fails
     def seek(message)
       closed_consumer_check(__method__)
 
@@ -503,10 +487,8 @@ module Rdkafka
     #
     # @param list [TopicPartitionList,nil] The topic with partitions to commit
     # @param async [Boolean] Whether to commit async or wait for the commit to finish
-    #
-    # @raise [RdkafkaError] When committing fails
-    #
     # @return [nil]
+    # @raise [RdkafkaError] When committing fails
     def commit(list=nil, async=false)
       closed_consumer_check(__method__)
 
@@ -532,10 +514,8 @@ module Rdkafka
     # Poll for the next message on one of the subscribed topics
     #
     # @param timeout_ms [Integer] Timeout of this poll
-    #
-    # @raise [RdkafkaError] When polling fails
-    #
     # @return [Message, nil] A message or nil if there was no new message within the timeout
+    # @raise [RdkafkaError] When polling fails
     def poll(timeout_ms)
       closed_consumer_check(__method__)
 
@@ -561,17 +541,40 @@ module Rdkafka
       end
     end
 
+    # Polls the main rdkafka queue (not the consumer one). Do **NOT** use it if `consumer_poll_set`
+    #   was set to `true`.
+    #
+    # Events will cause application-provided callbacks to be called.
+    #
+    # Events (in the context of the consumer):
+    #   - error callbacks
+    #   - stats callbacks
+    #   - any other callbacks supported by librdkafka that are not part of the consumer_poll, that
+    #     would have a callback configured and activated.
+    #
+    # This method needs to be called at regular intervals to serve any queued callbacks waiting to
+    # be called. When in use, does **NOT** replace `#poll` but needs to run complementary with it.
+    #
+    # @param timeout_ms [Integer] poll timeout. If set to 0 will run async, when set to -1 will
+    #   block until any events available.
+    #
+    # @note This method technically should be called `#poll` and the current `#poll` should be
+    #   called `#consumer_poll` though we keep the current naming convention to make it backward
+    #   compatible.
+    def events_poll(timeout_ms = 0)
+      @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_poll(inner, timeout_ms)
+      end
+    end
+
     # Poll for new messages and yield for each received one. Iteration
     # will end when the consumer is closed.
     #
-    # If `enable.partition.eof` is turned on in the config this will raise an
-    # error when an eof is reached, so you probably want to disable that when
-    # using this method of iteration.
+    # If `enable.partition.eof` is turned on in the config this will raise an error when an eof is
+    # reached, so you probably want to disable that when using this method of iteration.
     #
     # @raise [RdkafkaError] When polling fails
-    #
     # @yieldparam message [Message] Received message
-    #
     # @return [nil]
     def each
       loop do
@@ -624,9 +627,7 @@ module Rdkafka
     # that you may or may not see again.
     #
     # @param max_items [Integer] Maximum size of the yielded array of messages
-    #
     # @param bytes_threshold [Integer] Threshold number of total message bytes in the yielded array of messages
-    #
     # @param timeout_ms [Integer] max time to wait for up to max_items
     #
     # @raise [RdkafkaError] When polling fails
@@ -673,10 +674,6 @@ module Rdkafka
     end
 
     private
-    def monotonic_now
-      # needed because Time.now can go backwards
-      Process.clock_gettime(Process::CLOCK_MONOTONIC)
-    end
 
     def closed_consumer_check(method)
       raise Rdkafka::ClosedConsumerError.new(method) if closed?

data/lib/rdkafka/helpers/time.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  # Namespace for some small utilities used in multiple components
+  module Helpers
+    # Time related methods used across Karafka
+    module Time
+      # @return [Float] current monotonic time in seconds with microsecond precision
+      def monotonic_now
+        ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      end
+    end
+  end
+end

data/lib/rdkafka/producer.rb

@@ -1,10 +1,10 @@
 # frozen_string_literal: true
 
-require "objspace"
-
 module Rdkafka
   # A producer for Kafka messages. To create a producer set up a {Config} and call {Config#producer producer} on that.
   class Producer
+    include Helpers::Time
+
     # Cache partitions count for 30 seconds
     PARTITIONS_COUNT_TTL = 30
 
@@ -167,18 +167,16 @@ module Rdkafka
     end
 
     # Partition count for a given topic.
-    # NOTE: If 'allow.auto.create.topics' is set to true in the broker, the topic will be auto-created after returning nil.
     #
     # @param topic [String] The topic name.
+    # @return [Integer] partition count for a given topic
     #
-    # @return partition count [Integer,nil]
-    #
-    # We cache the partition count for a given topic for given time
-    # This prevents us in case someone uses `partition_key` from querying for the count with
-    # each message. Instead we query once every 30 seconds at most
+    # @note If 'allow.auto.create.topics' is set to true in the broker, the topic will be
+    #   auto-created after returning nil.
     #
-    # @param topic [String] topic name
-    # @return [Integer] partition count for a given topic
+    # @note We cache the partition count for a given topic for given time.
+    #   This prevents us in case someone uses `partition_key` from querying for the count with
+    #   each message. Instead we query once every 30 seconds at most
     def partition_count(topic)
       closed_producer_check(__method__)
 
@@ -308,11 +306,6 @@ module Rdkafka
 
     private
 
-    def monotonic_now
-      # needed because Time.now can go backwards
-      Process.clock_gettime(Process::CLOCK_MONOTONIC)
-    end
-
     def closed_producer_check(method)
       raise Rdkafka::ClosedProducerError.new(method) if closed?
     end

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.13.8"
+  VERSION = "0.13.9"
   LIBRDKAFKA_VERSION = "2.2.0"
   LIBRDKAFKA_SOURCE_SHA256 = "af9a820cbecbc64115629471df7c7cecd40403b6c34bfdbb9223152677a47226"
 end

data/lib/rdkafka.rb

@@ -1,7 +1,12 @@
 # frozen_string_literal: true
 
-require "rdkafka/version"
+require "logger"
+require "objspace"
+require "ffi"
+require "json"
 
+require "rdkafka/version"
+require "rdkafka/helpers/time"
 require "rdkafka/abstract_handle"
 require "rdkafka/admin"
 require "rdkafka/admin/create_topic_handle"
@@ -24,3 +29,7 @@ require "rdkafka/native_kafka"
 require "rdkafka/producer"
 require "rdkafka/producer/delivery_handle"
 require "rdkafka/producer/delivery_report"
+
+# Main Rdkafka namespace of this gem
+module Rdkafka
+end

data/spec/rdkafka/abstract_handle_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::AbstractHandle do
   let(:response) { 0 }
   let(:result) { -1 }

data/spec/rdkafka/admin/create_topic_handle_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::Admin::CreateTopicHandle do
   let(:response) { 0 }
 

data/spec/rdkafka/admin/create_topic_report_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::Admin::CreateTopicReport do
   subject { Rdkafka::Admin::CreateTopicReport.new(
       FFI::MemoryPointer.from_string("error string"),

data/spec/rdkafka/admin/delete_topic_handle_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::Admin::DeleteTopicHandle do
   let(:response) { 0 }
 

data/spec/rdkafka/admin/delete_topic_report_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::Admin::DeleteTopicReport do
   subject { Rdkafka::Admin::DeleteTopicReport.new(
       FFI::MemoryPointer.from_string("error string"),

data/spec/rdkafka/admin_spec.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
 require "ostruct"
 
 describe Rdkafka::Admin do

data/spec/rdkafka/bindings_spec.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
 require 'zlib'
 
 describe Rdkafka::Bindings do

data/spec/rdkafka/callbacks_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::Callbacks do
 
   # The code in the call back functions is 100% covered by other specs.  Due to

data/spec/rdkafka/config_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::Config do
   context "logger" do
     it "should have a default logger" do
@@ -115,6 +113,14 @@ describe Rdkafka::Config do
       consumer.close
     end
 
+    it "should create a consumer with consumer_poll_set set to false" do
+      config = rdkafka_consumer_config
+      config.consumer_poll_set = false
+      consumer = config.consumer
+      expect(consumer).to be_a Rdkafka::Consumer
+      consumer.close
+    end
+
     it "should raise an error when creating a consumer with invalid config" do
       config = Rdkafka::Config.new('invalid.key' => 'value')
       expect {

data/spec/rdkafka/consumer/headers_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::Consumer::Headers do
   let(:headers) do
     { # Note String keys!

data/spec/rdkafka/consumer/message_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::Consumer::Message do
   let(:native_client) { new_native_client }
   let(:native_topic) { new_native_topic(native_client: native_client) }

data/spec/rdkafka/consumer/partition_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::Consumer::Partition do
   let(:offset) { 100 }
   let(:err) { 0 }

data/spec/rdkafka/consumer/topic_partition_list_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::Consumer::TopicPartitionList do
   it "should create a new list and add unassigned topics" do
     list = Rdkafka::Consumer::TopicPartitionList.new

data/spec/rdkafka/consumer_spec.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
 require "ostruct"
 require 'securerandom'
 
@@ -55,6 +54,30 @@ describe Rdkafka::Consumer do
         consumer.subscription
       }.to raise_error(Rdkafka::RdkafkaError)
     end
+
+    context "when using consumer without the poll set" do
+      let(:consumer) do
+        config = rdkafka_consumer_config
+        config.consumer_poll_set = false
+        config.consumer
+      end
+
+      it "should subscribe, unsubscribe and return the subscription" do
+        expect(consumer.subscription).to be_empty
+
+        consumer.subscribe("consume_test_topic")
+
+        expect(consumer.subscription).not_to be_empty
+        expected_subscription = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+          list.add_topic("consume_test_topic")
+        end
+        expect(consumer.subscription).to eq expected_subscription
+
+        consumer.unsubscribe
+
+        expect(consumer.subscription).to be_empty
+      end
+    end
   end
 
   describe "#pause and #resume" do
@@ -1077,6 +1100,29 @@ describe Rdkafka::Consumer do
     end
   end
 
+  # Only relevant in case of a consumer with separate queues
+  describe '#events_poll' do
+    let(:stats) { [] }
+
+    before { Rdkafka::Config.statistics_callback = ->(published) { stats << published } }
+
+    after { Rdkafka::Config.statistics_callback = nil }
+
+    let(:consumer) do
+      config = rdkafka_consumer_config('statistics.interval.ms': 100)
+      config.consumer_poll_set = false
+      config.consumer
+    end
+
+    it "expect to run events_poll, operate and propagate stats on events_poll and not poll" do
+      consumer.subscribe("consume_test_topic")
+      consumer.poll(1_000)
+      expect(stats).to be_empty
+      consumer.events_poll(-1)
+      expect(stats).not_to be_empty
+    end
+  end
+
   describe "a rebalance listener" do
     let(:consumer) do
       config = rdkafka_consumer_config

data/spec/rdkafka/error_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::RdkafkaError do
   it "should raise a type error for a nil response" do
     expect {

data/spec/rdkafka/metadata_spec.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
 require "securerandom"
 
 describe Rdkafka::Metadata do

data/spec/rdkafka/native_kafka_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::NativeKafka do
   let(:config) { rdkafka_producer_config }
   let(:native) { config.send(:native_kafka, config.send(:native_config), :rd_kafka_producer) }

data/spec/rdkafka/producer/delivery_handle_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::Producer::DeliveryHandle do
   let(:response) { 0 }
 

data/spec/rdkafka/producer/delivery_report_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::Producer::DeliveryReport do
   subject { Rdkafka::Producer::DeliveryReport.new(2, 100, "topic", -1) }
 

data/spec/rdkafka/producer_spec.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
 require "zlib"
 
 describe Rdkafka::Producer do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.13.8
+  version: 0.13.9
 platform: ruby
 authors:
 - Thijs Cadier
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2023-10-31 00:00:00.000000000 Z
+date: 2023-11-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -174,11 +174,13 @@ files:
 - ".github/workflows/ci.yml"
 - ".gitignore"
 - ".rspec"
+- ".ruby-gemset"
+- ".ruby-version"
 - ".yardopts"
 - CHANGELOG.md
 - Gemfile
 - Guardfile
-- LICENSE
+- MIT-LICENSE
 - README.md
 - Rakefile
 - certs/cert_chain.pem
@@ -204,6 +206,7 @@ files:
 - lib/rdkafka/consumer/partition.rb
 - lib/rdkafka/consumer/topic_partition_list.rb
 - lib/rdkafka/error.rb
+- lib/rdkafka/helpers/time.rb
 - lib/rdkafka/metadata.rb
 - lib/rdkafka/native_kafka.rb
 - lib/rdkafka/producer.rb
@@ -258,7 +261,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.4
+rubygems_version: 3.4.19
 signing_key: 
 specification_version: 4
 summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.