rdkafka 0.16.1 → 0.18.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2c95cc0f676ffb0921be329ffa7af3954d9d5111a2312906e06266bb80f19323
-  data.tar.gz: 712b70533b65a862c3783cf6cc9179d5d700ea1a657cf5c44d143d52027fc176
+  metadata.gz: 9c9dad3e9f793616387a5665ecba20271fe431755919995bef1921d595af20c1
+  data.tar.gz: ffa1c227773b625195f48c0fd3e0cdbf386f0bdad9438a41e472fe16acf4c856
 SHA512:
-  metadata.gz: f78b1b2b11a1d51fb93f9f533e92937aa82731cdf76b24a8940a2dc667e7e6216d639b6983a8de54ba00f3271c1076066caf393cf68028d82cb6297772f4de86
-  data.tar.gz: 2bc40080e7aed367bb7c6305ac10054b8e6569e645ddbec29390149124e2f11006661f0b470f8cb864edac18e26281e90649c62cc5b659626960c4a88dfc69f0
+  metadata.gz: bbc22c2a6d032bbd0485670062f42d3aa7bd4ec60119e76e669ec2f133f68b21e6529d7674e191b05a3b298ac0935b7e34d1166292486475ab5d70d5665fac9d
+  data.tar.gz: 77f248a1373d8220b142b4d632063f692918263a6535fa0f28238e3dcb1f5e63ed8b8b6a56bbffd1c9c5c99ca79ae6d2eaf59dd223ac58788f6144cbc36e8c6d

data/.ruby-version

@@ -1 +1 @@
-3.3.3
+3.3.4

data/CHANGELOG.md

@@ -1,5 +1,19 @@
 # Rdkafka Changelog
 
+## 0.18.0 (2024-09-02)
+- [Enhancement] Update `librdkafka` to `2.5.0`
+- [Enhancement] Do not release GVL on `rd_kafka_name` (ferrous26)
+- [Patch] Patch cooperative-sticky assignments in librdkafka.
+- [Fix] Mitigate a case where FFI would not restart the background events callback dispatcher in forks
+- [Fix] Fix unused variable reference in producer (lucasmvnascimento)
+
+## 0.17.0 (2024-08-03)
+- [Feature] Add `#seek_by` to be able to seek for a message by topic, partition and offset (zinahia)
+- [Enhancement] Update `librdkafka` to `2.4.0`
+- [Enhancement] Support ability to release patches to librdkafka.
+- [Change] Remove old producer timeout API warnings.
+- [Fix] Switch to local release of librdkafka to mitigate its unavailability.
+
 ## 0.16.1 (2024-07-10)
 - [Fix] Switch to local release of librdkafka to mitigate its unavailability.
 
@@ -21,6 +35,9 @@
 - [Fix] Background logger stops working after forking causing memory leaks (mensfeld)
 - [Fix] Fix bogus case/when syntax. Levels 1, 2, and 6 previously defaulted to UNKNOWN (jjowdy)
 
+## 0.15.2 (2024-07-10)
+- [Fix] Switch to local release of librdkafka to mitigate its unavailability.
+
 ## 0.15.1 (2024-01-30)
 - [Enhancement] Provide support for Nix OS (alexandriainfantino)
 - [Enhancement] Replace `rd_kafka_offset_store` with `rd_kafka_offsets_store` (mensfeld)
@@ -46,6 +63,9 @@
 - [Enhancement] Increase the `#lag` and `#query_watermark_offsets` default timeouts from 100ms to 1000ms. This will compensate for network glitches and remote clusters operations (mensfeld)
 - [Change] Use `SecureRandom.uuid` instead of `random` for test consumer groups (mensfeld)
 
+## 0.14.1 (2024-07-10)
+- [Fix] Switch to local release of librdkafka to mitigate its unavailability.
+
 ## 0.14.0 (2023-11-21)
 - [Enhancement] Add `raise_response_error` flag to the `Rdkafka::AbstractHandle`.
 - [Enhancement] Allow for setting `statistics_callback` as nil to reset predefined settings configured by a different gem (mensfeld)
@@ -63,6 +83,9 @@
 - [Change] Update Kafka Docker with Confluent KRaft (mensfeld)
 - [Change] Update librdkafka repo reference from edenhill to confluentinc (mensfeld)
 
+## 0.13.1 (2024-07-10)
+- [Fix] Switch to local release of librdkafka to mitigate its unavailability.
+
 ## 0.13.0 (2023-07-24)
 - Support cooperative sticky partition assignment in the rebalance callback (methodmissing)
 - Support both string and symbol header keys (ColinDKelley)
@@ -79,6 +102,9 @@
 - Make metadata request timeout configurable (mensfeld)
 - call_on_partitions_assigned and call_on_partitions_revoked only get a tpl passed in (thijsc)
 
+## 0.12.1 (2024-07-11)
+- [Fix] Switch to local release of librdkafka to mitigate its unavailability.
+
 ## 0.12.0 (2022-06-17)
 - Bumps librdkafka to 1.9.0
 - Fix crash on empty partition key (mensfeld)

data/README.md

@@ -161,12 +161,14 @@ bundle exec rake produce_messages
 
 ## Versions
 
-| rdkafka-ruby | librdkafka |
-|-|-|
-| 0.16.0 (2024-06-13) | 2.3.0 (2023-10-25) |
-| 0.15.0 (2023-12-03) | 2.3.0 (2023-10-25) |
-| 0.14.0 (2023-11-21) | 2.2.0 (2023-07-12) |
-| 0.13.0 (2023-07-24) | 2.0.2 (2023-01-20) |
-| 0.12.0 (2022-06-17) | 1.9.0 (2022-06-16) |
-| 0.11.0 (2021-11-17) | 1.8.2 (2021-10-18) |
-| 0.10.0 (2021-09-07) | 1.5.0 (2020-07-20) |
+| rdkafka-ruby | librdkafka | patches |
+|-|-|-|
+| 0.18.0 (2024-09-02) | 2.5.0 (2024-06-10) | yes |
+| 0.17.0 (2024-08-03) | 2.4.0 (2024-05-07) | no  |
+| 0.16.0 (2024-06-13) | 2.3.0 (2023-10-25) | no  |
+| 0.15.0 (2023-12-03) | 2.3.0 (2023-10-25) | no  |
+| 0.14.0 (2023-11-21) | 2.2.0 (2023-07-12) | no  |
+| 0.13.0 (2023-07-24) | 2.0.2 (2023-01-20) | no  |
+| 0.12.0 (2022-06-17) | 1.9.0 (2022-06-16) | no  |
+| 0.11.0 (2021-11-17) | 1.8.2 (2021-10-18) | no  |
+| 0.10.0 (2021-09-07) | 1.5.0 (2020-07-20) | no  |

data/certs/cert.pem

@@ -0,0 +1,26 @@
+-----BEGIN CERTIFICATE-----
+MIIEcDCCAtigAwIBAgIBATANBgkqhkiG9w0BAQsFADA/MRAwDgYDVQQDDAdjb250
+YWN0MRcwFQYKCZImiZPyLGQBGRYHa2FyYWZrYTESMBAGCgmSJomT8ixkARkWAmlv
+MB4XDTI0MDgyMzEwMTkyMFoXDTQ5MDgxNzEwMTkyMFowPzEQMA4GA1UEAwwHY29u
+dGFjdDEXMBUGCgmSJomT8ixkARkWB2thcmFma2ExEjAQBgoJkiaJk/IsZAEZFgJp
+bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAKjLhLjQqUlNayxkXnO+
+PsmCDs/KFIzhrsYMfLZRZNaWmzV3ujljMOdDjd4snM2X06C41iVdQPWjpe3j8vVe
+ZXEWR/twSbOP6Eeg8WVH2wCOo0x5i7yhVn4UBLH4JpfEMCbemVcWQ9ry9OMg4WpH
+Uu4dRwxFV7hzCz3p0QfNLRI4miAxnGWcnlD98IJRjBAksTuR1Llj0vbOrDGsL9ZT
+JeXP2gdRLd8SqzAFJEWrbeTBCBU7gfSh3oMg5SVDLjaqf7Kz5wC/8bDZydzanOxB
+T6CDXPsCnllmvTNx2ei2T5rGYJOzJeNTmJLLK6hJWUlAvaQSvCwZRvFJ0tVGLEoS
+flqSr6uGyyl1eMUsNmsH4BqPEYcAV6P2PKTv2vUR8AP0raDvZ3xL1TKvfRb8xRpo
+vPopCGlY5XBWEc6QERHfVLTIVsjnls2/Ujj4h8/TSfqqYnaHKefIMLbuD/tquMjD
+iWQsW2qStBV0T+U7FijKxVfrfqZP7GxQmDAc9o1iiyAa3QIDAQABo3cwdTAJBgNV
+HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQU3O4dTXmvE7YpAkszGzR9DdL9
+sbEwHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
+bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEAVKTfoLXn7mqdSxIR
+eqxcR6Huudg1jes81s1+X0uiRTR3hxxKZ3Y82cPsee9zYWyBrN8TA4KA0WILTru7
+Ygxvzha0SRPsSiaKLmgOJ+61ebI4+bOORzIJLpD6GxCxu1r7MI4+0r1u1xe0EWi8
+agkVo1k4Vi8cKMLm6Gl9b3wG9zQBw6fcgKwmpjKiNnOLP+OytzUANrIUJjoq6oal
+TC+f/Uc0TLaRqUaW/bejxzDWWHoM3SU6aoLPuerglzp9zZVzihXwx3jPLUVKDFpF
+Rl2lcBDxlpYGueGo0/oNzGJAAy6js8jhtHC9+19PD53vk7wHtFTZ/0ugDQYnwQ+x
+oml2fAAuVWpTBCgOVFe6XCQpMKopzoxQ1PjKztW2KYxgJdIBX87SnL3aWuBQmhRd
+i9zWxov0mr44TWegTVeypcWGd/0nxu1+QHVNHJrpqlPBRvwQsUm7fwmRInGpcaB8
+ap8wNYvryYzrzvzUxIVFBVM5PacgkFqRmolCa8I7tdKQN+R1
+-----END CERTIFICATE-----

data/dist/patches/rdkafka_sticky_assignor.c.patch

@@ -0,0 +1,26 @@
+# This patch is released under the 2-clause BSD license, same as librdkafka
+# Fixes: https://github.com/confluentinc/librdkafka/issues/4783
+#
+--- librdkafka_2.5.0/src/rdkafka_sticky_assignor.c	2024-07-08 09:47:43.000000000 +0200
++++ librdkafka_2.5.0/src/rdkafka_sticky_assignor.c	2024-07-30 09:44:38.529759640 +0200
+@@ -769,7 +769,7 @@
+         const rd_kafka_topic_partition_list_t *partitions;
+         const char *consumer;
+         const rd_map_elem_t *elem;
+-        int i;
++        int i, j;
+ 
+         /* The assignment is balanced if minimum and maximum numbers of
+          * partitions assigned to consumers differ by at most one. */
+@@ -836,9 +836,9 @@
+ 
+                 /* Otherwise make sure it can't get any more partitions */
+ 
+-                for (i = 0; i < potentialTopicPartitions->cnt; i++) {
++                for (j = 0; j < potentialTopicPartitions->cnt; j++) {
+                         const rd_kafka_topic_partition_t *partition =
+-                            &potentialTopicPartitions->elems[i];
++                            &potentialTopicPartitions->elems[j];
+                         const char *otherConsumer;
+                         int otherConsumerPartitionCount;
+ 

data/docker-compose.yml

@@ -3,7 +3,7 @@ version: '2'
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:7.6.1
+    image: confluentinc/cp-kafka:7.7.0
 
     ports:
       - 9092:9092

data/ext/README.md

@@ -1,7 +1,7 @@
 # Ext
 
-This gem depends on the `librdkafka` C library. It is downloaded when
-this gem is installed.
+This gem depends on the `librdkafka` C library. It is downloaded, stored in
+`dist/` directory, and checked into source control.
 
 To update the `librdkafka` version follow the following steps:
 
@@ -9,8 +9,9 @@ To update the `librdkafka` version follow the following steps:
   version number and asset checksum for `tar.gz`.
 * Change the version in `lib/rdkafka/version.rb`
 * Change the `sha256` in `lib/rdkafka/version.rb`
-* Run `bundle exec rake` in the `ext` directory to download and build
-  the new version
+* Run `bundle exec rake dist:download` in the `ext` directory to download the
+  new release and place it in the `dist/` for you
+* Run `bundle exec rake` in the `ext` directory to build the new version
 * Run `docker-compose pull` in the main gem directory to ensure the docker
   images used by the tests and run `docker-compose up`
 * Finally, run `bundle exec rspec` in the main gem directory to execute

data/ext/Rakefile

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require File.expand_path('../../lib/rdkafka/version', __FILE__)
+require "digest"
 require "fileutils"
 require "open-uri"
 
@@ -30,6 +31,8 @@ task :default => :clean do
     }
     recipe.configure_options = ["--host=#{recipe.host}"]
 
+    recipe.patch_files = Dir[File.join(releases, 'patches', "*.patch")].sort
+
     # Disable using libc regex engine in favor of the embedded one
     # The default regex engine of librdkafka does not always work exactly as most of the users
     # would expect, hence this flag allows for changing it to the other one
@@ -71,6 +74,42 @@ task :clean do
   FileUtils.rm_rf File.join(File.dirname(__FILE__), "tmp")
 end
 
+namespace :dist do
+  task :dir do
+    ENV["RDKAFKA_DIST_PATH"] ||= File.expand_path(File.join(File.dirname(__FILE__), '..', 'dist'))
+  end
+
+  task :file => "dist:dir" do
+    ENV["RDKAFKA_DIST_FILE"] ||= File.join(ENV["RDKAFKA_DIST_PATH"], "librdkafka_#{Rdkafka::LIBRDKAFKA_VERSION}.tar.gz")
+  end
+
+  task :clean => "dist:file" do
+    Dir.glob(File.join("#{ENV['RDKAFKA_DIST_PATH']}", "*")).each do |filename|
+      next if filename.include? ENV["RDKAFKA_DIST_FILE"]
+
+      FileUtils.rm_rf filename
+    end
+  end
+
+  task :download => "dist:file" do
+    version = Rdkafka::LIBRDKAFKA_VERSION
+    librdkafka_download = "https://codeload.github.com/confluentinc/librdkafka/tar.gz/v#{version}"
+
+    URI.open(librdkafka_download) do |file|
+      filename = ENV["RDKAFKA_DIST_FILE"]
+      data     = file.read
+
+      if Digest::SHA256.hexdigest(data) != Rdkafka::LIBRDKAFKA_SOURCE_SHA256
+        raise "SHA256 does not match downloaded file"
+      end
+
+      File.write(filename, data)
+    end
+  end
+
+  task :update => %w[dist:download dist:clean]
+end
+
 namespace :build do
   desc "Build librdkafka at the given git sha or tag"
   task :git, [:ref] do |task, args|
@@ -78,8 +117,9 @@ namespace :build do
     version = "git-#{ref}"
 
     recipe = MiniPortile.new("librdkafka", version)
-    recipe.files << "https://github.com/edenhill/librdkafka/archive/#{ref}.tar.gz"
+    recipe.files << "https://github.com/confluentinc/librdkafka/archive/#{ref}.tar.gz"
     recipe.configure_options = ["--host=#{recipe.host}","--enable-static", "--enable-zstd"]
+    recipe.patch_files = Dir[File.join(releases, 'patches', "*.patch")].sort
     recipe.cook
 
     ext = recipe.host.include?("darwin") ? "dylib" : "so"

data/lib/rdkafka/abstract_handle.rb

@@ -16,9 +16,6 @@ module Rdkafka
     REGISTRY = {}
     # Default wait timeout is 31 years
     MAX_WAIT_TIMEOUT_FOREVER = 10_000_000_000
-    # Deprecation message for wait_timeout argument in wait method
-    WAIT_TIMEOUT_DEPRECATION_MESSAGE = "The 'wait_timeout' argument is deprecated and will be removed in future versions without replacement. " \
-      "We don't rely on it's value anymore. Please refactor your code to remove references to it."
 
     private_constant :MAX_WAIT_TIMEOUT_FOREVER
 
@@ -59,16 +56,13 @@ module Rdkafka
     #
     # @param max_wait_timeout [Numeric, nil] Amount of time to wait before timing out.
     #   If this is nil we will wait forever
-    # @param wait_timeout [nil] deprecated
     # @param raise_response_error [Boolean] should we raise error when waiting finishes
     #
     # @return [Object] Operation-specific result
     #
     # @raise [RdkafkaError] When the operation failed
     # @raise [WaitTimeoutError] When the timeout has been reached and the handle is still pending
-    def wait(max_wait_timeout: 60, wait_timeout: nil, raise_response_error: true)
-      Kernel.warn(WAIT_TIMEOUT_DEPRECATION_MESSAGE) unless wait_timeout.nil?
-
+    def wait(max_wait_timeout: 60, raise_response_error: true)
       timeout = max_wait_timeout ? monotonic_now + max_wait_timeout : MAX_WAIT_TIMEOUT_FOREVER
 
       @mutex.synchronize do

data/lib/rdkafka/bindings.rb

@@ -38,7 +38,7 @@ module Rdkafka
 
     # Metadata
 
-    attach_function :rd_kafka_name, [:pointer], :string, blocking: true
+    attach_function :rd_kafka_name, [:pointer], :string
     attach_function :rd_kafka_memberid, [:pointer], :string, blocking: true
     attach_function :rd_kafka_clusterid, [:pointer], :string, blocking: true
     attach_function :rd_kafka_metadata, [:pointer, :int, :pointer, :pointer, :int], :int, blocking: true

data/lib/rdkafka/callbacks.rb

@@ -2,7 +2,6 @@
 
 module Rdkafka
   module Callbacks
-
     # Extracts attributes of a rd_kafka_topic_result_t
     #
     # @private
@@ -149,13 +148,6 @@ module Rdkafka
       end
     end
 
-    # FFI Function used for Create Topic and Delete Topic callbacks
-    BackgroundEventCallbackFunction = FFI::Function.new(
-        :void, [:pointer, :pointer, :pointer]
-    ) do |client_ptr, event_ptr, opaque_ptr|
-      BackgroundEventCallback.call(client_ptr, event_ptr, opaque_ptr)
-    end
-
     # @private
     class BackgroundEventCallback
       def self.call(_, event_ptr, _)
@@ -348,13 +340,6 @@ module Rdkafka
       end
     end
 
-    # FFI Function used for Message Delivery callbacks
-    DeliveryCallbackFunction = FFI::Function.new(
-        :void, [:pointer, :pointer, :pointer]
-    ) do |client_ptr, message_ptr, opaque_ptr|
-      DeliveryCallback.call(client_ptr, message_ptr, opaque_ptr)
-    end
-
     # @private
     class DeliveryCallback
       def self.call(_, message_ptr, opaque_ptr)
@@ -387,5 +372,44 @@ module Rdkafka
         end
       end
     end
+
+    @@mutex = Mutex.new
+    @@current_pid = nil
+
+    class << self
+      # Defines or recreates after fork callbacks that require FFI thread so the callback thread
+      # is always correctly initialized
+      #
+      # @see https://github.com/ffi/ffi/issues/1114
+      def ensure_ffi_running
+        @@mutex.synchronize do
+          return if @@current_pid == ::Process.pid
+
+          if const_defined?(:BackgroundEventCallbackFunction, false)
+            send(:remove_const, :BackgroundEventCallbackFunction)
+            send(:remove_const, :DeliveryCallbackFunction)
+          end
+
+          # FFI Function used for Create Topic and Delete Topic callbacks
+          background_event_callback_function = FFI::Function.new(
+              :void, [:pointer, :pointer, :pointer]
+          ) do |client_ptr, event_ptr, opaque_ptr|
+            BackgroundEventCallback.call(client_ptr, event_ptr, opaque_ptr)
+          end
+
+          # FFI Function used for Message Delivery callbacks
+          delivery_callback_function = FFI::Function.new(
+              :void, [:pointer, :pointer, :pointer]
+          ) do |client_ptr, message_ptr, opaque_ptr|
+            DeliveryCallback.call(client_ptr, message_ptr, opaque_ptr)
+          end
+
+          const_set(:BackgroundEventCallbackFunction, background_event_callback_function)
+          const_set(:DeliveryCallbackFunction, delivery_callback_function)
+
+          @@current_pid = ::Process.pid
+        end
+      end
+    end
   end
 end

data/lib/rdkafka/config.rb

@@ -146,6 +146,8 @@ module Rdkafka
     #
     # @return [Config]
     def initialize(config_hash = {})
+      Callbacks.ensure_ffi_running
+
       @config_hash = DEFAULT_CONFIG.merge(config_hash)
       @consumer_rebalance_listener = nil
       @consumer_poll_set = true

data/lib/rdkafka/consumer.rb

@@ -423,6 +423,19 @@ module Rdkafka
     # @return [nil]
     # @raise [RdkafkaError] When seeking fails
     def seek(message)
+      seek_by(message.topic, message.partition, message.offset)
+    end
+
+    # Seek to a particular message by providing the topic, partition and offset.
+    # The next poll on the topic/partition will return the
+    # message at the given offset.
+    #
+    # @param topic [String] The topic in which to seek
+    # @param partition [Integer] The partition number to seek
+    # @param offset [Integer] The partition offset to seek
+    # @return [nil]
+    # @raise [RdkafkaError] When seeking fails
+    def seek_by(topic, partition, offset)
       closed_consumer_check(__method__)
 
       # rd_kafka_offset_store is one of the few calls that does not support
@@ -430,14 +443,14 @@ module Rdkafka
       native_topic = @native_kafka.with_inner do |inner|
         Rdkafka::Bindings.rd_kafka_topic_new(
           inner,
-          message.topic,
+          topic,
           nil
         )
       end
       response = Rdkafka::Bindings.rd_kafka_seek(
         native_topic,
-        message.partition,
-        message.offset,
+        partition,
+        offset,
         0 # timeout
       )
       if response != 0

data/lib/rdkafka/producer.rb

@@ -298,7 +298,7 @@ module Rdkafka
         partitioner_name = @topics_configs.dig(topic, topic_config_hash, :partitioner) || @partitioner_name
 
         # If the topic is not present, set to -1
-        partition = Rdkafka::Bindings.partitioner(partition_key, partition_count, @partitioner_name) if partition_count.positive?
+        partition = Rdkafka::Bindings.partitioner(partition_key, partition_count, partitioner_name) if partition_count.positive?
       end
 
       # If partition is nil, use -1 to let librdafka set the partition randomly or

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.16.1"
-  LIBRDKAFKA_VERSION = "2.3.0"
-  LIBRDKAFKA_SOURCE_SHA256 = "2d49c35c77eeb3d42fa61c43757fcbb6a206daa560247154e60642bcdcc14d12"
+  VERSION = "0.18.0"
+  LIBRDKAFKA_VERSION = "2.5.0"
+  LIBRDKAFKA_SOURCE_SHA256 = "3dc62de731fd516dfb1032861d9a580d4d0b5b0856beb0f185d06df8e6c26259"
 end

data/rdkafka.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
   gem.version = Rdkafka::VERSION
   gem.required_ruby_version = '>= 3.0'
   gem.extensions = %w(ext/Rakefile)
-  gem.cert_chain = %w[certs/cert_chain.pem]
+  gem.cert_chain = %w[certs/cert.pem]
 
   if $PROGRAM_NAME.end_with?('gem')
     gem.signing_key = File.expand_path('~/.ssh/gem-private_key.pem')

data/spec/rdkafka/abstract_handle_spec.rb

@@ -80,7 +80,6 @@ describe Rdkafka::AbstractHandle do
       let(:pending_handle) { true }
 
       it "should wait until the timeout and then raise an error" do
-        expect(Kernel).not_to receive(:warn)
         expect {
           subject.wait(max_wait_timeout: 0.1)
         }.to raise_error Rdkafka::AbstractHandle::WaitTimeoutError, /test_operation/
@@ -90,22 +89,15 @@ describe Rdkafka::AbstractHandle do
     context 'when pending_handle false' do
       let(:pending_handle) { false }
 
-      it 'should show a deprecation warning when wait_timeout is set' do
-        expect(Kernel).to receive(:warn).with(Rdkafka::AbstractHandle::WAIT_TIMEOUT_DEPRECATION_MESSAGE)
-        subject.wait(wait_timeout: 0.1)
-      end
-
       context "without error" do
         let(:result) { 1 }
 
         it "should return a result" do
-          expect(Kernel).not_to receive(:warn)
           wait_result = subject.wait
           expect(wait_result).to eq(result)
         end
 
         it "should wait without a timeout" do
-          expect(Kernel).not_to receive(:warn)
           wait_result = subject.wait(max_wait_timeout: nil)
           expect(wait_result).to eq(result)
         end
@@ -115,7 +107,6 @@ describe Rdkafka::AbstractHandle do
         let(:response) { 20 }
 
         it "should raise an rdkafka error" do
-          expect(Kernel).not_to receive(:warn)
           expect {
             subject.wait
           }.to raise_error Rdkafka::RdkafkaError

data/spec/rdkafka/admin_spec.rb

@@ -34,7 +34,7 @@ describe Rdkafka::Admin do
   describe '#describe_errors' do
     let(:errors) { admin.class.describe_errors }
 
-    it { expect(errors.size).to eq(162) }
+    it { expect(errors.size).to eq(170) }
     it { expect(errors[-184]).to eq(code: -184, description: 'Local: Queue full', name: '_QUEUE_FULL') }
     it { expect(errors[21]).to eq(code: 21, description: 'Broker: Invalid required acks value', name: 'INVALID_REQUIRED_ACKS') }
   end
@@ -737,4 +737,18 @@ expect(ex.broker_message).to match(/Topic name.*is invalid: .* contains one or m
       end
     end
   end
+
+  context "when operating from a fork" do
+    # @see https://github.com/ffi/ffi/issues/1114
+    it 'expect to be able to create topics and run other admin operations without hanging' do
+      # If the FFI issue is not mitigated, this will hang forever
+      pid = fork do
+        admin
+          .create_topic(topic_name, topic_partition_count, topic_replication_factor)
+          .wait
+      end
+
+      Process.wait(pid)
+    end
+  end
 end

data/spec/rdkafka/consumer_spec.rb

@@ -258,6 +258,95 @@ describe Rdkafka::Consumer do
     end
   end
 
+  describe "#seek_by" do
+    let(:topic) { "consume_test_topic" }
+    let(:partition) { 0 }
+    let(:offset) { 0 }
+
+    it "should raise an error when seeking fails" do
+      expect(Rdkafka::Bindings).to receive(:rd_kafka_seek).and_return(20)
+      expect {
+        consumer.seek_by(topic, partition, offset)
+      }.to raise_error Rdkafka::RdkafkaError
+    end
+
+    context "subscription" do
+      let(:timeout) { 1000 }
+
+      before do
+        consumer.subscribe(topic)
+
+        # 1. partitions are assigned
+        wait_for_assignment(consumer)
+        expect(consumer.assignment).not_to be_empty
+
+        # 2. eat unrelated messages
+        while(consumer.poll(timeout)) do; end
+      end
+      after { consumer.unsubscribe }
+
+      def send_one_message(val)
+        producer.produce(
+          topic:     topic,
+          payload:   "payload #{val}",
+          key:       "key 1",
+          partition: 0
+        ).wait
+      end
+
+      it "works when a partition is paused" do
+        # 3. get reference message
+        send_one_message(:a)
+        message1 = consumer.poll(timeout)
+        expect(message1&.payload).to eq "payload a"
+
+        # 4. pause the subscription
+        tpl = Rdkafka::Consumer::TopicPartitionList.new
+        tpl.add_topic(topic, 1)
+        consumer.pause(tpl)
+
+        # 5. seek by the previous message fields
+        consumer.seek_by(message1.topic, message1.partition, message1.offset)
+
+        # 6. resume the subscription
+        tpl = Rdkafka::Consumer::TopicPartitionList.new
+        tpl.add_topic(topic, 1)
+        consumer.resume(tpl)
+
+        # 7. ensure same message is read again
+        message2 = consumer.poll(timeout)
+
+        # This is needed because `enable.auto.offset.store` is true but when running in CI that
+        # is overloaded, offset store lags
+        sleep(2)
+
+        consumer.commit
+        expect(message1.offset).to eq message2.offset
+        expect(message1.payload).to eq message2.payload
+      end
+
+      it "allows skipping messages" do
+        # 3. send messages
+        send_one_message(:a)
+        send_one_message(:b)
+        send_one_message(:c)
+
+        # 4. get reference message
+        message = consumer.poll(timeout)
+        expect(message&.payload).to eq "payload a"
+
+        # 5. seek over one message
+        consumer.seek_by(message.topic, message.partition, message.offset + 2)
+
+        # 6. ensure that only one message is available
+        records = consumer.poll(timeout)
+        expect(records&.payload).to eq "payload c"
+        records = consumer.poll(timeout)
+        expect(records).to be_nil
+      end
+    end
+  end
+
   describe "#assign and #assignment" do
     it "should return an empty assignment if nothing is assigned" do
       expect(consumer.assignment).to be_empty

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.16.1
+  version: 0.18.0
 platform: ruby
 authors:
 - Thijs Cadier
@@ -13,30 +13,30 @@ cert_chain:
   -----BEGIN CERTIFICATE-----
   MIIEcDCCAtigAwIBAgIBATANBgkqhkiG9w0BAQsFADA/MRAwDgYDVQQDDAdjb250
   YWN0MRcwFQYKCZImiZPyLGQBGRYHa2FyYWZrYTESMBAGCgmSJomT8ixkARkWAmlv
-  MB4XDTIzMDgyMTA3MjU1NFoXDTI0MDgyMDA3MjU1NFowPzEQMA4GA1UEAwwHY29u
+  MB4XDTI0MDgyMzEwMTkyMFoXDTQ5MDgxNzEwMTkyMFowPzEQMA4GA1UEAwwHY29u
   dGFjdDEXMBUGCgmSJomT8ixkARkWB2thcmFma2ExEjAQBgoJkiaJk/IsZAEZFgJp
-  bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAOuZpyQKEwsTG9plLat7
-  8bUaNuNBEnouTsNMr6X+XTgvyrAxTuocdsyP1sNCjdS1B8RiiDH1/Nt9qpvlBWon
-  sdJ1SYhaWNVfqiYStTDnCx3PRMmHRdD4KqUWKpN6VpZ1O/Zu+9Mw0COmvXgZuuO9
-  wMSJkXRo6dTCfMedLAIxjMeBIxtoLR2e6Jm6MR8+8WYYVWrO9kSOOt5eKQLBY7aK
-  b/Dc40EcJKPg3Z30Pia1M9ZyRlb6SOj6SKpHRqc7vbVQxjEw6Jjal1lZ49m3YZMd
-  ArMAs9lQZNdSw5/UX6HWWURLowg6k10RnhTUtYyzO9BFev0JFJftHnmuk8vtb+SD
-  5VPmjFXg2VOcw0B7FtG75Vackk8QKfgVe3nSPhVpew2CSPlbJzH80wChbr19+e3+
-  YGr1tOiaJrL6c+PNmb0F31NXMKpj/r+n15HwlTMRxQrzFcgjBlxf2XFGnPQXHhBm
-  kp1OFnEq4GG9sON4glRldkwzi/f/fGcZmo5fm3d+0ZdNgwIDAQABo3cwdTAJBgNV
-  HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQUPVH5+dLA80A1kJ2Uz5iGwfOa
-  1+swHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
-  bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEAnpa0jcN7JzREHMTQ
-  bfZ+xcvlrzuROMY6A3zIZmQgbnoZZNuX4cMRrT1p1HuwXpxdpHPw7dDjYqWw3+1h
-  3mXLeMuk7amjQpYoSWU/OIZMhIsARra22UN8qkkUlUj3AwTaChVKN/bPJOM2DzfU
-  kz9vUgLeYYFfQbZqeI6SsM7ltilRV4W8D9yNUQQvOxCFxtLOetJ00fC/E7zMUzbK
-  IBwYFQYsbI6XQzgAIPW6nGSYKgRhkfpmquXSNKZRIQ4V6bFrufa+DzD0bt2ZA3ah
-  fMmJguyb5L2Gf1zpDXzFSPMG7YQFLzwYz1zZZvOU7/UCpQsHpID/YxqDp4+Dgb+Y
-  qma0whX8UG/gXFV2pYWpYOfpatvahwi+A1TwPQsuZwkkhi1OyF1At3RY+hjSXyav
-  AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
-  msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
+  bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAKjLhLjQqUlNayxkXnO+
+  PsmCDs/KFIzhrsYMfLZRZNaWmzV3ujljMOdDjd4snM2X06C41iVdQPWjpe3j8vVe
+  ZXEWR/twSbOP6Eeg8WVH2wCOo0x5i7yhVn4UBLH4JpfEMCbemVcWQ9ry9OMg4WpH
+  Uu4dRwxFV7hzCz3p0QfNLRI4miAxnGWcnlD98IJRjBAksTuR1Llj0vbOrDGsL9ZT
+  JeXP2gdRLd8SqzAFJEWrbeTBCBU7gfSh3oMg5SVDLjaqf7Kz5wC/8bDZydzanOxB
+  T6CDXPsCnllmvTNx2ei2T5rGYJOzJeNTmJLLK6hJWUlAvaQSvCwZRvFJ0tVGLEoS
+  flqSr6uGyyl1eMUsNmsH4BqPEYcAV6P2PKTv2vUR8AP0raDvZ3xL1TKvfRb8xRpo
+  vPopCGlY5XBWEc6QERHfVLTIVsjnls2/Ujj4h8/TSfqqYnaHKefIMLbuD/tquMjD
+  iWQsW2qStBV0T+U7FijKxVfrfqZP7GxQmDAc9o1iiyAa3QIDAQABo3cwdTAJBgNV
+  HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQU3O4dTXmvE7YpAkszGzR9DdL9
+  sbEwHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
+  bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEAVKTfoLXn7mqdSxIR
+  eqxcR6Huudg1jes81s1+X0uiRTR3hxxKZ3Y82cPsee9zYWyBrN8TA4KA0WILTru7
+  Ygxvzha0SRPsSiaKLmgOJ+61ebI4+bOORzIJLpD6GxCxu1r7MI4+0r1u1xe0EWi8
+  agkVo1k4Vi8cKMLm6Gl9b3wG9zQBw6fcgKwmpjKiNnOLP+OytzUANrIUJjoq6oal
+  TC+f/Uc0TLaRqUaW/bejxzDWWHoM3SU6aoLPuerglzp9zZVzihXwx3jPLUVKDFpF
+  Rl2lcBDxlpYGueGo0/oNzGJAAy6js8jhtHC9+19PD53vk7wHtFTZ/0ugDQYnwQ+x
+  oml2fAAuVWpTBCgOVFe6XCQpMKopzoxQ1PjKztW2KYxgJdIBX87SnL3aWuBQmhRd
+  i9zWxov0mr44TWegTVeypcWGd/0nxu1+QHVNHJrpqlPBRvwQsUm7fwmRInGpcaB8
+  ap8wNYvryYzrzvzUxIVFBVM5PacgkFqRmolCa8I7tdKQN+R1
   -----END CERTIFICATE-----
-date: 2024-07-10 00:00:00.000000000 Z
+date: 2024-09-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -185,8 +185,9 @@ files:
 - MIT-LICENSE
 - README.md
 - Rakefile
-- certs/cert_chain.pem
-- dist/librdkafka_2.3.0.tar.gz
+- certs/cert.pem
+- dist/librdkafka_2.5.0.tar.gz
+- dist/patches/rdkafka_sticky_assignor.c.patch
 - docker-compose.yml
 - ext/README.md
 - ext/Rakefile

data/certs/cert_chain.pem

@@ -1,26 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIEcDCCAtigAwIBAgIBATANBgkqhkiG9w0BAQsFADA/MRAwDgYDVQQDDAdjb250
-YWN0MRcwFQYKCZImiZPyLGQBGRYHa2FyYWZrYTESMBAGCgmSJomT8ixkARkWAmlv
-MB4XDTIzMDgyMTA3MjU1NFoXDTI0MDgyMDA3MjU1NFowPzEQMA4GA1UEAwwHY29u
-dGFjdDEXMBUGCgmSJomT8ixkARkWB2thcmFma2ExEjAQBgoJkiaJk/IsZAEZFgJp
-bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAOuZpyQKEwsTG9plLat7
-8bUaNuNBEnouTsNMr6X+XTgvyrAxTuocdsyP1sNCjdS1B8RiiDH1/Nt9qpvlBWon
-sdJ1SYhaWNVfqiYStTDnCx3PRMmHRdD4KqUWKpN6VpZ1O/Zu+9Mw0COmvXgZuuO9
-wMSJkXRo6dTCfMedLAIxjMeBIxtoLR2e6Jm6MR8+8WYYVWrO9kSOOt5eKQLBY7aK
-b/Dc40EcJKPg3Z30Pia1M9ZyRlb6SOj6SKpHRqc7vbVQxjEw6Jjal1lZ49m3YZMd
-ArMAs9lQZNdSw5/UX6HWWURLowg6k10RnhTUtYyzO9BFev0JFJftHnmuk8vtb+SD
-5VPmjFXg2VOcw0B7FtG75Vackk8QKfgVe3nSPhVpew2CSPlbJzH80wChbr19+e3+
-YGr1tOiaJrL6c+PNmb0F31NXMKpj/r+n15HwlTMRxQrzFcgjBlxf2XFGnPQXHhBm
-kp1OFnEq4GG9sON4glRldkwzi/f/fGcZmo5fm3d+0ZdNgwIDAQABo3cwdTAJBgNV
-HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQUPVH5+dLA80A1kJ2Uz5iGwfOa
-1+swHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
-bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEAnpa0jcN7JzREHMTQ
-bfZ+xcvlrzuROMY6A3zIZmQgbnoZZNuX4cMRrT1p1HuwXpxdpHPw7dDjYqWw3+1h
-3mXLeMuk7amjQpYoSWU/OIZMhIsARra22UN8qkkUlUj3AwTaChVKN/bPJOM2DzfU
-kz9vUgLeYYFfQbZqeI6SsM7ltilRV4W8D9yNUQQvOxCFxtLOetJ00fC/E7zMUzbK
-IBwYFQYsbI6XQzgAIPW6nGSYKgRhkfpmquXSNKZRIQ4V6bFrufa+DzD0bt2ZA3ah
-fMmJguyb5L2Gf1zpDXzFSPMG7YQFLzwYz1zZZvOU7/UCpQsHpID/YxqDp4+Dgb+Y
-qma0whX8UG/gXFV2pYWpYOfpatvahwi+A1TwPQsuZwkkhi1OyF1At3RY+hjSXyav
-AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
-msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
------END CERTIFICATE-----