karafka-rdkafka 0.17.2.rc1 → 0.17.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f9321c1841f24770f9e375013312c710058066384cdeef25a8e25d5b29d2b869
-  data.tar.gz: 944cd7539a70a2ed8400340e10a3705b34a0aec51d25a666def0f575069027de
+  metadata.gz: 9c8adc4477dceb9b8d347a515b83cd250334fc159a862ac2315a3747fab36a3f
+  data.tar.gz: 25aa5b86190b3cf23ad67aae7ff5d66a5032d0bd17c80e34a776cb2af517e3cc
 SHA512:
-  metadata.gz: 04ae1810964a9ae6d96b402c1c52b8a6e506c8d4f7ff97299f5a35f79ce66a5f415aab2156ec037da1fce2e71b4302e72f1c7509acd6358fb52c2c360a0417c1
-  data.tar.gz: aef4f2144adfe02b54c449cfbdf84eabf8f50aab5df180a7afadb7aa7daa22b243ba00f8ce0686daa7da53441b5b0f81d450204f86cf941011014a0d76146edd
+  metadata.gz: 80c5696125626d2acd2c9cffea4e54fef6cf92269b2e8fbbff3b653bff2598bf06b4948e8fdcaff5285d245569b2c77c45958d783913ae9af79b8a816526837f
+  data.tar.gz: bd309b7292041e5d369a9504f85bd0f2dd61bf2dc0006dc0c9f7d58d69f367e44dbe4fce800e5f8157277c0edb316a54d5396639040fc21292f85a9c289eccad

data/CHANGELOG.md

@@ -1,5 +1,8 @@
 # Rdkafka Changelog
 
+## 0.17.3 (2024-08-09)
+- [Fix] Mitigate a case where FFI would not restart the background events callback dispatcher in forks.
+
 ## 0.17.2 (2024-08-07)
 - [Enhancement] Support returning `#details` for errors that do have topic/partition related extra info.
 

data/lib/rdkafka/callbacks.rb

@@ -2,7 +2,6 @@
 
 module Rdkafka
   module Callbacks
-
     # Extracts attributes of a rd_kafka_topic_result_t
     #
     # @private
@@ -149,13 +148,6 @@ module Rdkafka
       end
     end
 
-    # FFI Function used for Create Topic and Delete Topic callbacks
-    BackgroundEventCallbackFunction = FFI::Function.new(
-        :void, [:pointer, :pointer, :pointer]
-    ) do |client_ptr, event_ptr, opaque_ptr|
-      BackgroundEventCallback.call(client_ptr, event_ptr, opaque_ptr)
-    end
-
     # @private
     class BackgroundEventCallback
       def self.call(_, event_ptr, _)
@@ -348,13 +340,6 @@ module Rdkafka
       end
     end
 
-    # FFI Function used for Message Delivery callbacks
-    DeliveryCallbackFunction = FFI::Function.new(
-        :void, [:pointer, :pointer, :pointer]
-    ) do |client_ptr, message_ptr, opaque_ptr|
-      DeliveryCallback.call(client_ptr, message_ptr, opaque_ptr)
-    end
-
     # @private
     class DeliveryCallback
       def self.call(_, message_ptr, opaque_ptr)
@@ -387,5 +372,44 @@ module Rdkafka
         end
       end
     end
+
+    @@mutex = Mutex.new
+    @@current_pid = nil
+
+    class << self
+      # Defines or recreates after fork callbacks that require FFI thread so the callback thread
+      # is always correctly initialized
+      #
+      # @see https://github.com/ffi/ffi/issues/1114
+      def ensure_ffi_running
+        @@mutex.synchronize do
+          return if @@current_pid == ::Process.pid
+
+          if const_defined?(:BackgroundEventCallbackFunction, false)
+            send(:remove_const, :BackgroundEventCallbackFunction)
+            send(:remove_const, :DeliveryCallbackFunction)
+          end
+
+          # FFI Function used for Create Topic and Delete Topic callbacks
+          background_event_callback_function = FFI::Function.new(
+              :void, [:pointer, :pointer, :pointer]
+          ) do |client_ptr, event_ptr, opaque_ptr|
+            BackgroundEventCallback.call(client_ptr, event_ptr, opaque_ptr)
+          end
+
+          # FFI Function used for Message Delivery callbacks
+          delivery_callback_function = FFI::Function.new(
+              :void, [:pointer, :pointer, :pointer]
+          ) do |client_ptr, message_ptr, opaque_ptr|
+            DeliveryCallback.call(client_ptr, message_ptr, opaque_ptr)
+          end
+
+          const_set(:BackgroundEventCallbackFunction, background_event_callback_function)
+          const_set(:DeliveryCallbackFunction, delivery_callback_function)
+
+          @@current_pid = ::Process.pid
+        end
+      end
+    end
   end
 end

data/lib/rdkafka/config.rb

@@ -146,6 +146,8 @@ module Rdkafka
     #
     # @return [Config]
     def initialize(config_hash = {})
+      Callbacks.ensure_ffi_running
+
       @config_hash = DEFAULT_CONFIG.merge(config_hash)
       @consumer_rebalance_listener = nil
       @consumer_poll_set = true

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.17.2.rc1"
+  VERSION = "0.17.3"
   LIBRDKAFKA_VERSION = "2.5.0"
   LIBRDKAFKA_SOURCE_SHA256 = "3dc62de731fd516dfb1032861d9a580d4d0b5b0856beb0f185d06df8e6c26259"
 end

data/spec/rdkafka/admin_spec.rb

@@ -737,4 +737,18 @@ describe Rdkafka::Admin do
       end
     end
   end
+
+  context "when operating from a fork" do
+    # @see https://github.com/ffi/ffi/issues/1114
+    it 'expect to be able to create topics and run other admin operations without hanging' do
+      # If the FFI issue is not mitigated, this will hang forever
+      pid = fork do
+        admin
+          .create_topic(topic_name, topic_partition_count, topic_replication_factor)
+          .wait
+      end
+
+      Process.wait(pid)
+    end
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.17.2.rc1
+  version: 0.17.3
 platform: ruby
 authors:
 - Thijs Cadier
@@ -36,7 +36,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2024-08-07 00:00:00.000000000 Z
+date: 2024-08-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi