karafka-rdkafka 0.17.1 → 0.17.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c24a6e6092e6f39f66f81e4a1ab3e07fff2afb92df937232f5237e7602d1eea9
-  data.tar.gz: 1fcda4b35822c4c89c9a3a92ee67de8f2cb1f059f4e84f23cc12462c74bbd701
+  metadata.gz: 9c8adc4477dceb9b8d347a515b83cd250334fc159a862ac2315a3747fab36a3f
+  data.tar.gz: 25aa5b86190b3cf23ad67aae7ff5d66a5032d0bd17c80e34a776cb2af517e3cc
 SHA512:
-  metadata.gz: dfc563c4b071bf6ee6a2595fbc98122eff9758fdfa90f297cd27541b14a39ad71ffa11290d0ddc02687c5aa6cbbdf8cf7a26594582595403f6bb05de9ef25273
-  data.tar.gz: 8d9644f980260d159cd26624fbbbb56360a3dd3e94cebabe0eb89775143379aeef1c46e96b8b5e82daa2e7a296c97b65cba821b48eb12f6b80c3e940ddaa04d6
+  metadata.gz: 80c5696125626d2acd2c9cffea4e54fef6cf92269b2e8fbbff3b653bff2598bf06b4948e8fdcaff5285d245569b2c77c45958d783913ae9af79b8a816526837f
+  data.tar.gz: bd309b7292041e5d369a9504f85bd0f2dd61bf2dc0006dc0c9f7d58d69f367e44dbe4fce800e5f8157277c0edb316a54d5396639040fc21292f85a9c289eccad

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Rdkafka Changelog
 
+## 0.17.3 (2024-08-09)
+- [Fix] Mitigate a case where FFI would not restart the background events callback dispatcher in forks.
+
+## 0.17.2 (2024-08-07)
+- [Enhancement] Support returning `#details` for errors that do have topic/partition related extra info.
+
 ## 0.17.1 (2024-08-01)
 - [Enhancement] Support ability to release patches to librdkafka.
 - [Patch] Patch cooperative-sticky assignments in librdkafka.

data/lib/rdkafka/callbacks.rb

@@ -2,7 +2,6 @@
 
 module Rdkafka
   module Callbacks
-
     # Extracts attributes of a rd_kafka_topic_result_t
     #
     # @private
@@ -149,13 +148,6 @@ module Rdkafka
       end
     end
 
-    # FFI Function used for Create Topic and Delete Topic callbacks
-    BackgroundEventCallbackFunction = FFI::Function.new(
-        :void, [:pointer, :pointer, :pointer]
-    ) do |client_ptr, event_ptr, opaque_ptr|
-      BackgroundEventCallback.call(client_ptr, event_ptr, opaque_ptr)
-    end
-
     # @private
     class BackgroundEventCallback
       def self.call(_, event_ptr, _)
@@ -348,13 +340,6 @@ module Rdkafka
       end
     end
 
-    # FFI Function used for Message Delivery callbacks
-    DeliveryCallbackFunction = FFI::Function.new(
-        :void, [:pointer, :pointer, :pointer]
-    ) do |client_ptr, message_ptr, opaque_ptr|
-      DeliveryCallback.call(client_ptr, message_ptr, opaque_ptr)
-    end
-
     # @private
     class DeliveryCallback
       def self.call(_, message_ptr, opaque_ptr)
@@ -387,5 +372,44 @@ module Rdkafka
         end
       end
     end
+
+    @@mutex = Mutex.new
+    @@current_pid = nil
+
+    class << self
+      # Defines or recreates after fork callbacks that require FFI thread so the callback thread
+      # is always correctly initialized
+      #
+      # @see https://github.com/ffi/ffi/issues/1114
+      def ensure_ffi_running
+        @@mutex.synchronize do
+          return if @@current_pid == ::Process.pid
+
+          if const_defined?(:BackgroundEventCallbackFunction, false)
+            send(:remove_const, :BackgroundEventCallbackFunction)
+            send(:remove_const, :DeliveryCallbackFunction)
+          end
+
+          # FFI Function used for Create Topic and Delete Topic callbacks
+          background_event_callback_function = FFI::Function.new(
+              :void, [:pointer, :pointer, :pointer]
+          ) do |client_ptr, event_ptr, opaque_ptr|
+            BackgroundEventCallback.call(client_ptr, event_ptr, opaque_ptr)
+          end
+
+          # FFI Function used for Message Delivery callbacks
+          delivery_callback_function = FFI::Function.new(
+              :void, [:pointer, :pointer, :pointer]
+          ) do |client_ptr, message_ptr, opaque_ptr|
+            DeliveryCallback.call(client_ptr, message_ptr, opaque_ptr)
+          end
+
+          const_set(:BackgroundEventCallbackFunction, background_event_callback_function)
+          const_set(:DeliveryCallbackFunction, delivery_callback_function)
+
+          @@current_pid = ::Process.pid
+        end
+      end
+    end
   end
 end

data/lib/rdkafka/config.rb

@@ -146,6 +146,8 @@ module Rdkafka
     #
     # @return [Config]
     def initialize(config_hash = {})
+      Callbacks.ensure_ffi_running
+
       @config_hash = DEFAULT_CONFIG.merge(config_hash)
       @consumer_rebalance_listener = nil
       @consumer_poll_set = true

data/lib/rdkafka/error.rb

@@ -6,6 +6,9 @@ module Rdkafka
 
   # Error returned by the underlying rdkafka library.
   class RdkafkaError < BaseError
+    # Empty hash for details default allocation
+    EMPTY_HASH = {}.freeze
+
     # The underlying raw error response
     # @return [Integer]
     attr_reader :rdkafka_response
@@ -18,6 +21,10 @@ module Rdkafka
     # @return [String]
     attr_reader :broker_message
 
+    # Optional details hash specific to a given error or empty hash if none or not supported
+    # @return [Hash]
+    attr_reader :details
+
     class << self
       def build_from_c(response_ptr, message_prefix = nil, broker_message: nil)
         code = Rdkafka::Bindings.rd_kafka_error_code(response_ptr)
@@ -54,7 +61,21 @@ module Rdkafka
             message_prefix ||= response_ptr_or_code[:payload].read_string(response_ptr_or_code[:len])
           end
 
-          new(response_ptr_or_code[:err], message_prefix, broker_message: broker_message)
+          details = if response_ptr_or_code[:rkt].null?
+                      EMPTY_HASH
+                    else
+                      {
+                        partition: response_ptr_or_code[:partition],
+                        offset: response_ptr_or_code[:offset],
+                        topic: Bindings.rd_kafka_topic_name(response_ptr_or_code[:rkt])
+                      }.freeze
+                    end
+          new(
+            response_ptr_or_code[:err],
+            message_prefix,
+            broker_message: broker_message,
+            details: details
+          )
         else
           build_from_c(response_ptr_or_code, message_prefix)
         end
@@ -73,7 +94,8 @@ module Rdkafka
       broker_message: nil,
       fatal: false,
       retryable: false,
-      abortable: false
+      abortable: false,
+      details: EMPTY_HASH
     )
       raise TypeError.new("Response has to be an integer") unless response.is_a? Integer
       @rdkafka_response = response
@@ -82,6 +104,7 @@ module Rdkafka
       @fatal = fatal
       @retryable = retryable
       @abortable = abortable
+      @details = details
     end
 
     # This error's code, for example `:partition_eof`, `:msg_size_too_large`.

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.17.1"
+  VERSION = "0.17.3"
   LIBRDKAFKA_VERSION = "2.5.0"
   LIBRDKAFKA_SOURCE_SHA256 = "3dc62de731fd516dfb1032861d9a580d4d0b5b0856beb0f185d06df8e6c26259"
 end

data/spec/rdkafka/admin_spec.rb

@@ -737,4 +737,18 @@ describe Rdkafka::Admin do
       end
     end
   end
+
+  context "when operating from a fork" do
+    # @see https://github.com/ffi/ffi/issues/1114
+    it 'expect to be able to create topics and run other admin operations without hanging' do
+      # If the FFI issue is not mitigated, this will hang forever
+      pid = fork do
+        admin
+          .create_topic(topic_name, topic_partition_count, topic_replication_factor)
+          .wait
+      end
+
+      Process.wait(pid)
+    end
+  end
 end

data/spec/rdkafka/consumer_spec.rb

@@ -1467,4 +1467,39 @@ describe Rdkafka::Consumer do
       end
     end
   end
+
+  describe "when reaching eof on a topic and eof reporting enabled" do
+    let(:consumer) { rdkafka_consumer_config(:"enable.partition.eof" => true).consumer }
+
+    it "should return proper details" do
+      (0..2).each do |i|
+        producer.produce(
+          topic:     "consume_test_topic",
+          key:       "key lag #{i}",
+          partition: i
+        ).wait
+      end
+
+      # Consume to the end
+      consumer.subscribe("consume_test_topic")
+      eof_count = 0
+      eof_error = nil
+
+      loop do
+        begin
+          consumer.poll(100)
+        rescue Rdkafka::RdkafkaError => error
+          if error.is_partition_eof?
+            eof_error = error
+          end
+          break if eof_error
+        end
+      end
+
+      expect(eof_error.code).to eq(:partition_eof)
+      expect(eof_error.details[:topic]).to eq('consume_test_topic')
+      expect(eof_error.details[:partition]).to be_a(Integer)
+      expect(eof_error.details[:offset]).to be_a(Integer)
+    end
+  end
 end

data/spec/rdkafka/error_spec.rb

@@ -11,6 +11,12 @@ describe Rdkafka::RdkafkaError do
     expect(Rdkafka::RdkafkaError.new(10, "message prefix").message_prefix).to eq "message prefix"
   end
 
+  it "should have empty frozen details by default" do
+    error = Rdkafka::RdkafkaError.new(10, "message prefix")
+    expect(error.details).to eq({})
+    expect(error.details).to be_frozen
+  end
+
   it "should create an error with a broker message" do
     expect(Rdkafka::RdkafkaError.new(10, broker_message: "broker message").broker_message).to eq "broker message"
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.17.1
+  version: 0.17.3
 platform: ruby
 authors:
 - Thijs Cadier
@@ -36,7 +36,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2024-08-01 00:00:00.000000000 Z
+date: 2024-08-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi