karafka-rdkafka 0.17.2 → 0.17.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f5ecb2dbdfdc88a9167413058b211874672fe257976d4b183e77513020c4b3a8
-  data.tar.gz: 77b4fecea4f415b7ab123b46e60c6008138c0ab6c50b93563ad498d450205012
+  metadata.gz: ec66e856af65d68379f2d4bd8ec61aec8fc84fb29cd1262f0330b0cb4c19e3ef
+  data.tar.gz: be1f4baea61b9847228f5a14db9ebd7c615ab39fc7d5e92a88cf698b0d716903
 SHA512:
-  metadata.gz: cdca2436c74a067065c064c9bf46300ee8ff58de81ee65d912534a6fe8a8244c75fbc1628f5a9ae71afd249603609b82e1f193ffee32df7b2c588df54b419da0
-  data.tar.gz: 3b44316e071d3c1c78bb8b0bb47b32f6e36f27e14d7a053f43306a98fd795f4b95360066ca4781b2ee0048c17f5870caf0962e1bce8f117df61d23d73e66a594
+  metadata.gz: 95e3685a12a1d382bfeb2e9612ddb2b9a09a75b6ae77836f1e89bf11e990df5e8bf28561a8e1ec6df0cd74d35704a5b12e98d680ad852b86e5f1dfc353e2b0a2
+  data.tar.gz: d03a3c10dba060c7f1992dfc8c64f22eb52acac7806165a15ee32843747cb499f777c2e1ba9a568b9d8e7895945af464c205c61f4ca75d20203a8fef97785a03

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Rdkafka Changelog
 
+## 0.17.4 (2024-09-02)
+- [Enhancement] Bump librdkafka to 2.5.3
+- [Enhancement] Do not release GVL on `rd_kafka_name` (ferrous26)
+- [Fix] Fix unused variable reference in producer (lucasmvnascimento)
+
+## 0.17.3 (2024-08-09)
+- [Fix] Mitigate a case where FFI would not restart the background events callback dispatcher in forks.
+
 ## 0.17.2 (2024-08-07)
 - [Enhancement] Support returning `#details` for errors that do have topic/partition related extra info.
 

data/README.md

@@ -163,8 +163,8 @@ bundle exec rake produce_messages
 
 | rdkafka-ruby | librdkafka | patches |
 |-|-|-|
+| 0.17.4 (2024-09-02) | 2.5.3 (2024-09-02) | yes |
 | 0.17.0 (2024-08-01) | 2.5.0 (2024-07-10) | yes |
-| 0.17.0 (2024-07-21) | 2.5.0 (2024-07-10) | no  |
 | 0.16.0 (2024-06-13) | 2.4.0 (2024-05-07) | no  |
 | 0.15.0 (2023-12-03) | 2.3.0 (2023-10-25) | no  |
 | 0.14.0 (2023-11-21) | 2.2.0 (2023-07-12) | no  |

data/certs/cert.pem

@@ -0,0 +1,26 @@
+-----BEGIN CERTIFICATE-----
+MIIEcDCCAtigAwIBAgIBATANBgkqhkiG9w0BAQsFADA/MRAwDgYDVQQDDAdjb250
+YWN0MRcwFQYKCZImiZPyLGQBGRYHa2FyYWZrYTESMBAGCgmSJomT8ixkARkWAmlv
+MB4XDTI0MDgyMzEwMTkyMFoXDTQ5MDgxNzEwMTkyMFowPzEQMA4GA1UEAwwHY29u
+dGFjdDEXMBUGCgmSJomT8ixkARkWB2thcmFma2ExEjAQBgoJkiaJk/IsZAEZFgJp
+bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAKjLhLjQqUlNayxkXnO+
+PsmCDs/KFIzhrsYMfLZRZNaWmzV3ujljMOdDjd4snM2X06C41iVdQPWjpe3j8vVe
+ZXEWR/twSbOP6Eeg8WVH2wCOo0x5i7yhVn4UBLH4JpfEMCbemVcWQ9ry9OMg4WpH
+Uu4dRwxFV7hzCz3p0QfNLRI4miAxnGWcnlD98IJRjBAksTuR1Llj0vbOrDGsL9ZT
+JeXP2gdRLd8SqzAFJEWrbeTBCBU7gfSh3oMg5SVDLjaqf7Kz5wC/8bDZydzanOxB
+T6CDXPsCnllmvTNx2ei2T5rGYJOzJeNTmJLLK6hJWUlAvaQSvCwZRvFJ0tVGLEoS
+flqSr6uGyyl1eMUsNmsH4BqPEYcAV6P2PKTv2vUR8AP0raDvZ3xL1TKvfRb8xRpo
+vPopCGlY5XBWEc6QERHfVLTIVsjnls2/Ujj4h8/TSfqqYnaHKefIMLbuD/tquMjD
+iWQsW2qStBV0T+U7FijKxVfrfqZP7GxQmDAc9o1iiyAa3QIDAQABo3cwdTAJBgNV
+HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQU3O4dTXmvE7YpAkszGzR9DdL9
+sbEwHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
+bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEAVKTfoLXn7mqdSxIR
+eqxcR6Huudg1jes81s1+X0uiRTR3hxxKZ3Y82cPsee9zYWyBrN8TA4KA0WILTru7
+Ygxvzha0SRPsSiaKLmgOJ+61ebI4+bOORzIJLpD6GxCxu1r7MI4+0r1u1xe0EWi8
+agkVo1k4Vi8cKMLm6Gl9b3wG9zQBw6fcgKwmpjKiNnOLP+OytzUANrIUJjoq6oal
+TC+f/Uc0TLaRqUaW/bejxzDWWHoM3SU6aoLPuerglzp9zZVzihXwx3jPLUVKDFpF
+Rl2lcBDxlpYGueGo0/oNzGJAAy6js8jhtHC9+19PD53vk7wHtFTZ/0ugDQYnwQ+x
+oml2fAAuVWpTBCgOVFe6XCQpMKopzoxQ1PjKztW2KYxgJdIBX87SnL3aWuBQmhRd
+i9zWxov0mr44TWegTVeypcWGd/0nxu1+QHVNHJrpqlPBRvwQsUm7fwmRInGpcaB8
+ap8wNYvryYzrzvzUxIVFBVM5PacgkFqRmolCa8I7tdKQN+R1
+-----END CERTIFICATE-----

data/karafka-rdkafka.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
   gem.version = Rdkafka::VERSION
   gem.required_ruby_version = '>= 3.0'
   gem.extensions = %w(ext/Rakefile)
-  gem.cert_chain = %w[certs/cert_chain.pem]
+  gem.cert_chain = %w[certs/cert.pem]
 
   if $PROGRAM_NAME.end_with?('gem')
     gem.signing_key = File.expand_path('~/.ssh/gem-private_key.pem')

data/lib/rdkafka/bindings.rb

@@ -47,7 +47,7 @@ module Rdkafka
 
     # Metadata
 
-    attach_function :rd_kafka_name, [:pointer], :string, blocking: true
+    attach_function :rd_kafka_name, [:pointer], :string
     attach_function :rd_kafka_memberid, [:pointer], :string, blocking: true
     attach_function :rd_kafka_clusterid, [:pointer], :string, blocking: true
     attach_function :rd_kafka_metadata, [:pointer, :int, :pointer, :pointer, :int], :int, blocking: true

data/lib/rdkafka/callbacks.rb

@@ -2,7 +2,6 @@
 
 module Rdkafka
   module Callbacks
-
     # Extracts attributes of a rd_kafka_topic_result_t
     #
     # @private
@@ -149,13 +148,6 @@ module Rdkafka
       end
     end
 
-    # FFI Function used for Create Topic and Delete Topic callbacks
-    BackgroundEventCallbackFunction = FFI::Function.new(
-        :void, [:pointer, :pointer, :pointer]
-    ) do |client_ptr, event_ptr, opaque_ptr|
-      BackgroundEventCallback.call(client_ptr, event_ptr, opaque_ptr)
-    end
-
     # @private
     class BackgroundEventCallback
       def self.call(_, event_ptr, _)
@@ -348,13 +340,6 @@ module Rdkafka
       end
     end
 
-    # FFI Function used for Message Delivery callbacks
-    DeliveryCallbackFunction = FFI::Function.new(
-        :void, [:pointer, :pointer, :pointer]
-    ) do |client_ptr, message_ptr, opaque_ptr|
-      DeliveryCallback.call(client_ptr, message_ptr, opaque_ptr)
-    end
-
     # @private
     class DeliveryCallback
       def self.call(_, message_ptr, opaque_ptr)
@@ -387,5 +372,44 @@ module Rdkafka
         end
       end
     end
+
+    @@mutex = Mutex.new
+    @@current_pid = nil
+
+    class << self
+      # Defines or recreates after fork callbacks that require FFI thread so the callback thread
+      # is always correctly initialized
+      #
+      # @see https://github.com/ffi/ffi/issues/1114
+      def ensure_ffi_running
+        @@mutex.synchronize do
+          return if @@current_pid == ::Process.pid
+
+          if const_defined?(:BackgroundEventCallbackFunction, false)
+            send(:remove_const, :BackgroundEventCallbackFunction)
+            send(:remove_const, :DeliveryCallbackFunction)
+          end
+
+          # FFI Function used for Create Topic and Delete Topic callbacks
+          background_event_callback_function = FFI::Function.new(
+              :void, [:pointer, :pointer, :pointer]
+          ) do |client_ptr, event_ptr, opaque_ptr|
+            BackgroundEventCallback.call(client_ptr, event_ptr, opaque_ptr)
+          end
+
+          # FFI Function used for Message Delivery callbacks
+          delivery_callback_function = FFI::Function.new(
+              :void, [:pointer, :pointer, :pointer]
+          ) do |client_ptr, message_ptr, opaque_ptr|
+            DeliveryCallback.call(client_ptr, message_ptr, opaque_ptr)
+          end
+
+          const_set(:BackgroundEventCallbackFunction, background_event_callback_function)
+          const_set(:DeliveryCallbackFunction, delivery_callback_function)
+
+          @@current_pid = ::Process.pid
+        end
+      end
+    end
   end
 end

data/lib/rdkafka/config.rb

@@ -146,6 +146,8 @@ module Rdkafka
     #
     # @return [Config]
     def initialize(config_hash = {})
+      Callbacks.ensure_ffi_running
+
       @config_hash = DEFAULT_CONFIG.merge(config_hash)
       @consumer_rebalance_listener = nil
       @consumer_poll_set = true

data/lib/rdkafka/producer.rb

@@ -360,7 +360,7 @@ module Rdkafka
         partitioner_name = @topics_configs.dig(topic, topic_config_hash, :partitioner) || @partitioner_name
 
         # If the topic is not present, set to -1
-        partition = Rdkafka::Bindings.partitioner(partition_key, partition_count, @partitioner_name) if partition_count.positive?
+        partition = Rdkafka::Bindings.partitioner(partition_key, partition_count, partitioner_name) if partition_count.positive?
       end
 
       # If partition is nil, use -1 to let librdafka set the partition randomly or

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.17.2"
-  LIBRDKAFKA_VERSION = "2.5.0"
-  LIBRDKAFKA_SOURCE_SHA256 = "3dc62de731fd516dfb1032861d9a580d4d0b5b0856beb0f185d06df8e6c26259"
+  VERSION = "0.17.4"
+  LIBRDKAFKA_VERSION = "2.5.3"
+  LIBRDKAFKA_SOURCE_SHA256 = "eaa1213fdddf9c43e28834d9a832d9dd732377d35121e42f875966305f52b8ff"
 end

data/spec/rdkafka/admin_spec.rb

@@ -737,4 +737,18 @@ describe Rdkafka::Admin do
       end
     end
   end
+
+  context "when operating from a fork" do
+    # @see https://github.com/ffi/ffi/issues/1114
+    it 'expect to be able to create topics and run other admin operations without hanging' do
+      # If the FFI issue is not mitigated, this will hang forever
+      pid = fork do
+        admin
+          .create_topic(topic_name, topic_partition_count, topic_replication_factor)
+          .wait
+      end
+
+      Process.wait(pid)
+    end
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.17.2
+  version: 0.17.4
 platform: ruby
 authors:
 - Thijs Cadier
@@ -13,30 +13,30 @@ cert_chain:
   -----BEGIN CERTIFICATE-----
   MIIEcDCCAtigAwIBAgIBATANBgkqhkiG9w0BAQsFADA/MRAwDgYDVQQDDAdjb250
   YWN0MRcwFQYKCZImiZPyLGQBGRYHa2FyYWZrYTESMBAGCgmSJomT8ixkARkWAmlv
-  MB4XDTIzMDgyMTA3MjU1NFoXDTI0MDgyMDA3MjU1NFowPzEQMA4GA1UEAwwHY29u
+  MB4XDTI0MDgyMzEwMTkyMFoXDTQ5MDgxNzEwMTkyMFowPzEQMA4GA1UEAwwHY29u
   dGFjdDEXMBUGCgmSJomT8ixkARkWB2thcmFma2ExEjAQBgoJkiaJk/IsZAEZFgJp
-  bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAOuZpyQKEwsTG9plLat7
-  8bUaNuNBEnouTsNMr6X+XTgvyrAxTuocdsyP1sNCjdS1B8RiiDH1/Nt9qpvlBWon
-  sdJ1SYhaWNVfqiYStTDnCx3PRMmHRdD4KqUWKpN6VpZ1O/Zu+9Mw0COmvXgZuuO9
-  wMSJkXRo6dTCfMedLAIxjMeBIxtoLR2e6Jm6MR8+8WYYVWrO9kSOOt5eKQLBY7aK
-  b/Dc40EcJKPg3Z30Pia1M9ZyRlb6SOj6SKpHRqc7vbVQxjEw6Jjal1lZ49m3YZMd
-  ArMAs9lQZNdSw5/UX6HWWURLowg6k10RnhTUtYyzO9BFev0JFJftHnmuk8vtb+SD
-  5VPmjFXg2VOcw0B7FtG75Vackk8QKfgVe3nSPhVpew2CSPlbJzH80wChbr19+e3+
-  YGr1tOiaJrL6c+PNmb0F31NXMKpj/r+n15HwlTMRxQrzFcgjBlxf2XFGnPQXHhBm
-  kp1OFnEq4GG9sON4glRldkwzi/f/fGcZmo5fm3d+0ZdNgwIDAQABo3cwdTAJBgNV
-  HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQUPVH5+dLA80A1kJ2Uz5iGwfOa
-  1+swHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
-  bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEAnpa0jcN7JzREHMTQ
-  bfZ+xcvlrzuROMY6A3zIZmQgbnoZZNuX4cMRrT1p1HuwXpxdpHPw7dDjYqWw3+1h
-  3mXLeMuk7amjQpYoSWU/OIZMhIsARra22UN8qkkUlUj3AwTaChVKN/bPJOM2DzfU
-  kz9vUgLeYYFfQbZqeI6SsM7ltilRV4W8D9yNUQQvOxCFxtLOetJ00fC/E7zMUzbK
-  IBwYFQYsbI6XQzgAIPW6nGSYKgRhkfpmquXSNKZRIQ4V6bFrufa+DzD0bt2ZA3ah
-  fMmJguyb5L2Gf1zpDXzFSPMG7YQFLzwYz1zZZvOU7/UCpQsHpID/YxqDp4+Dgb+Y
-  qma0whX8UG/gXFV2pYWpYOfpatvahwi+A1TwPQsuZwkkhi1OyF1At3RY+hjSXyav
-  AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
-  msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
+  bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAKjLhLjQqUlNayxkXnO+
+  PsmCDs/KFIzhrsYMfLZRZNaWmzV3ujljMOdDjd4snM2X06C41iVdQPWjpe3j8vVe
+  ZXEWR/twSbOP6Eeg8WVH2wCOo0x5i7yhVn4UBLH4JpfEMCbemVcWQ9ry9OMg4WpH
+  Uu4dRwxFV7hzCz3p0QfNLRI4miAxnGWcnlD98IJRjBAksTuR1Llj0vbOrDGsL9ZT
+  JeXP2gdRLd8SqzAFJEWrbeTBCBU7gfSh3oMg5SVDLjaqf7Kz5wC/8bDZydzanOxB
+  T6CDXPsCnllmvTNx2ei2T5rGYJOzJeNTmJLLK6hJWUlAvaQSvCwZRvFJ0tVGLEoS
+  flqSr6uGyyl1eMUsNmsH4BqPEYcAV6P2PKTv2vUR8AP0raDvZ3xL1TKvfRb8xRpo
+  vPopCGlY5XBWEc6QERHfVLTIVsjnls2/Ujj4h8/TSfqqYnaHKefIMLbuD/tquMjD
+  iWQsW2qStBV0T+U7FijKxVfrfqZP7GxQmDAc9o1iiyAa3QIDAQABo3cwdTAJBgNV
+  HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQU3O4dTXmvE7YpAkszGzR9DdL9
+  sbEwHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
+  bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEAVKTfoLXn7mqdSxIR
+  eqxcR6Huudg1jes81s1+X0uiRTR3hxxKZ3Y82cPsee9zYWyBrN8TA4KA0WILTru7
+  Ygxvzha0SRPsSiaKLmgOJ+61ebI4+bOORzIJLpD6GxCxu1r7MI4+0r1u1xe0EWi8
+  agkVo1k4Vi8cKMLm6Gl9b3wG9zQBw6fcgKwmpjKiNnOLP+OytzUANrIUJjoq6oal
+  TC+f/Uc0TLaRqUaW/bejxzDWWHoM3SU6aoLPuerglzp9zZVzihXwx3jPLUVKDFpF
+  Rl2lcBDxlpYGueGo0/oNzGJAAy6js8jhtHC9+19PD53vk7wHtFTZ/0ugDQYnwQ+x
+  oml2fAAuVWpTBCgOVFe6XCQpMKopzoxQ1PjKztW2KYxgJdIBX87SnL3aWuBQmhRd
+  i9zWxov0mr44TWegTVeypcWGd/0nxu1+QHVNHJrpqlPBRvwQsUm7fwmRInGpcaB8
+  ap8wNYvryYzrzvzUxIVFBVM5PacgkFqRmolCa8I7tdKQN+R1
   -----END CERTIFICATE-----
-date: 2024-08-07 00:00:00.000000000 Z
+date: 2024-09-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -185,8 +185,8 @@ files:
 - MIT-LICENSE
 - README.md
 - Rakefile
-- certs/cert_chain.pem
-- dist/librdkafka_2.5.0.tar.gz
+- certs/cert.pem
+- dist/librdkafka_2.5.3.tar.gz
 - dist/patches/rdkafka_sticky_assignor.c.patch
 - docker-compose.yml
 - ext/README.md

data/certs/cert_chain.pem

@@ -1,26 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIEcDCCAtigAwIBAgIBATANBgkqhkiG9w0BAQsFADA/MRAwDgYDVQQDDAdjb250
-YWN0MRcwFQYKCZImiZPyLGQBGRYHa2FyYWZrYTESMBAGCgmSJomT8ixkARkWAmlv
-MB4XDTIzMDgyMTA3MjU1NFoXDTI0MDgyMDA3MjU1NFowPzEQMA4GA1UEAwwHY29u
-dGFjdDEXMBUGCgmSJomT8ixkARkWB2thcmFma2ExEjAQBgoJkiaJk/IsZAEZFgJp
-bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAOuZpyQKEwsTG9plLat7
-8bUaNuNBEnouTsNMr6X+XTgvyrAxTuocdsyP1sNCjdS1B8RiiDH1/Nt9qpvlBWon
-sdJ1SYhaWNVfqiYStTDnCx3PRMmHRdD4KqUWKpN6VpZ1O/Zu+9Mw0COmvXgZuuO9
-wMSJkXRo6dTCfMedLAIxjMeBIxtoLR2e6Jm6MR8+8WYYVWrO9kSOOt5eKQLBY7aK
-b/Dc40EcJKPg3Z30Pia1M9ZyRlb6SOj6SKpHRqc7vbVQxjEw6Jjal1lZ49m3YZMd
-ArMAs9lQZNdSw5/UX6HWWURLowg6k10RnhTUtYyzO9BFev0JFJftHnmuk8vtb+SD
-5VPmjFXg2VOcw0B7FtG75Vackk8QKfgVe3nSPhVpew2CSPlbJzH80wChbr19+e3+
-YGr1tOiaJrL6c+PNmb0F31NXMKpj/r+n15HwlTMRxQrzFcgjBlxf2XFGnPQXHhBm
-kp1OFnEq4GG9sON4glRldkwzi/f/fGcZmo5fm3d+0ZdNgwIDAQABo3cwdTAJBgNV
-HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQUPVH5+dLA80A1kJ2Uz5iGwfOa
-1+swHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
-bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEAnpa0jcN7JzREHMTQ
-bfZ+xcvlrzuROMY6A3zIZmQgbnoZZNuX4cMRrT1p1HuwXpxdpHPw7dDjYqWw3+1h
-3mXLeMuk7amjQpYoSWU/OIZMhIsARra22UN8qkkUlUj3AwTaChVKN/bPJOM2DzfU
-kz9vUgLeYYFfQbZqeI6SsM7ltilRV4W8D9yNUQQvOxCFxtLOetJ00fC/E7zMUzbK
-IBwYFQYsbI6XQzgAIPW6nGSYKgRhkfpmquXSNKZRIQ4V6bFrufa+DzD0bt2ZA3ah
-fMmJguyb5L2Gf1zpDXzFSPMG7YQFLzwYz1zZZvOU7/UCpQsHpID/YxqDp4+Dgb+Y
-qma0whX8UG/gXFV2pYWpYOfpatvahwi+A1TwPQsuZwkkhi1OyF1At3RY+hjSXyav
-AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
-msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
------END CERTIFICATE-----