karafka-rdkafka 0.14.9 → 0.14.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8a3805cc154b0e1ca0d71fe06a852013c3e8d8dfd8148d4de8902aed66a2774d
-  data.tar.gz: 908236ad1ab0be52549f01ddf90ed37355a3b6e9e3373b867eeb83fa05fdf39c
+  metadata.gz: 46b697b4fffd0a92901c4ba71c41fe81627828a2146f2f9a4bfb7f6127de44b9
+  data.tar.gz: 430bb67bd7de9c7aad6abc037d091aac9dd45b96146e0fc01d0b83e7c9fa0e1f
 SHA512:
-  metadata.gz: 6460881076e6c45ae5feb38a3cfd4da3160f13a092f6a7acdbbe268e297b10e27ad792dab32a8c193824a5ef44fc9c2f40c5685ee5c489370056f1c408297f83
-  data.tar.gz: c2f42212e4df244d36f2312506dd69e52e8064c3485dc1fd0692d223ac9f2512a8b8102b7f473766386eb997dc7bb490fc4fc3c4a2815c528d6c1a8cddee7fcd
+  metadata.gz: ee0329ac06eee8cd12d9ce7c0707dc097d73346d176e9e846621c575fc6fb7a199a8b44136ed07c685fe36bd0b51eb259440aaf21bc502e3b0a951d4481f5376
+  data.tar.gz: 5b55d118468203c5ea367d8b3902cb37baa018f8f0147b7ed5c2eb3d496a56641e3d3f238d90f2a0d9bcabbbf445158dfb6f185c7878882212d9ead56ec09026

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Rdkafka Changelog
 
+## 0.14.11 (2024-07-10)
+- [Fix] Switch to local release of librdkafka to mitigate its unavailability.
+
+## 0.14.10 (2024-02-08)
+- [Fix] Background logger stops working after forking causing memory leaks (mensfeld).
+
 ## 0.14.9 (2024-01-29)
 - [Fix] Partition cache caches invalid `nil` result for `PARTITIONS_COUNT_TTL`.
 - [Enhancement] Report `-1` instead of `nil` in case `partition_count` failure.

data/ext/Rakefile

@@ -22,11 +22,21 @@ task :default => :clean do
       ENV["LDFLAGS"] = "-L#{homebrew_prefix}/lib" unless ENV["LDFLAGS"]
     end
 
+    releases = File.expand_path(File.join(File.dirname(__FILE__), '../dist'))
+
     recipe.files << {
-      :url => "https://codeload.github.com/edenhill/librdkafka/tar.gz/v#{Rdkafka::LIBRDKAFKA_VERSION}",
+      :url => "file://#{releases}/librdkafka_#{Rdkafka::LIBRDKAFKA_VERSION}.tar.gz",
       :sha256 => Rdkafka::LIBRDKAFKA_SOURCE_SHA256
     }
     recipe.configure_options = ["--host=#{recipe.host}"]
+
+    # Disable using libc regex engine in favor of the embedded one
+    # The default regex engine of librdkafka does not always work exactly as most of the users
+    # would expect, hence this flag allows for changing it to the other one
+    if ENV.key?('RDKAFKA_DISABLE_REGEX_EXT')
+      recipe.configure_options << '--disable-regex-ext'
+    end
+
     recipe.cook
     # Move dynamic library we're interested in
     if recipe.host.include?('darwin')

data/karafka-rdkafka.gemspec

@@ -3,10 +3,10 @@
 require File.expand_path('lib/rdkafka/version', __dir__)
 
 Gem::Specification.new do |gem|
-  gem.authors = ['Thijs Cadier']
+  gem.authors = ['Thijs Cadier', 'Maciej Mensfeld']
   gem.email = ["contact@karafka.io"]
   gem.description = "Modern Kafka client library for Ruby based on librdkafka"
-  gem.summary = "The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka. It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+ and Ruby 2.4+."
+  gem.summary = "The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka. It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+ and Ruby 2.7+."
   gem.license = 'MIT'
 
   gem.files = `git ls-files`.split($\)

data/lib/rdkafka/bindings.rb

@@ -147,6 +147,8 @@ module Rdkafka
                  else
                    Logger::UNKNOWN
                  end
+
+      Rdkafka::Config.ensure_log_thread
       Rdkafka::Config.log_queue << [severity, "rdkafka: #{line}"]
     end
 

data/lib/rdkafka/config.rb

@@ -15,13 +15,12 @@ module Rdkafka
     @@opaques = ObjectSpace::WeakMap.new
     # @private
     @@log_queue = Queue.new
-
-    Thread.start do
-      loop do
-        severity, msg = @@log_queue.pop
-        @@logger.add(severity, msg)
-      end
-    end
+    # @private
+    # We memoize thread on the first log flush
+    # This allows us also to restart logger thread on forks
+    @@log_thread = nil
+    # @private
+    @@log_mutex = Mutex.new
 
     # Returns the current logger, by default this is a logger to stdout.
     #
@@ -30,6 +29,24 @@ module Rdkafka
       @@logger
     end
 
+    # Makes sure that there is a thread for consuming logs
+    # We do not spawn thread immediately and we need to check if it operates to support forking
+    def self.ensure_log_thread
+      return if @@log_thread && @@log_thread.alive?
+
+      @@log_mutex.synchronize do
+        # Restart if dead (fork, crash)
+        @@log_thread = nil if @@log_thread && !@@log_thread.alive?
+
+        @@log_thread ||= Thread.start do
+          loop do
+            severity, msg = @@log_queue.pop
+            @@logger.add(severity, msg)
+          end
+        end
+      end
+    end
+
     # Returns a queue whose contents will be passed to the configured logger. Each entry
     # should follow the format [Logger::Severity, String]. The benefit over calling the
     # logger directly is that this is safe to use from trap contexts.

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.14.9"
+  VERSION = "0.14.11"
   LIBRDKAFKA_VERSION = "2.3.0"
   LIBRDKAFKA_SOURCE_SHA256 = "2d49c35c77eeb3d42fa61c43757fcbb6a206daa560247154e60642bcdcc14d12"
 end

data/spec/rdkafka/config_spec.rb

@@ -22,6 +22,7 @@ describe Rdkafka::Config do
     it "supports logging queue" do
       log = StringIO.new
       Rdkafka::Config.logger = Logger.new(log)
+      Rdkafka::Config.ensure_log_thread
 
       Rdkafka::Config.log_queue << [Logger::FATAL, "I love testing"]
       20.times do
@@ -31,6 +32,25 @@ describe Rdkafka::Config do
 
       expect(log.string).to include "FATAL -- : I love testing"
     end
+
+    it "expect to start new logger thread after fork and work" do
+      reader, writer = IO.pipe
+
+      pid = fork do
+        $stdout.reopen(writer)
+        Rdkafka::Config.logger = Logger.new($stdout)
+        reader.close
+        producer = rdkafka_producer_config(debug: 'all').producer
+        producer.close
+        writer.close
+        sleep(1)
+      end
+
+      writer.close
+      Process.wait(pid)
+      output = reader.read
+      expect(output.split("\n").size).to be >= 20
+    end
   end
 
   context "statistics callback" do

data/spec/rdkafka/consumer_spec.rb

@@ -211,6 +211,11 @@ describe Rdkafka::Consumer do
 
         # 7. ensure same message is read again
         message2 = consumer.poll(timeout)
+
+        # This is needed because `enable.auto.offset.store` is true but when running in CI that
+        # is overloaded, offset store lags
+        sleep(1)
+
         consumer.commit
         expect(message1.offset).to eq message2.offset
         expect(message1.payload).to eq message2.payload

metadata

@@ -1,10 +1,11 @@
 --- !ruby/object:Gem::Specification
 name: karafka-rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.14.9
+  version: 0.14.11
 platform: ruby
 authors:
 - Thijs Cadier
+- Maciej Mensfeld
 autorequire: 
 bindir: bin
 cert_chain:
@@ -35,7 +36,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2024-01-29 00:00:00.000000000 Z
+date: 2024-07-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -185,6 +186,7 @@ files:
 - README.md
 - Rakefile
 - certs/cert_chain.pem
+- dist/librdkafka_2.3.0.tar.gz
 - docker-compose.yml
 - ext/README.md
 - ext/Rakefile
@@ -282,7 +284,7 @@ signing_key:
 specification_version: 4
 summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.
   It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+
-  and Ruby 2.4+.
+  and Ruby 2.7+.
 test_files:
 - spec/rdkafka/abstract_handle_spec.rb
 - spec/rdkafka/admin/create_acl_handle_spec.rb

metadata.gz.sig

@@ -1,4 +1,2 @@
-�v�
f).c̀K����^�K�aY��7�+�ju�E�e�:9��Qmi�V����R-�����Rm��.Ю0w��d���-B�VR0+h��
-+��-��G�o�ބ㵀,�N�l�4�u�c�8��\�|�n/�h�R�55��U4Ї�E rj���Qdz8�Y���ޘv���f���i^�����x����(볬�|�%SY�N����r񟊯���d"���4I1�6S���a��5l�4�k]�-��S��Jgz�A��I��l3�� X�p/����z
-�y�(c�s�dA���,�
-F(���
+6.��(����:��c�i�
���������^�9�ty��z	CF�����������1��0]Q�@��+*�=E���v=!5	�jb�('#����U�gXNC�iƮ�ij�[e������s�'"N.
�X�"C�����Z4�s/4k��1>E�n$�޸�}Q/KaϽ��k;����*
+�֤�[�a�\�J��Z#A��wK�cm����k6��w���^�s
QR�!�3.�"E&!�f��Z���u:0�����r��;�Yf�lP׬�RK�W�1��6w�u��ۃ(r��'�َԟ�x�ԁ�r��D��|���;E��H���!�6�+/̧�a�3U��v̈́!�i�/,߶p��wl��X*0���I�sPnǵ�