karafka-rdkafka 0.14.7 → 0.14.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 90a3f576c4973a7ee361afd645b426d011234f844ce8cb12228ee9ae6a7a9c2d
-  data.tar.gz: 1d3945367f88877b27eb038281cf88436f289d228824adce48bea6e6a8bd9c3d
+  metadata.gz: 8a3805cc154b0e1ca0d71fe06a852013c3e8d8dfd8148d4de8902aed66a2774d
+  data.tar.gz: 908236ad1ab0be52549f01ddf90ed37355a3b6e9e3373b867eeb83fa05fdf39c
 SHA512:
-  metadata.gz: 260c1e17e41aa25175af3d2ea1acd093872704602179ddf5238903b644316c9631fbbeafbdc062ee03953f6d5d1fc19a4e73643a01630eae48d3d179d70cc72b
-  data.tar.gz: fa8193f334c6bed1084218842c2762586fdb4cb16c15ddc095ae0215e1d51f9b3ccbaa40d92b4379fef6003cccc94396e68db906cff546c3ef8e4ac4f8a7c360
+  metadata.gz: 6460881076e6c45ae5feb38a3cfd4da3160f13a092f6a7acdbbe268e297b10e27ad792dab32a8c193824a5ef44fc9c2f40c5685ee5c489370056f1c408297f83
+  data.tar.gz: c2f42212e4df244d36f2312506dd69e52e8064c3485dc1fd0692d223ac9f2512a8b8102b7f473766386eb997dc7bb490fc4fc3c4a2815c528d6c1a8cddee7fcd

data/.github/workflows/ci.yml

@@ -29,7 +29,6 @@ jobs:
           - '3.0'
           - '3.0.0'
           - '2.7'
-          - '2.7.0'
         include:
           - ruby: '3.3'
             coverage: 'true'

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Rdkafka Changelog
 
+## 0.14.9 (2024-01-29)
+- [Fix] Partition cache caches invalid `nil` result for `PARTITIONS_COUNT_TTL`.
+- [Enhancement] Report `-1` instead of `nil` in case `partition_count` failure.
+
+## 0.14.8 (2024-01-24)
+- [Enhancement] Provide support for Nix OS (alexandriainfantino)
+- [Enhancement] Skip intermediate array creation on delivery report callback execution (one per message) (mensfeld)
+
 ## 0.14.7 (2023-12-29)
 - [Fix] Recognize that Karafka uses a custom partition object (fixed in 2.3.0) and ensure it is recognized.
 

data/README.md

@@ -32,6 +32,7 @@ The most important pieces of a Kafka client are implemented, and we aim to provi
   * [Message Publishing Libraries](#message-publishing-libraries)
 - [Development](#development)
 - [Example](#example)
+- [Versions](#versions)
 
 ## Project Scope
 
@@ -147,3 +148,14 @@ To see everything working, run these in separate tabs:
 bundle exec rake consume_messages
 bundle exec rake produce_messages
 ```
+
+## Versions
+
+| rdkafka-ruby | librdkafka |
+|-|-|
+| 0.15.0 (2023-12-03) | 2.3.0 (2023-10-25) |
+| 0.14.0 (2023-11-21) | 2.2.0 (2023-07-12) |
+| 0.13.0 (2023-07-24) | 2.0.2 (2023-01-20) |
+| 0.12.0 (2022-06-17) | 1.9.0 (2022-06-16) |
+| 0.11.0 (2021-11-17) | 1.8.2 (2021-10-18) |
+| 0.10.0 (2021-09-07) | 1.5.0 (2020-07-20) |

data/ext/Rakefile

@@ -1,40 +1,57 @@
 # frozen_string_literal: true
 
 require File.expand_path('../../lib/rdkafka/version', __FILE__)
-require "mini_portile2"
 require "fileutils"
 require "open-uri"
 
 task :default => :clean do
-  # Download and compile librdkafka
-  recipe = MiniPortile.new("librdkafka", Rdkafka::LIBRDKAFKA_VERSION)
+  # For nix users, nix can't locate the file paths because the packages it's requiring aren't managed by the system but are
+  # managed by nix itself, so using the normal file paths doesn't work for nix users.
+  #
+  # Mini_portile causes an issue because it's dependencies are downloaded on the fly and therefore don't exist/aren't
+  # accessible in the nix environment
+  if ENV.fetch('RDKAFKA_EXT_PATH', '').empty?
+    # Download and compile librdkafka if RDKAFKA_EXT_PATH is not set
+    require "mini_portile2"
+    recipe = MiniPortile.new("librdkafka", Rdkafka::LIBRDKAFKA_VERSION)
 
-  # Use default homebrew openssl if we're on mac and the directory exists
-  # and each of flags is not empty
-  if recipe.host&.include?("darwin") && system("which brew &> /dev/null") && Dir.exist?("#{homebrew_prefix = %x(brew --prefix openssl).strip}")
-    ENV["CPPFLAGS"] = "-I#{homebrew_prefix}/include" unless ENV["CPPFLAGS"]
-    ENV["LDFLAGS"] = "-L#{homebrew_prefix}/lib" unless ENV["LDFLAGS"]
-  end
+    # Use default homebrew openssl if we're on mac and the directory exists
+    # and each of flags is not empty
+    if recipe.host&.include?("darwin") && system("which brew &> /dev/null") && Dir.exist?("#{homebrew_prefix = %x(brew --prefix openssl).strip}")
+      ENV["CPPFLAGS"] = "-I#{homebrew_prefix}/include" unless ENV["CPPFLAGS"]
+      ENV["LDFLAGS"] = "-L#{homebrew_prefix}/lib" unless ENV["LDFLAGS"]
+    end
 
-  recipe.files << {
-    :url => "https://codeload.github.com/confluentinc/librdkafka/tar.gz/v#{Rdkafka::LIBRDKAFKA_VERSION}",
-    :sha256 => Rdkafka::LIBRDKAFKA_SOURCE_SHA256
-  }
-  recipe.configure_options = ["--host=#{recipe.host}"]
-  recipe.cook
-  # Move dynamic library we're interested in
-  if recipe.host.include?('darwin')
-    from_extension = '1.dylib'
-    to_extension   = 'dylib'
+    recipe.files << {
+      :url => "https://codeload.github.com/edenhill/librdkafka/tar.gz/v#{Rdkafka::LIBRDKAFKA_VERSION}",
+      :sha256 => Rdkafka::LIBRDKAFKA_SOURCE_SHA256
+    }
+    recipe.configure_options = ["--host=#{recipe.host}"]
+    recipe.cook
+    # Move dynamic library we're interested in
+    if recipe.host.include?('darwin')
+      from_extension = '1.dylib'
+      to_extension   = 'dylib'
+    else
+      from_extension = 'so.1'
+      to_extension = 'so'
+    end
+    lib_path = File.join(File.dirname(__FILE__), "ports/#{recipe.host}/librdkafka/#{Rdkafka::LIBRDKAFKA_VERSION}/lib/librdkafka.#{from_extension}")
+    FileUtils.mv(lib_path, File.join(File.dirname(__FILE__), "librdkafka.#{to_extension}"))
+    # Cleanup files created by miniportile we don't need in the gem
+    FileUtils.rm_rf File.join(File.dirname(__FILE__), "tmp")
+    FileUtils.rm_rf File.join(File.dirname(__FILE__), "ports")
   else
-    from_extension = 'so.1'
-    to_extension = 'so'
+    # Otherwise, copy existing libraries to ./ext
+    if ENV['RDKAFKA_EXT_PATH'].nil? || ENV['RDKAFKA_EXT_PATH'].empty?
+      raise "RDKAFKA_EXT_PATH must be set in your nix config when running under nix"
+    end
+    files = [
+      File.join(ENV['RDKAFKA_EXT_PATH'], 'lib', 'librdkafka.dylib'),
+      File.join(ENV['RDKAFKA_EXT_PATH'], 'lib', 'librdkafka.so')
+    ]
+    files.each { |ext| FileUtils.cp(ext, File.dirname(__FILE__)) if File.exist?(ext) }
   end
-  lib_path = File.join(File.dirname(__FILE__), "ports/#{recipe.host}/librdkafka/#{Rdkafka::LIBRDKAFKA_VERSION}/lib/librdkafka.#{from_extension}")
-  FileUtils.mv(lib_path, File.join(File.dirname(__FILE__), "librdkafka.#{to_extension}"))
-  # Cleanup files created by miniportile we don't need in the gem
-  FileUtils.rm_rf File.join(File.dirname(__FILE__), "tmp")
-  FileUtils.rm_rf File.join(File.dirname(__FILE__), "ports")
 end
 
 task :clean do

data/lib/rdkafka/consumer.rb

@@ -30,13 +30,6 @@ module Rdkafka
       ->(_) { close }
     end
 
-    # @return [String] consumer name
-    def name
-      @name ||= @native_kafka.with_inner do |inner|
-        ::Rdkafka::Bindings.rd_kafka_name(inner)
-      end
-    end
-
     # Close this consumer
     # @return [nil]
     def close

data/lib/rdkafka/producer.rb

@@ -23,6 +23,9 @@ module Rdkafka
     attr_reader :delivery_callback_arity
 
     # @private
+    # @param native_kafka [NativeKafka]
+    # @param partitioner_name [String, nil] name of the partitioner we want to use or nil to use
+    #   the "consistent_random" default
     def initialize(native_kafka, partitioner_name)
       @native_kafka = native_kafka
       @partitioner_name = partitioner_name || "consistent_random"
@@ -37,10 +40,16 @@ module Rdkafka
           topic_metadata = ::Rdkafka::Metadata.new(inner, topic).topics&.first
         end
 
-        cache[topic] = [
-          monotonic_now,
-          topic_metadata ? topic_metadata[:partition_count] : nil
-        ]
+        partition_count = topic_metadata ? topic_metadata[:partition_count] : -1
+
+        # This approach caches the failure to fetch only for 1 second. This will make sure, that
+        # we do not cache the failure for too long but also "buys" us a bit of time in case there
+        # would be issues in the cluster so we won't overaload it with consecutive requests
+        cache[topic] = if partition_count.positive?
+                         [monotonic_now, partition_count]
+                       else
+                         [monotonic_now - PARTITIONS_COUNT_TTL + 5, partition_count]
+                       end
       end
     end
 
@@ -196,14 +205,15 @@ module Rdkafka
     # Partition count for a given topic.
     #
     # @param topic [String] The topic name.
-    # @return [Integer] partition count for a given topic
+    # @return [Integer] partition count for a given topic or `-1` if it could not be obtained.
     #
     # @note If 'allow.auto.create.topics' is set to true in the broker, the topic will be
     #   auto-created after returning nil.
     #
     # @note We cache the partition count for a given topic for given time.
     #   This prevents us in case someone uses `partition_key` from querying for the count with
-    #   each message. Instead we query once every 30 seconds at most
+    #   each message. Instead we query once every 30 seconds at most if we have a valid partition
+    #   count or every 5 seconds in case we were not able to obtain number of partitions
     def partition_count(topic)
       closed_producer_check(__method__)
 
@@ -253,7 +263,7 @@ module Rdkafka
       if partition_key
         partition_count = partition_count(topic)
         # If the topic is not present, set to -1
-        partition = Rdkafka::Bindings.partitioner(partition_key, partition_count, @partitioner_name) if partition_count
+        partition = Rdkafka::Bindings.partitioner(partition_key, partition_count, @partitioner_name) if partition_count.positive?
       end
 
       # If partition is nil, use -1 to let librdafka set the partition randomly or
@@ -320,13 +330,27 @@ module Rdkafka
       delivery_handle
     end
 
+    # Calls (if registered) the delivery callback
+    #
+    # @param delivery_report [Producer::DeliveryReport]
+    # @param delivery_handle [Producer::DeliveryHandle]
     def call_delivery_callback(delivery_report, delivery_handle)
       return unless @delivery_callback
 
-      args = [delivery_report, delivery_handle].take(@delivery_callback_arity)
-      @delivery_callback.call(*args)
+      case @delivery_callback_arity
+      when 0
+        @delivery_callback.call
+      when 1
+        @delivery_callback.call(delivery_report)
+      else
+        @delivery_callback.call(delivery_report, delivery_handle)
+      end
     end
 
+    # Figures out the arity of a given block/method
+    #
+    # @param callback [#call, Proc]
+    # @return [Integer] arity of the provided block/method
     def arity(callback)
       return callback.arity if callback.respond_to?(:arity)
 
@@ -335,6 +359,10 @@ module Rdkafka
 
     private
 
+    # Ensures, no operations can happen on a closed producer
+    #
+    # @param method [Symbol] name of the method that invoked producer
+    # @raise [Rdkafka::ClosedProducerError]
     def closed_producer_check(method)
       raise Rdkafka::ClosedProducerError.new(method) if closed?
     end

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.14.7"
+  VERSION = "0.14.9"
   LIBRDKAFKA_VERSION = "2.3.0"
   LIBRDKAFKA_SOURCE_SHA256 = "2d49c35c77eeb3d42fa61c43757fcbb6a206daa560247154e60642bcdcc14d12"
 end

data/spec/rdkafka/consumer_spec.rb

@@ -1161,10 +1161,12 @@ describe Rdkafka::Consumer do
   end
 
   describe '#consumer_group_metadata_pointer' do
-    it 'expect to yield pointer' do
-      consumer.consumer_group_metadata_pointer do |pointer|
-        expect(pointer).to be_a(FFI::Pointer)
-      end
+    let(:pointer) { consumer.consumer_group_metadata_pointer }
+
+    after { Rdkafka::Bindings.rd_kafka_consumer_group_metadata_destroy(pointer) }
+
+    it 'expect to return a pointer' do
+      expect(pointer).to be_a(FFI::Pointer)
     end
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.14.7
+  version: 0.14.9
 platform: ruby
 authors:
 - Thijs Cadier
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2023-12-29 00:00:00.000000000 Z
+date: 2024-01-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi