karafka-rdkafka 0.14.8 → 0.14.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 610db3521dfe3b63d906f2d8814756c04882cc68603a218842cc41b487124133
-  data.tar.gz: d870f56e2d7fbc1ae8d14cda5757a02d01d121740da7656deab08341904f4677
+  metadata.gz: 8a3805cc154b0e1ca0d71fe06a852013c3e8d8dfd8148d4de8902aed66a2774d
+  data.tar.gz: 908236ad1ab0be52549f01ddf90ed37355a3b6e9e3373b867eeb83fa05fdf39c
 SHA512:
-  metadata.gz: 17bca0d972de4d24d2de1c1b39ac38bab36eff5f856dd65448215fb53a88fcc8bc0b61e856cf18802ad6649938f52b2d25fdc76b0055396821866841c8067e3a
-  data.tar.gz: 87742fba542fc3df5af16e41baa36d5a7d0c33304f37e54d6ffc50121a4dcf7173a717a0e37353b77cac8828ec707aab54056e3eba53ecfd2780dd87ef9b21ec
+  metadata.gz: 6460881076e6c45ae5feb38a3cfd4da3160f13a092f6a7acdbbe268e297b10e27ad792dab32a8c193824a5ef44fc9c2f40c5685ee5c489370056f1c408297f83
+  data.tar.gz: c2f42212e4df244d36f2312506dd69e52e8064c3485dc1fd0692d223ac9f2512a8b8102b7f473766386eb997dc7bb490fc4fc3c4a2815c528d6c1a8cddee7fcd

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Rdkafka Changelog
 
+## 0.14.9 (2024-01-29)
+- [Fix] Partition cache caches invalid `nil` result for `PARTITIONS_COUNT_TTL`.
+- [Enhancement] Report `-1` instead of `nil` in case `partition_count` failure.
+
 ## 0.14.8 (2024-01-24)
 - [Enhancement] Provide support for Nix OS (alexandriainfantino)
 - [Enhancement] Skip intermediate array creation on delivery report callback execution (one per message) (mensfeld)

data/lib/rdkafka/producer.rb

@@ -40,10 +40,16 @@ module Rdkafka
           topic_metadata = ::Rdkafka::Metadata.new(inner, topic).topics&.first
         end
 
-        cache[topic] = [
-          monotonic_now,
-          topic_metadata ? topic_metadata[:partition_count] : nil
-        ]
+        partition_count = topic_metadata ? topic_metadata[:partition_count] : -1
+
+        # This approach caches the failure to fetch only for 1 second. This will make sure, that
+        # we do not cache the failure for too long but also "buys" us a bit of time in case there
+        # would be issues in the cluster so we won't overaload it with consecutive requests
+        cache[topic] = if partition_count.positive?
+                         [monotonic_now, partition_count]
+                       else
+                         [monotonic_now - PARTITIONS_COUNT_TTL + 5, partition_count]
+                       end
       end
     end
 
@@ -199,14 +205,15 @@ module Rdkafka
     # Partition count for a given topic.
     #
     # @param topic [String] The topic name.
-    # @return [Integer] partition count for a given topic
+    # @return [Integer] partition count for a given topic or `-1` if it could not be obtained.
     #
     # @note If 'allow.auto.create.topics' is set to true in the broker, the topic will be
     #   auto-created after returning nil.
     #
     # @note We cache the partition count for a given topic for given time.
     #   This prevents us in case someone uses `partition_key` from querying for the count with
-    #   each message. Instead we query once every 30 seconds at most
+    #   each message. Instead we query once every 30 seconds at most if we have a valid partition
+    #   count or every 5 seconds in case we were not able to obtain number of partitions
     def partition_count(topic)
       closed_producer_check(__method__)
 
@@ -256,7 +263,7 @@ module Rdkafka
       if partition_key
         partition_count = partition_count(topic)
         # If the topic is not present, set to -1
-        partition = Rdkafka::Bindings.partitioner(partition_key, partition_count, @partitioner_name) if partition_count
+        partition = Rdkafka::Bindings.partitioner(partition_key, partition_count, @partitioner_name) if partition_count.positive?
       end
 
       # If partition is nil, use -1 to let librdafka set the partition randomly or

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.14.8"
+  VERSION = "0.14.9"
   LIBRDKAFKA_VERSION = "2.3.0"
   LIBRDKAFKA_SOURCE_SHA256 = "2d49c35c77eeb3d42fa61c43757fcbb6a206daa560247154e60642bcdcc14d12"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.14.8
+  version: 0.14.9
 platform: ruby
 authors:
 - Thijs Cadier
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2024-01-24 00:00:00.000000000 Z
+date: 2024-01-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi