karafka 2.1.6 → 2.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a6994a6d579728a877f84c87086d093aae8a1f830b891fcb4904883085432fe4
-  data.tar.gz: 13b21009a471194a72971ca81ddc718e044bb96587db0e8f186974f554e9ec62
+  metadata.gz: f405521c7a6706cc95e764a4740e7570935f7595d34481bbe33fb617e5537978
+  data.tar.gz: cd6671c441c07e31050bbddab290ba4d31e4a580a646cfd965edf58c19ff150c
 SHA512:
-  metadata.gz: e4711880bde1d2cd1cb34959f740459979b74ff4d28a671a232f88adbe7473cf67e366fc2b492fac761c572f3a6dfc147a59d46fc08e1c5e18df8ac5f108afdd
-  data.tar.gz: c094600c2bd421ce309c0125d60ea82ed0106d5ce4566b3bb8c1aab13c553e7bd2f6651b98029e42ac831b132563b2c502dd1c76defbf8307cd9bd2393b258f7
+  metadata.gz: 7b5e343a0d2c6e1f885c6eac6509de2f411b54e1a30ce12fac6fa18bb813d82ef666444345b92d8348ac4955cdabfc47ad3658312482f6c500ca169814f10517
+  data.tar.gz: 1b0c319f85dde3bc20b21a842da220d513351b436b3e4de08d56e69a02c36c7c2cd4187c879596ffc73f5dffc2cc3f032c6a8cdbd958ce34138866d27aa00b2b

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Karafka framework changelog
 
+## 2.1.7 (2023-07-22)
+- [Improvement] Always query for watermarks in the Iterator to improve the initial response time.
+- [Improvement] Add `max_wait_time` option to the Iterator.
+- [Fix] Fix a case where `Admin#read_topic` would wait for poll interval on non-existing messages instead of early exit.
+- [Fix] Fix a case where Iterator with per partition offsets with negative lookups would go below the number of available messages.
+- [Fix] Remove unused constant from Admin module.
+- [Fix] Add missing `connection.client.rebalance_callback.error` to the `LoggerListener` instrumentation hook.
+
 ## 2.1.6 (2023-06-29)
 - [Improvement] Provide time support for iterator
 - [Improvement] Provide time support for admin `#read_topic`
@@ -63,7 +71,7 @@
 2. Replace `Karafka::Pro::BaseConsumer` references to `Karafka::BaseConsumer`.
 3. Replace `Karafka::Instrumentation::Vendors::Datadog:Listener` with `Karafka::Instrumentation::Vendors::Datadog::MetricsListener`.
 
-## 2.0.41 (2023-14-19)
+## 2.0.41 (2023-04-19)
 - **[Feature]** Provide `Karafka::Pro::Iterator` for anonymous topic/partitions iterations and messages lookups (#1389 and #1427).
 - [Improvement] Optimize topic lookup for `read_topic` admin method usage.
 - [Improvement] Report via `LoggerListener` information about the partition on which a given job has started and finished.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.1.6)
+    karafka (2.1.7)
       karafka-core (>= 2.1.1, < 2.2.0)
       thor (>= 0.20)
       waterdrop (>= 2.6.2, < 3.0.0)
@@ -10,10 +10,10 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activejob (7.0.5)
-      activesupport (= 7.0.5)
+    activejob (7.0.6)
+      activesupport (= 7.0.6)
       globalid (>= 0.3.6)
-    activesupport (7.0.5)
+    activesupport (7.0.6)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -33,7 +33,7 @@ GEM
     karafka-core (2.1.1)
       concurrent-ruby (>= 1.1)
       karafka-rdkafka (>= 0.13.1, < 0.14.0)
-    karafka-rdkafka (0.13.1)
+    karafka-rdkafka (0.13.3)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
@@ -47,7 +47,7 @@ GEM
     minitest (5.18.1)
     rack (3.0.8)
     rake (13.0.6)
-    roda (3.69.0)
+    roda (3.70.0)
       rack
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
@@ -58,10 +58,10 @@ GEM
     rspec-expectations (3.12.3)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
-    rspec-mocks (3.12.5)
+    rspec-mocks (3.12.6)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
-    rspec-support (3.12.0)
+    rspec-support (3.12.1)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
@@ -72,8 +72,8 @@ GEM
     tilt (2.2.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.6.2)
-      karafka-core (>= 2.1.0, < 3.0.0)
+    waterdrop (2.6.4)
+      karafka-core (>= 2.1.1, < 3.0.0)
       zeitwerk (~> 2.3)
     zeitwerk (2.6.8)
 

data/lib/karafka/admin.rb

@@ -9,11 +9,6 @@ module Karafka
   # @note It always uses the primary defined cluster and does not support multi-cluster work.
   #   If you need this, just replace the cluster info for the time you use this
   module Admin
-    # A fake admin topic representation that we use for messages fetched using this API
-    # We cannot use the topics directly because we may want to request data from topics that we
-    # do not have in the routing
-    Topic = Struct.new(:name, :deserializer)
-
     # We wait only for this amount of time before raising error as we intercept this error and
     # retry after checking that the operation was finished or failed using external factor.
     MAX_WAIT_TIMEOUT = 1
@@ -37,7 +32,7 @@ module Karafka
       'enable.auto.commit': false
     }.freeze
 
-    private_constant :Topic, :CONFIG_DEFAULTS, :MAX_WAIT_TIMEOUT, :TPL_REQUEST_TIMEOUT,
+    private_constant :CONFIG_DEFAULTS, :MAX_WAIT_TIMEOUT, :TPL_REQUEST_TIMEOUT,
                      :MAX_ATTEMPTS
 
     class << self
@@ -71,7 +66,7 @@ module Karafka
           requested_range = (start_offset..start_offset + (count - 1))
           # Establish theoretical available range. Note, that this does not handle cases related to
           # log retention or compaction
-          available_range = (low_offset..high_offset)
+          available_range = (low_offset..(high_offset - 1))
           # Select only offset that we can select. This will remove all the potential offsets that
           # are below the low watermark offset
           possible_range = requested_range.select { |offset| available_range.include?(offset) }

data/lib/karafka/instrumentation/logger_listener.rb

@@ -277,6 +277,9 @@ module Karafka
         when 'connection.client.poll.error'
           error "Data polling error occurred: #{error}"
           error details
+        when 'connection.client.rebalance_callback.error'
+          error "Rebalance callback error occurred: #{error}"
+          error details
         else
           # This should never happen. Please contact the maintainers
           raise Errors::UnsupportedCaseError, event

data/lib/karafka/pro/iterator/tpl_builder.rb

@@ -93,12 +93,27 @@ module Karafka
             next unless partitions.is_a?(Hash)
 
             partitions.each do |partition, offset|
+              # Care only about numerical offsets
+              #
+              # For time based we already resolve them via librdkafka lookup API
+              next unless offset.is_a?(Integer)
+
+              low_offset, high_offset = @consumer.query_watermark_offsets(name, partition)
+
               # Care only about negative offsets (last n messages)
-              next unless offset.is_a?(Integer) && offset.negative?
+              #
+              # We reject the above results but we **NEED** to run the `#query_watermark_offsets`
+              # for each topic partition nonetheless. Without this, librdkafka fetches a lot more
+              # metadata about each topic and each partition and this takes much more time than
+              # just getting watermarks. If we do not run watermark, at least an extra second
+              # is added at the beginning of iterator flow
+              #
+              # This may not be significant when this runs in the background but in case of
+              # using iterator in thins like Puma, it heavily impacts the end user experience
+              next unless offset.negative?
 
-              _, high_watermark_offset = @consumer.query_watermark_offsets(name, partition)
               # We add because this offset is negative
-              @mapped_topics[name][partition] = high_watermark_offset + offset
+              @mapped_topics[name][partition] = [high_offset + offset, low_offset].max
             end
           end
         end

data/lib/karafka/pro/iterator.rb

@@ -39,6 +39,7 @@ module Karafka
       #   overwritten, you may want to include `auto.offset.reset` to match your case.
       # @param yield_nil [Boolean] should we yield also `nil` values when poll returns nothing.
       #   Useful in particular for long-living iterators.
+      # @param max_wait_time [Integer] max wait in ms when iterator did not receive any messages
       #
       # @note It is worth keeping in mind, that this API also needs to operate within
       #   `max.poll.interval.ms` limitations on each iteration
@@ -48,7 +49,8 @@ module Karafka
       def initialize(
         topics,
         settings: { 'auto.offset.reset': 'beginning' },
-        yield_nil: false
+        yield_nil: false,
+        max_wait_time: 200
       )
         @topics_with_partitions = Expander.new.call(topics)
 
@@ -62,6 +64,7 @@ module Karafka
 
         @settings = settings
         @yield_nil = yield_nil
+        @max_wait_time = max_wait_time
       end
 
       # Iterates over requested topic partitions and yields the results with the iterator itself
@@ -80,7 +83,7 @@ module Karafka
           # Stream data until we reach the end of all the partitions or until the end user
           # indicates that they are done
           until done?
-            message = poll(200)
+            message = poll
 
             # Skip nils if not explicitly required
             next if message.nil? && !@yield_nil
@@ -131,10 +134,9 @@ module Karafka
 
       private
 
-      # @param timeout [Integer] timeout in ms
       # @return [Rdkafka::Consumer::Message, nil] message or nil if nothing to do
-      def poll(timeout)
-        @current_consumer.poll(timeout)
+      def poll
+        @current_consumer.poll(@max_wait_time)
       rescue Rdkafka::RdkafkaError => e
         # End of partition
         if e.code == :partition_eof

data/lib/karafka/setup/attributes_map.rb

@@ -25,6 +25,7 @@ module Karafka
         broker.version.fallback
         builtin.features
         check.crcs
+        client.dns.lookup
         client.id
         client.rack
         closesocket_cb
@@ -161,6 +162,7 @@ module Karafka
         broker.address.ttl
         broker.version.fallback
         builtin.features
+        client.dns.lookup
         client.id
         client.rack
         closesocket_cb

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.1.6'
+  VERSION = '2.1.7'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.1.6
+  version: 2.1.7
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2023-06-29 00:00:00.000000000 Z
+date: 2023-07-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core