karafka 2.1.5 → 2.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 82a8b28b55f1db0808be3d1e48616f6b2389466332c9116e263e37cab992fc65
-  data.tar.gz: 2f29bb9bb1c3f949d206c5c8453b35ad163219babb48687e2270e13914e78aba
+  metadata.gz: f405521c7a6706cc95e764a4740e7570935f7595d34481bbe33fb617e5537978
+  data.tar.gz: cd6671c441c07e31050bbddab290ba4d31e4a580a646cfd965edf58c19ff150c
 SHA512:
-  metadata.gz: 93a66f4aeb49cea810bfd90cf424b3334d1dae992035e0bd9613bbd3c42f642f94fd0efd979d57df5083a46f66f522a7d3952c9e24340b8a4dc4c23aff165a0f
-  data.tar.gz: 4ee03b442b3029aecf0ffd636ddccb054e51f2a448c3dd642993464bfc32aa45595f26835db8a9b5b01940ab5b532e0bc22a9a3cdbcc9899320b55010473c749
+  metadata.gz: 7b5e343a0d2c6e1f885c6eac6509de2f411b54e1a30ce12fac6fa18bb813d82ef666444345b92d8348ac4955cdabfc47ad3658312482f6c500ca169814f10517
+  data.tar.gz: 1b0c319f85dde3bc20b21a842da220d513351b436b3e4de08d56e69a02c36c7c2cd4187c879596ffc73f5dffc2cc3f032c6a8cdbd958ce34138866d27aa00b2b

checksums.yaml.gz.sig

@@ -1,2 +1,2 @@
-\��2F`��i��F��@��L�;�)�*���_S
OT':,��r ��v�i��_�B�݈:�-��?-ױ�^6��.�/�</A�������1�.'�c�X����w�}���3?����5t��ū��)X��.�9&؝=�����?A�'upm���F2Ȟ��۞����fb�R�������P��v_��c�.�=֧��+_uߨHJfԳt@7S���p�<�[{�
1�������u(��Iη!�0\#HPp�AJ����W��X_�y���(o��𔒋�Wp������~�1ٌ�����)�fqb�,+�u�2��d^_ay4����ߺĵG ��`۪�t�z�:+��nN�|Mk`�vX��
-�->,��SW
+�9�$��<�8��5��O��/ �<�_�:)*τޜ��|�!�s�}���6�5�[�B_��E���s��;k���'Z�p��'Ŕ������${�j�.P������/}0�`T��+��V�>Ez1��
+�

data/CHANGELOG.md

@@ -1,5 +1,30 @@
 # Karafka framework changelog
 
+## 2.1.7 (2023-07-22)
+- [Improvement] Always query for watermarks in the Iterator to improve the initial response time.
+- [Improvement] Add `max_wait_time` option to the Iterator.
+- [Fix] Fix a case where `Admin#read_topic` would wait for poll interval on non-existing messages instead of early exit.
+- [Fix] Fix a case where Iterator with per partition offsets with negative lookups would go below the number of available messages.
+- [Fix] Remove unused constant from Admin module.
+- [Fix] Add missing `connection.client.rebalance_callback.error` to the `LoggerListener` instrumentation hook.
+
+## 2.1.6 (2023-06-29)
+- [Improvement] Provide time support for iterator
+- [Improvement] Provide time support for admin `#read_topic`
+- [Improvement] Provide time support for consumer `#seek`.
+- [Improvement] Remove no longer needed locks for client operations.
+- [Improvement] Raise `Karafka::Errors::TopicNotFoundError` when trying to iterate over non-existing topic.
+- [Improvement] Ensure that Kafka multi-command operations run under mutex together.
+- [Change] Require `waterdrop` `>= 2.6.2`
+- [Change] Require `karafka-core` `>= 2.1.1`
+- [Refactor] Clean-up iterator code.
+- [Fix]  Improve performance in dev environment for a Rails app (juike)
+- [Fix] Rename `InvalidRealOffsetUsage` to `InvalidRealOffsetUsageError` to align with naming of other errors.
+- [Fix] Fix unstable spec.
+- [Fix] Fix a case where automatic `#seek` would overwrite manual seek of a user when running LRJ.
+- [Fix] Make sure, that user direct `#seek` and `#pause` operations take precedence over system actions.
+- [Fix] Make sure, that `#pause` and `#resume` with one underlying connection do not race-condition.
+
 ## 2.1.5 (2023-06-19)
 - [Improvement] Drastically improve `#revoked?` response quality by checking the real time assignment lost state on librdkafka.
 - [Improvement] Improve eviction of saturated jobs that would run on already revoked assignments.
@@ -46,7 +71,7 @@
 2. Replace `Karafka::Pro::BaseConsumer` references to `Karafka::BaseConsumer`.
 3. Replace `Karafka::Instrumentation::Vendors::Datadog:Listener` with `Karafka::Instrumentation::Vendors::Datadog::MetricsListener`.
 
-## 2.0.41 (2023-14-19)
+## 2.0.41 (2023-04-19)
 - **[Feature]** Provide `Karafka::Pro::Iterator` for anonymous topic/partitions iterations and messages lookups (#1389 and #1427).
 - [Improvement] Optimize topic lookup for `read_topic` admin method usage.
 - [Improvement] Report via `LoggerListener` information about the partition on which a given job has started and finished.

data/Gemfile.lock

@@ -1,19 +1,19 @@
 PATH
   remote: .
   specs:
-    karafka (2.1.5)
-      karafka-core (>= 2.1.0, < 2.2.0)
+    karafka (2.1.7)
+      karafka-core (>= 2.1.1, < 2.2.0)
       thor (>= 0.20)
-      waterdrop (>= 2.6.1, < 3.0.0)
+      waterdrop (>= 2.6.2, < 3.0.0)
       zeitwerk (~> 2.3)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activejob (7.0.5)
-      activesupport (= 7.0.5)
+    activejob (7.0.6)
+      activesupport (= 7.0.6)
       globalid (>= 0.3.6)
-    activesupport (7.0.5)
+    activesupport (7.0.6)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -30,14 +30,14 @@ GEM
       activesupport (>= 5.0)
     i18n (1.14.1)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.1.0)
+    karafka-core (2.1.1)
       concurrent-ruby (>= 1.1)
-      karafka-rdkafka (>= 0.13.0, < 0.14.0)
-    karafka-rdkafka (0.13.0)
+      karafka-rdkafka (>= 0.13.1, < 0.14.0)
+    karafka-rdkafka (0.13.3)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    karafka-web (0.6.0)
+    karafka-web (0.6.1)
       erubi (~> 1.4)
       karafka (>= 2.1.4, < 3.0.0)
       karafka-core (>= 2.0.13, < 3.0.0)
@@ -47,7 +47,7 @@ GEM
     minitest (5.18.1)
     rack (3.0.8)
     rake (13.0.6)
-    roda (3.69.0)
+    roda (3.70.0)
       rack
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
@@ -58,10 +58,10 @@ GEM
     rspec-expectations (3.12.3)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
-    rspec-mocks (3.12.5)
+    rspec-mocks (3.12.6)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
-    rspec-support (3.12.0)
+    rspec-support (3.12.1)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
@@ -72,8 +72,8 @@ GEM
     tilt (2.2.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.6.1)
-      karafka-core (>= 2.1.0, < 3.0.0)
+    waterdrop (2.6.4)
+      karafka-core (>= 2.1.1, < 3.0.0)
       zeitwerk (~> 2.3)
     zeitwerk (2.6.8)
 

data/karafka.gemspec

@@ -21,9 +21,9 @@ Gem::Specification.new do |spec|
     without having to focus on things that are not your business domain.
   DESC
 
-  spec.add_dependency 'karafka-core', '>= 2.1.0', '< 2.2.0'
+  spec.add_dependency 'karafka-core', '>= 2.1.1', '< 2.2.0'
   spec.add_dependency 'thor', '>= 0.20'
-  spec.add_dependency 'waterdrop', '>= 2.6.1', '< 3.0.0'
+  spec.add_dependency 'waterdrop', '>= 2.6.2', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
   if $PROGRAM_NAME.end_with?('gem')

data/lib/karafka/admin.rb

@@ -9,15 +9,13 @@ module Karafka
   # @note It always uses the primary defined cluster and does not support multi-cluster work.
   #   If you need this, just replace the cluster info for the time you use this
   module Admin
-    # A fake admin topic representation that we use for messages fetched using this API
-    # We cannot use the topics directly because we may want to request data from topics that we
-    # do not have in the routing
-    Topic = Struct.new(:name, :deserializer)
-
     # We wait only for this amount of time before raising error as we intercept this error and
     # retry after checking that the operation was finished or failed using external factor.
     MAX_WAIT_TIMEOUT = 1
 
+    # Max time for a TPL request. We increase it to compensate for remote clusters latency
+    TPL_REQUEST_TIMEOUT = 2_000
+
     # How many times should be try. 1 x 60 => 60 seconds wait in total
     MAX_ATTEMPTS = 60
 
@@ -34,7 +32,8 @@ module Karafka
       'enable.auto.commit': false
     }.freeze
 
-    private_constant :Topic, :CONFIG_DEFAULTS, :MAX_WAIT_TIMEOUT, :MAX_ATTEMPTS
+    private_constant :CONFIG_DEFAULTS, :MAX_WAIT_TIMEOUT, :TPL_REQUEST_TIMEOUT,
+                     :MAX_ATTEMPTS
 
     class << self
       # Allows us to read messages from the topic
@@ -42,8 +41,9 @@ module Karafka
       # @param name [String, Symbol] topic name
       # @param partition [Integer] partition
       # @param count [Integer] how many messages we want to get at most
-      # @param start_offset [Integer] offset from which we should start. If -1 is provided
-      #   (default) we will start from the latest offset
+      # @param start_offset [Integer, Time] offset from which we should start. If -1 is provided
+      #   (default) we will start from the latest offset. If time is provided, the appropriate
+      #   offset will be resolved.
       # @param settings [Hash] kafka extra settings (optional)
       #
       # @return [Array<Karafka::Messages::Message>] array with messages
@@ -53,6 +53,9 @@ module Karafka
         low_offset, high_offset = nil
 
         with_consumer(settings) do |consumer|
+          # Convert the time offset (if needed)
+          start_offset = resolve_offset(consumer, name.to_s, partition, start_offset)
+
           low_offset, high_offset = consumer.query_watermark_offsets(name, partition)
 
           # Select offset dynamically if -1 or less
@@ -63,7 +66,7 @@ module Karafka
           requested_range = (start_offset..start_offset + (count - 1))
           # Establish theoretical available range. Note, that this does not handle cases related to
           # log retention or compaction
-          available_range = (low_offset..high_offset)
+          available_range = (low_offset..(high_offset - 1))
           # Select only offset that we can select. This will remove all the potential offsets that
           # are below the low watermark offset
           possible_range = requested_range.select { |offset| available_range.include?(offset) }
@@ -243,6 +246,29 @@ module Karafka
 
         ::Rdkafka::Config.new(config_hash)
       end
+
+      # Resolves the offset if offset is in a time format. Otherwise returns the offset without
+      # resolving.
+      # @param consumer [::Rdkafka::Consumer]
+      # @param name [String, Symbol] expected topic name
+      # @param partition [Integer]
+      # @param offset [Integer, Time]
+      # @return [Integer] expected offset
+      def resolve_offset(consumer, name, partition, offset)
+        if offset.is_a?(Time)
+          tpl = ::Rdkafka::Consumer::TopicPartitionList.new
+          tpl.add_topic_and_partitions_with_offsets(
+            name, partition => offset
+          )
+
+          real_offsets = consumer.offsets_for_times(tpl, TPL_REQUEST_TIMEOUT)
+          detected_offset = real_offsets.to_h.dig(name, partition)
+
+          detected_offset&.offset || raise(Errors::InvalidTimeBasedOffsetError)
+        else
+          offset
+        end
+      end
     end
   end
 end

data/lib/karafka/base_consumer.rb

@@ -70,6 +70,7 @@ module Karafka
     #
     # @return [Boolean] true if there was no exception, otherwise false.
     #
+    # @private
     # @note We keep the seek offset tracking, and use it to compensate for async offset flushing
     #   that may not yet kick in when error occurs. That way we pause always on the last processed
     #   message.
@@ -203,8 +204,15 @@ module Karafka
 
     # Seeks in the context of current topic and partition
     #
-    # @param offset [Integer] offset where we want to seek
-    def seek(offset)
+    # @param offset [Integer, Time] offset where we want to seek or time of the offset where we
+    #   want to seek.
+    # @param manual_seek [Boolean] Flag to differentiate between user seek and system/strategy
+    #   based seek. User seek operations should take precedence over system actions, hence we need
+    #   to know who invoked it.
+    # @note Please note, that if you are seeking to a time offset, getting the offset is blocking
+    def seek(offset, manual_seek = true)
+      coordinator.manual_seek if manual_seek
+
       client.seek(
         Karafka::Messages::Seek.new(
           topic.name,

data/lib/karafka/connection/client.rb

@@ -20,11 +20,14 @@ module Karafka
       # How many times should we retry polling in case of a failure
       MAX_POLL_RETRIES = 20
 
+      # Max time for a TPL request. We increase it to compensate for remote clusters latency
+      TPL_REQUEST_TIMEOUT = 2_000
+
       # We want to make sure we never close several clients in the same moment to prevent
       # potential race conditions and other issues
       SHUTDOWN_MUTEX = Mutex.new
 
-      private_constant :MAX_POLL_RETRIES, :SHUTDOWN_MUTEX
+      private_constant :MAX_POLL_RETRIES, :SHUTDOWN_MUTEX, :TPL_REQUEST_TIMEOUT
 
       # Creates a new consumer instance.
       #
@@ -35,12 +38,16 @@ module Karafka
         @id = SecureRandom.hex(6)
         # Name is set when we build consumer
         @name = ''
-        @mutex = Mutex.new
         @closed = false
         @subscription_group = subscription_group
         @buffer = RawMessagesBuffer.new
         @rebalance_manager = RebalanceManager.new
         @kafka = build_consumer
+        # There are few operations that can happen in parallel from the listener threads as well
+        # as from the workers. They are not fully thread-safe because they may be composed out of
+        # few calls to Kafka or out of few internal state changes. That is why we mutex them.
+        # It mostly revolves around pausing and resuming.
+        @mutex = Mutex.new
         # We need to keep track of what we have paused for resuming
         # In case we loose partition, we still need to resume it, otherwise it won't be fetched
         # again if we get reassigned to it later on. We need to keep them as after revocation we
@@ -101,16 +108,12 @@ module Karafka
       #
       # @param message [Karafka::Messages::Message]
       def store_offset(message)
-        @mutex.synchronize do
-          internal_store_offset(message)
-        end
+        internal_store_offset(message)
       end
 
       # @return [Boolean] true if our current assignment has been lost involuntarily.
       def assignment_lost?
-        @mutex.synchronize do
-          @kafka.assignment_lost?
-        end
+        @kafka.assignment_lost?
       end
 
       # Commits the offset on a current consumer in a non-blocking or blocking way.
@@ -127,11 +130,7 @@ module Karafka
       #   it does **not** resolve to `lost_assignment?`. It returns only the commit state operation
       #   result.
       def commit_offsets(async: true)
-        @mutex.lock
-
         internal_commit_offsets(async: async)
-      ensure
-        @mutex.unlock
       end
 
       # Commits offset in a synchronous way.
@@ -144,13 +143,11 @@ module Karafka
       # Seek to a particular message. The next poll on the topic/partition will return the
       # message at the given offset.
       #
-      # @param message [Messages::Message, Messages::Seek] message to which we want to seek to
+      # @param message [Messages::Message, Messages::Seek] message to which we want to seek to.
+      #   It can have the time based offset.
+      # @note Please note, that if you are seeking to a time offset, getting the offset is blocking
       def seek(message)
-        @mutex.lock
-
-        @kafka.seek(message)
-      ensure
-        @mutex.unlock
+        @mutex.synchronize { internal_seek(message) }
       end
 
       # Pauses given partition and moves back to last successful offset processed.
@@ -161,37 +158,34 @@ module Karafka
       #   be reprocessed after getting back to processing)
       # @note This will pause indefinitely and requires manual `#resume`
       def pause(topic, partition, offset)
-        @mutex.lock
-
-        # Do not pause if the client got closed, would not change anything
-        return if @closed
-
-        pause_msg = Messages::Seek.new(topic, partition, offset)
+        @mutex.synchronize do
+          # Do not pause if the client got closed, would not change anything
+          return if @closed
 
-        internal_commit_offsets(async: true)
+          pause_msg = Messages::Seek.new(topic, partition, offset)
 
-        # Here we do not use our cached tpls because we should not try to pause something we do
-        # not own anymore.
-        tpl = topic_partition_list(topic, partition)
+          internal_commit_offsets(async: true)
 
-        return unless tpl
+          # Here we do not use our cached tpls because we should not try to pause something we do
+          # not own anymore.
+          tpl = topic_partition_list(topic, partition)
 
-        Karafka.monitor.instrument(
-          'client.pause',
-          caller: self,
-          subscription_group: @subscription_group,
-          topic: topic,
-          partition: partition,
-          offset: offset
-        )
+          return unless tpl
 
-        @paused_tpls[topic][partition] = tpl
+          Karafka.monitor.instrument(
+            'client.pause',
+            caller: self,
+            subscription_group: @subscription_group,
+            topic: topic,
+            partition: partition,
+            offset: offset
+          )
 
-        @kafka.pause(tpl)
+          @paused_tpls[topic][partition] = tpl
 
-        @kafka.seek(pause_msg)
-      ensure
-        @mutex.unlock
+          @kafka.pause(tpl)
+          internal_seek(pause_msg)
+        end
       end
 
       # Resumes processing of a give topic partition after it was paused.
@@ -199,33 +193,31 @@ module Karafka
       # @param topic [String] topic name
       # @param partition [Integer] partition
       def resume(topic, partition)
-        @mutex.lock
-
-        return if @closed
+        @mutex.synchronize do
+          return if @closed
 
-        # We now commit offsets on rebalances, thus we can do it async just to make sure
-        internal_commit_offsets(async: true)
+          # We now commit offsets on rebalances, thus we can do it async just to make sure
+          internal_commit_offsets(async: true)
 
-        # If we were not able, let's try to reuse the one we have (if we have)
-        tpl = topic_partition_list(topic, partition) || @paused_tpls[topic][partition]
+          # If we were not able, let's try to reuse the one we have (if we have)
+          tpl = topic_partition_list(topic, partition) || @paused_tpls[topic][partition]
 
-        return unless tpl
+          return unless tpl
 
-        # If we did not have it, it means we never paused this partition, thus no resume should
-        # happen in the first place
-        return unless @paused_tpls[topic].delete(partition)
+          # If we did not have it, it means we never paused this partition, thus no resume should
+          # happen in the first place
+          return unless @paused_tpls[topic].delete(partition)
 
-        Karafka.monitor.instrument(
-          'client.resume',
-          caller: self,
-          subscription_group: @subscription_group,
-          topic: topic,
-          partition: partition
-        )
+          Karafka.monitor.instrument(
+            'client.resume',
+            caller: self,
+            subscription_group: @subscription_group,
+            topic: topic,
+            partition: partition
+          )
 
-        @kafka.resume(tpl)
-      ensure
-        @mutex.unlock
+          @kafka.resume(tpl)
+        end
       end
 
       # Gracefully stops topic consumption.
@@ -262,11 +254,9 @@ module Karafka
       def reset
         close
 
-        @mutex.synchronize do
-          @closed = false
-          @paused_tpls.clear
-          @kafka = build_consumer
-        end
+        @closed = false
+        @paused_tpls.clear
+        @kafka = build_consumer
       end
 
       # Runs a single poll ignoring all the potential errors
@@ -323,28 +313,55 @@ module Karafka
         raise e
       end
 
+      # Non-mutexed seek that should be used only internally. Outside we expose `#seek` that is
+      # wrapped with a mutex.
+      #
+      # @param message [Messages::Message, Messages::Seek] message to which we want to seek to.
+      #   It can have the time based offset.
+      def internal_seek(message)
+        # If the seek message offset is in a time format, we need to find the closest "real"
+        # offset matching before we seek
+        if message.offset.is_a?(Time)
+          tpl = ::Rdkafka::Consumer::TopicPartitionList.new
+          tpl.add_topic_and_partitions_with_offsets(
+            message.topic,
+            message.partition => message.offset
+          )
+
+          # Now we can overwrite the seek message offset with our resolved offset and we can
+          # then seek to the appropriate message
+          # We set the timeout to 2_000 to make sure that remote clusters handle this well
+          real_offsets = @kafka.offsets_for_times(tpl, TPL_REQUEST_TIMEOUT)
+          detected_partition = real_offsets.to_h.dig(message.topic, message.partition)
+
+          # There always needs to be an offset. In case we seek into the future, where there
+          # are no offsets yet, we get -1 which indicates the most recent offset
+          # We should always detect offset, whether it is 0, -1 or a corresponding
+          message.offset = detected_partition&.offset || raise(Errors::InvalidTimeBasedOffsetError)
+        end
+
+        @kafka.seek(message)
+      end
+
       # Commits the stored offsets in a sync way and closes the consumer.
       def close
         # Allow only one client to be closed at the same time
         SHUTDOWN_MUTEX.synchronize do
-          # Make sure that no other operations are happening on this client when we close it
-          @mutex.synchronize do
-            # Once client is closed, we should not close it again
-            # This could only happen in case of a race-condition when forceful shutdown happens
-            # and triggers this from a different thread
-            return if @closed
-
-            @closed = true
-
-            # Remove callbacks runners that were registered
-            ::Karafka::Core::Instrumentation.statistics_callbacks.delete(@subscription_group.id)
-            ::Karafka::Core::Instrumentation.error_callbacks.delete(@subscription_group.id)
-
-            @kafka.close
-            @buffer.clear
-            # @note We do not clear rebalance manager here as we may still have revocation info
-            # here that we want to consider valid prior to running another reconnection
-          end
+          # Once client is closed, we should not close it again
+          # This could only happen in case of a race-condition when forceful shutdown happens
+          # and triggers this from a different thread
+          return if @closed
+
+          @closed = true
+
+          # Remove callbacks runners that were registered
+          ::Karafka::Core::Instrumentation.statistics_callbacks.delete(@subscription_group.id)
+          ::Karafka::Core::Instrumentation.error_callbacks.delete(@subscription_group.id)
+
+          @kafka.close
+          @buffer.clear
+          # @note We do not clear rebalance manager here as we may still have revocation info
+          # here that we want to consider valid prior to running another reconnection
         end
       end
 

data/lib/karafka/errors.rb

@@ -48,6 +48,9 @@ module Karafka
     StrategyNotFoundError = Class.new(BaseError)
 
     # This should never happen. Please open an issue if it does.
-    InvalidRealOffsetUsage = Class.new(BaseError)
+    InvalidRealOffsetUsageError = Class.new(BaseError)
+
+    # This should never happen. Please open an issue if it does.
+    InvalidTimeBasedOffsetError = Class.new(BaseError)
   end
 end

data/lib/karafka/instrumentation/logger_listener.rb

@@ -277,6 +277,9 @@ module Karafka
         when 'connection.client.poll.error'
           error "Data polling error occurred: #{error}"
           error details
+        when 'connection.client.rebalance_callback.error'
+          error "Rebalance callback error occurred: #{error}"
+          error details
         else
           # This should never happen. Please contact the maintainers
           raise Errors::UnsupportedCaseError, event

data/lib/karafka/messages/seek.rb

@@ -4,6 +4,9 @@ module Karafka
   module Messages
     # "Fake" message that we use as an abstraction layer when seeking back.
     # This allows us to encapsulate a seek with a simple abstraction
+    #
+    # @note `#offset` can be either the offset value or the time of the offset
+    # (first equal or greater)
     Seek = Struct.new(:topic, :partition, :offset)
   end
 end