karafka 2.1.5 → 2.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 82a8b28b55f1db0808be3d1e48616f6b2389466332c9116e263e37cab992fc65
-  data.tar.gz: 2f29bb9bb1c3f949d206c5c8453b35ad163219babb48687e2270e13914e78aba
+  metadata.gz: a6994a6d579728a877f84c87086d093aae8a1f830b891fcb4904883085432fe4
+  data.tar.gz: 13b21009a471194a72971ca81ddc718e044bb96587db0e8f186974f554e9ec62
 SHA512:
-  metadata.gz: 93a66f4aeb49cea810bfd90cf424b3334d1dae992035e0bd9613bbd3c42f642f94fd0efd979d57df5083a46f66f522a7d3952c9e24340b8a4dc4c23aff165a0f
-  data.tar.gz: 4ee03b442b3029aecf0ffd636ddccb054e51f2a448c3dd642993464bfc32aa45595f26835db8a9b5b01940ab5b532e0bc22a9a3cdbcc9899320b55010473c749
+  metadata.gz: e4711880bde1d2cd1cb34959f740459979b74ff4d28a671a232f88adbe7473cf67e366fc2b492fac761c572f3a6dfc147a59d46fc08e1c5e18df8ac5f108afdd
+  data.tar.gz: c094600c2bd421ce309c0125d60ea82ed0106d5ce4566b3bb8c1aab13c553e7bd2f6651b98029e42ac831b132563b2c502dd1c76defbf8307cd9bd2393b258f7

data/CHANGELOG.md

@@ -1,5 +1,22 @@
 # Karafka framework changelog
 
+## 2.1.6 (2023-06-29)
+- [Improvement] Provide time support for iterator
+- [Improvement] Provide time support for admin `#read_topic`
+- [Improvement] Provide time support for consumer `#seek`.
+- [Improvement] Remove no longer needed locks for client operations.
+- [Improvement] Raise `Karafka::Errors::TopicNotFoundError` when trying to iterate over non-existing topic.
+- [Improvement] Ensure that Kafka multi-command operations run under mutex together.
+- [Change] Require `waterdrop` `>= 2.6.2`
+- [Change] Require `karafka-core` `>= 2.1.1`
+- [Refactor] Clean-up iterator code.
+- [Fix]  Improve performance in dev environment for a Rails app (juike)
+- [Fix] Rename `InvalidRealOffsetUsage` to `InvalidRealOffsetUsageError` to align with naming of other errors.
+- [Fix] Fix unstable spec.
+- [Fix] Fix a case where automatic `#seek` would overwrite manual seek of a user when running LRJ.
+- [Fix] Make sure, that user direct `#seek` and `#pause` operations take precedence over system actions.
+- [Fix] Make sure, that `#pause` and `#resume` with one underlying connection do not race-condition.
+
 ## 2.1.5 (2023-06-19)
 - [Improvement] Drastically improve `#revoked?` response quality by checking the real time assignment lost state on librdkafka.
 - [Improvement] Improve eviction of saturated jobs that would run on already revoked assignments.

data/Gemfile.lock

@@ -1,10 +1,10 @@
 PATH
   remote: .
   specs:
-    karafka (2.1.5)
-      karafka-core (>= 2.1.0, < 2.2.0)
+    karafka (2.1.6)
+      karafka-core (>= 2.1.1, < 2.2.0)
       thor (>= 0.20)
-      waterdrop (>= 2.6.1, < 3.0.0)
+      waterdrop (>= 2.6.2, < 3.0.0)
       zeitwerk (~> 2.3)
 
 GEM
@@ -30,14 +30,14 @@ GEM
       activesupport (>= 5.0)
     i18n (1.14.1)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.1.0)
+    karafka-core (2.1.1)
       concurrent-ruby (>= 1.1)
-      karafka-rdkafka (>= 0.13.0, < 0.14.0)
-    karafka-rdkafka (0.13.0)
+      karafka-rdkafka (>= 0.13.1, < 0.14.0)
+    karafka-rdkafka (0.13.1)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    karafka-web (0.6.0)
+    karafka-web (0.6.1)
       erubi (~> 1.4)
       karafka (>= 2.1.4, < 3.0.0)
       karafka-core (>= 2.0.13, < 3.0.0)
@@ -72,7 +72,7 @@ GEM
     tilt (2.2.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.6.1)
+    waterdrop (2.6.2)
       karafka-core (>= 2.1.0, < 3.0.0)
       zeitwerk (~> 2.3)
     zeitwerk (2.6.8)

data/karafka.gemspec

@@ -21,9 +21,9 @@ Gem::Specification.new do |spec|
     without having to focus on things that are not your business domain.
   DESC
 
-  spec.add_dependency 'karafka-core', '>= 2.1.0', '< 2.2.0'
+  spec.add_dependency 'karafka-core', '>= 2.1.1', '< 2.2.0'
   spec.add_dependency 'thor', '>= 0.20'
-  spec.add_dependency 'waterdrop', '>= 2.6.1', '< 3.0.0'
+  spec.add_dependency 'waterdrop', '>= 2.6.2', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
   if $PROGRAM_NAME.end_with?('gem')

data/lib/karafka/admin.rb

@@ -18,6 +18,9 @@ module Karafka
     # retry after checking that the operation was finished or failed using external factor.
     MAX_WAIT_TIMEOUT = 1
 
+    # Max time for a TPL request. We increase it to compensate for remote clusters latency
+    TPL_REQUEST_TIMEOUT = 2_000
+
     # How many times should be try. 1 x 60 => 60 seconds wait in total
     MAX_ATTEMPTS = 60
 
@@ -34,7 +37,8 @@ module Karafka
       'enable.auto.commit': false
     }.freeze
 
-    private_constant :Topic, :CONFIG_DEFAULTS, :MAX_WAIT_TIMEOUT, :MAX_ATTEMPTS
+    private_constant :Topic, :CONFIG_DEFAULTS, :MAX_WAIT_TIMEOUT, :TPL_REQUEST_TIMEOUT,
+                     :MAX_ATTEMPTS
 
     class << self
       # Allows us to read messages from the topic
@@ -42,8 +46,9 @@ module Karafka
       # @param name [String, Symbol] topic name
       # @param partition [Integer] partition
       # @param count [Integer] how many messages we want to get at most
-      # @param start_offset [Integer] offset from which we should start. If -1 is provided
-      #   (default) we will start from the latest offset
+      # @param start_offset [Integer, Time] offset from which we should start. If -1 is provided
+      #   (default) we will start from the latest offset. If time is provided, the appropriate
+      #   offset will be resolved.
       # @param settings [Hash] kafka extra settings (optional)
       #
       # @return [Array<Karafka::Messages::Message>] array with messages
@@ -53,6 +58,9 @@ module Karafka
         low_offset, high_offset = nil
 
         with_consumer(settings) do |consumer|
+          # Convert the time offset (if needed)
+          start_offset = resolve_offset(consumer, name.to_s, partition, start_offset)
+
           low_offset, high_offset = consumer.query_watermark_offsets(name, partition)
 
           # Select offset dynamically if -1 or less
@@ -243,6 +251,29 @@ module Karafka
 
         ::Rdkafka::Config.new(config_hash)
       end
+
+      # Resolves the offset if offset is in a time format. Otherwise returns the offset without
+      # resolving.
+      # @param consumer [::Rdkafka::Consumer]
+      # @param name [String, Symbol] expected topic name
+      # @param partition [Integer]
+      # @param offset [Integer, Time]
+      # @return [Integer] expected offset
+      def resolve_offset(consumer, name, partition, offset)
+        if offset.is_a?(Time)
+          tpl = ::Rdkafka::Consumer::TopicPartitionList.new
+          tpl.add_topic_and_partitions_with_offsets(
+            name, partition => offset
+          )
+
+          real_offsets = consumer.offsets_for_times(tpl, TPL_REQUEST_TIMEOUT)
+          detected_offset = real_offsets.to_h.dig(name, partition)
+
+          detected_offset&.offset || raise(Errors::InvalidTimeBasedOffsetError)
+        else
+          offset
+        end
+      end
     end
   end
 end

data/lib/karafka/base_consumer.rb

@@ -70,6 +70,7 @@ module Karafka
     #
     # @return [Boolean] true if there was no exception, otherwise false.
     #
+    # @private
     # @note We keep the seek offset tracking, and use it to compensate for async offset flushing
     #   that may not yet kick in when error occurs. That way we pause always on the last processed
     #   message.
@@ -203,8 +204,15 @@ module Karafka
 
     # Seeks in the context of current topic and partition
     #
-    # @param offset [Integer] offset where we want to seek
-    def seek(offset)
+    # @param offset [Integer, Time] offset where we want to seek or time of the offset where we
+    #   want to seek.
+    # @param manual_seek [Boolean] Flag to differentiate between user seek and system/strategy
+    #   based seek. User seek operations should take precedence over system actions, hence we need
+    #   to know who invoked it.
+    # @note Please note, that if you are seeking to a time offset, getting the offset is blocking
+    def seek(offset, manual_seek = true)
+      coordinator.manual_seek if manual_seek
+
       client.seek(
         Karafka::Messages::Seek.new(
           topic.name,

data/lib/karafka/connection/client.rb

@@ -20,11 +20,14 @@ module Karafka
       # How many times should we retry polling in case of a failure
       MAX_POLL_RETRIES = 20
 
+      # Max time for a TPL request. We increase it to compensate for remote clusters latency
+      TPL_REQUEST_TIMEOUT = 2_000
+
       # We want to make sure we never close several clients in the same moment to prevent
       # potential race conditions and other issues
       SHUTDOWN_MUTEX = Mutex.new
 
-      private_constant :MAX_POLL_RETRIES, :SHUTDOWN_MUTEX
+      private_constant :MAX_POLL_RETRIES, :SHUTDOWN_MUTEX, :TPL_REQUEST_TIMEOUT
 
       # Creates a new consumer instance.
       #
@@ -35,12 +38,16 @@ module Karafka
         @id = SecureRandom.hex(6)
         # Name is set when we build consumer
         @name = ''
-        @mutex = Mutex.new
         @closed = false
         @subscription_group = subscription_group
         @buffer = RawMessagesBuffer.new
         @rebalance_manager = RebalanceManager.new
         @kafka = build_consumer
+        # There are few operations that can happen in parallel from the listener threads as well
+        # as from the workers. They are not fully thread-safe because they may be composed out of
+        # few calls to Kafka or out of few internal state changes. That is why we mutex them.
+        # It mostly revolves around pausing and resuming.
+        @mutex = Mutex.new
         # We need to keep track of what we have paused for resuming
         # In case we loose partition, we still need to resume it, otherwise it won't be fetched
         # again if we get reassigned to it later on. We need to keep them as after revocation we
@@ -101,16 +108,12 @@ module Karafka
       #
       # @param message [Karafka::Messages::Message]
       def store_offset(message)
-        @mutex.synchronize do
-          internal_store_offset(message)
-        end
+        internal_store_offset(message)
       end
 
       # @return [Boolean] true if our current assignment has been lost involuntarily.
       def assignment_lost?
-        @mutex.synchronize do
-          @kafka.assignment_lost?
-        end
+        @kafka.assignment_lost?
       end
 
       # Commits the offset on a current consumer in a non-blocking or blocking way.
@@ -127,11 +130,7 @@ module Karafka
       #   it does **not** resolve to `lost_assignment?`. It returns only the commit state operation
       #   result.
       def commit_offsets(async: true)
-        @mutex.lock
-
         internal_commit_offsets(async: async)
-      ensure
-        @mutex.unlock
       end
 
       # Commits offset in a synchronous way.
@@ -144,13 +143,11 @@ module Karafka
       # Seek to a particular message. The next poll on the topic/partition will return the
       # message at the given offset.
       #
-      # @param message [Messages::Message, Messages::Seek] message to which we want to seek to
+      # @param message [Messages::Message, Messages::Seek] message to which we want to seek to.
+      #   It can have the time based offset.
+      # @note Please note, that if you are seeking to a time offset, getting the offset is blocking
       def seek(message)
-        @mutex.lock
-
-        @kafka.seek(message)
-      ensure
-        @mutex.unlock
+        @mutex.synchronize { internal_seek(message) }
       end
 
       # Pauses given partition and moves back to last successful offset processed.
@@ -161,37 +158,34 @@ module Karafka
       #   be reprocessed after getting back to processing)
       # @note This will pause indefinitely and requires manual `#resume`
       def pause(topic, partition, offset)
-        @mutex.lock
-
-        # Do not pause if the client got closed, would not change anything
-        return if @closed
-
-        pause_msg = Messages::Seek.new(topic, partition, offset)
+        @mutex.synchronize do
+          # Do not pause if the client got closed, would not change anything
+          return if @closed
 
-        internal_commit_offsets(async: true)
+          pause_msg = Messages::Seek.new(topic, partition, offset)
 
-        # Here we do not use our cached tpls because we should not try to pause something we do
-        # not own anymore.
-        tpl = topic_partition_list(topic, partition)
+          internal_commit_offsets(async: true)
 
-        return unless tpl
+          # Here we do not use our cached tpls because we should not try to pause something we do
+          # not own anymore.
+          tpl = topic_partition_list(topic, partition)
 
-        Karafka.monitor.instrument(
-          'client.pause',
-          caller: self,
-          subscription_group: @subscription_group,
-          topic: topic,
-          partition: partition,
-          offset: offset
-        )
+          return unless tpl
 
-        @paused_tpls[topic][partition] = tpl
+          Karafka.monitor.instrument(
+            'client.pause',
+            caller: self,
+            subscription_group: @subscription_group,
+            topic: topic,
+            partition: partition,
+            offset: offset
+          )
 
-        @kafka.pause(tpl)
+          @paused_tpls[topic][partition] = tpl
 
-        @kafka.seek(pause_msg)
-      ensure
-        @mutex.unlock
+          @kafka.pause(tpl)
+          internal_seek(pause_msg)
+        end
       end
 
       # Resumes processing of a give topic partition after it was paused.
@@ -199,33 +193,31 @@ module Karafka
       # @param topic [String] topic name
       # @param partition [Integer] partition
       def resume(topic, partition)
-        @mutex.lock
-
-        return if @closed
+        @mutex.synchronize do
+          return if @closed
 
-        # We now commit offsets on rebalances, thus we can do it async just to make sure
-        internal_commit_offsets(async: true)
+          # We now commit offsets on rebalances, thus we can do it async just to make sure
+          internal_commit_offsets(async: true)
 
-        # If we were not able, let's try to reuse the one we have (if we have)
-        tpl = topic_partition_list(topic, partition) || @paused_tpls[topic][partition]
+          # If we were not able, let's try to reuse the one we have (if we have)
+          tpl = topic_partition_list(topic, partition) || @paused_tpls[topic][partition]
 
-        return unless tpl
+          return unless tpl
 
-        # If we did not have it, it means we never paused this partition, thus no resume should
-        # happen in the first place
-        return unless @paused_tpls[topic].delete(partition)
+          # If we did not have it, it means we never paused this partition, thus no resume should
+          # happen in the first place
+          return unless @paused_tpls[topic].delete(partition)
 
-        Karafka.monitor.instrument(
-          'client.resume',
-          caller: self,
-          subscription_group: @subscription_group,
-          topic: topic,
-          partition: partition
-        )
+          Karafka.monitor.instrument(
+            'client.resume',
+            caller: self,
+            subscription_group: @subscription_group,
+            topic: topic,
+            partition: partition
+          )
 
-        @kafka.resume(tpl)
-      ensure
-        @mutex.unlock
+          @kafka.resume(tpl)
+        end
       end
 
       # Gracefully stops topic consumption.
@@ -262,11 +254,9 @@ module Karafka
       def reset
         close
 
-        @mutex.synchronize do
-          @closed = false
-          @paused_tpls.clear
-          @kafka = build_consumer
-        end
+        @closed = false
+        @paused_tpls.clear
+        @kafka = build_consumer
       end
 
       # Runs a single poll ignoring all the potential errors
@@ -323,28 +313,55 @@ module Karafka
         raise e
       end
 
+      # Non-mutexed seek that should be used only internally. Outside we expose `#seek` that is
+      # wrapped with a mutex.
+      #
+      # @param message [Messages::Message, Messages::Seek] message to which we want to seek to.
+      #   It can have the time based offset.
+      def internal_seek(message)
+        # If the seek message offset is in a time format, we need to find the closest "real"
+        # offset matching before we seek
+        if message.offset.is_a?(Time)
+          tpl = ::Rdkafka::Consumer::TopicPartitionList.new
+          tpl.add_topic_and_partitions_with_offsets(
+            message.topic,
+            message.partition => message.offset
+          )
+
+          # Now we can overwrite the seek message offset with our resolved offset and we can
+          # then seek to the appropriate message
+          # We set the timeout to 2_000 to make sure that remote clusters handle this well
+          real_offsets = @kafka.offsets_for_times(tpl, TPL_REQUEST_TIMEOUT)
+          detected_partition = real_offsets.to_h.dig(message.topic, message.partition)
+
+          # There always needs to be an offset. In case we seek into the future, where there
+          # are no offsets yet, we get -1 which indicates the most recent offset
+          # We should always detect offset, whether it is 0, -1 or a corresponding
+          message.offset = detected_partition&.offset || raise(Errors::InvalidTimeBasedOffsetError)
+        end
+
+        @kafka.seek(message)
+      end
+
       # Commits the stored offsets in a sync way and closes the consumer.
       def close
         # Allow only one client to be closed at the same time
         SHUTDOWN_MUTEX.synchronize do
-          # Make sure that no other operations are happening on this client when we close it
-          @mutex.synchronize do
-            # Once client is closed, we should not close it again
-            # This could only happen in case of a race-condition when forceful shutdown happens
-            # and triggers this from a different thread
-            return if @closed
-
-            @closed = true
-
-            # Remove callbacks runners that were registered
-            ::Karafka::Core::Instrumentation.statistics_callbacks.delete(@subscription_group.id)
-            ::Karafka::Core::Instrumentation.error_callbacks.delete(@subscription_group.id)
-
-            @kafka.close
-            @buffer.clear
-            # @note We do not clear rebalance manager here as we may still have revocation info
-            # here that we want to consider valid prior to running another reconnection
-          end
+          # Once client is closed, we should not close it again
+          # This could only happen in case of a race-condition when forceful shutdown happens
+          # and triggers this from a different thread
+          return if @closed
+
+          @closed = true
+
+          # Remove callbacks runners that were registered
+          ::Karafka::Core::Instrumentation.statistics_callbacks.delete(@subscription_group.id)
+          ::Karafka::Core::Instrumentation.error_callbacks.delete(@subscription_group.id)
+
+          @kafka.close
+          @buffer.clear
+          # @note We do not clear rebalance manager here as we may still have revocation info
+          # here that we want to consider valid prior to running another reconnection
         end
       end
 

data/lib/karafka/errors.rb

@@ -48,6 +48,9 @@ module Karafka
     StrategyNotFoundError = Class.new(BaseError)
 
     # This should never happen. Please open an issue if it does.
-    InvalidRealOffsetUsage = Class.new(BaseError)
+    InvalidRealOffsetUsageError = Class.new(BaseError)
+
+    # This should never happen. Please open an issue if it does.
+    InvalidTimeBasedOffsetError = Class.new(BaseError)
   end
 end

data/lib/karafka/messages/seek.rb

@@ -4,6 +4,9 @@ module Karafka
   module Messages
     # "Fake" message that we use as an abstraction layer when seeking back.
     # This allows us to encapsulate a seek with a simple abstraction
+    #
+    # @note `#offset` can be either the offset value or the time of the offset
+    # (first equal or greater)
     Seek = Struct.new(:topic, :partition, :offset)
   end
 end

data/lib/karafka/pro/iterator/expander.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    class Iterator
+      # There are various ways you can provide topics information for iterating.
+      #
+      # This mapper normalizes this data, resolves offsets and maps the time based offsets into
+      # appropriate once
+      #
+      # Following formats are accepted:
+      #
+      # - 'topic1' - just a string with one topic name
+      # - ['topic1', 'topic2'] - just the names
+      # - { 'topic1' => -100 } - names with negative lookup offset
+      # - { 'topic1' => { 0 => 5 } } - names with exact partitions offsets
+      # - { 'topic1' => { 0 => -5 }, 'topic2' => { 1 => 5 } } - with per partition negative offsets
+      # - { 'topic1' => 100 } - means we run all partitions from the offset 100
+      # - { 'topic1' => Time.now - 60 } - we run all partitions from the message from 60s ago
+      # - { 'topic1' => { 1 => Time.now - 60 } } - partition1 from message 60s ago
+      #
+      class Expander
+        # Expands topics to which we want to subscribe with partitions information in case this
+        # info is not provided.
+        #
+        # @param topics [Array, Hash, String] topics definitions
+        # @return [Hash] expanded and normalized requested topics and partitions data
+        def call(topics)
+          expanded = Hash.new { |h, k| h[k] = {} }
+
+          normalize_format(topics).map do |topic, details|
+            if details.is_a?(Hash)
+              details.each do |partition, offset|
+                expanded[topic][partition] = offset
+              end
+            else
+              partition_count(topic).times do |partition|
+                # If no offsets are provided, we just start from zero
+                expanded[topic][partition] = details || 0
+              end
+            end
+          end
+
+          expanded
+        end
+
+        private
+
+        # Input can be provided in multiple formats. Here we normalize it to one (hash).
+        #
+        # @param topics [Array, Hash, String] requested topics
+        # @return [Hash] normalized hash with topics data
+        def normalize_format(topics)
+          # Simplification for the single topic case
+          topics = [topics] if topics.is_a?(String)
+
+          # If we've got just array with topics, we need to convert that into a representation
+          # that we can expand with offsets
+          topics = topics.map { |name| [name, false] }.to_h if topics.is_a?(Array)
+          # We remap by creating new hash, just in case the hash came as the argument for this
+          # expanded. We do not want to modify user provided hash
+          topics.transform_keys(&:to_s)
+        end
+
+        # List of topics with their partition information for expansion
+        # We cache it so we do not have to run consecutive requests to obtain data about multiple
+        # topics
+        def topics
+          @topics ||= Admin.cluster_info.topics
+        end
+
+        # @param name [String] topic name
+        # @return [Integer] number of partitions of the topic we want to iterate over
+        def partition_count(name)
+          topics
+            .find { |topic| topic.fetch(:topic_name) == name }
+            .tap { |topic| topic || raise(Errors::TopicNotFoundError, name) }
+            .fetch(:partitions)
+            .count
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/iterator/tpl_builder.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    class Iterator
+      # Max time for a TPL request. We increase it to compensate for remote clusters latency
+      TPL_REQUEST_TIMEOUT = 2_000
+
+      private_constant :TPL_REQUEST_TIMEOUT
+
+      # Because we have various formats in which we can provide the offsets, before we can
+      # subscribe to them, there needs to be a bit of normalization.
+      #
+      # For some of the cases, we need to go to Kafka and get the real offsets or watermarks.
+      #
+      # This builder resolves that and builds a tpl to which we can safely subscribe the way
+      # we want it.
+      class TplBuilder
+        # @param consumer [::Rdkafka::Consumer] consumer instance needed to talk with Kafka
+        # @param expanded_topics [Hash] hash with expanded and normalized topics data
+        def initialize(consumer, expanded_topics)
+          @consumer = consumer
+          @expanded_topics = expanded_topics
+          @mapped_topics = Hash.new { |h, k| h[k] = {} }
+        end
+
+        # @return [Rdkafka::Consumer::TopicPartitionList] final tpl we can use to subscribe
+        def call
+          resolve_partitions_without_offsets
+          resolve_partitions_with_exact_offsets
+          resolve_partitions_with_negative_offsets
+          resolve_partitions_with_time_offsets
+
+          # Final tpl with all the data
+          tpl = Rdkafka::Consumer::TopicPartitionList.new
+
+          @mapped_topics.each do |name, partitions|
+            tpl.add_topic_and_partitions_with_offsets(name, partitions)
+          end
+
+          tpl
+        end
+
+        private
+
+        # First we expand on those partitions that do not have offsets defined.
+        # When we operate in case like this, we just start from beginning
+        def resolve_partitions_without_offsets
+          @expanded_topics.each do |name, partitions|
+            # We can here only about the case where we have partitions without offsets
+            next unless partitions.is_a?(Array) || partitions.is_a?(Range)
+
+            # When no offsets defined, we just start from zero
+            @mapped_topics[name] = partitions.map { |partition| [partition, 0] }.to_h
+          end
+        end
+
+        # If we get exact numeric offsets, we can just start from them without any extra work
+        def resolve_partitions_with_exact_offsets
+          @expanded_topics.each do |name, partitions|
+            next unless partitions.is_a?(Hash)
+
+            partitions.each do |partition, offset|
+              # Skip negative and time based offsets
+              next unless offset.is_a?(Integer) && offset >= 0
+
+              # Exact offsets can be used as they are
+              # No need for extra operations
+              @mapped_topics[name][partition] = offset
+            end
+          end
+        end
+
+        # If the offsets are negative, it means we want to fetch N last messages and we need to
+        # figure out the appropriate offsets
+        #
+        # We do it by getting the watermark offsets and just calculating it. This means that for
+        # heavily compacted topics, this may return less than the desired number but it is a
+        # limitation that is documented.
+        def resolve_partitions_with_negative_offsets
+          @expanded_topics.each do |name, partitions|
+            next unless partitions.is_a?(Hash)
+
+            partitions.each do |partition, offset|
+              # Care only about negative offsets (last n messages)
+              next unless offset.is_a?(Integer) && offset.negative?
+
+              _, high_watermark_offset = @consumer.query_watermark_offsets(name, partition)
+              # We add because this offset is negative
+              @mapped_topics[name][partition] = high_watermark_offset + offset
+            end
+          end
+        end
+
+        # For time based offsets we first need to aggregate them and request the proper offsets.
+        # We want to get all times in one go for all tpls defined with times, so we accumulate
+        # them here and we will make one sync request to kafka for all.
+        def resolve_partitions_with_time_offsets
+          time_tpl = Rdkafka::Consumer::TopicPartitionList.new
+
+          # First we need to collect the time based once
+          @expanded_topics.each do |name, partitions|
+            next unless partitions.is_a?(Hash)
+
+            time_based = {}
+
+            partitions.each do |partition, offset|
+              next unless offset.is_a?(Time)
+
+              time_based[partition] = offset
+            end
+
+            next if time_based.empty?
+
+            time_tpl.add_topic_and_partitions_with_offsets(name, time_based)
+          end
+
+          # If there were no time-based, no need to query Kafka
+          return if time_tpl.empty?
+
+          real_offsets = @consumer.offsets_for_times(time_tpl, TPL_REQUEST_TIMEOUT)
+
+          real_offsets.to_h.each do |name, results|
+            results.each do |result|
+              raise(Errors::InvalidTimeBasedOffsetError) unless result
+
+              @mapped_topics[name][result.partition] = result.offset
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/iterator.rb

@@ -50,7 +50,7 @@ module Karafka
         settings: { 'auto.offset.reset': 'beginning' },
         yield_nil: false
       )
-        @topics_with_partitions = expand_topics_with_partitions(topics)
+        @topics_with_partitions = Expander.new.call(topics)
 
         @routing_topics = @topics_with_partitions.map do |name, _|
           [name, ::Karafka::Routing::Router.find_or_initialize_by_name(name)]
@@ -71,7 +71,7 @@ module Karafka
       # only eat up resources.
       def each
         Admin.with_consumer(@settings) do |consumer|
-          tpl = tpl_with_expanded_offsets(consumer)
+          tpl = TplBuilder.new(consumer, @topics_with_partitions).call
           consumer.assign(tpl)
 
           # We need this for self-referenced APIs like pausing
@@ -131,43 +131,6 @@ module Karafka
 
       private
 
-      # Expands topics to which we want to subscribe with partitions information in case this
-      # info is not provided. For our convenience we want to support 5 formats of defining
-      # the subscribed topics:
-      #
-      # - 'topic1' - just a string with one topic name
-      # - ['topic1', 'topic2'] - just the names
-      # - { 'topic1' => -100 } - names with negative lookup offset
-      # - { 'topic1' => { 0 => 5 } } - names with exact partitions offsets
-      # - { 'topic1' => { 0 => -5 }, 'topic2' => { 1 => 5 } } - with per partition negative offsets
-      #
-      # @param topics [Array, Hash] topics definitions
-      # @return [Hash] hash with topics containing partitions definitions
-      def expand_topics_with_partitions(topics)
-        # Simplification for the single topic case
-        topics = [topics] if topics.is_a?(String)
-        # If we've got just array with topics, we need to convert that into a representation
-        # that we can expand with offsets
-        topics = topics.map { |name| [name, false] }.to_h if topics.is_a?(Array)
-
-        expanded = Hash.new { |h, k| h[k] = {} }
-
-        topics.map do |topic, details|
-          if details.is_a?(Hash)
-            details.each do |partition, offset|
-              expanded[topic][partition] = offset
-            end
-          else
-            partition_count(topic.to_s).times do |partition|
-              # If no offsets are provided, we just start from zero
-              expanded[topic][partition] = details || 0
-            end
-          end
-        end
-
-        expanded
-      end
-
       # @param timeout [Integer] timeout in ms
       # @return [Rdkafka::Consumer::Message, nil] message or nil if nothing to do
       def poll(timeout)
@@ -200,54 +163,6 @@ module Karafka
       def done?
         @stopped_partitions >= @total_partitions
       end
-
-      # Builds the tpl representing all the subscriptions we want to run
-      #
-      # Additionally for negative offsets, does the watermark calculation where to start
-      #
-      # @param consumer [Rdkafka::Consumer] consumer we need in case of negative offsets as
-      #   negative are going to be used to do "give me last X". We use the already initialized
-      #   consumer instance, not to start another one again.
-      # @return [Rdkafka::Consumer::TopicPartitionList]
-      def tpl_with_expanded_offsets(consumer)
-        tpl = Rdkafka::Consumer::TopicPartitionList.new
-
-        @topics_with_partitions.each do |name, partitions|
-          partitions_with_offsets = {}
-
-          # When no offsets defined, we just start from zero
-          if partitions.is_a?(Array) || partitions.is_a?(Range)
-            partitions_with_offsets = partitions.map { |partition| [partition, 0] }.to_h
-          else
-            # When offsets defined, we can either use them if positive or expand and move back
-            # in case of negative (-1000 means last 1000 messages, etc)
-            partitions.each do |partition, offset|
-              if offset.negative?
-                _, high_watermark_offset = consumer.query_watermark_offsets(name, partition)
-                # We add because this offset is negative
-                partitions_with_offsets[partition] = high_watermark_offset + offset
-              else
-                partitions_with_offsets[partition] = offset
-              end
-            end
-          end
-
-          tpl.add_topic_and_partitions_with_offsets(name, partitions_with_offsets)
-        end
-
-        tpl
-      end
-
-      # @param name [String] topic name
-      # @return [Integer] number of partitions of the topic we want to iterate over
-      def partition_count(name)
-        Admin
-          .cluster_info
-          .topics
-          .find { |topic| topic.fetch(:topic_name) == name }
-          .fetch(:partitions)
-          .count
-      end
     end
   end
 end

data/lib/karafka/pro/processing/filters_applier.rb

@@ -81,6 +81,7 @@ module Karafka
         # The first message we do need to get next time we poll. We use the minimum not to jump
         # accidentally by over any.
         # @return [Karafka::Messages::Message, nil] cursor message or nil if none
+        # @note Cursor message can also return the offset in the time format
         def cursor
           return nil unless active?
 

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom.rb

@@ -44,7 +44,9 @@ module Karafka
                   if coordinator.filtered? && !revoked?
                     handle_post_filtering
                   elsif !revoked?
-                    seek(coordinator.seek_offset)
+                    # no need to check for manual seek because AJ consumer is internal and
+                    # fully controlled by us
+                    seek(coordinator.seek_offset, false)
                     resume
                   else
                     resume

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom_vp.rb

@@ -50,7 +50,9 @@ module Karafka
                   if coordinator.filtered? && !revoked?
                     handle_post_filtering
                   elsif !revoked?
-                    seek(coordinator.seek_offset)
+                    # no need to check for manual seek because AJ consumer is internal and
+                    # fully controlled by us
+                    seek(coordinator.seek_offset, false)
                     resume
                   else
                     resume

data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom.rb

@@ -42,7 +42,9 @@ module Karafka
                 if coordinator.success?
                   coordinator.pause_tracker.reset
 
-                  seek(coordinator.seek_offset) unless revoked?
+                  # no need to check for manual seek because AJ consumer is internal and
+                  # fully controlled by us
+                  seek(coordinator.seek_offset, false) unless revoked?
 
                   resume
                 elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries

data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom_vp.rb

@@ -46,7 +46,9 @@ module Karafka
                   # Since we have VP here we do not commit intermediate offsets and need to commit
                   # them here. We do commit in collapsed mode but this is generalized.
                   mark_as_consumed(last_group_message) unless revoked?
-                  seek(coordinator.seek_offset) unless revoked?
+                  # no need to check for manual seek because AJ consumer is internal and
+                  # fully controlled by us
+                  seek(coordinator.seek_offset, false) unless revoked?
 
                   resume
                 elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries

data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom_vp.rb

@@ -46,7 +46,9 @@ module Karafka
                   if coordinator.filtered? && !revoked?
                     handle_post_filtering
                   elsif !revoked?
-                    seek(coordinator.seek_offset)
+                    # no need to check for manual seek because AJ consumer is internal and
+                    # fully controlled by us
+                    seek(coordinator.seek_offset, false)
                     resume
                   else
                     resume

data/lib/karafka/pro/processing/strategies/aj/lrj_mom_vp.rb

@@ -48,7 +48,10 @@ module Karafka
                   coordinator.pause_tracker.reset
 
                   mark_as_consumed(last_group_message) unless revoked?
-                  seek(coordinator.seek_offset) unless revoked?
+
+                  # no need to check for manual seek because AJ consumer is internal and
+                  # fully controlled by us
+                  seek(coordinator.seek_offset, false) unless revoked?
 
                   resume
                 else

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb

@@ -47,8 +47,8 @@ module Karafka
 
                   if coordinator.filtered? && !revoked?
                     handle_post_filtering
-                  elsif !revoked?
-                    seek(coordinator.seek_offset)
+                  elsif !revoked? && !coordinator.manual_seek?
+                    seek(coordinator.seek_offset, false)
                     resume
                   else
                     resume

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb

@@ -42,8 +42,8 @@ module Karafka
 
                   if coordinator.filtered? && !revoked?
                     handle_post_filtering
-                  elsif !revoked?
-                    seek(last_group_message.offset + 1)
+                  elsif !revoked? && !coordinator.manual_seek?
+                    seek(last_group_message.offset + 1, false)
                     resume
                   else
                     resume

data/lib/karafka/pro/processing/strategies/dlq/lrj.rb

@@ -38,7 +38,8 @@ module Karafka
                   return if coordinator.manual_pause?
 
                   mark_as_consumed(last_group_message) unless revoked?
-                  seek(coordinator.seek_offset) unless revoked?
+                  # We should not overwrite user manual seel request with our seek
+                  seek(coordinator.seek_offset, false) unless revoked? || coordinator.manual_seek?
 
                   resume
                 elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries

data/lib/karafka/pro/processing/strategies/dlq/lrj_mom.rb

@@ -37,7 +37,9 @@ module Karafka
 
                   return if coordinator.manual_pause?
 
-                  seek(last_group_message.offset + 1) unless revoked?
+                  unless revoked? || coordinator.manual_seek?
+                    seek(last_group_message.offset + 1, false)
+                  end
 
                   resume
                 elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries

data/lib/karafka/pro/processing/strategies/ftr/default.rb

@@ -70,6 +70,10 @@ module Karafka
               when :skip
                 nil
               when :seek
+                # User direct actions take priority over automatic operations
+                # If we've already seeked we can just resume operations, nothing extra needed
+                return resume if coordinator.manual_seek?
+
                 throttle_message = filter.cursor
 
                 Karafka.monitor.instrument(
@@ -77,11 +81,14 @@ module Karafka
                   caller: self,
                   message: throttle_message
                 ) do
-                  seek(throttle_message.offset)
+                  seek(throttle_message.offset, false)
                 end
 
                 resume
               when :pause
+                # User direct actions take priority over automatic operations
+                return nil if coordinator.manual_pause?
+
                 throttle_message = filter.cursor
 
                 Karafka.monitor.instrument(

data/lib/karafka/pro/processing/strategies/lrj/default.rb

@@ -53,7 +53,7 @@ module Karafka
                   return if coordinator.manual_pause?
 
                   mark_as_consumed(last_group_message) unless revoked?
-                  seek(coordinator.seek_offset) unless revoked?
+                  seek(coordinator.seek_offset, false) unless revoked? || coordinator.manual_seek?
 
                   resume
                 else

data/lib/karafka/pro/processing/strategies/lrj/ftr.rb

@@ -45,10 +45,10 @@ module Karafka
                   # If still not revoked and was throttled, we need to apply throttling logic
                   if coordinator.filtered? && !revoked?
                     handle_post_filtering
-                  elsif !revoked?
+                  elsif !revoked? && !coordinator.manual_seek?
                     # If not revoked and not throttled, we move to where we were suppose to and
                     # resume
-                    seek(coordinator.seek_offset)
+                    seek(coordinator.seek_offset, false)
                     resume
                   else
                     resume

data/lib/karafka/pro/processing/strategies/lrj/ftr_mom.rb

@@ -43,10 +43,10 @@ module Karafka
                   # If still not revoked and was throttled, we need to apply filtering logic
                   if coordinator.filtered? && !revoked?
                     handle_post_filtering
-                  elsif !revoked?
+                  elsif !revoked? && !coordinator.manual_seek?
                     # If not revoked and not throttled, we move to where we were suppose to and
                     # resume
-                    seek(last_group_message.offset + 1)
+                    seek(last_group_message.offset + 1, false)
                     resume
                   else
                     resume

data/lib/karafka/pro/processing/strategies/lrj/mom.rb

@@ -52,7 +52,9 @@ module Karafka
 
                   return if coordinator.manual_pause?
 
-                  seek(last_group_message.offset + 1) unless revoked?
+                  unless revoked? || coordinator.manual_seek?
+                    seek(last_group_message.offset + 1, false)
+                  end
 
                   resume
                 else

data/lib/karafka/pro/processing/virtual_offset_manager.rb

@@ -118,7 +118,7 @@ module Karafka
 
         # @return [Messages::Seek] markable message for real offset marking
         def markable
-          raise Errors::InvalidRealOffsetUsage unless markable?
+          raise Errors::InvalidRealOffsetUsageError unless markable?
 
           Messages::Seek.new(
             @topic,

data/lib/karafka/processing/coordinator.rb

@@ -23,6 +23,7 @@ module Karafka
         @consumptions = {}
         @running_jobs = 0
         @manual_pause = false
+        @manual_seek = false
         @mutex = Mutex.new
         @marked = false
         @failure = false
@@ -41,6 +42,9 @@ module Karafka
         # When starting to run, no pause is expected and no manual pause as well
         @manual_pause = false
 
+        # No user invoked seeks on a new run
+        @manual_seek = false
+
         # We set it on the first encounter and never again, because then the offset setting
         # should be up to the consumers logic (our or the end user)
         # Seek offset needs to be always initialized as for case where manual offset management
@@ -148,6 +152,16 @@ module Karafka
         @pause_tracker.paused? && @manual_pause
       end
 
+      # Marks seek as manual for coordination purposes
+      def manual_seek
+        @manual_seek = true
+      end
+
+      # @return [Boolean] did a user invoke seek in the current operations scope
+      def manual_seek?
+        @manual_seek
+      end
+
       # Allows to run synchronized (locked) code that can operate in between virtual partitions
       # @param block [Proc] code we want to run in the synchronized mode
       def synchronize(&block)

data/lib/karafka/railtie.rb

@@ -70,11 +70,11 @@ if Karafka.rails?
         # We can have many listeners, but it does not matter in which we will reload the code
         # as long as all the consumers will be re-created as Rails reload is thread-safe
         ::Karafka::App.monitor.subscribe('connection.listener.fetch_loop') do
-          # Reload code each time there is a change in the code
-          next unless Rails.application.reloaders.any?(&:updated?)
           # If consumer persistence is enabled, no reason to reload because we will still keep
           # old consumer instances in memory.
           next if Karafka::App.config.consumer_persistence
+          # Reload code each time there is a change in the code
+          next unless Rails.application.reloaders.any?(&:updated?)
 
           Rails.application.reloader.reload!
         end

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.1.5'
+  VERSION = '2.1.6'
 end

data.tar.gz.sig

@@ -1,5 +1,2 @@
-=QV��7�x;��=��������.,�Zм� �˅�8�KS-����灥����q��S��nˀ�����"������诜	�R#��E�s�u�R�V�.��9W��a�0�ҳ�� ����,���|8�OG�C�簽,w��X��ӒH�#+M)�m��f�v'��]bP]K��Y�r ֘;��*ܮ��GX}a@�)�P��̀��{�X Dij����ob
-�
-�,S
�����C8�<�OqCy?f��S3���m�VɂN�DDZ@���?�����\ *�ek�L��
-�b35�'���r�j�<>4vI1��(P��)27�|�:���	c�y4���8X9�۟��
-�0S�@D
+mf̈́x1��2�'�lj!���.����U��
KL1.�fv��$����N��Oi���U/)3lS��ݑ�%}�aE��}/J\�_��1In� N�t+����mV��G�����C��_Y;��X3�Y�8(J,}���=���:��p	�(��2��{:�O�K"$��7W�A�5k��|l���]�Bd^�I�4*\���9���PL'������ޝmh�]o�n(��	��e˱վ
+PJ�5���

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.1.5
+  version: 2.1.6
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2023-06-19 00:00:00.000000000 Z
+date: 2023-06-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -43,7 +43,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.1.0
+        version: 2.1.1
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.2.0
@@ -53,7 +53,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.1.0
+        version: 2.1.1
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.2.0
@@ -77,7 +77,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.6.1
+        version: 2.6.2
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -87,7 +87,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.6.1
+        version: 2.6.2
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -226,6 +226,8 @@ files:
 - lib/karafka/pro/encryption/messages/parser.rb
 - lib/karafka/pro/encryption/setup/config.rb
 - lib/karafka/pro/iterator.rb
+- lib/karafka/pro/iterator/expander.rb
+- lib/karafka/pro/iterator/tpl_builder.rb
 - lib/karafka/pro/loader.rb
 - lib/karafka/pro/performance_tracker.rb
 - lib/karafka/pro/processing/collapser.rb