karafka 2.3.2 → 2.4.0.beta1

data/lib/karafka/pro/iterator.rb

@@ -20,7 +20,9 @@ module Karafka
     # the end. It also allows for signaling, when a given message should be last out of certain
     # partition, but we still want to continue iterating in other messages.
     #
-    # It does **not** create a consumer group and does not have any offset management.
+    # It does **not** create a consumer group and does not have any offset management until first
+    # consumer offset marking happens. So can be use for quick seeks as well as iterative,
+    # repetitive data fetching from rake, etc.
     class Iterator
       # A simple API allowing to iterate over topic/partition data, without having to subscribe
       # and deal with rebalances. This API allows for multi-partition streaming and is optimized
@@ -92,6 +94,7 @@ module Karafka
             end
           end
 
+          @current_consumer.commit_offsets(async: false) if @stored_offsets
           @current_message = nil
           @current_consumer = nil
         end
@@ -127,6 +130,29 @@ module Karafka
         )
       end
 
+      # Stops all the iterating
+      # @note `break` can also be used but in such cases commits stored async will not be flushed
+      #   to Kafka. This is why `#stop` is the recommended method.
+      def stop
+        @stopped = true
+      end
+
+      # Marks given message as consumed.
+      #
+      # @param message [Karafka::Messages::Message] message that we want to mark as processed
+      def mark_as_consumed(message)
+        @current_consumer.store_offset(message, nil)
+        @stored_offsets = true
+      end
+
+      # Marks given message as consumed and commits offsets
+      #
+      # @param message [Karafka::Messages::Message] message that we want to mark as processed
+      def mark_as_consumed!(message)
+        mark_as_consumed(message)
+        @current_consumer.commit_offsets(async: false)
+      end
+
       private
 
       # @return [Rdkafka::Consumer::Message, nil] message or nil if nothing to do
@@ -158,7 +184,7 @@ module Karafka
       # Do we have all the data we wanted or did every topic partition has reached eof.
       # @return [Boolean]
       def done?
-        @stopped_partitions >= @total_partitions
+        (@stopped_partitions >= @total_partitions) || @stopped
       end
     end
   end

data/lib/karafka/pro/loader.rb

@@ -63,6 +63,9 @@ module Karafka
         # @param config [Karafka::Core::Configurable::Node]
         def post_setup_all(config)
           features.each { |feature| feature.post_setup(config) }
+
+          # We initialize it here so we don't initialize it during multi-threading work
+          Processing::SubscriptionGroupsCoordinator.instance
         end
 
         private

data/lib/karafka/pro/processing/coordinator.rb

@@ -99,8 +99,9 @@ module Karafka
         end
 
         # @return [Boolean] is the coordinated work finished or not
+        # @note Used only in the consume operation context
         def finished?
-          @running_jobs.zero?
+          @running_jobs[:consume].zero?
         end
 
         # Runs synchronized code once for a collective of virtual partitions prior to work being
@@ -122,7 +123,7 @@ module Karafka
           end
         end
 
-        # Runs once when all the work that is suppose to be coordinated is finished
+        # Runs given code once when all the work that is suppose to be coordinated is finished
         # It runs once per all the coordinated jobs and should be used to run any type of post
         # jobs coordination processing execution
         def on_finished
@@ -143,6 +144,18 @@ module Karafka
           end
         end
 
+        # @param interval [Integer] milliseconds of activity
+        # @return [Boolean] was this partition in activity within last `interval` milliseconds
+        # @note Will return true also if currently active
+        def active_within?(interval)
+          # its always active if there's any job related to this coordinator that is still
+          # enqueued or running
+          return true if @running_jobs.values.any?(:positive?)
+
+          # Otherwise we check last time any job of this coordinator was active
+          @changed_at + interval > monotonic_now
+        end
+
         private
 
         # Checks if given action is executable once. If it is and true is returned, this method

data/lib/karafka/pro/processing/expansions_selector.rb

@@ -23,7 +23,9 @@ module Karafka
         def find(topic)
           # Start with the non-pro expansions
           expansions = super
+          expansions << Pro::Processing::Piping::Consumer
           expansions << Pro::Processing::OffsetMetadata::Consumer if topic.offset_metadata?
+          expansions << Pro::Processing::PeriodicJob::Consumer if topic.periodic_job?
           expansions
         end
       end

data/lib/karafka/pro/processing/jobs_queue.rb

@@ -19,17 +19,36 @@ module Karafka
       #
       # Aside from the OSS queue capabilities it allows for jobless locking for advanced schedulers
       class JobsQueue < Karafka::Processing::JobsQueue
+        include Core::Helpers::Time
+
         attr_accessor :in_processing
 
+        # How long should we keep async lock (31 years)
+        WAIT_TIMEOUT = 10_000_000_000
+
+        private_constant :WAIT_TIMEOUT
+
         # @return [Karafka::Pro::Processing::JobsQueue]
         def initialize
           super
 
           @in_waiting = Hash.new { |h, k| h[k] = [] }
+          @locks = Hash.new { |h, k| h[k] = {} }
+          @async_locking = false
 
           @statistics[:waiting] = 0
         end
 
+        # Registers semaphore and a lock hash
+        #
+        # @param group_id [String]
+        def register(group_id)
+          super
+          @mutex.synchronize do
+            @locks[group_id]
+          end
+        end
+
         # Method that allows us to lock queue on a given subscription group without enqueuing the a
         # job. This can be used when building complex schedulers that want to postpone enqueuing
         # before certain conditions are met.
@@ -64,6 +83,48 @@ module Karafka
           end
         end
 
+        # Allows for explicit locking of the queue of a given subscription group.
+        #
+        # This can be used for cross-topic synchronization.
+        #
+        # @param group_id [String] id of the group we want to lock
+        # @param lock_id [Object] unique id we want to use to identify our lock
+        # @param timeout [Integer] number of ms how long this lock should be valid. Useful for
+        #   auto-expiring locks used to delay further processing without explicit pausing on
+        #   the consumer
+        #
+        # @note We do not raise `Errors::JobsQueueSynchronizationError` similar to `#lock` here
+        #   because we want to have ability to prolong time limited locks
+        def lock_async(group_id, lock_id, timeout: WAIT_TIMEOUT)
+          return if @queue.closed?
+
+          @async_locking = true
+
+          @mutex.synchronize do
+            @locks[group_id][lock_id] = monotonic_now + timeout
+
+            # We need to tick so our new time sensitive lock can reload time constraints on sleep
+            tick(group_id)
+          end
+        end
+
+        # Allows for explicit unlocking of locked queue of a group
+        #
+        # @param group_id [String] id of the group we want to unlock
+        # @param lock_id [Object] unique id we want to use to identify our lock
+        #
+        def unlock_async(group_id, lock_id)
+          @mutex.synchronize do
+            if @locks[group_id].delete(lock_id)
+              tick(group_id)
+
+              return
+            end
+
+            raise(Errors::JobsQueueSynchronizationError, [group_id, lock_id])
+          end
+        end
+
         # Clears the processing states for a provided group. Useful when a recovery happens and we
         # need to clean up state but only for a given subscription group.
         #
@@ -74,6 +135,8 @@ module Karafka
 
             @statistics[:waiting] -= @in_waiting[group_id].size
             @in_waiting[group_id].clear
+            @locks[group_id].clear
+            @async_locking = false
 
             # We unlock it just in case it was blocked when clearing started
             tick(group_id)
@@ -87,21 +150,75 @@ module Karafka
         def empty?(group_id)
           @mutex.synchronize do
             @in_processing[group_id].empty? &&
-              @in_waiting[group_id].empty?
+              @in_waiting[group_id].empty? &&
+              !locked_async?(group_id)
+          end
+        end
+
+        # Blocks when there are things in the queue in a given group and waits until all the
+        #   blocking jobs from a given group are completed or any of the locks times out
+        # @param group_id [String] id of the group in which jobs we're interested.
+        # @see `Karafka::Processing::JobsQueue`
+        #
+        # @note Because checking that async locking is on happens on regular ticking, first lock
+        #   on a group can take up to one tick. That is expected.
+        #
+        # @note This implementation takes into consideration temporary async locks that can happen.
+        #   Thanks to the fact that we use the minimum lock time as a timeout, we do not have to
+        #   wait a whole ticking period to unlock async locks.
+        def wait(group_id)
+          return super unless @async_locking
+
+          # We do not generalize this flow because this one is more expensive as it has to allocate
+          # extra objects. That's why we only use it when locks are actually in use
+          base_interval = tick_interval / 1_000.0
+
+          while wait?(group_id)
+            yield if block_given?
+
+            now = monotonic_now
+
+            wait_times = @locks[group_id].values.map! do |lock_time|
+              # Convert ms to seconds, seconds are required by Ruby queue engine
+              (lock_time - now) / 1_000
+            end
+
+            wait_times.delete_if(&:negative?)
+            wait_times << base_interval
+
+            @semaphores.fetch(group_id).pop(timeout: wait_times.min)
           end
         end
 
         private
 
+        # Tells us if given group is locked
+        #
+        # @param group_id [String] id of the group in which we're interested.
+        # @return [Boolean] true if there are any active locks on the group, otherwise false
+        def locked_async?(group_id)
+          return false unless @async_locking
+
+          group = @locks[group_id]
+
+          return false if group.empty?
+
+          now = monotonic_now
+
+          group.delete_if { |_, wait_timeout| wait_timeout < now }
+
+          !group.empty?
+        end
+
         # @param group_id [String] id of the group in which jobs we're interested.
         # @return [Boolean] should we keep waiting or not
         # @note We do not wait for non-blocking jobs. Their flow should allow for `poll` running
         #   as they may exceed `max.poll.interval`
         def wait?(group_id)
-          !(
-            @in_processing[group_id].all?(&:non_blocking?) &&
-            @in_waiting[group_id].all?(&:non_blocking?)
-          )
+          return true unless @in_processing[group_id].all?(&:non_blocking?)
+          return true unless @in_waiting[group_id].all?(&:non_blocking?)
+
+          locked_async?(group_id)
         end
       end
     end

data/lib/karafka/pro/processing/periodic_job/consumer.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Namespace for periodic jobs related processing APIs
+      module PeriodicJob
+        # Consumer extra methods useful only when periodic jobs are in use
+        module Consumer
+          class << self
+            # Defines an empty `#tick` method if not present
+            #
+            # We define it that way due to our injection strategy flow.
+            #
+            # @param consumer_singleton_class [Karafka::BaseConsumer] consumer singleton class
+            #   that is being enriched with periodic jobs API
+            def included(consumer_singleton_class)
+              # Do not define empty tick method on consumer if it already exists
+              # We only define it when it does not exist to have empty periodic ticking
+              #
+              # We need to check both cases (public and private) since user is not expected to
+              # have this method public
+              return if consumer_singleton_class.instance_methods.include?(:tick)
+              return if consumer_singleton_class.private_instance_methods.include?(:tick)
+
+              # Create empty ticking method
+              consumer_singleton_class.class_eval do
+                def tick; end
+              end
+            end
+          end
+
+          # Runs the on-schedule tick periodic operations
+          # This method is an alias but is part of the naming convention used for other flows, this
+          # is why we do not reference the `handle_before_schedule_tick` directly
+          def on_before_schedule_tick
+            handle_before_schedule_tick
+          end
+
+          # Used by the executor to trigger consumer tick
+          # @private
+          def on_tick
+            handle_tick
+          rescue StandardError => e
+            Karafka.monitor.instrument(
+              'error.occurred',
+              error: e,
+              caller: self,
+              type: 'consumer.tick.error'
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/piping/consumer.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # All code needed for messages piping in Karafka
+      module Piping
+        # Consumer piping functionality
+        #
+        # It provides way to pipe data in a consistent way with extra traceability headers similar
+        # to those in the enhanced DLQ.
+        module Consumer
+          # Empty hash to save on memory allocations
+          EMPTY_HASH = {}.freeze
+
+          private_constant :EMPTY_HASH
+
+          # Pipes given message to the provided topic with expected details. Useful for
+          # pass-through operations where deserialization is not needed. Upon usage it will include
+          # all the original headers + meta headers about the source of message.
+          #
+          # @param topic [String, Symbol] where we want to send the message
+          # @param message [Karafka::Messages::Message] original message to pipe
+          #
+          # @note It will NOT deserialize the payload so it is fast
+          #
+          # @note We assume that there can be different number of partitions in the target topic,
+          #   this is why we use `key` based on the original topic partition number and not the
+          #   partition id itself. This will not utilize partitions beyond the number of partitions
+          #   of original topic, but will accommodate for topics with less partitions.
+          def pipe_async(topic:, message:)
+            produce_async(
+              build_pipe_message(topic: topic, message: message)
+            )
+          end
+
+          # Sync version of pipe for one message
+          #
+          # @param topic [String, Symbol] where we want to send the message
+          # @param message [Karafka::Messages::Message] original message to pipe
+          # @see [#pipe_async]
+          def pipe_sync(topic:, message:)
+            produce_sync(
+              build_pipe_message(topic: topic, message: message)
+            )
+          end
+
+          # Async multi-message pipe
+          #
+          # @param topic [String, Symbol] where we want to send the message
+          # @param messages [Array<Karafka::Messages::Message>] original messages to pipe
+          #
+          # @note If transactional producer in use and dispatch is not wrapped with a transaction,
+          #   it will automatically wrap the dispatch with a transaction
+          def pipe_many_async(topic:, messages:)
+            messages = messages.map do |message|
+              build_pipe_message(topic: topic, message: message)
+            end
+
+            produce_many_async(messages)
+          end
+
+          # Sync multi-message pipe
+          #
+          # @param topic [String, Symbol] where we want to send the message
+          # @param messages [Array<Karafka::Messages::Message>] original messages to pipe
+          #
+          # @note If transactional producer in use and dispatch is not wrapped with a transaction,
+          #   it will automatically wrap the dispatch with a transaction
+          def pipe_many_sync(topic:, messages:)
+            messages = messages.map do |message|
+              build_pipe_message(topic: topic, message: message)
+            end
+
+            produce_many_sync(messages)
+          end
+
+          private
+
+          # @param topic [String, Symbol] where we want to send the message
+          # @param message [Karafka::Messages::Message] original message to pipe
+          # @return [Hash] hash with message to pipe.
+          #
+          # @note If you need to alter this, please define the `#enhance_pipe_message` method
+          def build_pipe_message(topic:, message:)
+            original_partition = message.partition.to_s
+
+            pipe_message = {
+              topic: topic,
+              key: original_partition,
+              payload: message.raw_payload,
+              headers: message.headers.merge(
+                'original_topic' => message.topic,
+                'original_partition' => original_partition,
+                'original_offset' => message.offset.to_s,
+                'original_consumer_group' => self.topic.consumer_group.id
+              )
+            }
+
+            # Optional method user can define in consumer to enhance the dlq message hash with
+            # some extra details if needed or to replace payload, etc
+            if respond_to?(:enhance_pipe_message, true)
+              enhance_pipe_message(
+                pipe_message,
+                message
+              )
+            end
+
+            pipe_message
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom.rb

@@ -55,7 +55,7 @@ module Karafka
                   apply_dlq_flow do
                     skippable_message, = find_skippable_message
                     dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                    mark_as_consumed(skippable_message)
+                    mark_dispatched_to_dlq(skippable_message)
                   end
                 end
               end

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom_vp.rb

@@ -61,7 +61,7 @@ module Karafka
                   apply_dlq_flow do
                     skippable_message, = find_skippable_message
                     dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                    mark_as_consumed(skippable_message)
+                    mark_dispatched_to_dlq(skippable_message)
                   end
                 end
               end

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom.rb

@@ -57,7 +57,7 @@ module Karafka
                     # We can commit the offset here because we know that we skip it "forever" and
                     # since AJ consumer commits the offset after each job, we also know that the
                     # previous job was successful
-                    mark_as_consumed(skippable_message)
+                    mark_dispatched_to_dlq(skippable_message)
                   end
                 end
               end

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom_vp.rb

@@ -55,7 +55,7 @@ module Karafka
                     # We can commit the offset here because we know that we skip it "forever" and
                     # since AJ consumer commits the offset after each job, we also know that the
                     # previous job was successful
-                    mark_as_consumed(skippable_message)
+                    mark_dispatched_to_dlq(skippable_message)
                   end
                 end
               end

data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom.rb

@@ -51,7 +51,7 @@ module Karafka
                   apply_dlq_flow do
                     skippable_message, = find_skippable_message
                     dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                    mark_as_consumed(skippable_message)
+                    mark_dispatched_to_dlq(skippable_message)
                   end
                 end
               end

data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom_vp.rb

@@ -55,7 +55,7 @@ module Karafka
                   apply_dlq_flow do
                     skippable_message, = find_skippable_message
                     dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                    mark_as_consumed(skippable_message)
+                    mark_dispatched_to_dlq(skippable_message)
                   end
                 end
               end

data/lib/karafka/pro/processing/strategies/aj/dlq_mom.rb

@@ -49,7 +49,7 @@ module Karafka
                     # We can commit the offset here because we know that we skip it "forever" and
                     # since AJ consumer commits the offset after each job, we also know that the
                     # previous job was successful
-                    mark_as_consumed(skippable_message)
+                    mark_dispatched_to_dlq(skippable_message)
                   end
                 end
               end

data/lib/karafka/pro/processing/strategies/aj/dlq_mom_vp.rb

@@ -54,7 +54,7 @@ module Karafka
                     # Aj::DlqMom
                     skippable_message, = find_skippable_message
                     dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                    mark_as_consumed(skippable_message)
+                    mark_dispatched_to_dlq(skippable_message)
                   end
                 end
               end

data/lib/karafka/pro/processing/strategies/aj/lrj_mom_vp.rb

@@ -73,6 +73,8 @@ module Karafka
               Karafka.monitor.instrument('consumer.revoked', caller: self) do
                 revoked
               end
+            ensure
+              coordinator.decrement(:revoked)
             end
           end
         end

data/lib/karafka/pro/processing/strategies/default.rb

@@ -220,7 +220,7 @@ module Karafka
           ensure
             # We need to decrease number of jobs that this coordinator coordinates as it has
             # finished
-            coordinator.decrement
+            coordinator.decrement(:consume)
           end
 
           # Standard flow without any features
@@ -254,6 +254,8 @@ module Karafka
             Karafka.monitor.instrument('consumer.revoked', caller: self) do
               revoked
             end
+          ensure
+            coordinator.decrement(:revoked)
           end
 
           # No action needed for the tick standard flow
@@ -269,6 +271,8 @@ module Karafka
             Karafka.monitor.instrument('consumer.ticked', caller: self) do
               tick
             end
+          ensure
+            coordinator.decrement(:periodic)
           end
         end
       end

data/lib/karafka/pro/processing/strategies/dlq/default.rb

@@ -111,7 +111,8 @@ module Karafka
               # should not be cleaned as it should go to the DLQ
               raise(Cleaner::Errors::MessageCleanedError) if skippable_message.cleaned?
 
-              producer.produce_async(
+              producer.public_send(
+                topic.dead_letter_queue.dispatch_method,
                 build_dlq_message(
                   skippable_message
                 )
@@ -134,7 +135,7 @@ module Karafka
 
               dispatch = lambda do
                 dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                mark_as_consumed(skippable_message)
+                mark_dispatched_to_dlq(skippable_message)
               end
 
               if dispatch_in_a_transaction?
@@ -157,7 +158,8 @@ module Karafka
                   'original_topic' => topic.name,
                   'original_partition' => original_partition,
                   'original_offset' => skippable_message.offset.to_s,
-                  'original_consumer_group' => topic.consumer_group.id
+                  'original_consumer_group' => topic.consumer_group.id,
+                  'original_attempts' => attempt.to_s
                 )
               }
 
@@ -210,14 +212,28 @@ module Karafka
                 raise Karafka::UnsupportedCaseError, flow
               end
 
+              yield
+
               # We reset the pause to indicate we will now consider it as "ok".
               coordinator.pause_tracker.reset
 
-              yield
-
               # Always backoff after DLQ dispatch even on skip to prevent overloads on errors
               pause(coordinator.seek_offset, nil, false)
             end
+
+            # Marks message that went to DLQ (if applicable) based on the requested method
+            # @param skippable_message [Karafka::Messages::Message]
+            def mark_dispatched_to_dlq(skippable_message)
+              case topic.dead_letter_queue.marking_method
+              when :mark_as_consumed
+                mark_as_consumed(skippable_message)
+              when :mark_as_consumed!
+                mark_as_consumed!(skippable_message)
+              else
+                # This should never happen. Bug if encountered. Please report
+                raise Karafka::Errors::UnsupportedCaseError
+              end
+            end
           end
         end
       end

data/lib/karafka/pro/processing/strategies/lrj/default.rb

@@ -76,6 +76,8 @@ module Karafka
               Karafka.monitor.instrument('consumer.revoked', caller: self) do
                 revoked
               end
+            ensure
+              coordinator.decrement(:revoked)
             end
 
             # Allows for LRJ to synchronize its work. It may be needed because LRJ can run