karafka 1.4.12 → 2.2.10

data/lib/karafka/processing/jobs_builder.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Class responsible for deciding what type of job should we build to run a given command and
+    # for building a proper job for it.
+    class JobsBuilder
+      # @param executor [Karafka::Processing::Executor]
+      # @param messages [Karafka::Messages::Messages] messages batch to be consumed
+      # @return [Karafka::Processing::Jobs::Consume] consumption job
+      def consume(executor, messages)
+        Jobs::Consume.new(executor, messages)
+      end
+
+      # @param executor [Karafka::Processing::Executor]
+      # @return [Karafka::Processing::Jobs::Revoked] revocation job
+      def revoked(executor)
+        Jobs::Revoked.new(executor)
+      end
+
+      # @param executor [Karafka::Processing::Executor]
+      # @return [Karafka::Processing::Jobs::Shutdown] shutdown job
+      def shutdown(executor)
+        Jobs::Shutdown.new(executor)
+      end
+    end
+  end
+end

data/lib/karafka/processing/jobs_queue.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # This is the key work component for Karafka jobs distribution. It provides API for running
+    # jobs in parallel while operating within more than one subscription group.
+    #
+    # We need to take into consideration fact, that more than one subscription group can operate
+    # on this queue, that's why internally we keep track of processing per group.
+    #
+    # We work with the assumption, that partitions data is evenly distributed.
+    class JobsQueue
+      # @return [Karafka::Processing::JobsQueue]
+      def initialize
+        @queue = Queue.new
+        # Those queues will act as semaphores internally. Since we need an indicator for waiting
+        # we could use Thread.pass but this is expensive. Instead we can just lock until any
+        # of the workers finishes their work and we can re-check. This means that in the worse
+        # scenario, we will context switch 10 times per poll instead of getting this thread
+        # scheduled by Ruby hundreds of thousands of times per group.
+        # We cannot use a single semaphore as it could potentially block in listeners that should
+        # process with their data and also could unlock when a given group needs to remain locked
+        @semaphores = Concurrent::Map.new do |h, k|
+          h.compute_if_absent(k) { Queue.new }
+        end
+
+        @in_processing = Hash.new { |h, k| h[k] = [] }
+
+        @mutex = Mutex.new
+      end
+
+      # Returns number of jobs that are either enqueued or in processing (but not finished)
+      # @return [Integer] number of elements in the queue
+      # @note Using `#pop` won't decrease this number as only marking job as completed does this
+      def size
+        @in_processing.values.map(&:size).sum
+      end
+
+      # Adds the job to the internal main queue, scheduling it for execution in a worker and marks
+      # this job as in processing pipeline.
+      #
+      # @param job [Jobs::Base] job that we want to run
+      def <<(job)
+        # We do not push the job if the queue is closed as it means that it would anyhow not be
+        # executed
+        return if @queue.closed?
+
+        @mutex.synchronize do
+          group = @in_processing[job.group_id]
+
+          raise(Errors::JobsQueueSynchronizationError, job.group_id) if group.include?(job)
+
+          group << job
+
+          @queue << job
+        end
+      end
+
+      # @return [Jobs::Base, nil] waits for a job from the main queue and returns it once available
+      #   or returns nil if the queue has been stopped and there won't be anything more to process
+      #   ever.
+      # @note This command is blocking and will wait until any job is available on the main queue
+      def pop
+        @queue.pop
+      end
+
+      # Causes the wait lock to re-check the lock conditions and potential unlock.
+      # @param group_id [String] id of the group we want to unlock for one tick
+      # @note This does not release the wait lock. It just causes a conditions recheck
+      def tick(group_id)
+        @semaphores[group_id] << true
+      end
+
+      # Marks a given job from a given group as completed. When there are no more jobs from a given
+      # group to be executed, we won't wait.
+      #
+      # @param [Jobs::Base] job that was completed
+      def complete(job)
+        @mutex.synchronize do
+          @in_processing[job.group_id].delete(job)
+          tick(job.group_id)
+        end
+      end
+
+      # Clears the processing states for a provided group. Useful when a recovery happens and we
+      # need to clean up state but only for a given subscription group.
+      #
+      # @param group_id [String]
+      def clear(group_id)
+        @mutex.synchronize do
+          @in_processing[group_id].clear
+          # We unlock it just in case it was blocked when clearing started
+          tick(group_id)
+        end
+      end
+
+      # Stops the whole processing queue.
+      def close
+        @mutex.synchronize do
+          return if @queue.closed?
+
+          @queue.close
+          @semaphores.each_value(&:close)
+        end
+      end
+
+      # @param group_id [String]
+      #
+      # @return [Boolean] tell us if we have anything in the processing (or for processing) from
+      # a given group.
+      def empty?(group_id)
+        @mutex.synchronize do
+          @in_processing[group_id].empty?
+        end
+      end
+
+      # Blocks when there are things in the queue in a given group and waits until all the blocking
+      #   jobs from a given group are completed
+      #
+      # @param group_id [String] id of the group in which jobs we're interested.
+      # @note This method is blocking.
+      def wait(group_id)
+        # Go doing other things while we cannot process and wait for anyone to finish their work
+        # and re-check the wait status
+        @semaphores[group_id].pop while wait?(group_id)
+      end
+
+      # - `busy` - number of jobs that are currently being processed (active work)
+      # - `enqueued` - number of jobs in the queue that are waiting to be picked up by a worker
+      #
+      # @return [Hash] hash with basic usage statistics of this queue.
+      def statistics
+        {
+          busy: size - @queue.size,
+          enqueued: @queue.size
+        }.freeze
+      end
+
+      private
+
+      # @param group_id [String] id of the group in which jobs we're interested.
+      # @return [Boolean] should we keep waiting or not
+      # @note We do not wait for non-blocking jobs. Their flow should allow for `poll` running
+      #   as they may exceed `max.poll.interval`
+      def wait?(group_id)
+        !@in_processing[group_id].all?(&:non_blocking?)
+      end
+    end
+  end
+end

data/lib/karafka/processing/partitioner.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Basic partitioner for work division
+    # It does not divide any work.
+    class Partitioner
+      # @param subscription_group [Karafka::Routing::SubscriptionGroup] subscription group
+      def initialize(subscription_group)
+        @subscription_group = subscription_group
+      end
+
+      # @param _topic [String] topic name
+      # @param messages [Array<Karafka::Messages::Message>] karafka messages
+      # @param _coordinator [Karafka::Processing::Coordinator] processing coordinator that will
+      #   be used with those messages
+      # @yieldparam [Integer] group id
+      # @yieldparam [Array<Karafka::Messages::Message>] karafka messages
+      def call(_topic, messages, _coordinator)
+        yield(0, messages)
+      end
+    end
+  end
+end

data/lib/karafka/processing/result.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # A simple object that allows us to keep track of processing state.
+    # It allows to indicate if given thing moved from success to a failure or the other way around
+    # Useful for tracking consumption state
+    class Result
+      attr_reader :cause
+
+      def initialize
+        @success = true
+        @cause = false
+      end
+
+      # @return [Boolean]
+      def success?
+        @success
+      end
+
+      # Marks state as successful
+      def success!
+        @success = true
+        # We set cause to false so the previous error that occurred does not leak when error is
+        # no longer present
+        @cause = false
+      end
+
+      # Marks state as failure
+      # @param cause [StandardError] error that occurred and caused failure
+      def failure!(cause)
+        @success = false
+        @cause = cause
+      end
+
+      # @return [Boolean] true if processing failed
+      def failure?
+        !@success
+      end
+    end
+  end
+end

data/lib/karafka/processing/scheduler.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # FIFO scheduler for messages coming from various topics and partitions
+    class Scheduler
+      # Schedules jobs in the fifo order
+      #
+      # @param queue [Karafka::Processing::JobsQueue] queue where we want to put the jobs
+      # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
+      def schedule_consumption(queue, jobs_array)
+        jobs_array.each do |job|
+          queue << job
+        end
+      end
+
+      # Both revocation and shutdown jobs can also run in fifo by default
+      alias schedule_revocation schedule_consumption
+      alias schedule_shutdown schedule_consumption
+    end
+  end
+end

data/lib/karafka/processing/strategies/aj_dlq_mom.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module Strategies
+      # ActiveJob strategy to cooperate with the DLQ.
+      #
+      # While AJ is uses MOM by default because it delegates the offset management to the AJ
+      # consumer. With DLQ however there is an extra case for skipping broken jobs with offset
+      # marking due to ordered processing.
+      module AjDlqMom
+        include DlqMom
+
+        # Apply strategy when only when using AJ with MOM and DLQ
+        FEATURES = %i[
+          active_job
+          dead_letter_queue
+          manual_offset_management
+        ].freeze
+
+        # How should we post-finalize consumption.
+        def handle_after_consume
+          return if revoked?
+
+          if coordinator.success?
+            # Do NOT commit offsets, they are comitted after each job in the AJ consumer.
+            coordinator.pause_tracker.reset
+          elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+            retry_after_pause
+          else
+            coordinator.pause_tracker.reset
+            skippable_message, = find_skippable_message
+            dispatch_to_dlq(skippable_message)
+            # We can commit the offset here because we know that we skip it "forever" and
+            # since AJ consumer commits the offset after each job, we also know that the
+            # previous job was successful
+            mark_as_consumed(skippable_message)
+            pause(coordinator.seek_offset, nil, false)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/strategies/aj_mom.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module Strategies
+      # ActiveJob enabled
+      # Manual offset management enabled
+      #
+      # This is the default AJ strategy since AJ cannot be used without MOM
+      module AjMom
+        include Mom
+
+        # Apply strategy when only when using AJ with MOM
+        FEATURES = %i[
+          active_job
+          manual_offset_management
+        ].freeze
+      end
+    end
+  end
+end

data/lib/karafka/processing/strategies/base.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Our processing patterns differ depending on various features configurations
+    # In this namespace we collect strategies for particular feature combinations to simplify the
+    # design. Based on features combinations we can then select handling strategy for a given case.
+    #
+    # @note The lack of common code here is intentional. It would get complex if there would be
+    #   any type of composition, so each strategy is expected to be self-sufficient
+    module Strategies
+      # Base strategy that should be included in each strategy, just to ensure the API
+      module Base
+        # What should happen before jobs are enqueued
+        # @note This runs from the listener thread, not recommended to put anything slow here
+        def handle_before_enqueue
+          raise NotImplementedError, 'Implement in a subclass'
+        end
+
+        # What should happen before we kick in the processing
+        def handle_before_consume
+          raise NotImplementedError, 'Implement in a subclass'
+        end
+
+        # What should happen in the processing
+        def handle_consume
+          raise NotImplementedError, 'Implement in a subclass'
+        end
+
+        # Post-consumption handling
+        def handle_after_consume
+          raise NotImplementedError, 'Implement in a subclass'
+        end
+
+        # Idle run handling
+        def handle_idle
+          raise NotImplementedError, 'Implement in a subclass'
+        end
+
+        # Revocation handling
+        def handle_revoked
+          raise NotImplementedError, 'Implement in a subclass'
+        end
+
+        # Shutdown handling
+        def handle_shutdown
+          raise NotImplementedError, 'Implement in a subclass'
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/strategies/default.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module Strategies
+      # No features enabled.
+      # No manual offset management
+      # No long running jobs
+      # Nothing. Just standard, automatic flow
+      module Default
+        include Base
+
+        # Apply strategy for a non-feature based flow
+        FEATURES = %i[].freeze
+
+        # Marks message as consumed in an async way.
+        #
+        # @param message [Messages::Message] last successfully processed message.
+        # @return [Boolean] true if we were able to mark the offset, false otherwise.
+        #   False indicates that we were not able and that we have lost the partition.
+        #
+        # @note We keep track of this offset in case we would mark as consumed and got error when
+        #   processing another message. In case like this we do not pause on the message we've
+        #   already processed but rather at the next one. This applies to both sync and async
+        #   versions of this method.
+        def mark_as_consumed(message)
+          # Ignore earlier offsets than the one we already committed
+          return true if coordinator.seek_offset > message.offset
+          return false if revoked?
+          return revoked? unless client.mark_as_consumed(message)
+
+          coordinator.seek_offset = message.offset + 1
+
+          true
+        end
+
+        # Marks message as consumed in a sync way.
+        #
+        # @param message [Messages::Message] last successfully processed message.
+        # @return [Boolean] true if we were able to mark the offset, false otherwise.
+        #   False indicates that we were not able and that we have lost the partition.
+        def mark_as_consumed!(message)
+          # Ignore earlier offsets than the one we already committed
+          return true if coordinator.seek_offset > message.offset
+          return false if revoked?
+
+          return revoked? unless client.mark_as_consumed!(message)
+
+          coordinator.seek_offset = message.offset + 1
+
+          true
+        end
+
+        # Triggers an async offset commit
+        #
+        # @param async [Boolean] should we use async (default) or sync commit
+        # @return [Boolean] true if we still own the partition.
+        # @note Due to its async nature, this may not fully represent the offset state in some
+        #   edge cases (like for example going beyond max.poll.interval)
+        def commit_offsets(async: true)
+          # Do not commit if we already lost the assignment
+          return false if revoked?
+          return true if client.commit_offsets(async: async)
+
+          # This will once more check the librdkafka revocation status and will revoke the
+          # coordinator in case it was not revoked
+          revoked?
+        end
+
+        # Triggers a synchronous offsets commit to Kafka
+        #
+        # @return [Boolean] true if we still own the partition, false otherwise.
+        # @note This is fully synchronous, hence the result of this can be used in DB transactions
+        #   etc as a way of making sure, that we still own the partition.
+        def commit_offsets!
+          commit_offsets(async: false)
+        end
+
+        # No actions needed for the standard flow here
+        def handle_before_enqueue
+          nil
+        end
+
+        # Increment number of attempts
+        def handle_before_consume
+          coordinator.pause_tracker.increment
+        end
+
+        # Run the user consumption code
+        def handle_consume
+          Karafka.monitor.instrument('consumer.consume', caller: self)
+          Karafka.monitor.instrument('consumer.consumed', caller: self) do
+            consume
+          end
+
+          # Mark job as successful
+          coordinator.success!(self)
+        rescue StandardError => e
+          coordinator.failure!(self, e)
+
+          # Re-raise so reported in the consumer
+          raise e
+        ensure
+          # We need to decrease number of jobs that this coordinator coordinates as it has finished
+          coordinator.decrement
+        end
+
+        # Standard flow marks work as consumed and moves on if everything went ok.
+        # If there was a processing error, we will pause and continue from the next message
+        # (next that is +1 from the last one that was successfully marked as consumed)
+        def handle_after_consume
+          return if revoked?
+
+          if coordinator.success?
+            coordinator.pause_tracker.reset
+
+            # We should not move the offset automatically when the partition was paused
+            # If we would not do this upon a revocation during the pause time, a different process
+            # would pick not from the place where we paused but from the offset that would be
+            # automatically committed here
+            return if coordinator.manual_pause?
+
+            mark_as_consumed(messages.last)
+          else
+            retry_after_pause
+          end
+        end
+
+        # Code that should run on idle runs without messages available
+        def handle_idle
+          nil
+        end
+
+        # We need to always un-pause the processing in case we have lost a given partition.
+        # Otherwise the underlying librdkafka would not know we may want to continue processing and
+        # the pause could in theory last forever
+        def handle_revoked
+          resume
+
+          coordinator.revoke
+
+          Karafka.monitor.instrument('consumer.revoke', caller: self)
+          Karafka.monitor.instrument('consumer.revoked', caller: self) do
+            revoked
+          end
+        end
+
+        # Runs the shutdown code
+        def handle_shutdown
+          Karafka.monitor.instrument('consumer.shutting_down', caller: self)
+          Karafka.monitor.instrument('consumer.shutdown', caller: self) do
+            shutdown
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/strategies/dlq.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module Strategies
+      # When using dead letter queue, processing won't stop after defined number of retries
+      # upon encountering non-critical errors but the messages that error will be moved to a
+      # separate topic with their payload and metadata, so they can be handled differently.
+      module Dlq
+        include Default
+
+        # Apply strategy when only dead letter queue is turned on
+        FEATURES = %i[
+          dead_letter_queue
+        ].freeze
+
+        # When manual offset management is on, we do not mark anything as consumed automatically
+        # and we rely on the user to figure things out
+        def handle_after_consume
+          return if revoked?
+
+          if coordinator.success?
+            coordinator.pause_tracker.reset
+
+            return if coordinator.manual_pause?
+
+            mark_as_consumed(messages.last)
+          elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+            retry_after_pause
+          # If we've reached number of retries that we could, we need to skip the first message
+          # that was not marked as consumed, pause and continue, while also moving this message
+          # to the dead topic
+          else
+            # We reset the pause to indicate we will now consider it as "ok".
+            coordinator.pause_tracker.reset
+
+            skippable_message, = find_skippable_message
+
+            # Send skippable message to the dql topic
+            dispatch_to_dlq(skippable_message)
+
+            # We mark the broken message as consumed and move on
+            mark_as_consumed(skippable_message)
+
+            return if revoked?
+
+            # We pause to backoff once just in case.
+            pause(coordinator.seek_offset, nil, false)
+          end
+        end
+
+        # Finds the message may want to skip (all, starting from first)
+        # @private
+        # @return [Array<Karafka::Messages::Message, Boolean>] message we may want to skip and
+        #   information if this message was from marked offset or figured out via mom flow
+        def find_skippable_message
+          skippable_message = messages.find do |msg|
+            coordinator.marked? && msg.offset == coordinator.seek_offset
+          end
+
+          # If we don't have the message matching the last comitted offset, it means that
+          # user operates with manual offsets and we're beyond the batch in which things
+          # broke for the first time. Then we skip the first (as no markings) and we
+          # move on one by one.
+          skippable_message ? [skippable_message, true] : [messages.first, false]
+        end
+
+        # Moves the broken message into a separate queue defined via the settings
+        # @private
+        # @param skippable_message [Karafka::Messages::Message] message we are skipping that also
+        #   should go to the dlq topic
+        def dispatch_to_dlq(skippable_message)
+          producer.produce_async(
+            topic: topic.dead_letter_queue.topic,
+            payload: skippable_message.raw_payload
+          )
+
+          # Notify about dispatch on the events bus
+          Karafka.monitor.instrument(
+            'dead_letter_queue.dispatched',
+            caller: self,
+            message: skippable_message
+          )
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/strategies/dlq_mom.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module Strategies
+      # Same as pure dead letter queue but we do not marked failed message as consumed
+      module DlqMom
+        include Dlq
+
+        # Apply strategy when dlq is on with manual offset management
+        FEATURES = %i[
+          dead_letter_queue
+          manual_offset_management
+        ].freeze
+
+        # When manual offset management is on, we do not mark anything as consumed automatically
+        # and we rely on the user to figure things out
+        def handle_after_consume
+          return if revoked?
+
+          if coordinator.success?
+            coordinator.pause_tracker.reset
+          elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+            retry_after_pause
+          # If we've reached number of retries that we could, we need to skip the first message
+          # that was not marked as consumed, pause and continue, while also moving this message
+          # to the dead topic
+          else
+            # We reset the pause to indicate we will now consider it as "ok".
+            coordinator.pause_tracker.reset
+
+            skippable_message, = find_skippable_message
+
+            dispatch_to_dlq(skippable_message)
+
+            # Save the next offset we want to go with after moving given message to DLQ
+            # Without this, we would not be able to move forward and we would end up
+            # in an infinite loop trying to un-pause from the message we've already processed
+            # Of course, since it's a MoM a rebalance or kill, will move it back as no
+            # offsets are being committed
+            coordinator.seek_offset = skippable_message.offset + 1
+
+            pause(coordinator.seek_offset, nil, false)
+          end
+        end
+      end
+    end
+  end
+end