karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/processing/consumer_groups/jobs/eofed.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
+    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
+    module ConsumerGroups
+      # Consumer-group-specific job types
+      module Jobs
+        # Job that runs the eofed operation when we receive eof without messages alongside.
+        class Eofed < ::Karafka::Processing::Jobs::Base
+          self.action = :eofed
+
+          # @param executor [Karafka::Processing::ConsumerGroups::Executor] executor that is supposed to run the job
+          # @return [Eofed]
+          def initialize(executor)
+            @executor = executor
+            super()
+          end
+
+          # Runs code prior to scheduling this eofed job
+          def before_schedule
+            executor.before_schedule_eofed
+          end
+
+          # Runs the eofed job via an executor.
+          def call
+            executor.eofed
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/consumer_groups/jobs/idle.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module ConsumerGroups
+      module Jobs
+        # Type of job that we may use to run some extra handling that happens without the user
+        # related lifecycle event like consumption, revocation, etc.
+        class Idle < ::Karafka::Processing::Jobs::Base
+          self.action = :idle
+
+          # @param executor [Karafka::Processing::ConsumerGroups::Executor] executor that is supposed to run a given
+          #   job on an active consumer
+          # @return [Idle]
+          def initialize(executor)
+            @executor = executor
+            super()
+          end
+
+          # Runs code prior to scheduling this idle job
+          def before_schedule
+            executor.before_schedule_idle
+          end
+
+          # Run the idle work via the executor
+          def call
+            executor.idle
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/consumer_groups/jobs/revoked.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
+    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
+    module ConsumerGroups
+      # Consumer-group-specific job types
+      module Jobs
+        # Job that runs the revoked operation when we lose a partition on a consumer that lost it.
+        class Revoked < ::Karafka::Processing::Jobs::Base
+          self.action = :revoked
+
+          # @param executor [Karafka::Processing::ConsumerGroups::Executor] executor that is supposed to run the job
+          # @return [Revoked]
+          def initialize(executor)
+            @executor = executor
+            super()
+          end
+
+          # Runs code prior to scheduling this revoked job
+          def before_schedule
+            executor.before_schedule_revoked
+          end
+
+          # Runs the revoking job via an executor.
+          def call
+            executor.revoked
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/consumer_groups/jobs/shutdown.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module ConsumerGroups
+      module Jobs
+        # Job that runs on each active consumer upon process shutdown (one job per consumer).
+        class Shutdown < ::Karafka::Processing::Jobs::Base
+          self.action = :shutdown
+
+          # @param executor [Karafka::Processing::ConsumerGroups::Executor] executor that is supposed to run a given
+          #   job on an active consumer
+          # @return [Shutdown]
+          def initialize(executor)
+            @executor = executor
+            super()
+          end
+
+          # Runs code prior to scheduling this shutdown job
+          def before_schedule
+            executor.before_schedule_shutdown
+          end
+
+          # Runs the shutdown job via an executor.
+          def call
+            executor.shutdown
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/consumer_groups/jobs_builder.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module ConsumerGroups
+      # Class responsible for deciding what type of job should we build to run a given command and
+      # for building a proper job for it.
+      class JobsBuilder
+        # @param executor [Karafka::Processing::ConsumerGroups::Executor]
+        # @param messages [Karafka::Messages::Messages] messages batch to be consumed
+        # @return [Karafka::Processing::ConsumerGroups::Jobs::Consume] consumption job
+        def consume(executor, messages)
+          Jobs::Consume.new(executor, messages)
+        end
+
+        # @param executor [Karafka::Processing::ConsumerGroups::Executor]
+        # @return [Karafka::Processing::ConsumerGroups::Jobs::Eofed] eofed job
+        def eofed(executor)
+          Jobs::Eofed.new(executor)
+        end
+
+        # @param executor [Karafka::Processing::ConsumerGroups::Executor]
+        # @return [Karafka::Processing::ConsumerGroups::Jobs::Revoked] revocation job
+        def revoked(executor)
+          Jobs::Revoked.new(executor)
+        end
+
+        # @param executor [Karafka::Processing::ConsumerGroups::Executor]
+        # @return [Karafka::Processing::ConsumerGroups::Jobs::Shutdown] shutdown job
+        def shutdown(executor)
+          Jobs::Shutdown.new(executor)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/consumer_groups/partitioner.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
+    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
+    module ConsumerGroups
+      # Basic partitioner for work division
+      # It does not divide any work.
+      class Partitioner
+        # @param subscription_group [Karafka::Routing::SubscriptionGroup] subscription group
+        def initialize(subscription_group)
+          @subscription_group = subscription_group
+        end
+
+        # @param _topic [String] topic name
+        # @param messages [Array<Karafka::Messages::Message>] karafka messages
+        # @param _coordinator [Karafka::Processing::ConsumerGroups::Coordinator] processing coordinator that will
+        #   be used with those messages
+        # @yieldparam [Integer] group id
+        # @yieldparam [Array<Karafka::Messages::Message>] karafka messages
+        def call(_topic, messages, _coordinator)
+          yield(0, messages)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/consumer_groups/strategies/aj_dlq_mom.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
+    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
+    module ConsumerGroups
+      module Strategies
+        # ActiveJob strategy to cooperate with the DLQ.
+        #
+        # While AJ is uses MOM by default because it delegates the offset management to the AJ
+        # consumer. With DLQ however there is an extra case for skipping broken jobs with offset
+        # marking due to ordered processing.
+        module AjDlqMom
+          include DlqMom
+
+          # Apply strategy when only when using AJ with MOM and DLQ
+          FEATURES = %i[
+            active_job
+            dead_letter_queue
+            manual_offset_management
+          ].freeze
+
+          # How should we post-finalize consumption.
+          def handle_after_consume
+            return if revoked?
+
+            if coordinator.success?
+              # Do NOT commit offsets, they are comitted after each job in the AJ consumer.
+              coordinator.pause_tracker.reset
+            elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+              retry_after_pause
+            else
+              coordinator.pause_tracker.reset
+              skippable_message, = find_skippable_message
+              dispatch_to_dlq(skippable_message)
+              # We can commit the offset here because we know that we skip it "forever" and
+              # since AJ consumer commits the offset after each job, we also know that the
+              # previous job was successful
+              mark_dispatched_to_dlq(skippable_message)
+              pause(seek_offset, nil, false)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/consumer_groups/strategies/aj_mom.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
+    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
+    module ConsumerGroups
+      module Strategies
+        # ActiveJob enabled
+        # Manual offset management enabled
+        #
+        # This is the default AJ strategy since AJ cannot be used without MOM
+        module AjMom
+          include Mom
+
+          # Apply strategy when only when using AJ with MOM
+          FEATURES = %i[
+            active_job
+            manual_offset_management
+          ].freeze
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/consumer_groups/strategies/base.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
+    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
+    module ConsumerGroups
+      # Our processing patterns differ depending on various features configurations
+      # In this namespace we collect strategies for particular feature combinations to simplify the
+      # design. Based on features combinations we can then select handling strategy for a given case.
+      #
+      # @note The lack of common code here is intentional. It would get complex if there would be
+      #   any type of composition, so each strategy is expected to be self-sufficient
+      module Strategies
+        # Base strategy that should be included in each strategy, just to ensure the API
+        module Base
+          # Defines all the before schedule handlers for appropriate actions
+          %i[
+            consume
+            idle
+            revoked
+            shutdown
+          ].each do |action|
+            class_eval <<~RUBY, __FILE__, __LINE__ + 1
+              def handle_before_schedule_#{action}
+                # What should happen before scheduling this work
+                raise NotImplementedError, 'Implement in a subclass'
+              end
+            RUBY
+          end
+
+          # What should happen before we kick in the processing
+          def handle_before_consume
+            raise NotImplementedError, "Implement in a subclass"
+          end
+
+          # What should happen in the processing
+          def handle_consume
+            raise NotImplementedError, "Implement in a subclass"
+          end
+
+          # Post-consumption handling
+          def handle_after_consume
+            raise NotImplementedError, "Implement in a subclass"
+          end
+
+          # Idle run handling
+          def handle_idle
+            raise NotImplementedError, "Implement in a subclass"
+          end
+
+          # Revocation handling
+          def handle_revoked
+            raise NotImplementedError, "Implement in a subclass"
+          end
+
+          # Shutdown handling
+          def handle_shutdown
+            raise NotImplementedError, "Implement in a subclass"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/consumer_groups/strategies/default.rb

@@ -0,0 +1,218 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
+    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
+    module ConsumerGroups
+      module Strategies
+        # No features enabled.
+        # No manual offset management
+        # No long running jobs
+        # Nothing. Just standard, automatic flow
+        module Default
+          include Base
+
+          # Apply strategy for a non-feature based flow
+          FEATURES = %i[].freeze
+
+          # By default on all "before schedule" we just run instrumentation, nothing more
+          %i[
+            consume
+            idle
+            eofed
+            revoked
+            shutdown
+          ].each do |action|
+            class_eval <<~RUBY, __FILE__, __LINE__ + 1
+              # No actions needed for the standard flow here
+              def handle_before_schedule_#{action}
+                monitor.instrument('consumer.before_schedule_#{action}', caller: self)
+
+                nil
+              end
+            RUBY
+          end
+
+          # Runs the post-creation, post-assignment code
+          # @note It runs in the listener loop. Should **not** be used for anything heavy or
+          #   with any potential errors. Mostly for initialization of states, etc.
+          def handle_initialized
+            monitor.instrument("consumer.initialize", caller: self)
+            monitor.instrument("consumer.initialized", caller: self) do
+              initialized
+            end
+          end
+
+          # Marks message as consumed in an async way.
+          #
+          # @param message [Messages::Message] last successfully processed message.
+          # @return [Boolean] true if we were able to mark the offset, false otherwise.
+          #   False indicates that we were not able and that we have lost the partition.
+          #
+          # @note We keep track of this offset in case we would mark as consumed and got error when
+          #   processing another message. In case like this we do not pause on the message we've
+          #   already processed but rather at the next one. This applies to both sync and async
+          #   versions of this method.
+          def mark_as_consumed(message)
+            # seek offset can be nil only in case `#seek` was invoked with offset reset request
+            # In case like this we ignore marking
+            return true if seek_offset.nil?
+            # Ignore double markings of the same offset
+            return true if (seek_offset - 1) == message.offset
+            return false if revoked?
+            return revoked? unless client.mark_as_consumed(message)
+
+            self.seek_offset = message.offset + 1
+
+            true
+          end
+
+          # Marks message as consumed in a sync way.
+          #
+          # @param message [Messages::Message] last successfully processed message.
+          # @return [Boolean] true if we were able to mark the offset, false otherwise.
+          #   False indicates that we were not able and that we have lost the partition.
+          def mark_as_consumed!(message)
+            # seek offset can be nil only in case `#seek` was invoked with offset reset request
+            # In case like this we ignore marking
+            return true if seek_offset.nil?
+            # Ignore double markings of the same offset
+            return true if (seek_offset - 1) == message.offset
+            return false if revoked?
+
+            return revoked? unless client.mark_as_consumed!(message)
+
+            self.seek_offset = message.offset + 1
+
+            true
+          end
+
+          # Triggers an async offset commit
+          #
+          # @param async [Boolean] should we use async (default) or sync commit
+          # @return [Boolean] true if we still own the partition.
+          # @note Due to its async nature, this may not fully represent the offset state in some
+          #   edge cases (like for example going beyond max.poll.interval)
+          def commit_offsets(async: true)
+            # Do not commit if we already lost the assignment
+            return false if revoked?
+            return true if client.commit_offsets(async: async)
+
+            # This will once more check the librdkafka revocation status and will revoke the
+            # coordinator in case it was not revoked
+            revoked?
+          end
+
+          # Triggers a synchronous offsets commit to Kafka
+          #
+          # @return [Boolean] true if we still own the partition, false otherwise.
+          # @note This is fully synchronous, hence the result of this can be used in DB transactions
+          #   etc as a way of making sure, that we still own the partition.
+          def commit_offsets!
+            commit_offsets(async: false)
+          end
+
+          # Increment number of attempts
+          def handle_before_consume
+            coordinator.pause_tracker.increment
+          end
+
+          # Runs the wrapping to execute appropriate action wrapped with the wrapper method code
+          #
+          # @param action [Symbol]
+          # @param block [Proc]
+          def handle_wrap(action, &block)
+            monitor.instrument("consumer.wrap", caller: self)
+            monitor.instrument("consumer.wrapped", caller: self) do
+              wrap(action, &block)
+            end
+          end
+
+          # Run the user consumption code
+          def handle_consume
+            monitor.instrument("consumer.consume", caller: self)
+            monitor.instrument("consumer.consumed", caller: self) do
+              consume
+            end
+
+            # Mark job as successful
+            coordinator.success!(self)
+          rescue => e
+            coordinator.failure!(self, e)
+
+            # Re-raise so reported in the consumer
+            raise e
+          ensure
+            # We need to decrease number of jobs that this coordinator coordinates as it has finished
+            coordinator.decrement(:consume)
+          end
+
+          # Standard flow marks work as consumed and moves on if everything went ok.
+          # If there was a processing error, we will pause and continue from the next message
+          # (next that is +1 from the last one that was successfully marked as consumed)
+          def handle_after_consume
+            return if revoked?
+
+            if coordinator.success?
+              coordinator.pause_tracker.reset
+
+              # We should not move the offset automatically when the partition was paused
+              # If we would not do this upon a revocation during the pause time, a different process
+              # would pick not from the place where we paused but from the offset that would be
+              # automatically committed here
+              return if coordinator.manual_pause?
+
+              mark_as_consumed(messages.last)
+            else
+              retry_after_pause
+            end
+          end
+
+          # Code that should run on idle runs without messages available
+          def handle_idle
+            nil
+          ensure
+            coordinator.decrement(:idle)
+          end
+
+          # Runs the consumer `#eofed` method with reporting
+          def handle_eofed
+            monitor.instrument("consumer.eof", caller: self)
+            monitor.instrument("consumer.eofed", caller: self) do
+              eofed
+            end
+          ensure
+            coordinator.decrement(:eofed)
+          end
+
+          # We need to always un-pause the processing in case we have lost a given partition.
+          # Otherwise the underlying librdkafka would not know we may want to continue processing and
+          # the pause could in theory last forever
+          def handle_revoked
+            resume
+
+            coordinator.revoke
+
+            monitor.instrument("consumer.revoke", caller: self)
+            monitor.instrument("consumer.revoked", caller: self) do
+              revoked
+            end
+          ensure
+            coordinator.decrement(:revoked)
+          end
+
+          # Runs the shutdown code
+          def handle_shutdown
+            monitor.instrument("consumer.shutting_down", caller: self)
+            monitor.instrument("consumer.shutdown", caller: self) do
+              shutdown
+            end
+          ensure
+            coordinator.decrement(:shutdown)
+          end
+        end
+      end
+    end
+  end
+end