karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/processing/jobs/consume.rb

@@ -1,47 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Processing
-    module Jobs
-      # The main job type. It runs the executor that triggers given topic partition messages
-      # processing in an underlying consumer instance.
-      class Consume < Base
-        # @return [Array<Rdkafka::Consumer::Message>] array with messages
-        attr_reader :messages
-
-        self.action = :consume
-
-        # @param executor [Karafka::Processing::Executor] executor that is suppose to run a given
-        #   job
-        # @param messages [Karafka::Messages::Messages] karafka messages batch
-        # @return [Consume]
-        def initialize(executor, messages)
-          @executor = executor
-          @messages = messages
-          super()
-        end
-
-        # Runs all the preparation code on the executor that needs to happen before the job is
-        # scheduled.
-        def before_schedule
-          executor.before_schedule_consume(@messages)
-        end
-
-        # Runs the before consumption preparations on the executor
-        def before_call
-          executor.before_consume
-        end
-
-        # Runs the given executor
-        def call
-          executor.consume
-        end
-
-        # Runs any error handling and other post-consumption stuff on the executor
-        def after_call
-          executor.after_consume
-        end
-      end
-    end
-  end
-end

data/lib/karafka/processing/jobs/eofed.rb

@@ -1,29 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Processing
-    module Jobs
-      # Job that runs the eofed operation when we receive eof without messages alongside.
-      class Eofed < Base
-        self.action = :eofed
-
-        # @param executor [Karafka::Processing::Executor] executor that is suppose to run the job
-        # @return [Eofed]
-        def initialize(executor)
-          @executor = executor
-          super()
-        end
-
-        # Runs code prior to scheduling this eofed job
-        def before_schedule
-          executor.before_schedule_eofed
-        end
-
-        # Runs the eofed job via an executor.
-        def call
-          executor.eofed
-        end
-      end
-    end
-  end
-end

data/lib/karafka/processing/jobs/idle.rb

@@ -1,31 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Processing
-    module Jobs
-      # Type of job that we may use to run some extra handling that happens without the user
-      # related lifecycle event like consumption, revocation, etc.
-      class Idle < Base
-        self.action = :idle
-
-        # @param executor [Karafka::Processing::Executor] executor that is suppose to run a given
-        #   job on an active consumer
-        # @return [Shutdown]
-        def initialize(executor)
-          @executor = executor
-          super()
-        end
-
-        # Runs code prior to scheduling this idle job
-        def before_schedule
-          executor.before_schedule_idle
-        end
-
-        # Run the idle work via the executor
-        def call
-          executor.idle
-        end
-      end
-    end
-  end
-end

data/lib/karafka/processing/jobs/revoked.rb

@@ -1,29 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Processing
-    module Jobs
-      # Job that runs the revoked operation when we loose a partition on a consumer that lost it.
-      class Revoked < Base
-        self.action = :revoked
-
-        # @param executor [Karafka::Processing::Executor] executor that is suppose to run the job
-        # @return [Revoked]
-        def initialize(executor)
-          @executor = executor
-          super()
-        end
-
-        # Runs code prior to scheduling this revoked job
-        def before_schedule
-          executor.before_schedule_revoked
-        end
-
-        # Runs the revoking job via an executor.
-        def call
-          executor.revoked
-        end
-      end
-    end
-  end
-end

data/lib/karafka/processing/jobs/shutdown.rb

@@ -1,30 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Processing
-    module Jobs
-      # Job that runs on each active consumer upon process shutdown (one job per consumer).
-      class Shutdown < Base
-        self.action = :shutdown
-
-        # @param executor [Karafka::Processing::Executor] executor that is suppose to run a given
-        #   job on an active consumer
-        # @return [Shutdown]
-        def initialize(executor)
-          @executor = executor
-          super()
-        end
-
-        # Runs code prior to scheduling this shutdown job
-        def before_schedule
-          executor.before_schedule_shutdown
-        end
-
-        # Runs the shutdown job via an executor.
-        def call
-          executor.shutdown
-        end
-      end
-    end
-  end
-end

data/lib/karafka/processing/jobs_builder.rb

@@ -1,34 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Processing
-    # Class responsible for deciding what type of job should we build to run a given command and
-    # for building a proper job for it.
-    class JobsBuilder
-      # @param executor [Karafka::Processing::Executor]
-      # @param messages [Karafka::Messages::Messages] messages batch to be consumed
-      # @return [Karafka::Processing::Jobs::Consume] consumption job
-      def consume(executor, messages)
-        Jobs::Consume.new(executor, messages)
-      end
-
-      # @param executor [Karafka::Processing::Executor]
-      # @return [Karafka::Processing::Jobs::Eofed] eofed job
-      def eofed(executor)
-        Jobs::Eofed.new(executor)
-      end
-
-      # @param executor [Karafka::Processing::Executor]
-      # @return [Karafka::Processing::Jobs::Revoked] revocation job
-      def revoked(executor)
-        Jobs::Revoked.new(executor)
-      end
-
-      # @param executor [Karafka::Processing::Executor]
-      # @return [Karafka::Processing::Jobs::Shutdown] shutdown job
-      def shutdown(executor)
-        Jobs::Shutdown.new(executor)
-      end
-    end
-  end
-end

data/lib/karafka/processing/strategies/aj_dlq_mom.rb

@@ -1,44 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Processing
-    module Strategies
-      # ActiveJob strategy to cooperate with the DLQ.
-      #
-      # While AJ is uses MOM by default because it delegates the offset management to the AJ
-      # consumer. With DLQ however there is an extra case for skipping broken jobs with offset
-      # marking due to ordered processing.
-      module AjDlqMom
-        include DlqMom
-
-        # Apply strategy when only when using AJ with MOM and DLQ
-        FEATURES = %i[
-          active_job
-          dead_letter_queue
-          manual_offset_management
-        ].freeze
-
-        # How should we post-finalize consumption.
-        def handle_after_consume
-          return if revoked?
-
-          if coordinator.success?
-            # Do NOT commit offsets, they are comitted after each job in the AJ consumer.
-            coordinator.pause_tracker.reset
-          elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-            retry_after_pause
-          else
-            coordinator.pause_tracker.reset
-            skippable_message, = find_skippable_message
-            dispatch_to_dlq(skippable_message)
-            # We can commit the offset here because we know that we skip it "forever" and
-            # since AJ consumer commits the offset after each job, we also know that the
-            # previous job was successful
-            mark_dispatched_to_dlq(skippable_message)
-            pause(seek_offset, nil, false)
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/processing/strategies/aj_mom.rb

@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Processing
-    module Strategies
-      # ActiveJob enabled
-      # Manual offset management enabled
-      #
-      # This is the default AJ strategy since AJ cannot be used without MOM
-      module AjMom
-        include Mom
-
-        # Apply strategy when only when using AJ with MOM
-        FEATURES = %i[
-          active_job
-          manual_offset_management
-        ].freeze
-      end
-    end
-  end
-end

data/lib/karafka/processing/strategies/base.rb

@@ -1,61 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Processing
-    # Our processing patterns differ depending on various features configurations
-    # In this namespace we collect strategies for particular feature combinations to simplify the
-    # design. Based on features combinations we can then select handling strategy for a given case.
-    #
-    # @note The lack of common code here is intentional. It would get complex if there would be
-    #   any type of composition, so each strategy is expected to be self-sufficient
-    module Strategies
-      # Base strategy that should be included in each strategy, just to ensure the API
-      module Base
-        # Defines all the before schedule handlers for appropriate actions
-        %i[
-          consume
-          idle
-          revoked
-          shutdown
-        ].each do |action|
-          class_eval <<~RUBY, __FILE__, __LINE__ + 1
-            def handle_before_schedule_#{action}
-              # What should happen before scheduling this work
-              raise NotImplementedError, 'Implement in a subclass'
-            end
-          RUBY
-        end
-
-        # What should happen before we kick in the processing
-        def handle_before_consume
-          raise NotImplementedError, "Implement in a subclass"
-        end
-
-        # What should happen in the processing
-        def handle_consume
-          raise NotImplementedError, "Implement in a subclass"
-        end
-
-        # Post-consumption handling
-        def handle_after_consume
-          raise NotImplementedError, "Implement in a subclass"
-        end
-
-        # Idle run handling
-        def handle_idle
-          raise NotImplementedError, "Implement in a subclass"
-        end
-
-        # Revocation handling
-        def handle_revoked
-          raise NotImplementedError, "Implement in a subclass"
-        end
-
-        # Shutdown handling
-        def handle_shutdown
-          raise NotImplementedError, "Implement in a subclass"
-        end
-      end
-    end
-  end
-end

data/lib/karafka/processing/strategies/default.rb

@@ -1,214 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Processing
-    module Strategies
-      # No features enabled.
-      # No manual offset management
-      # No long running jobs
-      # Nothing. Just standard, automatic flow
-      module Default
-        include Base
-
-        # Apply strategy for a non-feature based flow
-        FEATURES = %i[].freeze
-
-        # By default on all "before schedule" we just run instrumentation, nothing more
-        %i[
-          consume
-          idle
-          eofed
-          revoked
-          shutdown
-        ].each do |action|
-          class_eval <<~RUBY, __FILE__, __LINE__ + 1
-            # No actions needed for the standard flow here
-            def handle_before_schedule_#{action}
-              monitor.instrument('consumer.before_schedule_#{action}', caller: self)
-
-              nil
-            end
-          RUBY
-        end
-
-        # Runs the post-creation, post-assignment code
-        # @note It runs in the listener loop. Should **not** be used for anything heavy or
-        #   with any potential errors. Mostly for initialization of states, etc.
-        def handle_initialized
-          monitor.instrument("consumer.initialize", caller: self)
-          monitor.instrument("consumer.initialized", caller: self) do
-            initialized
-          end
-        end
-
-        # Marks message as consumed in an async way.
-        #
-        # @param message [Messages::Message] last successfully processed message.
-        # @return [Boolean] true if we were able to mark the offset, false otherwise.
-        #   False indicates that we were not able and that we have lost the partition.
-        #
-        # @note We keep track of this offset in case we would mark as consumed and got error when
-        #   processing another message. In case like this we do not pause on the message we've
-        #   already processed but rather at the next one. This applies to both sync and async
-        #   versions of this method.
-        def mark_as_consumed(message)
-          # seek offset can be nil only in case `#seek` was invoked with offset reset request
-          # In case like this we ignore marking
-          return true if seek_offset.nil?
-          # Ignore double markings of the same offset
-          return true if (seek_offset - 1) == message.offset
-          return false if revoked?
-          return revoked? unless client.mark_as_consumed(message)
-
-          self.seek_offset = message.offset + 1
-
-          true
-        end
-
-        # Marks message as consumed in a sync way.
-        #
-        # @param message [Messages::Message] last successfully processed message.
-        # @return [Boolean] true if we were able to mark the offset, false otherwise.
-        #   False indicates that we were not able and that we have lost the partition.
-        def mark_as_consumed!(message)
-          # seek offset can be nil only in case `#seek` was invoked with offset reset request
-          # In case like this we ignore marking
-          return true if seek_offset.nil?
-          # Ignore double markings of the same offset
-          return true if (seek_offset - 1) == message.offset
-          return false if revoked?
-
-          return revoked? unless client.mark_as_consumed!(message)
-
-          self.seek_offset = message.offset + 1
-
-          true
-        end
-
-        # Triggers an async offset commit
-        #
-        # @param async [Boolean] should we use async (default) or sync commit
-        # @return [Boolean] true if we still own the partition.
-        # @note Due to its async nature, this may not fully represent the offset state in some
-        #   edge cases (like for example going beyond max.poll.interval)
-        def commit_offsets(async: true)
-          # Do not commit if we already lost the assignment
-          return false if revoked?
-          return true if client.commit_offsets(async: async)
-
-          # This will once more check the librdkafka revocation status and will revoke the
-          # coordinator in case it was not revoked
-          revoked?
-        end
-
-        # Triggers a synchronous offsets commit to Kafka
-        #
-        # @return [Boolean] true if we still own the partition, false otherwise.
-        # @note This is fully synchronous, hence the result of this can be used in DB transactions
-        #   etc as a way of making sure, that we still own the partition.
-        def commit_offsets!
-          commit_offsets(async: false)
-        end
-
-        # Increment number of attempts
-        def handle_before_consume
-          coordinator.pause_tracker.increment
-        end
-
-        # Runs the wrapping to execute appropriate action wrapped with the wrapper method code
-        #
-        # @param action [Symbol]
-        # @param block [Proc]
-        def handle_wrap(action, &block)
-          monitor.instrument("consumer.wrap", caller: self)
-          monitor.instrument("consumer.wrapped", caller: self) do
-            wrap(action, &block)
-          end
-        end
-
-        # Run the user consumption code
-        def handle_consume
-          monitor.instrument("consumer.consume", caller: self)
-          monitor.instrument("consumer.consumed", caller: self) do
-            consume
-          end
-
-          # Mark job as successful
-          coordinator.success!(self)
-        rescue => e
-          coordinator.failure!(self, e)
-
-          # Re-raise so reported in the consumer
-          raise e
-        ensure
-          # We need to decrease number of jobs that this coordinator coordinates as it has finished
-          coordinator.decrement(:consume)
-        end
-
-        # Standard flow marks work as consumed and moves on if everything went ok.
-        # If there was a processing error, we will pause and continue from the next message
-        # (next that is +1 from the last one that was successfully marked as consumed)
-        def handle_after_consume
-          return if revoked?
-
-          if coordinator.success?
-            coordinator.pause_tracker.reset
-
-            # We should not move the offset automatically when the partition was paused
-            # If we would not do this upon a revocation during the pause time, a different process
-            # would pick not from the place where we paused but from the offset that would be
-            # automatically committed here
-            return if coordinator.manual_pause?
-
-            mark_as_consumed(messages.last)
-          else
-            retry_after_pause
-          end
-        end
-
-        # Code that should run on idle runs without messages available
-        def handle_idle
-          nil
-        ensure
-          coordinator.decrement(:idle)
-        end
-
-        # Runs the consumer `#eofed` method with reporting
-        def handle_eofed
-          monitor.instrument("consumer.eof", caller: self)
-          monitor.instrument("consumer.eofed", caller: self) do
-            eofed
-          end
-        ensure
-          coordinator.decrement(:eofed)
-        end
-
-        # We need to always un-pause the processing in case we have lost a given partition.
-        # Otherwise the underlying librdkafka would not know we may want to continue processing and
-        # the pause could in theory last forever
-        def handle_revoked
-          resume
-
-          coordinator.revoke
-
-          monitor.instrument("consumer.revoke", caller: self)
-          monitor.instrument("consumer.revoked", caller: self) do
-            revoked
-          end
-        ensure
-          coordinator.decrement(:revoked)
-        end
-
-        # Runs the shutdown code
-        def handle_shutdown
-          monitor.instrument("consumer.shutting_down", caller: self)
-          monitor.instrument("consumer.shutdown", caller: self) do
-            shutdown
-          end
-        ensure
-          coordinator.decrement(:shutdown)
-        end
-      end
-    end
-  end
-end