karafka 2.4.0 → 2.4.18

data/lib/karafka/processing/jobs/shutdown.rb

@@ -5,6 +5,8 @@ module Karafka
     module Jobs
       # Job that runs on each active consumer upon process shutdown (one job per consumer).
       class Shutdown < Base
+        self.action = :shutdown
+
         # @param executor [Karafka::Processing::Executor] executor that is suppose to run a given
         #   job on an active consumer
         # @return [Shutdown]

data/lib/karafka/processing/jobs_builder.rb

@@ -12,6 +12,12 @@ module Karafka
         Jobs::Consume.new(executor, messages)
       end
 
+      # @param executor [Karafka::Processing::Executor]
+      # @return [Karafka::Processing::Jobs::Eofed] eofed job
+      def eofed(executor)
+        Jobs::Eofed.new(executor)
+      end
+
       # @param executor [Karafka::Processing::Executor]
       # @return [Karafka::Processing::Jobs::Revoked] revocation job
       def revoked(executor)

data/lib/karafka/processing/schedulers/default.rb

@@ -24,6 +24,7 @@ module Karafka
         alias on_schedule_revocation on_schedule_consumption
         alias on_schedule_shutdown on_schedule_consumption
         alias on_schedule_idle on_schedule_consumption
+        alias on_schedule_eofed on_schedule_consumption
 
         # This scheduler does not have anything to manage as it is a pass through and has no state
         def on_manage

data/lib/karafka/processing/strategies/aj_dlq_mom.rb

@@ -35,7 +35,7 @@ module Karafka
             # since AJ consumer commits the offset after each job, we also know that the
             # previous job was successful
             mark_dispatched_to_dlq(skippable_message)
-            pause(coordinator.seek_offset, nil, false)
+            pause(seek_offset, nil, false)
           end
         end
       end

data/lib/karafka/processing/strategies/default.rb

@@ -17,19 +17,30 @@ module Karafka
         %i[
           consume
           idle
+          eofed
           revoked
           shutdown
         ].each do |action|
           class_eval <<~RUBY, __FILE__, __LINE__ + 1
             # No actions needed for the standard flow here
             def handle_before_schedule_#{action}
-              Karafka.monitor.instrument('consumer.before_schedule_#{action}', caller: self)
+              monitor.instrument('consumer.before_schedule_#{action}', caller: self)
 
               nil
             end
           RUBY
         end
 
+        # Runs the post-creation, post-assignment code
+        # @note It runs in the listener loop. Should **not** be used for anything heavy or
+        #   with any potential errors. Mostly for initialization of states, etc.
+        def handle_initialized
+          monitor.instrument('consumer.initialize', caller: self)
+          monitor.instrument('consumer.initialized', caller: self) do
+            initialized
+          end
+        end
+
         # Marks message as consumed in an async way.
         #
         # @param message [Messages::Message] last successfully processed message.
@@ -43,13 +54,13 @@ module Karafka
         def mark_as_consumed(message)
           # seek offset can be nil only in case `#seek` was invoked with offset reset request
           # In case like this we ignore marking
-          return true if coordinator.seek_offset.nil?
+          return true if seek_offset.nil?
           # Ignore earlier offsets than the one we already committed
-          return true if coordinator.seek_offset > message.offset
+          return true if seek_offset > message.offset
           return false if revoked?
           return revoked? unless client.mark_as_consumed(message)
 
-          coordinator.seek_offset = message.offset + 1
+          self.seek_offset = message.offset + 1
 
           true
         end
@@ -62,14 +73,14 @@ module Karafka
         def mark_as_consumed!(message)
           # seek offset can be nil only in case `#seek` was invoked with offset reset request
           # In case like this we ignore marking
-          return true if coordinator.seek_offset.nil?
+          return true if seek_offset.nil?
           # Ignore earlier offsets than the one we already committed
-          return true if coordinator.seek_offset > message.offset
+          return true if seek_offset > message.offset
           return false if revoked?
 
           return revoked? unless client.mark_as_consumed!(message)
 
-          coordinator.seek_offset = message.offset + 1
+          self.seek_offset = message.offset + 1
 
           true
         end
@@ -104,10 +115,21 @@ module Karafka
           coordinator.pause_tracker.increment
         end
 
+        # Runs the wrapping to execute appropriate action wrapped with the wrapper method code
+        #
+        # @param action [Symbol]
+        # @param block [Proc]
+        def handle_wrap(action, &block)
+          monitor.instrument('consumer.wrap', caller: self)
+          monitor.instrument('consumer.wrapped', caller: self) do
+            wrap(action, &block)
+          end
+        end
+
         # Run the user consumption code
         def handle_consume
-          Karafka.monitor.instrument('consumer.consume', caller: self)
-          Karafka.monitor.instrument('consumer.consumed', caller: self) do
+          monitor.instrument('consumer.consume', caller: self)
+          monitor.instrument('consumer.consumed', caller: self) do
             consume
           end
 
@@ -151,6 +173,16 @@ module Karafka
           coordinator.decrement(:idle)
         end
 
+        # Runs the consumer `#eofed` method with reporting
+        def handle_eofed
+          monitor.instrument('consumer.eof', caller: self)
+          monitor.instrument('consumer.eofed', caller: self) do
+            eofed
+          end
+        ensure
+          coordinator.decrement(:eofed)
+        end
+
         # We need to always un-pause the processing in case we have lost a given partition.
         # Otherwise the underlying librdkafka would not know we may want to continue processing and
         # the pause could in theory last forever
@@ -159,8 +191,8 @@ module Karafka
 
           coordinator.revoke
 
-          Karafka.monitor.instrument('consumer.revoke', caller: self)
-          Karafka.monitor.instrument('consumer.revoked', caller: self) do
+          monitor.instrument('consumer.revoke', caller: self)
+          monitor.instrument('consumer.revoked', caller: self) do
             revoked
           end
         ensure
@@ -169,8 +201,8 @@ module Karafka
 
         # Runs the shutdown code
         def handle_shutdown
-          Karafka.monitor.instrument('consumer.shutting_down', caller: self)
-          Karafka.monitor.instrument('consumer.shutdown', caller: self) do
+          monitor.instrument('consumer.shutting_down', caller: self)
+          monitor.instrument('consumer.shutdown', caller: self) do
             shutdown
           end
         ensure

data/lib/karafka/processing/strategies/dlq.rb

@@ -76,12 +76,16 @@ module Karafka
             dispatch_to_dlq(skippable_message)
 
             # We mark the broken message as consumed and move on
-            mark_dispatched_to_dlq(skippable_message)
+            if mark_after_dispatch?
+              mark_dispatched_to_dlq(skippable_message)
 
-            return if revoked?
+              return if revoked?
+            else
+              self.seek_offset = skippable_message.offset + 1
+            end
 
             # We pause to backoff once just in case.
-            pause(coordinator.seek_offset, nil, false)
+            pause(seek_offset, nil, false)
           end
         end
 
@@ -91,7 +95,7 @@ module Karafka
         #   information if this message was from marked offset or figured out via mom flow
         def find_skippable_message
           skippable_message = messages.find do |msg|
-            coordinator.marked? && msg.offset == coordinator.seek_offset
+            coordinator.marked? && msg.offset == seek_offset
           end
 
           # If we don't have the message matching the last comitted offset, it means that
@@ -113,13 +117,23 @@ module Karafka
           )
 
           # Notify about dispatch on the events bus
-          Karafka.monitor.instrument(
+          monitor.instrument(
             'dead_letter_queue.dispatched',
             caller: self,
             message: skippable_message
           )
         end
 
+        # @return [Boolean] should we mark given message as consumed after dispatch. For default
+        #  non MOM strategies if user did not explicitly tell us not to, we mark it. Default is
+        #  `nil`, which means `true` in this case. If user provided alternative value, we go
+        #  with it.
+        def mark_after_dispatch?
+          return true if topic.dead_letter_queue.mark_after_dispatch.nil?
+
+          topic.dead_letter_queue.mark_after_dispatch
+        end
+
         # Marks message that went to DLQ (if applicable) based on the requested method
         # @param skippable_message [Karafka::Messages::Message]
         def mark_dispatched_to_dlq(skippable_message)

data/lib/karafka/processing/strategies/dlq_mom.rb

@@ -33,16 +33,35 @@ module Karafka
 
             dispatch_to_dlq(skippable_message)
 
-            # Save the next offset we want to go with after moving given message to DLQ
-            # Without this, we would not be able to move forward and we would end up
-            # in an infinite loop trying to un-pause from the message we've already processed
-            # Of course, since it's a MoM a rebalance or kill, will move it back as no
-            # offsets are being committed
-            coordinator.seek_offset = skippable_message.offset + 1
-
-            pause(coordinator.seek_offset, nil, false)
+            # We mark the broken message as consumed and move on
+            if mark_after_dispatch?
+              mark_dispatched_to_dlq(skippable_message)
+
+              return if revoked?
+            else
+              # Save the next offset we want to go with after moving given message to DLQ
+              # Without this, we would not be able to move forward and we would end up
+              # in an infinite loop trying to un-pause from the message we've already processed
+              # Of course, since it's a MoM a rebalance or kill, will move it back as no
+              # offsets are being committed
+              self.seek_offset = skippable_message.offset + 1
+            end
+
+            pause(seek_offset, nil, false)
           end
         end
+
+        # @return [Boolean] should we mark given message as consumed after dispatch. For
+        #  MOM strategies if user did not explicitly tell us to mark, we do not mark. Default is
+        #  `nil`, which means `false` in this case. If user provided alternative value, we go
+        #  with it.
+        #
+        # @note Please note, this is the opposite behavior than in case of AOM strategies.
+        def mark_after_dispatch?
+          return false if topic.dead_letter_queue.mark_after_dispatch.nil?
+
+          topic.dead_letter_queue.mark_after_dispatch
+        end
       end
     end
   end

data/lib/karafka/processing/worker.rb

@@ -18,6 +18,10 @@ module Karafka
     #                not user code but need to run after user code base is executed.
     class Worker
       include Helpers::Async
+      include Helpers::ConfigImporter.new(
+        worker_job_call_wrapper: %i[internal processing worker_job_call_wrapper],
+        monitor: %i[monitor]
+      )
 
       # @return [String] id of this worker
       attr_reader :id
@@ -27,6 +31,7 @@ module Karafka
       def initialize(jobs_queue)
         @id = SecureRandom.hex(6)
         @jobs_queue = jobs_queue
+        @non_wrapped_flow = worker_job_call_wrapper == false
       end
 
       private
@@ -49,22 +54,30 @@ module Karafka
         instrument_details = { caller: self, job: job, jobs_queue: @jobs_queue }
 
         if job
-          Karafka.monitor.instrument('worker.process', instrument_details)
+          job.wrap do
+            monitor.instrument('worker.process', instrument_details)
 
-          Karafka.monitor.instrument('worker.processed', instrument_details) do
-            job.before_call
+            monitor.instrument('worker.processed', instrument_details) do
+              job.before_call
 
-            # If a job is marked as non blocking, we can run a tick in the job queue and if there
-            # are no other blocking factors, the job queue will be unlocked.
-            # If this does not run, all the things will be blocking and job queue won't allow to
-            # pass it until done.
-            @jobs_queue.tick(job.group_id) if job.non_blocking?
+              # If a job is marked as non blocking, we can run a tick in the job queue and if there
+              # are no other blocking factors, the job queue will be unlocked.
+              # If this does not run, all the things will be blocking and job queue won't allow to
+              # pass it until done.
+              @jobs_queue.tick(job.group_id) if job.non_blocking?
 
-            job.call
+              if @non_wrapped_flow
+                job.call
+              else
+                worker_job_call_wrapper.wrap do
+                  job.call
+                end
+              end
 
-            job.after_call
+              job.after_call
 
-            true
+              true
+            end
           end
         else
           false
@@ -73,7 +86,7 @@ module Karafka
       # rubocop:disable Lint/RescueException
       rescue Exception => e
         # rubocop:enable Lint/RescueException
-        Karafka.monitor.instrument(
+        monitor.instrument(
           'error.occurred',
           caller: self,
           job: job,
@@ -89,7 +102,7 @@ module Karafka
         end
 
         # Always publish info, that we completed all the work despite its result
-        Karafka.monitor.instrument('worker.completed', instrument_details)
+        monitor.instrument('worker.completed', instrument_details)
       end
     end
   end

data/lib/karafka/railtie.rb

@@ -65,48 +65,6 @@ if Karafka.rails?
         app.config.autoload_paths += %w[app/consumers]
       end
 
-      initializer 'karafka.configure_rails_code_reloader' do
-        # There are components that won't work with older Rails version, so we check it and
-        # provide a failover
-        rails6plus = Rails.gem_version >= Gem::Version.new('6.0.0')
-
-        next unless Rails.env.development?
-        next unless ENV.key?('KARAFKA_CLI')
-        next unless rails6plus
-
-        # We can have many listeners, but it does not matter in which we will reload the code
-        # as long as all the consumers will be re-created as Rails reload is thread-safe
-        ::Karafka::App.monitor.subscribe('connection.listener.fetch_loop') do
-          # If consumer persistence is enabled, no reason to reload because we will still keep
-          # old consumer instances in memory.
-          next if Karafka::App.config.consumer_persistence
-          # Reload code each time there is a change in the code
-          next unless Rails.application.reloaders.any?(&:updated?)
-
-          Rails.application.reloader.reload!
-        end
-      end
-
-      initializer 'karafka.release_active_record_connections' do
-        rails7plus = Rails.gem_version >= Gem::Version.new('7.0.0')
-
-        ActiveSupport.on_load(:active_record) do
-          ::Karafka::App.monitor.subscribe('worker.completed') do
-            # Always release the connection after processing is done. Otherwise thread may hang
-            # blocking the reload and further processing
-            # @see https://github.com/rails/rails/issues/44183
-            #
-            # The change technically happens in 7.1 but 7.0 already supports this so we can make
-            # a proper change for 7.0+
-            if rails7plus
-              ActiveRecord::Base.connection_handler.clear_active_connections!
-            else
-              ActiveRecord::Base.clear_active_connections!
-            end
-          end
-        end
-      end
-
       initializer 'karafka.require_karafka_boot_file' do |app|
         rails6plus = Rails.gem_version >= Gem::Version.new('6.0.0')
 
@@ -136,6 +94,17 @@ if Karafka.rails?
           end
         end
       end
+
+      initializer 'karafka.configure_worker_external_executor' do |app|
+        app.config.after_initialize do
+          app_config = Karafka::App.config
+
+          # We need to wrap execution of the core user code with a wrapper in case of Rails, so
+          # the auto-reload works as expected
+          worker_job_call_wrapper = app_config.consumer_persistence ? app.executor : app.reloader
+          app_config.internal.processing.worker_job_call_wrapper = worker_job_call_wrapper
+        end
+      end
     end
   end
 end

data/lib/karafka/routing/builder.rb

@@ -48,6 +48,10 @@ module Karafka
 
           instance_eval(&block)
 
+          # Ensures high-level routing details consistency
+          # Contains checks that require knowledge about all the consumer groups to operate
+          Contracts::Routing.new.validate!(map(&:to_h))
+
           each do |consumer_group|
             # Validate consumer group settings
             Contracts::ConsumerGroup.new.validate!(consumer_group.to_h)
@@ -63,6 +67,20 @@ module Karafka
         end
       end
 
+      # Clear out the drawn routes.
+      alias array_clear clear
+      private :array_clear
+
+      # Clear routes and draw them again with the given block. Helpful for testing purposes.
+      # @param block [Proc] block we will evaluate within the builder context
+      def redraw(&block)
+        @mutex.synchronize do
+          @draws.clear
+          array_clear
+        end
+        draw(&block)
+      end
+
       # @return [Array<Karafka::Routing::ConsumerGroup>] only active consumer groups that
       #   we want to use. Since Karafka supports multi-process setup, we need to be able
       #   to pick only those consumer groups that should be active in our given process context
@@ -75,7 +93,7 @@ module Karafka
         @mutex.synchronize do
           @defaults = EMPTY_DEFAULTS
           @draws.clear
-          super
+          array_clear
         end
       end
 

data/lib/karafka/routing/consumer_group.rb

@@ -8,6 +8,12 @@ module Karafka
     # @note A single consumer group represents Kafka consumer group, but it may not match 1:1 with
     #   subscription groups. There can be more subscription groups than consumer groups
     class ConsumerGroup
+      include Helpers::ConfigImporter.new(
+        activity_manager: %i[internal routing activity_manager],
+        builder: %i[internal routing builder],
+        subscription_groups_builder: %i[internal routing subscription_groups_builder]
+      )
+
       attr_reader :id, :topics, :name
 
       # This is a "virtual" attribute that is not building subscription groups.
@@ -32,7 +38,7 @@ module Karafka
 
       # @return [Boolean] true if this consumer group should be active in our current process
       def active?
-        config.internal.routing.activity_manager.active?(:consumer_groups, name)
+        activity_manager.active?(:consumer_groups, name)
       end
 
       # Builds a topic representation inside of a current consumer group route
@@ -43,7 +49,7 @@ module Karafka
         topic = Topic.new(name, self)
         @topics << Proxy.new(
           topic,
-          config.internal.routing.builder.defaults,
+          builder.defaults,
           &block
         ).target
         built_topic = @topics.last
@@ -72,11 +78,7 @@ module Karafka
       # @return [Array<Routing::SubscriptionGroup>] all the subscription groups build based on
       #   the consumer group topics
       def subscription_groups
-        @subscription_groups ||= config
-                                 .internal
-                                 .routing
-                                 .subscription_groups_builder
-                                 .call(topics)
+        @subscription_groups ||= subscription_groups_builder.call(topics)
       end
 
       # Hashed version of consumer group that can be used for validation purposes
@@ -88,13 +90,6 @@ module Karafka
           id: id
         }.freeze
       end
-
-      private
-
-      # @return [Karafka::Core::Configurable::Node] root node config
-      def config
-        ::Karafka::App.config
-      end
     end
   end
 end

data/lib/karafka/routing/features/dead_letter_queue/config.rb

@@ -21,6 +21,9 @@ module Karafka
           :dispatch_method,
           # Should we use `#mark_as_consumed` or `#mark_as_consumed!` (in flows that mark)
           :marking_method,
+          # Should we mark as consumed after dispatch or not. True for most cases, except MOM where
+          # it is on user to decide (false by default)
+          :mark_after_dispatch,
           # Initialize with kwargs
           keyword_init: true
         ) do

data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb

@@ -21,6 +21,7 @@ module Karafka
               required(:independent) { |val| [true, false].include?(val) }
               required(:max_retries) { |val| val.is_a?(Integer) && val >= 0 }
               required(:transactional) { |val| [true, false].include?(val) }
+              required(:mark_after_dispatch) { |val| [true, false, nil].include?(val) }
 
               required(:dispatch_method) do |val|
                 %i[produce_async produce_sync].include?(val)

data/lib/karafka/routing/features/dead_letter_queue/topic.rb

@@ -22,6 +22,9 @@ module Karafka
           #   whether dispatch on dlq should be sync or async (async by default)
           # @param marking_method [Symbol] `:mark_as_consumed` or `:mark_as_consumed!`. Describes
           #   whether marking on DLQ should be async or sync (async by default)
+          # @param mark_after_dispatch [Boolean, nil] Should we mark after dispatch. `nil` means
+          #   that the default strategy approach to marking will be used. `true` or `false`
+          #   overwrites the default
           # @return [Config] defined config
           def dead_letter_queue(
             max_retries: DEFAULT_MAX_RETRIES,
@@ -29,7 +32,8 @@ module Karafka
             independent: false,
             transactional: true,
             dispatch_method: :produce_async,
-            marking_method: :mark_as_consumed
+            marking_method: :mark_as_consumed,
+            mark_after_dispatch: nil
           )
             @dead_letter_queue ||= Config.new(
               active: !topic.nil?,
@@ -38,7 +42,8 @@ module Karafka
               independent: independent,
               transactional: transactional,
               dispatch_method: dispatch_method,
-              marking_method: marking_method
+              marking_method: marking_method,
+              mark_after_dispatch: mark_after_dispatch
             )
           end
 

data/lib/karafka/routing/features/eofed/config.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class Eofed < Base
+        # Config of this feature
+        Config = Struct.new(
+          :active,
+          keyword_init: true
+        ) { alias_method :active?, :active }
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/eofed/contracts/topic.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class Eofed < Base
+        # Eofed related contracts namespace
+        module Contracts
+          # Contract for eofed topic setup
+          class Topic < Karafka::Contracts::Base
+            configure do |config|
+              config.error_messages = YAML.safe_load(
+                File.read(
+                  File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+                )
+              ).fetch('en').fetch('validations').fetch('topic')
+            end
+
+            nested :eofed do
+              required(:active) { |val| [true, false].include?(val) }
+            end
+
+            virtual do |data, errors|
+              next unless errors.empty?
+
+              eofed = data[:eofed]
+
+              next unless eofed[:active]
+
+              next if data[:kafka][:'enable.partition.eof']
+
+              [[%i[eofed kafka], :enable]]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/eofed/topic.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class Eofed < Base
+        # Routing topic eofed API
+        module Topic
+          # @param active [Boolean] should the `#eofed` job run on eof
+          def eofed(active = false)
+            @eofed ||= Config.new(
+              active: active
+            )
+          end
+
+          # @return [Boolean] Are `#eofed` jobs active
+          def eofed?
+            eofed.active?
+          end
+
+          # @return [Hash] topic setup hash
+          def to_h
+            super.merge(
+              eofed: eofed.to_h
+            ).freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/eofed.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      # Namespace for feature allowing to enable the `#eofed` jobs.
+      # We do not enable it always because users may only be interested in fast eofed yielding
+      # without running the `#eofed` operation at all. This safes on empty cycles of running
+      # pointless empty jobs.
+      class Eofed < Base
+      end
+    end
+  end
+end