karafka 2.0.21 → 2.0.23

data/lib/karafka/pro/processing/strategies/aj_dlq_mom.rb

@@ -42,7 +42,7 @@ module Karafka
                 # Do NOT commit offsets, they are comitted after each job in the AJ consumer.
                 coordinator.pause_tracker.reset
               elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                pause(coordinator.seek_offset)
+                pause(coordinator.seek_offset, nil, false)
               else
                 coordinator.pause_tracker.reset
                 skippable_message = find_skippable_message
@@ -51,7 +51,7 @@ module Karafka
                 # since AJ consumer commits the offset after each job, we also know that the
                 # previous job was successful
                 mark_as_consumed(skippable_message)
-                pause(coordinator.seek_offset)
+                pause(coordinator.seek_offset, nil, false)
               end
             end
           end

data/lib/karafka/pro/processing/strategies/aj_lrj_mom_vp.rb

@@ -33,7 +33,7 @@ module Karafka
           # No actions needed for the standard flow here
           def handle_before_enqueue
             coordinator.on_enqueued do
-              pause(coordinator.seek_offset, Lrj::MAX_PAUSE_TIME)
+              pause(coordinator.seek_offset, Lrj::MAX_PAUSE_TIME, false)
             end
           end
 
@@ -51,7 +51,7 @@ module Karafka
                 # If processing failed, we need to pause
                 # For long running job this will overwrite the default never-ending pause and will
                 # cause the processing to keep going after the error backoff
-                pause(coordinator.seek_offset)
+                pause(coordinator.seek_offset, nil, false)
               end
             end
           end

data/lib/karafka/pro/processing/strategies/aj_mom_vp.rb

@@ -47,7 +47,7 @@ module Karafka
 
                 mark_as_consumed(last_group_message)
               else
-                pause(coordinator.seek_offset)
+                pause(coordinator.seek_offset, nil, false)
               end
             end
           end

data/lib/karafka/pro/processing/strategies/default.rb

@@ -22,6 +22,7 @@ module Karafka
         # Nothing. Just standard, automatic flow
         module Default
           include Base
+          include ::Karafka::Processing::Strategies::Default
 
           # Apply strategy for a non-feature based flow
           FEATURES = %i[].freeze
@@ -39,6 +40,31 @@ module Karafka
             end
           end
 
+          # Run the user consumption code
+          def handle_consume
+            # We should not run the work at all on a partition that was revoked
+            # This can happen primarily when an LRJ job gets to the internal worker queue and
+            # this partition is revoked prior processing.
+            unless revoked?
+              Karafka.monitor.instrument('consumer.consumed', caller: self) do
+                consume
+              end
+            end
+
+            # Mark job as successful
+            coordinator.consumption(self).success!
+          rescue StandardError => e
+            # If failed, mark as failed
+            coordinator.consumption(self).failure!(e)
+
+            # Re-raise so reported in the consumer
+            raise e
+          ensure
+            # We need to decrease number of jobs that this coordinator coordinates as it has
+            # finished
+            coordinator.decrement
+          end
+
           # Standard flow without any features
           def handle_after_consume
             coordinator.on_finished do |last_group_message|
@@ -47,9 +73,13 @@ module Karafka
               if coordinator.success?
                 coordinator.pause_tracker.reset
 
+                # Do not mark last message if pause happened. This prevents a scenario where pause
+                # is overridden upon rebalance by marking
+                return if coordinator.manual_pause?
+
                 mark_as_consumed(last_group_message)
               else
-                pause(coordinator.seek_offset)
+                pause(coordinator.seek_offset, nil, false)
               end
             end
           end

data/lib/karafka/pro/processing/strategies/dlq.rb

@@ -32,9 +32,11 @@ module Karafka
               if coordinator.success?
                 coordinator.pause_tracker.reset
 
+                return if coordinator.manual_pause?
+
                 mark_as_consumed(messages.last)
               elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                pause(coordinator.seek_offset)
+                pause(coordinator.seek_offset, nil, false)
               # If we've reached number of retries that we could, we need to skip the first message
               # that was not marked as consumed, pause and continue, while also moving this message
               # to the dead topic
@@ -44,7 +46,7 @@ module Karafka
                 skippable_message = find_skippable_message
                 dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
                 mark_as_consumed(skippable_message)
-                pause(coordinator.seek_offset)
+                pause(coordinator.seek_offset, nil, false)
               end
             end
           end

data/lib/karafka/pro/processing/strategies/dlq_lrj.rb

@@ -34,12 +34,14 @@ module Karafka
               if coordinator.success?
                 coordinator.pause_tracker.reset
 
+                return if coordinator.manual_pause?
+
                 mark_as_consumed(last_group_message) unless revoked?
                 seek(coordinator.seek_offset) unless revoked?
 
                 resume
               elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                pause(coordinator.seek_offset)
+                pause(coordinator.seek_offset, nil, false)
               else
                 coordinator.pause_tracker.reset
 

data/lib/karafka/pro/processing/strategies/dlq_lrj_mom.rb

@@ -38,7 +38,7 @@ module Karafka
 
                 resume
               elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                pause(coordinator.seek_offset)
+                pause(coordinator.seek_offset, nil, false)
               else
                 coordinator.pause_tracker.reset
 

data/lib/karafka/pro/processing/strategies/dlq_mom.rb

@@ -35,7 +35,7 @@ module Karafka
               if coordinator.success?
                 coordinator.pause_tracker.reset
               elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                pause(coordinator.seek_offset)
+                pause(coordinator.seek_offset, nil, false)
               # If we've reached number of retries that we could, we need to skip the first message
               # that was not marked as consumed, pause and continue, while also moving this message
               # to the dead topic.
@@ -51,7 +51,7 @@ module Karafka
                   dispatch_to_dlq(skippable_message)
                 end
 
-                pause(coordinator.seek_offset)
+                pause(coordinator.seek_offset, nil, false)
               end
             end
           end

data/lib/karafka/pro/processing/strategies/lrj.rb

@@ -37,7 +37,7 @@ module Karafka
               #
               # For VP it applies the same way and since VP cannot be used with MOM we should not
               # have any edge cases here.
-              pause(coordinator.seek_offset, MAX_PAUSE_TIME)
+              pause(coordinator.seek_offset, MAX_PAUSE_TIME, false)
             end
           end
 
@@ -47,6 +47,8 @@ module Karafka
               if coordinator.success?
                 coordinator.pause_tracker.reset
 
+                return if coordinator.manual_pause?
+
                 mark_as_consumed(last_group_message) unless revoked?
                 seek(coordinator.seek_offset) unless revoked?
 
@@ -55,7 +57,7 @@ module Karafka
                 # If processing failed, we need to pause
                 # For long running job this will overwrite the default never-ending pause and will
                 # cause the processing to keep going after the error backoff
-                pause(coordinator.seek_offset)
+                pause(coordinator.seek_offset, nil, false)
               end
             end
           end

data/lib/karafka/pro/processing/strategies/lrj_mom.rb

@@ -36,7 +36,7 @@ module Karafka
               #
               # For VP it applies the same way and since VP cannot be used with MOM we should not
               # have any edge cases here.
-              pause(coordinator.seek_offset, Lrj::MAX_PAUSE_TIME)
+              pause(coordinator.seek_offset, Lrj::MAX_PAUSE_TIME, false)
             end
           end
 
@@ -50,7 +50,7 @@ module Karafka
 
                 resume
               else
-                pause(coordinator.seek_offset)
+                pause(coordinator.seek_offset, false)
               end
             end
           end

data/lib/karafka/pro/processing/strategies/mom.rb

@@ -32,7 +32,7 @@ module Karafka
               if coordinator.success?
                 coordinator.pause_tracker.reset
               else
-                pause(coordinator.seek_offset)
+                pause(coordinator.seek_offset, nil, false)
               end
             end
           end

data/lib/karafka/processing/coordinator.rb

@@ -21,6 +21,7 @@ module Karafka
         @revoked = false
         @consumptions = {}
         @running_jobs = 0
+        @manual_pause = false
         @mutex = Mutex.new
       end
 
@@ -34,6 +35,9 @@ module Karafka
           # consumption results of consumer instances we no longer control
           @consumptions.clear
 
+          # When starting to run, no pause is expected and no manual pause as well
+          @manual_pause = false
+
           # We set it on the first encounter and never again, because then the offset setting
           # should be up to the consumers logic (our or the end user)
           # Seek offset needs to be always initialized as for case where manual offset management
@@ -98,6 +102,17 @@ module Karafka
       def revoked?
         @revoked
       end
+
+      # Store in the coordinator info, that this pause was done manually by the end user and not
+      # by the system itself
+      def manual_pause
+        @mutex.synchronize { @manual_pause = true }
+      end
+
+      # @return [Boolean] are we in a pause that was initiated by the user
+      def manual_pause?
+        @pause_tracker.paused? && @manual_pause
+      end
     end
   end
 end

data/lib/karafka/processing/jobs_queue.rb

@@ -100,7 +100,7 @@ module Karafka
           return if @queue.closed?
 
           @queue.close
-          @semaphores.values.each(&:close)
+          @semaphores.each_value(&:close)
         end
       end
 

data/lib/karafka/processing/strategies/aj_dlq_mom.rb

@@ -26,7 +26,7 @@ module Karafka
             # Do NOT commit offsets, they are comitted after each job in the AJ consumer.
             coordinator.pause_tracker.reset
           elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-            pause(coordinator.seek_offset)
+            pause(coordinator.seek_offset, nil, false)
           else
             coordinator.pause_tracker.reset
             skippable_message = find_skippable_message
@@ -35,7 +35,7 @@ module Karafka
             # since AJ consumer commits the offset after each job, we also know that the
             # previous job was successful
             mark_as_consumed(skippable_message)
-            pause(coordinator.seek_offset)
+            pause(coordinator.seek_offset, nil, false)
           end
         end
       end

data/lib/karafka/processing/strategies/base.rb

@@ -22,6 +22,11 @@ module Karafka
           raise NotImplementedError, 'Implement in a subclass'
         end
 
+        # What should happen in the processing
+        def handle_consume
+          raise NotImplementedError, 'Implement in a subclass'
+        end
+
         # Post-consumption handling
         def handle_after_consume
           raise NotImplementedError, 'Implement in a subclass'

data/lib/karafka/processing/strategies/default.rb

@@ -23,6 +23,25 @@ module Karafka
           coordinator.pause_tracker.increment
         end
 
+        # Run the user consumption code
+        def handle_consume
+          Karafka.monitor.instrument('consumer.consumed', caller: self) do
+            consume
+          end
+
+          # Mark job as successful
+          coordinator.consumption(self).success!
+        rescue StandardError => e
+          # If failed, mark as failed
+          coordinator.consumption(self).failure!(e)
+
+          # Re-raise so reported in the consumer
+          raise e
+        ensure
+          # We need to decrease number of jobs that this coordinator coordinates as it has finished
+          coordinator.decrement
+        end
+
         # Standard flow marks work as consumed and moves on if everything went ok.
         # If there was a processing error, we will pause and continue from the next message
         # (next that is +1 from the last one that was successfully marked as consumed)
@@ -32,9 +51,15 @@ module Karafka
           if coordinator.success?
             coordinator.pause_tracker.reset
 
+            # We should not move the offset automatically when the partition was paused
+            # If we would not do this upon a revocation during the pause time, a different process
+            # would pick not from the place where we paused but from the offset that would be
+            # automatically committed here
+            return if coordinator.manual_pause?
+
             mark_as_consumed(messages.last)
           else
-            pause(coordinator.seek_offset)
+            pause(coordinator.seek_offset, nil, false)
           end
         end
 

data/lib/karafka/processing/strategies/dlq.rb

@@ -22,9 +22,11 @@ module Karafka
           if coordinator.success?
             coordinator.pause_tracker.reset
 
+            return if coordinator.manual_pause?
+
             mark_as_consumed(messages.last)
           elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-            pause(coordinator.seek_offset)
+            pause(coordinator.seek_offset, nil, false)
           # If we've reached number of retries that we could, we need to skip the first message
           # that was not marked as consumed, pause and continue, while also moving this message
           # to the dead topic
@@ -43,7 +45,7 @@ module Karafka
             return if revoked?
 
             # We pause to backoff once just in case.
-            pause(coordinator.seek_offset)
+            pause(coordinator.seek_offset, nil, false)
           end
         end
 

data/lib/karafka/processing/strategies/dlq_mom.rb

@@ -21,7 +21,7 @@ module Karafka
           if coordinator.success?
             coordinator.pause_tracker.reset
           elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-            pause(coordinator.seek_offset)
+            pause(coordinator.seek_offset, nil, false)
           # If we've reached number of retries that we could, we need to skip the first message
           # that was not marked as consumed, pause and continue, while also moving this message
           # to the dead topic
@@ -33,7 +33,7 @@ module Karafka
             dispatch_to_dlq(skippable_message)
 
             # We pause to backoff once just in case.
-            pause(coordinator.seek_offset)
+            pause(coordinator.seek_offset, nil, false)
           end
         end
       end

data/lib/karafka/processing/strategies/mom.rb

@@ -20,7 +20,7 @@ module Karafka
           if coordinator.success?
             coordinator.pause_tracker.reset
           else
-            pause(coordinator.seek_offset)
+            pause(coordinator.seek_offset, nil, false)
           end
         end
       end

data/lib/karafka/railtie.rb

@@ -45,6 +45,9 @@ if rails
 
         next unless Rails.env.development?
         next unless ENV.key?('KARAFKA_CLI')
+        # If we are already publishing to STDOUT, no need to add it again.
+        # If added again, would print stuff twice
+        next if ActiveSupport::Logger.logger_outputs_to?(Rails.logger, $stdout)
 
         logger = ActiveSupport::Logger.new($stdout)
         # Inherit the logger level from Rails, otherwise would always run with the debug level

data/lib/karafka/routing/builder.rb

@@ -82,7 +82,7 @@ module Karafka
       # @param block [Proc] further topics definitions
       def subscription_group(subscription_group_name = SecureRandom.uuid, &block)
         consumer_group('app') do
-          target.public_send(:subscription_group=, subscription_group_name, &block)
+          target.public_send(:subscription_group=, subscription_group_name.to_s, &block)
         end
       end
 

data/lib/karafka/routing/consumer_group.rb

@@ -31,7 +31,10 @@ module Karafka
 
       # @return [Boolean] true if this consumer group should be active in our current process
       def active?
-        Karafka::Server.consumer_groups.include?(name)
+        cgs = Karafka::App.config.internal.routing.active.consumer_groups
+
+        # When empty it means no groups were specified, hence all should be used
+        cgs.empty? || cgs.include?(name)
       end
 
       # Builds a topic representation inside of a current consumer group route
@@ -50,9 +53,9 @@ module Karafka
 
       # Assigns the current subscription group id based on the defined one and allows for further
       # topic definition
-      # @param name [String, Symbol]
+      # @param name [String, Symbol] name of the current subscription group
       # @param block [Proc] block that may include topics definitions
-      def subscription_group=(name, &block)
+      def subscription_group=(name = SecureRandom.uuid, &block)
         # We cast it here, so the routing supports symbol based but that's anyhow later on
         # validated as a string
         @current_subscription_group_id = name

data/lib/karafka/routing/subscription_group.rb

@@ -8,7 +8,7 @@ module Karafka
     # @note One subscription group will always belong to one consumer group, but one consumer
     #   group can have multiple subscription groups.
     class SubscriptionGroup
-      attr_reader :id, :topics, :kafka
+      attr_reader :id, :name, :topics, :kafka
 
       # @param position [Integer] position of this subscription group in all the subscriptions
       #   groups array. We need to have this value for sake of static group memberships, where
@@ -16,7 +16,8 @@ module Karafka
       # @param topics [Karafka::Routing::Topics] all the topics that share the same key settings
       # @return [SubscriptionGroup] built subscription group
       def initialize(position, topics)
-        @id = "#{topics.first.subscription_group}_#{position}"
+        @name = topics.first.subscription_group
+        @id = "#{@name}_#{position}"
         @position = position
         @topics = topics
         @kafka = build_kafka
@@ -38,6 +39,14 @@ module Karafka
         @topics.first.max_wait_time
       end
 
+      # @return [Boolean] is this subscription group one of active once
+      def active?
+        sgs = Karafka::App.config.internal.routing.active.subscription_groups
+
+        # When empty it means no groups were specified, hence all should be used
+        sgs.empty? || sgs.include?(name)
+      end
+
       private
 
       # @return [Hash] kafka settings are a bit special. They are exactly the same for all of the

data/lib/karafka/routing/topic.rb

@@ -75,6 +75,14 @@ module Karafka
         consumer
       end
 
+      # @return [Boolean] should this topic be in use
+      def active?
+        topics = Karafka::App.config.internal.routing.active.topics
+
+        # When empty it means no topics were specified, hence all should be used
+        topics.empty? || topics.include?(name)
+      end
+
       # @return [Hash] hash with all the topic attributes
       # @note This is being used when we validate the consumer_group and its topics
       def to_h

data/lib/karafka/routing/topics.rb

@@ -23,6 +23,14 @@ module Karafka
         @accumulator.each(&block)
       end
 
+      # Allows us to remove elements from the topics
+      #
+      # Block to decide what to delete
+      # @param block [Proc]
+      def delete_if(&block)
+        @accumulator.delete_if(&block)
+      end
+
       # Finds topic by its name
       #
       # @param topic_name [String] topic name

data/lib/karafka/server.rb

@@ -20,11 +20,19 @@ module Karafka
       # Set of workers
       attr_accessor :workers
 
-      # Writer for list of consumer groups that we want to consume in our current process context
-      attr_writer :consumer_groups
-
       # Method which runs app
       def run
+        self.listeners = []
+        self.workers = []
+
+        # We need to validate this prior to running because it may be executed also from the
+        # embedded
+        # We cannot validate this during the start because config needs to be populated and routes
+        # need to be defined.
+        Contracts::ServerCliOptions.new.validate!(
+          Karafka::App.config.internal.routing.active.to_h
+        )
+
         process.on_sigint { stop }
         process.on_sigquit { stop }
         process.on_sigterm { stop }
@@ -49,13 +57,6 @@ module Karafka
         raise e
       end
 
-      # @return [Array<String>] array with names of consumer groups that should be consumed in a
-      #   current server context
-      def consumer_groups
-        # If not specified, a server will listen on all the topics
-        @consumer_groups ||= Karafka::App.consumer_groups.map(&:name).freeze
-      end
-
       # Starts Karafka with a supervision
       # @note We don't need to sleep because Karafka::Fetcher is locking and waiting to
       # finish loop (and it won't happen until we explicitly want to stop)

data/lib/karafka/setup/config.rb

@@ -107,6 +107,14 @@ module Karafka
           # option subscription_groups_builder [Routing::SubscriptionGroupsBuilder] subscription
           #   group builder
           setting :subscription_groups_builder, default: Routing::SubscriptionGroupsBuilder.new
+
+          # Internally assigned list of limits on routings active for the current process
+          # This should be overwritten by the CLI command
+          setting :active do
+            setting :consumer_groups, default: [].freeze
+            setting :subscription_groups, default: [].freeze
+            setting :topics, default: [].freeze
+          end
         end
 
         setting :processing do
@@ -142,16 +150,18 @@ module Karafka
         # Configuring method
         # @param block [Proc] block we want to execute with the config instance
         def setup(&block)
+          # Will prepare and verify license if present
+          Licenser.prepare_and_verify(config.license)
+          # Will configure all the pro components
+          # This needs to happen before end user configuration as the end user may overwrite some
+          # of the pro defaults with custom components
+          Pro::Loader.setup(config) if Karafka.pro?
+
           configure(&block)
           merge_kafka_defaults!(config)
 
           Contracts::Config.new.validate!(config.to_h)
 
-          licenser = Licenser.new
-
-          # Tries to load our license gem and if present will try to load the correct license
-          licenser.prepare_and_verify(config.license)
-
           configure_components
 
           Karafka::App.initialized!
@@ -188,12 +198,6 @@ module Karafka
             producer_config.kafka = AttributesMap.producer(config.kafka.dup)
             producer_config.logger = config.logger
           end
-
-          return unless Karafka.pro?
-
-          # Runs the pro loader that includes all the pro components
-          require 'karafka/pro/loader'
-          Pro::Loader.setup(config)
         end
       end
     end

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.21'
+  VERSION = '2.0.23'
 end

data/lib/karafka.rb

@@ -100,5 +100,14 @@ loader.eager_load
 # nor included here
 ::Karafka::Routing::Features::Base.load_all
 
+# We need to detect and require (not setup) Pro components during the gem load, because we need
+# to make pro components available in case anyone wants to use them as a base to their own
+# custom components. Otherwise inheritance would not work.
+Karafka::Licenser.detect do
+  require 'karafka/pro/loader'
+
+  Karafka::Pro::Loader.require_all
+end
+
 # Load railtie after everything else is ready so we know we can rely on it.
 require 'karafka/railtie'