karafka 2.2.12 → 2.2.14

data/lib/karafka/pro/instrumentation/performance_tracker.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Namespace for Pro components instrumentation related code
+    module Instrumentation
+      # Tracker used to keep track of performance metrics
+      # It provides insights that can be used to optimize processing flow
+      # @note Even if we have some race-conditions here it is relevant due to the quantity of data.
+      #   This is why we do not mutex it.
+      class PerformanceTracker
+        include Singleton
+
+        # How many samples do we collect per topic partition
+        SAMPLES_COUNT = 200
+
+        private_constant :SAMPLES_COUNT
+
+        # Builds up nested concurrent hash for data tracking
+        def initialize
+          @processing_times = Hash.new do |topics_hash, topic|
+            topics_hash[topic] = Hash.new do |partitions_hash, partition|
+              partitions_hash[partition] = []
+            end
+          end
+        end
+
+        # @param topic [String]
+        # @param partition [Integer]
+        # @return [Float] p95 processing time of a single message from a single topic partition
+        def processing_time_p95(topic, partition)
+          values = @processing_times[topic][partition]
+
+          return 0 if values.empty?
+          return values.first if values.size == 1
+
+          percentile(0.95, values)
+        end
+
+        # @private
+        # @param event [Karafka::Core::Monitoring::Event] event details
+        # Tracks time taken to process a single message of a given topic partition
+        def on_consumer_consumed(event)
+          consumer = event[:caller]
+          messages = consumer.messages
+          topic = messages.metadata.topic
+          partition = messages.metadata.partition
+
+          samples = @processing_times[topic][partition]
+          samples << event[:time] / messages.count
+
+          return unless samples.size > SAMPLES_COUNT
+
+          samples.shift
+        end
+
+        private
+
+        # Computers the requested percentile out of provided values
+        # @param percentile [Float]
+        # @param values [Array<String>] all the values based on which we should
+        # @return [Float] computed percentile
+        def percentile(percentile, values)
+          values_sorted = values.sort
+
+          floor = (percentile * (values_sorted.length - 1) + 1).floor - 1
+          mod = (percentile * (values_sorted.length - 1) + 1).modulo(1)
+
+          values_sorted[floor] + (mod * (values_sorted[floor + 1] - values_sorted[floor]))
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/loader.rb

@@ -84,7 +84,8 @@ module Karafka
 
           icfg.processing.coordinator_class = Processing::Coordinator
           icfg.processing.partitioner_class = Processing::Partitioner
-          icfg.processing.scheduler = Processing::Scheduler.new
+          icfg.processing.scheduler_class = Processing::Schedulers::Default
+          icfg.processing.jobs_queue_class = Processing::JobsQueue
           icfg.processing.jobs_builder = Processing::JobsBuilder.new
           icfg.processing.strategy_selector = Processing::StrategySelector.new
 
@@ -92,7 +93,7 @@ module Karafka
           icfg.active_job.dispatcher = ActiveJob::Dispatcher.new
           icfg.active_job.job_options_contract = ActiveJob::JobOptionsContract.new
 
-          config.monitor.subscribe(PerformanceTracker.instance)
+          config.monitor.subscribe(Instrumentation::PerformanceTracker.instance)
         end
 
         # Loads the Pro features of Karafka

data/lib/karafka/pro/processing/coordinator.rb

@@ -21,14 +21,20 @@ module Karafka
 
         def_delegators :@collapser, :collapsed?, :collapse_until!
 
-        attr_reader :filter, :virtual_offset_manager
+        attr_reader :filter, :virtual_offset_manager, :shared_mutex
 
         # @param args [Object] anything the base coordinator accepts
         def initialize(*args)
           super
 
           @executed = []
-          @flow_lock = Mutex.new
+          @flow_mutex = Mutex.new
+          # Lock for user code synchronization
+          # We do not want to mix coordinator lock with the user lock not to create cases where
+          # user imposed lock would lock the internal operations of Karafka
+          # This shared lock can be used by the end user as it is not used internally by the
+          # framework and can be used for user-facing locking
+          @shared_mutex = Mutex.new
           @collapser = Collapser.new
           @filter = FiltersApplier.new(self)
 
@@ -89,7 +95,7 @@ module Karafka
         # Runs synchronized code once for a collective of virtual partitions prior to work being
         # enqueued
         def on_enqueued
-          @flow_lock.synchronize do
+          @flow_mutex.synchronize do
             return unless executable?(:on_enqueued)
 
             yield(@last_message)
@@ -98,7 +104,7 @@ module Karafka
 
         # Runs given code only once per all the coordinated jobs upon starting first of them
         def on_started
-          @flow_lock.synchronize do
+          @flow_mutex.synchronize do
             return unless executable?(:on_started)
 
             yield(@last_message)
@@ -109,7 +115,7 @@ module Karafka
         # It runs once per all the coordinated jobs and should be used to run any type of post
         # jobs coordination processing execution
         def on_finished
-          @flow_lock.synchronize do
+          @flow_mutex.synchronize do
             return unless finished?
             return unless executable?(:on_finished)
 
@@ -119,7 +125,7 @@ module Karafka
 
         # Runs once after a partition is revoked
         def on_revoked
-          @flow_lock.synchronize do
+          @flow_mutex.synchronize do
             return unless executable?(:on_revoked)
 
             yield(@last_message)

data/lib/karafka/pro/processing/jobs_queue.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Enhanced processing queue that provides ability to build complex work-distribution
+      # schedulers dedicated to particular job types
+      #
+      # Aside from the OSS queue capabilities it allows for jobless locking for advanced schedulers
+      class JobsQueue < Karafka::Processing::JobsQueue
+        attr_accessor :in_processing
+
+        # @return [Karafka::Pro::Processing::JobsQueue]
+        def initialize
+          super
+
+          @in_waiting = Hash.new { |h, k| h[k] = [] }
+
+          @statistics[:waiting] = 0
+        end
+
+        # Method that allows us to lock queue on a given subscription group without enqueuing the a
+        # job. This can be used when building complex schedulers that want to postpone enqueuing
+        # before certain conditions are met.
+        #
+        # @param job [Jobs::Base] job used for locking
+        def lock(job)
+          @mutex.synchronize do
+            group = @in_waiting[job.group_id]
+
+            # This should never happen. Same job should not be locked twice
+            raise(Errors::JobsQueueSynchronizationError, job.group_id) if group.include?(job)
+
+            @statistics[:waiting] += 1
+
+            group << job
+          end
+        end
+
+        # Method for unlocking the given subscription group queue space that was locked with a
+        # given job that was **not** added to the queue but used via `#lock`.
+        #
+        # @param job [Jobs::Base] job that locked the queue
+        def unlock(job)
+          @mutex.synchronize do
+            @statistics[:waiting] -= 1
+
+            return if @in_waiting[job.group_id].delete(job)
+
+            # This should never happen. It means there was a job being unlocked that was never
+            # locked in the first place
+            raise(Errors::JobsQueueSynchronizationError, job.group_id)
+          end
+        end
+
+        # Clears the processing states for a provided group. Useful when a recovery happens and we
+        # need to clean up state but only for a given subscription group.
+        #
+        # @param group_id [String]
+        def clear(group_id)
+          @mutex.synchronize do
+            @in_processing[group_id].clear
+
+            @statistics[:waiting] -= @in_waiting[group_id].size
+            @in_waiting[group_id].clear
+
+            # We unlock it just in case it was blocked when clearing started
+            tick(group_id)
+          end
+        end
+
+        # @param group_id [String]
+        #
+        # @return [Boolean] tell us if we have anything in the processing (or for processing) from
+        # a given group.
+        def empty?(group_id)
+          @mutex.synchronize do
+            @in_processing[group_id].empty? &&
+              @in_waiting[group_id].empty?
+          end
+        end
+
+        private
+
+        # @param group_id [String] id of the group in which jobs we're interested.
+        # @return [Boolean] should we keep waiting or not
+        # @note We do not wait for non-blocking jobs. Their flow should allow for `poll` running
+        #   as they may exceed `max.poll.interval`
+        def wait?(group_id)
+          !(
+            @in_processing[group_id].all?(&:non_blocking?) &&
+            @in_waiting[group_id].all?(&:non_blocking?)
+          )
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/schedulers/base.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Namespace for Pro schedulers related components
+      module Schedulers
+        # Base for all the Pro custom schedulers
+        #
+        # It wraps the Scheduler API with mutex to ensure, that during scheduling we do not start
+        # scheduling other work that could impact the decision making in between multiple
+        # subscription groups running in separate threads.
+        #
+        # @note All the `on_` methods can be redefined with a non-thread-safe versions without
+        #   locks if needed, however when doing so, ensure that your scheduler is stateless.
+        class Base
+          # @param queue [Karafka::Processing::JobsQueue] queue where we want to put the jobs
+          def initialize(queue)
+            @queue = queue
+            @mutex = Mutex.new
+          end
+
+          # Runs the consumption jobs scheduling flow under a mutex
+          #
+          # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs for scheduling
+          def on_schedule_consumption(jobs_array)
+            @mutex.synchronize do
+              schedule_consumption(jobs_array)
+            end
+          end
+
+          # Should schedule the consumption jobs
+          #
+          # @param _jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs for scheduling
+          def schedule_consumption(_jobs_array)
+            raise NotImplementedError, 'Implement in a subclass'
+          end
+
+          # Runs the revocation jobs scheduling flow under a mutex
+          #
+          # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs for scheduling
+          def on_schedule_revocation(jobs_array)
+            @mutex.synchronize do
+              schedule_revocation(jobs_array)
+            end
+          end
+
+          # Schedules the revocation jobs.
+          #
+          # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs for scheduling
+          #
+          # @note We provide a default scheduler logic here because by default revocation jobs
+          #   should be scheduled as fast as possible.
+          def schedule_revocation(jobs_array)
+            jobs_array.each do |job|
+              @queue << job
+            end
+          end
+
+          # Runs the shutdown jobs scheduling flow under a mutex
+          #
+          # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs for scheduling
+          def on_schedule_shutdown(jobs_array)
+            @mutex.synchronize do
+              schedule_shutdown(jobs_array)
+            end
+          end
+
+          # Schedules the shutdown jobs.
+          #
+          # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs for scheduling
+          #
+          # @note We provide a default scheduler logic here because by default revocation jobs
+          #   should be scheduled as fast as possible.
+          def schedule_shutdown(jobs_array)
+            jobs_array.each do |job|
+              @queue << job
+            end
+          end
+
+          # Runs the manage tick under mutex
+          def on_manage
+            @mutex.synchronize { manage }
+          end
+
+          # Should manage scheduling on jobs state changes
+          #
+          # By default does nothing as default schedulers are stateless
+          def manage
+            nil
+          end
+
+          # Runs clearing under mutex
+          #
+          # @param group_id [String] Subscription group id
+          def on_clear(group_id)
+            @mutex.synchronize { clear(group_id) }
+          end
+
+          # By default schedulers are stateless, so nothing to clear.
+          #
+          # @param _group_id [String] Subscription group id
+          def clear(_group_id)
+            nil
+          end
+
+          private
+
+          # @return [Karafka::Processing::JobsQueue] jobs queue reference for internal usage
+          #   inside of the scheduler
+          attr_reader :queue
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/schedulers/default.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Namespace for Pro schedulers
+      module Schedulers
+        # Optimizes scheduler that takes into consideration of execution time needed to process
+        # messages from given topics partitions. It uses the non-preemptive LJF algorithm
+        #
+        # This scheduler is designed to optimize execution times on jobs that perform IO operations
+        # as when taking IO into consideration, the can achieve optimized parallel processing.
+        #
+        # This scheduler can also work with virtual partitions.
+        #
+        # Aside from consumption jobs, other jobs do not run often, thus we can leave them with
+        # default FIFO scheduler from the default Karafka scheduler
+        #
+        # @note This is a stateless scheduler, thus we can override the `#on_` API.
+        class Default < Base
+          # Schedules jobs in the LJF order for consumption
+          #
+          # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
+          def on_schedule_consumption(jobs_array)
+            perf_tracker = Instrumentation::PerformanceTracker.instance
+
+            ordered = []
+
+            jobs_array.each do |job|
+              ordered << [
+                job,
+                processing_cost(perf_tracker, job)
+              ]
+            end
+
+            ordered.sort_by!(&:last)
+            ordered.reverse!
+            ordered.map!(&:first)
+
+            ordered.each do |job|
+              @queue << job
+            end
+          end
+
+          # Schedules jobs in the fifo order
+          #
+          # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
+          def on_schedule_revocation(jobs_array)
+            jobs_array.each do |job|
+              @queue << job
+            end
+          end
+
+          # Schedules jobs in the fifo order
+          #
+          # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
+          def on_schedule_shutdown(jobs_array)
+            jobs_array.each do |job|
+              @queue << job
+            end
+          end
+
+          # This scheduler does not have anything to manage as it is a pass through and has no
+          # state
+          def on_manage
+            nil
+          end
+
+          # This scheduler does not need to be cleared because it is stateless
+          #
+          # @param _group_id [String] Subscription group id
+          def on_clear(_group_id)
+            nil
+          end
+
+          private
+
+          # @param perf_tracker [PerformanceTracker]
+          # @param job [Karafka::Processing::Jobs::Base] job we will be processing
+          # @return [Numeric] estimated cost of processing this job
+          def processing_cost(perf_tracker, job)
+            if job.is_a?(::Karafka::Processing::Jobs::Consume)
+              messages = job.messages
+              message = messages.first
+
+              perf_tracker.processing_time_p95(message.topic, message.partition) * messages.size
+            else
+              # LJF will set first the most expensive, but we want to run the zero cost jobs
+              # related to the lifecycle always first. That is why we "emulate" that they
+              # the longest possible jobs that anyone can run
+              Float::INFINITY
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj/lrj_mom_vp.rb

@@ -33,7 +33,7 @@ module Karafka
             ].freeze
 
             # No actions needed for the standard flow here
-            def handle_before_enqueue
+            def handle_before_schedule_consume
               super
 
               coordinator.on_enqueued do

data/lib/karafka/pro/processing/strategies/default.rb

@@ -28,7 +28,9 @@ module Karafka
           FEATURES = %i[].freeze
 
           # No actions needed for the standard flow here
-          def handle_before_enqueue
+          def handle_before_schedule_consume
+            Karafka.monitor.instrument('consumer.before_schedule_consume', caller: self)
+
             nil
           end
 

data/lib/karafka/pro/processing/strategies/lrj/default.rb

@@ -29,7 +29,7 @@ module Karafka
             ].freeze
 
             # We always need to pause prior to doing any jobs for LRJ
-            def handle_before_enqueue
+            def handle_before_schedule_consume
               super
 
               # This ensures that when running LRJ with VP, things operate as expected run only
@@ -77,6 +77,15 @@ module Karafka
                 revoked
               end
             end
+
+            # Allows for LRJ to synchronize its work. It may be needed because LRJ can run
+            # lifecycle events like revocation while the LRJ work is running and there may be a
+            # need for a critical section.
+            #
+            # @param block [Proc] block we want to run in a mutex to prevent race-conditions
+            def synchronize(&block)
+              coordinator.shared_mutex.synchronize(&block)
+            end
           end
         end
       end

data/lib/karafka/pro/processing/strategies/lrj/mom.rb

@@ -29,7 +29,7 @@ module Karafka
             ].freeze
 
             # We always need to pause prior to doing any jobs for LRJ
-            def handle_before_enqueue
+            def handle_before_schedule_consume
               super
 
               # This ensures that when running LRJ with VP, things operate as expected run only

data/lib/karafka/pro/processing/strategies/vp/default.rb

@@ -94,13 +94,15 @@ module Karafka
 
             # Allows for cross-virtual-partition consumers locks
             #
-            # This is not needed in the non-VP flows because there is always only one consumer
-            # per partition at the same time, so no coordination is needed directly for the
-            # end users
+            # This is not needed in the non-VP flows except LRJ because there is always only one
+            # consumer per partition at the same time, so no coordination is needed directly for
+            # the end users. With LRJ it is needed and provided in the `LRJ::Default` strategy,
+            # because lifecycle events on revocation can run in parallel to the LRJ job as it is
+            # non-blocking.
             #
             # @param block [Proc] block we want to run in a mutex to prevent race-conditions
             def synchronize(&block)
-              coordinator.synchronize(&block)
+              coordinator.shared_mutex.synchronize(&block)
             end
 
             private
@@ -110,7 +112,9 @@ module Karafka
             #
             # @note This can be done without the mutex, because it happens from the same thread
             #   for all the work (listener thread)
-            def handle_before_enqueue
+            def handle_before_schedule_consume
+              super
+
               coordinator.virtual_offset_manager.register(
                 messages.map(&:offset)
               )

data/lib/karafka/processing/coordinator.rb

@@ -162,11 +162,24 @@ module Karafka
         @manual_seek
       end
 
+      # @param consumer [Object] karafka consumer (normal or pro)
+      # @return [Karafka::Processing::Result] result object which we can use to indicate
+      #   consumption processing state.
+      def consumption(consumer)
+        @consumptions[consumer] ||= Processing::Result.new
+      end
+
       # Allows to run synchronized (locked) code that can operate only from a given thread
       #
       # @param block [Proc] code we want to run in the synchronized mode
+      #
       # @note We check if mutex is not owned already by the current thread so we won't end up with
       #   a deadlock in case user runs coordinated code from inside of his own lock
+      #
+      # @note This is internal and should **not** be used to synchronize user-facing code.
+      #   Otherwise user indirectly could cause deadlocks or prolonged locks by running his logic.
+      #   This can and should however be used for multi-thread strategy applications and other
+      #   internal operations locks.
       def synchronize(&block)
         if @mutex.owned?
           yield
@@ -174,13 +187,6 @@ module Karafka
           @mutex.synchronize(&block)
         end
       end
-
-      # @param consumer [Object] karafka consumer (normal or pro)
-      # @return [Karafka::Processing::Result] result object which we can use to indicate
-      #   consumption processing state.
-      def consumption(consumer)
-        @consumptions[consumer] ||= Processing::Result.new
-      end
     end
   end
 end