karafka 2.2.13 → 2.2.14

data/lib/karafka/pro/processing/schedulers/base.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Namespace for Pro schedulers related components
+      module Schedulers
+        # Base for all the Pro custom schedulers
+        #
+        # It wraps the Scheduler API with mutex to ensure, that during scheduling we do not start
+        # scheduling other work that could impact the decision making in between multiple
+        # subscription groups running in separate threads.
+        #
+        # @note All the `on_` methods can be redefined with a non-thread-safe versions without
+        #   locks if needed, however when doing so, ensure that your scheduler is stateless.
+        class Base
+          # @param queue [Karafka::Processing::JobsQueue] queue where we want to put the jobs
+          def initialize(queue)
+            @queue = queue
+            @mutex = Mutex.new
+          end
+
+          # Runs the consumption jobs scheduling flow under a mutex
+          #
+          # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs for scheduling
+          def on_schedule_consumption(jobs_array)
+            @mutex.synchronize do
+              schedule_consumption(jobs_array)
+            end
+          end
+
+          # Should schedule the consumption jobs
+          #
+          # @param _jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs for scheduling
+          def schedule_consumption(_jobs_array)
+            raise NotImplementedError, 'Implement in a subclass'
+          end
+
+          # Runs the revocation jobs scheduling flow under a mutex
+          #
+          # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs for scheduling
+          def on_schedule_revocation(jobs_array)
+            @mutex.synchronize do
+              schedule_revocation(jobs_array)
+            end
+          end
+
+          # Schedules the revocation jobs.
+          #
+          # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs for scheduling
+          #
+          # @note We provide a default scheduler logic here because by default revocation jobs
+          #   should be scheduled as fast as possible.
+          def schedule_revocation(jobs_array)
+            jobs_array.each do |job|
+              @queue << job
+            end
+          end
+
+          # Runs the shutdown jobs scheduling flow under a mutex
+          #
+          # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs for scheduling
+          def on_schedule_shutdown(jobs_array)
+            @mutex.synchronize do
+              schedule_shutdown(jobs_array)
+            end
+          end
+
+          # Schedules the shutdown jobs.
+          #
+          # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs for scheduling
+          #
+          # @note We provide a default scheduler logic here because by default revocation jobs
+          #   should be scheduled as fast as possible.
+          def schedule_shutdown(jobs_array)
+            jobs_array.each do |job|
+              @queue << job
+            end
+          end
+
+          # Runs the manage tick under mutex
+          def on_manage
+            @mutex.synchronize { manage }
+          end
+
+          # Should manage scheduling on jobs state changes
+          #
+          # By default does nothing as default schedulers are stateless
+          def manage
+            nil
+          end
+
+          # Runs clearing under mutex
+          #
+          # @param group_id [String] Subscription group id
+          def on_clear(group_id)
+            @mutex.synchronize { clear(group_id) }
+          end
+
+          # By default schedulers are stateless, so nothing to clear.
+          #
+          # @param _group_id [String] Subscription group id
+          def clear(_group_id)
+            nil
+          end
+
+          private
+
+          # @return [Karafka::Processing::JobsQueue] jobs queue reference for internal usage
+          #   inside of the scheduler
+          attr_reader :queue
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/schedulers/default.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Namespace for Pro schedulers
+      module Schedulers
+        # Optimizes scheduler that takes into consideration of execution time needed to process
+        # messages from given topics partitions. It uses the non-preemptive LJF algorithm
+        #
+        # This scheduler is designed to optimize execution times on jobs that perform IO operations
+        # as when taking IO into consideration, the can achieve optimized parallel processing.
+        #
+        # This scheduler can also work with virtual partitions.
+        #
+        # Aside from consumption jobs, other jobs do not run often, thus we can leave them with
+        # default FIFO scheduler from the default Karafka scheduler
+        #
+        # @note This is a stateless scheduler, thus we can override the `#on_` API.
+        class Default < Base
+          # Schedules jobs in the LJF order for consumption
+          #
+          # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
+          def on_schedule_consumption(jobs_array)
+            perf_tracker = Instrumentation::PerformanceTracker.instance
+
+            ordered = []
+
+            jobs_array.each do |job|
+              ordered << [
+                job,
+                processing_cost(perf_tracker, job)
+              ]
+            end
+
+            ordered.sort_by!(&:last)
+            ordered.reverse!
+            ordered.map!(&:first)
+
+            ordered.each do |job|
+              @queue << job
+            end
+          end
+
+          # Schedules jobs in the fifo order
+          #
+          # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
+          def on_schedule_revocation(jobs_array)
+            jobs_array.each do |job|
+              @queue << job
+            end
+          end
+
+          # Schedules jobs in the fifo order
+          #
+          # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
+          def on_schedule_shutdown(jobs_array)
+            jobs_array.each do |job|
+              @queue << job
+            end
+          end
+
+          # This scheduler does not have anything to manage as it is a pass through and has no
+          # state
+          def on_manage
+            nil
+          end
+
+          # This scheduler does not need to be cleared because it is stateless
+          #
+          # @param _group_id [String] Subscription group id
+          def on_clear(_group_id)
+            nil
+          end
+
+          private
+
+          # @param perf_tracker [PerformanceTracker]
+          # @param job [Karafka::Processing::Jobs::Base] job we will be processing
+          # @return [Numeric] estimated cost of processing this job
+          def processing_cost(perf_tracker, job)
+            if job.is_a?(::Karafka::Processing::Jobs::Consume)
+              messages = job.messages
+              message = messages.first
+
+              perf_tracker.processing_time_p95(message.topic, message.partition) * messages.size
+            else
+              # LJF will set first the most expensive, but we want to run the zero cost jobs
+              # related to the lifecycle always first. That is why we "emulate" that they
+              # the longest possible jobs that anyone can run
+              Float::INFINITY
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj/lrj_mom_vp.rb

@@ -33,7 +33,7 @@ module Karafka
             ].freeze
 
             # No actions needed for the standard flow here
-            def handle_before_enqueue
+            def handle_before_schedule_consume
               super
 
               coordinator.on_enqueued do

data/lib/karafka/pro/processing/strategies/default.rb

@@ -28,8 +28,8 @@ module Karafka
           FEATURES = %i[].freeze
 
           # No actions needed for the standard flow here
-          def handle_before_enqueue
-            Karafka.monitor.instrument('consumer.before_enqueue', caller: self)
+          def handle_before_schedule_consume
+            Karafka.monitor.instrument('consumer.before_schedule_consume', caller: self)
 
             nil
           end

data/lib/karafka/pro/processing/strategies/lrj/default.rb

@@ -29,7 +29,7 @@ module Karafka
             ].freeze
 
             # We always need to pause prior to doing any jobs for LRJ
-            def handle_before_enqueue
+            def handle_before_schedule_consume
               super
 
               # This ensures that when running LRJ with VP, things operate as expected run only

data/lib/karafka/pro/processing/strategies/lrj/mom.rb

@@ -29,7 +29,7 @@ module Karafka
             ].freeze
 
             # We always need to pause prior to doing any jobs for LRJ
-            def handle_before_enqueue
+            def handle_before_schedule_consume
               super
 
               # This ensures that when running LRJ with VP, things operate as expected run only

data/lib/karafka/pro/processing/strategies/vp/default.rb

@@ -112,7 +112,7 @@ module Karafka
             #
             # @note This can be done without the mutex, because it happens from the same thread
             #   for all the work (listener thread)
-            def handle_before_enqueue
+            def handle_before_schedule_consume
               super
 
               coordinator.virtual_offset_manager.register(

data/lib/karafka/processing/executor.rb

@@ -11,6 +11,15 @@ module Karafka
     #
     # @note Executors are not removed after partition is revoked. They are not that big and will
     #   be re-used in case of a re-claim
+    #
+    # @note Since given consumer can run various operations, executor manages that and its
+    #   lifecycle. There are following types of operations with appropriate before/after, etc:
+    #
+    #   - consume - primary operation related to running user consumption code
+    #   - idle - cleanup job that runs on idle runs where no messages would be passed to the end
+    #     user. This is used for complex flows with filters, etc
+    #   - revoked - runs after the partition was revoked
+    #   - shutdown - runs when process is going to shutdown
     class Executor
       extend Forwardable
 
@@ -39,11 +48,11 @@ module Karafka
       end
 
       # Allows us to prepare the consumer in the listener thread prior to the job being send to
-      # the queue. It also allows to run some code that is time sensitive and cannot wait in the
+      # be scheduled. It also allows to run some code that is time sensitive and cannot wait in the
       # queue as it could cause starvation.
       #
       # @param messages [Array<Karafka::Messages::Message>]
-      def before_enqueue(messages)
+      def before_schedule_consume(messages)
         # Recreate consumer with each batch if persistence is not enabled
         # We reload the consumers with each batch instead of relying on some external signals
         # when needed for consistency. That way devs may have it on or off and not in this
@@ -60,7 +69,7 @@ module Karafka
           Time.now
         )
 
-        consumer.on_before_enqueue
+        consumer.on_before_schedule_consume
       end
 
       # Runs setup and warm-up code in the worker prior to running the consumption
@@ -79,6 +88,11 @@ module Karafka
         consumer.on_after_consume
       end
 
+      # Runs the code needed before idle work is scheduled
+      def before_schedule_idle
+        consumer.on_before_schedule_idle
+      end
+
       # Runs consumer idle operations
       # This may include house-keeping or other state management changes that can occur but that
       # not mean there are any new messages available for the end user to process
@@ -96,6 +110,11 @@ module Karafka
         consumer.on_idle
       end
 
+      # Runs code needed before revoked job is scheduled
+      def before_schedule_revoked
+        consumer.on_before_schedule_revoked if @consumer
+      end
+
       # Runs the controller `#revoked` method that should be triggered when a given consumer is
       # no longer needed due to partitions reassignment.
       #
@@ -112,6 +131,11 @@ module Karafka
         consumer.on_revoked if @consumer
       end
 
+      # Runs code needed before shutdown job is scheduled
+      def before_schedule_shutdown
+        consumer.on_before_schedule_shutdown if @consumer
+      end
+
       # Runs the controller `#shutdown` method that should be triggered when a given consumer is
       # no longer needed as we're closing the process.
       #

data/lib/karafka/processing/executors_buffer.rb

@@ -54,9 +54,9 @@ module Karafka
       # @yieldparam [Integer] partition number
       # @yieldparam [Executor] given executor
       def each
-        @buffer.each do |_, partitions|
-          partitions.each do |_, executors|
-            executors.each do |_, executor|
+        @buffer.each_value do |partitions|
+          partitions.each_value do |executors|
+            executors.each_value do |executor|
               yield(executor)
             end
           end

data/lib/karafka/processing/jobs/base.rb

@@ -20,11 +20,14 @@ module Karafka
           # All jobs are blocking by default and they can release the lock when blocking operations
           # are done (if needed)
           @non_blocking = false
+          @status = :pending
         end
 
-        # When redefined can run any code prior to the job being enqueued
+        # When redefined can run any code prior to the job being scheduled
         # @note This will run in the listener thread and not in the worker
-        def before_enqueue; end
+        def before_schedule
+          raise NotImplementedError, 'Please implement in a subclass'
+        end
 
         # When redefined can run any code that should run before executing the proper code
         def before_call; end
@@ -49,6 +52,20 @@ module Karafka
         def non_blocking?
           @non_blocking
         end
+
+        # @return [Boolean] was this job finished.
+        def finished?
+          @status == :finished
+        end
+
+        # Marks the job as finished. Used by the worker to indicate, that this job is done.
+        #
+        # @note Since the scheduler knows exactly when it schedules jobs and when it keeps them
+        #   pending, we do not need advanced state tracking and the only information from the
+        #   "outside" is whether it was finished or not after it was scheduled for execution.
+        def finish!
+          @status = :finished
+        end
       end
     end
   end

data/lib/karafka/processing/jobs/consume.rb

@@ -20,9 +20,9 @@ module Karafka
         end
 
         # Runs all the preparation code on the executor that needs to happen before the job is
-        # enqueued.
-        def before_enqueue
-          executor.before_enqueue(@messages)
+        # scheduled.
+        def before_schedule
+          executor.before_schedule_consume(@messages)
         end
 
         # Runs the before consumption preparations on the executor

data/lib/karafka/processing/jobs/idle.rb

@@ -14,6 +14,11 @@ module Karafka
           super()
         end
 
+        # Runs code prior to scheduling this idle job
+        def before_schedule
+          executor.before_schedule_idle
+        end
+
         # Run the idle work via the executor
         def call
           executor.idle

data/lib/karafka/processing/jobs/revoked.rb

@@ -12,6 +12,11 @@ module Karafka
           super()
         end
 
+        # Runs code prior to scheduling this revoked job
+        def before_schedule
+          executor.before_schedule_revoked
+        end
+
         # Runs the revoking job via an executor.
         def call
           executor.revoked

data/lib/karafka/processing/jobs/shutdown.rb

@@ -13,6 +13,11 @@ module Karafka
           super()
         end
 
+        # Runs code prior to scheduling this shutdown job
+        def before_schedule
+          executor.before_schedule_shutdown
+        end
+
         # Runs the shutdown job via an executor.
         def call
           executor.shutdown

data/lib/karafka/processing/jobs_queue.rb

@@ -23,12 +23,7 @@ module Karafka
         # scheduled by Ruby hundreds of thousands of times per group.
         # We cannot use a single semaphore as it could potentially block in listeners that should
         # process with their data and also could unlock when a given group needs to remain locked
-        @semaphores = Concurrent::Map.new do |h, k|
-          # Ruby prior to 3.2 did not have queue with a timeout on `#pop`, that is why for those
-          # versions we use our custom queue wrapper
-          h.compute_if_absent(k) { RUBY_VERSION < '3.2' ? TimedQueue.new : Queue.new }
-        end
-
+        @semaphores = {}
         @concurrency = Karafka::App.config.concurrency
         @tick_interval = ::Karafka::App.config.internal.tick_interval
         @in_processing = Hash.new { |h, k| h[k] = [] }
@@ -37,6 +32,22 @@ module Karafka
         @mutex = Mutex.new
       end
 
+      # Registers given subscription group id in the queue. It is needed so we do not dynamically
+      # create semaphore, hence avoiding potential race conditions
+      #
+      # @param group_id [String]
+      def register(group_id)
+        # Ruby prior to 3.2 did not have queue with a timeout on `#pop`, that is why for those
+        @mutex.synchronize do
+          # versions we use our custom queue wrapper
+          #
+          # Initializes this semaphore from the mutex, so it is never auto-created
+          # Since we always schedule a job before waiting using semaphores, there won't be any
+          # concurrency problems
+          @semaphores[group_id] = RUBY_VERSION < '3.2' ? TimedQueue.new : Queue.new
+        end
+      end
+
       # Adds the job to the internal main queue, scheduling it for execution in a worker and marks
       # this job as in processing pipeline.
       #
@@ -79,7 +90,7 @@ module Karafka
       # @param group_id [String] id of the group we want to unlock for one tick
       # @note This does not release the wait lock. It just causes a conditions recheck
       def tick(group_id)
-        @semaphores[group_id] << true
+        @semaphores.fetch(group_id) << true
       end
 
       # Marks a given job from a given group as completed. When there are no more jobs from a given
@@ -149,7 +160,7 @@ module Karafka
         while wait?(group_id)
           yield if block_given?
 
-          @semaphores[group_id].pop(timeout: @tick_interval / 1_000.0)
+          @semaphores.fetch(group_id).pop(timeout: @tick_interval / 1_000.0)
         end
       end
 

data/lib/karafka/processing/schedulers/default.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Namespace for Karafka OSS schedulers
+    module Schedulers
+      # FIFO scheduler for messages coming from various topics and partitions
+      class Default
+        # @param queue [Karafka::Processing::JobsQueue] queue where we want to put the jobs
+        def initialize(queue)
+          @queue = queue
+        end
+
+        # Schedules jobs in the fifo order
+        #
+        # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
+        def on_schedule_consumption(jobs_array)
+          jobs_array.each do |job|
+            @queue << job
+          end
+        end
+
+        # Both revocation and shutdown jobs can also run in fifo by default
+        alias on_schedule_revocation on_schedule_consumption
+        alias on_schedule_shutdown on_schedule_consumption
+
+        # This scheduler does not have anything to manage as it is a pass through and has no state
+        def on_manage
+          nil
+        end
+
+        # This scheduler does not need to be cleared because it is stateless
+        #
+        # @param _group_id [String] Subscription group id
+        def on_clear(_group_id)
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/strategies/base.rb

@@ -11,10 +11,19 @@ module Karafka
     module Strategies
       # Base strategy that should be included in each strategy, just to ensure the API
       module Base
-        # What should happen before jobs are enqueued
-        # @note This runs from the listener thread, not recommended to put anything slow here
-        def handle_before_enqueue
-          raise NotImplementedError, 'Implement in a subclass'
+        # Defines all the before schedule handlers for appropriate actions
+        %i[
+          consume
+          idle
+          revoked
+          shutdown
+        ].each do |action|
+          class_eval <<~RUBY, __FILE__, __LINE__ + 1
+            def handle_before_schedule_#{action}
+              # What should happen before scheduling this work
+              raise NotImplementedError, 'Implement in a subclass'
+            end
+          RUBY
         end
 
         # What should happen before we kick in the processing

data/lib/karafka/processing/strategies/default.rb

@@ -13,6 +13,23 @@ module Karafka
         # Apply strategy for a non-feature based flow
         FEATURES = %i[].freeze
 
+        # By default on all "before schedule" we just run instrumentation, nothing more
+        %i[
+          consume
+          idle
+          revoked
+          shutdown
+        ].each do |action|
+          class_eval <<~RUBY, __FILE__, __LINE__ + 1
+            # No actions needed for the standard flow here
+            def handle_before_schedule_#{action}
+              Karafka.monitor.instrument('consumer.before_schedule_#{action}', caller: self)
+
+              nil
+            end
+          RUBY
+        end
+
         # Marks message as consumed in an async way.
         #
         # @param message [Messages::Message] last successfully processed message.
@@ -76,13 +93,6 @@ module Karafka
           commit_offsets(async: false)
         end
 
-        # No actions needed for the standard flow here
-        def handle_before_enqueue
-          Karafka.monitor.instrument('consumer.before_enqueue', caller: self)
-
-          nil
-        end
-
         # Increment number of attempts
         def handle_before_consume
           coordinator.pause_tracker.increment

data/lib/karafka/processing/worker.rb

@@ -83,7 +83,10 @@ module Karafka
         )
       ensure
         # job can be nil when the queue is being closed
-        @jobs_queue.complete(job) if job
+        if job
+          @jobs_queue.complete(job)
+          job.finish!
+        end
 
         # Always publish info, that we completed all the work despite its result
         Karafka.monitor.instrument('worker.completed', instrument_details)

data/lib/karafka/routing/proxy.rb

@@ -10,11 +10,12 @@ module Karafka
       # @param target [Object] target object to which we proxy any DSL call
       # @param defaults [Proc] defaults for target that should be applicable after the proper
       #   proxy context (if needed)
-      # @param block [Proc] block that we want to evaluate in the proxy context
+      # @param block [Proc, nil] block that we want to evaluate in the proxy context or nil if no
+      #   proxy block context for example because whole context is taken from defaults
       def initialize(target, defaults = ->(_) {}, &block)
         @target = target
-        instance_eval(&block)
-        instance_eval(&defaults)
+        instance_eval(&block) if block
+        instance_eval(&defaults) if defaults
       end
 
       # Ruby 2.7.0 to 2.7.2 do not have arg forwarding, so we fallback to the old way

data/lib/karafka/routing/topics.rb

@@ -9,7 +9,7 @@ module Karafka
       include Enumerable
       extend Forwardable
 
-      def_delegators :@accumulator, :[], :size, :empty?, :last, :<<
+      def_delegators :@accumulator, :[], :size, :empty?, :last, :<<, :map!, :sort_by!, :reverse!
 
       # @param topics_array [Array<Karafka::Routing::Topic>] array with topics
       def initialize(topics_array)