karafka 2.2.11 → 2.2.13

data/lib/karafka/pro/processing/jobs_queue.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Enhanced processing queue that provides ability to build complex work-distribution
+      # schedulers dedicated to particular job types
+      #
+      # Aside from the OSS queue capabilities it allows for jobless locking for advanced schedulers
+      class JobsQueue < Karafka::Processing::JobsQueue
+        attr_accessor :in_processing
+
+        # @return [Karafka::Pro::Processing::JobsQueue]
+        def initialize
+          super
+
+          @in_waiting = Hash.new { |h, k| h[k] = [] }
+
+          @statistics[:waiting] = 0
+        end
+
+        # Method that allows us to lock queue on a given subscription group without enqueuing the a
+        # job. This can be used when building complex schedulers that want to postpone enqueuing
+        # before certain conditions are met.
+        #
+        # @param job [Jobs::Base] job used for locking
+        def lock(job)
+          @mutex.synchronize do
+            group = @in_waiting[job.group_id]
+
+            # This should never happen. Same job should not be locked twice
+            raise(Errors::JobsQueueSynchronizationError, job.group_id) if group.include?(job)
+
+            @statistics[:waiting] += 1
+
+            group << job
+          end
+        end
+
+        # Method for unlocking the given subscription group queue space that was locked with a
+        # given job that was **not** added to the queue but used via `#lock`.
+        #
+        # @param job [Jobs::Base] job that locked the queue
+        def unlock(job)
+          @mutex.synchronize do
+            @statistics[:waiting] -= 1
+
+            return if @in_waiting[job.group_id].delete(job)
+
+            # This should never happen. It means there was a job being unlocked that was never
+            # locked in the first place
+            raise(Errors::JobsQueueSynchronizationError, job.group_id)
+          end
+        end
+
+        # Clears the processing states for a provided group. Useful when a recovery happens and we
+        # need to clean up state but only for a given subscription group.
+        #
+        # @param group_id [String]
+        def clear(group_id)
+          @mutex.synchronize do
+            @in_processing[group_id].clear
+
+            @statistics[:waiting] -= @in_waiting[group_id].size
+            @in_waiting[group_id].clear
+
+            # We unlock it just in case it was blocked when clearing started
+            tick(group_id)
+          end
+        end
+
+        # @param group_id [String]
+        #
+        # @return [Boolean] tell us if we have anything in the processing (or for processing) from
+        # a given group.
+        def empty?(group_id)
+          @mutex.synchronize do
+            @in_processing[group_id].empty? &&
+              @in_waiting[group_id].empty?
+          end
+        end
+
+        private
+
+        # @param group_id [String] id of the group in which jobs we're interested.
+        # @return [Boolean] should we keep waiting or not
+        # @note We do not wait for non-blocking jobs. Their flow should allow for `poll` running
+        #   as they may exceed `max.poll.interval`
+        def wait?(group_id)
+          !(
+            @in_processing[group_id].all?(&:non_blocking?) &&
+            @in_waiting[group_id].all?(&:non_blocking?)
+          )
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/scheduler.rb

@@ -27,10 +27,9 @@ module Karafka
       class Scheduler < ::Karafka::Processing::Scheduler
         # Schedules jobs in the LJF order for consumption
         #
-        # @param queue [Karafka::Processing::JobsQueue] queue where we want to put the jobs
         # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
         #
-        def schedule_consumption(queue, jobs_array)
+        def schedule_consumption(jobs_array)
           perf_tracker = PerformanceTracker.instance
 
           ordered = []
@@ -47,7 +46,7 @@ module Karafka
           ordered.map!(&:first)
 
           ordered.each do |job|
-            queue << job
+            @queue << job
           end
         end
 

data/lib/karafka/pro/processing/strategies/default.rb

@@ -29,6 +29,8 @@ module Karafka
 
           # No actions needed for the standard flow here
           def handle_before_enqueue
+            Karafka.monitor.instrument('consumer.before_enqueue', caller: self)
+
             nil
           end
 

data/lib/karafka/pro/processing/strategies/lrj/default.rb

@@ -77,6 +77,15 @@ module Karafka
                 revoked
               end
             end
+
+            # Allows for LRJ to synchronize its work. It may be needed because LRJ can run
+            # lifecycle events like revocation while the LRJ work is running and there may be a
+            # need for a critical section.
+            #
+            # @param block [Proc] block we want to run in a mutex to prevent race-conditions
+            def synchronize(&block)
+              coordinator.shared_mutex.synchronize(&block)
+            end
           end
         end
       end

data/lib/karafka/pro/processing/strategies/vp/default.rb

@@ -94,13 +94,15 @@ module Karafka
 
             # Allows for cross-virtual-partition consumers locks
             #
-            # This is not needed in the non-VP flows because there is always only one consumer
-            # per partition at the same time, so no coordination is needed directly for the
-            # end users
+            # This is not needed in the non-VP flows except LRJ because there is always only one
+            # consumer per partition at the same time, so no coordination is needed directly for
+            # the end users. With LRJ it is needed and provided in the `LRJ::Default` strategy,
+            # because lifecycle events on revocation can run in parallel to the LRJ job as it is
+            # non-blocking.
             #
             # @param block [Proc] block we want to run in a mutex to prevent race-conditions
             def synchronize(&block)
-              coordinator.synchronize(&block)
+              coordinator.shared_mutex.synchronize(&block)
             end
 
             private
@@ -111,6 +113,8 @@ module Karafka
             # @note This can be done without the mutex, because it happens from the same thread
             #   for all the work (listener thread)
             def handle_before_enqueue
+              super
+
               coordinator.virtual_offset_manager.register(
                 messages.map(&:offset)
               )

data/lib/karafka/processing/coordinator.rb

@@ -162,11 +162,24 @@ module Karafka
         @manual_seek
       end
 
+      # @param consumer [Object] karafka consumer (normal or pro)
+      # @return [Karafka::Processing::Result] result object which we can use to indicate
+      #   consumption processing state.
+      def consumption(consumer)
+        @consumptions[consumer] ||= Processing::Result.new
+      end
+
       # Allows to run synchronized (locked) code that can operate only from a given thread
       #
       # @param block [Proc] code we want to run in the synchronized mode
+      #
       # @note We check if mutex is not owned already by the current thread so we won't end up with
       #   a deadlock in case user runs coordinated code from inside of his own lock
+      #
+      # @note This is internal and should **not** be used to synchronize user-facing code.
+      #   Otherwise user indirectly could cause deadlocks or prolonged locks by running his logic.
+      #   This can and should however be used for multi-thread strategy applications and other
+      #   internal operations locks.
       def synchronize(&block)
         if @mutex.owned?
           yield
@@ -174,13 +187,6 @@ module Karafka
           @mutex.synchronize(&block)
         end
       end
-
-      # @param consumer [Object] karafka consumer (normal or pro)
-      # @return [Karafka::Processing::Result] result object which we can use to indicate
-      #   consumption processing state.
-      def consumption(consumer)
-        @consumptions[consumer] ||= Processing::Result.new
-      end
     end
   end
 end

data/lib/karafka/processing/inline_insights/consumer.rb

@@ -35,6 +35,8 @@ module Karafka
 
         alias statistics insights
         alias statistics? insights?
+        alias inline_insights insights
+        alias inline_insights? insights?
       end
     end
   end

data/lib/karafka/processing/jobs_queue.rb

@@ -9,6 +9,9 @@ module Karafka
     # on this queue, that's why internally we keep track of processing per group.
     #
     # We work with the assumption, that partitions data is evenly distributed.
+    #
+    # @note This job queue also keeps track / understands number of busy workers. This is because
+    #   we use a single workers poll that can have granular scheduling.
     class JobsQueue
       # @return [Karafka::Processing::JobsQueue]
       def initialize
@@ -21,21 +24,19 @@ module Karafka
         # We cannot use a single semaphore as it could potentially block in listeners that should
         # process with their data and also could unlock when a given group needs to remain locked
         @semaphores = Concurrent::Map.new do |h, k|
-          h.compute_if_absent(k) { Queue.new }
+          # Ruby prior to 3.2 did not have queue with a timeout on `#pop`, that is why for those
+          # versions we use our custom queue wrapper
+          h.compute_if_absent(k) { RUBY_VERSION < '3.2' ? TimedQueue.new : Queue.new }
         end
 
+        @concurrency = Karafka::App.config.concurrency
+        @tick_interval = ::Karafka::App.config.internal.tick_interval
         @in_processing = Hash.new { |h, k| h[k] = [] }
+        @statistics = { busy: 0, enqueued: 0 }
 
         @mutex = Mutex.new
       end
 
-      # Returns number of jobs that are either enqueued or in processing (but not finished)
-      # @return [Integer] number of elements in the queue
-      # @note Using `#pop` won't decrease this number as only marking job as completed does this
-      def size
-        @in_processing.values.map(&:size).sum
-      end
-
       # Adds the job to the internal main queue, scheduling it for execution in a worker and marks
       # this job as in processing pipeline.
       #
@@ -52,6 +53,16 @@ module Karafka
 
           group << job
 
+          # Assume that moving to queue means being picked up immediately not to create stats
+          # race conditions because of pop overhead. If there are workers available, we assume
+          # work is going to be handled as we never reject enqueued jobs
+          if @statistics[:busy] < @concurrency
+            @statistics[:busy] += 1
+          else
+            # If system is fully loaded, it means this job is indeed enqueued
+            @statistics[:enqueued] += 1
+          end
+
           @queue << job
         end
       end
@@ -77,7 +88,16 @@ module Karafka
       # @param [Jobs::Base] job that was completed
       def complete(job)
         @mutex.synchronize do
+          # We finish one job and if there is another, we pick it up
+          if @statistics[:enqueued].positive?
+            @statistics[:enqueued] -= 1
+          # If no more enqueued jobs, we will be just less busy
+          else
+            @statistics[:busy] -= 1
+          end
+
           @in_processing[job.group_id].delete(job)
+
           tick(job.group_id)
         end
       end
@@ -118,11 +138,19 @@ module Karafka
       #   jobs from a given group are completed
       #
       # @param group_id [String] id of the group in which jobs we're interested.
+      # @yieldparam [Block] block we want to run before each pop (in case of Ruby pre 3.2) or
+      #   before each pop and on every tick interval.
+      #   This allows us to run extra code that needs to be executed even when we are waiting on
+      #   the work to be finished.
       # @note This method is blocking.
       def wait(group_id)
         # Go doing other things while we cannot process and wait for anyone to finish their work
         # and re-check the wait status
-        @semaphores[group_id].pop while wait?(group_id)
+        while wait?(group_id)
+          yield if block_given?
+
+          @semaphores[group_id].pop(timeout: @tick_interval / 1_000.0)
+        end
       end
 
       # - `busy` - number of jobs that are currently being processed (active work)
@@ -130,10 +158,10 @@ module Karafka
       #
       # @return [Hash] hash with basic usage statistics of this queue.
       def statistics
-        {
-          busy: size - @queue.size,
-          enqueued: @queue.size
-        }.freeze
+        # Ensures there are no race conditions when returning this data
+        @mutex.synchronize do
+          @statistics.dup.freeze
+        end
       end
 
       private

data/lib/karafka/processing/scheduler.rb

@@ -4,19 +4,35 @@ module Karafka
   module Processing
     # FIFO scheduler for messages coming from various topics and partitions
     class Scheduler
+      # @param queue [Karafka::Processing::JobsQueue] queue where we want to put the jobs
+      def initialize(queue)
+        @queue = queue
+      end
+
       # Schedules jobs in the fifo order
       #
-      # @param queue [Karafka::Processing::JobsQueue] queue where we want to put the jobs
       # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
-      def schedule_consumption(queue, jobs_array)
+      def schedule_consumption(jobs_array)
         jobs_array.each do |job|
-          queue << job
+          @queue << job
         end
       end
 
       # Both revocation and shutdown jobs can also run in fifo by default
       alias schedule_revocation schedule_consumption
       alias schedule_shutdown schedule_consumption
+
+      # This scheduler does not have anything to manage as it is a pass through and has no state
+      def manage
+        nil
+      end
+
+      # This scheduler does not need to be cleared because it is stateless
+      #
+      # @param _group_id [String] Subscription group id
+      def clear(_group_id)
+        nil
+      end
     end
   end
 end

data/lib/karafka/processing/strategies/default.rb

@@ -78,6 +78,8 @@ module Karafka
 
         # No actions needed for the standard flow here
         def handle_before_enqueue
+          Karafka.monitor.instrument('consumer.before_enqueue', caller: self)
+
           nil
         end
 

data/lib/karafka/processing/timed_queue.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Minimal queue with timeout for Ruby 3.1 and lower.
+    #
+    # It is needed because only since 3.2, Ruby has a timeout on `#pop`
+    class TimedQueue
+      include Karafka::Core::Helpers::Time
+
+      def initialize
+        @queue = Queue.new
+        @mutex = Thread::Mutex.new
+        @resource = Thread::ConditionVariable.new
+      end
+
+      # Adds element to the queue
+      #
+      # @param obj [Object] pushes an element onto the queue
+      def push(obj)
+        @mutex.synchronize do
+          @queue << obj
+          @resource.broadcast
+        end
+      end
+
+      alias << push
+
+      # No timeout means waiting up to 31 years
+      #
+      # @param timeout [Integer] max number of seconds to wait on the pop
+      # @return [Object] element inserted on the array or `nil` on timeout
+      #
+      # @note We use timeout in seconds because this is how Ruby 3.2+ works and we want to have
+      #   the same API for newer and older Ruby versions
+      def pop(timeout: 10_000_000_000)
+        deadline = monotonic_now + timeout * 1000
+
+        @mutex.synchronize do
+          loop do
+            return @queue.pop unless @queue.empty?
+            return @queue.pop if @queue.closed?
+
+            to_wait = (deadline - monotonic_now) / 1_000.0
+
+            return nil if to_wait <= 0
+
+            @resource.wait(@mutex, to_wait)
+          end
+        end
+      end
+
+      # Closes the internal queue and releases the lock
+      def close
+        @mutex.synchronize do
+          @queue.close
+          @resource.broadcast
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/builder.rb

@@ -3,20 +3,25 @@
 module Karafka
   module Routing
     # Builder used as a DSL layer for building consumers and telling them which topics to consume
+    #
+    # @note We lock the access just in case this is used in patterns. The locks here do not have
+    #   any impact on routing usage unless being expanded, so no race conditions risks.
+    #
     # @example Build a simple (most common) route
     #   consumers do
     #     topic :new_videos do
     #       consumer NewVideosConsumer
     #     end
     #   end
-    class Builder < Concurrent::Array
+    class Builder < Array
       # Empty default per-topic config
       EMPTY_DEFAULTS = ->(_) {}.freeze
 
       private_constant :EMPTY_DEFAULTS
 
       def initialize
-        @draws = Concurrent::Array.new
+        @mutex = Mutex.new
+        @draws = []
         @defaults = EMPTY_DEFAULTS
         super
       end
@@ -34,21 +39,23 @@ module Karafka
       #     end
       #   end
       def draw(&block)
-        @draws << block
+        @mutex.synchronize do
+          @draws << block
 
-        instance_eval(&block)
+          instance_eval(&block)
 
-        each do |consumer_group|
-          # Validate consumer group settings
-          Contracts::ConsumerGroup.new.validate!(consumer_group.to_h)
+          each do |consumer_group|
+            # Validate consumer group settings
+            Contracts::ConsumerGroup.new.validate!(consumer_group.to_h)
 
-          # and then its topics settings
-          consumer_group.topics.each do |topic|
-            Contracts::Topic.new.validate!(topic.to_h)
-          end
+            # and then its topics settings
+            consumer_group.topics.each do |topic|
+              Contracts::Topic.new.validate!(topic.to_h)
+            end
 
-          # Initialize subscription groups after all the routing is done
-          consumer_group.subscription_groups
+            # Initialize subscription groups after all the routing is done
+            consumer_group.subscription_groups
+          end
         end
       end
 
@@ -61,9 +68,11 @@ module Karafka
 
       # Clears the builder and the draws memory
       def clear
-        @defaults = EMPTY_DEFAULTS
-        @draws.clear
-        super
+        @mutex.synchronize do
+          @defaults = EMPTY_DEFAULTS
+          @draws.clear
+          super
+        end
       end
 
       # @param block [Proc] block with per-topic evaluated defaults
@@ -71,7 +80,13 @@ module Karafka
       def defaults(&block)
         return @defaults unless block
 
-        @defaults = block
+        if @mutex.owned?
+          @defaults = block
+        else
+          @mutex.synchronize do
+            @defaults = block
+          end
+        end
       end
 
       private

data/lib/karafka/routing/subscription_group.rb

@@ -10,19 +10,24 @@ module Karafka
     class SubscriptionGroup
       attr_reader :id, :name, :topics, :kafka, :consumer_group
 
-      # Numeric for counting groups
-      GROUP_COUNT = Concurrent::AtomicFixnum.new
+      # Lock for generating new ids safely
+      ID_MUTEX = Mutex.new
 
-      private_constant :GROUP_COUNT
+      private_constant :ID_MUTEX
 
       class << self
         # Generates new subscription group id that will be used in case of anonymous subscription
         #   groups
         # @return [String] hex(6) compatible reproducible id
         def id
-          ::Digest::MD5.hexdigest(
-            GROUP_COUNT.increment.to_s
-          )[0..11]
+          ID_MUTEX.synchronize do
+            @group_counter ||= 0
+            @group_counter += 1
+
+            ::Digest::MD5.hexdigest(
+              @group_counter.to_s
+            )[0..11]
+          end
         end
       end
 

data/lib/karafka/runner.rb

@@ -8,7 +8,7 @@ module Karafka
     def call
       # Despite possibility of having several independent listeners, we aim to have one queue for
       # jobs across and one workers poll for that
-      jobs_queue = Processing::JobsQueue.new
+      jobs_queue = App.config.internal.processing.jobs_queue_class.new
 
       workers = Processing::WorkersBatch.new(jobs_queue)
       listeners = Connection::ListenersBatch.new(jobs_queue)

data/lib/karafka/setup/config.rb

@@ -152,6 +152,17 @@ module Karafka
         #   instances
         setting :process, default: Process.new
 
+        # Interval of "ticking". This is used to define the maximum time between consecutive
+        # polling of the main rdkafka queue. It should match also the `statistics.interval.ms`
+        # smallest value defined in any of the per-kafka settings, so metrics are published with
+        # the desired frequency. It is set to 5 seconds because `statistics.interval.ms` is also
+        # set to five seconds.
+        #
+        # It is NOT allowed to set it to a value less than 1 seconds because it could cause polling
+        # not to have enough time to run. This (not directly) defines also a single poll
+        # max timeout as to allow for frequent enough events polling
+        setting :tick_interval, default: 5_000
+
         # Namespace for CLI related settings
         setting :cli do
           # option contract [Object] cli setup validation contract (in the context of options and
@@ -198,8 +209,9 @@ module Karafka
         end
 
         setting :processing do
+          setting :jobs_queue_class, default: Processing::JobsQueue
           # option scheduler [Object] scheduler we will be using
-          setting :scheduler, default: Processing::Scheduler.new
+          setting :scheduler_class, default: Processing::Scheduler
           # option jobs_builder [Object] jobs builder we want to use
           setting :jobs_builder, default: Processing::JobsBuilder.new
           # option coordinator [Class] work coordinator we want to user for processing coordination

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.2.11'
+  VERSION = '2.2.13'
 end

data/lib/karafka.rb

@@ -16,7 +16,6 @@
   singleton
   digest
   zeitwerk
-  concurrent/atomic/atomic_fixnum
 ].each(&method(:require))
 
 # Karafka framework main namespace