karafka 2.2.12 → 2.2.14

data/lib/karafka/processing/executor.rb

@@ -11,6 +11,15 @@ module Karafka
     #
     # @note Executors are not removed after partition is revoked. They are not that big and will
     #   be re-used in case of a re-claim
+    #
+    # @note Since given consumer can run various operations, executor manages that and its
+    #   lifecycle. There are following types of operations with appropriate before/after, etc:
+    #
+    #   - consume - primary operation related to running user consumption code
+    #   - idle - cleanup job that runs on idle runs where no messages would be passed to the end
+    #     user. This is used for complex flows with filters, etc
+    #   - revoked - runs after the partition was revoked
+    #   - shutdown - runs when process is going to shutdown
     class Executor
       extend Forwardable
 
@@ -39,11 +48,11 @@ module Karafka
       end
 
       # Allows us to prepare the consumer in the listener thread prior to the job being send to
-      # the queue. It also allows to run some code that is time sensitive and cannot wait in the
+      # be scheduled. It also allows to run some code that is time sensitive and cannot wait in the
       # queue as it could cause starvation.
       #
       # @param messages [Array<Karafka::Messages::Message>]
-      def before_enqueue(messages)
+      def before_schedule_consume(messages)
         # Recreate consumer with each batch if persistence is not enabled
         # We reload the consumers with each batch instead of relying on some external signals
         # when needed for consistency. That way devs may have it on or off and not in this
@@ -60,7 +69,7 @@ module Karafka
           Time.now
         )
 
-        consumer.on_before_enqueue
+        consumer.on_before_schedule_consume
       end
 
       # Runs setup and warm-up code in the worker prior to running the consumption
@@ -79,6 +88,11 @@ module Karafka
         consumer.on_after_consume
       end
 
+      # Runs the code needed before idle work is scheduled
+      def before_schedule_idle
+        consumer.on_before_schedule_idle
+      end
+
       # Runs consumer idle operations
       # This may include house-keeping or other state management changes that can occur but that
       # not mean there are any new messages available for the end user to process
@@ -96,6 +110,11 @@ module Karafka
         consumer.on_idle
       end
 
+      # Runs code needed before revoked job is scheduled
+      def before_schedule_revoked
+        consumer.on_before_schedule_revoked if @consumer
+      end
+
       # Runs the controller `#revoked` method that should be triggered when a given consumer is
       # no longer needed due to partitions reassignment.
       #
@@ -112,6 +131,11 @@ module Karafka
         consumer.on_revoked if @consumer
       end
 
+      # Runs code needed before shutdown job is scheduled
+      def before_schedule_shutdown
+        consumer.on_before_schedule_shutdown if @consumer
+      end
+
       # Runs the controller `#shutdown` method that should be triggered when a given consumer is
       # no longer needed as we're closing the process.
       #

data/lib/karafka/processing/executors_buffer.rb

@@ -54,9 +54,9 @@ module Karafka
       # @yieldparam [Integer] partition number
       # @yieldparam [Executor] given executor
       def each
-        @buffer.each do |_, partitions|
-          partitions.each do |_, executors|
-            executors.each do |_, executor|
+        @buffer.each_value do |partitions|
+          partitions.each_value do |executors|
+            executors.each_value do |executor|
               yield(executor)
             end
           end

data/lib/karafka/processing/jobs/base.rb

@@ -20,11 +20,14 @@ module Karafka
           # All jobs are blocking by default and they can release the lock when blocking operations
           # are done (if needed)
           @non_blocking = false
+          @status = :pending
         end
 
-        # When redefined can run any code prior to the job being enqueued
+        # When redefined can run any code prior to the job being scheduled
         # @note This will run in the listener thread and not in the worker
-        def before_enqueue; end
+        def before_schedule
+          raise NotImplementedError, 'Please implement in a subclass'
+        end
 
         # When redefined can run any code that should run before executing the proper code
         def before_call; end
@@ -49,6 +52,20 @@ module Karafka
         def non_blocking?
           @non_blocking
         end
+
+        # @return [Boolean] was this job finished.
+        def finished?
+          @status == :finished
+        end
+
+        # Marks the job as finished. Used by the worker to indicate, that this job is done.
+        #
+        # @note Since the scheduler knows exactly when it schedules jobs and when it keeps them
+        #   pending, we do not need advanced state tracking and the only information from the
+        #   "outside" is whether it was finished or not after it was scheduled for execution.
+        def finish!
+          @status = :finished
+        end
       end
     end
   end

data/lib/karafka/processing/jobs/consume.rb

@@ -20,9 +20,9 @@ module Karafka
         end
 
         # Runs all the preparation code on the executor that needs to happen before the job is
-        # enqueued.
-        def before_enqueue
-          executor.before_enqueue(@messages)
+        # scheduled.
+        def before_schedule
+          executor.before_schedule_consume(@messages)
         end
 
         # Runs the before consumption preparations on the executor

data/lib/karafka/processing/jobs/idle.rb

@@ -14,6 +14,11 @@ module Karafka
           super()
         end
 
+        # Runs code prior to scheduling this idle job
+        def before_schedule
+          executor.before_schedule_idle
+        end
+
         # Run the idle work via the executor
         def call
           executor.idle

data/lib/karafka/processing/jobs/revoked.rb

@@ -12,6 +12,11 @@ module Karafka
           super()
         end
 
+        # Runs code prior to scheduling this revoked job
+        def before_schedule
+          executor.before_schedule_revoked
+        end
+
         # Runs the revoking job via an executor.
         def call
           executor.revoked

data/lib/karafka/processing/jobs/shutdown.rb

@@ -13,6 +13,11 @@ module Karafka
           super()
         end
 
+        # Runs code prior to scheduling this shutdown job
+        def before_schedule
+          executor.before_schedule_shutdown
+        end
+
         # Runs the shutdown job via an executor.
         def call
           executor.shutdown

data/lib/karafka/processing/jobs_queue.rb

@@ -9,6 +9,9 @@ module Karafka
     # on this queue, that's why internally we keep track of processing per group.
     #
     # We work with the assumption, that partitions data is evenly distributed.
+    #
+    # @note This job queue also keeps track / understands number of busy workers. This is because
+    #   we use a single workers poll that can have granular scheduling.
     class JobsQueue
       # @return [Karafka::Processing::JobsQueue]
       def initialize
@@ -20,23 +23,29 @@ module Karafka
         # scheduled by Ruby hundreds of thousands of times per group.
         # We cannot use a single semaphore as it could potentially block in listeners that should
         # process with their data and also could unlock when a given group needs to remain locked
-        @semaphores = Concurrent::Map.new do |h, k|
-          # Ruby prior to 3.2 did not have queue with a timeout on `#pop`, that is why for those
-          # versions we use our custom queue wrapper
-          h.compute_if_absent(k) { RUBY_VERSION < '3.2' ? TimedQueue.new : Queue.new }
-        end
-
+        @semaphores = {}
+        @concurrency = Karafka::App.config.concurrency
         @tick_interval = ::Karafka::App.config.internal.tick_interval
         @in_processing = Hash.new { |h, k| h[k] = [] }
+        @statistics = { busy: 0, enqueued: 0 }
 
         @mutex = Mutex.new
       end
 
-      # Returns number of jobs that are either enqueued or in processing (but not finished)
-      # @return [Integer] number of elements in the queue
-      # @note Using `#pop` won't decrease this number as only marking job as completed does this
-      def size
-        @in_processing.values.map(&:size).sum
+      # Registers given subscription group id in the queue. It is needed so we do not dynamically
+      # create semaphore, hence avoiding potential race conditions
+      #
+      # @param group_id [String]
+      def register(group_id)
+        # Ruby prior to 3.2 did not have queue with a timeout on `#pop`, that is why for those
+        @mutex.synchronize do
+          # versions we use our custom queue wrapper
+          #
+          # Initializes this semaphore from the mutex, so it is never auto-created
+          # Since we always schedule a job before waiting using semaphores, there won't be any
+          # concurrency problems
+          @semaphores[group_id] = RUBY_VERSION < '3.2' ? TimedQueue.new : Queue.new
+        end
       end
 
       # Adds the job to the internal main queue, scheduling it for execution in a worker and marks
@@ -55,6 +64,16 @@ module Karafka
 
           group << job
 
+          # Assume that moving to queue means being picked up immediately not to create stats
+          # race conditions because of pop overhead. If there are workers available, we assume
+          # work is going to be handled as we never reject enqueued jobs
+          if @statistics[:busy] < @concurrency
+            @statistics[:busy] += 1
+          else
+            # If system is fully loaded, it means this job is indeed enqueued
+            @statistics[:enqueued] += 1
+          end
+
           @queue << job
         end
       end
@@ -71,7 +90,7 @@ module Karafka
       # @param group_id [String] id of the group we want to unlock for one tick
       # @note This does not release the wait lock. It just causes a conditions recheck
       def tick(group_id)
-        @semaphores[group_id] << true
+        @semaphores.fetch(group_id) << true
       end
 
       # Marks a given job from a given group as completed. When there are no more jobs from a given
@@ -80,7 +99,16 @@ module Karafka
       # @param [Jobs::Base] job that was completed
       def complete(job)
         @mutex.synchronize do
+          # We finish one job and if there is another, we pick it up
+          if @statistics[:enqueued].positive?
+            @statistics[:enqueued] -= 1
+          # If no more enqueued jobs, we will be just less busy
+          else
+            @statistics[:busy] -= 1
+          end
+
           @in_processing[job.group_id].delete(job)
+
           tick(job.group_id)
         end
       end
@@ -132,7 +160,7 @@ module Karafka
         while wait?(group_id)
           yield if block_given?
 
-          @semaphores[group_id].pop(timeout: @tick_interval / 1_000.0)
+          @semaphores.fetch(group_id).pop(timeout: @tick_interval / 1_000.0)
         end
       end
 
@@ -141,10 +169,10 @@ module Karafka
       #
       # @return [Hash] hash with basic usage statistics of this queue.
       def statistics
-        {
-          busy: size - @queue.size,
-          enqueued: @queue.size
-        }.freeze
+        # Ensures there are no race conditions when returning this data
+        @mutex.synchronize do
+          @statistics.dup.freeze
+        end
       end
 
       private

data/lib/karafka/processing/schedulers/default.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Namespace for Karafka OSS schedulers
+    module Schedulers
+      # FIFO scheduler for messages coming from various topics and partitions
+      class Default
+        # @param queue [Karafka::Processing::JobsQueue] queue where we want to put the jobs
+        def initialize(queue)
+          @queue = queue
+        end
+
+        # Schedules jobs in the fifo order
+        #
+        # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
+        def on_schedule_consumption(jobs_array)
+          jobs_array.each do |job|
+            @queue << job
+          end
+        end
+
+        # Both revocation and shutdown jobs can also run in fifo by default
+        alias on_schedule_revocation on_schedule_consumption
+        alias on_schedule_shutdown on_schedule_consumption
+
+        # This scheduler does not have anything to manage as it is a pass through and has no state
+        def on_manage
+          nil
+        end
+
+        # This scheduler does not need to be cleared because it is stateless
+        #
+        # @param _group_id [String] Subscription group id
+        def on_clear(_group_id)
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/strategies/base.rb

@@ -11,10 +11,19 @@ module Karafka
     module Strategies
       # Base strategy that should be included in each strategy, just to ensure the API
       module Base
-        # What should happen before jobs are enqueued
-        # @note This runs from the listener thread, not recommended to put anything slow here
-        def handle_before_enqueue
-          raise NotImplementedError, 'Implement in a subclass'
+        # Defines all the before schedule handlers for appropriate actions
+        %i[
+          consume
+          idle
+          revoked
+          shutdown
+        ].each do |action|
+          class_eval <<~RUBY, __FILE__, __LINE__ + 1
+            def handle_before_schedule_#{action}
+              # What should happen before scheduling this work
+              raise NotImplementedError, 'Implement in a subclass'
+            end
+          RUBY
         end
 
         # What should happen before we kick in the processing

data/lib/karafka/processing/strategies/default.rb

@@ -13,6 +13,23 @@ module Karafka
         # Apply strategy for a non-feature based flow
         FEATURES = %i[].freeze
 
+        # By default on all "before schedule" we just run instrumentation, nothing more
+        %i[
+          consume
+          idle
+          revoked
+          shutdown
+        ].each do |action|
+          class_eval <<~RUBY, __FILE__, __LINE__ + 1
+            # No actions needed for the standard flow here
+            def handle_before_schedule_#{action}
+              Karafka.monitor.instrument('consumer.before_schedule_#{action}', caller: self)
+
+              nil
+            end
+          RUBY
+        end
+
         # Marks message as consumed in an async way.
         #
         # @param message [Messages::Message] last successfully processed message.
@@ -76,11 +93,6 @@ module Karafka
           commit_offsets(async: false)
         end
 
-        # No actions needed for the standard flow here
-        def handle_before_enqueue
-          nil
-        end
-
         # Increment number of attempts
         def handle_before_consume
           coordinator.pause_tracker.increment

data/lib/karafka/processing/worker.rb

@@ -83,7 +83,10 @@ module Karafka
         )
       ensure
         # job can be nil when the queue is being closed
-        @jobs_queue.complete(job) if job
+        if job
+          @jobs_queue.complete(job)
+          job.finish!
+        end
 
         # Always publish info, that we completed all the work despite its result
         Karafka.monitor.instrument('worker.completed', instrument_details)

data/lib/karafka/routing/builder.rb

@@ -3,20 +3,25 @@
 module Karafka
   module Routing
     # Builder used as a DSL layer for building consumers and telling them which topics to consume
+    #
+    # @note We lock the access just in case this is used in patterns. The locks here do not have
+    #   any impact on routing usage unless being expanded, so no race conditions risks.
+    #
     # @example Build a simple (most common) route
     #   consumers do
     #     topic :new_videos do
     #       consumer NewVideosConsumer
     #     end
     #   end
-    class Builder < Concurrent::Array
+    class Builder < Array
       # Empty default per-topic config
       EMPTY_DEFAULTS = ->(_) {}.freeze
 
       private_constant :EMPTY_DEFAULTS
 
       def initialize
-        @draws = Concurrent::Array.new
+        @mutex = Mutex.new
+        @draws = []
         @defaults = EMPTY_DEFAULTS
         super
       end
@@ -34,21 +39,23 @@ module Karafka
       #     end
       #   end
       def draw(&block)
-        @draws << block
+        @mutex.synchronize do
+          @draws << block
 
-        instance_eval(&block)
+          instance_eval(&block)
 
-        each do |consumer_group|
-          # Validate consumer group settings
-          Contracts::ConsumerGroup.new.validate!(consumer_group.to_h)
+          each do |consumer_group|
+            # Validate consumer group settings
+            Contracts::ConsumerGroup.new.validate!(consumer_group.to_h)
 
-          # and then its topics settings
-          consumer_group.topics.each do |topic|
-            Contracts::Topic.new.validate!(topic.to_h)
-          end
+            # and then its topics settings
+            consumer_group.topics.each do |topic|
+              Contracts::Topic.new.validate!(topic.to_h)
+            end
 
-          # Initialize subscription groups after all the routing is done
-          consumer_group.subscription_groups
+            # Initialize subscription groups after all the routing is done
+            consumer_group.subscription_groups
+          end
         end
       end
 
@@ -61,9 +68,11 @@ module Karafka
 
       # Clears the builder and the draws memory
       def clear
-        @defaults = EMPTY_DEFAULTS
-        @draws.clear
-        super
+        @mutex.synchronize do
+          @defaults = EMPTY_DEFAULTS
+          @draws.clear
+          super
+        end
       end
 
       # @param block [Proc] block with per-topic evaluated defaults
@@ -71,7 +80,13 @@ module Karafka
       def defaults(&block)
         return @defaults unless block
 
-        @defaults = block
+        if @mutex.owned?
+          @defaults = block
+        else
+          @mutex.synchronize do
+            @defaults = block
+          end
+        end
       end
 
       private

data/lib/karafka/routing/proxy.rb

@@ -10,11 +10,12 @@ module Karafka
       # @param target [Object] target object to which we proxy any DSL call
       # @param defaults [Proc] defaults for target that should be applicable after the proper
       #   proxy context (if needed)
-      # @param block [Proc] block that we want to evaluate in the proxy context
+      # @param block [Proc, nil] block that we want to evaluate in the proxy context or nil if no
+      #   proxy block context for example because whole context is taken from defaults
       def initialize(target, defaults = ->(_) {}, &block)
         @target = target
-        instance_eval(&block)
-        instance_eval(&defaults)
+        instance_eval(&block) if block
+        instance_eval(&defaults) if defaults
       end
 
       # Ruby 2.7.0 to 2.7.2 do not have arg forwarding, so we fallback to the old way

data/lib/karafka/routing/subscription_group.rb

@@ -10,19 +10,24 @@ module Karafka
     class SubscriptionGroup
       attr_reader :id, :name, :topics, :kafka, :consumer_group
 
-      # Numeric for counting groups
-      GROUP_COUNT = Concurrent::AtomicFixnum.new
+      # Lock for generating new ids safely
+      ID_MUTEX = Mutex.new
 
-      private_constant :GROUP_COUNT
+      private_constant :ID_MUTEX
 
       class << self
         # Generates new subscription group id that will be used in case of anonymous subscription
         #   groups
         # @return [String] hex(6) compatible reproducible id
         def id
-          ::Digest::MD5.hexdigest(
-            GROUP_COUNT.increment.to_s
-          )[0..11]
+          ID_MUTEX.synchronize do
+            @group_counter ||= 0
+            @group_counter += 1
+
+            ::Digest::MD5.hexdigest(
+              @group_counter.to_s
+            )[0..11]
+          end
         end
       end
 

data/lib/karafka/routing/topics.rb

@@ -9,7 +9,7 @@ module Karafka
       include Enumerable
       extend Forwardable
 
-      def_delegators :@accumulator, :[], :size, :empty?, :last, :<<
+      def_delegators :@accumulator, :[], :size, :empty?, :last, :<<, :map!, :sort_by!, :reverse!
 
       # @param topics_array [Array<Karafka::Routing::Topic>] array with topics
       def initialize(topics_array)

data/lib/karafka/runner.rb

@@ -8,7 +8,7 @@ module Karafka
     def call
       # Despite possibility of having several independent listeners, we aim to have one queue for
       # jobs across and one workers poll for that
-      jobs_queue = Processing::JobsQueue.new
+      jobs_queue = App.config.internal.processing.jobs_queue_class.new
 
       workers = Processing::WorkersBatch.new(jobs_queue)
       listeners = Connection::ListenersBatch.new(jobs_queue)

data/lib/karafka/setup/config.rb

@@ -209,8 +209,9 @@ module Karafka
         end
 
         setting :processing do
+          setting :jobs_queue_class, default: Processing::JobsQueue
           # option scheduler [Object] scheduler we will be using
-          setting :scheduler, default: Processing::Scheduler.new
+          setting :scheduler_class, default: Processing::Schedulers::Default
           # option jobs_builder [Object] jobs builder we want to use
           setting :jobs_builder, default: Processing::JobsBuilder.new
           # option coordinator [Class] work coordinator we want to user for processing coordination
@@ -277,6 +278,9 @@ module Karafka
           # are also configured
           Pro::Loader.post_setup_all(config) if Karafka.pro?
 
+          # Subscribe the assignments tracker so we can always query all current assignments
+          config.monitor.subscribe(Instrumentation::AssignmentsTracker.instance)
+
           Karafka::App.initialized!
         end
 

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.2.12'
+  VERSION = '2.2.14'
 end

data/lib/karafka.rb

@@ -16,7 +16,6 @@
   singleton
   digest
   zeitwerk
-  concurrent/atomic/atomic_fixnum
 ].each(&method(:require))
 
 # Karafka framework main namespace