karafka 2.0.0.alpha4 → 2.0.0.beta1

data/lib/karafka/connection/rebalance_manager.rb

@@ -9,35 +9,50 @@ module Karafka
     #
     # @note Since this does not happen really often, we try to stick with same objects for the
     #   empty states most of the time, so we don't create many objects during the manager life
+    #
+    # @note Internally in the rebalance manager we have a notion of lost partitions. Partitions
+    #   that are lost, are those that got revoked but did not get re-assigned back. We do not
+    #   expose this concept outside and we normalize to have them revoked, as it is irrelevant
+    #   from the rest of the code perspective as only those that are lost are truly revoked.
     class RebalanceManager
+      # Empty array for internal usage not to create new objects
+      EMPTY_ARRAY = [].freeze
+
+      private_constant :EMPTY_ARRAY
+
       # @return [RebalanceManager]
       def initialize
-        @assigned = {}
-        @revoked = {}
+        @assigned_partitions = {}
+        @revoked_partitions = {}
+        @lost_partitions = {}
       end
 
-      # @return [Hash<String, Array<Integer>>] hash where the keys are the names of topics for
-      #   which we've got new partitions assigned and array with ids of the partitions as the value
-      # @note Once assigned partitions are fetched, the state will be reset since the callbacks
-      #   for new assigned partitions are set only during a state change
-      def assigned_partitions
-        return @assigned if @assigned.empty?
-
-        result = @assigned.dup
-        @assigned.clear
-        result
+      # Resets the rebalance manager state
+      # This needs to be done before each polling loop as during the polling, the state may be
+      # changed
+      def clear
+        @assigned_partitions.clear
+        @revoked_partitions.clear
+        @lost_partitions.clear
       end
 
       # @return [Hash<String, Array<Integer>>] hash where the keys are the names of topics for
       #   which we've lost partitions and array with ids of the partitions as the value
-      # @note Once revoked partitions are fetched, the state will be reset since the callbacks
-      #   for new revoked partitions are set only during a state change
+      # @note We do not consider as lost topics and partitions that got revoked and assigned
       def revoked_partitions
-        return @revoked if @revoked.empty?
+        return @revoked_partitions if @revoked_partitions.empty?
+        return @lost_partitions unless @lost_partitions.empty?
+
+        @revoked_partitions.each do |topic, partitions|
+          @lost_partitions[topic] = partitions - @assigned_partitions.fetch(topic, EMPTY_ARRAY)
+        end
+
+        @lost_partitions
+      end
 
-        result = @revoked.dup
-        @revoked.clear
-        result
+      # @return [Boolean] true if any partitions were revoked
+      def revoked_partitions?
+        !revoked_partitions.empty?
       end
 
       # Callback that kicks in inside of rdkafka, when new partitions are assigned.
@@ -46,7 +61,7 @@ module Karafka
       # @param _ [Rdkafka::Consumer]
       # @param partitions [Rdkafka::Consumer::TopicPartitionList]
       def on_partitions_assigned(_, partitions)
-        @assigned = partitions.to_h.transform_values { |part| part.map(&:partition) }
+        @assigned_partitions = partitions.to_h.transform_values { |part| part.map(&:partition) }
       end
 
       # Callback that kicks in inside of rdkafka, when partitions are revoked.
@@ -55,7 +70,7 @@ module Karafka
       # @param _ [Rdkafka::Consumer]
       # @param partitions [Rdkafka::Consumer::TopicPartitionList]
       def on_partitions_revoked(_, partitions)
-        @revoked = partitions.to_h.transform_values { |part| part.map(&:partition) }
+        @revoked_partitions = partitions.to_h.transform_values { |part| part.map(&:partition) }
       end
     end
   end

data/lib/karafka/contracts/config.rb

@@ -32,6 +32,7 @@ module Karafka
           required(:routing_builder)
           required(:status)
           required(:process)
+          required(:scheduler)
           required(:subscription_groups_builder)
         end
       end

data/lib/karafka/instrumentation/{stdout_listener.rb → logger_listener.rb}

@@ -4,7 +4,7 @@ module Karafka
   module Instrumentation
     # Default listener that hooks up to our instrumentation and uses its events for logging
     # It can be removed/replaced or anything without any harm to the Karafka app flow.
-    class StdoutListener
+    class LoggerListener
       # Log levels that we use in this particular listener
       USED_LOG_LEVELS = %i[
         debug

data/lib/karafka/instrumentation/monitor.rb

@@ -22,7 +22,8 @@ module Karafka
         app.stopping
         app.stopped
 
-        consumer.consume
+        consumer.prepared
+        consumer.consumed
         consumer.revoked
         consumer.shutdown
 

data/lib/karafka/pro/active_job/dispatcher.rb

@@ -1,18 +1,18 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component.
-# All of the commercial components are present in the lib/karafka/pro directory of this repository
-# and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
-
 module Karafka
   module Pro
     # Karafka Pro ActiveJob components
     module ActiveJob
+      # This Karafka component is a Pro component.
+      # All of the commercial components are present in the lib/karafka/pro directory of this
+      # repository and their usage requires commercial license agreement.
+      #
+      # Karafka has also commercial-friendly license, commercial support and commercial components.
+      #
+      # By sending a pull request to the pro components, you are agreeing to transfer the copyright
+      # of your code to Maciej Mensfeld.
+
       # Pro dispatcher that sends the ActiveJob job to a proper topic based on the queue name
       # and that allows to inject additional options into the producer, effectively allowing for a
       # much better and more granular control over the dispatch and consumption process.

data/lib/karafka/pro/active_job/job_options_contract.rb

@@ -1,17 +1,17 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component.
-# All of the commercial components are present in the lib/karafka/pro directory of this repository
-# and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
-
 module Karafka
   module Pro
     module ActiveJob
+      # This Karafka component is a Pro component.
+      # All of the commercial components are present in the lib/karafka/pro directory of this
+      # repository and their usage requires commercial license agreement.
+      #
+      # Karafka has also commercial-friendly license, commercial support and commercial components.
+      #
+      # By sending a pull request to the pro components, you are agreeing to transfer the copyright
+      # of your code to Maciej Mensfeld.
+
       # Contract for validating the options that can be altered with `#karafka_options` per job
       # class that works with Pro features.
       class JobOptionsContract < ::Karafka::ActiveJob::JobOptionsContract

data/lib/karafka/pro/loader.rb

@@ -1,15 +1,16 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component.
-# All of the commercial components are present in the lib/karafka/pro directory of this repository
-# and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
 module Karafka
   module Pro
+    # This Karafka component is a Pro component.
+    # All of the commercial components are present in the lib/karafka/pro directory of this
+    # repository and their usage requires commercial license agreement.
+    #
+    # Karafka has also commercial-friendly license, commercial support and commercial components.
+    #
+    # By sending a pull request to the pro components, you are agreeing to transfer the copyright
+    # of your code to Maciej Mensfeld.
+
     # Loader requires and loads all the pro components only when they are needed
     class Loader
       class << self
@@ -17,11 +18,15 @@ module Karafka
         # @param config [Dry::Configurable::Config] whole app config that we can alter with pro
         #   components
         def setup(config)
+          require_relative 'performance_tracker'
           require_relative 'active_job/dispatcher'
           require_relative 'active_job/job_options_contract'
 
           config.internal.active_job.dispatcher = ActiveJob::Dispatcher.new
           config.internal.active_job.job_options_contract = ActiveJob::JobOptionsContract.new
+
+          # Monitor time needed to process each message from a single partition
+          config.monitor.subscribe(PerformanceTracker.instance)
         end
       end
     end

data/lib/karafka/pro/performance_tracker.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Pro
+    # This Karafka component is a Pro component.
+    # All of the commercial components are present in the lib/karafka/pro directory of this
+    # repository and their usage requires commercial license agreement.
+    #
+    # Karafka has also commercial-friendly license, commercial support and commercial components.
+    #
+    # By sending a pull request to the pro components, you are agreeing to transfer the copyright
+    # of your code to Maciej Mensfeld.
+
+    # Tracker used to keep track of performance metrics
+    # It provides insights that can be used to optimize processing flow
+    class PerformanceTracker
+      include Singleton
+
+      # How many samples do we collect per topic partition
+      SAMPLES_COUNT = 200
+
+      private_constant :SAMPLES_COUNT
+
+      # Builds up nested concurrent hash for data tracking
+      def initialize
+        @processing_times = Concurrent::Hash.new do |topics_hash, topic|
+          topics_hash[topic] = Concurrent::Hash.new do |partitions_hash, partition|
+            # This array does not have to be concurrent because we always access single partition
+            # data via instrumentation that operates in a single thread via consumer
+            partitions_hash[partition] = []
+          end
+        end
+      end
+
+      # @param topic [String]
+      # @param partition [Integer]
+      # @return [Float] p95 processing time of a single message from a single topic partition
+      def processing_time_p95(topic, partition)
+        values = @processing_times[topic][partition]
+
+        return 0 if values.empty?
+        return values.first if values.size == 1
+
+        percentile(0.95, values)
+      end
+
+      # @private
+      # @param event [Dry::Events::Event] event details
+      # Tracks time taken to process a single message of a given topic partition
+      def on_consumer_consumed(event)
+        consumer = event[:caller]
+        messages = consumer.messages
+        topic = messages.metadata.topic
+        partition = messages.metadata.partition
+
+        samples = @processing_times[topic][partition]
+        samples << event[:time] / messages.count
+
+        return unless samples.size > SAMPLES_COUNT
+
+        samples.shift
+      end
+
+      private
+
+      # Computers the requested percentile out of provided values
+      # @param percentile [Float]
+      # @param values [Array<String>] all the values based on which we should
+      # @return [Float] computed percentile
+      def percentile(percentile, values)
+        values_sorted = values.sort
+
+        floor = (percentile * (values_sorted.length - 1) + 1).floor - 1
+        mod = (percentile * (values_sorted.length - 1) + 1).modulo(1)
+
+        values_sorted[floor] + (mod * (values_sorted[floor + 1] - values_sorted[floor]))
+      end
+    end
+  end
+end

data/lib/karafka/processing/executor.rb

@@ -4,10 +4,10 @@ module Karafka
   # Namespace that encapsulates all the logic related to processing data.
   module Processing
     # Executors:
-    # - run consumers code with provided messages batch (for `#call`) or run given teardown
-    #   operations when needed from separate threads.
-    # - they re-create consumer instances in case of partitions that were revoked
-    #   and assigned back.
+    # - run consumers code (for `#call`) or run given preparation / teardown operations when needed
+    #   from separate threads.
+    # - they re-create consumer instances in case of partitions that were revoked and assigned
+    #   back.
     #
     # @note Executors are not removed after partition is revoked. They are not that big and will
     #   be re-used in case of a re-claim
@@ -21,21 +21,21 @@ module Karafka
       # @param group_id [String] id of the subscription group to which the executor belongs
       # @param client [Karafka::Connection::Client] kafka client
       # @param topic [Karafka::Routing::Topic] topic for which this executor will run
-      # @param pause [Karafka::TimeTrackers::Pause] fetch pause object for crash pausing
-      def initialize(group_id, client, topic, pause)
+      # @param pause_tracker [Karafka::TimeTrackers::Pause] fetch pause tracker for pausing
+      def initialize(group_id, client, topic, pause_tracker)
         @id = SecureRandom.uuid
         @group_id = group_id
         @client = client
         @topic = topic
-        @pause = pause
+        @pause_tracker = pause_tracker
       end
 
-      # Runs consumer data processing against given batch and handles failures and errors.
+      # Builds the consumer instance and sets all that is needed to run the user consumption logic
       #
       # @param messages [Array<Rdkafka::Consumer::Message>] raw rdkafka messages
       # @param received_at [Time] the moment we've received the batch (actually the moment we've)
       #   enqueued it, but good enough
-      def consume(messages, received_at)
+      def prepare(messages, received_at)
         # Recreate consumer with each batch if persistence is not enabled
         # We reload the consumers with each batch instead of relying on some external signals
         # when needed for consistency. That way devs may have it on or off and not in this
@@ -49,6 +49,11 @@ module Karafka
           received_at
         )
 
+        consumer.on_prepared
+      end
+
+      # Runs consumer data processing against given batch and handles failures and errors.
+      def consume
         # We run the consumer client logic...
         consumer.on_consume
       end
@@ -86,7 +91,7 @@ module Karafka
           consumer = @topic.consumer.new
           consumer.topic = @topic
           consumer.client = @client
-          consumer.pause = @pause
+          consumer.pause_tracker = @pause_tracker
           consumer.producer = ::Karafka::App.producer
           consumer
         end

data/lib/karafka/processing/jobs/base.rb

@@ -5,6 +5,8 @@ module Karafka
     # Namespace for all the jobs that are suppose to run in workers.
     module Jobs
       # Base class for all the jobs types that are suppose to run in workers threads.
+      # Each job can have 3 main entry-points: `#prepare`, `#call` and `#teardown`
+      # Only `#call` is required.
       class Base
         extend Forwardable
 
@@ -12,6 +14,20 @@ module Karafka
         def_delegators :executor, :id, :group_id
 
         attr_reader :executor
+
+        # When redefined can run any code that should run before executing the proper code
+        def prepare; end
+
+        # When redefined can run any code that should run after executing the proper code
+        def teardown; end
+
+        # @return [Boolean] is this a non-blocking job
+        # @note Blocking job is a job, that will cause the job queue to wait until it is finished
+        #   before removing the lock on new jobs being added
+        # @note All the jobs are blocking by default
+        def non_blocking?
+          false
+        end
       end
     end
   end

data/lib/karafka/processing/jobs/consume.rb

@@ -18,9 +18,14 @@ module Karafka
           super()
         end
 
-        # Runs the given executor.
+        # Runs the preparations on the executor
+        def prepare
+          executor.prepare(@messages, @created_at)
+        end
+
+        # Runs the given executor
         def call
-          executor.consume(@messages, @created_at)
+          executor.consume
         end
       end
     end

data/lib/karafka/processing/jobs_queue.rb

@@ -21,7 +21,7 @@ module Karafka
         # We cannot use a single semaphore as it could potentially block in listeners that should
         # process with their data and also could unlock when a given group needs to remain locked
         @semaphores = Hash.new { |h, k| h[k] = Queue.new }
-        @in_processing = Hash.new { |h, k| h[k] = {} }
+        @in_processing = Hash.new { |h, k| h[k] = [] }
         @mutex = Mutex.new
       end
 
@@ -44,9 +44,9 @@ module Karafka
         @mutex.synchronize do
           group = @in_processing[job.group_id]
 
-          raise(Errors::JobsQueueSynchronizationError, job.group_id) if group.key?(job.id)
+          raise(Errors::JobsQueueSynchronizationError, job.group_id) if group.include?(job)
 
-          group[job.id] = true
+          group << job
         end
 
         @queue << job
@@ -60,14 +60,21 @@ module Karafka
         @queue.pop
       end
 
+      # Causes the wait lock to re-check the lock conditions and potential unlock.
+      # @param group_id [String] id of the group we want to unlock for one tick
+      # @note This does not release the wait lock. It just causes a conditions recheck
+      def tick(group_id)
+        @semaphores[group_id] << true
+      end
+
       # Marks a given job from a given group as completed. When there are no more jobs from a given
       # group to be executed, we won't wait.
       #
       # @param [Jobs::Base] job that was completed
       def complete(job)
         @mutex.synchronize do
-          @in_processing[job.group_id].delete(job.id)
-          @semaphores[job.group_id] << true
+          @in_processing[job.group_id].delete(job)
+          tick(job.group_id)
         end
       end
 
@@ -79,7 +86,7 @@ module Karafka
         @mutex.synchronize do
           @in_processing[group_id].clear
           # We unlock it just in case it was blocked when clearing started
-          @semaphores[group_id] << true
+          tick(group_id)
         end
       end
 
@@ -108,13 +115,15 @@ module Karafka
       # @param group_id [String] id of the group in which jobs we're interested.
       # @return [Boolean] should we keep waiting or not
       def wait?(group_id)
+        group = @in_processing[group_id]
+
         # If it is stopping, all the previous messages that are processed at the moment need to
         # finish. Otherwise we may risk closing the client and committing offsets afterwards
-        return false if Karafka::App.stopping? && @in_processing[group_id].empty?
+        return false if Karafka::App.stopping? && group.empty?
         return false if @queue.closed?
-        return false if @in_processing[group_id].empty?
+        return false if group.empty?
 
-        true
+        !group.all?(&:non_blocking?)
       end
     end
   end

data/lib/karafka/processing/worker.rb

@@ -4,6 +4,18 @@ module Karafka
   module Processing
     # Workers are used to run jobs in separate threads.
     # Workers are the main processing units of the Karafka framework.
+    #
+    # Each job runs in three stages:
+    #   - prepare - here we can run any code that we would need to run blocking before we allow
+    #               the job to run fully async (non blocking). This will always run in a blocking
+    #               way and can be used to make sure all the resources and external dependencies
+    #               are satisfied before going async.
+    #
+    #   - call - actual processing logic that can run sync or async
+    #
+    #   - teardown - it should include any code that we want to run after we executed the user
+    #                code. This can be used to unlock certain resources or do other things that are
+    #                not user code but need to run after user code base is executed.
     class Worker
       extend Forwardable
 
@@ -33,7 +45,18 @@ module Karafka
         job = @jobs_queue.pop
 
         if job
+          job.prepare
+
+          # If a job is marked as non blocking, we can run a tick in the job queue and if there
+          # are no other blocking factors, the job queue will be unlocked.
+          # If this does not run, all the things will be blocking and job queue won't allow to
+          # pass it until done.
+          @jobs_queue.tick(job.group_id) if job.non_blocking?
+
           job.call
+
+          job.teardown
+
           true
         else
           false

data/lib/karafka/railtie.rb

@@ -82,8 +82,20 @@ if rails
       initializer 'karafka.require_karafka_boot_file' do |app|
         rails6plus = Rails.gem_version >= Gem::Version.new('6.0.0')
 
+        # If the boot file location is set to "false", we should not raise an exception and we
+        # should just not load karafka stuff. Setting this explicitly to false indicates, that
+        # karafka is part of the supply chain but it is not a first class citizen of a given
+        # system (may be just a dependency of a dependency), thus railtie should not kick in to
+        # load the non-existing boot file
+        next if Karafka.boot_file.to_s == 'false'
+
         karafka_boot_file = Rails.root.join(Karafka.boot_file.to_s).to_s
 
+        # Provide more comprehensive error for when no boot file
+        unless File.exist?(karafka_boot_file)
+          raise(Karafka::Errors::MissingBootFileError, karafka_boot_file)
+        end
+
         if rails6plus
           app.reloader.to_prepare do
             # Load Karafka boot file, so it can be used in Rails server context

data/lib/karafka/scheduler.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Karafka
+  # FIFO scheduler for messages coming from various topics and partitions
+  class Scheduler
+    # Yields messages from partitions in the fifo order
+    #
+    # @param messages_buffer [Karafka::Connection::MessagesBuffer] messages buffer with data from
+    #   multiple topics and partitions
+    # @yieldparam [String] topic name
+    # @yieldparam [Integer] partition number
+    # @yieldparam [Array<Rdkafka::Consumer::Message>] topic partition aggregated results
+    def call(messages_buffer)
+      messages_buffer.each do |topic, partitions|
+        partitions.each do |partition, messages|
+          yield(topic, partition, messages)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/setup/config.rb

@@ -60,7 +60,7 @@ module Karafka
       # option [Boolean] should we leave offset management to the user
       setting :manual_offset_management, default: false
       # options max_messages [Integer] how many messages do we want to fetch from Kafka in one go
-      setting :max_messages, default: 100_000
+      setting :max_messages, default: 1_000
       # option [Integer] number of milliseconds we can wait while fetching data
       setting :max_wait_time, default: 10_000
       # option shutdown_timeout [Integer] the number of milliseconds after which Karafka no
@@ -96,6 +96,8 @@ module Karafka
         # option subscription_groups_builder [Routing::SubscriptionGroupsBuilder] subscription
         #   group builder
         setting :subscription_groups_builder, default: Routing::SubscriptionGroupsBuilder.new
+        # option scheduler [Class] scheduler we will be using
+        setting :scheduler, default: Scheduler.new
 
         # Karafka components for ActiveJob
         setting :active_job do

data/lib/karafka/templates/karafka.rb.erb

@@ -40,7 +40,7 @@ class KarafkaApp < Karafka::App
   # interested in logging events for certain environments. Since instrumentation
   # notifications add extra boilerplate, if you want to achieve max performance,
   # listen to only what you really need for given environment.
-  Karafka.monitor.subscribe(Karafka::Instrumentation::StdoutListener.new)
+  Karafka.monitor.subscribe(Karafka::Instrumentation::LoggerListener.new)
   # Karafka.monitor.subscribe(Karafka::Instrumentation::ProctitleListener.new)
 
   routes.draw do

data/lib/karafka/time_trackers/pause.rb

@@ -41,9 +41,12 @@ module Karafka
 
       # Pauses the processing from now till the end of the interval (backoff or non-backoff)
       # and records the count.
-      def pause
+      # @param timeout [Integer] timeout value in milliseconds that overwrites the default timeout
+      # @note Providing this value can be useful when we explicitly want to pause for a certain
+      #   period of time, outside of any regular pausing logic
+      def pause(timeout = backoff_interval)
         @started_at = now
-        @ends_at = @started_at + backoff_interval
+        @ends_at = @started_at + timeout
         @count += 1
       end
 
@@ -53,6 +56,11 @@ module Karafka
         @ends_at = nil
       end
 
+      # Expires the pause, so it can be considered expired
+      def expire
+        @ends_at = nil
+      end
+
       # @return [Boolean] are we paused from processing
       def paused?
         !@started_at.nil?

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.0.alpha4'
+  VERSION = '2.0.0.beta1'
 end