karafka 2.2.13 → 2.2.14

data/lib/karafka/setup/config.rb

@@ -211,7 +211,7 @@ module Karafka
         setting :processing do
           setting :jobs_queue_class, default: Processing::JobsQueue
           # option scheduler [Object] scheduler we will be using
-          setting :scheduler_class, default: Processing::Scheduler
+          setting :scheduler_class, default: Processing::Schedulers::Default
           # option jobs_builder [Object] jobs builder we want to use
           setting :jobs_builder, default: Processing::JobsBuilder.new
           # option coordinator [Class] work coordinator we want to user for processing coordination
@@ -278,6 +278,9 @@ module Karafka
           # are also configured
           Pro::Loader.post_setup_all(config) if Karafka.pro?
 
+          # Subscribe the assignments tracker so we can always query all current assignments
+          config.monitor.subscribe(Instrumentation::AssignmentsTracker.instance)
+
           Karafka::App.initialized!
         end
 

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.2.13'
+  VERSION = '2.2.14'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.2.13
+  version: 2.2.14
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2023-11-17 00:00:00.000000000 Z
+date: 2023-12-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -183,6 +183,7 @@ files:
 - lib/karafka/helpers/colorize.rb
 - lib/karafka/helpers/interval_runner.rb
 - lib/karafka/helpers/multi_delegator.rb
+- lib/karafka/instrumentation/assignments_tracker.rb
 - lib/karafka/instrumentation/callbacks/error.rb
 - lib/karafka/instrumentation/callbacks/rebalance.rb
 - lib/karafka/instrumentation/callbacks/statistics.rb
@@ -229,11 +230,11 @@ files:
 - lib/karafka/pro/encryption/messages/middleware.rb
 - lib/karafka/pro/encryption/messages/parser.rb
 - lib/karafka/pro/encryption/setup/config.rb
+- lib/karafka/pro/instrumentation/performance_tracker.rb
 - lib/karafka/pro/iterator.rb
 - lib/karafka/pro/iterator/expander.rb
 - lib/karafka/pro/iterator/tpl_builder.rb
 - lib/karafka/pro/loader.rb
-- lib/karafka/pro/performance_tracker.rb
 - lib/karafka/pro/processing/collapser.rb
 - lib/karafka/pro/processing/coordinator.rb
 - lib/karafka/pro/processing/filters/base.rb
@@ -248,7 +249,8 @@ files:
 - lib/karafka/pro/processing/jobs_builder.rb
 - lib/karafka/pro/processing/jobs_queue.rb
 - lib/karafka/pro/processing/partitioner.rb
-- lib/karafka/pro/processing/scheduler.rb
+- lib/karafka/pro/processing/schedulers/base.rb
+- lib/karafka/pro/processing/schedulers/default.rb
 - lib/karafka/pro/processing/strategies.rb
 - lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom.rb
 - lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom_vp.rb
@@ -367,7 +369,7 @@ files:
 - lib/karafka/processing/jobs_queue.rb
 - lib/karafka/processing/partitioner.rb
 - lib/karafka/processing/result.rb
-- lib/karafka/processing/scheduler.rb
+- lib/karafka/processing/schedulers/default.rb
 - lib/karafka/processing/strategies/aj_dlq_mom.rb
 - lib/karafka/processing/strategies/aj_mom.rb
 - lib/karafka/processing/strategies/base.rb

data/lib/karafka/pro/performance_tracker.rb

@@ -1,84 +0,0 @@
-# frozen_string_literal: true
-
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
-
-module Karafka
-  module Pro
-    # Tracker used to keep track of performance metrics
-    # It provides insights that can be used to optimize processing flow
-    class PerformanceTracker
-      include Singleton
-
-      # How many samples do we collect per topic partition
-      SAMPLES_COUNT = 200
-
-      private_constant :SAMPLES_COUNT
-
-      # Builds up nested concurrent hash for data tracking
-      def initialize
-        @processing_times = Concurrent::Map.new do |topics_hash, topic|
-          topics_hash.compute_if_absent(topic) do
-            Concurrent::Map.new do |partitions_hash, partition|
-              # This array does not have to be concurrent because we always access single
-              # partition data via instrumentation that operates in a single thread via consumer
-              partitions_hash.compute_if_absent(partition) { [] }
-            end
-          end
-        end
-      end
-
-      # @param topic [String]
-      # @param partition [Integer]
-      # @return [Float] p95 processing time of a single message from a single topic partition
-      def processing_time_p95(topic, partition)
-        values = @processing_times[topic][partition]
-
-        return 0 if values.empty?
-        return values.first if values.size == 1
-
-        percentile(0.95, values)
-      end
-
-      # @private
-      # @param event [Karafka::Core::Monitoring::Event] event details
-      # Tracks time taken to process a single message of a given topic partition
-      def on_consumer_consumed(event)
-        consumer = event[:caller]
-        messages = consumer.messages
-        topic = messages.metadata.topic
-        partition = messages.metadata.partition
-
-        samples = @processing_times[topic][partition]
-        samples << event[:time] / messages.count
-
-        return unless samples.size > SAMPLES_COUNT
-
-        samples.shift
-      end
-
-      private
-
-      # Computers the requested percentile out of provided values
-      # @param percentile [Float]
-      # @param values [Array<String>] all the values based on which we should
-      # @return [Float] computed percentile
-      def percentile(percentile, values)
-        values_sorted = values.sort
-
-        floor = (percentile * (values_sorted.length - 1) + 1).floor - 1
-        mod = (percentile * (values_sorted.length - 1) + 1).modulo(1)
-
-        values_sorted[floor] + (mod * (values_sorted[floor + 1] - values_sorted[floor]))
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/scheduler.rb

@@ -1,74 +0,0 @@
-# frozen_string_literal: true
-
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
-
-module Karafka
-  module Pro
-    module Processing
-      # Optimizes scheduler that takes into consideration of execution time needed to process
-      # messages from given topics partitions. It uses the non-preemptive LJF algorithm
-      #
-      # This scheduler is designed to optimize execution times on jobs that perform IO operations
-      # as when taking IO into consideration, the can achieve optimized parallel processing.
-      #
-      # This scheduler can also work with virtual partitions.
-      #
-      # Aside from consumption jobs, other jobs do not run often, thus we can leave them with
-      # default FIFO scheduler from the default Karafka scheduler
-      class Scheduler < ::Karafka::Processing::Scheduler
-        # Schedules jobs in the LJF order for consumption
-        #
-        # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
-        #
-        def schedule_consumption(jobs_array)
-          perf_tracker = PerformanceTracker.instance
-
-          ordered = []
-
-          jobs_array.each do |job|
-            ordered << [
-              job,
-              processing_cost(perf_tracker, job)
-            ]
-          end
-
-          ordered.sort_by!(&:last)
-          ordered.reverse!
-          ordered.map!(&:first)
-
-          ordered.each do |job|
-            @queue << job
-          end
-        end
-
-        private
-
-        # @param perf_tracker [PerformanceTracker]
-        # @param job [Karafka::Processing::Jobs::Base] job we will be processing
-        # @return [Numeric] estimated cost of processing this job
-        def processing_cost(perf_tracker, job)
-          if job.is_a?(::Karafka::Processing::Jobs::Consume)
-            messages = job.messages
-            message = messages.first
-
-            perf_tracker.processing_time_p95(message.topic, message.partition) * messages.size
-          else
-            # LJF will set first the most expensive, but we want to run the zero cost jobs
-            # related to the lifecycle always first. That is why we "emulate" that they
-            # the longest possible jobs that anyone can run
-            Float::INFINITY
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/processing/scheduler.rb

@@ -1,38 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Processing
-    # FIFO scheduler for messages coming from various topics and partitions
-    class Scheduler
-      # @param queue [Karafka::Processing::JobsQueue] queue where we want to put the jobs
-      def initialize(queue)
-        @queue = queue
-      end
-
-      # Schedules jobs in the fifo order
-      #
-      # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
-      def schedule_consumption(jobs_array)
-        jobs_array.each do |job|
-          @queue << job
-        end
-      end
-
-      # Both revocation and shutdown jobs can also run in fifo by default
-      alias schedule_revocation schedule_consumption
-      alias schedule_shutdown schedule_consumption
-
-      # This scheduler does not have anything to manage as it is a pass through and has no state
-      def manage
-        nil
-      end
-
-      # This scheduler does not need to be cleared because it is stateless
-      #
-      # @param _group_id [String] Subscription group id
-      def clear(_group_id)
-        nil
-      end
-    end
-  end
-end