karafka 2.0.0.beta4 → 2.0.0.beta5

data/lib/karafka/setup/config.rb

@@ -85,21 +85,29 @@ module Karafka
 
       # Namespace for internal settings that should not be modified directly
       setting :internal do
-        # option routing_builder [Karafka::Routing::Builder] builder instance
-        setting :routing_builder, default: Routing::Builder.new
         # option status [Karafka::Status] app status
         setting :status, default: Status.new
         # option process [Karafka::Process] process status
         # @note In the future, we need to have a single process representation for all the karafka
         #   instances
         setting :process, default: Process.new
-        # option subscription_groups_builder [Routing::SubscriptionGroupsBuilder] subscription
-        #   group builder
-        setting :subscription_groups_builder, default: Routing::SubscriptionGroupsBuilder.new
-        # option scheduler [Class] scheduler we will be using
-        setting :scheduler, default: Scheduler.new
-        # option jobs_builder [Class] jobs builder we want to use
-        setting :jobs_builder, default: Processing::JobsBuilder.new
+
+        setting :routing do
+          # option builder [Karafka::Routing::Builder] builder instance
+          setting :builder, default: Routing::Builder.new
+          # option subscription_groups_builder [Routing::SubscriptionGroupsBuilder] subscription
+          #   group builder
+          setting :subscription_groups_builder, default: Routing::SubscriptionGroupsBuilder.new
+        end
+
+        setting :processing do
+          # option scheduler [Object] scheduler we will be using
+          setting :scheduler, default: Processing::Scheduler.new
+          # option jobs_builder [Object] jobs builder we want to use
+          setting :jobs_builder, default: Processing::JobsBuilder.new
+          # option coordinator [Class] work coordinator we want to user for processing coordination
+          setting :coordinator_class, default: Processing::Coordinator
+        end
 
         # Karafka components for ActiveJob
         setting :active_job do
@@ -109,7 +117,7 @@ module Karafka
           #   ensuring, that extra job options defined are valid
           setting :job_options_contract, default: ActiveJob::JobOptionsContract.new
           # option consumer [Class] consumer class that should be used to consume ActiveJob data
-          setting :consumer, default: ActiveJob::Consumer
+          setting :consumer_class, default: ActiveJob::Consumer
         end
       end
 

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.0.beta4'
+  VERSION = '2.0.0.beta5'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.0.beta4
+  version: 2.0.0.beta5
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -34,7 +34,7 @@ cert_chain:
   R2P11bWoCtr70BsccVrN8jEhzwXngMyI2gVt750Y+dbTu1KgRqZKp/ECe7ZzPzXj
   pIy9vHxTANKYVyI4qj8OrFdEM5BQNu8oQpL0iQ==
   -----END CERTIFICATE-----
-date: 2022-06-20 00:00:00.000000000 Z
+date: 2022-07-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dry-configurable
@@ -172,7 +172,8 @@ files:
 - bin/integrations
 - bin/karafka
 - bin/scenario
-- bin/stress
+- bin/stress_many
+- bin/stress_one
 - bin/wait_for_kafka
 - certs/karafka-pro.pem
 - certs/mensfeld.pem
@@ -236,11 +237,14 @@ files:
 - lib/karafka/pro/base_consumer.rb
 - lib/karafka/pro/loader.rb
 - lib/karafka/pro/performance_tracker.rb
+- lib/karafka/pro/processing/coordinator.rb
 - lib/karafka/pro/processing/jobs/consume_non_blocking.rb
 - lib/karafka/pro/processing/jobs_builder.rb
+- lib/karafka/pro/processing/scheduler.rb
 - lib/karafka/pro/routing/extensions.rb
-- lib/karafka/pro/scheduler.rb
 - lib/karafka/process.rb
+- lib/karafka/processing/coordinator.rb
+- lib/karafka/processing/coordinators_buffer.rb
 - lib/karafka/processing/executor.rb
 - lib/karafka/processing/executors_buffer.rb
 - lib/karafka/processing/jobs/base.rb
@@ -250,6 +254,7 @@ files:
 - lib/karafka/processing/jobs_builder.rb
 - lib/karafka/processing/jobs_queue.rb
 - lib/karafka/processing/result.rb
+- lib/karafka/processing/scheduler.rb
 - lib/karafka/processing/worker.rb
 - lib/karafka/processing/workers_batch.rb
 - lib/karafka/railtie.rb
@@ -263,7 +268,6 @@ files:
 - lib/karafka/routing/topic.rb
 - lib/karafka/routing/topics.rb
 - lib/karafka/runner.rb
-- lib/karafka/scheduler.rb
 - lib/karafka/serialization/json/deserializer.rb
 - lib/karafka/server.rb
 - lib/karafka/setup/config.rb

metadata.gz.sig

@@ -1 +1,4 @@
-"k (w��r�&L�.3!E긽w3m46��7�c�>zSL�:w1�g`���(��r�U0�����SϹ�b�ű�A��5M�%�EL-v���bw�Ah�B���>�P{amO�u�$7@��3�c�����ڐ��P��Hu:��͸��K�h�E"��EZ1�)h�i��wk?'�Jճ����w�Uu����}F(5zv��O֯ћ��L�^�J�6����B| ���|ujiG�W/ ��A��;}U�9%��۬�r4�Af�>���
LϞ"�����+��E'�N��Y?ӂ����c�Z꺼ڭH}�QĪ
�*��|��*���93��BX��쎾c�
K�y]�DB�W�Y�])��ߣ�I+k���F�쇔RJ��
+뙮��F��|-aa�}:�g*Z��b�eM~hv��bX;���2Խ�a�`/��̊��r!WY�-h΂�
Μ����]\t��fذ��M%5N���YDnu6�ի7�O`�rP���<����
�#Ӊ�؞0����Db��ȭ�E (��T}��\���2\=�TfΠ
+
+��e	>�
+z:O��N�1�g�i(Y�%$�w*�[-f��/�

data/lib/karafka/pro/scheduler.rb

@@ -1,54 +0,0 @@
-# frozen_string_literal: true
-
-# This Karafka component is a Pro component.
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
-
-module Karafka
-  module Pro
-    # Optimizes scheduler that takes into consideration of execution time needed to process
-    # messages from given topics partitions. It uses the non-preemptive LJF algorithm
-    #
-    # This scheduler is designed to optimize execution times on jobs that perform IO operations as
-    # when taking IO into consideration, the can achieve optimized parallel processing.
-    #
-    # This scheduler can also work with virtual partitions.
-    #
-    # Aside from consumption jobs, other jobs do not run often, thus we can leave them with
-    # default FIFO scheduler from the default Karafka scheduler
-    class Scheduler < ::Karafka::Scheduler
-      # Schedules jobs in the LJF order for consumption
-      #
-      # @param queue [Karafka::Processing::JobsQueue] queue where we want to put the jobs
-      # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
-      #
-      def schedule_consumption(queue, jobs_array)
-        pt = PerformanceTracker.instance
-
-        ordered = []
-
-        jobs_array.each do |job|
-          messages = job.messages
-          message = messages.first
-
-          cost = pt.processing_time_p95(message.topic, message.partition) * messages.size
-
-          ordered << [job, cost]
-        end
-
-        ordered.sort_by!(&:last)
-        ordered.reverse!
-        ordered.map!(&:first)
-
-        ordered.each do |job|
-          queue << job
-        end
-      end
-    end
-  end
-end

data/lib/karafka/scheduler.rb

@@ -1,20 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # FIFO scheduler for messages coming from various topics and partitions
-  class Scheduler
-    # Schedules jobs in the fifo order
-    #
-    # @param queue [Karafka::Processing::JobsQueue] queue where we want to put the jobs
-    # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
-    def schedule_consumption(queue, jobs_array)
-      jobs_array.each do |job|
-        queue << job
-      end
-    end
-
-    # Both revocation and shutdown jobs can also run in fifo by default
-    alias schedule_revocation schedule_consumption
-    alias schedule_shutdown schedule_consumption
-  end
-end