karafka 2.0.0.beta3 → 2.0.0.rc1

data/lib/karafka/setup/config.rb

@@ -85,21 +85,31 @@ module Karafka
 
       # Namespace for internal settings that should not be modified directly
       setting :internal do
-        # option routing_builder [Karafka::Routing::Builder] builder instance
-        setting :routing_builder, default: Routing::Builder.new
         # option status [Karafka::Status] app status
         setting :status, default: Status.new
         # option process [Karafka::Process] process status
         # @note In the future, we need to have a single process representation for all the karafka
         #   instances
         setting :process, default: Process.new
-        # option subscription_groups_builder [Routing::SubscriptionGroupsBuilder] subscription
-        #   group builder
-        setting :subscription_groups_builder, default: Routing::SubscriptionGroupsBuilder.new
-        # option scheduler [Class] scheduler we will be using
-        setting :scheduler, default: Scheduler.new
-        # option jobs_builder [Class] jobs builder we want to use
-        setting :jobs_builder, default: Processing::JobsBuilder.new
+
+        setting :routing do
+          # option builder [Karafka::Routing::Builder] builder instance
+          setting :builder, default: Routing::Builder.new
+          # option subscription_groups_builder [Routing::SubscriptionGroupsBuilder] subscription
+          #   group builder
+          setting :subscription_groups_builder, default: Routing::SubscriptionGroupsBuilder.new
+        end
+
+        setting :processing do
+          # option scheduler [Object] scheduler we will be using
+          setting :scheduler, default: Processing::Scheduler.new
+          # option jobs_builder [Object] jobs builder we want to use
+          setting :jobs_builder, default: Processing::JobsBuilder.new
+          # option coordinator [Class] work coordinator we want to user for processing coordination
+          setting :coordinator_class, default: Processing::Coordinator
+          # option partitioner_class [Class] partitioner we use against a batch of data
+          setting :partitioner_class, default: Processing::Partitioner
+        end
 
         # Karafka components for ActiveJob
         setting :active_job do
@@ -109,7 +119,7 @@ module Karafka
           #   ensuring, that extra job options defined are valid
           setting :job_options_contract, default: ActiveJob::JobOptionsContract.new
           # option consumer [Class] consumer class that should be used to consume ActiveJob data
-          setting :consumer, default: ActiveJob::Consumer
+          setting :consumer_class, default: ActiveJob::Consumer
         end
       end
 

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.0.beta3'
+  VERSION = '2.0.0.rc1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.0.beta3
+  version: 2.0.0.rc1
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -34,7 +34,7 @@ cert_chain:
   R2P11bWoCtr70BsccVrN8jEhzwXngMyI2gVt750Y+dbTu1KgRqZKp/ECe7ZzPzXj
   pIy9vHxTANKYVyI4qj8OrFdEM5BQNu8oQpL0iQ==
   -----END CERTIFICATE-----
-date: 2022-06-14 00:00:00.000000000 Z
+date: 2022-07-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dry-configurable
@@ -112,7 +112,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.3.0
+        version: 2.3.1
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -122,7 +122,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.3.0
+        version: 2.3.1
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -172,7 +172,9 @@ files:
 - bin/integrations
 - bin/karafka
 - bin/scenario
-- bin/stress
+- bin/stress_many
+- bin/stress_one
+- bin/wait_for_kafka
 - certs/karafka-pro.pem
 - certs/mensfeld.pem
 - config/errors.yml
@@ -232,14 +234,18 @@ files:
 - lib/karafka/pro/active_job/consumer.rb
 - lib/karafka/pro/active_job/dispatcher.rb
 - lib/karafka/pro/active_job/job_options_contract.rb
-- lib/karafka/pro/base_consumer_extensions.rb
+- lib/karafka/pro/base_consumer.rb
 - lib/karafka/pro/loader.rb
 - lib/karafka/pro/performance_tracker.rb
+- lib/karafka/pro/processing/coordinator.rb
 - lib/karafka/pro/processing/jobs/consume_non_blocking.rb
 - lib/karafka/pro/processing/jobs_builder.rb
+- lib/karafka/pro/processing/partitioner.rb
+- lib/karafka/pro/processing/scheduler.rb
 - lib/karafka/pro/routing/extensions.rb
-- lib/karafka/pro/scheduler.rb
 - lib/karafka/process.rb
+- lib/karafka/processing/coordinator.rb
+- lib/karafka/processing/coordinators_buffer.rb
 - lib/karafka/processing/executor.rb
 - lib/karafka/processing/executors_buffer.rb
 - lib/karafka/processing/jobs/base.rb
@@ -248,6 +254,9 @@ files:
 - lib/karafka/processing/jobs/shutdown.rb
 - lib/karafka/processing/jobs_builder.rb
 - lib/karafka/processing/jobs_queue.rb
+- lib/karafka/processing/partitioner.rb
+- lib/karafka/processing/result.rb
+- lib/karafka/processing/scheduler.rb
 - lib/karafka/processing/worker.rb
 - lib/karafka/processing/workers_batch.rb
 - lib/karafka/railtie.rb
@@ -261,7 +270,6 @@ files:
 - lib/karafka/routing/topic.rb
 - lib/karafka/routing/topics.rb
 - lib/karafka/runner.rb
-- lib/karafka/scheduler.rb
 - lib/karafka/serialization/json/deserializer.rb
 - lib/karafka/server.rb
 - lib/karafka/setup/config.rb

data/lib/karafka/pro/base_consumer_extensions.rb

@@ -1,66 +0,0 @@
-# frozen_string_literal: true
-
-# This Karafka component is a Pro component.
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
-
-module Karafka
-  module Pro
-    # Extensions to the base consumer that make it more pro and fancy
-    #
-    # @note In case of using lrj, manual pausing may not be the best idea as resume needs to happen
-    #   after each batch is processed.
-    #
-    # They need to be added to the consumer via `#prepend`
-    module BaseConsumerExtensions
-      # Pause for tops 31 years
-      MAX_PAUSE_TIME = 1_000_000_000_000
-
-      private_constant :MAX_PAUSE_TIME
-
-      # Pauses processing of a given partition until we're done with the processing
-      # This ensures, that we can easily poll not reaching the `max.poll.interval`
-      def on_prepare
-        # Pause at the first message in a batch. That way in case of a crash, we will not loose
-        # any messages
-        pause(messages.first.offset, MAX_PAUSE_TIME) if topic.long_running_job?
-
-        super
-      end
-
-      # After user code, we seek and un-pause our partition
-      def on_consume
-        # If anything went wrong here, we should not run any partition management as it's Karafka
-        # core that will handle the backoff
-        return unless super
-
-        return unless topic.long_running_job?
-
-        # Nothing to resume if it was revoked
-        return if revoked?
-
-        # Once processing is done, we move to the new offset based on commits
-        seek(@seek_offset || messages.first.offset)
-        resume
-      end
-
-      # Marks this consumer revoked state as true
-      # This allows us for things like lrj to finish early as this state may change during lrj
-      # execution
-      def on_revoked
-        @revoked = true
-        super
-      end
-
-      # @return [Boolean] true if partition was revoked from the current consumer
-      def revoked?
-        @revoked || false
-      end
-    end
-  end
-end

data/lib/karafka/pro/scheduler.rb

@@ -1,54 +0,0 @@
-# frozen_string_literal: true
-
-# This Karafka component is a Pro component.
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
-
-module Karafka
-  module Pro
-    # Optimizes scheduler that takes into consideration of execution time needed to process
-    # messages from given topics partitions. It uses the non-preemptive LJF algorithm
-    #
-    # This scheduler is designed to optimize execution times on jobs that perform IO operations as
-    # when taking IO into consideration, the can achieve optimized parallel processing.
-    #
-    # This scheduler can also work with virtual partitions.
-    #
-    # Aside from consumption jobs, other jobs do not run often, thus we can leave them with
-    # default FIFO scheduler from the default Karafka scheduler
-    class Scheduler < ::Karafka::Scheduler
-      # Schedules jobs in the LJF order for consumption
-      #
-      # @param queue [Karafka::Processing::JobsQueue] queue where we want to put the jobs
-      # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
-      #
-      def schedule_consumption(queue, jobs_array)
-        pt = PerformanceTracker.instance
-
-        ordered = []
-
-        jobs_array.each do |job|
-          messages = job.messages
-          message = messages.first
-
-          cost = pt.processing_time_p95(message.topic, message.partition) * messages.size
-
-          ordered << [job, cost]
-        end
-
-        ordered.sort_by!(&:last)
-        ordered.reverse!
-        ordered.map!(&:first)
-
-        ordered.each do |job|
-          queue << job
-        end
-      end
-    end
-  end
-end

data/lib/karafka/scheduler.rb

@@ -1,20 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # FIFO scheduler for messages coming from various topics and partitions
-  class Scheduler
-    # Schedules jobs in the fifo order
-    #
-    # @param queue [Karafka::Processing::JobsQueue] queue where we want to put the jobs
-    # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
-    def schedule_consumption(queue, jobs_array)
-      jobs_array.each do |job|
-        queue << job
-      end
-    end
-
-    # Both revocation and shutdown jobs can also run in fifo by default
-    alias schedule_revocation schedule_consumption
-    alias schedule_shutdown schedule_consumption
-  end
-end