karafka 2.0.0.beta4 → 2.0.0.rc2

data/lib/karafka/processing/jobs/consume.rb

@@ -12,17 +12,19 @@ module Karafka
         # @param executor [Karafka::Processing::Executor] executor that is suppose to run a given
         #   job
         # @param messages [Karafka::Messages::Messages] karafka messages batch
+        # @param coordinator [Karafka::Processing::Coordinator] processing coordinator
         # @return [Consume]
-        def initialize(executor, messages)
+        def initialize(executor, messages, coordinator)
           @executor = executor
           @messages = messages
+          @coordinator = coordinator
           @created_at = Time.now
           super()
         end
 
         # Runs the before consumption preparations on the executor
         def before_call
-          executor.before_consume(@messages, @created_at)
+          executor.before_consume(@messages, @created_at, @coordinator)
         end
 
         # Runs the given executor

data/lib/karafka/processing/jobs_builder.rb

@@ -7,9 +7,10 @@ module Karafka
     class JobsBuilder
       # @param executor [Karafka::Processing::Executor]
       # @param messages [Karafka::Messages::Messages] messages batch to be consumed
+      # @param coordinator [Karafka::Processing::Coordinator]
       # @return [Karafka::Processing::Jobs::Consume] consumption job
-      def consume(executor, messages)
-        Jobs::Consume.new(executor, messages)
+      def consume(executor, messages, coordinator)
+        Jobs::Consume.new(executor, messages, coordinator)
       end
 
       # @param executor [Karafka::Processing::Executor]

data/lib/karafka/processing/partitioner.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Basic partitioner for work division
+    # It does not divide any work.
+    class Partitioner
+      # @param subscription_group [Karafka::Routing::SubscriptionGroup] subscription group
+      def initialize(subscription_group)
+        @subscription_group = subscription_group
+      end
+
+      # @param _topic [String] topic name
+      # @param messages [Array<Karafka::Messages::Message>] karafka messages
+      # @yieldparam [Integer] group id
+      # @yieldparam [Array<Karafka::Messages::Message>] karafka messages
+      def call(_topic, messages)
+        yield(0, messages)
+      end
+    end
+  end
+end

data/lib/karafka/processing/result.rb

@@ -10,11 +10,6 @@ module Karafka
         @success = true
       end
 
-      # @return [Boolean]
-      def failure?
-        !success?
-      end
-
       # @return [Boolean]
       def success?
         @success

data/lib/karafka/processing/scheduler.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # FIFO scheduler for messages coming from various topics and partitions
+    class Scheduler
+      # Schedules jobs in the fifo order
+      #
+      # @param queue [Karafka::Processing::JobsQueue] queue where we want to put the jobs
+      # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
+      def schedule_consumption(queue, jobs_array)
+        jobs_array.each do |job|
+          queue << job
+        end
+      end
+
+      # Both revocation and shutdown jobs can also run in fifo by default
+      alias schedule_revocation schedule_consumption
+      alias schedule_shutdown schedule_consumption
+    end
+  end
+end

data/lib/karafka/routing/consumer_group.rb

@@ -38,7 +38,7 @@ module Karafka
       # @return [Array<Routing::SubscriptionGroup>] all the subscription groups build based on
       #   the consumer group topics
       def subscription_groups
-        App.config.internal.subscription_groups_builder.call(topics)
+        App.config.internal.routing.subscription_groups_builder.call(topics)
       end
 
       # Hashed version of consumer group that can be used for validation purposes

data/lib/karafka/routing/topic.rb

@@ -66,6 +66,15 @@ module Karafka
         end
       end
 
+      # @return [Class] consumer class that we should use
+      # @note This is just an alias to the `#consumer` method. We however want to use it internally
+      #   instead of referencing the `#consumer`. We use this to indicate that this method returns
+      #   class and not an instance. In the routing we want to keep the `#consumer Consumer`
+      #   routing syntax, but for references outside, we should use this one.
+      def consumer_class
+        consumer
+      end
+
       # @return [Boolean] true if this topic offset is handled by the end user
       def manual_offset_management?
         manual_offset_management

data/lib/karafka/setup/config.rb

@@ -12,7 +12,7 @@ module Karafka
     #   enough and will still keep the code simple
     # @see Karafka::Setup::Configurators::Base for more details about configurators api
     class Config
-      extend Dry::Configurable
+      extend ::WaterDrop::Configurable
 
       # Defaults for kafka settings, that will be overwritten only if not present already
       KAFKA_DEFAULTS = {
@@ -62,7 +62,7 @@ module Karafka
       # options max_messages [Integer] how many messages do we want to fetch from Kafka in one go
       setting :max_messages, default: 1_000
       # option [Integer] number of milliseconds we can wait while fetching data
-      setting :max_wait_time, default: 10_000
+      setting :max_wait_time, default: 5_000
       # option shutdown_timeout [Integer] the number of milliseconds after which Karafka no
       #   longer waits for the consumers to stop gracefully but instead we force terminate
       #   everything.
@@ -85,21 +85,31 @@ module Karafka
 
       # Namespace for internal settings that should not be modified directly
       setting :internal do
-        # option routing_builder [Karafka::Routing::Builder] builder instance
-        setting :routing_builder, default: Routing::Builder.new
         # option status [Karafka::Status] app status
         setting :status, default: Status.new
         # option process [Karafka::Process] process status
         # @note In the future, we need to have a single process representation for all the karafka
         #   instances
         setting :process, default: Process.new
-        # option subscription_groups_builder [Routing::SubscriptionGroupsBuilder] subscription
-        #   group builder
-        setting :subscription_groups_builder, default: Routing::SubscriptionGroupsBuilder.new
-        # option scheduler [Class] scheduler we will be using
-        setting :scheduler, default: Scheduler.new
-        # option jobs_builder [Class] jobs builder we want to use
-        setting :jobs_builder, default: Processing::JobsBuilder.new
+
+        setting :routing do
+          # option builder [Karafka::Routing::Builder] builder instance
+          setting :builder, default: Routing::Builder.new
+          # option subscription_groups_builder [Routing::SubscriptionGroupsBuilder] subscription
+          #   group builder
+          setting :subscription_groups_builder, default: Routing::SubscriptionGroupsBuilder.new
+        end
+
+        setting :processing do
+          # option scheduler [Object] scheduler we will be using
+          setting :scheduler, default: Processing::Scheduler.new
+          # option jobs_builder [Object] jobs builder we want to use
+          setting :jobs_builder, default: Processing::JobsBuilder.new
+          # option coordinator [Class] work coordinator we want to user for processing coordination
+          setting :coordinator_class, default: Processing::Coordinator
+          # option partitioner_class [Class] partitioner we use against a batch of data
+          setting :partitioner_class, default: Processing::Partitioner
+        end
 
         # Karafka components for ActiveJob
         setting :active_job do
@@ -109,10 +119,14 @@ module Karafka
           #   ensuring, that extra job options defined are valid
           setting :job_options_contract, default: ActiveJob::JobOptionsContract.new
           # option consumer [Class] consumer class that should be used to consume ActiveJob data
-          setting :consumer, default: ActiveJob::Consumer
+          setting :consumer_class, default: ActiveJob::Consumer
         end
       end
 
+      # This will load all the defaults that can be later overwritten.
+      # Thanks to that we have an initial state out of the box.
+      configure
+
       class << self
         # Configuring method
         # @param block [Proc] block we want to execute with the config instance

data/lib/karafka/templates/example_consumer.rb.erb

@@ -7,10 +7,10 @@ class ExampleConsumer < ApplicationConsumer
   end
 
   # Run anything upon partition being revoked
-  # def on_revoked
+  # def revoked
   # end
 
   # Define here any teardown things you want when Karafka server stops
-  # def on_shutdown
+  # def shutdown
   # end
 end

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.0.beta4'
+  VERSION = '2.0.0.rc2'
 end

data/lib/karafka.rb

@@ -12,8 +12,6 @@
   openssl
   base64
   date
-  dry-configurable
-  dry-validation
   dry/events/publisher
   dry/monitor/notifications
   zeitwerk

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.0.beta4
+  version: 2.0.0.rc2
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -34,22 +34,8 @@ cert_chain:
   R2P11bWoCtr70BsccVrN8jEhzwXngMyI2gVt750Y+dbTu1KgRqZKp/ECe7ZzPzXj
   pIy9vHxTANKYVyI4qj8OrFdEM5BQNu8oQpL0iQ==
   -----END CERTIFICATE-----
-date: 2022-06-20 00:00:00.000000000 Z
+date: 2022-07-19 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: dry-configurable
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.13'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.13'
 - !ruby/object:Gem::Dependency
   name: dry-monitor
   requirement: !ruby/object:Gem::Requirement
@@ -64,20 +50,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.5'
-- !ruby/object:Gem::Dependency
-  name: dry-validation
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.7'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.7'
 - !ruby/object:Gem::Dependency
   name: rdkafka
   requirement: !ruby/object:Gem::Requirement
@@ -112,7 +84,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.3.1
+        version: 2.3.3
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -122,7 +94,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.3.1
+        version: 2.3.3
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -172,7 +144,8 @@ files:
 - bin/integrations
 - bin/karafka
 - bin/scenario
-- bin/stress
+- bin/stress_many
+- bin/stress_one
 - bin/wait_for_kafka
 - certs/karafka-pro.pem
 - certs/mensfeld.pem
@@ -211,6 +184,7 @@ files:
 - lib/karafka/env.rb
 - lib/karafka/errors.rb
 - lib/karafka/helpers/async.rb
+- lib/karafka/helpers/colorize.rb
 - lib/karafka/helpers/multi_delegator.rb
 - lib/karafka/instrumentation.rb
 - lib/karafka/instrumentation/callbacks/error.rb
@@ -236,11 +210,15 @@ files:
 - lib/karafka/pro/base_consumer.rb
 - lib/karafka/pro/loader.rb
 - lib/karafka/pro/performance_tracker.rb
+- lib/karafka/pro/processing/coordinator.rb
 - lib/karafka/pro/processing/jobs/consume_non_blocking.rb
 - lib/karafka/pro/processing/jobs_builder.rb
+- lib/karafka/pro/processing/partitioner.rb
+- lib/karafka/pro/processing/scheduler.rb
 - lib/karafka/pro/routing/extensions.rb
-- lib/karafka/pro/scheduler.rb
 - lib/karafka/process.rb
+- lib/karafka/processing/coordinator.rb
+- lib/karafka/processing/coordinators_buffer.rb
 - lib/karafka/processing/executor.rb
 - lib/karafka/processing/executors_buffer.rb
 - lib/karafka/processing/jobs/base.rb
@@ -249,7 +227,9 @@ files:
 - lib/karafka/processing/jobs/shutdown.rb
 - lib/karafka/processing/jobs_builder.rb
 - lib/karafka/processing/jobs_queue.rb
+- lib/karafka/processing/partitioner.rb
 - lib/karafka/processing/result.rb
+- lib/karafka/processing/scheduler.rb
 - lib/karafka/processing/worker.rb
 - lib/karafka/processing/workers_batch.rb
 - lib/karafka/railtie.rb
@@ -263,7 +243,6 @@ files:
 - lib/karafka/routing/topic.rb
 - lib/karafka/routing/topics.rb
 - lib/karafka/runner.rb
-- lib/karafka/scheduler.rb
 - lib/karafka/serialization/json/deserializer.rb
 - lib/karafka/server.rb
 - lib/karafka/setup/config.rb
@@ -302,5 +281,5 @@ requirements: []
 rubygems_version: 3.3.7
 signing_key: 
 specification_version: 4
-summary: Ruby based framework for working with Apache Kafka
+summary: Ruby framework for working with Apache Kafka
 test_files: []

data/lib/karafka/pro/scheduler.rb

@@ -1,54 +0,0 @@
-# frozen_string_literal: true
-
-# This Karafka component is a Pro component.
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
-
-module Karafka
-  module Pro
-    # Optimizes scheduler that takes into consideration of execution time needed to process
-    # messages from given topics partitions. It uses the non-preemptive LJF algorithm
-    #
-    # This scheduler is designed to optimize execution times on jobs that perform IO operations as
-    # when taking IO into consideration, the can achieve optimized parallel processing.
-    #
-    # This scheduler can also work with virtual partitions.
-    #
-    # Aside from consumption jobs, other jobs do not run often, thus we can leave them with
-    # default FIFO scheduler from the default Karafka scheduler
-    class Scheduler < ::Karafka::Scheduler
-      # Schedules jobs in the LJF order for consumption
-      #
-      # @param queue [Karafka::Processing::JobsQueue] queue where we want to put the jobs
-      # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
-      #
-      def schedule_consumption(queue, jobs_array)
-        pt = PerformanceTracker.instance
-
-        ordered = []
-
-        jobs_array.each do |job|
-          messages = job.messages
-          message = messages.first
-
-          cost = pt.processing_time_p95(message.topic, message.partition) * messages.size
-
-          ordered << [job, cost]
-        end
-
-        ordered.sort_by!(&:last)
-        ordered.reverse!
-        ordered.map!(&:first)
-
-        ordered.each do |job|
-          queue << job
-        end
-      end
-    end
-  end
-end

data/lib/karafka/scheduler.rb

@@ -1,20 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # FIFO scheduler for messages coming from various topics and partitions
-  class Scheduler
-    # Schedules jobs in the fifo order
-    #
-    # @param queue [Karafka::Processing::JobsQueue] queue where we want to put the jobs
-    # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
-    def schedule_consumption(queue, jobs_array)
-      jobs_array.each do |job|
-        queue << job
-      end
-    end
-
-    # Both revocation and shutdown jobs can also run in fifo by default
-    alias schedule_revocation schedule_consumption
-    alias schedule_shutdown schedule_consumption
-  end
-end