karafka 2.0.0.beta4 → 2.0.0.rc2

data/lib/karafka/pro/active_job/consumer.rb

@@ -26,7 +26,7 @@ module Karafka
           messages.each do |message|
             # If for any reason we've lost this partition, not worth iterating over new messages
             # as they are no longer ours
-            return if revoked?
+            break if revoked?
             break if Karafka::App.stopping?
 
             ::ActiveJob::Base.execute(
@@ -34,13 +34,6 @@ module Karafka
             )
 
             mark_as_consumed(message)
-
-            # We check it twice as the job may be long running
-            # If marking fails, it also means it got revoked and we can stop consuming
-            return if revoked?
-
-            # Do not process more if we are shutting down
-            break if Karafka::App.stopping?
           end
         end
       end

data/lib/karafka/pro/active_job/job_options_contract.rb

@@ -14,13 +14,17 @@ module Karafka
     module ActiveJob
       # Contract for validating the options that can be altered with `#karafka_options` per job
       # class that works with Pro features.
-      class JobOptionsContract < ::Karafka::ActiveJob::JobOptionsContract
-        # Dry types
-        Types = include Dry.Types()
-
-        params do
-          optional(:partitioner).value(Types.Interface(:call))
+      class JobOptionsContract < Contracts::Base
+        configure do |config|
+          config.error_messages = YAML.safe_load(
+            File.read(
+              File.join(Karafka.gem_root, 'config', 'errors.yml')
+            )
+          ).fetch('en').fetch('validations').fetch('job_options')
         end
+
+        optional(:dispatch_method) { |val| %i[produce_async produce_sync].include?(val) }
+        optional(:partitioner) { |val| val.respond_to?(:call) }
       end
     end
   end

data/lib/karafka/pro/base_consumer.rb

@@ -26,26 +26,38 @@ module Karafka
       # Pauses processing of a given partition until we're done with the processing
       # This ensures, that we can easily poll not reaching the `max.poll.interval`
       def on_before_consume
-        # Pause at the first message in a batch. That way in case of a crash, we will not loose
-        # any messages
         return unless topic.long_running_job?
 
-        pause(messages.first.offset, MAX_PAUSE_TIME)
+        # This ensures, that when running LRJ with VP, things operate as expected
+        coordinator.on_started do |first_group_message|
+          # Pause at the first message in a batch. That way in case of a crash, we will not loose
+          # any messages
+          pause(first_group_message.offset, MAX_PAUSE_TIME)
+        end
       end
 
       # Runs extra logic after consumption that is related to handling long running jobs
       # @note This overwrites the '#on_after_consume' from the base consumer
       def on_after_consume
-        # Nothing to do if we lost the partition
-        return if revoked?
+        coordinator.on_finished do |first_group_message, last_group_message|
+          on_after_consume_regular(first_group_message, last_group_message)
+        end
+      end
+
+      private
 
-        if @consumption.success?
-          pause_tracker.reset
+      # Handles the post-consumption flow depending on topic settings
+      #
+      # @param first_message [Karafka::Messages::Message]
+      # @param last_message [Karafka::Messages::Message]
+      def on_after_consume_regular(first_message, last_message)
+        if coordinator.success?
+          coordinator.pause_tracker.reset
 
           # We use the non-blocking one here. If someone needs the blocking one, can implement it
           # with manual offset management
           # Mark as consumed only if manual offset management is not on
-          mark_as_consumed(messages.last) unless topic.manual_offset_management?
+          mark_as_consumed(last_message) unless topic.manual_offset_management? || revoked?
 
           # If this is not a long running job there is nothing for us to do here
           return unless topic.long_running_job?
@@ -53,24 +65,18 @@ module Karafka
           # Once processing is done, we move to the new offset based on commits
           # Here, in case manual offset management is off, we have the new proper offset of a
           # first message from another batch from `@seek_offset`. If manual offset management
-          # is on, we move to place where the user indicated it was finished.
-          seek(@seek_offset || messages.first.offset)
+          # is on, we move to place where the user indicated it was finished. This can create an
+          # interesting (yet valid) corner case, where with manual offset management on and no
+          # marking as consumed, we end up with an infinite loop processing same messages over and
+          # over again
+          seek(@seek_offset || first_message.offset)
+
           resume
         else
           # If processing failed, we need to pause
-          pause(@seek_offset || messages.first.offset)
+          pause(@seek_offset || first_message.offset)
         end
       end
-
-      # Marks this consumer revoked state as true
-      # This allows us for things like lrj to finish early as this state may change during lrj
-      # execution
-      def on_revoked
-        # @note This may already be set to true if we tried to commit offsets and failed. In case
-        # like this it will automatically be marked as revoked.
-        @revoked = true
-        super
-      end
     end
   end
 end

data/lib/karafka/pro/loader.rb

@@ -17,9 +17,11 @@ module Karafka
       COMPONENTS = %w[
         base_consumer
         performance_tracker
-        scheduler
+        processing/scheduler
         processing/jobs/consume_non_blocking
         processing/jobs_builder
+        processing/coordinator
+        processing/partitioner
         routing/extensions
         active_job/consumer
         active_job/dispatcher
@@ -35,11 +37,16 @@ module Karafka
         def setup(config)
           COMPONENTS.each { |component| require_relative(component) }
 
-          config.internal.scheduler = Scheduler.new
-          config.internal.jobs_builder = Processing::JobsBuilder.new
-          config.internal.active_job.consumer = ActiveJob::Consumer
-          config.internal.active_job.dispatcher = ActiveJob::Dispatcher.new
-          config.internal.active_job.job_options_contract = ActiveJob::JobOptionsContract.new
+          icfg = config.internal
+
+          icfg.processing.coordinator_class = Processing::Coordinator
+          icfg.processing.partitioner_class = Processing::Partitioner
+          icfg.processing.scheduler = Processing::Scheduler.new
+          icfg.processing.jobs_builder = Processing::JobsBuilder.new
+
+          icfg.active_job.consumer_class = ActiveJob::Consumer
+          icfg.active_job.dispatcher = ActiveJob::Dispatcher.new
+          icfg.active_job.job_options_contract = ActiveJob::JobOptionsContract.new
 
           ::Karafka::Routing::Topic.include(Routing::Extensions)
 

data/lib/karafka/pro/processing/coordinator.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Pro
+    module Processing
+      # Pro coordinator that provides extra orchestration methods useful for parallel processing
+      # within the same partition
+      class Coordinator < ::Karafka::Processing::Coordinator
+        # @param args [Object] anything the base coordinator accepts
+        def initialize(*args)
+          super
+          @on_started_invoked = false
+          @on_finished_invoked = false
+          @flow_lock = Mutex.new
+        end
+
+        # Starts the coordination process
+        # @param messages [Array<Karafka::Messages::Message>] messages for which processing we are
+        #   going to coordinate.
+        def start(messages)
+          super
+
+          @mutex.synchronize do
+            @on_started_invoked = false
+            @on_finished_invoked = false
+            @first_message = messages.first
+            @last_message = messages.last
+          end
+        end
+
+        # @return [Boolean] is the coordinated work finished or not
+        def finished?
+          @running_jobs.zero?
+        end
+
+        # Runs given code only once per all the coordinated jobs upon starting first of them
+        def on_started
+          @flow_lock.synchronize do
+            return if @on_started_invoked
+
+            @on_started_invoked = true
+
+            yield(@first_message, @last_message)
+          end
+        end
+
+        # Runs once when all the work that is suppose to be coordinated is finished
+        # It runs once per all the coordinated jobs and should be used to run any type of post
+        # jobs coordination processing execution
+        def on_finished
+          @flow_lock.synchronize do
+            return unless finished?
+            return if @on_finished_invoked
+
+            @on_finished_invoked = true
+
+            yield(@first_message, @last_message)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/jobs_builder.rb

@@ -16,11 +16,12 @@ module Karafka
       class JobsBuilder < ::Karafka::Processing::JobsBuilder
         # @param executor [Karafka::Processing::Executor]
         # @param messages [Karafka::Messages::Messages] messages batch to be consumed
+        # @param coordinator [Karafka::Processing::Coordinator]
         # @return [Karafka::Processing::Jobs::Consume] blocking job
         # @return [Karafka::Pro::Processing::Jobs::ConsumeNonBlocking] non blocking for lrj
-        def consume(executor, messages)
+        def consume(executor, messages, coordinator)
           if executor.topic.long_running_job?
-            Jobs::ConsumeNonBlocking.new(executor, messages)
+            Jobs::ConsumeNonBlocking.new(executor, messages, coordinator)
           else
             super
           end

data/lib/karafka/pro/processing/partitioner.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Pro partitioner that can distribute work based on the virtual partitioner settings
+      class Partitioner < ::Karafka::Processing::Partitioner
+        # @param topic [String] topic name
+        # @param messages [Array<Karafka::Messages::Message>] karafka messages
+        # @yieldparam [Integer] group id
+        # @yieldparam [Array<Karafka::Messages::Message>] karafka messages
+        def call(topic, messages)
+          ktopic = @subscription_group.topics.find(topic)
+
+          @concurrency ||= ::Karafka::App.config.concurrency
+
+          # We only partition work if we have a virtual partitioner and more than one thread to
+          # process the data. With one thread it is not worth partitioning the work as the work
+          # itself will be assigned to one thread (pointless work)
+          if ktopic.virtual_partitioner? && @concurrency > 1
+            messages
+              .group_by { |msg| ktopic.virtual_partitioner.call(msg).hash.abs % @concurrency }
+              .each { |group_id, messages_group| yield(group_id, messages_group) }
+          else
+            # When no virtual partitioner, works as regular one
+            yield(0, messages)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/scheduler.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Optimizes scheduler that takes into consideration of execution time needed to process
+      # messages from given topics partitions. It uses the non-preemptive LJF algorithm
+      #
+      # This scheduler is designed to optimize execution times on jobs that perform IO operations
+      # as when taking IO into consideration, the can achieve optimized parallel processing.
+      #
+      # This scheduler can also work with virtual partitions.
+      #
+      # Aside from consumption jobs, other jobs do not run often, thus we can leave them with
+      # default FIFO scheduler from the default Karafka scheduler
+      class Scheduler < ::Karafka::Processing::Scheduler
+        # Schedules jobs in the LJF order for consumption
+        #
+        # @param queue [Karafka::Processing::JobsQueue] queue where we want to put the jobs
+        # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
+        #
+        def schedule_consumption(queue, jobs_array)
+          pt = PerformanceTracker.instance
+
+          ordered = []
+
+          jobs_array.each do |job|
+            messages = job.messages
+            message = messages.first
+
+            cost = pt.processing_time_p95(message.topic, message.partition) * messages.size
+
+            ordered << [job, cost]
+          end
+
+          ordered.sort_by!(&:last)
+          ordered.reverse!
+          ordered.map!(&:first)
+
+          ordered.each do |job|
+            queue << job
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/extensions.rb

@@ -19,9 +19,15 @@ module Karafka
           # @param base [Class] class we extend
           def included(base)
             base.attr_accessor :long_running_job
+            base.attr_accessor :virtual_partitioner
           end
         end
 
+        # @return [Boolean] true if virtual partitioner is defined, false otherwise
+        def virtual_partitioner?
+          virtual_partitioner != nil
+        end
+
         # @return [Boolean] is a given job on a topic a long running one
         def long_running_job?
           @long_running_job || false

data/lib/karafka/processing/coordinator.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Basic coordinator that allows us to provide coordination objects into consumers.
+    #
+    # This is a wrapping layer to simplify management of work to be handled around consumption.
+    #
+    # @note This coordinator needs to be thread safe. Some operations are performed only in the
+    #   listener thread, but we go with thread-safe by default for all not to worry about potential
+    #   future mistakes.
+    class Coordinator
+      # @return [Karafka::TimeTrackers::Pause]
+      attr_reader :pause_tracker
+
+      # @param pause_tracker [Karafka::TimeTrackers::Pause] pause tracker for given topic partition
+      def initialize(pause_tracker)
+        @pause_tracker = pause_tracker
+        @revoked = false
+        @consumptions = {}
+        @running_jobs = 0
+        @mutex = Mutex.new
+      end
+
+      # Starts the coordinator for given consumption jobs
+      # @param _messages [Array<Karafka::Messages::Message>] batch of message for which we are
+      #   going to coordinate work. Not used with regular coordinator.
+      def start(_messages)
+        @mutex.synchronize do
+          @running_jobs = 0
+          # We need to clear the consumption results hash here, otherwise we could end up storing
+          # consumption results of consumer instances we no longer control
+          @consumptions.clear
+        end
+      end
+
+      # Increases number of jobs that we handle with this coordinator
+      def increment
+        @mutex.synchronize { @running_jobs += 1 }
+      end
+
+      # Decrements number of jobs we handle at the moment
+      def decrement
+        @mutex.synchronize do
+          @running_jobs -= 1
+
+          return @running_jobs unless @running_jobs.negative?
+
+          # This should never happen. If it does, something is heavily out of sync. Please reach
+          # out to us if you encounter this
+          raise Karafka::Errors::InvalidCoordinatorState, 'Was zero before decrementation'
+        end
+      end
+
+      # @param consumer [Object] karafka consumer (normal or pro)
+      # @return [Karafka::Processing::Result] result object which we can use to indicate
+      #   consumption processing state.
+      def consumption(consumer)
+        @mutex.synchronize do
+          @consumptions[consumer] ||= Processing::Result.new
+        end
+      end
+
+      # Is all the consumption done and finished successfully for this coordinator
+      def success?
+        @mutex.synchronize { @running_jobs.zero? && @consumptions.values.all?(&:success?) }
+      end
+
+      # Marks given coordinator for processing group as revoked
+      #
+      # This is invoked in two places:
+      #   - from the main listener loop when we detect revoked partitions
+      #   - from the consumer in case checkpointing fails
+      #
+      # This means, we can end up having consumer being aware that it was revoked prior to the
+      # listener loop dispatching the revocation job. It is ok, as effectively nothing will be
+      # processed until revocation jobs are done.
+      def revoke
+        @mutex.synchronize { @revoked = true }
+      end
+
+      # @return [Boolean] is the partition we are processing revoked or not
+      def revoked?
+        @revoked
+      end
+    end
+  end
+end

data/lib/karafka/processing/coordinators_buffer.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Coordinators builder used to build coordinators per topic partition
+    #
+    # It provides direct pauses access for revocation
+    #
+    # @note This buffer operates only from the listener loop, thus we do not have to make it
+    #   thread-safe.
+    class CoordinatorsBuffer
+      def initialize
+        @pauses_manager = Connection::PausesManager.new
+        @coordinator_class = ::Karafka::App.config.internal.processing.coordinator_class
+        @coordinators = Hash.new { |h, k| h[k] = {} }
+      end
+
+      # @param topic [String] topic name
+      # @param partition [Integer] partition number
+      def find_or_create(topic, partition)
+        @coordinators[topic][partition] ||= @coordinator_class.new(
+          @pauses_manager.fetch(topic, partition)
+        )
+      end
+
+      # Resumes processing of partitions for which pause time has ended.
+      # @param block we want to run for resumed topic partitions
+      # @yieldparam [String] topic name
+      # @yieldparam [Integer] partition number
+      def resume(&block)
+        @pauses_manager.resume(&block)
+      end
+
+      # @param topic [String] topic name
+      # @param partition [Integer] partition number
+      def revoke(topic, partition)
+        return unless @coordinators[topic].key?(partition)
+
+        # The fact that we delete here does not change the fact that the executor still holds the
+        # reference to this coordinator. We delete it here, as we will no longer process any
+        # new stuff with it and we may need a new coordinator if we regain this partition, but the
+        # coordinator may still be in use
+        @coordinators[topic].delete(partition).revoke
+      end
+
+      # Clears coordinators and re-created the pauses manager
+      # This should be used only for critical errors recovery
+      def reset
+        @pauses_manager = Connection::PausesManager.new
+        @coordinators.clear
+      end
+    end
+  end
+end

data/lib/karafka/processing/executor.rb

@@ -30,13 +30,11 @@ module Karafka
       # @param group_id [String] id of the subscription group to which the executor belongs
       # @param client [Karafka::Connection::Client] kafka client
       # @param topic [Karafka::Routing::Topic] topic for which this executor will run
-      # @param pause_tracker [Karafka::TimeTrackers::Pause] fetch pause tracker for pausing
-      def initialize(group_id, client, topic, pause_tracker)
+      def initialize(group_id, client, topic)
         @id = SecureRandom.uuid
         @group_id = group_id
         @client = client
         @topic = topic
-        @pause_tracker = pause_tracker
       end
 
       # Builds the consumer instance, builds messages batch and sets all that is needed to run the
@@ -45,20 +43,15 @@ module Karafka
       # @param messages [Array<Karafka::Messages::Message>]
       # @param received_at [Time] the moment we've received the batch (actually the moment we've)
       #   enqueued it, but good enough
-      def before_consume(messages, received_at)
+      # @param coordinator [Karafka::Processing::Coordinator] coordinator for processing management
+      def before_consume(messages, received_at, coordinator)
         # Recreate consumer with each batch if persistence is not enabled
         # We reload the consumers with each batch instead of relying on some external signals
         # when needed for consistency. That way devs may have it on or off and not in this
         # middle state, where re-creation of a consumer instance would occur only sometimes
-        @recreate = true unless ::Karafka::App.config.consumer_persistence
+        @consumer = nil unless ::Karafka::App.config.consumer_persistence
 
-        # If @recreate was set to true (aside from non persistent), it means, that revocation or
-        # a shutdown happened and we need to have a new instance for running another consume for
-        # this topic partition
-        if @recreate
-          @consumer = nil
-          @recreate = false
-        end
+        consumer.coordinator = coordinator
 
         # First we build messages batch...
         consumer.messages = Messages::Builders::Messages.call(
@@ -78,7 +71,7 @@ module Karafka
 
       # Runs consumer after consumption code
       def after_consume
-        consumer.on_after_consume if @consumer
+        consumer.on_after_consume
       end
 
       # Runs the controller `#revoked` method that should be triggered when a given consumer is
@@ -95,7 +88,6 @@ module Karafka
       #   consumer instance.
       def revoked
         consumer.on_revoked if @consumer
-        @recreate = true
       end
 
       # Runs the controller `#shutdown` method that should be triggered when a given consumer is
@@ -107,7 +99,6 @@ module Karafka
         # There is a case, where the consumer no longer exists because it was revoked, in case like
         # that we do not build a new instance and shutdown should not be triggered.
         consumer.on_shutdown if @consumer
-        @recreate = true
       end
 
       private
@@ -115,10 +106,9 @@ module Karafka
       # @return [Object] cached consumer instance
       def consumer
         @consumer ||= begin
-          consumer = @topic.consumer.new
+          consumer = @topic.consumer_class.new
           consumer.topic = @topic
           consumer.client = @client
-          consumer.pause_tracker = @pause_tracker
           consumer.producer = ::Karafka::App.producer
           consumer
         end

data/lib/karafka/processing/executors_buffer.rb

@@ -11,30 +11,48 @@ module Karafka
       def initialize(client, subscription_group)
         @subscription_group = subscription_group
         @client = client
-        @buffer = Hash.new { |h, k| h[k] = {} }
+        # We need two layers here to keep track of topics, partitions and processing groups
+        @buffer = Hash.new { |h, k| h[k] = Hash.new { |h2, k2| h2[k2] = {} } }
       end
 
+      # Finds or creates an executor based on the provided details
+      #
       # @param topic [String] topic name
       # @param partition [Integer] partition number
-      # @param pause [TimeTrackers::Pause] pause corresponding with provided topic and partition
+      # @param parallel_key [String] parallel group key
       # @return [Executor] consumer executor
-      def fetch(
-        topic,
-        partition,
-        pause
-      )
-        ktopic = @subscription_group.topics.find(topic)
+      def find_or_create(topic, partition, parallel_key)
+        ktopic = find_topic(topic)
 
-        ktopic || raise(Errors::TopicNotFoundError, topic)
-
-        @buffer[ktopic][partition] ||= Executor.new(
+        @buffer[ktopic][partition][parallel_key] ||= Executor.new(
           @subscription_group.id,
           @client,
-          ktopic,
-          pause
+          ktopic
         )
       end
 
+      # Revokes executors of a given topic partition, so they won't be used anymore for incoming
+      # messages
+      #
+      # @param topic [String] topic name
+      # @param partition [Integer] partition number
+      def revoke(topic, partition)
+        ktopic = find_topic(topic)
+
+        @buffer[ktopic][partition].clear
+      end
+
+      # Finds all the executors available for a given topic partition
+      #
+      # @param topic [String] topic name
+      # @param partition [Integer] partition number
+      # @return [Array<Executor>] executors in use for this topic + partition
+      def find_all(topic, partition)
+        ktopic = find_topic(topic)
+
+        @buffer[ktopic][partition].values
+      end
+
       # Iterates over all available executors and yields them together with topic and partition
       # info
       # @yieldparam [Routing::Topic] karafka routing topic object
@@ -42,8 +60,11 @@ module Karafka
       # @yieldparam [Executor] given executor
       def each
         @buffer.each do |ktopic, partitions|
-          partitions.each do |partition, executor|
-            yield(ktopic, partition, executor)
+          partitions.each do |partition, executors|
+            executors.each do |_parallel_key, executor|
+              # We skip the parallel key here as it does not serve any value when iterating
+              yield(ktopic, partition, executor)
+            end
           end
         end
       end
@@ -52,6 +73,16 @@ module Karafka
       def clear
         @buffer.clear
       end
+
+      private
+
+      # Finds topic based on its name
+      #
+      # @param topic [String] topic we're looking for
+      # @return [Karafka::Routing::Topic] topic we're interested in
+      def find_topic(topic)
+        @subscription_group.topics.find(topic) || raise(Errors::TopicNotFoundError, topic)
+      end
     end
   end
 end