karafka 2.0.0.beta3 → 2.0.0.rc1

data/lib/karafka/connection/client.rb

@@ -36,6 +36,12 @@ module Karafka
         # Marks if we need to offset. If we did not store offsets, we should not commit the offset
         # position as it will crash rdkafka
         @offsetting = false
+        # We need to keep track of what we have paused for resuming
+        # In case we loose partition, we still need to resume it, otherwise it won't be fetched
+        # again if we get reassigned to it later on. We need to keep them as after revocation we
+        # no longer may be able to fetch them from Kafka. We could build them but it is easier
+        # to just keep them here and use if needed when cannot be obtained
+        @paused_tpls = Hash.new { |h, k| h[k] = {} }
       end
 
       # Fetches messages within boundaries defined by the settings (time, size, topics, etc).
@@ -45,12 +51,13 @@ module Karafka
       # @note This method should not be executed from many threads at the same time
       def batch_poll
         time_poll = TimeTrackers::Poll.new(@subscription_group.max_wait_time)
-        time_poll.start
 
         @buffer.clear
         @rebalance_manager.clear
 
         loop do
+          time_poll.start
+
           # Don't fetch more messages if we do not have any time left
           break if time_poll.exceeded?
           # Don't fetch more messages if we've fetched max as we've wanted
@@ -69,7 +76,11 @@ module Karafka
           # If partition revocation happens, we need to remove messages from revoked partitions
           # as well as ensure we do not have duplicated due to the offset reset for partitions
           # that we got assigned
-          remove_revoked_and_duplicated_messages if @rebalance_manager.revoked_partitions?
+          # We also do early break, so the information about rebalance is used as soon as possible
+          if @rebalance_manager.changed?
+            remove_revoked_and_duplicated_messages
+            break
+          end
 
           # Finally once we've (potentially) removed revoked, etc, if no messages were returned
           # we can break.
@@ -144,10 +155,14 @@ module Karafka
 
         internal_commit_offsets(async: false)
 
+        # Here we do not use our cached tpls because we should not try to pause something we do
+        # not own anymore.
         tpl = topic_partition_list(topic, partition)
 
         return unless tpl
 
+        @paused_tpls[topic][partition] = tpl
+
         @kafka.pause(tpl)
 
         @kafka.seek(pause_msg)
@@ -169,9 +184,13 @@ module Karafka
         # We can skip performance penalty since resuming should not happen too often
         internal_commit_offsets(async: false)
 
-        tpl = topic_partition_list(topic, partition)
+        # If we were not able, let's try to reuse the one we have (if we have)
+        tpl = topic_partition_list(topic, partition) || @paused_tpls[topic][partition]
 
         return unless tpl
+        # If we did not have it, it means we never paused this partition, thus no resume should
+        # happen in the first place
+        return unless @paused_tpls[topic].delete(partition)
 
         @kafka.resume(tpl)
       ensure
@@ -190,6 +209,7 @@ module Karafka
       # Marks given message as consumed.
       #
       # @param [Karafka::Messages::Message] message that we want to mark as processed
+      # @return [Boolean] true if successful. False if we no longer own given partition
       # @note This method won't trigger automatic offsets commits, rather relying on the offset
       #   check-pointing trigger that happens with each batch processed
       def mark_as_consumed(message)
@@ -199,8 +219,10 @@ module Karafka
       # Marks a given message as consumed and commits the offsets in a blocking way.
       #
       # @param [Karafka::Messages::Message] message that we want to mark as processed
+      # @return [Boolean] true if successful. False if we no longer own given partition
       def mark_as_consumed!(message)
-        mark_as_consumed(message)
+        return false unless mark_as_consumed(message)
+
         commit_offsets!
       end
 
@@ -211,28 +233,42 @@ module Karafka
         @mutex.synchronize do
           @closed = false
           @offsetting = false
+          @paused_tpls.clear
           @kafka = build_consumer
         end
       end
 
       private
 
+      # When we cannot store an offset, it means we no longer own the partition
+      #
       # Non thread-safe offset storing method
       # @param message [Karafka::Messages::Message]
+      # @return [Boolean] true if we could store the offset (if we still own the partition)
       def internal_store_offset(message)
         @offsetting = true
         @kafka.store_offset(message)
+        true
+      rescue Rdkafka::RdkafkaError => e
+        return false if e.code == :assignment_lost
+        return false if e.code == :state
+
+        raise e
       end
 
       # Non thread-safe message committing method
       # @param async [Boolean] should the commit happen async or sync (async by default)
+      # @return [Boolean] true if offset commit worked, false if we've lost the assignment
       def internal_commit_offsets(async: true)
-        return unless @offsetting
+        return true unless @offsetting
 
         @kafka.commit(nil, async)
         @offsetting = false
+
+        true
       rescue Rdkafka::RdkafkaError => e
-        return if e.code == :no_offset
+        return false if e.code == :assignment_lost
+        return false if e.code == :no_offset
 
         raise e
       end
@@ -250,7 +286,8 @@ module Karafka
 
           @kafka.close
           @buffer.clear
-          @rebalance_manager.clear
+          # @note We do not clear rebalance manager here as we may still have revocation info here
+          # that we want to consider valid prior to running another reconnection
         end
       end
 
@@ -279,30 +316,39 @@ module Karafka
 
         time_poll.start
 
-        @kafka.poll(time_poll.remaining)
+        @kafka.poll(timeout)
       rescue ::Rdkafka::RdkafkaError => e
-        raise if time_poll.attempts > MAX_POLL_RETRIES
-        raise unless time_poll.retryable?
-
+        # We return nil, so we do not restart until running the whole loop
+        # This allows us to run revocation jobs and other things and we will pick up new work
+        # next time after dispatching all the things that are needed
+        #
+        # If we would retry here, the client reset would become transparent and we would not have
+        # a chance to take any actions
         case e.code
         when :max_poll_exceeded # -147
           reset
+          return nil
         when :transport # -195
           reset
+          return nil
         when :rebalance_in_progress # -27
           reset
+          return nil
         when :not_coordinator # 16
           reset
+          return nil
         when :network_exception # 13
           reset
+          return nil
         end
 
-        time_poll.checkpoint
-
+        raise if time_poll.attempts > MAX_POLL_RETRIES
         raise unless time_poll.retryable?
 
+        time_poll.checkpoint
         time_poll.backoff
 
+        # On unknown errors we do our best to retry and handle them before raising
         retry
       end
 
@@ -346,7 +392,7 @@ module Karafka
       # we are no longer responsible in a given process for processing those messages and they
       # should have been picked up by a different process.
       def remove_revoked_and_duplicated_messages
-        @rebalance_manager.revoked_partitions.each do |topic, partitions|
+        @rebalance_manager.lost_partitions.each do |topic, partitions|
           partitions.each do |partition|
             @buffer.delete(topic, partition)
           end

data/lib/karafka/connection/listener.rb

@@ -18,15 +18,18 @@ module Karafka
       # @param jobs_queue [Karafka::Processing::JobsQueue] queue where we should push work
       # @return [Karafka::Connection::Listener] listener instance
       def initialize(subscription_group, jobs_queue)
+        proc_config = ::Karafka::App.config.internal.processing
+
         @id = SecureRandom.uuid
         @subscription_group = subscription_group
         @jobs_queue = jobs_queue
-        @jobs_builder = ::Karafka::App.config.internal.jobs_builder
-        @pauses_manager = PausesManager.new
+        @coordinators = Processing::CoordinatorsBuffer.new
         @client = Client.new(@subscription_group)
         @executors = Processing::ExecutorsBuffer.new(@client, subscription_group)
+        @jobs_builder = proc_config.jobs_builder
+        @partitioner = proc_config.partitioner_class.new(subscription_group)
         # We reference scheduler here as it is much faster than fetching this each time
-        @scheduler = ::Karafka::App.config.internal.scheduler
+        @scheduler = proc_config.scheduler
         # We keep one buffer for messages to preserve memory and not allocate extra objects
         # We can do this that way because we always first schedule jobs using messages before we
         # fetch another batch.
@@ -86,6 +89,9 @@ module Karafka
           build_and_schedule_revoke_lost_partitions_jobs
 
           # We wait only on jobs from our subscription group. Other groups are independent.
+          # This will block on revoked jobs until they are finished. Those are not meant to last
+          # long and should not have any bigger impact on the system. Doing this in a blocking way
+          # simplifies the overall design and prevents from race conditions
           wait
 
           build_and_schedule_consumption_jobs
@@ -136,7 +142,7 @@ module Karafka
 
       # Resumes processing of partitions that were paused due to an error.
       def resume_paused_partitions
-        @pauses_manager.resume do |topic, partition|
+        @coordinators.resume do |topic, partition|
           @client.resume(topic, partition)
         end
       end
@@ -152,9 +158,21 @@ module Karafka
 
         revoked_partitions.each do |topic, partitions|
           partitions.each do |partition|
-            pause_tracker = @pauses_manager.fetch(topic, partition)
-            executor = @executors.fetch(topic, partition, pause_tracker)
-            jobs << @jobs_builder.revoked(executor)
+            @coordinators.revoke(topic, partition)
+
+            # There may be a case where we have lost partition of which data we have never
+            # processed (if it was assigned and revoked really fast), thus we may not have it
+            # here. In cases like this, we do not run a revocation job
+            @executors.find_all(topic, partition).each do |executor|
+              jobs << @jobs_builder.revoked(executor)
+            end
+
+            # We need to remove all the executors of a given topic partition that we have lost, so
+            # next time we pick up it's work, new executors kick in. This may be needed especially
+            # for LRJ where we could end up with a race condition
+            # This revocation needs to happen after the jobs are scheduled, otherwise they would
+            # be scheduled with new executors instead of old
+            @executors.revoke(topic, partition)
           end
         end
 
@@ -191,11 +209,19 @@ module Karafka
         jobs = []
 
         @messages_buffer.each do |topic, partition, messages|
-          pause_tracker = @pauses_manager.fetch(topic, partition)
+          coordinator = @coordinators.find_or_create(topic, partition)
+
+          # Start work coordination for this topic partition
+          coordinator.start(messages)
 
-          executor = @executors.fetch(topic, partition, pause_tracker)
+          @partitioner.call(topic, messages) do |group_id, partition_messages|
+            # Count the job we're going to create here
+            coordinator.increment
 
-          jobs << @jobs_builder.consume(executor, messages)
+            executor = @executors.find_or_create(topic, partition, group_id)
+
+            jobs << @jobs_builder.consume(executor, partition_messages, coordinator)
+          end
         end
 
         @scheduler.schedule_consumption(@jobs_queue, jobs)
@@ -231,7 +257,7 @@ module Karafka
         @jobs_queue.wait(@subscription_group.id)
         @jobs_queue.clear(@subscription_group.id)
         @client.reset
-        @pauses_manager = PausesManager.new
+        @coordinators.reset
         @executors = Processing::ExecutorsBuffer.new(@client, @subscription_group)
       end
     end

data/lib/karafka/connection/rebalance_manager.rb

@@ -18,13 +18,15 @@ module Karafka
       # Empty array for internal usage not to create new objects
       EMPTY_ARRAY = [].freeze
 
+      attr_reader :assigned_partitions, :revoked_partitions
+
       private_constant :EMPTY_ARRAY
 
       # @return [RebalanceManager]
       def initialize
         @assigned_partitions = {}
         @revoked_partitions = {}
-        @lost_partitions = {}
+        @changed = false
       end
 
       # Resets the rebalance manager state
@@ -33,26 +35,12 @@ module Karafka
       def clear
         @assigned_partitions.clear
         @revoked_partitions.clear
-        @lost_partitions.clear
-      end
-
-      # @return [Hash<String, Array<Integer>>] hash where the keys are the names of topics for
-      #   which we've lost partitions and array with ids of the partitions as the value
-      # @note We do not consider as lost topics and partitions that got revoked and assigned
-      def revoked_partitions
-        return @revoked_partitions if @revoked_partitions.empty?
-        return @lost_partitions unless @lost_partitions.empty?
-
-        @revoked_partitions.each do |topic, partitions|
-          @lost_partitions[topic] = partitions - @assigned_partitions.fetch(topic, EMPTY_ARRAY)
-        end
-
-        @lost_partitions
+        @changed = false
       end
 
-      # @return [Boolean] true if any partitions were revoked
-      def revoked_partitions?
-        !revoked_partitions.empty?
+      # @return [Boolean] indicates a state change in the partitions assignment
+      def changed?
+        @changed
       end
 
       # Callback that kicks in inside of rdkafka, when new partitions are assigned.
@@ -62,6 +50,7 @@ module Karafka
       # @param partitions [Rdkafka::Consumer::TopicPartitionList]
       def on_partitions_assigned(_, partitions)
         @assigned_partitions = partitions.to_h.transform_values { |part| part.map(&:partition) }
+        @changed = true
       end
 
       # Callback that kicks in inside of rdkafka, when partitions are revoked.
@@ -71,6 +60,18 @@ module Karafka
       # @param partitions [Rdkafka::Consumer::TopicPartitionList]
       def on_partitions_revoked(_, partitions)
         @revoked_partitions = partitions.to_h.transform_values { |part| part.map(&:partition) }
+        @changed = true
+      end
+
+      # We consider as lost only partitions that were taken away and not re-assigned back to us
+      def lost_partitions
+        lost_partitions = {}
+
+        revoked_partitions.each do |topic, partitions|
+          lost_partitions[topic] = partitions - assigned_partitions.fetch(topic, EMPTY_ARRAY)
+        end
+
+        lost_partitions
       end
     end
   end

data/lib/karafka/contracts/config.rb

@@ -30,12 +30,26 @@ module Karafka
 
         # We validate internals just to be sure, that they are present and working
         required(:internal).schema do
-          required(:routing_builder)
-          required(:subscription_groups_builder)
-          required(:jobs_builder)
           required(:status)
           required(:process)
-          required(:scheduler)
+
+          required(:routing).schema do
+            required(:builder)
+            required(:subscription_groups_builder)
+          end
+
+          required(:processing).schema do
+            required(:jobs_builder)
+            required(:scheduler)
+            required(:coordinator_class)
+            required(:partitioner_class)
+          end
+
+          required(:active_job).schema do
+            required(:dispatcher)
+            required(:job_options_contract)
+            required(:consumer_class)
+          end
         end
       end
 

data/lib/karafka/contracts/server_cli_options.rb

@@ -12,7 +12,7 @@ module Karafka
         # If there were no consumer_groups declared in the server cli, it means that we will
         # run all of them and no need to validate them here at all
         if !value.nil? &&
-           !(value - Karafka::App.config.internal.routing_builder.map(&:name)).empty?
+           !(value - Karafka::App.config.internal.routing.builder.map(&:name)).empty?
           key(:consumer_groups).failure(:consumer_groups_inclusion)
         end
       end

data/lib/karafka/errors.rb

@@ -47,5 +47,8 @@ module Karafka
     # Used to instrument this error into the error notifications
     # We do not raise it so we won't crash deployed systems
     ExpiredLicenseTokenError = Class.new(BaseError)
+
+    # This should never happen. Please open an issue if it does.
+    InvalidCoordinatorState = Class.new(BaseError)
   end
 end

data/lib/karafka/instrumentation/logger_listener.rb

@@ -98,9 +98,6 @@ module Karafka
         details = (error.backtrace || []).join("\n")
 
         case type
-        when 'consumer.prepared.error'
-          error "Consumer prepared error: #{error}"
-          error details
         when 'consumer.consume.error'
           error "Consumer consuming error: #{error}"
           error details

data/lib/karafka/instrumentation/monitor.rb

@@ -22,7 +22,6 @@ module Karafka
         app.stopping
         app.stopped
 
-        consumer.prepared
         consumer.consumed
         consumer.revoked
         consumer.shutdown

data/lib/karafka/pro/active_job/consumer.rb

@@ -20,26 +20,20 @@ module Karafka
       #
       # It contains slightly better revocation warranties than the regular blocking consumer as
       # it can stop processing batch of jobs in the middle after the revocation.
-      class Consumer < Karafka::ActiveJob::Consumer
+      class Consumer < Karafka::Pro::BaseConsumer
         # Runs ActiveJob jobs processing and handles lrj if needed
         def consume
           messages.each do |message|
             # If for any reason we've lost this partition, not worth iterating over new messages
             # as they are no longer ours
-            return if revoked?
+            break if revoked?
             break if Karafka::App.stopping?
 
             ::ActiveJob::Base.execute(
               ::ActiveSupport::JSON.decode(message.raw_payload)
             )
 
-            # We check it twice as the job may be long running
-            return if revoked?
-
             mark_as_consumed(message)
-
-            # Do not process more if we are shutting down
-            break if Karafka::App.stopping?
           end
         end
       end

data/lib/karafka/pro/base_consumer.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Karafka PRO consumer.
+    #
+    # If you use PRO, all your consumers should inherit (indirectly) from it.
+    #
+    # @note In case of using lrj, manual pausing may not be the best idea as resume needs to happen
+    #   after each batch is processed.
+    class BaseConsumer < Karafka::BaseConsumer
+      # Pause for tops 31 years
+      MAX_PAUSE_TIME = 1_000_000_000_000
+
+      private_constant :MAX_PAUSE_TIME
+
+      # Pauses processing of a given partition until we're done with the processing
+      # This ensures, that we can easily poll not reaching the `max.poll.interval`
+      def on_before_consume
+        return unless topic.long_running_job?
+
+        # This ensures, that when running LRJ with VP, things operate as expected
+        coordinator.on_started do |first_group_message|
+          # Pause at the first message in a batch. That way in case of a crash, we will not loose
+          # any messages
+          pause(first_group_message.offset, MAX_PAUSE_TIME)
+        end
+      end
+
+      # Runs extra logic after consumption that is related to handling long running jobs
+      # @note This overwrites the '#on_after_consume' from the base consumer
+      def on_after_consume
+        coordinator.on_finished do |first_group_message, last_group_message|
+          on_after_consume_regular(first_group_message, last_group_message)
+        end
+      end
+
+      private
+
+      # Handles the post-consumption flow depending on topic settings
+      #
+      # @param first_message [Karafka::Messages::Message]
+      # @param last_message [Karafka::Messages::Message]
+      def on_after_consume_regular(first_message, last_message)
+        if coordinator.success?
+          coordinator.pause_tracker.reset
+
+          # We use the non-blocking one here. If someone needs the blocking one, can implement it
+          # with manual offset management
+          # Mark as consumed only if manual offset management is not on
+          mark_as_consumed(last_message) unless topic.manual_offset_management? || revoked?
+
+          # If this is not a long running job there is nothing for us to do here
+          return unless topic.long_running_job?
+
+          # Once processing is done, we move to the new offset based on commits
+          # Here, in case manual offset management is off, we have the new proper offset of a
+          # first message from another batch from `@seek_offset`. If manual offset management
+          # is on, we move to place where the user indicated it was finished. This can create an
+          # interesting (yet valid) corner case, where with manual offset management on and no
+          # marking as consumed, we end up with an infinite loop processing same messages over and
+          # over again
+          seek(@seek_offset || first_message.offset)
+
+          resume
+        else
+          # If processing failed, we need to pause
+          pause(@seek_offset || first_message.offset)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/loader.rb

@@ -15,11 +15,13 @@ module Karafka
     class Loader
       # All the pro components that need to be loaded
       COMPONENTS = %w[
+        base_consumer
         performance_tracker
-        scheduler
-        base_consumer_extensions
+        processing/scheduler
         processing/jobs/consume_non_blocking
         processing/jobs_builder
+        processing/coordinator
+        processing/partitioner
         routing/extensions
         active_job/consumer
         active_job/dispatcher
@@ -35,14 +37,18 @@ module Karafka
         def setup(config)
           COMPONENTS.each { |component| require_relative(component) }
 
-          config.internal.scheduler = Scheduler.new
-          config.internal.jobs_builder = Processing::JobsBuilder.new
-          config.internal.active_job.consumer = ActiveJob::Consumer
-          config.internal.active_job.dispatcher = ActiveJob::Dispatcher.new
-          config.internal.active_job.job_options_contract = ActiveJob::JobOptionsContract.new
+          icfg = config.internal
+
+          icfg.processing.coordinator_class = Processing::Coordinator
+          icfg.processing.partitioner_class = Processing::Partitioner
+          icfg.processing.scheduler = Processing::Scheduler.new
+          icfg.processing.jobs_builder = Processing::JobsBuilder.new
+
+          icfg.active_job.consumer_class = ActiveJob::Consumer
+          icfg.active_job.dispatcher = ActiveJob::Dispatcher.new
+          icfg.active_job.job_options_contract = ActiveJob::JobOptionsContract.new
 
           ::Karafka::Routing::Topic.include(Routing::Extensions)
-          ::Karafka::BaseConsumer.prepend(BaseConsumerExtensions)
 
           config.monitor.subscribe(PerformanceTracker.instance)
         end

data/lib/karafka/pro/processing/coordinator.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Pro
+    module Processing
+      # Pro coordinator that provides extra orchestration methods useful for parallel processing
+      # within the same partition
+      class Coordinator < ::Karafka::Processing::Coordinator
+        # @param args [Object] anything the base coordinator accepts
+        def initialize(*args)
+          super
+          @on_started_invoked = false
+          @on_finished_invoked = false
+          @flow_lock = Mutex.new
+        end
+
+        # Starts the coordination process
+        # @param messages [Array<Karafka::Messages::Message>] messages for which processing we are
+        #   going to coordinate.
+        def start(messages)
+          super
+
+          @mutex.synchronize do
+            @on_started_invoked = false
+            @on_finished_invoked = false
+            @first_message = messages.first
+            @last_message = messages.last
+          end
+        end
+
+        # @return [Boolean] is the coordinated work finished or not
+        def finished?
+          @running_jobs.zero?
+        end
+
+        # Runs given code only once per all the coordinated jobs upon starting first of them
+        def on_started
+          @flow_lock.synchronize do
+            return if @on_started_invoked
+
+            @on_started_invoked = true
+
+            yield(@first_message, @last_message)
+          end
+        end
+
+        # Runs once when all the work that is suppose to be coordinated is finished
+        # It runs once per all the coordinated jobs and should be used to run any type of post
+        # jobs coordination processing execution
+        def on_finished
+          @flow_lock.synchronize do
+            return unless finished?
+            return if @on_finished_invoked
+
+            @on_finished_invoked = true
+
+            yield(@first_message, @last_message)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb

@@ -26,7 +26,7 @@ module Karafka
         #   management. This layer of the framework knows nothing about Kafka messages consumption.
         class ConsumeNonBlocking < ::Karafka::Processing::Jobs::Consume
           # Releases the blocking lock after it is done with the preparation phase for this job
-          def prepare
+          def before_call
             super
             @non_blocking = true
           end