karafka 2.0.0.beta4 → 2.0.0.beta5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e4e9430d2278617cbed38f5696011603d9c0d8c53813dfc180499dc6e4b97563
-  data.tar.gz: f082a95aa9841912f819dc0598591c4b96d7ef1199eff324e65ca0c601008dae
+  metadata.gz: 2c8e680ffdf69f88899a715c84cc484e8f568f4a93da9284195f4bf55a283ee1
+  data.tar.gz: 974356226a10ba2c77de770351a47180716533021a89040bcdc1aae57f452121
 SHA512:
-  metadata.gz: 7252c5503234ab4d35fa02d2bb0a18dd8239584fdddc5b451cfdf028a61f37d59a269bac804913d0abf46e2d3273188560e48aa9de40fbb319c766624c1a3b95
-  data.tar.gz: a4cc5d7c18d2a45483ee26acbacf62c9c13f8824697af96a3f2bf5bccb232d5b07097ed49cfb84a9b46e09f31405813d50b1564d6668f0a483023f449427428b
+  metadata.gz: 2427aaae1b1b07430df7c9f042d290bbae8380fb1f6ec7c26eecee92b8fe79e13ea9f3a99a36bf89b314ffba809c556618b22c0a87f0c0c83bb73cf8af72321b
+  data.tar.gz: 55e18448b5645acd38c4194967ea7df657c142d82a105699f7b204f222f8dfb2dbd14cce82b1f424ec177afb78049b3e7588642013674a3c2923a8848b6b87e7

data/.github/workflows/ci.yml

@@ -8,6 +8,10 @@ on:
   schedule:
     - cron:  '0 1 * * *'
 
+env:
+  BUNDLE_RETRY: 6
+  BUNDLE_JOBS: 4
+
 jobs:
   diffend:
     runs-on: ubuntu-latest
@@ -17,13 +21,16 @@ jobs:
       - uses: actions/checkout@v2
         with:
           fetch-depth: 0
+
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
           ruby-version: 3.1
           bundler-cache: true
+
       - name: Install Diffend plugin
         run: bundle plugin install diffend
+
       - name: Bundle Secure
         run: bundle secure
 
@@ -101,7 +108,17 @@ jobs:
         uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{matrix.ruby}}
-          bundler-cache: true
+
+      - name: Install latest Bundler
+        run: |
+          gem install bundler --no-document
+          gem update --system --no-document
+          bundle config set without 'tools benchmarks docs'
+
+      - name: Bundle install
+        run: |
+          bundle config set without development
+          bundle install
 
       - name: Ensure all needed Kafka topics are created and wait if not
         run: |

data/CHANGELOG.md

@@ -1,5 +1,20 @@
 # Karafka framework changelog
 
+## 2.0.0-beta5 (2022-07-05)
+- Always resume processing of a revoked partition upon assignment.
+- Improve specs stability.
+- Fix a case where revocation job would be executed on partition for which we never did any work.
+- Introduce a jobs group coordinator for easier jobs management.
+- Improve stability of resuming paused partitions that were revoked and re-assigned.
+- Optimize reaction time on partition ownership changes.
+- Fix a bug where despite setting long max wait time, we would return messages prior to it while not reaching the desired max messages count.
+- Add more integration specs related to polling limits.
+- Remove auto-detection of re-assigned partitions upon rebalance as for too fast rebalances it could not be accurate enough. It would also mess up in case of rebalances that would happen right after a `#seek` was issued for a partition.
+- Optimize the removal of pre-buffered lost partitions data.
+- Always rune `#revoked` when rebalance with revocation happens.
+- Evict executors upon rebalance, to prevent race-conditions.
+- Align topics names for integration specs.
+
 ## 2.0.0-beta4 (2022-06-20)
 - Rename job internal api methods from `#prepare` to `#before_call` and from `#teardown` to `#after_call` to abstract away jobs execution from any type of executors and consumers logic
 - Remove ability of running `before_consume` and `after_consume` completely. Those should be for internal usage only.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.0.beta4)
+    karafka (2.0.0.beta5)
       dry-configurable (~> 0.13)
       dry-monitor (~> 0.5)
       dry-validation (~> 1.7)

data/bin/benchmarks

@@ -39,8 +39,8 @@ if ENV['SEED']
 
   # We do not populate data of benchmarks_0_10 as we use it with life-stream data only
   %w[
-    benchmarks_0_01
-    benchmarks_0_05
+    benchmarks_00_01
+    benchmarks_00_05
   ].each do |topic_name|
     partitions_count = topic_name.split('_').last.to_i
 

data/bin/integrations

@@ -21,6 +21,9 @@ ROOT_PATH = Pathname.new(File.expand_path(File.join(File.dirname(__FILE__), '../
 # of CPU
 CONCURRENCY = ENV.key?('CI') ? 5 : Etc.nprocessors * 2
 
+# How may bytes do we want to keep from the stdout in the buffer for when we need to print it
+MAX_BUFFER_OUTPUT = 10_240
+
 # Abstraction around a single test scenario execution process
 class Scenario
   # How long a scenario can run before we kill it
@@ -84,9 +87,9 @@ class Scenario
     # We read it so it won't grow as we use our default logger that prints to both test.log and
     # to stdout. Otherwise after reaching the buffer size, it would hang
     buffer = ''
-    @stdout.read_nonblock(10_240, buffer, exception: false)
+    @stdout.read_nonblock(MAX_BUFFER_OUTPUT, buffer, exception: false)
     @stdout_tail << buffer
-    @stdout_tail = @stdout_tail[-10_024..-1] || @stdout_tail
+    @stdout_tail = @stdout_tail[-MAX_BUFFER_OUTPUT..-1] || @stdout_tail
 
     !@wait_thr.alive?
   end
@@ -114,11 +117,15 @@ class Scenario
     if success?
       print "\e[#{32}m#{'.'}\e[0m"
     else
+      buffer = ''
+
+      @stderr.read_nonblock(MAX_BUFFER_OUTPUT, buffer, exception: false)
+
       puts
       puts "\e[#{31}m#{'[FAILED]'}\e[0m #{name}"
       puts "Exit code: #{exit_code}"
       puts @stdout_tail
-      puts @stderr.read
+      puts buffer
       puts
     end
   end

data/bin/stress_one

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# Runs a single integration spec in an endless loop
+# This allows us to ensure (after long enough time) that the integration spec is stable and
+# that there are no anomalies when running it for a long period of time
+
+set -e
+
+while :
+do
+  reset
+  bin/scenario $1
+done

data/docker-compose.yml

@@ -16,26 +16,31 @@ services:
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
       KAFKA_CREATE_TOPICS:
-        "integrations_0_02:2:1,\
-         integrations_1_02:2:1,\
-         integrations_2_02:2:1,\
-         integrations_3_02:2:1,\
-         integrations_4_02:2:1,\
-         integrations_5_02:2:1,\
-         integrations_6_02:2:1,\
-         integrations_7_02:2:1,\
-         integrations_8_02:2:1,\
-         integrations_9_02:2:1,\
+        "integrations_00_02:2:1,\
+         integrations_01_02:2:1,\
+         integrations_02_02:2:1,\
+         integrations_03_02:2:1,\
+         integrations_04_02:2:1,\
+         integrations_05_02:2:1,\
+         integrations_06_02:2:1,\
+         integrations_07_02:2:1,\
+         integrations_08_02:2:1,\
+         integrations_09_02:2:1,\
          integrations_10_02:2:1,\
          integrations_11_02:2:1,\
          integrations_12_02:2:1,\
-         integrations_0_03:3:1,\
-         integrations_1_03:3:1,\
-         integrations_2_03:3:1,\
-         integrations_0_10:10:1,\
-         integrations_1_10:10:1,\
-         benchmarks_0_01:1:1,\
-         benchmarks_0_05:5:1,\
-         benchmarks_0_10:10:1"
+         integrations_13_02:2:1,\
+         integrations_14_02:2:1,\
+         integrations_15_02:2:1,\
+         integrations_16_02:2:1,\
+         integrations_00_03:3:1,\
+         integrations_01_03:3:1,\
+         integrations_02_03:3:1,\
+         integrations_03_03:3:1,\
+         integrations_00_10:10:1,\
+         integrations_01_10:10:1,\
+         benchmarks_00_01:1:1,\
+         benchmarks_00_05:5:1,\
+         benchmarks_00_10:10:1"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock

data/lib/karafka/active_job/routing/extensions.rb

@@ -13,7 +13,7 @@ module Karafka
         # @param block [Proc] block that we can use for some extra configuration
         def active_job_topic(name, &block)
           topic(name) do
-            consumer App.config.internal.active_job.consumer
+            consumer App.config.internal.active_job.consumer_class
 
             next unless block
 

data/lib/karafka/app.rb

@@ -10,7 +10,8 @@ module Karafka
       def consumer_groups
         config
           .internal
-          .routing_builder
+          .routing
+          .builder
       end
 
       # @return [Array<Karafka::Routing::SubscriptionGroup>] active subscription groups

data/lib/karafka/base_consumer.rb

@@ -10,17 +10,11 @@ module Karafka
     attr_accessor :messages
     # @return [Karafka::Connection::Client] kafka connection client
     attr_accessor :client
-    # @return [Karafka::TimeTrackers::Pause] current topic partition pause tracker
-    attr_accessor :pause_tracker
+    # @return [Karafka::Processing::Coordinator] coordinator
+    attr_accessor :coordinator
     # @return [Waterdrop::Producer] producer instance
     attr_accessor :producer
 
-    def initialize
-      # We re-use one to save on object allocation
-      # It also allows us to transfer the consumption notion to another batch
-      @consumption = Processing::Result.new
-    end
-
     # Can be used to run preparation code
     #
     # @private
@@ -41,9 +35,9 @@ module Karafka
         consume
       end
 
-      @consumption.success!
+      @coordinator.consumption(self).success!
     rescue StandardError => e
-      @consumption.failure!
+      @coordinator.consumption(self).failure!
 
       Karafka.monitor.instrument(
         'error.occurred',
@@ -51,14 +45,19 @@ module Karafka
         caller: self,
         type: 'consumer.consume.error'
       )
+    ensure
+      # We need to decrease number of jobs that this coordinator coordinates as it has finished
+      @coordinator.decrement
     end
 
     # @private
     # @note This should not be used by the end users as it is part of the lifecycle of things but
     #   not as part of the public api.
     def on_after_consume
-      if @consumption.success?
-        pause_tracker.reset
+      return if revoked?
+
+      if @coordinator.success?
+        coordinator.pause_tracker.reset
 
         # Mark as consumed only if manual offset management is not on
         return if topic.manual_offset_management?
@@ -75,6 +74,10 @@ module Karafka
     #
     # @private
     def on_revoked
+      coordinator.revoke
+
+      resume
+
       Karafka.monitor.instrument('consumer.revoked', caller: self) do
         revoked
       end
@@ -132,9 +135,11 @@ module Karafka
     #   processed but rather at the next one. This applies to both sync and async versions of this
     #   method.
     def mark_as_consumed(message)
-      @revoked = !client.mark_as_consumed(message)
+      unless client.mark_as_consumed(message)
+        coordinator.revoke
 
-      return false if revoked?
+        return false
+      end
 
       @seek_offset = message.offset + 1
 
@@ -147,9 +152,11 @@ module Karafka
     # @return [Boolean] true if we were able to mark the offset, false otherwise. False indicates
     #   that we were not able and that we have lost the partition.
     def mark_as_consumed!(message)
-      @revoked = !client.mark_as_consumed!(message)
+      unless client.mark_as_consumed!(message)
+        coordinator.revoke
 
-      return false if revoked?
+        return false
+      end
 
       @seek_offset = message.offset + 1
 
@@ -163,7 +170,7 @@ module Karafka
     # @param timeout [Integer, nil] how long in milliseconds do we want to pause or nil to use the
     #   default exponential pausing strategy defined for retries
     def pause(offset, timeout = nil)
-      timeout ? pause_tracker.pause(timeout) : pause_tracker.pause
+      timeout ? coordinator.pause_tracker.pause(timeout) : coordinator.pause_tracker.pause
 
       client.pause(
         messages.metadata.topic,
@@ -176,7 +183,7 @@ module Karafka
     def resume
       # This is sufficient to expire a partition pause, as with it will be resumed by the listener
       # thread before the next poll.
-      pause_tracker.expire
+      coordinator.pause_tracker.expire
     end
 
     # Seeks in the context of current topic and partition
@@ -196,7 +203,7 @@ module Karafka
     # @note We know that partition got revoked because when we try to mark message as consumed,
     #   unless if is successful, it will return false
     def revoked?
-      @revoked || false
+      coordinator.revoked?
     end
   end
 end

data/lib/karafka/connection/client.rb

@@ -36,6 +36,12 @@ module Karafka
         # Marks if we need to offset. If we did not store offsets, we should not commit the offset
         # position as it will crash rdkafka
         @offsetting = false
+        # We need to keep track of what we have paused for resuming
+        # In case we loose partition, we still need to resume it, otherwise it won't be fetched
+        # again if we get reassigned to it later on. We need to keep them as after revocation we
+        # no longer may be able to fetch them from Kafka. We could build them but it is easier
+        # to just keep them here and use if needed when cannot be obtained
+        @paused_tpls = Hash.new { |h, k| h[k] = {} }
       end
 
       # Fetches messages within boundaries defined by the settings (time, size, topics, etc).
@@ -45,12 +51,13 @@ module Karafka
       # @note This method should not be executed from many threads at the same time
       def batch_poll
         time_poll = TimeTrackers::Poll.new(@subscription_group.max_wait_time)
-        time_poll.start
 
         @buffer.clear
         @rebalance_manager.clear
 
         loop do
+          time_poll.start
+
           # Don't fetch more messages if we do not have any time left
           break if time_poll.exceeded?
           # Don't fetch more messages if we've fetched max as we've wanted
@@ -69,7 +76,11 @@ module Karafka
           # If partition revocation happens, we need to remove messages from revoked partitions
           # as well as ensure we do not have duplicated due to the offset reset for partitions
           # that we got assigned
-          remove_revoked_and_duplicated_messages if @rebalance_manager.revoked_partitions?
+          # We also do early break, so the information about rebalance is used as soon as possible
+          if @rebalance_manager.changed?
+            remove_revoked_and_duplicated_messages
+            break
+          end
 
           # Finally once we've (potentially) removed revoked, etc, if no messages were returned
           # we can break.
@@ -144,10 +155,14 @@ module Karafka
 
         internal_commit_offsets(async: false)
 
+        # Here we do not use our cached tpls because we should not try to pause something we do
+        # not own anymore.
         tpl = topic_partition_list(topic, partition)
 
         return unless tpl
 
+        @paused_tpls[topic][partition] = tpl
+
         @kafka.pause(tpl)
 
         @kafka.seek(pause_msg)
@@ -169,9 +184,13 @@ module Karafka
         # We can skip performance penalty since resuming should not happen too often
         internal_commit_offsets(async: false)
 
-        tpl = topic_partition_list(topic, partition)
+        # If we were not able, let's try to reuse the one we have (if we have)
+        tpl = topic_partition_list(topic, partition) || @paused_tpls[topic][partition]
 
         return unless tpl
+        # If we did not have it, it means we never paused this partition, thus no resume should
+        # happen in the first place
+        return unless @paused_tpls[topic].delete(partition)
 
         @kafka.resume(tpl)
       ensure
@@ -214,6 +233,7 @@ module Karafka
         @mutex.synchronize do
           @closed = false
           @offsetting = false
+          @paused_tpls.clear
           @kafka = build_consumer
         end
       end
@@ -369,7 +389,7 @@ module Karafka
       # we are no longer responsible in a given process for processing those messages and they
       # should have been picked up by a different process.
       def remove_revoked_and_duplicated_messages
-        @rebalance_manager.revoked_partitions.each do |topic, partitions|
+        @rebalance_manager.lost_partitions.each do |topic, partitions|
           partitions.each do |partition|
             @buffer.delete(topic, partition)
           end

data/lib/karafka/connection/listener.rb

@@ -21,12 +21,12 @@ module Karafka
         @id = SecureRandom.uuid
         @subscription_group = subscription_group
         @jobs_queue = jobs_queue
-        @jobs_builder = ::Karafka::App.config.internal.jobs_builder
-        @pauses_manager = PausesManager.new
+        @jobs_builder = ::Karafka::App.config.internal.processing.jobs_builder
+        @coordinators = Processing::CoordinatorsBuffer.new
         @client = Client.new(@subscription_group)
         @executors = Processing::ExecutorsBuffer.new(@client, subscription_group)
         # We reference scheduler here as it is much faster than fetching this each time
-        @scheduler = ::Karafka::App.config.internal.scheduler
+        @scheduler = ::Karafka::App.config.internal.processing.scheduler
         # We keep one buffer for messages to preserve memory and not allocate extra objects
         # We can do this that way because we always first schedule jobs using messages before we
         # fetch another batch.
@@ -79,6 +79,10 @@ module Karafka
             poll_and_remap_messages
           end
 
+          # This will ensure, that in the next poll, we continue processing (if we get them back)
+          # partitions that we have paused
+          resume_assigned_partitions
+
           # If there were revoked partitions, we need to wait on their jobs to finish before
           # distributing consuming jobs as upon revoking, we might get assigned to the same
           # partitions, thus getting their jobs. The revoking jobs need to finish before
@@ -86,6 +90,9 @@ module Karafka
           build_and_schedule_revoke_lost_partitions_jobs
 
           # We wait only on jobs from our subscription group. Other groups are independent.
+          # This will block on revoked jobs until they are finished. Those are not meant to last
+          # long and should not have any bigger impact on the system. Doing this in a blocking way
+          # simplifies the overall design and prevents from race conditions
           wait
 
           build_and_schedule_consumption_jobs
@@ -136,7 +143,7 @@ module Karafka
 
       # Resumes processing of partitions that were paused due to an error.
       def resume_paused_partitions
-        @pauses_manager.resume do |topic, partition|
+        @coordinators.resume do |topic, partition|
           @client.resume(topic, partition)
         end
       end
@@ -152,9 +159,23 @@ module Karafka
 
         revoked_partitions.each do |topic, partitions|
           partitions.each do |partition|
-            pause_tracker = @pauses_manager.fetch(topic, partition)
-            executor = @executors.fetch(topic, partition, pause_tracker)
-            jobs << @jobs_builder.revoked(executor)
+            # We revoke the coordinator here, so we do not have to revoke it in the revoke job
+            # itself (this happens prior to scheduling those jobs)
+            @coordinators.revoke(topic, partition)
+
+            # There may be a case where we have lost partition of which data we have never
+            # processed (if it was assigned and revoked really fast), thus we may not have it
+            # here. In cases like this, we do not run a revocation job
+            @executors.find_all(topic, partition).each do |executor|
+              jobs << @jobs_builder.revoked(executor)
+            end
+
+            # We need to remove all the executors of a given topic partition that we have lost, so
+            # next time we pick up it's work, new executors kick in. This may be needed especially
+            # for LRJ where we could end up with a race condition
+            # This revocation needs to happen after the jobs are scheduled, otherwise they would
+            # be scheduled with new executors instead of old
+            @executors.revoke(topic, partition)
           end
         end
 
@@ -183,6 +204,17 @@ module Karafka
         )
       end
 
+      # Revoked partition needs to be resumed if we were processing them earlier. This will do
+      # nothing to things that we are planning to process. Without this, things we get
+      # re-assigned would not be polled.
+      def resume_assigned_partitions
+        @client.rebalance_manager.assigned_partitions.each do |topic, partitions|
+          partitions.each do |partition|
+            @client.resume(topic, partition)
+          end
+        end
+      end
+
       # Takes the messages per topic partition and enqueues processing jobs in threads using
       # given scheduler.
       def build_and_schedule_consumption_jobs
@@ -191,11 +223,17 @@ module Karafka
         jobs = []
 
         @messages_buffer.each do |topic, partition, messages|
-          pause_tracker = @pauses_manager.fetch(topic, partition)
+          coordinator = @coordinators.find_or_create(topic, partition)
+
+          # Start work coordination for this topic partition
+          coordinator.start
+
+          # Count the job we're going to create here
+          coordinator.increment
 
-          executor = @executors.fetch(topic, partition, pause_tracker)
+          executor = @executors.find_or_create(topic, partition, 0)
 
-          jobs << @jobs_builder.consume(executor, messages)
+          jobs << @jobs_builder.consume(executor, messages, coordinator)
         end
 
         @scheduler.schedule_consumption(@jobs_queue, jobs)
@@ -231,7 +269,7 @@ module Karafka
         @jobs_queue.wait(@subscription_group.id)
         @jobs_queue.clear(@subscription_group.id)
         @client.reset
-        @pauses_manager = PausesManager.new
+        @coordinators.reset
         @executors = Processing::ExecutorsBuffer.new(@client, @subscription_group)
       end
     end

data/lib/karafka/connection/pauses_manager.rb

@@ -25,6 +25,14 @@ module Karafka
         )
       end
 
+      # Revokes pause tracker for a given topic partition
+      #
+      # @param topic [String] topic name
+      # @param partition [Integer] partition number
+      def revoke(topic, partition)
+        @pauses[topic].delete(partition)
+      end
+
       # Resumes processing of partitions for which pause time has ended.
       #
       # @yieldparam [String] topic name

data/lib/karafka/connection/rebalance_manager.rb

@@ -18,13 +18,15 @@ module Karafka
       # Empty array for internal usage not to create new objects
       EMPTY_ARRAY = [].freeze
 
+      attr_reader :assigned_partitions, :revoked_partitions
+
       private_constant :EMPTY_ARRAY
 
       # @return [RebalanceManager]
       def initialize
         @assigned_partitions = {}
         @revoked_partitions = {}
-        @lost_partitions = {}
+        @changed = false
       end
 
       # Resets the rebalance manager state
@@ -33,26 +35,12 @@ module Karafka
       def clear
         @assigned_partitions.clear
         @revoked_partitions.clear
-        @lost_partitions.clear
-      end
-
-      # @return [Hash<String, Array<Integer>>] hash where the keys are the names of topics for
-      #   which we've lost partitions and array with ids of the partitions as the value
-      # @note We do not consider as lost topics and partitions that got revoked and assigned
-      def revoked_partitions
-        return @revoked_partitions if @revoked_partitions.empty?
-        return @lost_partitions unless @lost_partitions.empty?
-
-        @revoked_partitions.each do |topic, partitions|
-          @lost_partitions[topic] = partitions - @assigned_partitions.fetch(topic, EMPTY_ARRAY)
-        end
-
-        @lost_partitions
+        @changed = false
       end
 
-      # @return [Boolean] true if any partitions were revoked
-      def revoked_partitions?
-        !revoked_partitions.empty?
+      # @return [Boolean] indicates a state change in the partitions assignment
+      def changed?
+        @changed
       end
 
       # Callback that kicks in inside of rdkafka, when new partitions are assigned.
@@ -62,6 +50,7 @@ module Karafka
       # @param partitions [Rdkafka::Consumer::TopicPartitionList]
       def on_partitions_assigned(_, partitions)
         @assigned_partitions = partitions.to_h.transform_values { |part| part.map(&:partition) }
+        @changed = true
       end
 
       # Callback that kicks in inside of rdkafka, when partitions are revoked.
@@ -71,6 +60,18 @@ module Karafka
       # @param partitions [Rdkafka::Consumer::TopicPartitionList]
       def on_partitions_revoked(_, partitions)
         @revoked_partitions = partitions.to_h.transform_values { |part| part.map(&:partition) }
+        @changed = true
+      end
+
+      # We consider as lost only partitions that were taken away and not re-assigned back to us
+      def lost_partitions
+        lost_partitions = {}
+
+        revoked_partitions.each do |topic, partitions|
+          lost_partitions[topic] = partitions - assigned_partitions.fetch(topic, EMPTY_ARRAY)
+        end
+
+        lost_partitions
       end
     end
   end