karafka 2.0.0.alpha5 → 2.0.0.alpha6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 062ee1f4c49d482daa48aa1af06aae08b83fb879ecfb4a6a14d5b2b34ed2975a
-  data.tar.gz: 28b92b8b5cea506641e339d59f53122fb18deb1637c20a2b34038ae2000b6f17
+  metadata.gz: 862df94b26c24809f82e07f71c39433b90ef08f68f053a004a87aa466b872dac
+  data.tar.gz: 2c533cbd6c271fe282f59c2030d9cc885555242bb8bc9316d0264a5ccfd694a0
 SHA512:
-  metadata.gz: '0497e94e2aa16ee20ded58e17313c9259bacb6f3fa91259e30f0cb0560b58a1eecb11447fb150e09cd659bb24d48e0cc732dc3fc7cc585a2aefe980df2e5b3f1'
-  data.tar.gz: 280407edd6298a7e62f970d2f9a2d6d1ff6eba4737ea4369eba20c4a83b9d00a0efa24ef36ceabd9def060839039fcf925c341f520efa2ab68532d484482f4fd
+  metadata.gz: 8ce3720e535d65f121bcbfd957286cbc41404f2aff2751622183367862db55f8231c286d5bbede4a18649eafcd085952a7fc8e7569a74dbd3633c9db906114e5
+  data.tar.gz: e77819ccd2be263b02958fcee71ce9228c93c7dee62fabbea61711fbbdbffa997bb9a85a7f668cf8bb596b7a98a773ace560ad5075c46e70f20cd302b354b0fe

data/CHANGELOG.md

@@ -1,5 +1,18 @@
 # Karafka framework changelog
 
+## 2.0.0-alpha6 (2022-04-17)
+- Fix a bug, where upon missing boot file and Rails, railtie would fail with a generic exception (#818) 
+- Fix an issue with parallel pristine specs colliding with each other during `bundle install` (#820)
+- Replace `consumer.consume` with `consumer.consumed` event to match the behaviour
+- Make sure, that offset committing happens before the `consumer.consumed` event is propagated
+- Fix for failing when not installed (just a dependency) (#817)
+- Evict messages from partitions that were lost upon rebalancing (#825)
+- Do **not** run `#revoked` on partitions that were lost and assigned back upon rebalancing (#825)
+- Remove potential duplicated that could occur upon rebalance with re-assigned partitions (#825)
+- Optimize integration test suite additional consumers shutdown process (#828)
+- Optimize messages eviction and duplicates removal on poll stopped due to lack of messages
+- Add static group membership integration spec
+
 ## 2.0.0-alpha5 (2022-04-03)
 - Rename StdoutListener to LoggerListener (#811)
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.0.alpha5)
+    karafka (2.0.0.alpha6)
       dry-configurable (~> 0.13)
       dry-monitor (~> 0.5)
       dry-validation (~> 1.7)

data/bin/integrations

@@ -44,17 +44,30 @@ class Scenario
   # @param path [String] path to the scenarios file
   def initialize(path)
     @path = path
-    @stdin, @stdout, @stderr, @wait_thr = Open3.popen3(init_and_build_cmd)
-    @started_at = current_time
     # Last 1024 characters from stdout
     @stdout_tail = ''
   end
 
+  # Starts running given scenario in a separate process
+  def start
+    @stdin, @stdout, @stderr, @wait_thr = Open3.popen3(init_and_build_cmd)
+    @started_at = current_time
+  end
+
   # @return [String] integration spec name
   def name
     @path.gsub("#{ROOT_PATH}/spec/integrations/", '')
   end
 
+  # @return [Boolean] true if spec is pristine
+  def pristine?
+    scenario_dir = File.dirname(@path)
+
+    # If there is a Gemfile in a scenario directory, it means it is a pristine spec and we need
+    # to run bundle install, etc in order to run it
+    File.exist?(File.join(scenario_dir, 'Gemfile'))
+  end
+
   # @return [Boolean] did this scenario finished or is it still running
   def finished?
     # If the thread is running too long, kill it
@@ -73,6 +86,13 @@ class Scenario
     !@wait_thr.alive?
   end
 
+  # @return [Boolean] did this scenario finish successfully or not
+  def success?
+    expected_exit_codes = EXIT_CODES[name] || EXIT_CODES[:default]
+
+    expected_exit_codes.include?(exit_code)
+  end
+
   # @return [Integer] pid of the process of this scenario
   def pid
     @wait_thr.pid
@@ -84,13 +104,6 @@ class Scenario
     @wait_thr.value&.exitstatus || 123
   end
 
-  # @return [Boolean] did this scenario finish successfully or not
-  def success?
-    expected_exit_codes = EXIT_CODES[name] || EXIT_CODES[:default]
-
-    expected_exit_codes.include?(exit_code)
-  end
-
   # Prints a status report when scenario is finished and stdout if it failed
   def report
     result = success? ? "\e[#{32}m#{'OK'}\e[0m" : "\e[#{31}m#{'FAILED'}\e[0m"
@@ -109,11 +122,10 @@ class Scenario
   # Sets up a proper environment for a given spec to run and returns the run command
   # @return [String] run command
   def init_and_build_cmd
-    scenario_dir = File.dirname(@path)
-
     # If there is a Gemfile in a scenario directory, it means it is a pristine spec and we need
     # to run bundle install, etc in order to run it
-    if File.exist?(File.join(scenario_dir, 'Gemfile'))
+    if pristine?
+      scenario_dir = File.dirname(@path)
       # We copy the spec into a temp dir, not to pollute the spec location with logs, etc
       temp_dir = Dir.mktmpdir
       file_name = File.basename(@path)
@@ -141,31 +153,6 @@ class Scenario
   end
 end
 
-# Simple array to keep track of active integration processes thread running with info on which
-# test scenario is running
-active_scenarios = []
-
-# Finished runners
-finished_scenarios = []
-
-# Waits for any of the processes to be finished and tracks exit codes
-#
-# @param active_scenarios [Array] active runners
-# @param finished_scenarios [Hash] finished forks exit codes
-def wait_and_track(active_scenarios, finished_scenarios)
-  exited = active_scenarios.find(&:finished?)
-
-  if exited
-    scenario = active_scenarios.delete(exited)
-
-    scenario.report
-
-    finished_scenarios << scenario
-  else
-    Thread.pass
-  end
-end
-
 # Load all the specs
 specs = Dir[ROOT_PATH.join('spec/integrations/**/*.rb')]
 
@@ -182,15 +169,40 @@ seed = (ENV['SEED'] || rand(0..10_000)).to_i
 
 puts "Random seed: #{seed}"
 
-specs.shuffle(random: Random.new(seed)).each do |integration_test|
-  scenario = Scenario.new(integration_test)
+scenarios = specs
+            .shuffle(random: Random.new(seed))
+            .map { |integration_test| Scenario.new(integration_test) }
 
-  active_scenarios << scenario
+regulars = scenarios.reject(&:pristine?)
+pristine = scenarios.select(&:pristine?)
 
-  wait_and_track(active_scenarios, finished_scenarios) until active_scenarios.size < CONCURRENCY
-end
+active_scenarios = []
+finished_scenarios = []
+
+while finished_scenarios.size < scenarios.size
+  # If we have space to run another scenario, we add it
+  if active_scenarios.size < CONCURRENCY
+    scenario = nil
+    # We can run only one pristine at the same time due to concurrency issues within bundler
+    # Since they usually take longer than others, we try to run them as fast as possible when there
+    # is a slot
+    scenario = pristine.pop unless active_scenarios.any?(&:pristine?)
+    scenario ||= regulars.pop
+
+    if scenario
+      scenario.start
+      active_scenarios << scenario
+    end
+  end
 
-wait_and_track(active_scenarios, finished_scenarios) while !active_scenarios.empty?
+  active_scenarios.select(&:finished?).each do |exited|
+    scenario = active_scenarios.delete(exited)
+    scenario.report
+    finished_scenarios << scenario
+  end
+
+  sleep(0.1)
+end
 
 # Fail all if any of the tests does not have expected exit code
 raise IntegrationTestError unless finished_scenarios.all?(&:success?)

data/docker-compose.yml

@@ -14,7 +14,9 @@ services:
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
       KAFKA_CREATE_TOPICS:
-        "integrations_0_03:3:1,\
+        "integrations_0_02:2:1,\
+         integrations_1_02:2:1,\
+         integrations_0_03:3:1,\
          integrations_1_03:3:1,\
          integrations_2_03:3:1,\
          integrations_0_10:10:1,\

data/lib/karafka/base_consumer.rb

@@ -21,18 +21,18 @@ module Karafka
     #   that may not yet kick in when error occurs. That way we pause always on the last processed
     #   message.
     def on_consume
-      Karafka.monitor.instrument('consumer.consume', caller: self) do
+      Karafka.monitor.instrument('consumer.consumed', caller: self) do
         consume
-      end
 
-      pause.reset
+        pause.reset
 
-      # Mark as consumed only if manual offset management is not on
-      return if topic.manual_offset_management
+        # Mark as consumed only if manual offset management is not on
+        return if topic.manual_offset_management
 
-      # We use the non-blocking one here. If someone needs the blocking one, can implement it with
-      # manual offset management
-      mark_as_consumed(messages.last)
+        # We use the non-blocking one here. If someone needs the blocking one, can implement it
+        # with manual offset management
+        mark_as_consumed(messages.last)
+      end
     rescue StandardError => e
       Karafka.monitor.instrument(
         'error.occurred',

data/lib/karafka/connection/client.rb

@@ -48,6 +48,7 @@ module Karafka
         time_poll.start
 
         @buffer.clear
+        @rebalance_manager.clear
 
         loop do
           # Don't fetch more messages if we do not have any time left
@@ -58,13 +59,23 @@ module Karafka
           # Fetch message within our time boundaries
           message = poll(time_poll.remaining)
 
-          # If there are no more messages, return what we have
-          break unless message
-
-          @buffer << message
+          # Put a message to the buffer if there is one
+          @buffer << message if message
 
           # Track time spent on all of the processing and polling
           time_poll.checkpoint
+
+          # Upon polling rebalance manager might have been updated.
+          # If partition revocation happens, we need to remove messages from revoked partitions
+          # as well as ensure we do not have duplicated due to the offset reset for partitions
+          # that we got assigned
+          remove_revoked_and_duplicated_messages if @rebalance_manager.revoked_partitions?
+
+          # Finally once we've (potentially) removed revoked, etc, if no messages were returned
+          # we can break.
+          # Worth keeping in mind, that the rebalance manager might have been updated despite no
+          # messages being returned during a poll
+          break unless message
         end
 
         @buffer
@@ -84,6 +95,9 @@ module Karafka
       # Ignoring a case where there would not be an offset (for example when rebalance occurs).
       #
       # @param async [Boolean] should the commit happen async or sync (async by default)
+      # @return [Boolean] did committing was successful. It may be not, when we no longer own
+      #   given partition.
+      #
       # @note This will commit all the offsets for the whole consumer. In order to achieve
       #   granular control over where the offset should be for particular topic partitions, the
       #   store_offset should be used to only store new offset when we want to to be flushed
@@ -212,6 +226,8 @@ module Karafka
           ::Karafka::Instrumentation.error_callbacks.delete(@subscription_group.id)
 
           @kafka.close
+          @buffer.clear
+          @rebalance_manager.clear
         end
       end
 
@@ -232,7 +248,7 @@ module Karafka
       # Performs a single poll operation.
       #
       # @param timeout [Integer] timeout for a single poll
-      # @return [Array<Rdkafka::Consumer::Message>, nil] fetched messages or nil if nothing polled
+      # @return [Rdkafka::Consumer::Message, nil] fetched message or nil if nothing polled
       def poll(timeout)
         time_poll ||= TimeTrackers::Poll.new(timeout)
 
@@ -301,6 +317,20 @@ module Karafka
 
         consumer
       end
+
+      # We may have a case where in the middle of data polling, we've lost a partition.
+      # In a case like this we should remove all the pre-buffered messages from list partitions as
+      # we are no longer responsible in a given process for processing those messages and they
+      # should have been picked up by a different process.
+      def remove_revoked_and_duplicated_messages
+        @rebalance_manager.revoked_partitions.each do |topic, partitions|
+          partitions.each do |partition|
+            @buffer.delete(topic, partition)
+          end
+        end
+
+        @buffer.uniq!
+      end
     end
   end
 end

data/lib/karafka/connection/messages_buffer.rb

@@ -42,6 +42,37 @@ module Karafka
         @groups[message.topic][message.partition] << message
       end
 
+      # Removes given topic and partition data out of the buffer
+      # This is used when there's a partition revocation
+      # @param topic [String] topic we're interested in
+      # @param partition [Integer] partition of which data we want to remove
+      def delete(topic, partition)
+        return unless @groups.key?(topic)
+        return unless @groups.fetch(topic).key?(partition)
+
+        topic_data = @groups.fetch(topic)
+        topic_data.delete(partition)
+
+        recount!
+
+        # If there are no more partitions to handle in a given topic, remove it completely
+        @groups.delete(topic) if topic_data.empty?
+      end
+
+      # Removes duplicated messages from the same partitions
+      # This should be used only when rebalance occurs, as we may get data again we already have
+      # due to the processing from the last offset. In cases like this, we may get same data
+      # again and we do want to ensure as few duplications as possible
+      def uniq!
+        @groups.each_value do |partitions|
+          partitions.each_value do |messages|
+            messages.uniq!(&:offset)
+          end
+        end
+
+        recount!
+      end
+
       # Removes all the data from the buffer.
       #
       # @note We do not clear the whole groups hash but rather we clear the partition hashes, so
@@ -52,6 +83,15 @@ module Karafka
         @size = 0
         @groups.each_value(&:clear)
       end
+
+      private
+
+      # Updates the messages count if we performed any operations that could change the state
+      def recount!
+        @size = @groups.each_value.sum do |partitions|
+          partitions.each_value.map(&:count).sum
+        end
+      end
     end
   end
 end

data/lib/karafka/connection/rebalance_manager.rb

@@ -9,35 +9,50 @@ module Karafka
     #
     # @note Since this does not happen really often, we try to stick with same objects for the
     #   empty states most of the time, so we don't create many objects during the manager life
+    #
+    # @note Internally in the rebalance manager we have a notion of lost partitions. Partitions
+    #   that are lost, are those that got revoked but did not get re-assigned back. We do not
+    #   expose this concept outside and we normalize to have them revoked, as it is irrelevant
+    #   from the rest of the code perspective as only those that are lost are truly revoked.
     class RebalanceManager
+      # Empty array for internal usage not to create new objects
+      EMPTY_ARRAY = [].freeze
+
+      private_constant :EMPTY_ARRAY
+
       # @return [RebalanceManager]
       def initialize
-        @assigned = {}
-        @revoked = {}
+        @assigned_partitions = {}
+        @revoked_partitions = {}
+        @lost_partitions = {}
       end
 
-      # @return [Hash<String, Array<Integer>>] hash where the keys are the names of topics for
-      #   which we've got new partitions assigned and array with ids of the partitions as the value
-      # @note Once assigned partitions are fetched, the state will be reset since the callbacks
-      #   for new assigned partitions are set only during a state change
-      def assigned_partitions
-        return @assigned if @assigned.empty?
-
-        result = @assigned.dup
-        @assigned.clear
-        result
+      # Resets the rebalance manager state
+      # This needs to be done before each polling loop as during the polling, the state may be
+      # changed
+      def clear
+        @assigned_partitions.clear
+        @revoked_partitions.clear
+        @lost_partitions.clear
       end
 
       # @return [Hash<String, Array<Integer>>] hash where the keys are the names of topics for
       #   which we've lost partitions and array with ids of the partitions as the value
-      # @note Once revoked partitions are fetched, the state will be reset since the callbacks
-      #   for new revoked partitions are set only during a state change
+      # @note We do not consider as lost topics and partitions that got revoked and assigned
       def revoked_partitions
-        return @revoked if @revoked.empty?
+        return @revoked_partitions if @revoked_partitions.empty?
+        return @lost_partitions unless @lost_partitions.empty?
+
+        @revoked_partitions.each do |topic, partitions|
+          @lost_partitions[topic] = partitions - @assigned_partitions.fetch(topic, EMPTY_ARRAY)
+        end
+
+        @lost_partitions
+      end
 
-        result = @revoked.dup
-        @revoked.clear
-        result
+      # @return [Boolean] true if any partitions were revoked
+      def revoked_partitions?
+        !revoked_partitions.empty?
       end
 
       # Callback that kicks in inside of rdkafka, when new partitions are assigned.
@@ -46,7 +61,7 @@ module Karafka
       # @param _ [Rdkafka::Consumer]
       # @param partitions [Rdkafka::Consumer::TopicPartitionList]
       def on_partitions_assigned(_, partitions)
-        @assigned = partitions.to_h.transform_values { |part| part.map(&:partition) }
+        @assigned_partitions = partitions.to_h.transform_values { |part| part.map(&:partition) }
       end
 
       # Callback that kicks in inside of rdkafka, when partitions are revoked.
@@ -55,7 +70,7 @@ module Karafka
       # @param _ [Rdkafka::Consumer]
       # @param partitions [Rdkafka::Consumer::TopicPartitionList]
       def on_partitions_revoked(_, partitions)
-        @revoked = partitions.to_h.transform_values { |part| part.map(&:partition) }
+        @revoked_partitions = partitions.to_h.transform_values { |part| part.map(&:partition) }
       end
     end
   end

data/lib/karafka/instrumentation/monitor.rb

@@ -22,7 +22,7 @@ module Karafka
         app.stopping
         app.stopped
 
-        consumer.consume
+        consumer.consumed
         consumer.revoked
         consumer.shutdown
 

data/lib/karafka/railtie.rb

@@ -82,8 +82,20 @@ if rails
       initializer 'karafka.require_karafka_boot_file' do |app|
         rails6plus = Rails.gem_version >= Gem::Version.new('6.0.0')
 
+        # If the boot file location is set to "false", we should not raise an exception and we
+        # should just not load karafka stuff. Setting this explicitly to false indicates, that
+        # karafka is part of the supply chain but it is not a first class citizen of a given
+        # system (may be just a dependency of a dependency), thus railtie should not kick in to
+        # load the non-existing boot file
+        next if Karafka.boot_file.to_s == 'false'
+
         karafka_boot_file = Rails.root.join(Karafka.boot_file.to_s).to_s
 
+        # Provide more comprehensive error for when no boot file
+        unless File.exist?(karafka_boot_file)
+          raise(Karafka::Errors::MissingBootFileError, karafka_boot_file)
+        end
+
         if rails6plus
           app.reloader.to_prepare do
             # Load Karafka boot file, so it can be used in Rails server context

data/lib/karafka/setup/config.rb

@@ -60,7 +60,7 @@ module Karafka
       # option [Boolean] should we leave offset management to the user
       setting :manual_offset_management, default: false
       # options max_messages [Integer] how many messages do we want to fetch from Kafka in one go
-      setting :max_messages, default: 100_000
+      setting :max_messages, default: 1_000
       # option [Integer] number of milliseconds we can wait while fetching data
       setting :max_wait_time, default: 10_000
       # option shutdown_timeout [Integer] the number of milliseconds after which Karafka no

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.0.alpha5'
+  VERSION = '2.0.0.alpha6'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.0.alpha5
+  version: 2.0.0.alpha6
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -34,7 +34,7 @@ cert_chain:
   R2P11bWoCtr70BsccVrN8jEhzwXngMyI2gVt750Y+dbTu1KgRqZKp/ECe7ZzPzXj
   pIy9vHxTANKYVyI4qj8OrFdEM5BQNu8oQpL0iQ==
   -----END CERTIFICATE-----
-date: 2022-04-03 00:00:00.000000000 Z
+date: 2022-04-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dry-configurable
@@ -282,7 +282,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
-rubygems_version: 3.3.4
+rubygems_version: 3.3.3
 signing_key: 
 specification_version: 4
 summary: Ruby based framework for working with Apache Kafka