karafka 2.0.27 → 2.0.29

data/lib/karafka/pro/processing/strategies/dlq_vp.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # Dead Letter Queue enabled
+        # Virtual Partitions enabled
+        #
+        # In general because we collapse processing in virtual partitions to one on errors, there
+        # is no special action that needs to be taken because we warranty that even with VPs
+        # on errors a retry collapses into a single state.
+        module DlqVp
+          # Features for this strategy
+          FEATURES = %i[
+            dead_letter_queue
+            virtual_partitions
+          ].freeze
+
+          include Dlq
+          include Vp
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/lrj_vp.rb

@@ -19,6 +19,7 @@ module Karafka
         # Virtual Partitions enabled
         module LrjVp
           # Same flow as the standard Lrj
+          include Vp
           include Lrj
 
           # Features for this strategy

data/lib/karafka/pro/processing/strategies/vp.rb

@@ -25,6 +25,11 @@ module Karafka
           FEATURES = %i[
             virtual_partitions
           ].freeze
+
+          # @return [Boolean] is the virtual processing collapsed in the context of given consumer.
+          def collapsed?
+            coordinator.collapsed?
+          end
         end
       end
     end

data/lib/karafka/pro/routing/features/dead_letter_queue/contract.rb

@@ -26,10 +26,10 @@ module Karafka
               ).fetch('en').fetch('validations').fetch('topic')
             end
 
-            # Make sure that we don't use DLQ with VP
-            # Using those two would cause many issues because the offset within VP is not
-            # manageable so in scenarios where we would fail on the last message, we would move by
-            # one and try again and fail, and move by one and try again and fail and so on...
+            # Make sure that when we use virtual partitions with DLQ, at least one retry is set
+            # We cannot use VP with DLQ without retries as we in order to provide ordering
+            # warranties on errors with VP, we need to collapse the VPs concurrency and retry
+            # without any indeterministic work
             virtual do |data, errors|
               next unless errors.empty?
 
@@ -38,8 +38,9 @@ module Karafka
 
               next unless dead_letter_queue[:active]
               next unless virtual_partitions[:active]
+              next if dead_letter_queue[:max_retries].positive?
 
-              [[%i[dead_letter_queue], :not_with_virtual_partitions]]
+              [[%i[dead_letter_queue], :with_virtual_partitions]]
             end
           end
         end

data/lib/karafka/processing/coordinator.rb

@@ -71,18 +71,28 @@ module Karafka
         end
       end
 
-      # @param consumer [Object] karafka consumer (normal or pro)
-      # @return [Karafka::Processing::Result] result object which we can use to indicate
-      #   consumption processing state.
-      def consumption(consumer)
+      # Is all the consumption done and finished successfully for this coordinator
+      def success?
         @mutex.synchronize do
-          @consumptions[consumer] ||= Processing::Result.new
+          @running_jobs.zero? && @consumptions.values.all?(&:success?)
         end
       end
 
-      # Is all the consumption done and finished successfully for this coordinator
-      def success?
-        @mutex.synchronize { @running_jobs.zero? && @consumptions.values.all?(&:success?) }
+      # Mark given consumption on consumer as successful
+      # @param consumer [Karafka::BaseConsumer] consumer that finished successfully
+      def success!(consumer)
+        @mutex.synchronize do
+          consumption(consumer).success!
+        end
+      end
+
+      # Mark given consumption on consumer as failed
+      # @param consumer [Karafka::BaseConsumer] consumer that failed
+      # @param error [StandardError] error that occurred
+      def failure!(consumer, error)
+        @mutex.synchronize do
+          consumption(consumer).failure!(error)
+        end
       end
 
       # Marks given coordinator for processing group as revoked
@@ -113,6 +123,15 @@ module Karafka
       def manual_pause?
         @pause_tracker.paused? && @manual_pause
       end
+
+      private
+
+      # @param consumer [Object] karafka consumer (normal or pro)
+      # @return [Karafka::Processing::Result] result object which we can use to indicate
+      #   consumption processing state.
+      def consumption(consumer)
+        @consumptions[consumer] ||= Processing::Result.new
+      end
     end
   end
 end

data/lib/karafka/processing/partitioner.rb

@@ -12,9 +12,11 @@ module Karafka
 
       # @param _topic [String] topic name
       # @param messages [Array<Karafka::Messages::Message>] karafka messages
+      # @param _coordinator [Karafka::Processing::Coordinator] processing coordinator that will
+      #   be used with those messages
       # @yieldparam [Integer] group id
       # @yieldparam [Array<Karafka::Messages::Message>] karafka messages
-      def call(_topic, messages)
+      def call(_topic, messages, _coordinator)
         yield(0, messages)
       end
     end

data/lib/karafka/processing/strategies/default.rb

@@ -31,10 +31,9 @@ module Karafka
           end
 
           # Mark job as successful
-          coordinator.consumption(self).success!
+          coordinator.success!(self)
         rescue StandardError => e
-          # If failed, mark as failed
-          coordinator.consumption(self).failure!(e)
+          coordinator.failure!(self, e)
 
           # Re-raise so reported in the consumer
           raise e

data/lib/karafka/railtie.rb

@@ -2,14 +2,7 @@
 
 # This file contains Railtie for auto-configuration
 
-rails = false
-
-begin
-  # Do not load Rails again if already loaded
-  Object.const_defined?('Rails::Railtie') || require('rails')
-
-  rails = true
-rescue LoadError
+unless Karafka.rails?
   # Without defining this in any way, Zeitwerk ain't happy so we do it that way
   module Karafka
     class Railtie
@@ -17,7 +10,7 @@ rescue LoadError
   end
 end
 
-if rails
+if Karafka.rails?
   # Load ActiveJob adapter
   require 'active_job/karafka'
 

data/lib/karafka/setup/config.rb

@@ -16,7 +16,6 @@ module Karafka
 
       # Defaults for kafka settings, that will be overwritten only if not present already
       KAFKA_DEFAULTS = {
-        'client.id': 'karafka',
         # We emit the statistics by default, so all the instrumentation and web-ui work out of
         # the box, without requiring users to take any extra actions aside from enabling.
         'statistics.interval.ms': 5_000
@@ -193,6 +192,9 @@ module Karafka
             config.kafka[key] = value
           end
 
+          # Use Karafka client_id as kafka client id if not set
+          config.kafka[:'client.id'] ||= config.client_id
+
           return if Karafka::App.env.production?
 
           KAFKA_DEV_DEFAULTS.each do |key, value|

data/lib/karafka/templates/karafka.rb.erb

@@ -40,10 +40,16 @@ class KarafkaApp < Karafka::App
   Karafka.monitor.subscribe(Karafka::Instrumentation::LoggerListener.new)
   # Karafka.monitor.subscribe(Karafka::Instrumentation::ProctitleListener.new)
 
+  # This logger prints the producer development info using the Karafka logger.
+  # It is similar to the consumer logger listener but producer oriented.
+  Karafka.producer.monitor.subscribe(
+    WaterDrop::Instrumentation::LoggerListener.new(Karafka.logger)
+  )
+
   routes.draw do
 <% if rails? -%>
     # Uncomment this if you use Karafka with ActiveJob
-    # You ned to define the topic per each queue name you use
+    # You need to define the topic per each queue name you use
     # active_job_topic :default
 <% end -%>
     topic :example do

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.27'
+  VERSION = '2.0.29'
 end

data/lib/karafka.rb

@@ -70,6 +70,19 @@ module Karafka
       App.config.license.token != false
     end
 
+    # @return [Boolean] Do we run within/with Rails. We use this to initialize Railtie and proxy
+    #   the console invocation to Rails
+    def rails?
+      return @rails if instance_variable_defined?('@rails')
+
+      # Do not load Rails again if already loaded
+      Object.const_defined?('Rails::Railtie') || require('rails')
+
+      @rails = true
+    rescue LoadError
+      @rails = false
+    end
+
     # @return [String] path to a default file that contains booting procedure etc
     # @note By default it is a file called 'karafka.rb' but it can be specified as you wish if you
     #   have Karafka that is merged into a Sinatra/Rails app and karafka.rb is taken.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.27
+  version: 2.0.29
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2023-01-11 00:00:00.000000000 Z
+date: 2023-01-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -77,7 +77,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.9
+        version: 2.4.10
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -87,7 +87,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.9
+        version: 2.4.10
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -224,6 +224,7 @@ files:
 - lib/karafka/pro/encryption/setup/config.rb
 - lib/karafka/pro/loader.rb
 - lib/karafka/pro/performance_tracker.rb
+- lib/karafka/pro/processing/collapser.rb
 - lib/karafka/pro/processing/coordinator.rb
 - lib/karafka/pro/processing/jobs/consume_non_blocking.rb
 - lib/karafka/pro/processing/jobs/revoked_non_blocking.rb
@@ -231,7 +232,9 @@ files:
 - lib/karafka/pro/processing/partitioner.rb
 - lib/karafka/pro/processing/scheduler.rb
 - lib/karafka/pro/processing/strategies/aj_dlq_lrj_mom.rb
+- lib/karafka/pro/processing/strategies/aj_dlq_lrj_mom_vp.rb
 - lib/karafka/pro/processing/strategies/aj_dlq_mom.rb
+- lib/karafka/pro/processing/strategies/aj_dlq_mom_vp.rb
 - lib/karafka/pro/processing/strategies/aj_lrj_mom.rb
 - lib/karafka/pro/processing/strategies/aj_lrj_mom_vp.rb
 - lib/karafka/pro/processing/strategies/aj_mom.rb
@@ -241,7 +244,9 @@ files:
 - lib/karafka/pro/processing/strategies/dlq.rb
 - lib/karafka/pro/processing/strategies/dlq_lrj.rb
 - lib/karafka/pro/processing/strategies/dlq_lrj_mom.rb
+- lib/karafka/pro/processing/strategies/dlq_lrj_vp.rb
 - lib/karafka/pro/processing/strategies/dlq_mom.rb
+- lib/karafka/pro/processing/strategies/dlq_vp.rb
 - lib/karafka/pro/processing/strategies/lrj.rb
 - lib/karafka/pro/processing/strategies/lrj_mom.rb
 - lib/karafka/pro/processing/strategies/lrj_vp.rb