karafka 2.0.14 → 2.0.16

data/lib/karafka/pro/loader.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component.
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
 # All of the commercial components are present in the lib/karafka/pro directory of this
 # repository and their usage requires commercial license agreement.
 #
@@ -13,37 +15,41 @@ module Karafka
   module Pro
     # Loader requires and loads all the pro components only when they are needed
     class Loader
-      # All the pro components that need to be loaded
-      COMPONENTS = %w[
-        base_consumer
-        performance_tracker
-        processing/scheduler
-        processing/jobs/consume_non_blocking
-        processing/jobs_builder
-        processing/coordinator
-        processing/partitioner
-        contracts/base
-        contracts/consumer_group
-        contracts/consumer_group_topic
-        routing/topic_extensions
-        routing/builder_extensions
-        active_job/consumer
+      # There seems to be a conflict in between using two Zeitwerk instances and it makes lookups
+      # for nested namespaces instead of creating them.
+      # We require those not to deal with this and then all works as expected
+      FORCE_LOADED = %w[
         active_job/dispatcher
-        active_job/job_options_contract
+        processing/jobs/consume_non_blocking
+        processing/strategies/base
+        routing/features/base
       ].freeze
 
-      private_constant :COMPONENTS
+      # Zeitwerk pro loader
+      # We need to have one per process, that's why it's set as a constant
+      PRO_LOADER = Zeitwerk::Loader.new
+
+      private_constant :PRO_LOADER
 
       class << self
+        # Requires all the components without using them anywhere
+        def require_all
+          FORCE_LOADED.each { |file| require_relative(file) }
+
+          PRO_LOADER.push_dir(Karafka.core_root.join('pro'), namespace: Karafka::Pro)
+          PRO_LOADER.setup
+          PRO_LOADER.eager_load
+        end
+
         # Loads all the pro components and configures them wherever it is expected
         # @param config [Karafka::Core::Configurable::Node] app config that we can alter with pro
         #   components
         def setup(config)
-          COMPONENTS.each { |component| require_relative(component) }
+          require_all
 
           reconfigure(config)
 
-          load_routing_extensions
+          load_topic_features
         end
 
         private
@@ -57,6 +63,7 @@ module Karafka
           icfg.processing.partitioner_class = Processing::Partitioner
           icfg.processing.scheduler = Processing::Scheduler.new
           icfg.processing.jobs_builder = Processing::JobsBuilder.new
+          icfg.processing.strategy_selector = Processing::StrategySelector.new
 
           icfg.active_job.consumer_class = ActiveJob::Consumer
           icfg.active_job.dispatcher = ActiveJob::Dispatcher.new
@@ -65,10 +72,10 @@ module Karafka
           config.monitor.subscribe(PerformanceTracker.instance)
         end
 
-        # Loads routing extensions
-        def load_routing_extensions
-          ::Karafka::Routing::Topic.prepend(Routing::TopicExtensions)
-          ::Karafka::Routing::Builder.prepend(Routing::BuilderExtensions)
+        # Loads the Pro features of Karafka
+        # @note Object space lookup is not the fastest but we do it once during boot, so it's ok
+        def load_topic_features
+          ::Karafka::Pro::Routing::Features::Base.load_all
         end
       end
     end

data/lib/karafka/pro/performance_tracker.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component.
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
 # All of the commercial components are present in the lib/karafka/pro directory of this
 # repository and their usage requires commercial license agreement.
 #

data/lib/karafka/pro/processing/coordinator.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component.
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
 # All of the commercial components are present in the lib/karafka/pro directory of this
 # repository and their usage requires commercial license agreement.
 #
@@ -21,6 +23,7 @@ module Karafka
           @on_enqueued_invoked = false
           @on_started_invoked = false
           @on_finished_invoked = false
+          @on_revoked_invoked = false
           @flow_lock = Mutex.new
         end
 
@@ -79,6 +82,18 @@ module Karafka
             yield(@last_message)
           end
         end
+
+        # Runs once when a partition is revoked
+        def on_revoked
+          @flow_lock.synchronize do
+            return unless finished?
+            return if @on_revoked_invoked
+
+            @on_revoked_invoked = true
+
+            yield(@last_message)
+          end
+        end
       end
     end
   end

data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component.
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
 # All of the commercial components are present in the lib/karafka/pro directory of this
 # repository and their usage requires commercial license agreement.
 #

data/lib/karafka/pro/processing/jobs_builder.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component.
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
 # All of the commercial components are present in the lib/karafka/pro directory of this
 # repository and their usage requires commercial license agreement.
 #

data/lib/karafka/pro/processing/partitioner.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component.
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
 # All of the commercial components are present in the lib/karafka/pro directory of this
 # repository and their usage requires commercial license agreement.
 #
@@ -25,28 +27,23 @@ module Karafka
           # process the data. With one thread it is not worth partitioning the work as the work
           # itself will be assigned to one thread (pointless work)
           if ktopic.virtual_partitions? && ktopic.virtual_partitions.max_partitions > 1
-            # We need to reduce it to number of threads, so the group_id is not a direct effect
+            # We need to reduce it to the max concurrency, so the group_id is not a direct effect
             # of the end user action. Otherwise the persistence layer for consumers would cache
             # it forever and it would cause memory leaks
-            groupings = messages
-                        .group_by { |msg| ktopic.virtual_partitions.partitioner.call(msg) }
-                        .values
-
-            # Reduce the number of virtual partitions to a size that matches the max_partitions
-            # As mentioned above we cannot use the partitioning keys directly as it could cause
-            # memory leaks
             #
-            # The algorithm here is simple, we assume that the most costly in terms of processing,
-            # will be processing of the biggest group and we reduce the smallest once to have
-            # max of groups equal to max_partitions
-            while groupings.size > ktopic.virtual_partitions.max_partitions
-              groupings.sort_by! { |grouping| -grouping.size }
+            # This also needs to be consistent because the aggregation here needs to warrant, that
+            # the same partitioned message will always be assigned to the same virtual partition.
+            # Otherwise in case of a window aggregation with VP spanning across several polls, the
+            # data could not be complete.
+            groupings = messages.group_by do |msg|
+              key = ktopic.virtual_partitions.partitioner.call(msg).to_s.sum
 
-              # Offset order needs to be maintained for virtual partitions
-              groupings << (groupings.pop + groupings.pop).sort_by!(&:offset)
+              key % ktopic.virtual_partitions.max_partitions
             end
 
-            groupings.each_with_index { |messages_group, index| yield(index, messages_group) }
+            groupings.each do |key, messages_group|
+              yield(key, messages_group)
+            end
           else
             # When no virtual partitioner, works as regular one
             yield(0, messages)

data/lib/karafka/pro/processing/scheduler.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component.
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
 # All of the commercial components are present in the lib/karafka/pro directory of this
 # repository and their usage requires commercial license agreement.
 #

data/lib/karafka/pro/processing/strategies/aj_dlq_lrj_mom.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # ActiveJob enabled
+        # DLQ enabled
+        # Long-Running Job enabled
+        # Manual offset management enabled
+        #
+        # This case is a bit of special. Please see the `AjDlqMom` for explanation on how the
+        # offset management works in this case.
+        module AjDlqLrjMom
+          include AjLrjMom
+          include AjDlqMom
+
+          # Features for this strategy
+          FEATURES = %i[
+            active_job
+            long_running_job
+            manual_offset_management
+            dead_letter_queue
+          ].freeze
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj_dlq_mom.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # ActiveJob enabled
+        # DLQ enabled
+        # Manual offset management enabled
+        #
+        # AJ has manual offset management on by default and the offset management is delegated to
+        # the AJ consumer. This means, we cannot mark as consumed always. We can only mark as
+        # consumed when we skip given job upon errors. In all the other scenarions marking as
+        # consumed needs to happen in the AJ consumer on a per job basis.
+        module AjDlqMom
+          include DlqMom
+
+          # Features for this strategy
+          FEATURES = %i[
+            active_job
+            dead_letter_queue
+            manual_offset_management
+          ].freeze
+
+          # How should we post-finalize consumption.
+          def handle_after_consume
+            coordinator.on_finished do
+              return if revoked?
+
+              if coordinator.success?
+                # Do NOT commit offsets, they are comitted after each job in the AJ consumer.
+                coordinator.pause_tracker.reset
+              elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                pause(coordinator.seek_offset)
+              else
+                coordinator.pause_tracker.reset
+                skippable_message = find_skippable_message
+                dispatch_to_dlq(skippable_message)
+                # We can commit the offset here because we know that we skip it "forever" and
+                # since AJ consumer commits the offset after each job, we also know that the
+                # previous job was successful
+                mark_as_consumed(skippable_message)
+                pause(coordinator.seek_offset)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj_lrj_mom.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # ActiveJob enabled
+        # Long-Running Job enabled
+        # Manual offset management enabled
+        module AjLrjMom
+          # Same behaviour as LrjMom
+          include LrjMom
+
+          # Features for this strategy
+          FEATURES = %i[
+            active_job
+            long_running_job
+            manual_offset_management
+          ].freeze
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj_lrj_mom_vp.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # ActiveJob enabled
+        # Long-Running Job enabled
+        # Manual offset management enabled
+        # Virtual Partitions enabled
+        module AjLrjMomVp
+          include Default
+
+          # Features for this strategy
+          FEATURES = %i[
+            active_job
+            long_running_job
+            manual_offset_management
+            virtual_partitions
+          ].freeze
+
+          # No actions needed for the standard flow here
+          def handle_before_enqueue
+            coordinator.on_enqueued do
+              pause(coordinator.seek_offset, Lrj::MAX_PAUSE_TIME)
+            end
+          end
+
+          # Standard flow without any features
+          def handle_after_consume
+            coordinator.on_finished do |last_group_message|
+              if coordinator.success?
+                coordinator.pause_tracker.reset
+
+                mark_as_consumed(last_group_message) unless revoked? || Karafka::App.stopping?
+                seek(coordinator.seek_offset) unless revoked?
+
+                resume
+              else
+                # If processing failed, we need to pause
+                # For long running job this will overwrite the default never-ending pause and will
+                # cause the processing to keep going after the error backoff
+                pause(coordinator.seek_offset)
+              end
+            end
+          end
+
+          # LRJ cannot resume here. Only in handling the after consumption
+          def handle_revoked
+            coordinator.on_revoked do
+              coordinator.revoke
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj_mom.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # ActiveJob enabled
+        # Manual Offset management enabled
+        module AjMom
+          # Standard ActiveJob strategy is the same one we use for Mom
+          include Mom
+
+          # Features for this strategy
+          FEATURES = %i[
+            active_job
+            manual_offset_management
+          ].freeze
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj_mom_vp.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # ActiveJob enabled
+        # Manual offset management enabled
+        # Virtual Partitions enabled
+        module AjMomVp
+          include Default
+
+          # Features for this strategy
+          FEATURES = %i[
+            active_job
+            manual_offset_management
+            virtual_partitions
+          ].freeze
+
+          # Standard flow without any features
+          def handle_after_consume
+            coordinator.on_finished do |last_group_message|
+              if coordinator.success?
+                coordinator.pause_tracker.reset
+
+                # When this is an ActiveJob running via Pro with virtual partitions, we cannot mark
+                # intermediate jobs as processed not to mess up with the ordering.
+                # Only when all the jobs are processed and we did not loose the partition
+                # assignment and we are not stopping (Pro ActiveJob has an early break) we can
+                # commit offsets on this as only then we can be sure, that all the jobs were
+                # processed.
+                # For a non virtual partitions case, the flow is regular and state is marked after
+                # each successfully processed job
+                return if revoked?
+                return if Karafka::App.stopping?
+
+                mark_as_consumed(last_group_message)
+              else
+                pause(coordinator.seek_offset)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/base.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Pro processing strategies namespace
+      module Strategies
+        # Base strategy for Pro
+        module Base
+          include ::Karafka::Processing::Strategies::Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/default.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # No features enabled.
+        # No manual offset management
+        # No long running jobs
+        # No virtual partitions
+        # Nothing. Just standard, automatic flow
+        module Default
+          include Base
+
+          # Apply strategy for a non-feature based flow
+          FEATURES = %i[].freeze
+
+          # No actions needed for the standard flow here
+          def handle_before_enqueue
+            nil
+          end
+
+          # Increment number of attempts per one "full" job. For all VP on a single topic partition
+          # this also should run once.
+          def handle_before_consume
+            coordinator.on_started do
+              coordinator.pause_tracker.increment
+            end
+          end
+
+          # Standard flow without any features
+          def handle_after_consume
+            coordinator.on_finished do |last_group_message|
+              return if revoked?
+
+              if coordinator.success?
+                coordinator.pause_tracker.reset
+
+                mark_as_consumed(last_group_message)
+              else
+                pause(coordinator.seek_offset)
+              end
+            end
+          end
+
+          # Standard
+          def handle_revoked
+            coordinator.on_revoked do
+              resume
+
+              coordinator.revoke
+            end
+          end
+        end
+      end
+    end
+  end
+end