karafka 2.2.13 → 2.3.0.alpha1

data/lib/karafka/pro/processing/strategies/dlq/default.rb

@@ -26,6 +26,41 @@ module Karafka
               dead_letter_queue
             ].freeze
 
+            # Override of the standard `#mark_as_consumed` in order to handle the pause tracker
+            # reset in case DLQ is marked as fully independent. When DLQ is marked independent,
+            # any offset marking causes the pause count tracker to reset. This is useful when
+            # the error is not due to the collective batch operations state but due to intermediate
+            # "crawling" errors that move with it
+            #
+            # @see `Strategies::Default#mark_as_consumed` for more details
+            # @param message [Messages::Message]
+            # @param offset_metadata [String, nil]
+            def mark_as_consumed(message, offset_metadata = nil)
+              return super unless retrying?
+              return super unless topic.dead_letter_queue.independent?
+              return false unless super
+
+              coordinator.pause_tracker.reset
+
+              true
+            end
+
+            # Override of the standard `#mark_as_consumed!`. Resets the pause tracker count in case
+            # DLQ was configured with the `independent` flag.
+            #
+            # @see `Strategies::Default#mark_as_consumed!` for more details
+            # @param message [Messages::Message]
+            # @param offset_metadata [String, nil]
+            def mark_as_consumed!(message, offset_metadata = nil)
+              return super unless retrying?
+              return super unless topic.dead_letter_queue.independent?
+              return false unless super
+
+              coordinator.pause_tracker.reset
+
+              true
+            end
+
             # When we encounter non-recoverable message, we skip it and go on with our lives
             def handle_after_consume
               coordinator.on_finished do |last_group_message|

data/lib/karafka/pro/processing/strategies/lrj/default.rb

@@ -29,7 +29,7 @@ module Karafka
             ].freeze
 
             # We always need to pause prior to doing any jobs for LRJ
-            def handle_before_enqueue
+            def handle_before_schedule_consume
               super
 
               # This ensures that when running LRJ with VP, things operate as expected run only

data/lib/karafka/pro/processing/strategies/lrj/mom.rb

@@ -29,7 +29,7 @@ module Karafka
             ].freeze
 
             # We always need to pause prior to doing any jobs for LRJ
-            def handle_before_enqueue
+            def handle_before_schedule_consume
               super
 
               # This ensures that when running LRJ with VP, things operate as expected run only

data/lib/karafka/pro/processing/strategies/vp/default.rb

@@ -29,40 +29,74 @@ module Karafka
             ].freeze
 
             # @param message [Karafka::Messages::Message] marks message as consumed
+            # @param offset_metadata [String, nil]
             # @note This virtual offset management uses a regular default marking API underneath.
             #   We do not alter the "real" marking API, as VPs are just one of many cases we want
             #   to support and we do not want to impact them with collective offsets management
-            def mark_as_consumed(message)
-              return super if collapsed?
-
-              manager = coordinator.virtual_offset_manager
-
-              coordinator.synchronize do
-                manager.mark(message)
-                # If this is last marking on a finished flow, we can use the original
-                # last message and in order to do so, we need to mark all previous messages as
-                # consumed as otherwise the computed offset could be different
-                # We mark until our offset just in case of a DLQ flow or similar, where we do not
-                # want to mark all but until the expected location
-                manager.mark_until(message) if coordinator.finished?
-
-                return revoked? unless manager.markable?
-
-                manager.markable? ? super(manager.markable) : revoked?
+            def mark_as_consumed(message, offset_metadata = nil)
+              if @_in_transaction && !collapsed?
+                mark_in_transaction(message, offset_metadata, true)
+              elsif collapsed?
+                super
+              else
+                manager = coordinator.virtual_offset_manager
+
+                coordinator.synchronize do
+                  manager.mark(message, offset_metadata)
+                  # If this is last marking on a finished flow, we can use the original
+                  # last message and in order to do so, we need to mark all previous messages as
+                  # consumed as otherwise the computed offset could be different
+                  # We mark until our offset just in case of a DLQ flow or similar, where we do not
+                  # want to mark all but until the expected location
+                  manager.mark_until(message, offset_metadata) if coordinator.finished?
+
+                  return revoked? unless manager.markable?
+
+                  manager.markable? ? super(*manager.markable) : revoked?
+                end
               end
             end
 
             # @param message [Karafka::Messages::Message] blocking marks message as consumed
-            def mark_as_consumed!(message)
-              return super if collapsed?
+            # @param offset_metadata [String, nil]
+            def mark_as_consumed!(message, offset_metadata = nil)
+              if @_in_transaction && !collapsed?
+                mark_in_transaction(message, offset_metadata, false)
+              elsif collapsed?
+                super
+              else
+                manager = coordinator.virtual_offset_manager
+
+                coordinator.synchronize do
+                  manager.mark(message, offset_metadata)
+                  manager.mark_until(message, offset_metadata) if coordinator.finished?
+                  manager.markable? ? super(*manager.markable) : revoked?
+                end
+              end
+            end
 
-              manager = coordinator.virtual_offset_manager
+            # Stores the next offset for processing inside of the transaction when collapsed and
+            # accumulates marking as consumed in the local buffer.
+            #
+            # Due to nature of VPs we cannot provide full EOS support but we can simulate it,
+            # making sure that no offset are stored unless transaction is finished. We do it by
+            # accumulating the post-transaction marking requests and after it is successfully done
+            # we mark each as consumed. This effectively on errors "rollbacks" the state and
+            # prevents offset storage.
+            #
+            # Since the EOS here is "weak", we do not have to worry about the race-conditions and
+            # we do not have to have any mutexes.
+            #
+            # @param message [Messages::Message] message we want to commit inside of a transaction
+            # @param offset_metadata [String, nil] offset metadata or nil if none
+            # @param async [Boolean] should we mark in async or sync way (applicable only to post
+            #   transaction state synchronization usage as within transaction it is always sync)
+            def mark_in_transaction(message, offset_metadata, async)
+              raise Errors::TransactionRequiredError unless @_in_transaction
 
-              coordinator.synchronize do
-                manager.mark(message)
-                manager.mark_until(message) if coordinator.finished?
-                manager.markable? ? super(manager.markable) : revoked?
-              end
+              return super if collapsed?
+
+              @_transaction_marked << [message, offset_metadata, async]
             end
 
             # @return [Boolean] is the virtual processing collapsed in the context of given
@@ -112,7 +146,7 @@ module Karafka
             #
             # @note This can be done without the mutex, because it happens from the same thread
             #   for all the work (listener thread)
-            def handle_before_enqueue
+            def handle_before_schedule_consume
               super
 
               coordinator.virtual_offset_manager.register(

data/lib/karafka/pro/processing/virtual_offset_manager.rb

@@ -30,22 +30,29 @@ module Karafka
 
         # @param topic [String]
         # @param partition [Integer]
+        # @param offset_metadata_strategy [Symbol] what metadata should we select. That is, should
+        #   we use the most recent or one picked from the offset that is going to be committed
         #
         # @note We need topic and partition because we use a seek message (virtual) for real offset
         #   management. We could keep real message reference but this can be memory consuming
         #   and not worth it.
-        def initialize(topic, partition)
+        def initialize(topic, partition, offset_metadata_strategy)
           @topic = topic
           @partition = partition
           @groups = []
           @marked = {}
+          @offsets_metadata = {}
           @real_offset = -1
+          @offset_metadata_strategy = offset_metadata_strategy
+          @current_offset_metadata = nil
         end
 
         # Clears the manager for a next collective operation
         def clear
           @groups.clear
-          @marked = {}
+          @offsets_metadata.clear
+          @current_offset_metadata = nil
+          @marked.clear
           @real_offset = -1
         end
 
@@ -65,9 +72,14 @@ module Karafka
         # and we can refresh our real offset representation based on that as it might have changed
         # to a newer real offset.
         # @param message [Karafka::Messages::Message] message coming from VP we want to mark
-        def mark(message)
+        # @param offset_metadata [String, nil] offset metadata. `nil` if none
+        def mark(message, offset_metadata)
           offset = message.offset
 
+          # Store metadata when we materialize the most stable offset
+          @offsets_metadata[offset] = offset_metadata
+          @current_offset_metadata = offset_metadata
+
           group = @groups.find { |reg_group| reg_group.include?(offset) }
 
           # This case can happen when someone uses MoM and wants to mark message from a previous
@@ -81,6 +93,9 @@ module Karafka
 
           # Mark all previous messages from the same group also as virtually consumed
           group[0..position].each do |markable_offset|
+            # Set previous messages metadata offset as the offset of higher one for overwrites
+            # unless a different metadata were set explicitely
+            @offsets_metadata[markable_offset] ||= offset_metadata
             @marked[markable_offset] = true
           end
 
@@ -91,13 +106,15 @@ module Karafka
         # Mark all from all groups including the `message`.
         # Useful when operating in a collapsed state for marking
         # @param message [Karafka::Messages::Message]
-        def mark_until(message)
-          mark(message)
+        # @param offset_metadata [String, nil]
+        def mark_until(message, offset_metadata)
+          mark(message, offset_metadata)
 
           @groups.each do |group|
             group.each do |offset|
               next if offset > message.offset
 
+              @offsets_metadata[offset] = offset_metadata
               @marked[offset] = true
             end
           end
@@ -116,15 +133,28 @@ module Karafka
           !@real_offset.negative?
         end
 
-        # @return [Messages::Seek] markable message for real offset marking
+        # @return [Array<Messages::Seek, String>] markable message for real offset marking and
+        #   its associated metadata
         def markable
           raise Errors::InvalidRealOffsetUsageError unless markable?
 
-          Messages::Seek.new(
-            @topic,
-            @partition,
-            @real_offset
-          )
+          offset_metadata = case @offset_metadata_strategy
+                            when :exact
+                              @offsets_metadata.fetch(@real_offset)
+                            when :current
+                              @current_offset_metadata
+                            else
+                              raise Errors::UnsupportedCaseError, @offset_metadata_strategy
+                            end
+
+          [
+            Messages::Seek.new(
+              @topic,
+              @partition,
+              @real_offset
+            ),
+            offset_metadata
+          ]
         end
 
         private

data/lib/karafka/pro/routing/features/long_running_job/topic.rb

@@ -23,6 +23,8 @@ module Karafka
               @long_running_job ||= Config.new(active: active)
             end
 
+            alias long_running long_running_job
+
             # @return [Boolean] is a given job on a topic a long-running one
             def long_running_job?
               long_running_job.active?

data/lib/karafka/pro/routing/features/multiplexing/config.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Multiplexing < Base
+          # Multiplexing configuration
+          Config = Struct.new(
+            :active,
+            :min,
+            :max,
+            :boot,
+            keyword_init: true
+          ) do
+            alias_method :active?, :active
+
+            # @return [Boolean] true if we are allowed to upscale and downscale
+            def dynamic?
+              min < max
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/multiplexing/contracts/topic.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Multiplexing < Base
+          # Namespace for multiplexing feature contracts
+          module Contracts
+            # Validates the subscription group multiplexing setup
+            # We validate it on the topic level as subscription groups are not built during the
+            # routing as they are pre-run dynamically built.
+            #
+            # multiplexing attributes are optional since multiplexing may not be enabled
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+              end
+
+              nested(:subscription_group_details) do
+                optional(:multiplexing_min) { |val| val.is_a?(Integer) && val >= 1 }
+                optional(:multiplexing_max) { |val| val.is_a?(Integer) && val >= 1 }
+                optional(:multiplexing_boot) { |val| val.is_a?(Integer) && val >= 1 }
+              end
+
+              # Makes sure min is not more than max
+              virtual do |data, errors|
+                next unless errors.empty?
+                next unless min(data)
+                next unless max(data)
+
+                min = min(data)
+                max = max(data)
+
+                next if min <= max
+
+                [[%w[subscription_group_details], :multiplexing_min_max_mismatch]]
+              end
+
+              # Makes sure, that boot is between min and max
+              virtual do |data, errors|
+                next unless errors.empty?
+                next unless min(data)
+                next unless max(data)
+                next unless boot(data)
+
+                min = min(data)
+                max = max(data)
+                boot = boot(data)
+
+                next if boot >= min && boot <= max
+
+                [[%w[subscription_group_details], :multiplexing_boot_mismatch]]
+              end
+
+              # Makes sure, that boot is equal to min and max when not in dynamic mode
+              virtual do |data, errors|
+                next unless errors.empty?
+                next unless min(data)
+                next unless max(data)
+                next unless boot(data)
+
+                min = min(data)
+                max = max(data)
+                boot = boot(data)
+
+                # In dynamic mode there are other rules to check boot
+                next if min != max
+                next if boot == min
+
+                [[%w[subscription_group_details], :multiplexing_boot_not_dynamic]]
+              end
+
+              class << self
+                # @param data [Hash] topic details
+                # @return [Integer, false] min or false if missing
+                def min(data)
+                  data[:subscription_group_details].fetch(:multiplexing_min, false)
+                end
+
+                # @param data [Hash] topic details
+                # @return [Integer, false] max or false if missing
+                def max(data)
+                  data[:subscription_group_details].fetch(:multiplexing_max, false)
+                end
+
+                # @param data [Hash] topic details
+                # @return [Integer, false] boot or false if missing
+                def boot(data)
+                  data[:subscription_group_details].fetch(:multiplexing_boot, false)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/multiplexing/patches/contracts/consumer_group.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Multiplexing < Base
+          # Patches to Karafka OSS
+          module Patches
+            # Contracts patches
+            module Contracts
+              # Consumer group contract patches
+              module ConsumerGroup
+                # Redefines the setup allowing for multiple sgs as long as with different names
+                #
+                # @param topic [Hash] topic config hash
+                # @return [Array] topic unique key for validators
+                def topic_unique_key(topic)
+                  [
+                    topic[:name],
+                    topic[:subscription_group_details]
+                  ]
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/multiplexing/proxy.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Multiplexing < Base
+          # Allows for multiplexing setup inside a consumer group definition
+          module Proxy
+            # @param min [Integer, nil] min multiplexing count or nil to set it to max, effectively
+            #   disabling dynamic multiplexing
+            # @param max [Integer] max multiplexing count
+            # @param boot [Integer] how many listeners should we start during boot by default
+            def multiplexing(min: nil, max: 1, boot: nil)
+              @target.current_subscription_group_details.merge!(
+                multiplexing_min: min || max,
+                multiplexing_max: max,
+                # Picks half of max by default as long as possible. Otherwise goes with min
+                multiplexing_boot: boot || [min || max, (max / 2)].max
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/multiplexing/subscription_group.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Multiplexing < Base
+          # Adds methods needed for the multiplexing to work
+          module SubscriptionGroup
+            # @return [Config] multiplexing config
+            def multiplexing
+              @multiplexing ||= begin
+                max = @details.fetch(:multiplexing_max, 1)
+                min = @details.fetch(:multiplexing_min, max)
+                boot = @details.fetch(:multiplexing_boot, max / 2)
+                active = max > 1
+
+                Config.new(active: active, min: min, max: max, boot: boot)
+              end
+            end
+
+            # @return [Boolean] is multiplexing active
+            def multiplexing?
+              multiplexing.active?
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/multiplexing/subscription_groups_builder.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Multiplexing < Base
+          # Expands the builder to multiply multiplexed groups
+          module SubscriptionGroupsBuilder
+            # Takes into consideration multiplexing and builds the more groups
+            #
+            # @param topics_array [Array<Routing::Topic>] group of topics that have the same
+            #   settings and can use the same connection
+            # @return [Array<Array<Routing::Topics>>] expanded groups
+            def expand(topics_array)
+              factor = topics_array.first.subscription_group_details.fetch(:multiplexing_max, 1)
+
+              Array.new(factor) do |i|
+                ::Karafka::Routing::Topics.new(
+                  i.zero? ? topics_array : topics_array.map(&:dup)
+                )
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/multiplexing.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Namespace for Pro routing enhancements
+    module Routing
+      # Namespace for additional Pro features
+      module Features
+        # Multiplexing allows for creating multiple subscription groups for the same topic inside
+        # of the same subscription group allowing for better parallelism with limited number
+        # of processes
+        class Multiplexing < Base
+          class << self
+            # @param _config [Karafka::Core::Configurable::Node] app config node
+            def pre_setup(_config)
+              # Make sure we use proper unique validator for topics definitions
+              ::Karafka::Contracts::ConsumerGroup.singleton_class.prepend(
+                Patches::Contracts::ConsumerGroup
+              )
+            end
+
+            # If needed installs the needed listener and initializes tracker
+            #
+            # @param _config [Karafka::Core::Configurable::Node] app config
+            def post_setup(_config)
+              ::Karafka::App.monitor.subscribe('app.running') do
+                # Do not install the manager and listener to control multiplexing unless there is
+                # multiplexing enabled and it is dynamic.
+                # We only need to control multiplexing when it is in a dynamic state
+                next unless ::Karafka::App
+                            .subscription_groups
+                            .values
+                            .flat_map(&:itself)
+                            .any? { |sg| sg.multiplexing? && sg.multiplexing.dynamic? }
+
+                # Subscribe for events and possibility to manage via the Pro connection manager
+                # that supports multiplexing
+                ::Karafka.monitor.subscribe(
+                  ::Karafka::Pro::Connection::Multiplexing::Listener.new
+                )
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end