karafka 2.2.14 → 2.3.0.alpha2

data/lib/karafka/pro/processing/strategies/default.rb

@@ -27,6 +27,144 @@ module Karafka
           # Apply strategy for a non-feature based flow
           FEATURES = %i[].freeze
 
+          # Allows to set offset metadata that will be used with the upcoming marking as consumed
+          # as long as a different offset metadata was not used. After it was used either via
+          # `#mark_as_consumed` or `#mark_as_consumed!` it will be set back to `nil`. It is done
+          # that way to provide the end user with ability to influence metadata on the non-user
+          # initiated markings in complex flows.
+          #
+          # @param offset_metadata [String, nil] metadata we want to store with the upcoming
+          #   marking as consumed
+          #
+          # @note Please be aware, that offset metadata set this way will be passed to any marking
+          #   as consumed even if it was not user initiated. For example in the DLQ flow.
+          def store_offset_metadata(offset_metadata)
+            @_current_offset_metadata = offset_metadata
+          end
+
+          # Marks message as consumed in an async way.
+          #
+          # @param message [Messages::Message] last successfully processed message.
+          # @param offset_metadata [String, nil] offset metadata string or nil if nothing
+          # @return [Boolean] true if we were able to mark the offset, false otherwise.
+          #   False indicates that we were not able and that we have lost the partition.
+          #
+          # @note We keep track of this offset in case we would mark as consumed and got error when
+          #   processing another message. In case like this we do not pause on the message we've
+          #   already processed but rather at the next one. This applies to both sync and async
+          #   versions of this method.
+          def mark_as_consumed(message, offset_metadata = @_current_offset_metadata)
+            if @_in_transaction
+              mark_in_transaction(message, offset_metadata, true)
+            else
+              # seek offset can be nil only in case `#seek` was invoked with offset reset request
+              # In case like this we ignore marking
+              return true if coordinator.seek_offset.nil?
+              # Ignore earlier offsets than the one we already committed
+              return true if coordinator.seek_offset > message.offset
+              return false if revoked?
+              return revoked? unless client.mark_as_consumed(message, offset_metadata)
+
+              coordinator.seek_offset = message.offset + 1
+            end
+
+            true
+          ensure
+            @_current_offset_metadata = nil
+          end
+
+          # Marks message as consumed in a sync way.
+          #
+          # @param message [Messages::Message] last successfully processed message.
+          # @param offset_metadata [String, nil] offset metadata string or nil if nothing
+          # @return [Boolean] true if we were able to mark the offset, false otherwise.
+          #   False indicates that we were not able and that we have lost the partition.
+          def mark_as_consumed!(message, offset_metadata = @_current_offset_metadata)
+            if @_in_transaction
+              mark_in_transaction(message, offset_metadata, false)
+            else
+              # seek offset can be nil only in case `#seek` was invoked with offset reset request
+              # In case like this we ignore marking
+              return true if coordinator.seek_offset.nil?
+              # Ignore earlier offsets than the one we already committed
+              return true if coordinator.seek_offset > message.offset
+              return false if revoked?
+
+              return revoked? unless client.mark_as_consumed!(message, offset_metadata)
+
+              coordinator.seek_offset = message.offset + 1
+            end
+
+            true
+          ensure
+            @_current_offset_metadata = nil
+          end
+
+          # Starts producer transaction, saves the transaction context for transactional marking
+          # and runs user code in this context
+          #
+          # Transactions on a consumer level differ from those initiated by the producer as they
+          # allow to mark offsets inside of the transaction. If the transaction is initialized
+          # only from the consumer, the offset will be stored in a regular fashion.
+          #
+          # @param block [Proc] code that we want to run in a transaction
+          def transaction(&block)
+            transaction_started = false
+
+            # Prevent from nested transactions. It would not make any sense
+            raise Errors::TransactionAlreadyInitializedError if @_in_transaction
+
+            transaction_started = true
+            @_transaction_marked = []
+            @_in_transaction = true
+
+            producer.transaction(&block)
+
+            @_in_transaction = false
+
+            # This offset is already stored in transaction but we set it here anyhow because we
+            # want to make sure our internal in-memory state is aligned with the transaction
+            #
+            # @note We never need to use the blocking `#mark_as_consumed!` here because the offset
+            #   anyhow was already stored during the transaction
+            #
+            # @note In theory we could only keep reference to the most recent marking and reject
+            #   others. We however do not do it for two reasons:
+            #   - User may have non standard flow relying on some alternative order and we want to
+            #     mimic this
+            #   - Complex strategies like VPs can use this in VPs to mark in parallel without
+            #     having to redefine the transactional flow completely
+            @_transaction_marked.each do |marking|
+              marking.pop ? mark_as_consumed(*marking) : mark_as_consumed!(*marking)
+            end
+          ensure
+            if transaction_started
+              @_transaction_marked.clear
+              @_in_transaction = false
+            end
+          end
+
+          # Stores the next offset for processing inside of the transaction and stores it in a
+          # local accumulator for post-transaction status update
+          #
+          # @param message [Messages::Message] message we want to commit inside of a transaction
+          # @param offset_metadata [String, nil] offset metadata or nil if none
+          # @param async [Boolean] should we mark in async or sync way (applicable only to post
+          #   transaction state synchronization usage as within transaction it is always sync)
+          def mark_in_transaction(message, offset_metadata, async)
+            raise Errors::TransactionRequiredError unless @_in_transaction
+            raise Errors::AssignmentLostError if revoked?
+
+            producer.transaction_mark_as_consumed(
+              client,
+              message,
+              offset_metadata
+            )
+
+            @_transaction_marked ||= []
+            @_transaction_marked << [message, offset_metadata, async]
+          end
+
           # No actions needed for the standard flow here
           def handle_before_schedule_consume
             Karafka.monitor.instrument('consumer.before_schedule_consume', caller: self)
@@ -87,7 +225,7 @@ module Karafka
             end
           end
 
-          # Standard
+          # Standard flow for revocation
           def handle_revoked
             coordinator.on_revoked do
               resume
@@ -100,6 +238,21 @@ module Karafka
               revoked
             end
           end
+
+          # No action needed for the tick standard flow
+          def handle_before_schedule_tick
+            Karafka.monitor.instrument('consumer.before_schedule_tick', caller: self)
+
+            nil
+          end
+
+          # Runs the consumer `#tick` method with reporting
+          def handle_tick
+            Karafka.monitor.instrument('consumer.tick', caller: self)
+            Karafka.monitor.instrument('consumer.ticked', caller: self) do
+              tick
+            end
+          end
         end
       end
     end

data/lib/karafka/pro/processing/strategies/dlq/default.rb

@@ -26,6 +26,45 @@ module Karafka
               dead_letter_queue
             ].freeze
 
+            # Override of the standard `#mark_as_consumed` in order to handle the pause tracker
+            # reset in case DLQ is marked as fully independent. When DLQ is marked independent,
+            # any offset marking causes the pause count tracker to reset. This is useful when
+            # the error is not due to the collective batch operations state but due to intermediate
+            # "crawling" errors that move with it
+            #
+            # @see `Strategies::Default#mark_as_consumed` for more details
+            # @param message [Messages::Message]
+            # @param offset_metadata [String, nil]
+            def mark_as_consumed(message, offset_metadata = @_current_offset_metadata)
+              return super unless retrying?
+              return super unless topic.dead_letter_queue.independent?
+              return false unless super
+
+              coordinator.pause_tracker.reset
+
+              true
+            ensure
+              @_current_offset_metadata = nil
+            end
+
+            # Override of the standard `#mark_as_consumed!`. Resets the pause tracker count in case
+            # DLQ was configured with the `independent` flag.
+            #
+            # @see `Strategies::Default#mark_as_consumed!` for more details
+            # @param message [Messages::Message]
+            # @param offset_metadata [String, nil]
+            def mark_as_consumed!(message, offset_metadata = @_current_offset_metadata)
+              return super unless retrying?
+              return super unless topic.dead_letter_queue.independent?
+              return false unless super
+
+              coordinator.pause_tracker.reset
+
+              true
+            ensure
+              @_current_offset_metadata = nil
+            end
+
             # When we encounter non-recoverable message, we skip it and go on with our lives
             def handle_after_consume
               coordinator.on_finished do |last_group_message|

data/lib/karafka/pro/processing/strategies/vp/default.rb

@@ -29,40 +29,80 @@ module Karafka
             ].freeze
 
             # @param message [Karafka::Messages::Message] marks message as consumed
+            # @param offset_metadata [String, nil]
             # @note This virtual offset management uses a regular default marking API underneath.
             #   We do not alter the "real" marking API, as VPs are just one of many cases we want
             #   to support and we do not want to impact them with collective offsets management
-            def mark_as_consumed(message)
-              return super if collapsed?
-
-              manager = coordinator.virtual_offset_manager
-
-              coordinator.synchronize do
-                manager.mark(message)
-                # If this is last marking on a finished flow, we can use the original
-                # last message and in order to do so, we need to mark all previous messages as
-                # consumed as otherwise the computed offset could be different
-                # We mark until our offset just in case of a DLQ flow or similar, where we do not
-                # want to mark all but until the expected location
-                manager.mark_until(message) if coordinator.finished?
-
-                return revoked? unless manager.markable?
-
-                manager.markable? ? super(manager.markable) : revoked?
+            def mark_as_consumed(message, offset_metadata = @_current_offset_metadata)
+              if @_in_transaction && !collapsed?
+                mark_in_transaction(message, offset_metadata, true)
+              elsif collapsed?
+                super
+              else
+                manager = coordinator.virtual_offset_manager
+
+                coordinator.synchronize do
+                  manager.mark(message, offset_metadata)
+                  # If this is last marking on a finished flow, we can use the original
+                  # last message and in order to do so, we need to mark all previous messages as
+                  # consumed as otherwise the computed offset could be different
+                  # We mark until our offset just in case of a DLQ flow or similar, where we do not
+                  # want to mark all but until the expected location
+                  manager.mark_until(message, offset_metadata) if coordinator.finished?
+
+                  return revoked? unless manager.markable?
+
+                  manager.markable? ? super(*manager.markable) : revoked?
+                end
               end
+            ensure
+              @_current_offset_metadata = nil
             end
 
             # @param message [Karafka::Messages::Message] blocking marks message as consumed
-            def mark_as_consumed!(message)
-              return super if collapsed?
+            # @param offset_metadata [String, nil]
+            def mark_as_consumed!(message, offset_metadata = @_current_offset_metadata)
+              if @_in_transaction && !collapsed?
+                mark_in_transaction(message, offset_metadata, false)
+              elsif collapsed?
+                super
+              else
+                manager = coordinator.virtual_offset_manager
+
+                coordinator.synchronize do
+                  manager.mark(message, offset_metadata)
+                  manager.mark_until(message, offset_metadata) if coordinator.finished?
+                  manager.markable? ? super(*manager.markable) : revoked?
+                end
+              end
+            ensure
+              @_current_offset_metadata = nil
+            end
 
-              manager = coordinator.virtual_offset_manager
+            # Stores the next offset for processing inside of the transaction when collapsed and
+            # accumulates marking as consumed in the local buffer.
+            #
+            # Due to nature of VPs we cannot provide full EOS support but we can simulate it,
+            # making sure that no offset are stored unless transaction is finished. We do it by
+            # accumulating the post-transaction marking requests and after it is successfully done
+            # we mark each as consumed. This effectively on errors "rollbacks" the state and
+            # prevents offset storage.
+            #
+            # Since the EOS here is "weak", we do not have to worry about the race-conditions and
+            # we do not have to have any mutexes.
+            #
+            # @param message [Messages::Message] message we want to commit inside of a transaction
+            # @param offset_metadata [String, nil] offset metadata or nil if none
+            # @param async [Boolean] should we mark in async or sync way (applicable only to post
+            #   transaction state synchronization usage as within transaction it is always sync)
+            def mark_in_transaction(message, offset_metadata, async)
+              raise Errors::TransactionRequiredError unless @_in_transaction
+              # Prevent from attempts of offset storage when we no longer own the assignment
+              raise Errors::AssignmentLostError if revoked?
 
-              coordinator.synchronize do
-                manager.mark(message)
-                manager.mark_until(message) if coordinator.finished?
-                manager.markable? ? super(manager.markable) : revoked?
-              end
+              return super if collapsed?
+
+              @_transaction_marked << [message, offset_metadata, async]
             end
 
             # @return [Boolean] is the virtual processing collapsed in the context of given

data/lib/karafka/pro/processing/virtual_offset_manager.rb

@@ -30,22 +30,29 @@ module Karafka
 
         # @param topic [String]
         # @param partition [Integer]
+        # @param offset_metadata_strategy [Symbol] what metadata should we select. That is, should
+        #   we use the most recent or one picked from the offset that is going to be committed
         #
         # @note We need topic and partition because we use a seek message (virtual) for real offset
         #   management. We could keep real message reference but this can be memory consuming
         #   and not worth it.
-        def initialize(topic, partition)
+        def initialize(topic, partition, offset_metadata_strategy)
           @topic = topic
           @partition = partition
           @groups = []
           @marked = {}
+          @offsets_metadata = {}
           @real_offset = -1
+          @offset_metadata_strategy = offset_metadata_strategy
+          @current_offset_metadata = nil
         end
 
         # Clears the manager for a next collective operation
         def clear
           @groups.clear
-          @marked = {}
+          @offsets_metadata.clear
+          @current_offset_metadata = nil
+          @marked.clear
           @real_offset = -1
         end
 
@@ -65,9 +72,14 @@ module Karafka
         # and we can refresh our real offset representation based on that as it might have changed
         # to a newer real offset.
         # @param message [Karafka::Messages::Message] message coming from VP we want to mark
-        def mark(message)
+        # @param offset_metadata [String, nil] offset metadata. `nil` if none
+        def mark(message, offset_metadata)
           offset = message.offset
 
+          # Store metadata when we materialize the most stable offset
+          @offsets_metadata[offset] = offset_metadata
+          @current_offset_metadata = offset_metadata
+
           group = @groups.find { |reg_group| reg_group.include?(offset) }
 
           # This case can happen when someone uses MoM and wants to mark message from a previous
@@ -81,6 +93,9 @@ module Karafka
 
           # Mark all previous messages from the same group also as virtually consumed
           group[0..position].each do |markable_offset|
+            # Set previous messages metadata offset as the offset of higher one for overwrites
+            # unless a different metadata were set explicitely
+            @offsets_metadata[markable_offset] ||= offset_metadata
             @marked[markable_offset] = true
           end
 
@@ -91,13 +106,15 @@ module Karafka
         # Mark all from all groups including the `message`.
         # Useful when operating in a collapsed state for marking
         # @param message [Karafka::Messages::Message]
-        def mark_until(message)
-          mark(message)
+        # @param offset_metadata [String, nil]
+        def mark_until(message, offset_metadata)
+          mark(message, offset_metadata)
 
           @groups.each do |group|
             group.each do |offset|
               next if offset > message.offset
 
+              @offsets_metadata[offset] = offset_metadata
               @marked[offset] = true
             end
           end
@@ -116,15 +133,28 @@ module Karafka
           !@real_offset.negative?
         end
 
-        # @return [Messages::Seek] markable message for real offset marking
+        # @return [Array<Messages::Seek, String>] markable message for real offset marking and
+        #   its associated metadata
         def markable
           raise Errors::InvalidRealOffsetUsageError unless markable?
 
-          Messages::Seek.new(
-            @topic,
-            @partition,
-            @real_offset
-          )
+          offset_metadata = case @offset_metadata_strategy
+                            when :exact
+                              @offsets_metadata.fetch(@real_offset)
+                            when :current
+                              @current_offset_metadata
+                            else
+                              raise Errors::UnsupportedCaseError, @offset_metadata_strategy
+                            end
+
+          [
+            Messages::Seek.new(
+              @topic,
+              @partition,
+              @real_offset
+            ),
+            offset_metadata
+          ]
         end
 
         private

data/lib/karafka/pro/routing/features/long_running_job/topic.rb

@@ -23,6 +23,8 @@ module Karafka
               @long_running_job ||= Config.new(active: active)
             end
 
+            alias long_running long_running_job
+
             # @return [Boolean] is a given job on a topic a long-running one
             def long_running_job?
               long_running_job.active?

data/lib/karafka/pro/routing/features/multiplexing/config.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Multiplexing < Base
+          # Multiplexing configuration
+          Config = Struct.new(
+            :active,
+            :min,
+            :max,
+            :boot,
+            keyword_init: true
+          ) do
+            alias_method :active?, :active
+
+            # @return [Boolean] true if we are allowed to upscale and downscale
+            def dynamic?
+              min < max
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/multiplexing/contracts/topic.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Multiplexing < Base
+          # Namespace for multiplexing feature contracts
+          module Contracts
+            # Validates the subscription group multiplexing setup
+            # We validate it on the topic level as subscription groups are not built during the
+            # routing as they are pre-run dynamically built.
+            #
+            # multiplexing attributes are optional since multiplexing may not be enabled
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+              end
+
+              nested(:subscription_group_details) do
+                optional(:multiplexing_min) { |val| val.is_a?(Integer) && val >= 1 }
+                optional(:multiplexing_max) { |val| val.is_a?(Integer) && val >= 1 }
+                optional(:multiplexing_boot) { |val| val.is_a?(Integer) && val >= 1 }
+              end
+
+              # Makes sure min is not more than max
+              virtual do |data, errors|
+                next unless errors.empty?
+                next unless min(data)
+                next unless max(data)
+
+                min = min(data)
+                max = max(data)
+
+                next if min <= max
+
+                [[%w[subscription_group_details], :multiplexing_min_max_mismatch]]
+              end
+
+              # Makes sure, that boot is between min and max
+              virtual do |data, errors|
+                next unless errors.empty?
+                next unless min(data)
+                next unless max(data)
+                next unless boot(data)
+
+                min = min(data)
+                max = max(data)
+                boot = boot(data)
+
+                next if boot >= min && boot <= max
+
+                [[%w[subscription_group_details], :multiplexing_boot_mismatch]]
+              end
+
+              # Makes sure, that boot is equal to min and max when not in dynamic mode
+              virtual do |data, errors|
+                next unless errors.empty?
+                next unless min(data)
+                next unless max(data)
+                next unless boot(data)
+
+                min = min(data)
+                max = max(data)
+                boot = boot(data)
+
+                # In dynamic mode there are other rules to check boot
+                next if min != max
+                next if boot == min
+
+                [[%w[subscription_group_details], :multiplexing_boot_not_dynamic]]
+              end
+
+              class << self
+                # @param data [Hash] topic details
+                # @return [Integer, false] min or false if missing
+                def min(data)
+                  data[:subscription_group_details].fetch(:multiplexing_min, false)
+                end
+
+                # @param data [Hash] topic details
+                # @return [Integer, false] max or false if missing
+                def max(data)
+                  data[:subscription_group_details].fetch(:multiplexing_max, false)
+                end
+
+                # @param data [Hash] topic details
+                # @return [Integer, false] boot or false if missing
+                def boot(data)
+                  data[:subscription_group_details].fetch(:multiplexing_boot, false)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/multiplexing/patches/contracts/consumer_group.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Multiplexing < Base
+          # Patches to Karafka OSS
+          module Patches
+            # Contracts patches
+            module Contracts
+              # Consumer group contract patches
+              module ConsumerGroup
+                # Redefines the setup allowing for multiple sgs as long as with different names
+                #
+                # @param topic [Hash] topic config hash
+                # @return [Array] topic unique key for validators
+                def topic_unique_key(topic)
+                  [
+                    topic[:name],
+                    topic[:subscription_group_details]
+                  ]
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end