karafka 2.0.0 → 2.1.0

data/lib/karafka/pro/processing/virtual_offset_manager.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Manager that keeps track of our offsets with the virtualization layer that are local
+      # to given partition assignment. It allows for easier offset management for virtual
+      # virtual partition cases as it provides us ability to mark as consumed and move the
+      # real offset behind as expected.
+      #
+      # @note We still use the regular coordinator "real" offset management as we want to have
+      #   them as separated as possible because the real seek offset management is also used for
+      #   pausing, filtering and others and should not be impacted by the virtual one
+      #
+      # @note This manager is **not** thread-safe by itself. It should operate from coordinator
+      #   locked locations.
+      class VirtualOffsetManager
+        attr_reader :groups
+
+        # @param topic [String]
+        # @param partition [Integer]
+        #
+        # @note We need topic and partition because we use a seek message (virtual) for real offset
+        #   management. We could keep real message reference but this can be memory consuming
+        #   and not worth it.
+        def initialize(topic, partition)
+          @topic = topic
+          @partition = partition
+          @groups = []
+          @marked = {}
+          @real_offset = -1
+        end
+
+        # Clears the manager for a next collective operation
+        def clear
+          @groups.clear
+          @marked = {}
+          @real_offset = -1
+        end
+
+        # Registers an offset group coming from one virtual consumer. In order to move the real
+        # underlying offset accordingly, we need to make sure to track the virtual consumers
+        # offsets groups independently and only materialize the end result.
+        #
+        # @param offsets_group [Array<Integer>] offsets from one virtual consumer
+        def register(offsets_group)
+          @groups << offsets_group
+
+          offsets_group.each { |offset| @marked[offset] = false }
+        end
+
+        # Marks given message as marked (virtually consumed).
+        # We mark given message offset and other earlier offsets from the same group as done
+        # and we can refresh our real offset representation based on that as it might have changed
+        # to a newer real offset.
+        # @param message [Karafka::Messages::Message] message coming from VP we want to mark
+        def mark(message)
+          offset = message.offset
+
+          group = @groups.find { |reg_group| reg_group.include?(offset) }
+
+          # This case can happen when someone uses MoM and wants to mark message from a previous
+          # batch as consumed. We can add it, since the real offset refresh will point to it
+          unless group
+            group = [offset]
+            @groups << group
+          end
+
+          position = group.index(offset)
+
+          # Mark all previous messages from the same group also as virtually consumed
+          group[0..position].each do |markable_offset|
+            @marked[markable_offset] = true
+          end
+
+          # Recompute the real offset representation
+          materialize_real_offset
+        end
+
+        # Mark all from all groups including the `message`.
+        # Useful when operating in a collapsed state for marking
+        # @param message [Karafka::Messages::Message]
+        def mark_until(message)
+          mark(message)
+
+          @groups.each do |group|
+            group.each do |offset|
+              next if offset > message.offset
+
+              @marked[offset] = true
+            end
+          end
+
+          materialize_real_offset
+        end
+
+        # @return [Array<Integer>] Offsets of messages already marked as consumed virtually
+        def marked
+          @marked.select { |_, status| status }.map(&:first).sort
+        end
+
+        # Is there a real offset we can mark as consumed
+        # @return [Boolean]
+        def markable?
+          !@real_offset.negative?
+        end
+
+        # @return [Messages::Seek] markable message for real offset marking
+        def markable
+          raise Errors::InvalidRealOffsetUsage unless markable?
+
+          Messages::Seek.new(
+            @topic,
+            @partition,
+            @real_offset
+          )
+        end
+
+        private
+
+        # Recomputes the biggest possible real offset we can have.
+        # It picks the the biggest offset that has uninterrupted stream of virtually marked as
+        # consumed because this will be the collective offset.
+        def materialize_real_offset
+          @marked.to_a.sort_by(&:first).each do |offset, marked|
+            break unless marked
+
+            @real_offset = offset
+          end
+
+          @real_offset = (@marked.keys.min - 1) if @real_offset.negative?
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/base.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Base class for all the Pro features
+        class Base < Karafka::Routing::Features::Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/dead_letter_queue/contract.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class DeadLetterQueue < Base
+          # Extended rules for dead letter queue settings
+          class Contract < Contracts::Base
+            configure do |config|
+              config.error_messages = YAML.safe_load(
+                File.read(
+                  File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                )
+              ).fetch('en').fetch('validations').fetch('topic')
+            end
+
+            # Make sure that when we use virtual partitions with DLQ, at least one retry is set
+            # We cannot use VP with DLQ without retries as we in order to provide ordering
+            # warranties on errors with VP, we need to collapse the VPs concurrency and retry
+            # without any indeterministic work
+            virtual do |data, errors|
+              next unless errors.empty?
+
+              dead_letter_queue = data[:dead_letter_queue]
+              virtual_partitions = data[:virtual_partitions]
+
+              next unless dead_letter_queue[:active]
+              next unless virtual_partitions[:active]
+              next if dead_letter_queue[:max_retries].positive?
+
+              [[%i[dead_letter_queue], :with_virtual_partitions]]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/dead_letter_queue.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Enhanced Dead Letter Queue
+        #
+        # This version behaves similar to the regular version but provides ordering warranties and
+        # allows for batch skipping
+        class DeadLetterQueue < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/{builder_extensions.rb → features/delaying/config.rb}

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component.
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
 # All of the commercial components are present in the lib/karafka/pro directory of this
 # repository and their usage requires commercial license agreement.
 #
@@ -11,17 +13,12 @@
 
 module Karafka
   module Pro
-    # Pro routing components
     module Routing
-      # Routing extensions for builder to be able to validate Pro components correct usage
-      module BuilderExtensions
-        # Validate consumer groups with pro contracts
-        # @param block [Proc] routing defining block
-        def draw(&block)
-          super
-
-          each do |consumer_group|
-            ::Karafka::Pro::Contracts::ConsumerGroup.new.validate!(consumer_group.to_h)
+      module Features
+        class Delaying < Base
+          # Delaying feature configuration
+          Config = Struct.new(:active, :delay, keyword_init: true) do
+            alias_method :active?, :active
           end
         end
       end

data/lib/karafka/pro/routing/features/delaying/contract.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Delaying < Base
+          # Contract to validate configuration of the expiring feature
+          class Contract < Contracts::Base
+            configure do |config|
+              config.error_messages = YAML.safe_load(
+                File.read(
+                  File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                )
+              ).fetch('en').fetch('validations').fetch('topic')
+            end
+
+            nested(:delaying) do
+              required(:active) { |val| [true, false].include?(val) }
+              required(:delay) { |val| val.nil? || (val.is_a?(Integer) && val.positive?) }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/delaying/topic.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Delaying < Base
+          # Topic delaying API extensions
+          module Topic
+            # @param delay [Integer, nil] minimum age of a message we want to process
+            def delaying(delay = nil)
+              # Those settings are used for validation
+              @delaying ||= begin
+                config = Config.new(active: !delay.nil?, delay: delay)
+
+                if config.active?
+                  factory = ->(*) { Pro::Processing::Filters::Delayer.new(delay) }
+                  filter(factory)
+                end
+
+                config
+              end
+            end
+
+            # Just an alias for nice API
+            #
+            # @param args [Array] Anything `#delaying` accepts
+            def delay_by(*args)
+              delaying(*args)
+            end
+
+            # @return [Boolean] is a given job delaying
+            def delaying?
+              delaying.active?
+            end
+
+            # @return [Hash] topic with all its native configuration options plus delaying
+            def to_h
+              super.merge(
+                delaying: delaying.to_h
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/delaying.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Delaying allows us to delay processing of certain topics. This is useful when we want
+        # to for any reason to wait until processing data from a topic. It does not sleep and
+        # instead uses pausing to manage delays. This allows us to free up processing resources
+        # and not block the polling thread.
+        #
+        # Delaying is a virtual feature realized via the filters
+        class Delaying < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/expiring/config.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Expiring < Base
+          # Expiring feature configuration
+          Config = Struct.new(:active, :ttl, keyword_init: true) do
+            alias_method :active?, :active
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/expiring/contract.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Expiring < Base
+          # Contract to validate configuration of the expiring feature
+          class Contract < Contracts::Base
+            configure do |config|
+              config.error_messages = YAML.safe_load(
+                File.read(
+                  File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                )
+              ).fetch('en').fetch('validations').fetch('topic')
+            end
+
+            nested(:expiring) do
+              required(:active) { |val| [true, false].include?(val) }
+              required(:ttl) { |val| val.nil? || (val.is_a?(Integer) && val.positive?) }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/expiring/topic.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Expiring < Base
+          # Topic expiring API extensions
+          module Topic
+            # @param ttl [Integer, nil] maximum time in ms a message is considered alive
+            def expiring(ttl = nil)
+              # Those settings are used for validation
+              @expiring ||= begin
+                config = Config.new(active: !ttl.nil?, ttl: ttl)
+
+                if config.active?
+                  factory = ->(*) { Pro::Processing::Filters::Expirer.new(ttl) }
+                  filter(factory)
+                end
+
+                config
+              end
+            end
+
+            # Just an alias for nice API
+            #
+            # @param args [Array] Anything `#expiring` accepts
+            def expire_in(*args)
+              expiring(*args)
+            end
+
+            # @return [Boolean] is a given job expiring
+            def expiring?
+              expiring.active?
+            end
+
+            # @return [Hash] topic with all its native configuration options plus expiring
+            def to_h
+              super.merge(
+                expiring: expiring.to_h
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/expiring.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Expiring allows us to filter out messages that are older than our expectation.
+        # This can also be done in a consumer, but applying the filtering prior to the jobs
+        # enqueuing allows us to improve operations with virtual partitions and limit the number of
+        # not necessary messages being ever seen
+        class Expiring < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/filtering/config.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Filtering < Base
+          # Filtering feature configuration
+          Config = Struct.new(:factories, keyword_init: true) do
+            # @return [Boolean] is this feature in use. Are any filters defined
+            def active?
+              !factories.empty?
+            end
+
+            # @return [Array<Object>] array of filters applicable to a topic partition
+            def filters
+              factories.map(&:call)
+            end
+
+            # @return [Hash] this config hash
+            def to_h
+              super.merge(active: active?)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/filtering/contract.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Filtering < Base
+          # Contract to validate configuration of the filtering feature
+          class Contract < Contracts::Base
+            configure do |config|
+              config.error_messages = YAML.safe_load(
+                File.read(
+                  File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                )
+              ).fetch('en').fetch('validations').fetch('topic')
+            end
+
+            nested(:filtering) do
+              required(:active) { |val| [true, false].include?(val) }
+
+              required(:factories) do |val|
+                val.is_a?(Array) && val.all? { |factory| factory.respond_to?(:call) }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end