karafka 2.4.0 → 2.4.18

data/lib/karafka/pro/routing/features/scheduled_messages/builder.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class ScheduledMessages < Base
+          # Routing extensions for scheduled messages
+          module Builder
+            # Enabled scheduled messages operations and adds needed topics and other stuff.
+            #
+            # @param group_name [String, false] name for scheduled messages topic that is also used
+            #   as a group identifier. Users can have multiple schedule topics flows to prevent key
+            #   collisions, prioritize and do other stuff. `false` if not active.
+            # @param block [Proc] optional reconfiguration of the topics definitions.
+            # @note Namespace for topics should include the divider as it is not automatically
+            #   added.
+            def scheduled_messages(group_name = false, &block)
+              return unless group_name
+
+              # Load zlib only if user enables scheduled messages
+              require 'zlib'
+
+              # We set it to 5 so we have enough space to handle more events. All related topics
+              # should have same partition count.
+              default_partitions = 5
+              msg_cfg = App.config.scheduled_messages
+
+              consumer_group msg_cfg.group_id do
+                # Registers the primary topic that we use to control schedules execution. This is
+                # the one that we use to trigger scheduled messages.
+                messages_topic = topic(group_name) do
+                  instance_eval(&block) if block && block.arity.zero?
+
+                  consumer msg_cfg.consumer_class
+
+                  deserializers(
+                    headers: msg_cfg.deserializers.headers
+                  )
+
+                  # Because the topic method name as well as builder proxy method name is the same
+                  # we need to reference it via target directly
+                  target.scheduled_messages(true)
+
+                  # We manage offsets directly because messages can have both schedules and
+                  # commands and we need to apply them only when we need to
+                  manual_offset_management(true)
+
+                  # We use multi-batch operations and in-memory state for schedules. This needs to
+                  # always operate without re-creation.
+                  consumer_persistence(true)
+
+                  # This needs to be enabled for the eof to work correctly
+                  kafka('enable.partition.eof': true, inherit: true)
+                  eofed(true)
+
+                  # Since this is a topic that gets replayed because of schedule management, we do
+                  # want to get more data faster during recovery
+                  max_messages(10_000)
+
+                  max_wait_time(1_000)
+
+                  # This is a setup that should allow messages to be compacted fairly fast. Since
+                  # each dispatched message should be removed via tombstone, they do not have to
+                  # be present in the topic for too long.
+                  config(
+                    partitions: default_partitions,
+                    # Will ensure, that after tombstone is present, given scheduled message, that
+                    # has been dispatched is removed by Kafka
+                    'cleanup.policy': 'compact',
+                    # When 10% or more dispatches are done, compact data
+                    'min.cleanable.dirty.ratio': 0.1,
+                    # Frequent segment rotation to support intense compaction
+                    'segment.ms': 3_600_000,
+                    'delete.retention.ms': 3_600_000,
+                    'segment.bytes': 52_428_800
+                  )
+
+                  # This is the core of execution. Since we dispatch data in time intervals, we
+                  # need to be able to do this even when no new data is coming
+                  periodic(
+                    interval: msg_cfg.interval,
+                    during_pause: false,
+                    during_retry: false
+                  )
+
+                  # If this is the direct schedules redefinition style, we run it
+                  # The second one (see end of this method) allows for linear reconfiguration of
+                  # both the topics
+                  instance_eval(&block) if block && block.arity.zero?
+                end
+
+                # Holds states of scheduler per each of the partitions since they tick
+                # independently. We only hold future statistics not to have to deal with
+                # any type of state restoration
+                states_topic = topic("#{group_name}#{msg_cfg.states_postfix}") do
+                  active(false)
+                  target.scheduled_messages(true)
+                  config(
+                    partitions: default_partitions,
+                    'cleanup.policy': 'compact',
+                    'min.cleanable.dirty.ratio': 0.1,
+                    'segment.ms': 3_600_000,
+                    'delete.retention.ms': 3_600_000,
+                    'segment.bytes': 52_428_800
+                  )
+                  deserializers(
+                    payload: msg_cfg.deserializers.payload
+                  )
+                end
+
+                yield(messages_topic, states_topic) if block && block.arity.positive?
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/scheduled_messages/config.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class ScheduledMessages < Base
+          # Scheduled messages configuration
+          Config = Struct.new(
+            :active,
+            keyword_init: true
+          ) { alias_method :active?, :active }
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/scheduled_messages/contracts/topic.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class ScheduledMessages < Base
+          # Namespace for scheduled messages contracts
+          module Contracts
+            # Rules around scheduled messages settings
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+              end
+
+              nested(:scheduled_messages) do
+                required(:active) { |val| [true, false].include?(val) }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/scheduled_messages/proxy.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class ScheduledMessages < Base
+          # Routing proxy extensions for scheduled messages
+          module Proxy
+            include Builder
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/scheduled_messages/topic.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class ScheduledMessages < Base
+          # Topic extensions to be able to check if given topic is a scheduled messages topic
+          # Please note, that this applies to both the schedules topic and logs topic
+          module Topic
+            # @param active [Boolean] should this topic be considered related to scheduled messages
+            def scheduled_messages(active = false)
+              @scheduled_messages ||= Config.new(active: active)
+            end
+
+            # @return [Boolean] is this an ActiveJob topic
+            def scheduled_messages?
+              scheduled_messages.active?
+            end
+
+            # @return [Hash] topic with all its native configuration options plus scheduled
+            # messages namespace settings
+            def to_h
+              super.merge(
+                scheduled_messages: scheduled_messages.to_h
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/scheduled_messages.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Scheduled messages functionality routing namespace
+        class ScheduledMessages < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/swarm/config.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/routing/features/swarm/contracts/routing.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/routing/features/swarm/contracts/topic.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/routing/features/swarm/topic.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/routing/features/swarm.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/routing/features/throttling/config.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/routing/features/throttling/contracts/topic.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/routing/features/throttling/topic.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/routing/features/throttling.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/routing/features/virtual_partitions/config.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -22,6 +14,7 @@ module Karafka
             :partitioner,
             :max_partitions,
             :offset_metadata_strategy,
+            :reducer,
             keyword_init: true
           ) { alias_method :active?, :active }
         end

data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -31,6 +23,7 @@ module Karafka
               nested(:virtual_partitions) do
                 required(:active) { |val| [true, false].include?(val) }
                 required(:partitioner) { |val| val.nil? || val.respond_to?(:call) }
+                required(:reducer) { |val| val.respond_to?(:call) }
                 required(:max_partitions) { |val| val.is_a?(Integer) && val >= 1 }
                 required(:offset_metadata_strategy) { |val| %i[exact current].include?(val) }
               end

data/lib/karafka/pro/routing/features/virtual_partitions/topic.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -26,18 +18,24 @@ module Karafka
             # @param offset_metadata_strategy [Symbol] how we should match the metadata for the
             #   offset. `:exact` will match the offset matching metadata and `:current` will select
             #   the most recently reported metadata
+            # @param reducer [nil, #call] reducer for VPs key. It allows for using a custom
+            #   reducer to achieve enhanced parallelization when the default reducer is not enough.
             # @return [VirtualPartitions] method that allows to set the virtual partitions details
             #   during the routing configuration and then allows to retrieve it
             def virtual_partitions(
               max_partitions: Karafka::App.config.concurrency,
               partitioner: nil,
-              offset_metadata_strategy: :current
+              offset_metadata_strategy: :current,
+              reducer: nil
             )
               @virtual_partitions ||= Config.new(
                 active: !partitioner.nil?,
                 max_partitions: max_partitions,
                 partitioner: partitioner,
-                offset_metadata_strategy: offset_metadata_strategy
+                offset_metadata_strategy: offset_metadata_strategy,
+                # If no reducer provided, we use this one. It just runs a modulo on the sum of
+                # a stringified version, providing fairly good distribution.
+                reducer: reducer || ->(virtual_key) { virtual_key.to_s.sum % max_partitions }
               )
             end
 

data/lib/karafka/pro/routing/features/virtual_partitions.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro