karafka 2.2.14 → 2.3.0.alpha1

data/lib/karafka/pro/routing/features/multiplexing/proxy.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Multiplexing < Base
+          # Allows for multiplexing setup inside a consumer group definition
+          module Proxy
+            # @param min [Integer, nil] min multiplexing count or nil to set it to max, effectively
+            #   disabling dynamic multiplexing
+            # @param max [Integer] max multiplexing count
+            # @param boot [Integer] how many listeners should we start during boot by default
+            def multiplexing(min: nil, max: 1, boot: nil)
+              @target.current_subscription_group_details.merge!(
+                multiplexing_min: min || max,
+                multiplexing_max: max,
+                # Picks half of max by default as long as possible. Otherwise goes with min
+                multiplexing_boot: boot || [min || max, (max / 2)].max
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/multiplexing/subscription_group.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Multiplexing < Base
+          # Adds methods needed for the multiplexing to work
+          module SubscriptionGroup
+            # @return [Config] multiplexing config
+            def multiplexing
+              @multiplexing ||= begin
+                max = @details.fetch(:multiplexing_max, 1)
+                min = @details.fetch(:multiplexing_min, max)
+                boot = @details.fetch(:multiplexing_boot, max / 2)
+                active = max > 1
+
+                Config.new(active: active, min: min, max: max, boot: boot)
+              end
+            end
+
+            # @return [Boolean] is multiplexing active
+            def multiplexing?
+              multiplexing.active?
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/multiplexing/subscription_groups_builder.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Multiplexing < Base
+          # Expands the builder to multiply multiplexed groups
+          module SubscriptionGroupsBuilder
+            # Takes into consideration multiplexing and builds the more groups
+            #
+            # @param topics_array [Array<Routing::Topic>] group of topics that have the same
+            #   settings and can use the same connection
+            # @return [Array<Array<Routing::Topics>>] expanded groups
+            def expand(topics_array)
+              factor = topics_array.first.subscription_group_details.fetch(:multiplexing_max, 1)
+
+              Array.new(factor) do |i|
+                ::Karafka::Routing::Topics.new(
+                  i.zero? ? topics_array : topics_array.map(&:dup)
+                )
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/multiplexing.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Namespace for Pro routing enhancements
+    module Routing
+      # Namespace for additional Pro features
+      module Features
+        # Multiplexing allows for creating multiple subscription groups for the same topic inside
+        # of the same subscription group allowing for better parallelism with limited number
+        # of processes
+        class Multiplexing < Base
+          class << self
+            # @param _config [Karafka::Core::Configurable::Node] app config node
+            def pre_setup(_config)
+              # Make sure we use proper unique validator for topics definitions
+              ::Karafka::Contracts::ConsumerGroup.singleton_class.prepend(
+                Patches::Contracts::ConsumerGroup
+              )
+            end
+
+            # If needed installs the needed listener and initializes tracker
+            #
+            # @param _config [Karafka::Core::Configurable::Node] app config
+            def post_setup(_config)
+              ::Karafka::App.monitor.subscribe('app.running') do
+                # Do not install the manager and listener to control multiplexing unless there is
+                # multiplexing enabled and it is dynamic.
+                # We only need to control multiplexing when it is in a dynamic state
+                next unless ::Karafka::App
+                            .subscription_groups
+                            .values
+                            .flat_map(&:itself)
+                            .any? { |sg| sg.multiplexing? && sg.multiplexing.dynamic? }
+
+                # Subscribe for events and possibility to manage via the Pro connection manager
+                # that supports multiplexing
+                ::Karafka.monitor.subscribe(
+                  ::Karafka::Pro::Connection::Multiplexing::Listener.new
+                )
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/non_blocking_job/topic.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class NonBlockingJob < Base
+          # Non-Blocking Jobs topic API extensions
+          module Topic
+            # @param args [Array] anything accepted by the `#long_running_job` API
+            def non_blocking_job(*args)
+              long_running_job(*args)
+            end
+
+            alias non_blocking non_blocking_job
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/non_blocking_job.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Namespace for Pro routing enhancements
+    module Routing
+      # Namespace for additional Pro features
+      module Features
+        # Non Blocking Job is just an alias for LRJ.
+        #
+        # We however have it as a separate feature because its use-case may vary from LRJ.
+        #
+        # While LRJ is used mainly for long-running jobs that would take more than max poll
+        # interval time, non-blocking can be applied to make sure that we do not wait with polling
+        # of different partitions and topics that are subscribed together.
+        #
+        # This effectively allows for better resources utilization
+        #
+        # All the underlying code is the same but use-case is different and this should be
+        # reflected in the routing, hence this "virtual" feature.
+        class NonBlockingJob < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/offset_metadata/config.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class OffsetMetadata < Base
+          # Config for commit metadata feature
+          Config = Struct.new(
+            :active,
+            :deserializer,
+            :cache,
+            keyword_init: true
+          ) do
+            alias_method :active?, :active
+            alias_method :cache?, :cache
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/offset_metadata/contracts/topic.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class OffsetMetadata < Base
+          # Namespace for offset metadata feature contracts
+          module Contracts
+            # Contract to validate configuration of the expiring feature
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+              end
+
+              nested(:offset_metadata) do
+                required(:active) { |val| val == true }
+                required(:cache) { |val| [true, false].include?(val) }
+                required(:deserializer) { |val| val.respond_to?(:call) }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/offset_metadata/topic.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # This feature allows for saving and retrieving offset metadata with custom deserialization
+        # support. It allows for storing extra data during commits that can be then used to alter
+        # the processing flow after a rebalance.
+        #
+        # @note Because this feature has zero performance impact and makes no queries to Kafka
+        #   unless requested, it is always enabled.
+        class OffsetMetadata < Base
+          # Empty string not to create it on each deserialization
+          EMPTY_STRING = ''
+
+          # Default deserializer just ensures we always get a string as without metadata by
+          # default it would be nil
+          STRING_DESERIALIZER = ->(raw_metadata) { raw_metadata || EMPTY_STRING }.freeze
+
+          private_constant :STRING_DESERIALIZER, :EMPTY_STRING
+
+          # Commit Metadata API extensions
+          module Topic
+            # @param cache [Boolean] should we cache the response until rebalance
+            # @param deserializer [#call] deserializer that will get raw data and should return
+            #   deserialized metadata
+            # @return [Config] this feature config
+            def offset_metadata(cache: true, deserializer: STRING_DESERIALIZER)
+              @offset_metadata ||= Config.new(
+                active: true,
+                cache: cache,
+                deserializer: deserializer
+              )
+            end
+
+            # @return [true] is offset metadata active (it always is)
+            def offset_metadata?
+              offset_metadata.active?
+            end
+
+            # @return [Hash] topic with all its native configuration options plus offset metadata
+            #   settings
+            def to_h
+              super.merge(
+                offset_metadata: offset_metadata.to_h
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/offset_metadata.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Offset Metadata Support with a custom deserializer
+        class OffsetMetadata < Base
+          class << self
+            # If needed installs the needed listener and initializes tracker
+            #
+            # @param _config [Karafka::Core::Configurable::Node] app config
+            def post_setup(_config)
+              ::Karafka::App.monitor.subscribe('app.running') do
+                # Initialize the tracker prior to becoming multi-threaded
+                ::Karafka::Processing::InlineInsights::Tracker.instance
+
+                # Subscribe to the statistics reports and collect them
+                ::Karafka.monitor.subscribe(
+                  ::Karafka::Pro::Processing::OffsetMetadata::Listener.new
+                )
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb

@@ -46,7 +46,11 @@ module Karafka
                   next unless errors.empty?
 
                   regexp_strings = data[:patterns].map { |pattern| pattern.fetch(:regexp_string) }
+                  regexp_names = data[:patterns].map { |pattern| pattern.fetch(:name) }
 
+                  # If all names are the same for the same regexp, it means its a multiplex and
+                  # we can allow it
+                  next if regexp_names.uniq.size == 1 && regexp_strings.uniq.size == 1
                   next if regexp_strings.empty?
                   next if regexp_strings.uniq.size == regexp_strings.size
 

data/lib/karafka/pro/routing/features/patterns/detector.rb

@@ -22,6 +22,12 @@ module Karafka
           # @note This is NOT thread-safe and should run in a thread-safe context that warranties
           #   that there won't be any race conditions
           class Detector
+            # Mutex for making sure that we do not modify same consumer group in runtime at the
+            # same time from multiple subscription groups if they operate in a multiplexed mode
+            MUTEX = Mutex.new
+
+            private_constant :MUTEX
+
             # Checks if the provided topic matches any of the patterns and when detected, expands
             # the routing with it.
             #
@@ -29,16 +35,18 @@ module Karafka
             #   topics.
             # @param new_topic [String] new topic that we have detected
             def expand(sg_topics, new_topic)
-              sg_topics
-                .map(&:patterns)
-                .select(&:active?)
-                .select(&:matcher?)
-                .map(&:pattern)
-                .then { |pts| pts.empty? ? return : pts }
-                .then { |pts| Patterns.new(pts) }
-                .find(new_topic)
-                .then { |pattern| pattern || return }
-                .then { |pattern| install(pattern, new_topic, sg_topics) }
+              MUTEX.synchronize do
+                sg_topics
+                  .map(&:patterns)
+                  .select(&:active?)
+                  .select(&:matcher?)
+                  .map(&:pattern)
+                  .then { |pts| pts.empty? ? return : pts }
+                  .then { |pts| Patterns.new(pts) }
+                  .find(new_topic)
+                  .then { |pattern| pattern || return }
+                  .then { |pattern| install(pattern, new_topic, sg_topics) }
+              end
             end
 
             private

data/lib/karafka/pro/routing/features/periodic_job/config.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class PeriodicJob < Base
+          # Config for periodics topics feature
+          Config = Struct.new(
+            :active,
+            :during_pause,
+            :during_retry,
+            :interval,
+            :materialized,
+            keyword_init: true
+          ) do
+            alias_method :active?, :active
+            alias_method :during_pause?, :during_pause
+            alias_method :during_retry?, :during_retry
+            alias_method :materialized?, :materialized
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/periodic_job/contracts/topic.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class PeriodicJob < Base
+          # Namespace for periodics messages contracts
+          module Contracts
+            # Contract to validate configuration of the periodics feature
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+              end
+
+              nested(:periodic_job) do
+                required(:active) { |val| [true, false].include?(val) }
+                required(:interval) { |val| val.is_a?(Integer) && val >= 100 }
+                required(:during_pause) { |val| [true, false].include?(val) }
+                required(:during_retry) { |val| [true, false].include?(val) }
+                required(:materialized) { |val| [true, false].include?(val) }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/periodic_job/topic.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class PeriodicJob < Base
+          # Periodic topic action flows extensions
+          module Topic
+            # Defines topic as periodic. Periodic topics consumers will invoke `#tick` with each
+            # poll where messages were not received.
+            # @param active [Boolean] should ticking happen for this topic assignments.
+            # @param interval [Integer] minimum interval to run periodic jobs on given topic.
+            # @param during_pause [Boolean, nil] Should periodic jobs run when partition is paused.
+            #   It is set to `nil` by default allowing for detection when this value is not
+            #   configured but should be built dynamically based on LRJ status.
+            # @param during_retry [Boolean, nil] Should we run when there was an error and we are
+            #   in a retry flow. Please note that for this to work, `during_pause` also needs to be
+            #   set to true as errors retry happens after pause.
+            def periodic_job(
+              active = false,
+              interval: nil,
+              during_pause: nil,
+              during_retry: nil
+            )
+              @periodic_job ||= begin
+                # Set to active if any of the values was configured
+                active = true unless interval.nil?
+                active = true unless during_pause.nil?
+                active = true unless during_retry.nil?
+                # Default is not to retry during retry flow
+                during_retry = false if during_retry.nil?
+
+                # If no interval, use default
+                interval ||= ::Karafka::App.config.internal.tick_interval
+
+                Config.new(
+                  active: active,
+                  interval: interval,
+                  during_pause: during_pause,
+                  during_retry: during_retry,
+                  # This is internal setting for state management, not part of the configuration
+                  # Do not overwrite.
+                  # If `during_pause` is explicit, we do not select it based on LRJ setup and we
+                  # consider if fully ready out of the box
+                  materialized: !during_pause.nil?
+                )
+              end
+
+              return @periodic_job if @periodic_job.materialized?
+              return @periodic_job unless @long_running_job
+
+              # If not configured in any way, we want not to process during pause for LRJ.
+              # LRJ pauses by default when processing and during this time we do not want to
+              # tick at all. This prevents us from running periodic jobs while LRJ jobs are
+              # running. This of course has a side effect of not running when paused for any
+              # other reason but it is a compromise in the default settings
+              @periodic_job.during_pause = !long_running_job?
+              @periodic_job.materialized = true
+
+              @periodic_job
+            end
+
+            alias periodic periodic_job
+
+            # @return [Boolean] is periodics active
+            def periodic_job?
+              periodic_job.active?
+            end
+
+            # @return [Hash] topic with all its native configuration options plus periodics flows
+            #   settings
+            def to_h
+              super.merge(
+                periodic_job: periodic_job.to_h
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end