karafka 2.2.13 → 2.3.0.alpha1

data/lib/karafka/pro/routing/features/non_blocking_job/topic.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class NonBlockingJob < Base
+          # Non-Blocking Jobs topic API extensions
+          module Topic
+            # @param args [Array] anything accepted by the `#long_running_job` API
+            def non_blocking_job(*args)
+              long_running_job(*args)
+            end
+
+            alias non_blocking non_blocking_job
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/non_blocking_job.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Namespace for Pro routing enhancements
+    module Routing
+      # Namespace for additional Pro features
+      module Features
+        # Non Blocking Job is just an alias for LRJ.
+        #
+        # We however have it as a separate feature because its use-case may vary from LRJ.
+        #
+        # While LRJ is used mainly for long-running jobs that would take more than max poll
+        # interval time, non-blocking can be applied to make sure that we do not wait with polling
+        # of different partitions and topics that are subscribed together.
+        #
+        # This effectively allows for better resources utilization
+        #
+        # All the underlying code is the same but use-case is different and this should be
+        # reflected in the routing, hence this "virtual" feature.
+        class NonBlockingJob < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/offset_metadata/config.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class OffsetMetadata < Base
+          # Config for commit metadata feature
+          Config = Struct.new(
+            :active,
+            :deserializer,
+            :cache,
+            keyword_init: true
+          ) do
+            alias_method :active?, :active
+            alias_method :cache?, :cache
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/offset_metadata/contracts/topic.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class OffsetMetadata < Base
+          # Namespace for offset metadata feature contracts
+          module Contracts
+            # Contract to validate configuration of the expiring feature
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+              end
+
+              nested(:offset_metadata) do
+                required(:active) { |val| val == true }
+                required(:cache) { |val| [true, false].include?(val) }
+                required(:deserializer) { |val| val.respond_to?(:call) }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/offset_metadata/topic.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # This feature allows for saving and retrieving offset metadata with custom deserialization
+        # support. It allows for storing extra data during commits that can be then used to alter
+        # the processing flow after a rebalance.
+        #
+        # @note Because this feature has zero performance impact and makes no queries to Kafka
+        #   unless requested, it is always enabled.
+        class OffsetMetadata < Base
+          # Empty string not to create it on each deserialization
+          EMPTY_STRING = ''
+
+          # Default deserializer just ensures we always get a string as without metadata by
+          # default it would be nil
+          STRING_DESERIALIZER = ->(raw_metadata) { raw_metadata || EMPTY_STRING }.freeze
+
+          private_constant :STRING_DESERIALIZER, :EMPTY_STRING
+
+          # Commit Metadata API extensions
+          module Topic
+            # @param cache [Boolean] should we cache the response until rebalance
+            # @param deserializer [#call] deserializer that will get raw data and should return
+            #   deserialized metadata
+            # @return [Config] this feature config
+            def offset_metadata(cache: true, deserializer: STRING_DESERIALIZER)
+              @offset_metadata ||= Config.new(
+                active: true,
+                cache: cache,
+                deserializer: deserializer
+              )
+            end
+
+            # @return [true] is offset metadata active (it always is)
+            def offset_metadata?
+              offset_metadata.active?
+            end
+
+            # @return [Hash] topic with all its native configuration options plus offset metadata
+            #   settings
+            def to_h
+              super.merge(
+                offset_metadata: offset_metadata.to_h
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/offset_metadata.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Offset Metadata Support with a custom deserializer
+        class OffsetMetadata < Base
+          class << self
+            # If needed installs the needed listener and initializes tracker
+            #
+            # @param _config [Karafka::Core::Configurable::Node] app config
+            def post_setup(_config)
+              ::Karafka::App.monitor.subscribe('app.running') do
+                # Initialize the tracker prior to becoming multi-threaded
+                ::Karafka::Processing::InlineInsights::Tracker.instance
+
+                # Subscribe to the statistics reports and collect them
+                ::Karafka.monitor.subscribe(
+                  ::Karafka::Pro::Processing::OffsetMetadata::Listener.new
+                )
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb

@@ -46,7 +46,11 @@ module Karafka
                   next unless errors.empty?
 
                   regexp_strings = data[:patterns].map { |pattern| pattern.fetch(:regexp_string) }
+                  regexp_names = data[:patterns].map { |pattern| pattern.fetch(:name) }
 
+                  # If all names are the same for the same regexp, it means its a multiplex and
+                  # we can allow it
+                  next if regexp_names.uniq.size == 1 && regexp_strings.uniq.size == 1
                   next if regexp_strings.empty?
                   next if regexp_strings.uniq.size == regexp_strings.size
 

data/lib/karafka/pro/routing/features/patterns/detector.rb

@@ -22,6 +22,12 @@ module Karafka
           # @note This is NOT thread-safe and should run in a thread-safe context that warranties
           #   that there won't be any race conditions
           class Detector
+            # Mutex for making sure that we do not modify same consumer group in runtime at the
+            # same time from multiple subscription groups if they operate in a multiplexed mode
+            MUTEX = Mutex.new
+
+            private_constant :MUTEX
+
             # Checks if the provided topic matches any of the patterns and when detected, expands
             # the routing with it.
             #
@@ -29,16 +35,18 @@ module Karafka
             #   topics.
             # @param new_topic [String] new topic that we have detected
             def expand(sg_topics, new_topic)
-              sg_topics
-                .map(&:patterns)
-                .select(&:active?)
-                .select(&:matcher?)
-                .map(&:pattern)
-                .then { |pts| pts.empty? ? return : pts }
-                .then { |pts| Patterns.new(pts) }
-                .find(new_topic)
-                .then { |pattern| pattern || return }
-                .then { |pattern| install(pattern, new_topic, sg_topics) }
+              MUTEX.synchronize do
+                sg_topics
+                  .map(&:patterns)
+                  .select(&:active?)
+                  .select(&:matcher?)
+                  .map(&:pattern)
+                  .then { |pts| pts.empty? ? return : pts }
+                  .then { |pts| Patterns.new(pts) }
+                  .find(new_topic)
+                  .then { |pattern| pattern || return }
+                  .then { |pattern| install(pattern, new_topic, sg_topics) }
+              end
             end
 
             private

data/lib/karafka/pro/routing/features/periodic_job/config.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class PeriodicJob < Base
+          # Config for periodics topics feature
+          Config = Struct.new(
+            :active,
+            :during_pause,
+            :during_retry,
+            :interval,
+            :materialized,
+            keyword_init: true
+          ) do
+            alias_method :active?, :active
+            alias_method :during_pause?, :during_pause
+            alias_method :during_retry?, :during_retry
+            alias_method :materialized?, :materialized
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/periodic_job/contracts/topic.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class PeriodicJob < Base
+          # Namespace for periodics messages contracts
+          module Contracts
+            # Contract to validate configuration of the periodics feature
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+              end
+
+              nested(:periodic_job) do
+                required(:active) { |val| [true, false].include?(val) }
+                required(:interval) { |val| val.is_a?(Integer) && val >= 100 }
+                required(:during_pause) { |val| [true, false].include?(val) }
+                required(:during_retry) { |val| [true, false].include?(val) }
+                required(:materialized) { |val| [true, false].include?(val) }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/periodic_job/topic.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class PeriodicJob < Base
+          # Periodic topic action flows extensions
+          module Topic
+            # Defines topic as periodic. Periodic topics consumers will invoke `#tick` with each
+            # poll where messages were not received.
+            # @param active [Boolean] should ticking happen for this topic assignments.
+            # @param interval [Integer] minimum interval to run periodic jobs on given topic.
+            # @param during_pause [Boolean, nil] Should periodic jobs run when partition is paused.
+            #   It is set to `nil` by default allowing for detection when this value is not
+            #   configured but should be built dynamically based on LRJ status.
+            # @param during_retry [Boolean, nil] Should we run when there was an error and we are
+            #   in a retry flow. Please note that for this to work, `during_pause` also needs to be
+            #   set to true as errors retry happens after pause.
+            def periodic_job(
+              active = false,
+              interval: nil,
+              during_pause: nil,
+              during_retry: nil
+            )
+              @periodic_job ||= begin
+                # Set to active if any of the values was configured
+                active = true unless interval.nil?
+                active = true unless during_pause.nil?
+                active = true unless during_retry.nil?
+                # Default is not to retry during retry flow
+                during_retry = false if during_retry.nil?
+
+                # If no interval, use default
+                interval ||= ::Karafka::App.config.internal.tick_interval
+
+                Config.new(
+                  active: active,
+                  interval: interval,
+                  during_pause: during_pause,
+                  during_retry: during_retry,
+                  # This is internal setting for state management, not part of the configuration
+                  # Do not overwrite.
+                  # If `during_pause` is explicit, we do not select it based on LRJ setup and we
+                  # consider if fully ready out of the box
+                  materialized: !during_pause.nil?
+                )
+              end
+
+              return @periodic_job if @periodic_job.materialized?
+              return @periodic_job unless @long_running_job
+
+              # If not configured in any way, we want not to process during pause for LRJ.
+              # LRJ pauses by default when processing and during this time we do not want to
+              # tick at all. This prevents us from running periodic jobs while LRJ jobs are
+              # running. This of course has a side effect of not running when paused for any
+              # other reason but it is a compromise in the default settings
+              @periodic_job.during_pause = !long_running_job?
+              @periodic_job.materialized = true
+
+              @periodic_job
+            end
+
+            alias periodic periodic_job
+
+            # @return [Boolean] is periodics active
+            def periodic_job?
+              periodic_job.active?
+            end
+
+            # @return [Hash] topic with all its native configuration options plus periodics flows
+            #   settings
+            def to_h
+              super.merge(
+                periodic_job: periodic_job.to_h
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/periodic_job.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Feature allowing to run consumer operations even when no data is present on periodic
+        # interval.
+        # This allows for advanced window-based operations regardless of income of new data and
+        # other advanced cases where the consumer is needed even when no data is coming
+        class PeriodicJob < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/virtual_partitions/config.rb

@@ -21,6 +21,7 @@ module Karafka
             :active,
             :partitioner,
             :max_partitions,
+            :offset_metadata_strategy,
             keyword_init: true
           ) { alias_method :active?, :active }
         end

data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb

@@ -32,6 +32,7 @@ module Karafka
                 required(:active) { |val| [true, false].include?(val) }
                 required(:partitioner) { |val| val.nil? || val.respond_to?(:call) }
                 required(:max_partitions) { |val| val.is_a?(Integer) && val >= 1 }
+                required(:offset_metadata_strategy) { |val| %i[exact current].include?(val) }
               end
 
               # When virtual partitions are defined, partitioner needs to respond to `#call` and it

data/lib/karafka/pro/routing/features/virtual_partitions/topic.rb

@@ -23,16 +23,21 @@ module Karafka
             #   create more work than workers. When less, can ensure we have spare resources to
             #   process other things in parallel.
             # @param partitioner [nil, #call] nil or callable partitioner
+            # @param offset_metadata_strategy [Symbol] how we should match the metadata for the
+            #   offset. `:exact` will match the offset matching metadata and `:current` will select
+            #   the most recently reported metadata
             # @return [VirtualPartitions] method that allows to set the virtual partitions details
             #   during the routing configuration and then allows to retrieve it
             def virtual_partitions(
               max_partitions: Karafka::App.config.concurrency,
-              partitioner: nil
+              partitioner: nil,
+              offset_metadata_strategy: :current
             )
               @virtual_partitions ||= Config.new(
                 active: !partitioner.nil?,
                 max_partitions: max_partitions,
-                partitioner: partitioner
+                partitioner: partitioner,
+                offset_metadata_strategy: offset_metadata_strategy
               )
             end
 

data/lib/karafka/process.rb

@@ -25,9 +25,11 @@ module Karafka
       #     Karafka.logger.info('Log something here')
       #     exit
       #   end
-      define_method :"on_#{signal.to_s.downcase}" do |&block|
-        @callbacks[signal] << block
-      end
+      class_eval <<~RUBY, __FILE__, __LINE__ + 1
+        def on_#{signal.to_s.downcase}(&block)
+          @callbacks[:#{signal}] << block
+        end
+      RUBY
     end
 
     # Creates an instance of process and creates empty hash for callbacks

data/lib/karafka/processing/coordinator.rb

@@ -10,8 +10,12 @@ module Karafka
     #   listener thread, but we go with thread-safe by default for all not to worry about potential
     #   future mistakes.
     class Coordinator
+      extend Forwardable
+
       attr_reader :pause_tracker, :seek_offset, :topic, :partition
 
+      def_delegators :@pause_tracker, :attempt, :paused?
+
       # @param topic [Karafka::Routing::Topic]
       # @param partition [Integer]
       # @param pause_tracker [Karafka::TimeTrackers::Pause] pause tracker for given topic partition
@@ -149,7 +153,7 @@ module Karafka
 
       # @return [Boolean] are we in a pause that was initiated by the user
       def manual_pause?
-        @pause_tracker.paused? && @manual_pause
+        paused? && @manual_pause
       end
 
       # Marks seek as manual for coordination purposes