karafka 2.5.0.rc2 → 2.5.1.beta1

data/lib/karafka/pro/routing/features/expiring/topic.rb

@@ -10,6 +10,15 @@ module Karafka
         class Expiring < Base
           # Topic expiring API extensions
           module Topic
+            # This method calls the parent class initializer and then sets up the
+            # extra instance variable to nil. The explicit initialization
+            # to nil is included as an optimization for Ruby's object shapes system,
+            # which improves memory layout and access performance.
+            def initialize(...)
+              super
+              @expiring = nil
+            end
+
             # @param ttl [Integer, nil] maximum time in ms a message is considered alive
             def expiring(ttl = nil)
               # Those settings are used for validation

data/lib/karafka/pro/routing/features/filtering/contracts/topic.rb

@@ -17,7 +17,7 @@ module Karafka
                   File.read(
                     File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
                   )
-                ).fetch('en').fetch('validations').fetch('topic')
+                ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
               end
 
               nested(:filtering) do

data/lib/karafka/pro/routing/features/filtering/topic.rb

@@ -10,6 +10,15 @@ module Karafka
         class Filtering < Base
           # Filtering feature topic extensions
           module Topic
+            # This method calls the parent class initializer and then sets up the
+            # extra instance variable to nil. The explicit initialization
+            # to nil is included as an optimization for Ruby's object shapes system,
+            # which improves memory layout and access performance.
+            def initialize(...)
+              super
+              @filtering = nil
+            end
+
             # @param factory [#call, nil] Callable that can produce new filters instances per
             #   assigned topic partition. nil as default so this feature is disabled
             def filter(factory = nil)

data/lib/karafka/pro/routing/features/inline_insights/contracts/topic.rb

@@ -17,7 +17,7 @@ module Karafka
                   File.read(
                     File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
                   )
-                ).fetch('en').fetch('validations').fetch('topic')
+                ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
               end
 
               nested :inline_insights do

data/lib/karafka/pro/routing/features/inline_insights/topic.rb

@@ -10,6 +10,15 @@ module Karafka
         class InlineInsights < Base
           # Routing topic inline insights API
           module Topic
+            # This method calls the parent class initializer and then sets up the
+            # extra instance variable to nil. The explicit initialization
+            # to nil is included as an optimization for Ruby's object shapes system,
+            # which improves memory layout and access performance.
+            def initialize(...)
+              super
+              @inline_insights = nil
+            end
+
             # @param active [Boolean] should inline insights be activated
             # @param required [Boolean] are the insights required to operate
             def inline_insights(active = -1, required: -1)

data/lib/karafka/pro/routing/features/long_running_job/contracts/topic.rb

@@ -17,7 +17,7 @@ module Karafka
                   File.read(
                     File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
                   )
-                ).fetch('en').fetch('validations').fetch('topic')
+                ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
               end
 
               nested(:long_running_job) do

data/lib/karafka/pro/routing/features/long_running_job/topic.rb

@@ -10,6 +10,15 @@ module Karafka
         class LongRunningJob < Base
           # Long-Running Jobs topic API extensions
           module Topic
+            # This method calls the parent class initializer and then sets up the
+            # extra instance variable to nil. The explicit initialization
+            # to nil is included as an optimization for Ruby's object shapes system,
+            # which improves memory layout and access performance.
+            def initialize(...)
+              super
+              @long_running_job = nil
+            end
+
             # @param active [Boolean] do we want to enable long-running job feature for this topic
             def long_running_job(active = false)
               @long_running_job ||= Config.new(active: active)

data/lib/karafka/pro/routing/features/multiplexing/contracts/topic.rb

@@ -21,7 +21,7 @@ module Karafka
                   File.read(
                     File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
                   )
-                ).fetch('en').fetch('validations').fetch('topic')
+                ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
               end
 
               nested(:subscription_group_details) do

data/lib/karafka/pro/routing/features/multiplexing.rb

@@ -17,7 +17,7 @@ module Karafka
             # @param _config [Karafka::Core::Configurable::Node] app config node
             def pre_setup(_config)
               # Make sure we use proper unique validator for topics definitions
-              ::Karafka::Contracts::ConsumerGroup.singleton_class.prepend(
+              ::Karafka::Routing::Contracts::ConsumerGroup.singleton_class.prepend(
                 Patches::Contracts::ConsumerGroup
               )
             end

data/lib/karafka/pro/routing/features/offset_metadata/contracts/topic.rb

@@ -17,7 +17,7 @@ module Karafka
                   File.read(
                     File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
                   )
-                ).fetch('en').fetch('validations').fetch('topic')
+                ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
               end
 
               nested(:offset_metadata) do

data/lib/karafka/pro/routing/features/offset_metadata/topic.rb

@@ -25,6 +25,15 @@ module Karafka
 
           # Commit Metadata API extensions
           module Topic
+            # This method calls the parent class initializer and then sets up the
+            # extra instance variable to nil. The explicit initialization
+            # to nil is included as an optimization for Ruby's object shapes system,
+            # which improves memory layout and access performance.
+            def initialize(...)
+              super
+              @offset_metadata = nil
+            end
+
             # @param cache [Boolean] should we cache the response until rebalance
             # @param deserializer [#call] deserializer that will get raw data and should return
             #   deserialized metadata

data/lib/karafka/pro/routing/features/parallel_segments/contracts/consumer_group.rb

@@ -17,7 +17,7 @@ module Karafka
                   File.read(
                     File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
                   )
-                ).fetch('en').fetch('validations').fetch('consumer_group')
+                ).fetch('en').fetch('validations').fetch('routing').fetch('consumer_group')
 
                 nested(:parallel_segments) do
                   required(:active) { |val| [true, false].include?(val) }

data/lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb

@@ -16,7 +16,7 @@ module Karafka
                   File.read(
                     File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
                   )
-                ).fetch('en').fetch('validations').fetch('consumer_group')
+                ).fetch('en').fetch('validations').fetch('routing').fetch('consumer_group')
 
                 required(:patterns) { |val| val.is_a?(Array) && val.all? { |el| el.is_a?(Hash) } }
 

data/lib/karafka/pro/routing/features/patterns/contracts/topic.rb

@@ -17,7 +17,7 @@ module Karafka
                   File.read(
                     File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
                   )
-                ).fetch('en').fetch('validations').fetch('topic')
+                ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
               end
 
               nested(:patterns) do

data/lib/karafka/pro/routing/features/patterns/topic.rb

@@ -10,6 +10,15 @@ module Karafka
         class Patterns < Base
           # Patterns feature topic extensions
           module Topic
+            # This method calls the parent class initializer and then sets up the
+            # extra instance variable to nil. The explicit initialization
+            # to nil is included as an optimization for Ruby's object shapes system,
+            # which improves memory layout and access performance.
+            def initialize(...)
+              super
+              @patterns = nil
+            end
+
             # @return [String] subscription name or the regexp string representing matching of
             #   new topics that should be detected.
             def subscription_name

data/lib/karafka/pro/routing/features/pausing/contracts/topic.rb

@@ -17,7 +17,7 @@ module Karafka
                   File.read(
                     File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
                   )
-                ).fetch('en').fetch('validations').fetch('topic')
+                ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
 
                 required(:pause_timeout) { |val| val.is_a?(Integer) && val.positive? }
                 required(:pause_max_timeout) { |val| val.is_a?(Integer) && val.positive? }

data/lib/karafka/pro/routing/features/periodic_job/contracts/topic.rb

@@ -17,7 +17,7 @@ module Karafka
                   File.read(
                     File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
                   )
-                ).fetch('en').fetch('validations').fetch('topic')
+                ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
               end
 
               nested(:periodic_job) do

data/lib/karafka/pro/routing/features/periodic_job/topic.rb

@@ -10,6 +10,15 @@ module Karafka
         class PeriodicJob < Base
           # Periodic topic action flows extensions
           module Topic
+            # This method calls the parent class initializer and then sets up the
+            # extra instance variable to nil. The explicit initialization
+            # to nil is included as an optimization for Ruby's object shapes system,
+            # which improves memory layout and access performance.
+            def initialize(...)
+              super
+              @periodic_job = nil
+            end
+
             # Defines topic as periodic. Periodic topics consumers will invoke `#tick` with each
             # poll where messages were not received.
             # @param active [Boolean] should ticking happen for this topic assignments.

data/lib/karafka/pro/routing/features/recurring_tasks/contracts/topic.rb

@@ -17,7 +17,7 @@ module Karafka
                   File.read(
                     File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
                   )
-                ).fetch('en').fetch('validations').fetch('topic')
+                ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
               end
 
               nested(:recurring_tasks) do

data/lib/karafka/pro/routing/features/recurring_tasks/topic.rb

@@ -11,6 +11,15 @@ module Karafka
           # Topic extensions to be able to check if given topic is a recurring tasks topic
           # Please note, that this applies to both the schedules topics and reports topics
           module Topic
+            # This method calls the parent class initializer and then sets up the
+            # extra instance variable to nil. The explicit initialization
+            # to nil is included as an optimization for Ruby's object shapes system,
+            # which improves memory layout and access performance.
+            def initialize(...)
+              super
+              @recurring_tasks = nil
+            end
+
             # @param active [Boolean] should this topic be considered related to recurring tasks
             def recurring_tasks(active = false)
               @recurring_tasks ||= Config.new(active: active)

data/lib/karafka/pro/routing/features/scheduled_messages/contracts/topic.rb

@@ -17,7 +17,7 @@ module Karafka
                   File.read(
                     File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
                   )
-                ).fetch('en').fetch('validations').fetch('topic')
+                ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
               end
 
               nested(:scheduled_messages) do

data/lib/karafka/pro/routing/features/scheduled_messages/topic.rb

@@ -11,6 +11,15 @@ module Karafka
           # Topic extensions to be able to check if given topic is a scheduled messages topic
           # Please note, that this applies to both the schedules topic and logs topic
           module Topic
+            # This method calls the parent class initializer and then sets up the
+            # extra instance variable to nil. The explicit initialization
+            # to nil is included as an optimization for Ruby's object shapes system,
+            # which improves memory layout and access performance.
+            def initialize(...)
+              super
+              @scheduled_messages = nil
+            end
+
             # @param active [Boolean] should this topic be considered related to scheduled messages
             def scheduled_messages(active = false)
               @scheduled_messages ||= Config.new(active: active)

data/lib/karafka/pro/routing/features/swarm/contracts/topic.rb

@@ -17,7 +17,7 @@ module Karafka
                   File.read(
                     File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
                   )
-                ).fetch('en').fetch('validations').fetch('topic')
+                ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
               end
 
               nested(:swarm) do

data/lib/karafka/pro/routing/features/swarm/topic.rb

@@ -10,6 +10,15 @@ module Karafka
         class Swarm < Base
           # Topic swarm API extensions
           module Topic
+            # This method calls the parent class initializer and then sets up the
+            # extra instance variable to nil. The explicit initialization
+            # to nil is included as an optimization for Ruby's object shapes system,
+            # which improves memory layout and access performance.
+            def initialize(...)
+              super
+              @swarm = nil
+            end
+
             # Allows defining swarm routing topic settings
             # @param nodes [Range, Array, Hash] range of nodes ids or array with nodes ids for
             #   which we should run given topic or hash with nodes expected partition assignments

data/lib/karafka/pro/routing/features/throttling/contracts/topic.rb

@@ -17,7 +17,7 @@ module Karafka
                   File.read(
                     File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
                   )
-                ).fetch('en').fetch('validations').fetch('topic')
+                ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
               end
 
               nested(:throttling) do

data/lib/karafka/pro/routing/features/throttling/topic.rb

@@ -10,6 +10,15 @@ module Karafka
         class Throttling < Base
           # Topic throttling API extensions
           module Topic
+            # This method calls the parent class initializer and then sets up the
+            # extra instance variable to nil. The explicit initialization
+            # to nil is included as an optimization for Ruby's object shapes system,
+            # which improves memory layout and access performance.
+            def initialize(...)
+              super
+              @throttling = nil
+            end
+
             # @param limit [Integer] max messages to process in an time interval
             # @param interval [Integer] time interval for processing
             def throttling(

data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb

@@ -17,7 +17,7 @@ module Karafka
                   File.read(
                     File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
                   )
-                ).fetch('en').fetch('validations').fetch('topic')
+                ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
               end
 
               nested(:virtual_partitions) do

data/lib/karafka/pro/routing/features/virtual_partitions/topic.rb

@@ -10,6 +10,15 @@ module Karafka
         class VirtualPartitions < Base
           # Topic extensions to be able to manage virtual partitions feature
           module Topic
+            # This method calls the parent class initializer and then sets up the
+            # extra instance variable to nil. The explicit initialization
+            # to nil is included as an optimization for Ruby's object shapes system,
+            # which improves memory layout and access performance.
+            def initialize(...)
+              super
+              @virtual_partitions = nil
+            end
+
             # @param max_partitions [Integer] max number of virtual partitions that can come out of
             #   the single distribution flow. When set to more than the Karafka threading, will
             #   create more work than workers. When less, can ensure we have spare resources to

data/lib/karafka/pro/scheduled_messages/contracts/config.rb

@@ -15,7 +15,7 @@ module Karafka
               File.read(
                 File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
               )
-            ).fetch('en').fetch('validations').fetch('config')
+            ).fetch('en').fetch('validations').fetch('setup').fetch('config')
           end
 
           nested(:scheduled_messages) do

data/lib/karafka/pro/scheduled_messages/daily_buffer.rb

@@ -58,9 +58,15 @@ module Karafka
             selected << [epoch, message]
           end
 
-          selected
-            .sort_by!(&:first)
-            .each { |_, message| yield(message) }
+          # When epoch is of the same value for multiple messages to be dispatched, we also sort
+          # on the offset to make sure that earlier messages are dispatched prior to newer
+          selected.sort! do |pck1, pck2|
+            cmp = pck1[0] <=> pck2[0]
+
+            cmp.zero? ? pck1[1].offset <=> pck2[1].offset : cmp
+          end
+
+          selected.each { |_, message| yield(message) }
         end
 
         # Removes given key from the accumulator

data/lib/karafka/pro/swarm/liveness_listener.rb

@@ -163,19 +163,34 @@ module Karafka
         end
 
         # @return [Integer] RSS in MB for the current process
-        # @note Since swarm is linux only, we do not have to worry about getting RSS on other OSes
         def rss_mb
+          RUBY_PLATFORM.include?('linux') ? rss_mb_linux : rss_mb_macos
+        end
+
+        # @return [Integer] RSS in MB for the current process on Linux
+        def rss_mb_linux
           kb_rss = 0
 
           IO.readlines("/proc/#{node.pid}/status").each do |line|
             next unless line.start_with?('VmRSS:')
 
             kb_rss = line.split[1].to_i
-
             break
           end
 
           (kb_rss / 1_024.to_i).round
+        rescue Errno::ENOENT, Errno::EACCES
+          # /proc file doesn't exist or no permission to read it
+          0
+        end
+
+        # @return [Integer] RSS in MB for the current process on macOS/BSD
+        def rss_mb_macos
+          output = `ps -o rss= -p #{node.pid}`.strip
+          output.empty? ? 0 : (output.to_i / 1_024.0).round
+        rescue Errno::ESRCH
+          # Process doesn't exist
+          0
         end
       end
     end

data/lib/karafka/processing/executor.rb

@@ -168,7 +168,6 @@ module Karafka
           topic = @coordinator.topic
 
           strategy = strategy_selector.find(topic)
-          expansions = expansions_selector.find(topic)
 
           consumer = topic.consumer_class.new
           # We use singleton class as the same consumer class may be used to process different
@@ -178,6 +177,7 @@ module Karafka
           # Specific features may expand consumer API beyond the injected strategy. The difference
           # here is that strategy impacts the flow of states while extra APIs just provide some
           # extra methods with informations, etc but do no deviate the flow behavior
+          expansions = expansions_selector.find(topic)
           expansions.each { |expansion| consumer.singleton_class.include(expansion) }
 
           consumer.client = @client

data/lib/karafka/routing/builder.rb

@@ -157,9 +157,6 @@ module Karafka
         end
       end
 
-      # In case we use simple style of routing, all topics will be assigned to the same consumer
-      # group that will be based on the client_id
-      #
       # @param topic_name [String, Symbol] name of a topic from which we want to consumer
       # @param block [Proc] proc we want to evaluate in the topic context
       def topic(topic_name, &block)

data/lib/karafka/routing/consumer_group.rb

@@ -22,10 +22,7 @@ module Karafka
       # groups
       attr_accessor :current_subscription_group_details
 
-      # @param name [String, Symbol] raw name of this consumer group. Raw means, that it does not
-      #   yet have an application client_id namespace, this will be added here by default.
-      #   We add it to make a multi-system development easier for people that don't use
-      #   kafka and don't understand the concept of consumer groups.
+      # @param name [String, Symbol] name of this consumer group.
       def initialize(name)
         @name = name.to_s
         # This used to be different when consumer mappers existed but now it is the same

data/lib/karafka/routing/contracts/consumer_group.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    # Namespace for routing related contracts
+    module Contracts
+      # Contract for single full route (consumer group + topics) validation.
+      class ConsumerGroup < Karafka::Contracts::Base
+        configure do |config|
+          config.error_messages = YAML.safe_load(
+            File.read(
+              File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+            )
+          ).fetch('en').fetch('validations').fetch('routing').fetch('consumer_group')
+        end
+
+        required(:id) { |val| val.is_a?(String) && Karafka::Contracts::TOPIC_REGEXP.match?(val) }
+        required(:topics) { |val| val.is_a?(Array) && !val.empty? }
+
+        virtual do |data, errors|
+          next unless errors.empty?
+
+          names = data.fetch(:topics).map { |topic| topic_unique_key(topic) }
+
+          next if names.size == names.uniq.size
+
+          [[%i[topics], :names_not_unique]]
+        end
+
+        # Prevent same topics subscriptions in one CG with different consumer classes
+        # This should prevent users from accidentally creating multi-sg one CG setup with weird
+        # different consumer usage. If you need to consume same topic twice, use distinct CGs.
+        virtual do |data, errors|
+          next unless errors.empty?
+
+          topics_consumers = Hash.new { |h, k| h[k] = Set.new }
+
+          data.fetch(:topics).map do |topic|
+            topics_consumers[topic[:name]] << topic[:consumer]
+          end
+
+          next if topics_consumers.values.map(&:size).all? { |count| count == 1 }
+
+          [[%i[topics], :many_consumers_same_topic]]
+        end
+
+        virtual do |data, errors|
+          next unless errors.empty?
+          next unless ::Karafka::App.config.strict_topics_namespacing
+
+          names = data.fetch(:topics).map { |topic| topic[:name] }
+          names_hash = names.each_with_object({}) { |n, h| h[n] = true }
+          error_occured = false
+          names.each do |n|
+            # Skip topic names that are not namespaced
+            next unless n.chars.find { |c| ['.', '_'].include?(c) }
+
+            if n.chars.include?('.')
+              # Check underscore styled topic
+              underscored_topic = n.tr('.', '_')
+              error_occured = names_hash[underscored_topic] ? true : false
+            else
+              # Check dot styled topic
+              dot_topic = n.tr('_', '.')
+              error_occured = names_hash[dot_topic] ? true : false
+            end
+          end
+
+          next unless error_occured
+
+          [[%i[topics], :topics_namespaced_names_not_unique]]
+        end
+
+        class << self
+          # @param topic [Hash] topic config hash
+          # @return [String] topic unique key for validators
+          def topic_unique_key(topic)
+            topic[:name]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/contracts/routing.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Contracts
+      # Ensures that routing wide rules are obeyed
+      class Routing < Karafka::Contracts::Base
+        configure do |config|
+          config.error_messages = YAML.safe_load(
+            File.read(
+              File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+            )
+          ).fetch('en').fetch('validations').fetch('routing')
+        end
+
+        # Ensures, that when declarative topics strict requirement is on, all topics have
+        # declarative definition (including DLQ topics)
+        # @note It will ignore routing pattern topics because those topics are virtual
+        virtual do |data, errors|
+          next unless errors.empty?
+          # Do not validate declaratives unless required and explicitly enabled
+          next unless Karafka::App.config.strict_declarative_topics
+
+          # Collects declarative topics. Please note, that any topic that has a `#topic` reference,
+          # will be declarative by default unless explicitly disabled. This however does not apply
+          # to the DLQ definitions
+          dec_topics = Set.new
+          # All topics including the DLQ topics names that are marked as active
+          topics = Set.new
+
+          data.each do |consumer_group|
+            consumer_group[:topics].each do |topic|
+              pat = topic[:patterns]
+              # Ignore pattern topics because they won't exist and should not be declarative
+              # managed
+              topics << topic[:name] if !pat || !pat[:active]
+
+              dlq = topic[:dead_letter_queue]
+              topics << dlq[:topic] if dlq[:active]
+
+              dec = topic[:declaratives]
+
+              dec_topics << topic[:name] if dec[:active]
+            end
+          end
+
+          missing_dec = topics - dec_topics
+
+          next if missing_dec.empty?
+
+          missing_dec.map do |topic_name|
+            [
+              [:topics, topic_name],
+              :without_declarative_definition
+            ]
+          end
+        end
+      end
+    end
+  end
+end