karafka 2.3.3 → 2.4.0.beta2

data/lib/karafka/pro/routing/features/direct_assignments/config.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class DirectAssignments < Base
+          # Direct assignments feature configuration
+          Config = Struct.new(:active, :partitions, keyword_init: true) do
+            alias_method :active?, :active
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/direct_assignments/contracts/consumer_group.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class DirectAssignments < Base
+          module Contracts
+            # Contract to validate configuration of the direct assignments feature
+            class ConsumerGroup < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('consumer_group')
+
+                virtual do |data, errors|
+                  next unless errors.empty?
+
+                  active = []
+
+                  data[:topics].each do |topic|
+                    active << topic[:direct_assignments][:active]
+                  end
+
+                  # If none active we use standard subscriptions
+                  next if active.none?
+                  # If all topics from a given CG are using direct assignments that is expected
+                  # and allowed
+                  next if active.all?
+
+                  [[%i[direct_assignments], :homogenous]]
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/direct_assignments/contracts/topic.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class DirectAssignments < Base
+          # Namespace for direct assignments feature contracts
+          module Contracts
+            # Contract to validate configuration of the direct assignments topic feature
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+              end
+
+              nested(:direct_assignments) do
+                required(:active) { |val| [true, false].include?(val) }
+
+                required(:partitions) do |val|
+                  next true if val == true
+                  next false unless val.is_a?(Hash)
+                  next false unless val.keys.all? { |part| part.is_a?(Integer) }
+                  next false unless val.values.all? { |flag| flag == true }
+
+                  true
+                end
+              end
+
+              virtual do |data, errors|
+                next unless errors.empty?
+
+                direct_assignments = data[:direct_assignments]
+                partitions = direct_assignments[:partitions]
+
+                next unless direct_assignments[:active]
+                next unless partitions.is_a?(Hash)
+                next unless partitions.empty?
+
+                [[%i[direct_assignments], :active_but_empty]]
+              end
+
+              # Make sure that when we use swarm, all requested partitions have allocation
+              # We cannot assign partitions and then not allocate them in swarm mode
+              # If this is the case, they should not be assigned in the first place
+              virtual do |data, errors|
+                next unless errors.empty?
+
+                direct_assignments = data[:direct_assignments]
+                swarm = data[:swarm]
+
+                next unless direct_assignments[:active]
+                next unless swarm[:active]
+
+                nodes = swarm[:nodes]
+
+                next unless nodes.is_a?(Hash)
+                # Can be true for all partitions assignment and in this case we do not check
+                next unless direct_assignments[:partitions].is_a?(Hash)
+
+                direct_partitions = direct_assignments[:partitions].keys
+                swarm_partitions = nodes.values.flatten
+
+                next unless swarm_partitions.all? { |partition| partition.is_a?(Integer) }
+                next if direct_partitions.sort == swarm_partitions.sort
+
+                # If we assigned more partitions than we distributed in swarm
+                if (direct_partitions - swarm_partitions).size.positive?
+                  [[%i[direct_assignments], :swarm_not_complete]]
+                else
+                  [[%i[direct_assignments], :swarm_overbooked]]
+                end
+              end
+
+              # Make sure that direct assignments are not used together with patterns as we
+              # cannot apply patterns on direct assignments
+              virtual do |data, errors|
+                next unless errors.empty?
+
+                direct_assignments = data[:direct_assignments]
+                patterns = data[:patterns]
+
+                next unless direct_assignments[:active]
+                next unless patterns[:active]
+
+                [[%i[direct_assignments], :patterns_active]]
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/direct_assignments/subscription_group.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Alterations to the direct assignments that allow us to do stable direct assignments
+        # without working with consumer groups dynamic assignments
+        class DirectAssignments < Base
+          # Extension allowing us to select correct subscriptions and assignments based on the
+          # expanded routing setup
+          module SubscriptionGroup
+            # @return [false, Array<String>] false if we do not have any subscriptions or array
+            #   with all the subscriptions for given subscription group
+            def subscriptions
+              topics
+                .select(&:active?)
+                .reject { |topic| topic.direct_assignments.active? }
+                .map(&:subscription_name)
+                .then { |subscriptions| subscriptions.empty? ? false : subscriptions }
+            end
+
+            # @param consumer [Karafka::Connection::Proxy] consumer for expanding  the partition
+            #   knowledge in case of certain topics assignments
+            # @return [Rdkafka::Consumer::TopicPartitionList] final tpl for assignments
+            def assignments(consumer)
+              topics
+                .select(&:active?)
+                .select { |topic| topic.direct_assignments.active? }
+                .map { |topic| build_assignments(topic) }
+                .to_h
+                .tap { |topics| return false if topics.empty? }
+                .then { |topics| Iterator::Expander.new.call(topics) }
+                .then { |topics| Iterator::TplBuilder.new(consumer, topics).call }
+            end
+
+            private
+
+            # Builds assignments based on all the routing stuff. Takes swarm into consideration.
+            #
+            # @param topic [Karafka::Routing::Topic]
+            # @return [Array<String, Hash>]
+            def build_assignments(topic)
+              node = Karafka::App.config.swarm.node
+
+              standard_setup = [
+                topic.subscription_name,
+                topic.direct_assignments.partitions
+              ]
+
+              return standard_setup unless node
+              # Unless user explicitly assigned particular partitions to particular nodes, we just
+              # go with full regular assignments
+              return standard_setup unless topic.swarm.nodes.is_a?(Hash)
+
+              [
+                topic.subscription_name,
+                topic.swarm.nodes.fetch(node.id).map { |partition| [partition, true] }.to_h
+              ]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/direct_assignments/topic.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class DirectAssignments < Base
+          # Topic extensions for direct assignments
+          module Topic
+            # Allows for direct assignment of
+            # @param partitions_or_all [true, Array<Integer>] informs Karafka that we want
+            #   to use direct assignments instead of automatic for this topic. It also allows us
+            #   to specify which partitions we're interested in or `true` if in all
+            #
+            # @example Assign all available partitions
+            #   direct_assignments(true)
+            #
+            # @example Assign partitions 2, 3 and 5
+            #   direct_assignments(2, 3, 5)
+            #
+            # @example Assign partitions from 0 to 3
+            #   direct_assignments(0..3)
+            def direct_assignments(*partitions_or_all)
+              @direct_assignments ||= if partitions_or_all == [true]
+                Config.new(
+                  active: true,
+                  partitions: true
+                )
+              elsif partitions_or_all.size == 1 && partitions_or_all.first.is_a?(Range)
+                partitions_or_all = partitions_or_all.first.to_a
+
+                Config.new(
+                  active: true,
+                  partitions: partitions_or_all.map { |partition| [partition, true] }.to_h
+                )
+              else
+                Config.new(
+                  active: !partitions_or_all.empty?,
+                  partitions: partitions_or_all.map { |partition| [partition, true] }.to_h
+                )
+              end
+            end
+
+            alias assign direct_assignments
+
+            # @return [Hash] topic with all its native configuration options plus direct
+            #   assignments
+            def to_h
+              super.merge(
+                direct_assignments: direct_assignments.to_h
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/direct_assignments.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Allows for building direct assignments that bypass consumer group auto-assignments
+        # Useful for building complex pipelines that have explicit assignment requirements
+        class DirectAssignments < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/patterns/builder.rb

@@ -26,7 +26,7 @@ module Karafka
             #   the regexp if we used named patterns
             # @param block [Proc]
             def pattern(regexp_or_name, regexp = nil, &block)
-              consumer_group('app') do
+              consumer_group(default_group_id) do
                 pattern(regexp_or_name, regexp, &block)
               end
             end

data/lib/karafka/pro/routing/features/swarm/contracts/routing.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Swarm < Base
+          module Contracts
+            # Special contract that validates prior to starting swarm that each node has
+            # at least one assignment.
+            #
+            # It is special because we cannot run it during routing definitions, because we can
+            # only run it when all routes are defined and full context is available.
+            #
+            # This is why we use it before warmup when everything is expected to be configured.
+            class Routing < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('routing')
+              end
+
+              # Validates that each node has at least one assignment.
+              #
+              # @param builder [Karafka::Routing::Builder]
+              def validate!(builder)
+                nodes_setup = Hash.new do |h, node_id|
+                  h[node_id] = { active: false, node_id: node_id }
+                end
+
+                # Initialize nodes in the hash so we can iterate over them
+                App.config.swarm.nodes.times { |node_id| nodes_setup[node_id] }
+                nodes_setup.freeze
+
+                builder.each do |consumer_group|
+                  consumer_group.topics.each do |topic|
+                    nodes_setup.each do |node_id, details|
+                      next unless topic.active?
+                      next unless topic.swarm.nodes.include?(node_id)
+
+                      details[:active] = true
+                    end
+                  end
+                end
+
+                nodes_setup.each_value do |details|
+                  super(details)
+                end
+              end
+
+              virtual do |data, errors|
+                next unless errors.empty?
+                next if data[:active]
+
+                [[%i[swarm_nodes], :not_used]]
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/swarm/contracts/topic.rb

@@ -32,10 +32,18 @@ module Karafka
                 required(:active) { |val| val == true }
 
                 required(:nodes) do |val|
-                  val.is_a?(Range) || (
-                    val.is_a?(Array) &&
-                    val.all? { |id| id.is_a?(Integer) }
-                  )
+                  next true if val.is_a?(Range)
+                  next true if val.is_a?(Array) && val.all? { |id| id.is_a?(Integer) }
+                  next false unless val.is_a?(Hash)
+                  next false unless val.keys.all? { |id| id.is_a?(Integer) }
+
+                  values = val.values
+
+                  next false unless values.all? { |ps| ps.is_a?(Array) || ps.is_a?(Range) }
+                  next true if values.flatten.all? { |part| part.is_a?(Integer) }
+                  next true if values.flatten.all? { |part| part.is_a?(Range) }
+
+                  false
                 end
               end
 
@@ -46,9 +54,12 @@ module Karafka
                 next unless errors.empty?
 
                 nodes = data[:swarm][:nodes]
+                # Hash can be used when we use direct assignments. In such cases nodes ids are
+                # keys in the hash and values are partitions
+                nodes = nodes.keys if nodes.is_a?(Hash)
 
                 # Defaults
-                next if nodes.first.zero? && nodes.last == Float::INFINITY
+                next if nodes.first.zero? && nodes.include?(Float::INFINITY)
 
                 # If our expectation towards which node should run things matches at least one
                 # node, then it's ok

data/lib/karafka/pro/routing/features/swarm/topic.rb

@@ -19,8 +19,31 @@ module Karafka
           # Topic swarm API extensions
           module Topic
             # Allows defining swarm routing topic settings
-            # @param nodes [Range, Array] range of nodes ids or array with nodes ids for which we
-            #   should run given topic
+            # @param nodes [Range, Array, Hash] range of nodes ids or array with nodes ids for
+            #   which we should run given topic or hash with nodes expected partition assignments
+            #   for the direct assignments API.
+            #
+            # @example Assign given topic only to node 1
+            #   swarm(nodes: [1])
+            #
+            # @example Assign given topic to nodes from 1 to 3
+            #   swarm(nodes: 1..3)
+            #
+            # @example Assign partitions 2 and 3 to node 0 and partitions 0, 1 to node 1
+            #   swarm(
+            #     nodes: {
+            #       0 => [2, 3],
+            #       1 => [0, 1]
+            #     }
+            #   )
+            #
+            # @example Assign partitions in ranges to nodes
+            #   swarm(
+            #     nodes: {
+            #       0 => (0..2),
+            #       1 => (3..5)
+            #     }
+            #   )
             def swarm(nodes: (0...Karafka::App.config.swarm.nodes))
               @swarm ||= Config.new(active: true, nodes: nodes)
             end

data/lib/karafka/pro/routing/features/swarm.rb

@@ -18,6 +18,17 @@ module Karafka
         # Karafka Pro Swarm extensions to the routing
         # They allow for more granular work assignment in the swarm
         class Swarm < Base
+          class << self
+            # Binds our routing validation contract prior to warmup in the supervisor, so we can
+            # run it when all the context should be there (config + full routing)
+            #
+            # @param config [Karafka::Core::Configurable::Node] app config
+            def post_setup(config)
+              config.monitor.subscribe('app.before_warmup') do
+                Contracts::Routing.new.validate!(config.internal.routing.builder)
+              end
+            end
+          end
         end
       end
     end

data/lib/karafka/pro/swarm/liveness_listener.rb

@@ -103,6 +103,26 @@ module Karafka
           end
         end
 
+        # Deregister the polling tracker for given listener
+        # @param _event [Karafka::Core::Monitoring::Event]
+        def on_connection_listener_stopping(_event)
+          # We are interested in disabling tracking for given listener only if it was requested
+          # when karafka was running. If we would always clear, it would not catch the shutdown
+          # polling requirements. The "running" listener shutdown operations happen only when
+          # the manager requests it for downscaling.
+          return if Karafka::App.done?
+
+          clear_polling_tick
+        end
+
+        # Deregister the polling tracker for given listener
+        # @param _event [Karafka::Core::Monitoring::Event]
+        def on_connection_listener_stopped(_event)
+          return if Karafka::App.done?
+
+          clear_polling_tick
+        end
+
         private
 
         # @return [Integer] object id of the current thread

data/lib/karafka/processing/coordinator.rb

@@ -11,6 +11,7 @@ module Karafka
     #   future mistakes.
     class Coordinator
       extend Forwardable
+      include Core::Helpers::Time
 
       attr_reader :pause_tracker, :seek_offset, :topic, :partition
 
@@ -25,12 +26,13 @@ module Karafka
         @pause_tracker = pause_tracker
         @revoked = false
         @consumptions = {}
-        @running_jobs = 0
+        @running_jobs = Hash.new { |h, k| h[k] = 0 }
         @manual_pause = false
         @manual_seek = false
         @mutex = Mutex.new
         @marked = false
         @failure = false
+        @changed_at = monotonic_now
       end
 
       # Starts the coordinator for given consumption jobs
@@ -38,7 +40,7 @@ module Karafka
       #   going to coordinate work. Not used with regular coordinator.
       def start(messages)
         @failure = false
-        @running_jobs = 0
+        @running_jobs[:consume] = 0
         # We need to clear the consumption results hash here, otherwise we could end up storing
         # consumption results of consumer instances we no longer control
         @consumptions.clear
@@ -70,16 +72,22 @@ module Karafka
       end
 
       # Increases number of jobs that we handle with this coordinator
-      def increment
-        synchronize { @running_jobs += 1 }
+      # @param job_type [Symbol] type of job that we want to increment
+      def increment(job_type)
+        synchronize do
+          @running_jobs[job_type] += 1
+          @changed_at = monotonic_now
+        end
       end
 
       # Decrements number of jobs we handle at the moment
-      def decrement
+      # @param job_type [Symbol] type of job that we want to decrement
+      def decrement(job_type)
         synchronize do
-          @running_jobs -= 1
+          @running_jobs[job_type] -= 1
+          @changed_at = monotonic_now
 
-          return @running_jobs unless @running_jobs.negative?
+          return @running_jobs[job_type] unless @running_jobs[job_type].negative?
 
           # This should never happen. If it does, something is heavily out of sync. Please reach
           # out to us if you encounter this
@@ -90,9 +98,10 @@ module Karafka
       # Is all the consumption done and finished successfully for this coordinator
       # We do not say we're successful until all work is done, because running work may still
       # crash.
+      # @note This is only used for consume synchronization
       def success?
         synchronize do
-          @running_jobs.zero? && @consumptions.values.all?(&:success?)
+          @running_jobs[:consume].zero? && @consumptions.values.all?(&:success?)
         end
       end
 

data/lib/karafka/processing/coordinators_buffer.rb

@@ -9,10 +9,13 @@ module Karafka
     # @note This buffer operates only from the listener loop, thus we do not have to make it
     #   thread-safe.
     class CoordinatorsBuffer
+      include Helpers::ConfigImporter.new(
+        coordinator_class: %i[internal processing coordinator_class]
+      )
+
       # @param topics [Karafka::Routing::Topics]
       def initialize(topics)
         @pauses_manager = Connection::PausesManager.new
-        @coordinator_class = ::Karafka::App.config.internal.processing.coordinator_class
         @coordinators = Hash.new { |h, k| h[k] = {} }
         @topics = topics
       end
@@ -23,7 +26,7 @@ module Karafka
         @coordinators[topic_name][partition] ||= begin
           routing_topic = @topics.find(topic_name)
 
-          @coordinator_class.new(
+          coordinator_class.new(
             routing_topic,
             partition,
             @pauses_manager.fetch(routing_topic, partition)