karafka 2.3.2 → 2.4.0.beta1

data/lib/karafka/pro/processing/strategies/lrj/mom.rb

@@ -68,6 +68,8 @@ module Karafka
               Karafka.monitor.instrument('consumer.revoked', caller: self) do
                 revoked
               end
+            ensure
+              coordinator.decrement(:revoked)
             end
           end
         end

data/lib/karafka/pro/processing/subscription_groups_coordinator.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Uses the jobs queue API to lock (pause) and unlock (resume) operations of a given
+      # subscription group. It is abstracted away from jobs queue on this layer because we do
+      # not want to introduce jobs queue as a concept to the consumers layer
+      class SubscriptionGroupsCoordinator
+        include Singleton
+
+        # @param subscription_group [Karafka::Routing::SubscriptionGroup] subscription group we
+        #   want to pause
+        # @param lock_id [Object] key we want to use if we want to set multiple locks on the same
+        #   subscription group
+        # @param kwargs [Object] Any keyword arguments accepted by the jobs queue lock.
+        def pause(subscription_group, lock_id = nil, **kwargs)
+          jobs_queue.lock_async(
+            subscription_group.id,
+            lock_id,
+            **kwargs
+          )
+        end
+
+        # @param subscription_group [Karafka::Routing::SubscriptionGroup] subscription group we
+        #   want to resume
+        # @param lock_id [Object] lock id (if it was used to pause)
+        def resume(subscription_group, lock_id = nil)
+          jobs_queue.unlock_async(subscription_group.id, lock_id)
+        end
+
+        private
+
+        # @return [Karafka::Pro::Processing::JobsQueue]
+        def jobs_queue
+          @jobs_queue ||= Karafka::Server.jobs_queue
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/direct_assignments/config.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class DirectAssignments < Base
+          # Direct assignments feature configuration
+          Config = Struct.new(:active, :partitions, keyword_init: true) do
+            alias_method :active?, :active
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/direct_assignments/contracts/consumer_group.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class DirectAssignments < Base
+          module Contracts
+            # Contract to validate configuration of the direct assignments feature
+            class ConsumerGroup < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('consumer_group')
+
+                virtual do |data, errors|
+                  next unless errors.empty?
+
+                  active = []
+
+                  data[:topics].each do |topic|
+                    active << topic[:direct_assignments][:active]
+                  end
+
+                  # If none active we use standard subscriptions
+                  next if active.none?
+                  # If all topics from a given CG are using direct assignments that is expected
+                  # and allowed
+                  next if active.all?
+
+                  [[%i[direct_assignments], :homogenous]]
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/direct_assignments/contracts/topic.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class DirectAssignments < Base
+          # Namespace for direct assignments feature contracts
+          module Contracts
+            # Contract to validate configuration of the direct assignments topic feature
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+              end
+
+              nested(:direct_assignments) do
+                required(:active) { |val| [true, false].include?(val) }
+
+                required(:partitions) do |val|
+                  next true if val == true
+                  next false unless val.is_a?(Hash)
+                  next false unless val.keys.all? { |part| part.is_a?(Integer) }
+                  next false unless val.values.all? { |flag| flag == true }
+
+                  true
+                end
+              end
+
+              virtual do |data, errors|
+                next unless errors.empty?
+
+                direct_assignments = data[:direct_assignments]
+                partitions = direct_assignments[:partitions]
+
+                next unless direct_assignments[:active]
+                next unless partitions.is_a?(Hash)
+                next unless partitions.empty?
+
+                [[%i[direct_assignments], :active_but_empty]]
+              end
+
+              # Make sure that when we use swarm, all requested partitions have allocation
+              # We cannot assign partitions and then not allocate them in swarm mode
+              # If this is the case, they should not be assigned in the first place
+              virtual do |data, errors|
+                next unless errors.empty?
+
+                direct_assignments = data[:direct_assignments]
+                swarm = data[:swarm]
+
+                next unless direct_assignments[:active]
+                next unless swarm[:active]
+
+                nodes = swarm[:nodes]
+
+                next unless nodes.is_a?(Hash)
+                # Can be true for all partitions assignment and in this case we do not check
+                next unless direct_assignments[:partitions].is_a?(Hash)
+
+                direct_partitions = direct_assignments[:partitions].keys
+                swarm_partitions = nodes.values.flatten
+
+                next unless swarm_partitions.all? { |partition| partition.is_a?(Integer) }
+                next if direct_partitions.sort == swarm_partitions.sort
+
+                # If we assigned more partitions than we distributed in swarm
+                if (direct_partitions - swarm_partitions).size.positive?
+                  [[%i[direct_assignments], :swarm_not_complete]]
+                else
+                  [[%i[direct_assignments], :swarm_overbooked]]
+                end
+              end
+
+              # Make sure that direct assignments are not used together with patterns as we
+              # cannot apply patterns on direct assignments
+              virtual do |data, errors|
+                next unless errors.empty?
+
+                direct_assignments = data[:direct_assignments]
+                patterns = data[:patterns]
+
+                next unless direct_assignments[:active]
+                next unless patterns[:active]
+
+                [[%i[direct_assignments], :patterns_active]]
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/direct_assignments/subscription_group.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Alterations to the direct assignments that allow us to do stable direct assignments
+        # without working with consumer groups dynamic assignments
+        class DirectAssignments < Base
+          # Extension allowing us to select correct subscriptions and assignments based on the
+          # expanded routing setup
+          module SubscriptionGroup
+            # @return [false, Array<String>] false if we do not have any subscriptions or array
+            #   with all the subscriptions for given subscription group
+            def subscriptions
+              topics
+                .select(&:active?)
+                .reject { |topic| topic.direct_assignments.active? }
+                .map(&:subscription_name)
+                .then { |subscriptions| subscriptions.empty? ? false : subscriptions }
+            end
+
+            # @param consumer [Karafka::Connection::Proxy] consumer for expanding  the partition
+            #   knowledge in case of certain topics assignments
+            # @return [Rdkafka::Consumer::TopicPartitionList] final tpl for assignments
+            def assignments(consumer)
+              topics
+                .select(&:active?)
+                .select { |topic| topic.direct_assignments.active? }
+                .map { |topic| build_assignments(topic) }
+                .to_h
+                .tap { |topics| return false if topics.empty? }
+                .then { |topics| Iterator::Expander.new.call(topics) }
+                .then { |topics| Iterator::TplBuilder.new(consumer, topics).call }
+            end
+
+            private
+
+            # Builds assignments based on all the routing stuff. Takes swarm into consideration.
+            #
+            # @param topic [Karafka::Routing::Topic]
+            # @return [Array<String, Hash>]
+            def build_assignments(topic)
+              node = Karafka::App.config.swarm.node
+
+              standard_setup = [
+                topic.subscription_name,
+                topic.direct_assignments.partitions
+              ]
+
+              return standard_setup unless node
+              # Unless user explicitly assigned particular partitions to particular nodes, we just
+              # go with full regular assignments
+              return standard_setup unless topic.swarm.nodes.is_a?(Hash)
+
+              [
+                topic.subscription_name,
+                topic.swarm.nodes.fetch(node.id).map { |partition| [partition, true] }.to_h
+              ]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/direct_assignments/topic.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class DirectAssignments < Base
+          # Topic extensions for direct assignments
+          module Topic
+            # Allows for direct assignment of
+            # @param partitions_or_all [true, Array<Integer>] informs Karafka that we want
+            #   to use direct assignments instead of automatic for this topic. It also allows us
+            #   to specify which partitions we're interested in or `true` if in all
+            #
+            # @example Assign all available partitions
+            #   direct_assignments(true)
+            #
+            # @example Assign partitions 2, 3 and 5
+            #   direct_assignments(2, 3, 5)
+            #
+            # @example Assign partitions from 0 to 3
+            #   direct_assignments(0..3)
+            def direct_assignments(*partitions_or_all)
+              @direct_assignments ||= if partitions_or_all == [true]
+                Config.new(
+                  active: true,
+                  partitions: true
+                )
+              elsif partitions_or_all.size == 1 && partitions_or_all.first.is_a?(Range)
+                partitions_or_all = partitions_or_all.first.to_a
+
+                Config.new(
+                  active: true,
+                  partitions: partitions_or_all.map { |partition| [partition, true] }.to_h
+                )
+              else
+                Config.new(
+                  active: !partitions_or_all.empty?,
+                  partitions: partitions_or_all.map { |partition| [partition, true] }.to_h
+                )
+              end
+            end
+
+            alias assign direct_assignments
+
+            # @return [Hash] topic with all its native configuration options plus direct
+            #   assignments
+            def to_h
+              super.merge(
+                direct_assignments: direct_assignments.to_h
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/direct_assignments.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Allows for building direct assignments that bypass consumer group auto-assignments
+        # Useful for building complex pipelines that have explicit assignment requirements
+        class DirectAssignments < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/patterns/builder.rb

@@ -26,7 +26,7 @@ module Karafka
             #   the regexp if we used named patterns
             # @param block [Proc]
             def pattern(regexp_or_name, regexp = nil, &block)
-              consumer_group('app') do
+              consumer_group(default_group_id) do
                 pattern(regexp_or_name, regexp, &block)
               end
             end

data/lib/karafka/pro/routing/features/swarm/config.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Swarm < Base
+          # Swarm feature configuration
+          Config = Struct.new(
+            :active,
+            :nodes,
+            keyword_init: true
+          ) do
+            alias_method :active?, :active
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/swarm/contracts/routing.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Swarm < Base
+          module Contracts
+            # Special contract that validates prior to starting swarm that each node has
+            # at least one assignment.
+            #
+            # It is special because we cannot run it during routing definitions, because we can
+            # only run it when all routes are defined and full context is available.
+            #
+            # This is why we use it before warmup when everything is expected to be configured.
+            class Routing < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('routing')
+              end
+
+              # Validates that each node has at least one assignment.
+              #
+              # @param builder [Karafka::Routing::Builder]
+              def validate!(builder)
+                nodes_setup = Hash.new do |h, node_id|
+                  h[node_id] = { active: false, node_id: node_id }
+                end
+
+                # Initialize nodes in the hash so we can iterate over them
+                App.config.swarm.nodes.times { |node_id| nodes_setup[node_id] }
+                nodes_setup.freeze
+
+                builder.each do |consumer_group|
+                  consumer_group.topics.each do |topic|
+                    nodes_setup.each do |node_id, details|
+                      next unless topic.active?
+                      next unless topic.swarm.nodes.include?(node_id)
+
+                      details[:active] = true
+                    end
+                  end
+                end
+
+                nodes_setup.each_value do |details|
+                  super(details)
+                end
+              end
+
+              virtual do |data, errors|
+                next unless errors.empty?
+                next if data[:active]
+
+                [[%i[swarm_nodes], :not_used]]
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/swarm/contracts/topic.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Swarm < Base
+          # Namespace for swarm contracts
+          module Contracts
+            # Contract to validate configuration of the swarm feature
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+              end
+
+              nested(:swarm) do
+                required(:active) { |val| val == true }
+
+                required(:nodes) do |val|
+                  next true if val.is_a?(Range)
+                  next true if val.is_a?(Array) && val.all? { |id| id.is_a?(Integer) }
+                  next false unless val.is_a?(Hash)
+                  next false unless val.keys.all? { |id| id.is_a?(Integer) }
+
+                  values = val.values
+
+                  next false unless values.all? { |ps| ps.is_a?(Array) || ps.is_a?(Range) }
+                  next true if values.flatten.all? { |part| part.is_a?(Integer) }
+                  next true if values.flatten.all? { |part| part.is_a?(Range) }
+
+                  false
+                end
+              end
+
+              # Make sure that if range is defined it fits number of nodes (except infinity)
+              # As it may be a common error to accidentally define a node that will never be
+              # reached
+              virtual do |data, errors|
+                next unless errors.empty?
+
+                nodes = data[:swarm][:nodes]
+                # Hash can be used when we use direct assignments. In such cases nodes ids are
+                # keys in the hash and values are partitions
+                nodes = nodes.keys if nodes.is_a?(Hash)
+
+                # Defaults
+                next if nodes.first.zero? && nodes.include?(Float::INFINITY)
+
+                # If our expectation towards which node should run things matches at least one
+                # node, then it's ok
+                next if Karafka::App.config.swarm.nodes.times.any? do |node_id|
+                  nodes.include?(node_id)
+                end
+
+                [[%i[swarm_nodes], :with_non_existent_nodes]]
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end