karafka 2.0.6 → 2.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f7c672af31943cc38566d944bc4cb0466145e98a2130295f3ca24c275dccdc64
-  data.tar.gz: 7eeaea273e18f31a29c79591a2627c2f79f0915663157cc9e31e1dd354c9deb0
+  metadata.gz: 0abed3f97a58be6b48f640468f7d7e6d48bc0960596b21d022b4616dd047be28
+  data.tar.gz: 48143253beee640e25e47a81474767c179e715e855d6173b59566483a57af5a8
 SHA512:
-  metadata.gz: '032765549abe7a8afb1866d6b8ac89ba711a8a04f1b7370018663fc140aa0d30a3966151d1524f49d12b1dfe7a4fafb72a008d0af2fe39ffc959e34c385f5469'
-  data.tar.gz: 10194fcd2ada3015b1bcd549ce286dafd5f94051b98a1752e2b5d9767d7c99ceaff47ce84429e345a99fb920f51cfabd4b5779d29da99b1e11fa6981e33b437f
+  metadata.gz: 9c9f8c170ac82fc0f1eb6ea41698dcd82cc525006931a59443d004c94eb18b56ffcb67eb1eb45fcc1fd557fee22e6e63ceb7a8a001245469e3e574d87c88c8e8
+  data.tar.gz: 47bc7e7dfe5ca3d503a3cb18da4e4b95c076197dc26b5633195e169d3f4d94da4effaf27bd4360ddff1481031b1ee20f61e465e24f6984570f6067ca4fbd51ea

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Karafka framework changelog
 
+## 2.0.7 (Unreleased)
+- [Breaking change] Redefine the Virtual Partitions routing DSL to accept concurrency
+- Allow for `concurrency` setting in Virtual Partitions to extend or limit number of jobs per regular partition. This allows to make sure, we do not use all the threads on virtual partitions jobs
+- Allow for creation of as many Virtual Partitions as needed, without taking global `concurrency` into consideration
+
 ## 2.0.6 (2022-09-02)
 - Improve client closing.
 - Fix for: Multiple LRJ topics fetched concurrently block ability for LRJ to kick in (#1002)

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.6)
+    karafka (2.0.7)
       karafka-core (>= 2.0.2, < 3.0.0)
       rdkafka (>= 0.12)
       thor (>= 0.20)

data/config/errors.yml

@@ -55,3 +55,5 @@ en:
 
     pro_consumer_group_topic:
       consumer_format: needs to inherit from Karafka::Pro::BaseConsumer and not Karafka::Consumer
+      virtual_partitions.partitioner_respond_to_call: needs to be defined and needs to respond to `#call`
+      virtual_partitions.concurrency_format: needs to be equl or more than 1

data/lib/karafka/pro/active_job/consumer.rb

@@ -35,7 +35,7 @@ module Karafka
 
             # We cannot mark jobs as done after each if there are virtual partitions. Otherwise
             # this could create random markings
-            next if topic.virtual_partitioner?
+            next if topic.virtual_partitions?
 
             mark_as_consumed(message)
           end

data/lib/karafka/pro/contracts/consumer_group_topic.rb

@@ -22,11 +22,31 @@ module Karafka
           ).fetch('en').fetch('validations').fetch('pro_consumer_group_topic')
         end
 
-        virtual do |data|
+        nested(:virtual_partitions) do
+          required(:active) { |val| [true, false].include?(val) }
+          required(:partitioner) { |val| val.nil? || val.respond_to?(:call) }
+          required(:concurrency) { |val| val.is_a?(Integer) && val >= 1 }
+        end
+
+        virtual do |data, errors|
+          next unless errors.empty?
           next if data[:consumer] < Karafka::Pro::BaseConsumer
 
           [[%i[consumer], :consumer_format]]
         end
+
+        # When virtual partitions are defined, partitioner needs to respond to `#call` and it
+        # cannot be nil
+        virtual do |data, errors|
+          next unless errors.empty?
+
+          virtual_partitions = data[:virtual_partitions]
+
+          next unless virtual_partitions[:active]
+          next if virtual_partitions[:partitioner].respond_to?(:call)
+
+          [[%i[virtual_partitions partitioner], :respond_to_call]]
+        end
       end
     end
   end

data/lib/karafka/pro/loader.rb

@@ -67,7 +67,7 @@ module Karafka
 
         # Loads routing extensions
         def load_routing_extensions
-          ::Karafka::Routing::Topic.include(Routing::TopicExtensions)
+          ::Karafka::Routing::Topic.prepend(Routing::TopicExtensions)
           ::Karafka::Routing::Builder.prepend(Routing::BuilderExtensions)
         end
       end

data/lib/karafka/pro/processing/partitioner.rb

@@ -21,17 +21,15 @@ module Karafka
         def call(topic, messages)
           ktopic = @subscription_group.topics.find(topic)
 
-          @concurrency ||= ::Karafka::App.config.concurrency
-
           # We only partition work if we have a virtual partitioner and more than one thread to
           # process the data. With one thread it is not worth partitioning the work as the work
           # itself will be assigned to one thread (pointless work)
-          if ktopic.virtual_partitioner? && @concurrency > 1
+          if ktopic.virtual_partitions? && ktopic.virtual_partitions.concurrency > 1
             # We need to reduce it to number of threads, so the group_id is not a direct effect
             # of the end user action. Otherwise the persistence layer for consumers would cache
             # it forever and it would cause memory leaks
             groupings = messages
-                        .group_by { |msg| ktopic.virtual_partitioner.call(msg) }
+                        .group_by { |msg| ktopic.virtual_partitions.partitioner.call(msg) }
                         .values
 
             # Reduce the max concurrency to a size that matches the concurrency
@@ -41,7 +39,7 @@ module Karafka
             # The algorithm here is simple, we assume that the most costly in terms of processing,
             # will be processing of the biggest group and we reduce the smallest once to have
             # max of groups equal to concurrency
-            while groupings.size > @concurrency
+            while groupings.size > ktopic.virtual_partitions.concurrency
               groupings.sort_by! { |grouping| -grouping.size }
 
               # Offset order needs to be maintained for virtual partitions

data/lib/karafka/pro/routing/topic_extensions.rb

@@ -15,23 +15,59 @@ module Karafka
     module Routing
       # Routing extensions that allow to configure some extra PRO routing options
       module TopicExtensions
+        # Internal representation of the virtual partitions settings and configuration
+        # This allows us to abstract away things in a nice manner
+        #
+        # For features with more options than just on/off we use this approach as it simplifies
+        # the code. We do not use it for all not to create unneeded complexity
+        VirtualPartitions = Struct.new(
+          :active,
+          :partitioner,
+          :concurrency,
+          keyword_init: true
+        ) { alias_method :active?, :active }
+
         class << self
           # @param base [Class] class we extend
-          def included(base)
+          def prepended(base)
             base.attr_accessor :long_running_job
-            base.attr_accessor :virtual_partitioner
           end
         end
 
-        # @return [Boolean] true if virtual partitioner is defined, false otherwise
-        def virtual_partitioner?
-          virtual_partitioner != nil
+        # @param concurrency [Integer] max number of virtual partitions that can come out of the
+        #   single distribution flow. When set to more than the Karafka threading, will create
+        #   more work than workers. When less, can ensure we have spare resources to process other
+        #   things in parallel.
+        # @param partitioner [nil, #call] nil or callable partitioner
+        # @return [VirtualPartitions] method that allows to set the virtual partitions details
+        #   during the routing configuration and then allows to retrieve it
+        def virtual_partitions(
+          concurrency: Karafka::App.config.concurrency,
+          partitioner: nil
+        )
+          @virtual_partitions ||= VirtualPartitions.new(
+            active: !partitioner.nil?,
+            concurrency: concurrency,
+            partitioner: partitioner
+          )
+        end
+
+        # @return [Boolean] are virtual partitions enabled for given topic
+        def virtual_partitions?
+          virtual_partitions.active?
         end
 
         # @return [Boolean] is a given job on a topic a long-running one
         def long_running_job?
           @long_running_job || false
         end
+
+        # @return [Hash] hash with topic details and the extensions details
+        def to_h
+          super.merge(
+            virtual_partitions: virtual_partitions.to_h
+          )
+        end
       end
     end
   end

data/lib/karafka/routing/proxy.rb

@@ -7,15 +7,6 @@ module Karafka
     class Proxy
       attr_reader :target
 
-      # We should proxy only non ? and = methods as we want to have a regular dsl
-      IGNORED_POSTFIXES = %w[
-        ?
-        =
-        !
-      ].freeze
-
-      private_constant :IGNORED_POSTFIXES
-
       # @param target [Object] target object to which we proxy any DSL call
       # @param block [Proc] block that we want to evaluate in the proxy context
       def initialize(target, &block)
@@ -25,21 +16,23 @@ module Karafka
 
       # Translates the no "=" DSL of routing into elements assignments on target
       # @param method_name [Symbol] name of the missing method
-      # @param arguments [Array] array with it's arguments
-      # @param block [Proc] block provided to the method
-      def method_missing(method_name, *arguments, &block)
+      def method_missing(method_name, ...)
         return super unless respond_to_missing?(method_name)
 
-        @target.public_send(:"#{method_name}=", *arguments, &block)
+        if @target.respond_to?(:"#{method_name}=")
+          @target.public_send(:"#{method_name}=", ...)
+        else
+          @target.public_send(method_name, ...)
+        end
       end
 
       # Tells whether or not a given element exists on the target
       # @param method_name [Symbol] name of the missing method
       # @param include_private [Boolean] should we include private in the check as well
       def respond_to_missing?(method_name, include_private = false)
-        return false if IGNORED_POSTFIXES.any? { |postfix| method_name.to_s.end_with?(postfix) }
-
-        @target.respond_to?(:"#{method_name}=", include_private) || super
+        @target.respond_to?(:"#{method_name}=", include_private) ||
+          @target.respond_to?(method_name, include_private) ||
+          super
       end
     end
   end

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.6'
+  VERSION = '2.0.7'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.6
+  version: 2.0.7
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2022-09-02 00:00:00.000000000 Z
+date: 2022-09-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core