karafka 2.0.6 → 2.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f7c672af31943cc38566d944bc4cb0466145e98a2130295f3ca24c275dccdc64
-  data.tar.gz: 7eeaea273e18f31a29c79591a2627c2f79f0915663157cc9e31e1dd354c9deb0
+  metadata.gz: 1a646d63d674e6ecb80625097426f48294fe8e8b3f3e9e020cf92645a5728251
+  data.tar.gz: 461c58c355f84b81c3396cb4164a426c3f1e87747b3101de1e7c6fbf2a876778
 SHA512:
-  metadata.gz: '032765549abe7a8afb1866d6b8ac89ba711a8a04f1b7370018663fc140aa0d30a3966151d1524f49d12b1dfe7a4fafb72a008d0af2fe39ffc959e34c385f5469'
-  data.tar.gz: 10194fcd2ada3015b1bcd549ce286dafd5f94051b98a1752e2b5d9767d7c99ceaff47ce84429e345a99fb920f51cfabd4b5779d29da99b1e11fa6981e33b437f
+  metadata.gz: 623832478b11b1fa61c7906f423417500838e30ef97256a59290d73dcfe17670cf7eb11adcf33d422ae590c6ce5f19215340c89d369f8ec5b4e609af07d0befb
+  data.tar.gz: 97e50ed131c939d09de884d3ddba826381e7d314d3bc4660ecf947fa6582d9e6f16e023dad121612eadffcd441a73874c96e287e5ec1885e008d843357e94b93

data/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Karafka framework changelog
 
+## 2.0.8 (2022-09-19)
+- [Breaking change] Rename Virtual Partitions `concurrency` to `max_partitions` to avoid confusion  (#1023).
+-  Allow for block based subscription groups management (#1030).
+
+## 2.0.7 (2022-09-05)
+- [Breaking change] Redefine the Virtual Partitions routing DSL to accept concurrency
+- Allow for `concurrency` setting in Virtual Partitions to extend or limit number of jobs per regular partition. This allows to make sure, we do not use all the threads on virtual partitions jobs
+- Allow for creation of as many Virtual Partitions as needed, without taking global `concurrency` into consideration
+
 ## 2.0.6 (2022-09-02)
 - Improve client closing.
 - Fix for: Multiple LRJ topics fetched concurrently block ability for LRJ to kick in (#1002)

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.6)
+    karafka (2.0.8)
       karafka-core (>= 2.0.2, < 3.0.0)
       rdkafka (>= 0.12)
       thor (>= 0.20)

data/config/errors.yml

@@ -55,3 +55,5 @@ en:
 
     pro_consumer_group_topic:
       consumer_format: needs to inherit from Karafka::Pro::BaseConsumer and not Karafka::Consumer
+      virtual_partitions.partitioner_respond_to_call: needs to be defined and needs to respond to `#call`
+      virtual_partitions.max_partitions_format: needs to be equl or more than 1

data/lib/karafka/base_consumer.rb

@@ -124,7 +124,7 @@ module Karafka
 
     # Method that will perform business logic and on data received from Kafka (it will consume
     #   the data)
-    # @note This method needs bo be implemented in a subclass. We stub it here as a failover if
+    # @note This method needs to be implemented in a subclass. We stub it here as a failover if
     #   someone forgets about it or makes on with typo
     def consume
       raise NotImplementedError, 'Implement this in a subclass'

data/lib/karafka/pro/active_job/consumer.rb

@@ -35,7 +35,7 @@ module Karafka
 
             # We cannot mark jobs as done after each if there are virtual partitions. Otherwise
             # this could create random markings
-            next if topic.virtual_partitioner?
+            next if topic.virtual_partitions?
 
             mark_as_consumed(message)
           end

data/lib/karafka/pro/base_consumer.rb

@@ -102,7 +102,7 @@ module Karafka
         else
           # If processing failed, we need to pause
           # For long running job this will overwrite the default never-ending pause and will cause
-          # the processing th keep going after the error backoff
+          # the processing to keep going after the error backoff
           pause(@seek_offset || first_message.offset)
         end
       end

data/lib/karafka/pro/contracts/consumer_group_topic.rb

@@ -22,11 +22,31 @@ module Karafka
           ).fetch('en').fetch('validations').fetch('pro_consumer_group_topic')
         end
 
-        virtual do |data|
+        nested(:virtual_partitions) do
+          required(:active) { |val| [true, false].include?(val) }
+          required(:partitioner) { |val| val.nil? || val.respond_to?(:call) }
+          required(:max_partitions) { |val| val.is_a?(Integer) && val >= 1 }
+        end
+
+        virtual do |data, errors|
+          next unless errors.empty?
           next if data[:consumer] < Karafka::Pro::BaseConsumer
 
           [[%i[consumer], :consumer_format]]
         end
+
+        # When virtual partitions are defined, partitioner needs to respond to `#call` and it
+        # cannot be nil
+        virtual do |data, errors|
+          next unless errors.empty?
+
+          virtual_partitions = data[:virtual_partitions]
+
+          next unless virtual_partitions[:active]
+          next if virtual_partitions[:partitioner].respond_to?(:call)
+
+          [[%i[virtual_partitions partitioner], :respond_to_call]]
+        end
       end
     end
   end

data/lib/karafka/pro/loader.rb

@@ -67,7 +67,7 @@ module Karafka
 
         # Loads routing extensions
         def load_routing_extensions
-          ::Karafka::Routing::Topic.include(Routing::TopicExtensions)
+          ::Karafka::Routing::Topic.prepend(Routing::TopicExtensions)
           ::Karafka::Routing::Builder.prepend(Routing::BuilderExtensions)
         end
       end

data/lib/karafka/pro/processing/partitioner.rb

@@ -21,27 +21,25 @@ module Karafka
         def call(topic, messages)
           ktopic = @subscription_group.topics.find(topic)
 
-          @concurrency ||= ::Karafka::App.config.concurrency
-
           # We only partition work if we have a virtual partitioner and more than one thread to
           # process the data. With one thread it is not worth partitioning the work as the work
           # itself will be assigned to one thread (pointless work)
-          if ktopic.virtual_partitioner? && @concurrency > 1
+          if ktopic.virtual_partitions? && ktopic.virtual_partitions.max_partitions > 1
             # We need to reduce it to number of threads, so the group_id is not a direct effect
             # of the end user action. Otherwise the persistence layer for consumers would cache
             # it forever and it would cause memory leaks
             groupings = messages
-                        .group_by { |msg| ktopic.virtual_partitioner.call(msg) }
+                        .group_by { |msg| ktopic.virtual_partitions.partitioner.call(msg) }
                         .values
 
-            # Reduce the max concurrency to a size that matches the concurrency
+            # Reduce the number of virtual partitions to a size that matches the max_partitions
             # As mentioned above we cannot use the partitioning keys directly as it could cause
             # memory leaks
             #
             # The algorithm here is simple, we assume that the most costly in terms of processing,
             # will be processing of the biggest group and we reduce the smallest once to have
-            # max of groups equal to concurrency
-            while groupings.size > @concurrency
+            # max of groups equal to max_partitions
+            while groupings.size > ktopic.virtual_partitions.max_partitions
               groupings.sort_by! { |grouping| -grouping.size }
 
               # Offset order needs to be maintained for virtual partitions

data/lib/karafka/pro/routing/topic_extensions.rb

@@ -15,23 +15,59 @@ module Karafka
     module Routing
       # Routing extensions that allow to configure some extra PRO routing options
       module TopicExtensions
+        # Internal representation of the virtual partitions settings and configuration
+        # This allows us to abstract away things in a nice manner
+        #
+        # For features with more options than just on/off we use this approach as it simplifies
+        # the code. We do not use it for all not to create unneeded complexity
+        VirtualPartitions = Struct.new(
+          :active,
+          :partitioner,
+          :max_partitions,
+          keyword_init: true
+        ) { alias_method :active?, :active }
+
         class << self
           # @param base [Class] class we extend
-          def included(base)
+          def prepended(base)
             base.attr_accessor :long_running_job
-            base.attr_accessor :virtual_partitioner
           end
         end
 
-        # @return [Boolean] true if virtual partitioner is defined, false otherwise
-        def virtual_partitioner?
-          virtual_partitioner != nil
+        # @param max_partitions [Integer] max number of virtual partitions that can come out of the
+        #   single distribution flow. When set to more than the Karafka threading, will create
+        #   more work than workers. When less, can ensure we have spare resources to process other
+        #   things in parallel.
+        # @param partitioner [nil, #call] nil or callable partitioner
+        # @return [VirtualPartitions] method that allows to set the virtual partitions details
+        #   during the routing configuration and then allows to retrieve it
+        def virtual_partitions(
+          max_partitions: Karafka::App.config.concurrency,
+          partitioner: nil
+        )
+          @virtual_partitions ||= VirtualPartitions.new(
+            active: !partitioner.nil?,
+            max_partitions: max_partitions,
+            partitioner: partitioner
+          )
+        end
+
+        # @return [Boolean] are virtual partitions enabled for given topic
+        def virtual_partitions?
+          virtual_partitions.active?
         end
 
         # @return [Boolean] is a given job on a topic a long-running one
         def long_running_job?
           @long_running_job || false
         end
+
+        # @return [Hash] hash with topic details and the extensions details
+        def to_h
+          super.merge(
+            virtual_partitions: virtual_partitions.to_h
+          )
+        end
       end
     end
   end

data/lib/karafka/processing/jobs/base.rb

@@ -2,7 +2,7 @@
 
 module Karafka
   module Processing
-    # Namespace for all the jobs that are suppose to run in workers.
+    # Namespace for all the jobs that are supposed to run in workers.
     module Jobs
       # Base class for all the jobs types that are suppose to run in workers threads.
       # Each job can have 3 main entry-points: `#before_call`, `#call` and `#after_call`
@@ -10,7 +10,7 @@ module Karafka
       class Base
         extend Forwardable
 
-        # @note Since one job has always one executer, we use the jobs id and group id as reference
+        # @note Since one job has always one executor, we use the jobs id and group id as reference
         def_delegators :executor, :id, :group_id
 
         attr_reader :executor

data/lib/karafka/processing/jobs_queue.rb

@@ -13,7 +13,7 @@ module Karafka
       # @return [Karafka::Processing::JobsQueue]
       def initialize
         @queue = Queue.new
-        # Those queues will act as a semaphores internally. Since we need an indicator for waiting
+        # Those queues will act as semaphores internally. Since we need an indicator for waiting
         # we could use Thread.pass but this is expensive. Instead we can just lock until any
         # of the workers finishes their work and we can re-check. This means that in the worse
         # scenario, we will context switch 10 times per poll instead of getting this thread

data/lib/karafka/routing/builder.rb

@@ -66,6 +66,16 @@ module Karafka
         end
       end
 
+      # Handles the simple routing case where we create one consumer group and allow for further
+      # subscription group customization
+      # @param subscription_group_name [String, Symbol] subscription group id
+      # @param block [Proc] further topics definitions
+      def subscription_group(subscription_group_name, &block)
+        consumer_group('app') do
+          target.public_send(:subscription_group=, subscription_group_name, &block)
+        end
+      end
+
       # In case we use simple style of routing, all topics will be assigned to the same consumer
       # group that will be based on the client_id
       #

data/lib/karafka/routing/consumer_group.rb

@@ -10,6 +10,12 @@ module Karafka
     class ConsumerGroup
       attr_reader :id, :topics, :name
 
+      # This is a "virtual" attribute that is not building subscription groups.
+      # It allows us to store the "current" subscription group defined in the routing
+      # This subscription group id is then injected into topics, so we can compute the subscription
+      # groups
+      attr_accessor :current_subscription_group_name
+
       # @param name [String, Symbol] raw name of this consumer group. Raw means, that it does not
       #   yet have an application client_id namespace, this will be added here by default.
       #   We add it to make a multi-system development easier for people that don't use
@@ -32,7 +38,27 @@ module Karafka
       def topic=(name, &block)
         topic = Topic.new(name, self)
         @topics << Proxy.new(topic, &block).target
-        @topics.last
+        built_topic = @topics.last
+        # We overwrite it conditionally in case it was not set by the user inline in the topic
+        # block definition
+        built_topic.subscription_group ||= current_subscription_group_name
+        built_topic
+      end
+
+      # Assigns the current subscription group id based on the defined one and allows for further
+      # topic definition
+      # @param name [String, Symbol]
+      # @param block [Proc] block that may include topics definitions
+      def subscription_group=(name, &block)
+        # We cast it here, so the routing supports symbol based but that's anyhow later on
+        # validated as a string
+        self.current_subscription_group_name = name.to_s
+
+        Proxy.new(self, &block)
+
+        # We need to reset the current subscription group after it is used, so it won't leak
+        # outside to other topics that would be defined without a defined subscription group
+        self.current_subscription_group_name = nil
       end
 
       # @return [Array<Routing::SubscriptionGroup>] all the subscription groups build based on

data/lib/karafka/routing/proxy.rb

@@ -7,15 +7,6 @@ module Karafka
     class Proxy
       attr_reader :target
 
-      # We should proxy only non ? and = methods as we want to have a regular dsl
-      IGNORED_POSTFIXES = %w[
-        ?
-        =
-        !
-      ].freeze
-
-      private_constant :IGNORED_POSTFIXES
-
       # @param target [Object] target object to which we proxy any DSL call
       # @param block [Proc] block that we want to evaluate in the proxy context
       def initialize(target, &block)
@@ -25,21 +16,23 @@ module Karafka
 
       # Translates the no "=" DSL of routing into elements assignments on target
       # @param method_name [Symbol] name of the missing method
-      # @param arguments [Array] array with it's arguments
-      # @param block [Proc] block provided to the method
-      def method_missing(method_name, *arguments, &block)
+      def method_missing(method_name, ...)
         return super unless respond_to_missing?(method_name)
 
-        @target.public_send(:"#{method_name}=", *arguments, &block)
+        if @target.respond_to?(:"#{method_name}=")
+          @target.public_send(:"#{method_name}=", ...)
+        else
+          @target.public_send(method_name, ...)
+        end
       end
 
       # Tells whether or not a given element exists on the target
       # @param method_name [Symbol] name of the missing method
       # @param include_private [Boolean] should we include private in the check as well
       def respond_to_missing?(method_name, include_private = false)
-        return false if IGNORED_POSTFIXES.any? { |postfix| method_name.to_s.end_with?(postfix) }
-
-        @target.respond_to?(:"#{method_name}=", include_private) || super
+        @target.respond_to?(:"#{method_name}=", include_private) ||
+          @target.respond_to?(method_name, include_private) ||
+          super
       end
     end
   end

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.6'
+  VERSION = '2.0.8'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.6
+  version: 2.0.8
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2022-09-02 00:00:00.000000000 Z
+date: 2022-09-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core