karafka 2.0.7 → 2.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0abed3f97a58be6b48f640468f7d7e6d48bc0960596b21d022b4616dd047be28
-  data.tar.gz: 48143253beee640e25e47a81474767c179e715e855d6173b59566483a57af5a8
+  metadata.gz: 1a646d63d674e6ecb80625097426f48294fe8e8b3f3e9e020cf92645a5728251
+  data.tar.gz: 461c58c355f84b81c3396cb4164a426c3f1e87747b3101de1e7c6fbf2a876778
 SHA512:
-  metadata.gz: 9c9f8c170ac82fc0f1eb6ea41698dcd82cc525006931a59443d004c94eb18b56ffcb67eb1eb45fcc1fd557fee22e6e63ceb7a8a001245469e3e574d87c88c8e8
-  data.tar.gz: 47bc7e7dfe5ca3d503a3cb18da4e4b95c076197dc26b5633195e169d3f4d94da4effaf27bd4360ddff1481031b1ee20f61e465e24f6984570f6067ca4fbd51ea
+  metadata.gz: 623832478b11b1fa61c7906f423417500838e30ef97256a59290d73dcfe17670cf7eb11adcf33d422ae590c6ce5f19215340c89d369f8ec5b4e609af07d0befb
+  data.tar.gz: 97e50ed131c939d09de884d3ddba826381e7d314d3bc4660ecf947fa6582d9e6f16e023dad121612eadffcd441a73874c96e287e5ec1885e008d843357e94b93

data/CHANGELOG.md

@@ -1,6 +1,10 @@
 # Karafka framework changelog
 
-## 2.0.7 (Unreleased)
+## 2.0.8 (2022-09-19)
+- [Breaking change] Rename Virtual Partitions `concurrency` to `max_partitions` to avoid confusion  (#1023).
+-  Allow for block based subscription groups management (#1030).
+
+## 2.0.7 (2022-09-05)
 - [Breaking change] Redefine the Virtual Partitions routing DSL to accept concurrency
 - Allow for `concurrency` setting in Virtual Partitions to extend or limit number of jobs per regular partition. This allows to make sure, we do not use all the threads on virtual partitions jobs
 - Allow for creation of as many Virtual Partitions as needed, without taking global `concurrency` into consideration

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.7)
+    karafka (2.0.8)
       karafka-core (>= 2.0.2, < 3.0.0)
       rdkafka (>= 0.12)
       thor (>= 0.20)

data/config/errors.yml

@@ -56,4 +56,4 @@ en:
     pro_consumer_group_topic:
       consumer_format: needs to inherit from Karafka::Pro::BaseConsumer and not Karafka::Consumer
       virtual_partitions.partitioner_respond_to_call: needs to be defined and needs to respond to `#call`
-      virtual_partitions.concurrency_format: needs to be equl or more than 1
+      virtual_partitions.max_partitions_format: needs to be equl or more than 1

data/lib/karafka/base_consumer.rb

@@ -124,7 +124,7 @@ module Karafka
 
     # Method that will perform business logic and on data received from Kafka (it will consume
     #   the data)
-    # @note This method needs bo be implemented in a subclass. We stub it here as a failover if
+    # @note This method needs to be implemented in a subclass. We stub it here as a failover if
     #   someone forgets about it or makes on with typo
     def consume
       raise NotImplementedError, 'Implement this in a subclass'

data/lib/karafka/pro/base_consumer.rb

@@ -102,7 +102,7 @@ module Karafka
         else
           # If processing failed, we need to pause
           # For long running job this will overwrite the default never-ending pause and will cause
-          # the processing th keep going after the error backoff
+          # the processing to keep going after the error backoff
           pause(@seek_offset || first_message.offset)
         end
       end

data/lib/karafka/pro/contracts/consumer_group_topic.rb

@@ -25,7 +25,7 @@ module Karafka
         nested(:virtual_partitions) do
           required(:active) { |val| [true, false].include?(val) }
           required(:partitioner) { |val| val.nil? || val.respond_to?(:call) }
-          required(:concurrency) { |val| val.is_a?(Integer) && val >= 1 }
+          required(:max_partitions) { |val| val.is_a?(Integer) && val >= 1 }
         end
 
         virtual do |data, errors|

data/lib/karafka/pro/processing/partitioner.rb

@@ -24,7 +24,7 @@ module Karafka
           # We only partition work if we have a virtual partitioner and more than one thread to
           # process the data. With one thread it is not worth partitioning the work as the work
           # itself will be assigned to one thread (pointless work)
-          if ktopic.virtual_partitions? && ktopic.virtual_partitions.concurrency > 1
+          if ktopic.virtual_partitions? && ktopic.virtual_partitions.max_partitions > 1
             # We need to reduce it to number of threads, so the group_id is not a direct effect
             # of the end user action. Otherwise the persistence layer for consumers would cache
             # it forever and it would cause memory leaks
@@ -32,14 +32,14 @@ module Karafka
                         .group_by { |msg| ktopic.virtual_partitions.partitioner.call(msg) }
                         .values
 
-            # Reduce the max concurrency to a size that matches the concurrency
+            # Reduce the number of virtual partitions to a size that matches the max_partitions
             # As mentioned above we cannot use the partitioning keys directly as it could cause
             # memory leaks
             #
             # The algorithm here is simple, we assume that the most costly in terms of processing,
             # will be processing of the biggest group and we reduce the smallest once to have
-            # max of groups equal to concurrency
-            while groupings.size > ktopic.virtual_partitions.concurrency
+            # max of groups equal to max_partitions
+            while groupings.size > ktopic.virtual_partitions.max_partitions
               groupings.sort_by! { |grouping| -grouping.size }
 
               # Offset order needs to be maintained for virtual partitions

data/lib/karafka/pro/routing/topic_extensions.rb

@@ -23,7 +23,7 @@ module Karafka
         VirtualPartitions = Struct.new(
           :active,
           :partitioner,
-          :concurrency,
+          :max_partitions,
           keyword_init: true
         ) { alias_method :active?, :active }
 
@@ -34,7 +34,7 @@ module Karafka
           end
         end
 
-        # @param concurrency [Integer] max number of virtual partitions that can come out of the
+        # @param max_partitions [Integer] max number of virtual partitions that can come out of the
         #   single distribution flow. When set to more than the Karafka threading, will create
         #   more work than workers. When less, can ensure we have spare resources to process other
         #   things in parallel.
@@ -42,12 +42,12 @@ module Karafka
         # @return [VirtualPartitions] method that allows to set the virtual partitions details
         #   during the routing configuration and then allows to retrieve it
         def virtual_partitions(
-          concurrency: Karafka::App.config.concurrency,
+          max_partitions: Karafka::App.config.concurrency,
           partitioner: nil
         )
           @virtual_partitions ||= VirtualPartitions.new(
             active: !partitioner.nil?,
-            concurrency: concurrency,
+            max_partitions: max_partitions,
             partitioner: partitioner
           )
         end

data/lib/karafka/processing/jobs/base.rb

@@ -2,7 +2,7 @@
 
 module Karafka
   module Processing
-    # Namespace for all the jobs that are suppose to run in workers.
+    # Namespace for all the jobs that are supposed to run in workers.
     module Jobs
       # Base class for all the jobs types that are suppose to run in workers threads.
       # Each job can have 3 main entry-points: `#before_call`, `#call` and `#after_call`
@@ -10,7 +10,7 @@ module Karafka
       class Base
         extend Forwardable
 
-        # @note Since one job has always one executer, we use the jobs id and group id as reference
+        # @note Since one job has always one executor, we use the jobs id and group id as reference
         def_delegators :executor, :id, :group_id
 
         attr_reader :executor

data/lib/karafka/processing/jobs_queue.rb

@@ -13,7 +13,7 @@ module Karafka
       # @return [Karafka::Processing::JobsQueue]
       def initialize
         @queue = Queue.new
-        # Those queues will act as a semaphores internally. Since we need an indicator for waiting
+        # Those queues will act as semaphores internally. Since we need an indicator for waiting
         # we could use Thread.pass but this is expensive. Instead we can just lock until any
         # of the workers finishes their work and we can re-check. This means that in the worse
         # scenario, we will context switch 10 times per poll instead of getting this thread

data/lib/karafka/routing/builder.rb

@@ -66,6 +66,16 @@ module Karafka
         end
       end
 
+      # Handles the simple routing case where we create one consumer group and allow for further
+      # subscription group customization
+      # @param subscription_group_name [String, Symbol] subscription group id
+      # @param block [Proc] further topics definitions
+      def subscription_group(subscription_group_name, &block)
+        consumer_group('app') do
+          target.public_send(:subscription_group=, subscription_group_name, &block)
+        end
+      end
+
       # In case we use simple style of routing, all topics will be assigned to the same consumer
       # group that will be based on the client_id
       #

data/lib/karafka/routing/consumer_group.rb

@@ -10,6 +10,12 @@ module Karafka
     class ConsumerGroup
       attr_reader :id, :topics, :name
 
+      # This is a "virtual" attribute that is not building subscription groups.
+      # It allows us to store the "current" subscription group defined in the routing
+      # This subscription group id is then injected into topics, so we can compute the subscription
+      # groups
+      attr_accessor :current_subscription_group_name
+
       # @param name [String, Symbol] raw name of this consumer group. Raw means, that it does not
       #   yet have an application client_id namespace, this will be added here by default.
       #   We add it to make a multi-system development easier for people that don't use
@@ -32,7 +38,27 @@ module Karafka
       def topic=(name, &block)
         topic = Topic.new(name, self)
         @topics << Proxy.new(topic, &block).target
-        @topics.last
+        built_topic = @topics.last
+        # We overwrite it conditionally in case it was not set by the user inline in the topic
+        # block definition
+        built_topic.subscription_group ||= current_subscription_group_name
+        built_topic
+      end
+
+      # Assigns the current subscription group id based on the defined one and allows for further
+      # topic definition
+      # @param name [String, Symbol]
+      # @param block [Proc] block that may include topics definitions
+      def subscription_group=(name, &block)
+        # We cast it here, so the routing supports symbol based but that's anyhow later on
+        # validated as a string
+        self.current_subscription_group_name = name.to_s
+
+        Proxy.new(self, &block)
+
+        # We need to reset the current subscription group after it is used, so it won't leak
+        # outside to other topics that would be defined without a defined subscription group
+        self.current_subscription_group_name = nil
       end
 
       # @return [Array<Routing::SubscriptionGroup>] all the subscription groups build based on

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.7'
+  VERSION = '2.0.8'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.7
+  version: 2.0.8
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2022-09-05 00:00:00.000000000 Z
+date: 2022-09-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core

metadata.gz.sig

@@ -1,2 +1,3 @@
-�`�j�CA�T(<`��C(�[Țl��B�� �|ۉS(�wMDQ��
�W�Ji�#��<��9�21�
-��N�Z�UAŲw��r���&nDu��F^��!4�����y��~��u��迂8��
+<�=�g>�g��M`��Fѐzv���V
U.o����
+�!�J����6���g�o�j]�f�{�����xOm�)e�������������Q��]���l"��Yj�R-�s��6FPb���W씕M]�kd�A��#��j�^����!�4������áe��߷���fMo!��3�B�{{�&�B�R�E�Fj}�m9���
+8ԏ��u�G�a�O�P'��~�@�ƌI�2��S}ňi�	F��W:�=Ѐ�t_��y���P��	���k��(+3n����t����%��,�|�ZF�Z�q����m�h��])M�T��#g=���H�_X�O�#Bf'>��2���+