karafka 2.0.30 → 2.0.31

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2c983b3fbf07db3e91f3c051a8b4de4b2d88619877fbf66a4e434e38bf67c96b
-  data.tar.gz: babb3c513e6bb3fd0610363e3bbd9e7fcf07b1820f02fdd80ccacc8a109aab08
+  metadata.gz: a4c598dcc6414c2a24f50452d5379c2a0aa26d7d1c7fd8d1b07f2af19a6e8125
+  data.tar.gz: 184a8b1e7bb1f62672f90904ce6c3fd5e2da5f1868bcb2a971a1e7872e8c3252
 SHA512:
-  metadata.gz: ea000c13a834ffba97c1837fa1f19dabbf78dbe8958354a8200bb5d2c0940852343521e6740113e07890f97a7e6edd108d8595a66688c4c14dbf313e86ec715f
-  data.tar.gz: 81ae980ef7950c6cae7d1bea2751d1826e5bbf4c1384be4736af895977fb7ff38f70c24a9b7b248cb18744e042ffb0ba0dc154171c29dcc73262733d87c31ea1
+  metadata.gz: a9d382efa4846f4419d86d1fa2742f663e2c255b203612b6810ab5bc44aafc8c9a1c6c299330889c0cc54cb952235ddbb8d45a3c2ed30c7132b8522fc46ef8d1
+  data.tar.gz: 5a5907e0e7b958a6277785ef4254ac5d8cf7f563e371fdf3fe7705719164362858b46efae62dda6c7a1f02646ffd57d05437fde496ebf2b7e36bdcb8ac57865d

data/.ruby-version

@@ -1 +1 @@
-3.2.0
+3.2.1

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Karafka framework changelog
 
+## 2.0.31 (2022-02-12)
+- [Feature] Allow for adding partitions via `Admin#create_partitions` API.
+- [Fix] Do not ignore admin errors upon invalid configuration (#1254)
+- [Fix] Topic name validation (#1300) - CandyFet
+- [Improvement] Increase the `max_wait_timeout` on admin operations to five minutes to make sure no timeout on heavily loaded clusters.
+- [Maintenance] Require `karafka-core` >= `2.0.11` and switch to shared RSpec locator.
+- [Maintenance] Require `karafka-rdkafka` >= `0.12.1`
+
 ## 2.0.30 (2022-01-31)
 - [Improvement] Alias `--consumer-groups` with `--include-consumer-groups`
 - [Improvement] Alias `--subscription-groups` with `--include-subscription-groups`

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.30)
-      karafka-core (>= 2.0.9, < 3.0.0)
+    karafka (2.0.31)
+      karafka-core (>= 2.0.11, < 3.0.0)
       thor (>= 0.20)
       waterdrop (>= 2.4.10, < 3.0.0)
       zeitwerk (~> 2.3)
@@ -29,10 +29,10 @@ GEM
       activesupport (>= 5.0)
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.0.9)
+    karafka-core (2.0.11)
       concurrent-ruby (>= 1.1)
-      karafka-rdkafka (>= 0.12)
-    karafka-rdkafka (0.12.0)
+      karafka-rdkafka (>= 0.12.1)
+    karafka-rdkafka (0.12.1)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
@@ -43,7 +43,7 @@ GEM
       rspec-core (~> 3.12.0)
       rspec-expectations (~> 3.12.0)
       rspec-mocks (~> 3.12.0)
-    rspec-core (3.12.0)
+    rspec-core (3.12.1)
       rspec-support (~> 3.12.0)
     rspec-expectations (3.12.2)
       diff-lcs (>= 1.2.0, < 2.0)
@@ -79,4 +79,4 @@ DEPENDENCIES
   simplecov
 
 BUNDLED WITH
-   2.4.5
+   2.4.6

data/config/locales/errors.yml

@@ -45,10 +45,12 @@ en:
       dead_letter_queue.topic_format: 'needs to be a string with a Kafka accepted format'
       dead_letter_queue.active_format: needs to be either true or false
       active_format: needs to be either true or false
+      inconsistent_namespacing: needs to be consistent namespacing style
 
     consumer_group:
       missing: needs to be present
       topics_names_not_unique: all topic names within a single consumer group must be unique
+      topics_namespaced_names_not_unique: all topic names within a single consumer group must be unique considering namespacing styles
       id_format: 'needs to be a string with a Kafka accepted format'
       topics_format: needs to be a non-empty array
 

data/karafka.gemspec

@@ -21,7 +21,7 @@ Gem::Specification.new do |spec|
     without having to focus on things that are not your business domain.
   DESC
 
-  spec.add_dependency 'karafka-core', '>= 2.0.9', '< 3.0.0'
+  spec.add_dependency 'karafka-core', '>= 2.0.11', '< 3.0.0'
   spec.add_dependency 'thor', '>= 0.20'
   spec.add_dependency 'waterdrop', '>= 2.4.10', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'

data/lib/karafka/admin.rb

@@ -14,6 +14,13 @@ module Karafka
     # do not have in the routing
     Topic = Struct.new(:name, :deserializer)
 
+    # We wait only for this amount of time before raising error as we intercept this error and
+    # retry after checking that the operation was finished or failed using external factor.
+    MAX_WAIT_TIMEOUT = 1
+
+    # How many times should be try. 1 x 60 => 60 seconds wait in total
+    MAX_ATTEMPTS = 60
+
     # Defaults for config
     CONFIG_DEFAULTS = {
       'group.id': 'karafka_admin',
@@ -22,7 +29,7 @@ module Karafka
       'statistics.interval.ms': 0
     }.freeze
 
-    private_constant :Topic, :CONFIG_DEFAULTS
+    private_constant :Topic, :CONFIG_DEFAULTS, :MAX_WAIT_TIMEOUT, :MAX_ATTEMPTS
 
     class << self
       # Allows us to read messages from the topic
@@ -86,9 +93,12 @@ module Karafka
       #   https://kafka.apache.org/documentation/#topicconfigs
       def create_topic(name, partitions, replication_factor, topic_config = {})
         with_admin do |admin|
-          admin.create_topic(name, partitions, replication_factor, topic_config)
+          handler = admin.create_topic(name, partitions, replication_factor, topic_config)
 
-          sleep(0.2) until topics_names.include?(name)
+          with_re_wait(
+            -> { handler.wait(max_wait_timeout: MAX_WAIT_TIMEOUT) },
+            -> { topics_names.include?(name) }
+          )
         end
       end
 
@@ -97,9 +107,27 @@ module Karafka
       # @param name [String] topic name
       def delete_topic(name)
         with_admin do |admin|
-          admin.delete_topic(name)
+          handler = admin.delete_topic(name)
 
-          sleep(0.2) while topics_names.include?(name)
+          with_re_wait(
+            -> { handler.wait(max_wait_timeout: MAX_WAIT_TIMEOUT) },
+            -> { !topics_names.include?(name) }
+          )
+        end
+      end
+
+      # Creates more partitions for a given topic
+      #
+      # @param name [String] topic name
+      # @param partitions [Integer] total number of partitions we expect to end up with
+      def create_partitions(name, partitions)
+        with_admin do |admin|
+          handler = admin.create_partitions(name, partitions)
+
+          with_re_wait(
+            -> { handler.wait(max_wait_timeout: MAX_WAIT_TIMEOUT) },
+            -> { topic(name).fetch(:partition_count) >= partitions }
+          )
         end
       end
 
@@ -117,6 +145,13 @@ module Karafka
         cluster_info.topics.map { |topic| topic.fetch(:topic_name) }
       end
 
+      # Finds details about given topic
+      # @param name [String] topic name
+      # @return [Hash] topic details
+      def topic(name)
+        cluster_info.topics.find { |topic| topic[:topic_name] == name }
+      end
+
       # Creates admin instance and yields it. After usage it closes the admin instance
       def with_admin
         admin = config(:producer).admin
@@ -133,6 +168,28 @@ module Karafka
         consumer&.close
       end
 
+      # There are some cases where rdkafka admin operations finish successfully but without the
+      # callback being triggered to materialize the post-promise object. Until this is fixed we
+      # can figure out, that operation we wanted to do finished successfully by checking that the
+      # effect of the command (new topic, more partitions, etc) is handled. Exactly for that we
+      # use the breaker. It we get a timeout, we can check that what we wanted to achieve has
+      # happened via the breaker check, hence we do not need to wait any longer.
+      #
+      # @param handler [Proc] the wait handler operation
+      # @param breaker [Proc] extra condition upon timeout that indicates things were finished ok
+      def with_re_wait(handler, breaker)
+        attempt ||= 0
+        attempt += 1
+
+        handler.call
+      rescue Rdkafka::AbstractHandle::WaitTimeoutError
+        return if breaker.call
+
+        retry if attempt <= MAX_ATTEMPTS
+
+        raise
+      end
+
       # @param type [Symbol] type of config we want
       # @return [::Rdkafka::Config] rdkafka config
       def config(type)

data/lib/karafka/contracts/consumer_group.rb

@@ -24,6 +24,32 @@ module Karafka
 
         [[%i[topics], :names_not_unique]]
       end
+
+      virtual do |data, errors|
+        next unless errors.empty?
+
+        names = data.fetch(:topics).map { |topic| topic[:name] }
+        names_hash = names.each_with_object({}) { |n, h| h[n] = true }
+        error_occured = false
+        names.each do |n|
+          # Skip topic names that are not namespaced
+          next unless n.chars.find { |c| ['.', '_'].include?(c) }
+
+          if n.chars.include?('.')
+            # Check underscore styled topic
+            underscored_topic = n.tr('.', '_')
+            error_occured = names_hash[underscored_topic] ? true : false
+          else
+            # Check dot styled topic
+            dot_topic = n.tr('_', '.')
+            error_occured = names_hash[dot_topic] ? true : false
+          end
+        end
+
+        next unless error_occured
+
+        [[%i[topics], :topics_namespaced_names_not_unique]]
+      end
     end
   end
 end

data/lib/karafka/contracts/topic.rb

@@ -48,6 +48,17 @@ module Karafka
           [[%w[kafka], e.message]]
         end
       end
+
+      virtual do |data, errors|
+        next unless errors.empty?
+
+        value = data.fetch(:name)
+        namespacing_chars_count = value.chars.find_all { |c| ['.', '_'].include?(c) }.uniq.count
+
+        next if namespacing_chars_count <= 1
+
+        [[%w[name], :inconsistent_namespacing]]
+      end
     end
   end
 end

data/lib/karafka/contracts.rb

@@ -5,6 +5,6 @@ module Karafka
   module Contracts
     # Regexp for validating format of groups and topics
     # @note It is not nested inside of the contracts, as it is used by couple of them
-    TOPIC_REGEXP = /\A(\w|-|\.)+\z/
+    TOPIC_REGEXP = /^[A-Za-z0-9\-_.]+$/
   end
 end

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.30'
+  VERSION = '2.0.31'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.30
+  version: 2.0.31
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2023-01-31 00:00:00.000000000 Z
+date: 2023-02-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -43,7 +43,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.9
+        version: 2.0.11
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -53,7 +53,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.9
+        version: 2.0.11
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -356,7 +356,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.1
+rubygems_version: 3.4.6
 signing_key: 
 specification_version: 4
 summary: Karafka is Ruby and Rails efficient Kafka processing framework.