karafka 2.0.35 → 2.0.37

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e2d2197fb0fe7eae8db43cc45ffbdcdc6da83d60731ae63b82dc4a24d852736e
-  data.tar.gz: 76b4b3f8ffb915b96a16f1eb83503283e5da46039fb952af383322773c4be551
+  metadata.gz: bab9c1d7bc952b4ecbfc4fad794d7e7c861cd3a332cc5d9058cef6c0bd9b57cb
+  data.tar.gz: 7662bd8dc5748d9112f3c72b2912619534e45750188f78df2f69a7e6ae1f9c31
 SHA512:
-  metadata.gz: 35d69a8fa9a26462f2167174d8700648909bbcf38363c5adef2f7ee42995b524c0d244bffe0bd76698f65df5475db78b38914f38d7eda239376c4031555ee885
-  data.tar.gz: 49a2256930ce83bc46b6471cf702b1ea2fe6f897a805f92e4dee7c3d9034a9625a44ecff0222b8396ed73858af5571cca4eaabca8d188a81312312c2225daa97
+  metadata.gz: 9a99a84d538a74bd27d5a0f585a12dbbe67eb76ab63cc1a0984cbe1562f230070ad482418f85393a3a479e81534a0957a0863c91f3a7f5b6433f74efd317c79e
+  data.tar.gz: 7da6129cd795f65d821bae897864648e4a5e37c0d07e8745f110f0d03a23d688d7717ab6c42652c4660fe3be3d26ee7051a8c2c66c1fcb62834a1c4159bd4ac4

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Karafka framework changelog
 
+## 2.0.37 (2023-03-20)
+- [Fix] Declarative topics execution on a secondary cluster run topics creation on the primary one (#1365)
+- [Fix]  Admin read operations commit offset when not needed (#1369)
+
+## 2.0.36 (2023-03-17)
+- [Refactor] Rename internal naming of `Structurable` to `Declaratives` for declarative topics feature.
+- [Fix] AJ + DLQ + MOM + LRJ is pausing indefinitely after the first job (#1362)
+
 ## 2.0.35 (2023-03-13)
 - **[Feature]** Allow for defining topics config via the DSL and its automatic creation via CLI command.
 - **[Feature]** Allow for full topics reset and topics repartitioning via the CLI. 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.35)
+    karafka (2.0.37)
       karafka-core (>= 2.0.12, < 3.0.0)
       thor (>= 0.20)
       waterdrop (>= 2.4.10, < 3.0.0)

data/config/locales/errors.yml

@@ -45,10 +45,10 @@ en:
       dead_letter_queue.topic_format: 'needs to be a string with a Kafka accepted format'
       dead_letter_queue.active_format: needs to be either true or false
       active_format: needs to be either true or false
-      structurable.partitions_format: needs to be more or equal to 1
-      structurable.active_format: needs to be true
-      structurable.replication_factor_format: needs to be more or equal to 1
-      structurable.details_format: needs to be a hash with only symbol keys
+      declaratives.partitions_format: needs to be more or equal to 1
+      declaratives.active_format: needs to be true
+      declaratives.replication_factor_format: needs to be more or equal to 1
+      declaratives.details_format: needs to be a hash with only symbol keys
       inconsistent_namespacing: |
         needs to be consistent namespacing style
         disable this validation by setting config.strict_topics_namespacing to false

data/lib/karafka/admin.rb

@@ -28,7 +28,10 @@ module Karafka
       'enable.partition.eof': true,
       'statistics.interval.ms': 0,
       # Fetch at most 5 MBs when using admin
-      'fetch.message.max.bytes': 5 * 1_048_576
+      'fetch.message.max.bytes': 5 * 1_048_576,
+      # Do not commit offset automatically, this prevents offset tracking for operations involving
+      # a consumer instance
+      'enable.auto.commit': false
     }.freeze
 
     private_constant :Topic, :CONFIG_DEFAULTS, :MAX_WAIT_TIMEOUT, :MAX_ATTEMPTS

data/lib/karafka/cli/topics.rb

@@ -29,7 +29,7 @@ module Karafka
 
       # Creates topics based on the routing setup and configuration
       def create
-        structurable_routing_topics.each do |topic|
+        declaratives_routing_topics.each do |topic|
           name = topic.name
 
           if existing_topics_names.include?(name)
@@ -38,9 +38,9 @@ module Karafka
             puts "Creating topic #{name}..."
             Admin.create_topic(
               name,
-              topic.structurable.partitions,
-              topic.structurable.replication_factor,
-              topic.structurable.details
+              topic.declaratives.partitions,
+              topic.declaratives.replication_factor,
+              topic.declaratives.details
             )
             puts "#{green('Created')} topic #{name}."
           end
@@ -49,7 +49,7 @@ module Karafka
 
       # Deletes routing based topics
       def delete
-        structurable_routing_topics.each do |topic|
+        declaratives_routing_topics.each do |topic|
           name = topic.name
 
           if existing_topics_names.include?(name)
@@ -89,7 +89,7 @@ module Karafka
           [topic.fetch(:topic_name), topic.fetch(:partition_count)]
         end.to_h
 
-        structurable_routing_topics.each do |topic|
+        declaratives_routing_topics.each do |topic|
           name = topic.name
 
           desired_count = topic.config.partitions
@@ -112,21 +112,24 @@ module Karafka
       # @note If topic is defined in multiple consumer groups, first config will be used. This
       #   means, that this CLI will not work for simultaneous management of multiple clusters from
       #   a single CLI command execution flow.
-      def structurable_routing_topics
-        return @structurable_routing_topics if @structurable_routing_topics
+      def declaratives_routing_topics
+        return @declaratives_routing_topics if @declaratives_routing_topics
 
         collected_topics = {}
+        default_servers = Karafka::App.config.kafka[:'bootstrap.servers']
 
         App.consumer_groups.each do |consumer_group|
           consumer_group.topics.each do |topic|
             # Skip topics that were explicitly disabled from management
-            next unless topic.structurable.active?
+            next unless topic.declaratives.active?
+            # If bootstrap servers are different, consider this a different cluster
+            next unless default_servers == topic.kafka[:'bootstrap.servers']
 
             collected_topics[topic.name] ||= topic
           end
         end
 
-        @structurable_routing_topics = collected_topics.values
+        @declaratives_routing_topics = collected_topics.values
       end
 
       # @return [Array<Hash>] existing topics details

data/lib/karafka/pro/processing/strategies/aj_dlq_lrj_mom.rb

@@ -23,8 +23,10 @@ module Karafka
         # This case is a bit of special. Please see the `AjDlqMom` for explanation on how the
         # offset management works in this case.
         module AjDlqLrjMom
-          include AjLrjMom
-          include AjDlqMom
+          # We can use the same code as for VP because non VP behaves like:
+          # - with one virtual partition
+          # - with "never ending" collapse
+          include AjDlqLrjMomVp
 
           # Features for this strategy
           FEATURES = %i[

data/lib/karafka/routing/features/{structurable → declaratives}/config.rb

@@ -3,8 +3,8 @@
 module Karafka
   module Routing
     module Features
-      class Structurable < Base
-        # Config for structurable feature
+      class Declaratives < Base
+        # Config for declarative topics feature
         Config = Struct.new(
           :active,
           :partitions,

data/lib/karafka/routing/features/{structurable → declaratives}/contract.rb

@@ -3,7 +3,7 @@
 module Karafka
   module Routing
     module Features
-      class Structurable < Base
+      class Declaratives < Base
         # Basic validation of the Kafka expected config details
         class Contract < Contracts::Base
           configure do |config|
@@ -14,7 +14,7 @@ module Karafka
             ).fetch('en').fetch('validations').fetch('topic')
           end
 
-          nested :structurable do
+          nested :declaratives do
             required(:active) { |val| [true, false].include?(val) }
             required(:partitions) { |val| val.is_a?(Integer) && val.positive? }
             required(:replication_factor) { |val| val.is_a?(Integer) && val.positive? }

data/lib/karafka/routing/features/{structurable → declaratives}/topic.rb

@@ -3,7 +3,7 @@
 module Karafka
   module Routing
     module Features
-      class Structurable < Base
+      class Declaratives < Base
         # Extension for managing Kafka topic configuration
         module Topic
           # @param active [Boolean] is the topic structure management feature active
@@ -12,7 +12,7 @@ module Karafka
           # @param details [Hash] extra configuration for the topic
           # @return [Config] defined structure
           def config(active: true, partitions: 1, replication_factor: 1, **details)
-            @structurable ||= Config.new(
+            @declaratives ||= Config.new(
               active: active,
               partitions: partitions,
               replication_factor: replication_factor,
@@ -21,20 +21,20 @@ module Karafka
           end
 
           # @return [Config] config details
-          def structurable
+          def declaratives
             config
           end
 
-          # @return [true] structurable is always active
-          def structurable?
-            structurable.active?
+          # @return [true] declaratives is always active
+          def declaratives?
+            declaratives.active?
           end
 
-          # @return [Hash] topic with all its native configuration options plus structurable
+          # @return [Hash] topic with all its native configuration options plus declaratives
           #   settings
           def to_h
             super.merge(
-              structurable: structurable.to_h
+              declaratives: declaratives.to_h
             ).freeze
           end
         end

data/lib/karafka/routing/features/{structurable.rb → declaratives.rb}

@@ -7,7 +7,7 @@ module Karafka
       # topics structure for test/development, etc. This allows to share the same set of settings
       # for topics despite the environment. Pretty much similar to how the `structure.sql` or
       # `schema.rb` operate for SQL dbs.
-      class Structurable < Base
+      class Declaratives < Base
       end
     end
   end

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.35'
+  VERSION = '2.0.37'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.35
+  version: 2.0.37
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2023-03-13 00:00:00.000000000 Z
+date: 2023-03-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -306,14 +306,14 @@ files:
 - lib/karafka/routing/features/dead_letter_queue/config.rb
 - lib/karafka/routing/features/dead_letter_queue/contract.rb
 - lib/karafka/routing/features/dead_letter_queue/topic.rb
+- lib/karafka/routing/features/declaratives.rb
+- lib/karafka/routing/features/declaratives/config.rb
+- lib/karafka/routing/features/declaratives/contract.rb
+- lib/karafka/routing/features/declaratives/topic.rb
 - lib/karafka/routing/features/manual_offset_management.rb
 - lib/karafka/routing/features/manual_offset_management/config.rb
 - lib/karafka/routing/features/manual_offset_management/contract.rb
 - lib/karafka/routing/features/manual_offset_management/topic.rb
-- lib/karafka/routing/features/structurable.rb
-- lib/karafka/routing/features/structurable/config.rb
-- lib/karafka/routing/features/structurable/contract.rb
-- lib/karafka/routing/features/structurable/topic.rb
 - lib/karafka/routing/proxy.rb
 - lib/karafka/routing/router.rb
 - lib/karafka/routing/subscription_group.rb