karafka 2.0.34 → 2.0.36

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 36d890d825aaeaee5349dcc653d888da3a023c01a837864544a905db977569c4
-  data.tar.gz: be442485812a05a030bab33da31a8e2fda684add8c4d59a0af78f517bb2519bd
+  metadata.gz: 128d19fa065cbb43fae7a23a424e44ec30077fe58c93d7c122da299ec5b98df1
+  data.tar.gz: 0b396fa75d93257f32a9ddf94f0dc89d2456b6ed07acad5c581908d148cf0a6f
 SHA512:
-  metadata.gz: d92be137485c436c1ed02435669785422e6e4da194ab19b97dca31f70b530fe7e0ae4e6b0c7c54895dbceca2d8fe4ef1bdcabdb287affe06e377942062777979
-  data.tar.gz: 0525b652373088a7a692134a6a4c89487e495e01934504d256a60ae7805c92f65a308d38c8d75ad806f39f5128d2d00865e10b1b4ae326c7dd60e7594c68558e
+  metadata.gz: 68d4da94eaf7a1033c95dd02bdc9a4bd8e69040df4db304dbcb6f8255997a40c003c999eea6442f6c1a1640c3e1ed44032c3826191e96085afbccc8600a8f2b6
+  data.tar.gz: 025fbc8edb26f6f8442e11ea90afdc3e07873efcba3f6778549cd66e0738dfec73d956d877c44ea39bc1064c22df9fca6d3c68a42ef97ad4438ed7812b690ea1

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Karafka framework changelog
 
+## 2.0.36 (2023-03-17)
+- [Refactor] Rename internal naming of `Structurable` to `Declaratives` for declarative topics feature.
+- [Fix] AJ + DLQ + MOM + LRJ is pausing indefinitely after the first job (#1362)
+
+## 2.0.35 (2023-03-13)
+- **[Feature]** Allow for defining topics config via the DSL and its automatic creation via CLI command.
+- **[Feature]** Allow for full topics reset and topics repartitioning via the CLI. 
+
 ## 2.0.34 (2023-03-04)
 - [Improvement] Attach an `embedded` tag to Karafka processes started using the embedded API.
 - [Change] Renamed `Datadog::Listener` to `Datadog::MetricsListener` for consistency. (#1124)

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.34)
+    karafka (2.0.36)
       karafka-core (>= 2.0.12, < 3.0.0)
       thor (>= 0.20)
       waterdrop (>= 2.4.10, < 3.0.0)
@@ -37,7 +37,7 @@ GEM
       mini_portile2 (~> 2.6)
       rake (> 12)
     mini_portile2 (2.8.1)
-    minitest (5.17.0)
+    minitest (5.18.0)
     rake (13.0.6)
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
@@ -61,12 +61,13 @@ GEM
     thor (1.2.1)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.4.11)
+    waterdrop (2.5.0)
       karafka-core (>= 2.0.12, < 3.0.0)
       zeitwerk (~> 2.3)
     zeitwerk (2.6.7)
 
 PLATFORMS
+  arm64-darwin-21
   x86_64-linux
 
 DEPENDENCIES
@@ -78,4 +79,4 @@ DEPENDENCIES
   simplecov
 
 BUNDLED WITH
-   2.4.6
+   2.4.7

data/config/locales/errors.yml

@@ -45,6 +45,10 @@ en:
       dead_letter_queue.topic_format: 'needs to be a string with a Kafka accepted format'
       dead_letter_queue.active_format: needs to be either true or false
       active_format: needs to be either true or false
+      declaratives.partitions_format: needs to be more or equal to 1
+      declaratives.active_format: needs to be true
+      declaratives.replication_factor_format: needs to be more or equal to 1
+      declaratives.details_format: needs to be a hash with only symbol keys
       inconsistent_namespacing: |
         needs to be consistent namespacing style
         disable this validation by setting config.strict_topics_namespacing to false

data/lib/karafka/cli/topics.rb

@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+module Karafka
+  class Cli < Thor
+    # CLI actions related to Kafka cluster topics management
+    class Topics < Base
+      include Helpers::Colorize
+
+      desc 'Allows for the topics management (create, delete, reset, repartition)'
+      # @param action [String] action we want to take
+      def call(action = 'missing')
+        case action
+        when 'create'
+          create
+        when 'delete'
+          delete
+        when 'reset'
+          reset
+        when 'repartition'
+          repartition
+        when 'migrate'
+          migrate
+        else
+          raise ::ArgumentError, "Invalid topics action: #{action}"
+        end
+      end
+
+      private
+
+      # Creates topics based on the routing setup and configuration
+      def create
+        declaratives_routing_topics.each do |topic|
+          name = topic.name
+
+          if existing_topics_names.include?(name)
+            puts "#{yellow('Skipping')} because topic #{name} already exists."
+          else
+            puts "Creating topic #{name}..."
+            Admin.create_topic(
+              name,
+              topic.declaratives.partitions,
+              topic.declaratives.replication_factor,
+              topic.declaratives.details
+            )
+            puts "#{green('Created')} topic #{name}."
+          end
+        end
+      end
+
+      # Deletes routing based topics
+      def delete
+        declaratives_routing_topics.each do |topic|
+          name = topic.name
+
+          if existing_topics_names.include?(name)
+            puts "Deleting topic #{name}..."
+            Admin.delete_topic(name)
+            puts "#{green('Deleted')} topic #{name}."
+          else
+            puts "#{yellow('Skipping')} because topic #{name} does not exist."
+          end
+        end
+      end
+
+      # Deletes routing based topics and re-creates them
+      def reset
+        delete
+
+        # We need to invalidate the metadata cache, otherwise we will think, that the topic
+        # already exists
+        @existing_topics = nil
+
+        create
+      end
+
+      # Creates missing topics and aligns the partitions count
+      def migrate
+        create
+
+        @existing_topics = nil
+
+        repartition
+      end
+
+      # Increases number of partitions on topics that have less partitions than defined
+      # Will **not** create topics if missing.
+      def repartition
+        existing_partitions = existing_topics.map do |topic|
+          [topic.fetch(:topic_name), topic.fetch(:partition_count)]
+        end.to_h
+
+        declaratives_routing_topics.each do |topic|
+          name = topic.name
+
+          desired_count = topic.config.partitions
+          existing_count = existing_partitions.fetch(name, false)
+
+          if existing_count && existing_count < desired_count
+            puts "Increasing number of partitions to #{desired_count} on topic #{name}..."
+            Admin.create_partitions(name, desired_count)
+            change = desired_count - existing_count
+            puts "#{green('Created')} #{change} additional partitions on topic #{name}."
+          elsif existing_count
+            puts "#{yellow('Skipping')} because topic #{name} has #{existing_count} partitions."
+          else
+            puts "#{yellow('Skipping')} because topic #{name} does not exist."
+          end
+        end
+      end
+
+      # @return [Array<Karafka::Routing::Topic>] all available topics that can be managed
+      # @note If topic is defined in multiple consumer groups, first config will be used. This
+      #   means, that this CLI will not work for simultaneous management of multiple clusters from
+      #   a single CLI command execution flow.
+      def declaratives_routing_topics
+        return @declaratives_routing_topics if @declaratives_routing_topics
+
+        collected_topics = {}
+
+        App.consumer_groups.each do |consumer_group|
+          consumer_group.topics.each do |topic|
+            # Skip topics that were explicitly disabled from management
+            next unless topic.declaratives.active?
+
+            collected_topics[topic.name] ||= topic
+          end
+        end
+
+        @declaratives_routing_topics = collected_topics.values
+      end
+
+      # @return [Array<Hash>] existing topics details
+      def existing_topics
+        @existing_topics ||= Admin.cluster_info.topics
+      end
+
+      # @return [Array<String>] names of already existing topics
+      def existing_topics_names
+        existing_topics.map { |topic| topic.fetch(:topic_name) }
+      end
+    end
+  end
+end

data/lib/karafka/helpers/colorize.rb

@@ -15,6 +15,12 @@ module Karafka
       def red(string)
         "\033[0;31m#{string}\033[0m"
       end
+
+      # @param string [String] string we want to have in yellow
+      # @return [String] yellow string
+      def yellow(string)
+        "\033[1;33m#{string}\033[0m"
+      end
     end
   end
 end

data/lib/karafka/pro/processing/strategies/aj_dlq_lrj_mom.rb

@@ -23,8 +23,10 @@ module Karafka
         # This case is a bit of special. Please see the `AjDlqMom` for explanation on how the
         # offset management works in this case.
         module AjDlqLrjMom
-          include AjLrjMom
-          include AjDlqMom
+          # We can use the same code as for VP because non VP behaves like:
+          # - with one virtual partition
+          # - with "never ending" collapse
+          include AjDlqLrjMomVp
 
           # Features for this strategy
           FEATURES = %i[

data/lib/karafka/routing/features/declaratives/config.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class Declaratives < Base
+        # Config for declarative topics feature
+        Config = Struct.new(
+          :active,
+          :partitions,
+          :replication_factor,
+          :details,
+          keyword_init: true
+        ) { alias_method :active?, :active }
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/declaratives/contract.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class Declaratives < Base
+        # Basic validation of the Kafka expected config details
+        class Contract < Contracts::Base
+          configure do |config|
+            config.error_messages = YAML.safe_load(
+              File.read(
+                File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+              )
+            ).fetch('en').fetch('validations').fetch('topic')
+          end
+
+          nested :declaratives do
+            required(:active) { |val| [true, false].include?(val) }
+            required(:partitions) { |val| val.is_a?(Integer) && val.positive? }
+            required(:replication_factor) { |val| val.is_a?(Integer) && val.positive? }
+            required(:details) do |val|
+              val.is_a?(Hash) &&
+                val.keys.all? { |key| key.is_a?(Symbol) }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/declaratives/topic.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class Declaratives < Base
+        # Extension for managing Kafka topic configuration
+        module Topic
+          # @param active [Boolean] is the topic structure management feature active
+          # @param partitions [Integer]
+          # @param replication_factor [Integer]
+          # @param details [Hash] extra configuration for the topic
+          # @return [Config] defined structure
+          def config(active: true, partitions: 1, replication_factor: 1, **details)
+            @declaratives ||= Config.new(
+              active: active,
+              partitions: partitions,
+              replication_factor: replication_factor,
+              details: details
+            )
+          end
+
+          # @return [Config] config details
+          def declaratives
+            config
+          end
+
+          # @return [true] declaratives is always active
+          def declaratives?
+            declaratives.active?
+          end
+
+          # @return [Hash] topic with all its native configuration options plus declaratives
+          #   settings
+          def to_h
+            super.merge(
+              declaratives: declaratives.to_h
+            ).freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/declaratives.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      # This feature allows to store per topic structure that can be later on used to bootstrap
+      # topics structure for test/development, etc. This allows to share the same set of settings
+      # for topics despite the environment. Pretty much similar to how the `structure.sql` or
+      # `schema.rb` operate for SQL dbs.
+      class Declaratives < Base
+      end
+    end
+  end
+end

data/lib/karafka/templates/karafka.rb.erb

@@ -53,6 +53,11 @@ class KarafkaApp < Karafka::App
     # active_job_topic :default
 <% end -%>
     topic :example do
+      # Uncomment this if you want Karafka to manage your topics configuration
+      # Managing topics configuration via routing will allow you to ensure config consistency
+      # across multiple environments
+      #
+      # config(partitions: 2, 'cleanup.policy': 'compact')
       consumer ExampleConsumer
     end
   end

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.34'
+  VERSION = '2.0.36'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.34
+  version: 2.0.36
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2023-03-04 00:00:00.000000000 Z
+date: 2023-03-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -168,6 +168,7 @@ files:
 - lib/karafka/cli/info.rb
 - lib/karafka/cli/install.rb
 - lib/karafka/cli/server.rb
+- lib/karafka/cli/topics.rb
 - lib/karafka/connection/client.rb
 - lib/karafka/connection/consumer_group_coordinator.rb
 - lib/karafka/connection/listener.rb
@@ -305,6 +306,10 @@ files:
 - lib/karafka/routing/features/dead_letter_queue/config.rb
 - lib/karafka/routing/features/dead_letter_queue/contract.rb
 - lib/karafka/routing/features/dead_letter_queue/topic.rb
+- lib/karafka/routing/features/declaratives.rb
+- lib/karafka/routing/features/declaratives/config.rb
+- lib/karafka/routing/features/declaratives/contract.rb
+- lib/karafka/routing/features/declaratives/topic.rb
 - lib/karafka/routing/features/manual_offset_management.rb
 - lib/karafka/routing/features/manual_offset_management/config.rb
 - lib/karafka/routing/features/manual_offset_management/contract.rb