karafka 2.5.4 → 2.5.6

data/lib/karafka/pro/cleaner/errors.rb

@@ -1,7 +1,24 @@
 # frozen_string_literal: true
 
-# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
-# See LICENSE for details.
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
 
 module Karafka
   module Pro

data/lib/karafka/pro/cleaner/messages/message.rb

@@ -1,7 +1,24 @@
 # frozen_string_literal: true
 
-# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
-# See LICENSE for details.
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
 
 module Karafka
   module Pro

data/lib/karafka/pro/cleaner/messages/messages.rb

@@ -1,7 +1,24 @@
 # frozen_string_literal: true
 
-# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
-# See LICENSE for details.
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
 
 module Karafka
   module Pro
@@ -36,6 +53,16 @@ module Karafka
               super(&)
             end
           end
+
+          # Cleans all messages in the batch
+          #
+          # @param metadata [Boolean] should we also clean metadata alongside the payload for
+          #   each message. `true` by default.
+          #
+          # @note This is a convenience method that calls `clean!` on each message in the batch.
+          def clean!(metadata: true)
+            each { |message| message.clean!(metadata: metadata) }
+          end
         end
       end
     end

data/lib/karafka/pro/cleaner/messages/metadata.rb

@@ -1,7 +1,24 @@
 # frozen_string_literal: true
 
-# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
-# See LICENSE for details.
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
 
 module Karafka
   module Pro

data/lib/karafka/pro/cleaner.rb

@@ -1,7 +1,24 @@
 # frozen_string_literal: true
 
-# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
-# See LICENSE for details.
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
 
 module Karafka
   module Pro

data/lib/karafka/pro/cli/contracts/server.rb

@@ -1,7 +1,24 @@
 # frozen_string_literal: true
 
-# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
-# See LICENSE for details.
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
 
 module Karafka
   module Pro
@@ -13,8 +30,8 @@ module Karafka
         class Server < Karafka::Cli::Contracts::Server
           configure do |config|
             config.error_messages = YAML.safe_load_file(
-              File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
-            ).fetch('en').fetch('validations').fetch('cli').fetch('server')
+              File.join(Karafka.gem_root, "config", "locales", "errors.yml")
+            ).fetch("en").fetch("validations").fetch("cli").fetch("server")
           end
 
           %i[
@@ -49,10 +66,10 @@ module Karafka
               next if value.empty?
 
               subscription_groups = Karafka::App
-                                    .consumer_groups
-                                    .map(&:subscription_groups)
-                                    .flatten
-                                    .map(&:name)
+                .consumer_groups
+                .map(&:subscription_groups)
+                .flatten
+                .map(&:name)
 
               next if (value - subscription_groups).empty?
 
@@ -70,12 +87,12 @@ module Karafka
               next if value.empty?
 
               topics = Karafka::App
-                       .consumer_groups
-                       .map(&:subscription_groups)
-                       .flatten
-                       .map(&:topics)
-                       .map { |gtopics| gtopics.map(&:name) }
-                       .flatten
+                .consumer_groups
+                .map(&:subscription_groups)
+                .flatten
+                .map(&:topics)
+                .map { |gtopics| gtopics.map(&:name) }
+                .flatten
 
               next if (value - topics).empty?
 

data/lib/karafka/pro/cli/parallel_segments/base.rb

@@ -1,7 +1,24 @@
 # frozen_string_literal: true
 
-# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
-# See LICENSE for details.
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
 
 module Karafka
   module Pro
@@ -30,9 +47,9 @@ module Karafka
             requested_groups = options[:groups].dup || []
 
             workable_groups = Karafka::App
-                              .routes
-                              .select(&:parallel_segments?)
-                              .group_by(&:segment_origin)
+              .routes
+              .select(&:parallel_segments?)
+              .group_by(&:segment_origin)
 
             # Use all if none provided
             return workable_groups if requested_groups.empty?
@@ -68,7 +85,7 @@ module Karafka
             consumer_groups = [segment_origin, segments.map(&:name)].flatten
 
             consumer_groups_with_topics = consumer_groups
-                                          .to_h { |name| [name, topics_names] }
+              .to_h { |name| [name, topics_names] }
 
             lags_with_offsets = Karafka::Admin.read_lags_with_offsets(
               consumer_groups_with_topics

data/lib/karafka/pro/cli/parallel_segments/collapse.rb

@@ -1,7 +1,24 @@
 # frozen_string_literal: true
 
-# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
-# See LICENSE for details.
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
 
 module Karafka
   module Pro
@@ -20,12 +37,12 @@ module Karafka
         class Collapse < Base
           # Runs the collapse operation
           def call
-            puts 'Starting parallel segments collapse...'
+            puts "Starting parallel segments collapse..."
 
             segments_count = applicable_groups.size
 
             if segments_count.zero?
-              puts "#{red('No')} consumer groups with parallel segments configuration found"
+              puts "#{red("No")} consumer groups with parallel segments configuration found"
 
               return
             end
@@ -57,7 +74,7 @@ module Karafka
             end
 
             puts
-            puts "Collapse completed #{green('successfully')}!"
+            puts "Collapse completed #{green("successfully")}!"
           end
 
           private
@@ -124,7 +141,7 @@ module Karafka
 
                 puts(
                   "  Inconclusive offsets for #{red(topic_name)}##{red(partition_id)}: " \
-                  "#{parallel_offsets.to_a.join(', ')}"
+                  "#{parallel_offsets.to_a.join(", ")}"
                 )
               end
             end
@@ -133,7 +150,7 @@ module Karafka
 
             raise(
               Karafka::Errors::CommandValidationError,
-              "Parallel segments for #{red(segment_origin)} have #{red('inconclusive')} offsets"
+              "Parallel segments for #{red(segment_origin)} have #{red("inconclusive")} offsets"
             )
           end
 

data/lib/karafka/pro/cli/parallel_segments/distribute.rb

@@ -1,7 +1,24 @@
 # frozen_string_literal: true
 
-# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
-# See LICENSE for details.
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
 
 module Karafka
   module Pro
@@ -27,12 +44,12 @@ module Karafka
         class Distribute < Base
           # Runs the distribution process
           def call
-            puts 'Starting parallel segments distribution...'
+            puts "Starting parallel segments distribution..."
 
             segments_count = applicable_groups.size
 
             if segments_count.zero?
-              puts "#{red('No')} consumer groups with parallel segments configuration found"
+              puts "#{red("No")} consumer groups with parallel segments configuration found"
 
               return
             end
@@ -62,7 +79,7 @@ module Karafka
             end
 
             puts
-            puts "Distribution completed #{green('successfully')}!"
+            puts "Distribution completed #{green("successfully")}!"
           end
 
           private

data/lib/karafka/pro/cli/parallel_segments.rb

@@ -1,7 +1,24 @@
 # frozen_string_literal: true
 
-# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
-# See LICENSE for details.
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
 
 module Karafka
   module Pro
@@ -14,11 +31,11 @@ module Karafka
           kafka_config: %i[kafka]
         )
 
-        desc 'Allows for parallel segments management'
+        desc "Allows for parallel segments management"
 
         option(
           :groups,
-          'Names of consumer groups on which we want to run the command. All if not provided',
+          "Names of consumer groups on which we want to run the command. All if not provided",
           Array,
           %w[
             --groups
@@ -33,7 +50,7 @@ module Karafka
         # their existing distributed offsets to be reset.
         option(
           :force,
-          'Should an operation on the parallel segments consumer group be forced',
+          "Should an operation on the parallel segments consumer group be forced",
           TrueClass,
           %w[
             --force
@@ -41,13 +58,13 @@ module Karafka
         )
 
         # @param action [String] action we want to take
-        def call(action = 'distribute')
+        def call(action = "distribute")
           case action
-          when 'distribute'
+          when "distribute"
             Distribute.new(options).call
-          when 'collapse'
+          when "collapse"
             Collapse.new(options).call
-          when 'reset'
+          when "reset"
             Collapse.new(options).call
             Distribute.new(options).call
           else

data/lib/karafka/pro/cli/topics/health.rb

@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Cli
+      module Topics
+        # Checks health of Kafka topics by analyzing replication and durability settings
+        # Identifies topics with:
+        # - No redundancy (RF=1)
+        # - Zero fault tolerance (RF <= min.insync.replicas)
+        # - Low durability (min.insync.replicas=1)
+        class Health < Karafka::Cli::Topics::Base
+          # Executes the health check across all topics in the cluster
+          # @return [Boolean] true if issues were found, false if all topics are healthy
+          def call
+            issues_found = false
+
+            supervised("Checking topics health") do
+              topics = existing_topics
+              puts "Found #{topics.count} topics to check..."
+              puts
+
+              topics.each do |topic|
+                # Skip internal Kafka topics
+                next if topic[:topic_name].start_with?("__")
+
+                issue = analyze_topic(topic)
+
+                if issue
+                  display_issue(issue)
+                  issues_found = true
+                else
+                  puts "#{green("✓")} #{topic[:topic_name]}"
+                end
+              end
+            end
+
+            puts
+            display_summary(issues_found)
+            puts
+
+            issues_found
+          end
+
+          private
+
+          # Analyzes a single topic for replication and durability issues
+          # @param topic [Hash] topic metadata from cluster info
+          # @return [Hash, nil] issue hash if problems found, nil if healthy
+          def analyze_topic(topic)
+            topic_name = topic[:topic_name]
+            rf = topic[:partitions].first&.fetch(:replica_count) || 0
+            min_isr = fetch_min_insync_replicas(topic_name)
+
+            check_replication_issue(topic_name, rf, min_isr)
+          end
+
+          # Fetches min.insync.replicas configuration for a topic
+          # @param topic_name [String] name of the topic
+          # @return [Integer] min.insync.replicas value
+          def fetch_min_insync_replicas(topic_name)
+            configs = Admin::Configs.describe(
+              Admin::Configs::Resource.new(type: :topic, name: topic_name)
+            ).first.configs
+
+            configs.find { |c| c.name == "min.insync.replicas" }.value.to_i
+          end
+
+          # Checks for replication and durability issues
+          # @param topic_name [String] name of the topic
+          # @param rf [Integer] replication factor
+          # @param min_isr [Integer] min.insync.replicas setting
+          # @return [Hash, nil] issue hash if problems found, nil if healthy
+          def check_replication_issue(topic_name, rf, min_isr)
+            return build_issue(topic_name, rf, min_isr, :critical, "RF=#{rf} (no redundancy)") if rf == 1
+
+            if rf <= min_isr
+              return build_issue(
+                topic_name,
+                rf,
+                min_isr,
+                :critical,
+                "RF=#{rf}, min.insync=#{min_isr} (zero fault tolerance)"
+              )
+            end
+
+            if min_isr == 1
+              return build_issue(
+                topic_name,
+                rf,
+                min_isr,
+                :warning,
+                "RF=#{rf}, min.insync=#{min_isr} (low durability)"
+              )
+            end
+
+            nil
+          end
+
+          # Builds an issue hash with consistent structure
+          # @param topic [String] topic name
+          # @param rf [Integer] replication factor
+          # @param min_isr [Integer] min.insync.replicas
+          # @param severity [Symbol] :critical or :warning
+          # @param message [String] human-readable issue description
+          # @return [Hash] issue details
+          def build_issue(topic, rf, min_isr, severity, message)
+            {
+              topic: topic,
+              rf: rf,
+              min_isr: min_isr,
+              severity: severity,
+              message: message
+            }
+          end
+
+          # Displays a single issue immediately as it's found
+          # @param issue [Hash] issue details
+          def display_issue(issue)
+            color_method = (issue[:severity] == :critical) ? :red : :yellow
+            symbol = (issue[:severity] == :critical) ? "\u2717" : "\u26A0"
+            puts "#{send(color_method, symbol)} #{issue[:topic]}: #{issue[:message]}"
+          end
+
+          # Displays final summary and recommendations
+          # @param issues_found [Boolean] whether any issues were found
+          def display_summary(issues_found)
+            if issues_found
+              puts
+              puts "#{grey("Recommendations")}:"
+              puts "  #{grey("\u2022")} Ensure RF >= 3 for production topics"
+              puts "  #{grey("\u2022")} Set min.insync.replicas to at least 2"
+              puts "  #{grey("\u2022")} Maintain RF > min.insync.replicas for fault tolerance"
+            else
+              puts "#{green("\u2713")} All topics are healthy"
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/cli/topics.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Cli
+      # Pro extension for the Topics CLI command
+      # Adds health checking functionality to the base topics command
+      module Topics
+        # Extends the base topics command to add health action support
+        module Extension
+          # @param action [String] action we want to take
+          def call(action = "help")
+            if action == "health"
+              detailed_exit_code = options.fetch(:detailed_exitcode, false)
+              issues_found = Pro::Cli::Topics::Health.new.call
+
+              return unless detailed_exit_code
+
+              # Exit with code 2 if issues found, code 0 if all healthy
+              issues_found ? exit(2) : exit(0)
+            else
+              super
+            end
+          end
+        end
+      end
+    end
+  end
+end
+
+# Extend the OSS Topics command with Pro functionality
+Karafka::Cli::Topics.prepend(Karafka::Pro::Cli::Topics::Extension)