karafka 2.5.5 → 2.5.7

data/lib/karafka/pro/cli/parallel_segments/collapse.rb

@@ -37,12 +37,12 @@ module Karafka
         class Collapse < Base
           # Runs the collapse operation
           def call
-            puts 'Starting parallel segments collapse...'
+            puts "Starting parallel segments collapse..."
 
             segments_count = applicable_groups.size
 
             if segments_count.zero?
-              puts "#{red('No')} consumer groups with parallel segments configuration found"
+              puts "#{red("No")} consumer groups with parallel segments configuration found"
 
               return
             end
@@ -74,7 +74,7 @@ module Karafka
             end
 
             puts
-            puts "Collapse completed #{green('successfully')}!"
+            puts "Collapse completed #{green("successfully")}!"
           end
 
           private
@@ -141,7 +141,7 @@ module Karafka
 
                 puts(
                   "  Inconclusive offsets for #{red(topic_name)}##{red(partition_id)}: " \
-                  "#{parallel_offsets.to_a.join(', ')}"
+                  "#{parallel_offsets.to_a.join(", ")}"
                 )
               end
             end
@@ -150,7 +150,7 @@ module Karafka
 
             raise(
               Karafka::Errors::CommandValidationError,
-              "Parallel segments for #{red(segment_origin)} have #{red('inconclusive')} offsets"
+              "Parallel segments for #{red(segment_origin)} have #{red("inconclusive")} offsets"
             )
           end
 

data/lib/karafka/pro/cli/parallel_segments/distribute.rb

@@ -44,12 +44,12 @@ module Karafka
         class Distribute < Base
           # Runs the distribution process
           def call
-            puts 'Starting parallel segments distribution...'
+            puts "Starting parallel segments distribution..."
 
             segments_count = applicable_groups.size
 
             if segments_count.zero?
-              puts "#{red('No')} consumer groups with parallel segments configuration found"
+              puts "#{red("No")} consumer groups with parallel segments configuration found"
 
               return
             end
@@ -79,7 +79,7 @@ module Karafka
             end
 
             puts
-            puts "Distribution completed #{green('successfully')}!"
+            puts "Distribution completed #{green("successfully")}!"
           end
 
           private

data/lib/karafka/pro/cli/parallel_segments.rb

@@ -31,11 +31,11 @@ module Karafka
           kafka_config: %i[kafka]
         )
 
-        desc 'Allows for parallel segments management'
+        desc "Allows for parallel segments management"
 
         option(
           :groups,
-          'Names of consumer groups on which we want to run the command. All if not provided',
+          "Names of consumer groups on which we want to run the command. All if not provided",
           Array,
           %w[
             --groups
@@ -50,7 +50,7 @@ module Karafka
         # their existing distributed offsets to be reset.
         option(
           :force,
-          'Should an operation on the parallel segments consumer group be forced',
+          "Should an operation on the parallel segments consumer group be forced",
           TrueClass,
           %w[
             --force
@@ -58,13 +58,13 @@ module Karafka
         )
 
         # @param action [String] action we want to take
-        def call(action = 'distribute')
+        def call(action = "distribute")
           case action
-          when 'distribute'
+          when "distribute"
             Distribute.new(options).call
-          when 'collapse'
+          when "collapse"
             Collapse.new(options).call
-          when 'reset'
+          when "reset"
             Collapse.new(options).call
             Distribute.new(options).call
           else

data/lib/karafka/pro/cli/topics/health.rb

@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Cli
+      module Topics
+        # Checks health of Kafka topics by analyzing replication and durability settings
+        # Identifies topics with:
+        # - No redundancy (RF=1)
+        # - Zero fault tolerance (RF <= min.insync.replicas)
+        # - Low durability (min.insync.replicas=1)
+        class Health < Karafka::Cli::Topics::Base
+          # Executes the health check across all topics in the cluster
+          # @return [Boolean] true if issues were found, false if all topics are healthy
+          def call
+            issues_found = false
+
+            supervised("Checking topics health") do
+              topics = existing_topics
+              puts "Found #{topics.count} topics to check..."
+              puts
+
+              topics.each do |topic|
+                # Skip internal Kafka topics
+                next if topic[:topic_name].start_with?("__")
+
+                issue = analyze_topic(topic)
+
+                if issue
+                  display_issue(issue)
+                  issues_found = true
+                else
+                  puts "#{green("✓")} #{topic[:topic_name]}"
+                end
+              end
+            end
+
+            puts
+            display_summary(issues_found)
+            puts
+
+            issues_found
+          end
+
+          private
+
+          # Analyzes a single topic for replication and durability issues
+          # @param topic [Hash] topic metadata from cluster info
+          # @return [Hash, nil] issue hash if problems found, nil if healthy
+          def analyze_topic(topic)
+            topic_name = topic[:topic_name]
+            rf = topic[:partitions].first&.fetch(:replica_count) || 0
+            min_isr = fetch_min_insync_replicas(topic_name)
+
+            check_replication_issue(topic_name, rf, min_isr)
+          end
+
+          # Fetches min.insync.replicas configuration for a topic
+          # @param topic_name [String] name of the topic
+          # @return [Integer] min.insync.replicas value
+          def fetch_min_insync_replicas(topic_name)
+            configs = Admin::Configs.describe(
+              Admin::Configs::Resource.new(type: :topic, name: topic_name)
+            ).first.configs
+
+            configs.find { |c| c.name == "min.insync.replicas" }.value.to_i
+          end
+
+          # Checks for replication and durability issues
+          # @param topic_name [String] name of the topic
+          # @param rf [Integer] replication factor
+          # @param min_isr [Integer] min.insync.replicas setting
+          # @return [Hash, nil] issue hash if problems found, nil if healthy
+          def check_replication_issue(topic_name, rf, min_isr)
+            return build_issue(topic_name, rf, min_isr, :critical, "RF=#{rf} (no redundancy)") if rf == 1
+
+            if rf <= min_isr
+              return build_issue(
+                topic_name,
+                rf,
+                min_isr,
+                :critical,
+                "RF=#{rf}, min.insync=#{min_isr} (zero fault tolerance)"
+              )
+            end
+
+            if min_isr == 1
+              return build_issue(
+                topic_name,
+                rf,
+                min_isr,
+                :warning,
+                "RF=#{rf}, min.insync=#{min_isr} (low durability)"
+              )
+            end
+
+            nil
+          end
+
+          # Builds an issue hash with consistent structure
+          # @param topic [String] topic name
+          # @param rf [Integer] replication factor
+          # @param min_isr [Integer] min.insync.replicas
+          # @param severity [Symbol] :critical or :warning
+          # @param message [String] human-readable issue description
+          # @return [Hash] issue details
+          def build_issue(topic, rf, min_isr, severity, message)
+            {
+              topic: topic,
+              rf: rf,
+              min_isr: min_isr,
+              severity: severity,
+              message: message
+            }
+          end
+
+          # Displays a single issue immediately as it's found
+          # @param issue [Hash] issue details
+          def display_issue(issue)
+            color_method = (issue[:severity] == :critical) ? :red : :yellow
+            symbol = (issue[:severity] == :critical) ? "\u2717" : "\u26A0"
+            puts "#{send(color_method, symbol)} #{issue[:topic]}: #{issue[:message]}"
+          end
+
+          # Displays final summary and recommendations
+          # @param issues_found [Boolean] whether any issues were found
+          def display_summary(issues_found)
+            if issues_found
+              puts
+              puts "#{grey("Recommendations")}:"
+              puts "  #{grey("\u2022")} Ensure RF >= 3 for production topics"
+              puts "  #{grey("\u2022")} Set min.insync.replicas to at least 2"
+              puts "  #{grey("\u2022")} Maintain RF > min.insync.replicas for fault tolerance"
+            else
+              puts "#{green("\u2713")} All topics are healthy"
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/cli/topics.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Cli
+      # Pro extension for the Topics CLI command
+      # Adds health checking functionality to the base topics command
+      module Topics
+        # Extends the base topics command to add health action support
+        module Extension
+          # @param action [String] action we want to take
+          def call(action = "help")
+            if action == "health"
+              detailed_exit_code = options.fetch(:detailed_exitcode, false)
+              issues_found = Pro::Cli::Topics::Health.new.call
+
+              return unless detailed_exit_code
+
+              # Exit with code 2 if issues found, code 0 if all healthy
+              issues_found ? exit(2) : exit(0)
+            else
+              super
+            end
+          end
+        end
+      end
+    end
+  end
+end
+
+# Extend the OSS Topics command with Pro functionality
+Karafka::Cli::Topics.prepend(Karafka::Pro::Cli::Topics::Extension)

data/lib/karafka/pro/connection/manager.rb

@@ -42,8 +42,8 @@ module Karafka
           @mutex = Mutex.new
           @changes = Hash.new do |h, k|
             h[k] = {
-              state: '',
-              join_state: '',
+              state: "",
+              join_state: "",
               state_age: 0,
               changed_at: monotonic_now
             }
@@ -86,9 +86,9 @@ module Karafka
           times = []
           # stateage is in microseconds
           # We monitor broker changes to make sure we do not introduce extra friction
-          times << (statistics['brokers'].values.map { |stats| stats['stateage'] }.min / 1_000)
-          times << statistics['cgrp']['rebalance_age']
-          times << statistics['cgrp']['stateage']
+          times << (statistics["brokers"].values.map { |stats| stats["stateage"] }.min / 1_000)
+          times << statistics["cgrp"]["rebalance_age"]
+          times << statistics["cgrp"]["stateage"]
 
           # Keep the previous change age for changes that were triggered by us
           previous_changed_at = @changes[subscription_group_id][:changed_at]
@@ -96,8 +96,8 @@ module Karafka
           @changes[subscription_group_id].merge!(
             state_age: times.min,
             changed_at: previous_changed_at,
-            join_state: statistics['cgrp']['join_state'],
-            state: statistics['cgrp']['state']
+            join_state: statistics["cgrp"]["join_state"],
+            state: statistics["cgrp"]["state"]
           )
         end
 
@@ -198,11 +198,11 @@ module Karafka
         # partitions assigned in sgs that can be scaled. If that is the case, we scale up.
         def scale_up
           multi_part_sgs_families = Karafka::App
-                                    .assignments
-                                    .select { |_, partitions| partitions.size > 1 }
-                                    .keys
-                                    .map { |sg| sg.subscription_group.name }
-                                    .uniq
+            .assignments
+            .select { |_, partitions| partitions.size > 1 }
+            .keys
+            .map { |sg| sg.subscription_group.name }
+            .uniq
 
           # Select connections for scaling up
           in_sg_families do |first_subscription_group, sg_listeners|
@@ -256,8 +256,8 @@ module Karafka
 
             state[:state_age] >= @scale_delay &&
               (monotonic_now - state[:changed_at]) >= @scale_delay &&
-              state[:state] == 'up' &&
-              state[:join_state] == 'steady'
+              state[:state] == "up" &&
+              state[:join_state] == "steady"
           end
         end
 

data/lib/karafka/pro/encryption/contracts/config.rb

@@ -29,8 +29,8 @@ module Karafka
         class Config < Karafka::Contracts::Base
           configure do |config|
             config.error_messages = YAML.safe_load_file(
-              File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
-            ).fetch('en').fetch('validations').fetch('setup').fetch('config')
+              File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
+            ).fetch("en").fetch("validations").fetch("setup").fetch("config")
           end
 
           nested(:encryption) do

data/lib/karafka/pro/encryption/messages/middleware.rb

@@ -41,12 +41,12 @@ module Karafka
             payload = message[:payload]
 
             message[:headers] ||= {}
-            message[:headers]['encryption'] = version
+            message[:headers]["encryption"] = version
             message[:payload] = cipher.encrypt(payload)
 
             return message unless fingerprinter
 
-            message[:headers]['encryption_fingerprint'] = fingerprinter.hexdigest(payload)
+            message[:headers]["encryption_fingerprint"] = fingerprinter.hexdigest(payload)
 
             message
           end

data/lib/karafka/pro/encryption/messages/parser.rb

@@ -39,8 +39,8 @@ module Karafka
           # @return [Object] deserialized payload
           def call(message)
             headers = message.headers
-            encryption = headers['encryption']
-            fingerprint = headers['encryption_fingerprint']
+            encryption = headers["encryption"]
+            fingerprint = headers["encryption_fingerprint"]
 
             return super unless active && encryption
 

data/lib/karafka/pro/encryption/setup/config.rb

@@ -35,12 +35,12 @@ module Karafka
           # Supporting versions allows us to be able to rotate private and public keys in case
           # we would need this. We can increase the version, rotate and Karafka when decrypting
           # will figure out proper private key based on the version
-          setting(:version, default: '1')
+          setting(:version, default: "1")
 
           # We always support one public key for producing messages
           # Public key needs to be always present even if we do not plan to produce messages from
           # a Karafka process. This is because of the web-ui and potentially other cases like this
-          setting(:public_key, default: '')
+          setting(:public_key, default: "")
 
           # Private keys in pem format, where the key is the version and value is the key.
           # This allows us to support key rotation

data/lib/karafka/pro/iterator/tpl_builder.rb

@@ -177,8 +177,8 @@ module Karafka
 
               next unless SUPPORTED_NAMED_POSITIONS.include?(named_offset)
 
-              @mapped_topics[name][partition] = -1 if named_offset == 'latest'
-              @mapped_topics[name][partition] = -2 if named_offset == 'earliest'
+              @mapped_topics[name][partition] = -1 if named_offset == "latest"
+              @mapped_topics[name][partition] = -2 if named_offset == "earliest"
             end
           end
         end

data/lib/karafka/pro/iterator.rb

@@ -54,7 +54,7 @@ module Karafka
       #   so we don't stop polling data even when reaching the end (end on a given moment)
       def initialize(
         topics,
-        settings: { 'auto.offset.reset': 'beginning' },
+        settings: { "auto.offset.reset": "beginning" },
         yield_nil: false,
         max_wait_time: 200
       )

data/lib/karafka/pro/loader.rb

@@ -37,6 +37,7 @@ module Karafka
         encryption/setup/config
         encryption/contracts/config
         encryption/messages/parser
+        cli/topics
       ].freeze
 
       # Zeitwerk pro loader
@@ -50,7 +51,7 @@ module Karafka
         def require_all
           FORCE_LOADED.each { |file| require_relative(file) }
 
-          PRO_LOADER.push_dir(Karafka.core_root.join('pro'), namespace: Karafka::Pro)
+          PRO_LOADER.push_dir(Karafka.core_root.join("pro"), namespace: Karafka::Pro)
           PRO_LOADER.setup
           PRO_LOADER.eager_load
         end

data/lib/karafka/pro/processing/adaptive_iterator/consumer.rb

@@ -36,7 +36,7 @@ module Karafka
             tracker = Tracker.new(
               adi_config.safety_margin,
               coordinator.last_polled_at,
-              topic.subscription_group.kafka.fetch(:'max.poll.interval.ms')
+              topic.subscription_group.kafka.fetch(:"max.poll.interval.ms")
             )
 
             messages.each(*args) do |message|

data/lib/karafka/pro/processing/coordinators/virtual_offset_manager.rb

@@ -91,17 +91,27 @@ module Karafka
             @offsets_metadata[offset] = offset_metadata
             @current_offset_metadata = offset_metadata
 
-            group = @groups.find { |reg_group| reg_group.include?(offset) }
+            group = nil
+            position = nil
+
+            @groups.each do |reg_group|
+              pos = reg_group.index(offset)
+
+              if pos
+                group = reg_group
+                position = pos
+                break
+              end
+            end
 
             # This case can happen when someone uses MoM and wants to mark message from a previous
             # batch as consumed. We can add it, since the real offset refresh will point to it
             unless group
               group = [offset]
+              position = 0
               @groups << group
             end
 
-            position = group.index(offset)
-
             # Mark all previous messages from the same group also as virtually consumed
             group[0..position].each do |markable_offset|
               # Set previous messages metadata offset as the offset of higher one for overwrites
@@ -135,7 +145,7 @@ module Karafka
 
           # @return [Array<Integer>] Offsets of messages already marked as consumed virtually
           def marked
-            @marked.select { |_, status| status }.map(&:first).sort
+            @marked.select { |_, status| status }.map { |offset, _| offset }.sort
           end
 
           # Is there a real offset we can mark as consumed
@@ -150,13 +160,13 @@ module Karafka
             raise Errors::InvalidRealOffsetUsageError unless markable?
 
             offset_metadata = case @offset_metadata_strategy
-                              when :exact
-                                @offsets_metadata.fetch(@real_offset)
-                              when :current
-                                @current_offset_metadata
-                              else
-                                raise Errors::UnsupportedCaseError, @offset_metadata_strategy
-                              end
+            when :exact
+              @offsets_metadata.fetch(@real_offset)
+            when :current
+              @current_offset_metadata
+            else
+              raise Errors::UnsupportedCaseError, @offset_metadata_strategy
+            end
 
             [
               Messages::Seek.new(
@@ -171,11 +181,11 @@ module Karafka
           private
 
           # Recomputes the biggest possible real offset we can have.
-          # It picks the the biggest offset that has uninterrupted stream of virtually marked as
+          # It picks the biggest offset that has uninterrupted stream of virtually marked as
           # consumed because this will be the collective offset.
           def materialize_real_offset
-            @marked.to_a.sort_by(&:first).each do |offset, marked|
-              break unless marked
+            @marked.keys.sort.each do |offset|
+              break unless @marked[offset]
 
               @real_offset = offset
             end

data/lib/karafka/pro/processing/filters/base.rb

@@ -46,7 +46,7 @@ module Karafka
           # @param messages [Array<Karafka::Messages::Message>] array with messages. Please keep
           #   in mind, this may already be partial due to execution of previous filters.
           def apply!(messages)
-            raise NotImplementedError, 'Implement in a subclass'
+            raise NotImplementedError, "Implement in a subclass"
           end
 
           # @return [Symbol] filter post-execution action on consumer. Either `:skip`, `:pause` or

data/lib/karafka/pro/processing/filters/delayer.rb

@@ -63,14 +63,14 @@ module Karafka
 
             timeout = (@delay / 1_000.0) - (::Time.now.utc - @cursor.timestamp)
 
-            timeout <= 0 ? 0 : timeout * 1_000
+            (timeout <= 0) ? 0 : timeout * 1_000
           end
 
           # @return [Symbol] action to take on post-filtering
           def action
             return :skip unless applied?
 
-            timeout <= 0 ? :seek : :pause
+            (timeout <= 0) ? :seek : :pause
           end
         end
       end

data/lib/karafka/pro/processing/filters/inline_insights_delayer.rb

@@ -73,7 +73,7 @@ module Karafka
 
           # @return [Integer, nil] ms timeout in case of pause or nil if not delaying
           def timeout
-            @cursor && applied? ? PAUSE_TIMEOUT : nil
+            (@cursor && applied?) ? PAUSE_TIMEOUT : nil
           end
 
           # Pause when we had to back-off or skip if delay is not needed

data/lib/karafka/pro/processing/offset_metadata/consumer.rb

@@ -45,7 +45,7 @@ module Karafka
             Fetcher.find(topic, partition, cache: cache)
           end
 
-          alias committed_offset_metadata offset_metadata
+          alias_method :committed_offset_metadata, :offset_metadata
         end
       end
     end

data/lib/karafka/pro/processing/parallel_segments/filters/base.rb

@@ -47,16 +47,16 @@ module Karafka
             # @return [String, Numeric] segment assignment key
             def partition(message)
               @partitioner.call(message)
-            rescue StandardError => e
+            rescue => e
               # This should not happen. If you are seeing this it means your partitioner code
               # failed and raised an error. We highly recommend mitigating partitioner level errors
               # on the user side because this type of collapse should be considered a last resort
               Karafka.monitor.instrument(
-                'error.occurred',
+                "error.occurred",
                 caller: self,
                 error: e,
                 message: message,
-                type: 'parallel_segments.partitioner.error'
+                type: "parallel_segments.partitioner.error"
               )
 
               :failure
@@ -70,14 +70,14 @@ module Karafka
               return 0 if message_segment_key == :failure
 
               @reducer.call(message_segment_key)
-            rescue StandardError => e
+            rescue => e
               # @see `#partition` method error handling doc
               Karafka.monitor.instrument(
-                'error.occurred',
+                "error.occurred",
                 caller: self,
                 error: e,
                 message_segment_key: message_segment_key,
-                type: 'parallel_segments.reducer.error'
+                type: "parallel_segments.reducer.error"
               )
 
               0