karafka-web 0.11.2 → 0.11.4

data/lib/karafka/web/pro/ui/controllers/topics/distributions_controller.rb

@@ -65,9 +65,7 @@ module Karafka
                     topic_name,
                     partition_count
                   )
-                rescue Rdkafka::RdkafkaError => e
-                  @form_error = e
-                rescue Rdkafka::Config::ConfigError => e
+                rescue Rdkafka::RdkafkaError, Rdkafka::Config::ConfigError => e
                   @form_error = e
                 end
 

data/lib/karafka/web/pro/ui/lib/branding/contracts/config.rb

@@ -14,10 +14,8 @@ module Karafka
               # Makes sure, all the expected UI branding config is defined as it should be
               class Config < ::Karafka::Contracts::Base
                 configure do |config|
-                  config.error_messages = YAML.safe_load(
-                    File.read(
-                      File.join(Karafka::Web.gem_root, 'config', 'locales', 'pro_errors.yml')
-                    )
+                  config.error_messages = YAML.safe_load_file(
+                    File.join(Karafka::Web.gem_root, 'config', 'locales', 'pro_errors.yml')
                   ).fetch('en').fetch('validations').fetch('config')
                 end
 

data/lib/karafka/web/pro/ui/lib/policies/contracts/config.rb

@@ -15,10 +15,8 @@ module Karafka
               # Makes sure, all the expected UI policies config is defined as it should be
               class Config < ::Karafka::Contracts::Base
                 configure do |config|
-                  config.error_messages = YAML.safe_load(
-                    File.read(
-                      File.join(Karafka::Web.gem_root, 'config', 'locales', 'pro_errors.yml')
-                    )
+                  config.error_messages = YAML.safe_load_file(
+                    File.join(Karafka::Web.gem_root, 'config', 'locales', 'pro_errors.yml')
                   ).fetch('en').fetch('validations').fetch('config')
                 end
 

data/lib/karafka/web/pro/ui/lib/search/contracts/config.rb

@@ -15,10 +15,8 @@ module Karafka
               # Makes sure, all the expected UI search config is defined as it should be
               class Config < ::Karafka::Contracts::Base
                 configure do |config|
-                  config.error_messages = YAML.safe_load(
-                    File.read(
-                      File.join(Karafka::Web.gem_root, 'config', 'locales', 'pro_errors.yml')
-                    )
+                  config.error_messages = YAML.safe_load_file(
+                    File.join(Karafka::Web.gem_root, 'config', 'locales', 'pro_errors.yml')
                   ).fetch('en').fetch('validations').fetch('config')
                 end
 
@@ -33,7 +31,7 @@ module Karafka
                     required(:limits) do |val|
                       next false unless val.is_a?(Array)
                       next false if val.empty?
-                      next false unless val.all? { |count| count.is_a?(Integer) }
+                      next false unless val.all?(Integer)
                       next false unless val.all?(&:positive?)
 
                       true

data/lib/karafka/web/pro/ui/lib/search/contracts/form.rb

@@ -20,10 +20,8 @@ module Karafka
               #   normalized to simplify the flow.
               class Form < Web::Contracts::Base
                 configure do |config|
-                  config.error_messages = YAML.safe_load(
-                    File.read(
-                      File.join(Karafka::Web.gem_root, 'config', 'locales', 'pro_errors.yml')
-                    )
+                  config.error_messages = YAML.safe_load_file(
+                    File.join(Karafka::Web.gem_root, 'config', 'locales', 'pro_errors.yml')
                   ).fetch('en').fetch('validations').fetch('search_form')
                 end
 
@@ -74,7 +72,7 @@ module Karafka
                   next false unless val.is_a?(Array)
                   next false if val.empty?
 
-                  val.all? { |ar_val| ar_val.is_a?(String) }
+                  val.all?(String)
                 end
 
                 # Special validation for timestamp to make sure it is not older than 2010

data/lib/karafka/web/pro/ui/lib/search/runner.rb

@@ -77,8 +77,21 @@ module Karafka
 
               private
 
+              # Dynamically defines accessor methods for each search criteria field.
+              # For example, if SEARCH_CRITERIA_FIELDS includes :topic, this creates:
+              #   def topic
+              #     @topic ||= @search_criteria.fetch(:topic)
+              #   end
               SEARCH_CRITERIA_FIELDS.each do |q|
+                # Example for q = :topic:
+                # def topic
+                #   @topic ||= @search_criteria.fetch(:topic)
+                # end
                 class_eval <<~RUBY, __FILE__, __LINE__ + 1
+                  # def topic
+                  #   @topic ||= @search_criteria.fetch(:topic)
+                  # end
+
                   def #{q}
                     @#{q} ||= @search_criteria.fetch(:#{q})
                   end
@@ -122,7 +135,7 @@ module Karafka
                         end
 
                 iterator_query = {
-                  @topic => partitions_to_search.map { |par| [par, start] }.to_h
+                  @topic => partitions_to_search.to_h { |par| [par, start] }
                 }
 
                 @iterator = Karafka::Pro::Iterator.new(iterator_query)

data/lib/karafka/web/pro/ui/routes/errors.rb

@@ -15,10 +15,10 @@ module Karafka
                 controller = build(Controllers::ErrorsController)
 
                 r.get :partition_id, Integer do |partition_id, offset|
-                  if params.current_offset != -1
-                    r.redirect root_path('errors', partition_id, params.current_offset)
-                  else
+                  if params.current_offset == -1
                     controller.show(partition_id, offset)
+                  else
+                    r.redirect root_path('errors', partition_id, params.current_offset)
                   end
                 end
 

data/lib/karafka/web/pro/ui/routes/explorer.rb

@@ -81,14 +81,14 @@ module Karafka
                   r.get String, :partition_id, Integer do |topic_id, partition_id, offset|
                     # If when viewing given message we get an offset of different message, we should
                     # redirect there. This allows us to support pagination with the current engine
-                    if params.current_offset != -1
+                    if params.current_offset == -1
+                      controller.show(topic_id, partition_id, offset)
+                    else
                       r.redirect explorer_topics_path(
                         topic_id,
                         partition_id,
                         params.current_offset
                       )
-                    else
-                      controller.show(topic_id, partition_id, offset)
                     end
                   end
 

data/lib/karafka/web/pro/ui/views/health/_no_partition_data.erb

@@ -0,0 +1,9 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
+<tr class="text-muted">
+  <td><%= partition_id %></td>
+  <td colspan="<%= colspan %>" class="text-center">
+    No data available. Partition may not be assigned or is not being consumed by any active process.
+  </td>
+</tr>

data/lib/karafka/web/pro/ui/views/health/_partitions_with_fallback.erb

@@ -0,0 +1,41 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
+<%#
+  Renders partitions with fallback for missing partition data
+
+  Required locals:
+    - topic_details: Hash containing :partitions and :partitions_count
+    - topic_name: Name of the topic
+    - partition_partial: Path to the partition partial to render
+    - colspan: Number of columns for the "no data" message
+%>
+
+<% partitions = topic_details[:partitions] %>
+
+<% partitions.each do |partition_id, details| %>
+  <%==
+    partial(
+      partition_partial,
+      locals: {
+        topic_name: topic_name,
+        partition_id: partition_id,
+        details: details
+      }
+    )
+  %>
+<% end %>
+
+<% topic_details[:partitions_count].times do |partition_id| %>
+  <% next if partitions.include?(partition_id) %>
+
+  <%==
+    partial(
+      'health/no_partition_data',
+      locals: {
+        partition_id: partition_id,
+        colspan: colspan
+      }
+    )
+  %>
+<% end %>

data/lib/karafka/web/pro/ui/views/health/changes.erb

@@ -14,7 +14,7 @@
     <h2 class="h2"><%= cg_name %></h2>
 
     <% topics = details[:topics] %>
-    <% topics.each_with_index do |(topic_name, partitions), index| %>
+    <% topics.each_with_index do |(topic_name, topic_details), index| %>
       <div class="data-table-wrapper">
         <table class="data-table">
           <thead>
@@ -38,18 +38,17 @@
             </tr>
           </thead>
           <tbody>
-              <% partitions.each do |partition_id, details| %>
-                <%==
-                  partial(
-                    'health/partition_times',
-                    locals: {
-                      topic_name: topic_name,
-                      partition_id: partition_id,
-                      details: details
-                    }
-                  )
-                %>
-              <% end %>
+              <%==
+                partial(
+                  'health/partitions_with_fallback',
+                  locals: {
+                    topic_details: topic_details,
+                    topic_name: topic_name,
+                    partition_partial: 'health/partition_times',
+                    colspan: 5
+                  }
+                )
+              %>
           </tbody>
         </table>
 

data/lib/karafka/web/pro/ui/views/health/lags.erb

@@ -14,7 +14,7 @@
     <h2 class="h2"><%= cg_name %></h2>
 
     <% topics = details[:topics] %>
-    <% topics.each_with_index do |(topic_name, partitions), index| %>
+    <% topics.each_with_index do |(topic_name, topic_details), index| %>
       <div class="data-table-wrapper">
         <table class="data-table">
           <thead>
@@ -39,18 +39,17 @@
             </tr>
           </thead>
           <tbody>
-              <% partitions.each do |partition_id, details| %>
-                <%==
-                  partial(
-                    'health/partition_lags',
-                    locals: {
-                      topic_name: topic_name,
-                      partition_id: partition_id,
-                      details: details
-                    }
-                  )
-                %>
-              <% end %>
+              <%==
+                partial(
+                  'health/partitions_with_fallback',
+                  locals: {
+                    topic_details: topic_details,
+                    topic_name: topic_name,
+                    partition_partial: 'health/partition_lags',
+                    colspan: 5
+                  }
+                )
+              %>
           </tbody>
         </table>
 

data/lib/karafka/web/pro/ui/views/health/offsets.erb

@@ -14,7 +14,7 @@
     <h2 class="h2"><%= cg_name %></h2>
 
     <% topics = details[:topics] %>
-    <% topics.each_with_index do |(topic_name, partitions), index| %>
+    <% topics.each_with_index do |(topic_name, topic_details), index| %>
       <div class="data-table-wrapper">
         <table class="data-table">
           <thead>
@@ -42,18 +42,17 @@
             </tr>
           </thead>
           <tbody>
-              <% partitions.each do |partition_id, details| %>
-                <%==
-                  partial(
-                    'health/partition_offset',
-                    locals: {
-                      topic_name: topic_name,
-                      partition_id: partition_id,
-                      details: details
-                    }
-                  )
-                %>
-              <% end %>
+              <%==
+                partial(
+                  'health/partitions_with_fallback',
+                  locals: {
+                    topic_details: topic_details,
+                    topic_name: topic_name,
+                    partition_partial: 'health/partition_offset',
+                    colspan: 11
+                  }
+                )
+              %>
           </tbody>
         </table>
 

data/lib/karafka/web/pro/ui/views/health/overview.erb

@@ -10,11 +10,11 @@
     <%== partial 'health/no_data' %>
   <% end %>
 
-  <% @stats.each_with_index do |(cg_name, details), index| %>
+  <% @stats.each_with_index do |(cg_name, cg_details), index| %>
     <h2 class="h2"><%= cg_name %></h2>
 
-    <% topics = details[:topics] %>
-    <% topics.each_with_index do |(topic_name, partitions), index| %>
+    <% topics = cg_details[:topics] %>
+    <% topics.each_with_index do |(topic_name, topic_details), index| %>
       <div class="data-table-wrapper">
         <table class="data-table">
           <thead>
@@ -50,22 +50,21 @@
             </tr>
           </thead>
           <tbody>
-              <% partitions.each do |partition_id, details| %>
-                <%==
-                  partial(
-                    'health/partition',
-                    locals: {
-                      topic_name: topic_name,
-                      partition_id: partition_id,
-                      details: details
-                    }
-                  )
-                %>
-              <% end %>
+              <%==
+                partial(
+                  'health/partitions_with_fallback',
+                  locals: {
+                    topic_details: topic_details,
+                    topic_name: topic_name,
+                    partition_partial: 'health/partition',
+                    colspan: 7
+                  }
+                )
+              %>
           </tbody>
         </table>
 
-        <%== partial 'health/table_metadata', locals: { details: details } %>
+        <%== partial 'health/table_metadata', locals: { details: cg_details } %>
       </div>
     <% end %>
   <% end %>

data/lib/karafka/web/processing/consumer.rb

@@ -19,6 +19,8 @@ module Karafka
 
           # If there is even one incompatible message, we need to stop
           consumers_messages.each do |message|
+            report = message.payload
+
             case @reports_schema_manager.call(message)
             when :current
               true
@@ -35,7 +37,9 @@ module Karafka
             # since in most cases this is intermediate due to rolling upgrades, this should not
             # significantly impact the state tracking and processing.
             when :older
-              @state_aggregator.add_state(message.payload, message.offset)
+              # Migrate old report format to current schema expectations (in-place)
+              @reports_migrator.call(report)
+              @state_aggregator.add_state(report, message.offset)
 
               next
             else
@@ -44,8 +48,8 @@ module Karafka
 
             # We need to run the aggregations on each message in order to compensate for
             # potential lags.
-            @state_aggregator.add(message.payload, message.offset)
-            @metrics_aggregator.add_report(message.payload)
+            @state_aggregator.add(report, message.offset)
+            @metrics_aggregator.add_report(report)
             @metrics_aggregator.add_stats(@state_aggregator.stats)
             # Indicates that we had at least one report we used to enrich data
             # If there were no state changes, there is no reason to flush data. This can occur
@@ -86,6 +90,7 @@ module Karafka
           @flush_interval = ::Karafka::Web.config.processing.interval
 
           @reports_schema_manager = Consumers::SchemaManager.new
+          @reports_migrator = Consumers::ReportsMigrator.new
           @state_aggregator = Consumers::Aggregators::State.new(@reports_schema_manager)
           @state_contract = Consumers::Contracts::State.new
 

data/lib/karafka/web/processing/consumers/aggregators/metrics.rb

@@ -68,7 +68,7 @@ module Karafka
             #   care about what a stopped process was doing and we can also remove it from active
             #   reports.
             def evict_expired_processes
-              max_ttl = @aggregated_from - ::Karafka::Web.config.ttl / 1_000
+              max_ttl = @aggregated_from - (::Karafka::Web.config.ttl / 1_000)
 
               @active_reports.delete_if do |_id, report|
                 report[:dispatched_at] < max_ttl || report[:process][:status] == 'stopped'

data/lib/karafka/web/processing/consumers/aggregators/state.rb

@@ -51,7 +51,11 @@ module Karafka
             # @param report [Hash]
             # @param offset [Integer]
             def add_state(report, offset)
-              process_id = report[:process][:id]
+              # When we deserialize the keys from the stored state, because we convert keys into
+              # symbols, we may have given process state already stored. This means that in order
+              # to update it, we do need to have the new report process id also as a symbol to
+              # act as the key
+              process_id = report[:process][:id].to_sym
 
               state[:processes][process_id] = {
                 dispatched_at: report[:dispatched_at],
@@ -103,7 +107,7 @@ module Karafka
             #   stopped processes for extra time within the ttl limitations. This makes tracking of
             #   things from UX perspective nicer.
             def evict_expired_processes
-              max_ttl = @aggregated_from - ::Karafka::Web.config.ttl / 1_000
+              max_ttl = @aggregated_from - (::Karafka::Web.config.ttl / 1_000)
 
               state[:processes].delete_if do |_id, details|
                 details[:dispatched_at] < max_ttl
@@ -165,13 +169,13 @@ module Karafka
             end
 
             # @param report [Hash]
-            def iterate_partitions(report)
+            # @param block [Proc]
+            # @yieldparam partition_stats [Hash] statistics for a single partition
+            def iterate_partitions(report, &block)
               report[:consumer_groups].each_value do |consumer_group|
                 consumer_group[:subscription_groups].each_value do |subscription_group|
                   subscription_group[:topics].each_value do |topic|
-                    topic[:partitions].each_value do |partition|
-                      yield(partition)
-                    end
+                    topic[:partitions].each_value(&block)
                   end
                 end
               end

data/lib/karafka/web/processing/consumers/contracts/state.rb

@@ -19,9 +19,14 @@ module Karafka
             required(:schema_version) { |val| val.is_a?(String) && !val.empty? }
             required(:dispatched_at) { |val| val.is_a?(Numeric) && val.positive? }
             required(:stats) { |val| val.is_a?(Hash) }
-            required(:processes) { |val| val.is_a?(Hash) }
             required(:schema_state) { |val| VALID_SCHEMA_STATES.include?(val) }
 
+            required(:processes) do |val|
+              next false unless val.is_a?(Hash)
+
+              val.keys.all?(Symbol)
+            end
+
             virtual do |data, errors|
               next unless errors.empty?
 

data/lib/karafka/web/processing/consumers/reports_migrator.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Processing
+      module Consumers
+        # Migrator for consumer reports that applies per-message transformations
+        #
+        # Unlike the Management::Migrator which operates on aggregated states,
+        # this migrator runs on each individual consumer report as it's processed.
+        #
+        # This is necessary because:
+        #   - Reports are continuously published during upgrades
+        #   - Reports are short-lived (TTL-based) and don't need persistent migrations
+        #   - Old schema reports may remain in Kafka topics for extended periods
+        #
+        # Migrations are lightweight transformations that normalize old report formats
+        # to work with current processing code. Migrations are stored in
+        # lib/karafka/web/management/migrations/consumers_reports/ alongside other migrations.
+        class ReportsMigrator
+          # Applies all applicable migrations to a consumer report
+          #
+          # @param report [Hash] deserialized consumer report
+          # @return [Hash] the same report object, potentially modified in-place
+          def call(report)
+            # Apply each applicable migration in order
+            migrations.each do |migration_class|
+              next unless migration_class.applicable?(report[:schema_version])
+
+              migration_class.new.migrate(report)
+            end
+
+            report
+          end
+
+          private
+
+          # Lazy-initialized cache of report migrations
+          # Only computed when first needed to avoid memory overhead if no old reports exist
+          def migrations
+            @migrations ||= Management::Migrations::Base
+                            .sorted_descendants
+                            .select { |migration_class| migration_class.type == :consumers_reports }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/processing/time_series_tracker.rb

@@ -53,7 +53,7 @@ module Karafka
         # @param existing [Hash] existing historical metrics (may be empty for the first state)
         def initialize(existing)
           # Builds an empty structure for potential time ranges we are interested in
-          @historicals = TIME_RANGES.keys.map { |name| [name, []] }.to_h
+          @historicals = TIME_RANGES.keys.to_h { |name| [name, []] }
 
           # Fetch the existing (if any) historical values that we already have
           import_existing(existing)

data/lib/karafka/web/tracking/consumers/contracts/report.rb

@@ -38,7 +38,7 @@ module Karafka
 
               required(:cpu_usage) do |val|
                 val.is_a?(Array) &&
-                  val.all? { |key| key.is_a?(Numeric) } &&
+                  val.all?(Numeric) &&
                   val.all? { |key| key >= -1 } &&
                   val.size == 3
               end

data/lib/karafka/web/tracking/consumers/contracts/topic.rb

@@ -10,6 +10,7 @@ module Karafka
             configure
 
             required(:name) { |val| val.is_a?(String) && !val.empty? }
+            required(:partitions_cnt) { |val| val.is_a?(Integer) && val >= 0 }
             required(:partitions) { |val| val.is_a?(Hash) }
 
             virtual do |data, errors|

data/lib/karafka/web/tracking/consumers/listeners/errors.rb

@@ -10,7 +10,7 @@ module Karafka
             include Tracking::Helpers::ErrorInfo
 
             # Schema used by consumers error reporting
-            SCHEMA_VERSION = '1.1.0'
+            SCHEMA_VERSION = '1.2.0'
 
             private_constant :SCHEMA_VERSION
 
@@ -38,6 +38,7 @@ module Karafka
               track do |sampler|
                 sampler.errors << {
                   schema_version: SCHEMA_VERSION,
+                  id: SecureRandom.uuid,
                   type: event[:type],
                   error_class: error_class,
                   error_message: error_message,

data/lib/karafka/web/tracking/consumers/listeners/processing.rb

@@ -25,7 +25,29 @@ module Karafka
               eofed
             ].each do |action|
               # Tracks the job that is going to be scheduled so we can also display pending jobs
+              # Dynamically creates methods like:
+              #   def on_consumer_before_schedule_consume(event)
+              #   def on_consumer_before_schedule_revoked(event)
+              #   etc.
+              # Example for action = :consume:
+              # # @param event [Karafka::Core::Monitoring::Event]
+              # def on_consumer_before_schedule_consume(event)
+              #   consumer = event.payload[:caller]
+              #   jid = job_id(consumer, 'consume')
+              #   job_details = job_details(consumer, 'consume')
+              #   job_details[:status] = 'pending'
+              #   track { |sampler| sampler.jobs[jid] = job_details }
+              # end
               class_eval <<~RUBY, __FILE__, __LINE__ + 1
+                # @param event [Karafka::Core::Monitoring::Event]
+                # def on_consumer_before_schedule_consume(event)
+                #   consumer = event.payload[:caller]
+                #   jid = job_id(consumer, 'consume')
+                #   job_details = job_details(consumer, 'consume')
+                #   job_details[:status] = 'pending'
+                #   track { |sampler| sampler.jobs[jid] = job_details }
+                # end
+
                 # @param event [Karafka::Core::Monitoring::Event]
                 def on_consumer_before_schedule_#{action}(event)
                   consumer = event.payload[:caller]
@@ -115,7 +137,31 @@ module Karafka
               [:tick, :ticked, 'tick'],
               [:eof, :eofed, 'eofed']
             ].each do |pre, post, action|
+              # Dynamically creates methods like:
+              #   def on_consumer_revoke(event)
+              #   def on_consumer_shutting_down(event)
+              #   etc.
+              # Example for pre = :revoke, action = :revoked:
+              # # Stores this job details
+              # #
+              # # @param event [Karafka::Core::Monitoring::Event]
+              # def on_consumer_revoke(event)
+              #   consumer = event.payload[:caller]
+              #   jid = job_id(consumer, 'revoked')
+              #   job_details = job_details(consumer, 'revoked')
+              #   track { |sampler| sampler.counters[:jobs] += 1; sampler.jobs[jid] = job_details }
+              # end
               class_eval <<~METHOD, __FILE__, __LINE__ + 1
+                # Stores this job details
+                #
+                # @param event [Karafka::Core::Monitoring::Event]
+                # def on_consumer_revoke(event)
+                #   consumer = event.payload[:caller]
+                #   jid = job_id(consumer, 'revoked')
+                #   job_details = job_details(consumer, 'revoked')
+                #   track { |sampler| sampler.counters[:jobs] += 1; sampler.jobs[jid] = job_details }
+                # end
+
                 # Stores this job details
                 #
                 # @param event [Karafka::Core::Monitoring::Event]

data/lib/karafka/web/tracking/consumers/listeners/statistics.rb

@@ -40,6 +40,7 @@ module Karafka
 
                     topic_details = topics_details[topic_name] ||= {
                       name: topic_name,
+                      partitions_cnt: partitions.keys.count { |k| k.to_i != -1 },
                       partitions: {}
                     }