karafka-web 0.7.1 → 0.7.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 291ef570840ca0156ae989c7bb729253e0a521d948e2f47688f88f8736568f5d
-  data.tar.gz: ee5b833af74db2c40cde6b2dafa7ead04308d16a3e5039b71c27e0f268cc7a8b
+  metadata.gz: dc9572e3dfbb4565361fdbc9521fb7a9a7833664f42bd8e9f0e63e940a397034
+  data.tar.gz: 07fb369bcd0d1aa6c80ef20ff96289989c38ff934a1eb795ab85f39b7adf1228
 SHA512:
-  metadata.gz: 9a05eefd03dab85d2576167c64a64ae3e13d944171df675626984eb57f7d0373d149d55a378fb44d4d8861c9db23b20ea6eef7368c53e9b101fda3dde4bc19bc
-  data.tar.gz: a7156e82313ff118fc1a35721bd14c9383ad670da877b31b00f7e4de95fb0d8b63d6385123efd54b9a15b68bd9734af9599c68136f8f84939f8851d01aab4856
+  metadata.gz: 51bc23c17be963c3b19143aa8034046c0bd85df4251dc9e2e911a8ab28ae194d9ae62e2ab1d3aa0b3e2f8dba33649ad51881d9e96fe669fffbdc971f406ce3eb
+  data.tar.gz: 332247644ed57d25687af816ed850f7847b6eb8e63646bf492460424f0d5855c8dace2f1127d0d84631bffa8f6a9fb71e42a64ece7141a6acb96d5b5f987896c

data/CHANGELOG.md

@@ -1,7 +1,18 @@
 # Karafka Web changelog
 
+## 0.7.3 (2023-09-18)
+- [Improvement] Mitigate a case where a race-condition during upgrade would crash data.
+
+## 0.7.2 (2023-09-18) 
+- [Improvement] Display hidden by accident errors for OSS metrics.
+- [Improvement] Use a five second cache for non-production environments to improve dev experience.
+- [Improvement] Limit number of partitions listed on the Consumers view if they exceed 10 to improve readability and indicate, that there are more in OSS similar to Pro.
+- [Improvement] Squash processes reports based on the key instead of payload skipping deserialization for duplicated reports.
+- [Improvement] Make sure, that the Karafka topics present data can be deserialized and report on the status page if not.
+- [Fix] Extensive data-poll on processes despite no processes being available.
+
 ## 0.7.1 (2023-09-15)
-- [Improvement] Limit number of partitions listed on the Consumers view if they exceed 10 to improve readability and indicate, that there are more.
+- [Improvement] Limit number of partitions listed on the Consumers view if they exceed 10 to improve readability and indicate, that there are more in Pro.
 - [Improvement] Make sure, that small messages size (less than 100 bytes) is correctly displayed.
 - [Fix] Validate refresh time.
 - [Fix] Fix invalid message payload size display (KB instead of B, etc).

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka-web (0.7.1)
+    karafka-web (0.7.3)
       erubi (~> 1.4)
       karafka (>= 2.2.3, < 3.0.0)
       karafka-core (>= 2.2.2, < 3.0.0)

data/lib/karafka/web/config.rb

@@ -92,7 +92,11 @@ module Karafka
         end
 
         # UI cache to improve performance of views that reuse states that are not often changed
-        setting :cache, default: Ui::Lib::TtlCache.new(60_000 * 5)
+        setting :cache, default: Ui::Lib::TtlCache.new(
+          # Use the TTL for internal cache in prod but invalidate quickly in other environments,
+          # as for example in development things may change frequently
+          Karafka.env.production? ? 60_000 * 5 : 5_000
+        )
 
         # Should we display internal topics of Kafka. The once starting with `__`
         # By default we do not display them as they are not usable from regular users perspective

data/lib/karafka/web/processing/consumers/aggregators/metrics.rb

@@ -106,17 +106,17 @@ module Karafka
                              .reject(&:negative?)
 
                       lags_stored = partitions_data
-                                    .map { |p_details| p_details.fetch(:lag_stored) }
+                                    .map { |p_details| p_details.fetch(:lag_stored, -1) }
                                     .reject(&:negative?)
 
                       offsets_hi = partitions_data
-                                   .map { |p_details| p_details.fetch(:hi_offset) }
+                                   .map { |p_details| p_details.fetch(:hi_offset, -1) }
                                    .reject(&:negative?)
 
                       # Last stable offsets freeze durations - we pick the max freeze to indicate
                       # the longest open transaction that potentially may be hanging
                       ls_offsets_fd = partitions_data
-                                      .map { |p_details| p_details.fetch(:ls_offset_fd) }
+                                      .map { |p_details| p_details.fetch(:ls_offset_fd, 0) }
                                       .reject(&:negative?)
 
                       # If there is no lag that would not be negative, it means we did not mark

data/lib/karafka/web/processing/consumers/aggregators/state.rb

@@ -152,8 +152,8 @@ module Karafka
                   stats[:listeners] += report_process[:listeners] || 0
                   stats[:processes] += 1
                   stats[:rss] += report_process[:memory_usage]
-                  stats[:lag] += lags.reject(&:negative?).sum
-                  stats[:lag_stored] += lags_stored.reject(&:negative?).sum
+                  stats[:lag] += lags.compact.reject(&:negative?).sum
+                  stats[:lag_stored] += lags_stored.compact.reject(&:negative?).sum
                   utilization += report_stats[:utilization]
                 end
 

data/lib/karafka/web/ui/models/processes.rb

@@ -18,9 +18,10 @@ module Karafka
             # @param state [State] current system state from which we can get processes metadata
             # @return [Array<Process>]
             def active(state)
-              processes = fetch_reports(state)
+              messages = fetch_reports(state)
+              messages = squash_processes_data(messages)
+              processes = messages.map(&:payload)
               evict_expired_processes(processes)
-              processes = squash_processes_data(processes)
               processes = sort_processes(processes)
 
               processes.map { |process_hash| Process.new(process_hash) }
@@ -32,7 +33,13 @@ module Karafka
             # @param state [State]
             # @return [Array<Hash>] array with deserialized processes reports
             def fetch_reports(state)
-              offsets = state[:processes]
+              processes = state[:processes]
+
+              # Short track when no processes not to run a read when nothing will be given
+              # This allows us to handle a case where we would load 10k of reports for nothing
+              return [] if processes.empty?
+
+              offsets = processes
                         .values
                         .map { |process| process[:offset] }
                         .sort
@@ -46,7 +53,7 @@ module Karafka
                 # was bypassed by state changes in the processes
                 10_000,
                 offsets.first || -1
-              ).map(&:payload)
+              )
             end
 
             # Collapses processes data and only keeps the most recent report for give process
@@ -55,7 +62,7 @@ module Karafka
             def squash_processes_data(processes)
               processes
                 .reverse
-                .uniq { |consumer| consumer[:process][:name] }
+                .uniq(&:key)
                 .reverse
             end
 

data/lib/karafka/web/ui/models/status.rb

@@ -134,46 +134,75 @@ module Karafka
             )
           end
 
-          # @return [Status::Step] Is the initial consumers state present in Kafka
+          # @return [Status::Step] Is the initial consumers state present in Kafka and that they
+          #   can be deserialized
           def initial_consumers_state
+            details = { issue_type: :presence }
+
             if replication.success?
-              @current_state ||= Models::ConsumersState.current
-              status = @current_state ? :success : :failure
+              begin
+                @current_state ||= Models::ConsumersState.current
+                status = @current_state ? :success : :failure
+              rescue JSON::ParserError
+                status = :failure
+                details[:issue_type] = :deserialization
+              end
             else
               status = :halted
             end
 
             Step.new(
               status,
-              nil
+              details
             )
           end
 
-          # @return [Status::Step] Is the initial consumers metrics record present in Kafka
+          # @return [Status::Step] Is the initial consumers metrics record present in Kafka and
+          #   that they can be deserialized
           def initial_consumers_metrics
+            details = { issue_type: :presence }
+
             if initial_consumers_state.success?
-              @current_metrics ||= Models::ConsumersMetrics.current
-              status = @current_metrics ? :success : :failure
+              begin
+                @current_metrics ||= Models::ConsumersMetrics.current
+                status = @current_metrics ? :success : :failure
+              rescue JSON::ParserError
+                status = :failure
+                details[:issue_type] = :deserialization
+              end
             else
               status = :halted
             end
 
             Step.new(
               status,
-              nil
+              details
             )
           end
 
-          # @return [Status::Step] Is there at least one active karafka server reporting to the
-          #   Web UI
-          def live_reporting
+          # @return [Status::Step] could we read and operate on the current processes data (if any)
+          def consumers_reports
             if initial_consumers_metrics.success?
               @processes ||= Models::Processes.active(@current_state)
-              status = @processes.empty? ? :failure : :success
+              status = :success
             else
               status = :halted
             end
 
+            Step.new(status, nil)
+          rescue JSON::ParserError
+            Step.new(:failure, nil)
+          end
+
+          # @return [Status::Step] Is there at least one active karafka server reporting to the
+          #   Web UI
+          def live_reporting
+            status = if consumers_reports.success?
+                       @processes.empty? ? :failure : :success
+                     else
+                       :halted
+                     end
+
             Step.new(
               status,
               nil

data/lib/karafka/web/ui/views/consumers/_consumer.erb

@@ -14,7 +14,11 @@
           <% subscription_group.topics.each do |topic| %>
             <span class="badge bg-secondary badge-topic" title="Consumer group: <%= consumer_group.id %>">
               <%= topic.name %>:
-              <%= topic.partitions.map(&:id).join(',') %>
+              <% if topic.partitions.size > 10 %>
+                <%= "#{topic.partitions.first(10).map(&:id).join(',')}..." %>
+              <% else %>
+                <%= topic.partitions.map(&:id).join(',') %>
+              <% end %>
             </span>
           <% end %>
         <% end %>

data/lib/karafka/web/ui/views/dashboard/index.erb

@@ -29,7 +29,7 @@
 
         <div class="tab-content">
           <div class="tab-pane show active" id="messages" role="tabpanel">
-            <% data = @aggregated_charts.with(:messages) %>
+            <% data = @aggregated_charts.with(:messages, :errors) %>
             <%== partial 'shared/chart', locals: { data: data, id: 'messages' } %>
           </div>
 

data/lib/karafka/web/ui/views/status/failures/_consumers_reports.erb

@@ -0,0 +1,11 @@
+<p>
+  At least one consumer report appears to be corrupted.
+</p>
+
+<p>
+  This issue typically arises when invalid messages have been sent to the Karafka consumers' reports topic or when the topic has been populated with data from a newer Karafka Web UI without updating it.
+</p>
+
+<p class="mb-0">
+  To resolve this, please first attempt to upgrade the Karafka Web UI. If the problem persists, execute <code>bundle exec karafka-web reset</code> to reset the Web UI.
+</p>

data/lib/karafka/web/ui/views/status/failures/_initial_consumers_metrics.erb

@@ -1,11 +1,25 @@
-<p>
-  The initial consumers metrics for the Web UI were not created.
-</p>
+<% if details[:issue_type] == :deserialization %>
+  <p>
+    The initial state of the consumers metrics appears to be corrupted.
+  </p>
 
-<p>
-  It means that the <code>bundle exec karafka-web migrate</code> was not executed or failed.
-</p>
+  <p>
+    This issue typically arises when invalid messages have been sent to the Karafka consumers' metrics topic or when the topic has been populated with data from a newer Karafka Web UI without updating it.
+  </p>
 
-<p class="mb-0">
-  To fix this, you need to ensure that the <code>bundle exec karafka-web migrate</code> runs successfully.
-</p>
+  <p class="mb-0">
+    To resolve this, please first attempt to upgrade the Karafka Web UI. If the problem persists, execute <code>bundle exec karafka-web reset</code> to reset the Web UI.
+  </p>
+<% else %>
+  <p>
+    The initial consumers metrics for the Web UI were not created.
+  </p>
+
+  <p>
+    It means that the <code>bundle exec karafka-web migrate</code> was not executed or failed.
+  </p>
+
+  <p class="mb-0">
+    To fix this, you need to ensure that the <code>bundle exec karafka-web migrate</code> runs successfully.
+  </p>
+<% end %>

data/lib/karafka/web/ui/views/status/failures/_initial_consumers_state.erb

@@ -1,11 +1,25 @@
-<p>
-  The initial consumers state for the Web UI was not created.
-</p>
+<% if details[:issue_type] == :deserialization %>
+  <p>
+    The initial state of the consumers appears to be corrupted.
+  </p>
 
-<p>
-  It means that the <code>bundle exec karafka-web install</code> was not executed or failed.
-</p>
+  <p>
+    This issue typically arises when invalid messages have been sent to the Karafka consumers' state topic or when the topic has been populated with data from a newer Karafka Web UI without updating it.
+  </p>
 
-<p class="mb-0">
-  To fix this, you need to ensure that the <code>bundle exec karafka-web migrate</code> runs successfully.
-</p>
+  <p class="mb-0">
+    To resolve this, please first attempt to upgrade the Karafka Web UI. If the problem persists, execute <code>bundle exec karafka-web reset</code> to reset the Web UI.
+  </p>
+<% else %>
+  <p>
+    The initial consumers state for the Web UI was not created.
+  </p>
+
+  <p>
+    It means that the <code>bundle exec karafka-web migrate</code> was not executed or failed.
+  </p>
+
+  <p class="mb-0">
+    To fix this, you need to ensure that the <code>bundle exec karafka-web migrate</code> runs successfully.
+  </p>
+<% end %>

data/lib/karafka/web/ui/views/status/show.erb

@@ -82,7 +82,7 @@
         partial(
           "status/#{@status.initial_consumers_state.to_s}",
           locals: {
-            title: 'Initial consumers state presence',
+            title: 'Initial consumers state',
             description: partial(
               'status/failures/initial_consumers_state',
               locals: {
@@ -97,7 +97,7 @@
         partial(
           "status/#{@status.initial_consumers_metrics.to_s}",
           locals: {
-            title: 'Initial consumers metrics presence',
+            title: 'Initial consumers metrics',
             description: partial(
               'status/failures/initial_consumers_metrics',
               locals: {
@@ -108,6 +108,21 @@
         )
       %>
 
+      <%==
+        partial(
+          "status/#{@status.consumers_reports.to_s}",
+          locals: {
+            title: 'Consumers reports',
+            description: partial(
+              'status/failures/consumers_reports',
+              locals: {
+                details: @status.consumers_reports.details
+              }
+            )
+          }
+        )
+      %>
+
       <%==
         partial(
           "status/#{@status.live_reporting.to_s}",

data/lib/karafka/web/version.rb

@@ -3,6 +3,6 @@
 module Karafka
   module Web
     # Current gem version
-    VERSION = '0.7.1'
+    VERSION = '0.7.3'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-web
 version: !ruby/object:Gem::Version
-  version: 0.7.1
+  version: 0.7.3
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2023-09-15 00:00:00.000000000 Z
+date: 2023-09-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: erubi
@@ -422,6 +422,7 @@ files:
 - lib/karafka/web/ui/views/status/_success.erb
 - lib/karafka/web/ui/views/status/_warning.erb
 - lib/karafka/web/ui/views/status/failures/_connection.erb
+- lib/karafka/web/ui/views/status/failures/_consumers_reports.erb
 - lib/karafka/web/ui/views/status/failures/_consumers_reports_schema_state.erb
 - lib/karafka/web/ui/views/status/failures/_enabled.erb
 - lib/karafka/web/ui/views/status/failures/_initial_consumers_metrics.erb