karafka-web 0.7.1 → 0.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 291ef570840ca0156ae989c7bb729253e0a521d948e2f47688f88f8736568f5d
-  data.tar.gz: ee5b833af74db2c40cde6b2dafa7ead04308d16a3e5039b71c27e0f268cc7a8b
+  metadata.gz: 3de31b98509126d85a621683b04f085f0f0f1e7682c28c1c1558920b87021d3d
+  data.tar.gz: 290dcfa64377e48d2ae03dc0495d2b535c32362d3dc82380ddb36b4b284b3679
 SHA512:
-  metadata.gz: 9a05eefd03dab85d2576167c64a64ae3e13d944171df675626984eb57f7d0373d149d55a378fb44d4d8861c9db23b20ea6eef7368c53e9b101fda3dde4bc19bc
-  data.tar.gz: a7156e82313ff118fc1a35721bd14c9383ad670da877b31b00f7e4de95fb0d8b63d6385123efd54b9a15b68bd9734af9599c68136f8f84939f8851d01aab4856
+  metadata.gz: 30ede1f0bed82fd9af59796a7f7e64e2eab29a0965b3e573f6d40c86fd8546c7b0e7dc21f073ccaf0cafbd6cd9f69760488fa70dd758496eea8a8aec43cb638d
+  data.tar.gz: eebb4f6cc73e8259ab2198ca1ccb9c7dd401586a5929b8e4efde8fafffebe50e8d2a0dbae1b5ba894bc3c07beb6cd01b39a343ece56989051c643cae262f8e87

data/CHANGELOG.md

@@ -1,7 +1,15 @@
 # Karafka Web changelog
 
+## 0.7.2 (2023-09-18) 
+- [Improvement] Display hidden by accident errors for OSS metrics.
+- [Improvement] Use a five second cache for non-production environments to improve dev experience.
+- [Improvement] Limit number of partitions listed on the Consumers view if they exceed 10 to improve readability and indicate, that there are more in OSS similar to Pro.
+- [Improvement] Squash processes reports based on the key instead of payload skipping deserialization for duplicated reports.
+- [Improvement] Make sure, that the Karafka topics present data can be deserialized and report on the status page if not.
+- [Fix] Extensive data-poll on processes despite no processes being available.
+
 ## 0.7.1 (2023-09-15)
-- [Improvement] Limit number of partitions listed on the Consumers view if they exceed 10 to improve readability and indicate, that there are more.
+- [Improvement] Limit number of partitions listed on the Consumers view if they exceed 10 to improve readability and indicate, that there are more in Pro.
 - [Improvement] Make sure, that small messages size (less than 100 bytes) is correctly displayed.
 - [Fix] Validate refresh time.
 - [Fix] Fix invalid message payload size display (KB instead of B, etc).

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka-web (0.7.1)
+    karafka-web (0.7.2)
       erubi (~> 1.4)
       karafka (>= 2.2.3, < 3.0.0)
       karafka-core (>= 2.2.2, < 3.0.0)

data/lib/karafka/web/config.rb

@@ -92,7 +92,11 @@ module Karafka
         end
 
         # UI cache to improve performance of views that reuse states that are not often changed
-        setting :cache, default: Ui::Lib::TtlCache.new(60_000 * 5)
+        setting :cache, default: Ui::Lib::TtlCache.new(
+          # Use the TTL for internal cache in prod but invalidate quickly in other environments,
+          # as for example in development things may change frequently
+          Karafka.env.production? ? 60_000 * 5 : 5_000
+        )
 
         # Should we display internal topics of Kafka. The once starting with `__`
         # By default we do not display them as they are not usable from regular users perspective

data/lib/karafka/web/ui/models/processes.rb

@@ -18,9 +18,10 @@ module Karafka
             # @param state [State] current system state from which we can get processes metadata
             # @return [Array<Process>]
             def active(state)
-              processes = fetch_reports(state)
+              messages = fetch_reports(state)
+              messages = squash_processes_data(messages)
+              processes = messages.map(&:payload)
               evict_expired_processes(processes)
-              processes = squash_processes_data(processes)
               processes = sort_processes(processes)
 
               processes.map { |process_hash| Process.new(process_hash) }
@@ -32,7 +33,13 @@ module Karafka
             # @param state [State]
             # @return [Array<Hash>] array with deserialized processes reports
             def fetch_reports(state)
-              offsets = state[:processes]
+              processes = state[:processes]
+
+              # Short track when no processes not to run a read when nothing will be given
+              # This allows us to handle a case where we would load 10k of reports for nothing
+              return [] if processes.empty?
+
+              offsets = processes
                         .values
                         .map { |process| process[:offset] }
                         .sort
@@ -46,7 +53,7 @@ module Karafka
                 # was bypassed by state changes in the processes
                 10_000,
                 offsets.first || -1
-              ).map(&:payload)
+              )
             end
 
             # Collapses processes data and only keeps the most recent report for give process
@@ -55,7 +62,7 @@ module Karafka
             def squash_processes_data(processes)
               processes
                 .reverse
-                .uniq { |consumer| consumer[:process][:name] }
+                .uniq(&:key)
                 .reverse
             end
 

data/lib/karafka/web/ui/models/status.rb

@@ -134,46 +134,75 @@ module Karafka
             )
           end
 
-          # @return [Status::Step] Is the initial consumers state present in Kafka
+          # @return [Status::Step] Is the initial consumers state present in Kafka and that they
+          #   can be deserialized
           def initial_consumers_state
+            details = { issue_type: :presence }
+
             if replication.success?
-              @current_state ||= Models::ConsumersState.current
-              status = @current_state ? :success : :failure
+              begin
+                @current_state ||= Models::ConsumersState.current
+                status = @current_state ? :success : :failure
+              rescue JSON::ParserError
+                status = :failure
+                details[:issue_type] = :deserialization
+              end
             else
               status = :halted
             end
 
             Step.new(
               status,
-              nil
+              details
             )
           end
 
-          # @return [Status::Step] Is the initial consumers metrics record present in Kafka
+          # @return [Status::Step] Is the initial consumers metrics record present in Kafka and
+          #   that they can be deserialized
           def initial_consumers_metrics
+            details = { issue_type: :presence }
+
             if initial_consumers_state.success?
-              @current_metrics ||= Models::ConsumersMetrics.current
-              status = @current_metrics ? :success : :failure
+              begin
+                @current_metrics ||= Models::ConsumersMetrics.current
+                status = @current_metrics ? :success : :failure
+              rescue JSON::ParserError
+                status = :failure
+                details[:issue_type] = :deserialization
+              end
             else
               status = :halted
             end
 
             Step.new(
               status,
-              nil
+              details
             )
           end
 
-          # @return [Status::Step] Is there at least one active karafka server reporting to the
-          #   Web UI
-          def live_reporting
+          # @return [Status::Step] could we read and operate on the current processes data (if any)
+          def consumers_reports
             if initial_consumers_metrics.success?
               @processes ||= Models::Processes.active(@current_state)
-              status = @processes.empty? ? :failure : :success
+              status = :success
             else
               status = :halted
             end
 
+            Step.new(status, nil)
+          rescue JSON::ParserError
+            Step.new(:failure, nil)
+          end
+
+          # @return [Status::Step] Is there at least one active karafka server reporting to the
+          #   Web UI
+          def live_reporting
+            status = if consumers_reports.success?
+                       @processes.empty? ? :failure : :success
+                     else
+                       :halted
+                     end
+
             Step.new(
               status,
               nil

data/lib/karafka/web/ui/views/consumers/_consumer.erb

@@ -14,7 +14,11 @@
           <% subscription_group.topics.each do |topic| %>
             <span class="badge bg-secondary badge-topic" title="Consumer group: <%= consumer_group.id %>">
               <%= topic.name %>:
-              <%= topic.partitions.map(&:id).join(',') %>
+              <% if topic.partitions.size > 10 %>
+                <%= "#{topic.partitions.first(10).map(&:id).join(',')}..." %>
+              <% else %>
+                <%= topic.partitions.map(&:id).join(',') %>
+              <% end %>
             </span>
           <% end %>
         <% end %>

data/lib/karafka/web/ui/views/dashboard/index.erb

@@ -29,7 +29,7 @@
 
         <div class="tab-content">
           <div class="tab-pane show active" id="messages" role="tabpanel">
-            <% data = @aggregated_charts.with(:messages) %>
+            <% data = @aggregated_charts.with(:messages, :errors) %>
             <%== partial 'shared/chart', locals: { data: data, id: 'messages' } %>
           </div>
 

data/lib/karafka/web/ui/views/status/failures/_consumers_reports.erb

@@ -0,0 +1,11 @@
+<p>
+  At least one consumer report appears to be corrupted.
+</p>
+
+<p>
+  This issue typically arises when invalid messages have been sent to the Karafka consumers' reports topic or when the topic has been populated with data from a newer Karafka Web UI without updating it.
+</p>
+
+<p class="mb-0">
+  To resolve this, please first attempt to upgrade the Karafka Web UI. If the problem persists, execute <code>bundle exec karafka-web reset</code> to reset the Web UI.
+</p>

data/lib/karafka/web/ui/views/status/failures/_initial_consumers_metrics.erb

@@ -1,11 +1,25 @@
-<p>
-  The initial consumers metrics for the Web UI were not created.
-</p>
+<% if details[:issue_type] == :deserialization %>
+  <p>
+    The initial state of the consumers metrics appears to be corrupted.
+  </p>
 
-<p>
-  It means that the <code>bundle exec karafka-web migrate</code> was not executed or failed.
-</p>
+  <p>
+    This issue typically arises when invalid messages have been sent to the Karafka consumers' metrics topic or when the topic has been populated with data from a newer Karafka Web UI without updating it.
+  </p>
 
-<p class="mb-0">
-  To fix this, you need to ensure that the <code>bundle exec karafka-web migrate</code> runs successfully.
-</p>
+  <p class="mb-0">
+    To resolve this, please first attempt to upgrade the Karafka Web UI. If the problem persists, execute <code>bundle exec karafka-web reset</code> to reset the Web UI.
+  </p>
+<% else %>
+  <p>
+    The initial consumers metrics for the Web UI were not created.
+  </p>
+
+  <p>
+    It means that the <code>bundle exec karafka-web migrate</code> was not executed or failed.
+  </p>
+
+  <p class="mb-0">
+    To fix this, you need to ensure that the <code>bundle exec karafka-web migrate</code> runs successfully.
+  </p>
+<% end %>

data/lib/karafka/web/ui/views/status/failures/_initial_consumers_state.erb

@@ -1,11 +1,25 @@
-<p>
-  The initial consumers state for the Web UI was not created.
-</p>
+<% if details[:issue_type] == :deserialization %>
+  <p>
+    The initial state of the consumers appears to be corrupted.
+  </p>
 
-<p>
-  It means that the <code>bundle exec karafka-web install</code> was not executed or failed.
-</p>
+  <p>
+    This issue typically arises when invalid messages have been sent to the Karafka consumers' state topic or when the topic has been populated with data from a newer Karafka Web UI without updating it.
+  </p>
 
-<p class="mb-0">
-  To fix this, you need to ensure that the <code>bundle exec karafka-web migrate</code> runs successfully.
-</p>
+  <p class="mb-0">
+    To resolve this, please first attempt to upgrade the Karafka Web UI. If the problem persists, execute <code>bundle exec karafka-web reset</code> to reset the Web UI.
+  </p>
+<% else %>
+  <p>
+    The initial consumers state for the Web UI was not created.
+  </p>
+
+  <p>
+    It means that the <code>bundle exec karafka-web migrate</code> was not executed or failed.
+  </p>
+
+  <p class="mb-0">
+    To fix this, you need to ensure that the <code>bundle exec karafka-web migrate</code> runs successfully.
+  </p>
+<% end %>

data/lib/karafka/web/ui/views/status/show.erb

@@ -82,7 +82,7 @@
         partial(
           "status/#{@status.initial_consumers_state.to_s}",
           locals: {
-            title: 'Initial consumers state presence',
+            title: 'Initial consumers state',
             description: partial(
               'status/failures/initial_consumers_state',
               locals: {
@@ -97,7 +97,7 @@
         partial(
           "status/#{@status.initial_consumers_metrics.to_s}",
           locals: {
-            title: 'Initial consumers metrics presence',
+            title: 'Initial consumers metrics',
             description: partial(
               'status/failures/initial_consumers_metrics',
               locals: {
@@ -108,6 +108,21 @@
         )
       %>
 
+      <%==
+        partial(
+          "status/#{@status.consumers_reports.to_s}",
+          locals: {
+            title: 'Consumers reports',
+            description: partial(
+              'status/failures/consumers_reports',
+              locals: {
+                details: @status.consumers_reports.details
+              }
+            )
+          }
+        )
+      %>
+
       <%==
         partial(
           "status/#{@status.live_reporting.to_s}",

data/lib/karafka/web/version.rb

@@ -3,6 +3,6 @@
 module Karafka
   module Web
     # Current gem version
-    VERSION = '0.7.1'
+    VERSION = '0.7.2'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-web
 version: !ruby/object:Gem::Version
-  version: 0.7.1
+  version: 0.7.2
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2023-09-15 00:00:00.000000000 Z
+date: 2023-09-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: erubi
@@ -422,6 +422,7 @@ files:
 - lib/karafka/web/ui/views/status/_success.erb
 - lib/karafka/web/ui/views/status/_warning.erb
 - lib/karafka/web/ui/views/status/failures/_connection.erb
+- lib/karafka/web/ui/views/status/failures/_consumers_reports.erb
 - lib/karafka/web/ui/views/status/failures/_consumers_reports_schema_state.erb
 - lib/karafka/web/ui/views/status/failures/_enabled.erb
 - lib/karafka/web/ui/views/status/failures/_initial_consumers_metrics.erb