karafka-web 0.7.0 → 0.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 651916e28aaa3964d58370954dc66d15afb727f5ca2b4e3f2d8909c41420f329
-  data.tar.gz: c4134a5c8a95a59fe9cd31954333d0663d242291a63718b256b4a95d03a865c0
+  metadata.gz: 3de31b98509126d85a621683b04f085f0f0f1e7682c28c1c1558920b87021d3d
+  data.tar.gz: 290dcfa64377e48d2ae03dc0495d2b535c32362d3dc82380ddb36b4b284b3679
 SHA512:
-  metadata.gz: 87f1a55595f44d1233a0a88433bf8d245dee7f11e2a1d0d78574eebba3a0471d47588ae7d4a6666674c22d0b35e84042113b37f083900d82a919339192443555
-  data.tar.gz: b618135edf9acdc3130e213fcf8d70d71975f1b813335252d9ef7b94a2f495cd653b891e9628f7bfc63ba8fcdc02dd9c28139508780d4fbbdd5e715e27150641
+  metadata.gz: 30ede1f0bed82fd9af59796a7f7e64e2eab29a0965b3e573f6d40c86fd8546c7b0e7dc21f073ccaf0cafbd6cd9f69760488fa70dd758496eea8a8aec43cb638d
+  data.tar.gz: eebb4f6cc73e8259ab2198ca1ccb9c7dd401586a5929b8e4efde8fafffebe50e8d2a0dbae1b5ba894bc3c07beb6cd01b39a343ece56989051c643cae262f8e87

data/CHANGELOG.md

@@ -1,5 +1,19 @@
 # Karafka Web changelog
 
+## 0.7.2 (2023-09-18) 
+- [Improvement] Display hidden by accident errors for OSS metrics.
+- [Improvement] Use a five second cache for non-production environments to improve dev experience.
+- [Improvement] Limit number of partitions listed on the Consumers view if they exceed 10 to improve readability and indicate, that there are more in OSS similar to Pro.
+- [Improvement] Squash processes reports based on the key instead of payload skipping deserialization for duplicated reports.
+- [Improvement] Make sure, that the Karafka topics present data can be deserialized and report on the status page if not.
+- [Fix] Extensive data-poll on processes despite no processes being available.
+
+## 0.7.1 (2023-09-15)
+- [Improvement] Limit number of partitions listed on the Consumers view if they exceed 10 to improve readability and indicate, that there are more in Pro.
+- [Improvement] Make sure, that small messages size (less than 100 bytes) is correctly displayed.
+- [Fix] Validate refresh time.
+- [Fix] Fix invalid message payload size display (KB instead of B, etc).
+
 ## 0.7.0 (2023-09-14)
 - **[Feature]** Introduce graphs.
 - **[Feature]** Introduce historical metrics storage.
@@ -96,6 +110,7 @@ Karafka::Web.enable!
 
 Because of the reporting schema update and new web-ui topics introduction, it is recommended to:
 
+0. Make sure you have upgraded to `0.6.3` before and that it was deployed. To all the environments you want to migrate to `0.7.0`.
 1. Upgrade the codebase based on the below details.
 2. **Stop** the consumer materializing Web-UI. Unless you are running a Web-UI dedicated consumer as recommended [here](https://karafka.io/docs/Web-UI-Development-vs-Production/), you will have to stop all the consumers. This is **crucial** because of schema changes. `karafka-web` `0.7.0` introduces the detection of schema changes, so this step should not be needed in the future.
 3. Run a migration command: `bundle exec karafka-web migrate` that will create missing states and missing topics. You **need** to run it for each of the environments where you use Karafka Web UI.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka-web (0.7.0)
+    karafka-web (0.7.2)
       erubi (~> 1.4)
       karafka (>= 2.2.3, < 3.0.0)
       karafka-core (>= 2.2.2, < 3.0.0)

data/lib/karafka/web/config.rb

@@ -92,7 +92,11 @@ module Karafka
         end
 
         # UI cache to improve performance of views that reuse states that are not often changed
-        setting :cache, default: Ui::Lib::TtlCache.new(60_000 * 5)
+        setting :cache, default: Ui::Lib::TtlCache.new(
+          # Use the TTL for internal cache in prod but invalidate quickly in other environments,
+          # as for example in development things may change frequently
+          Karafka.env.production? ? 60_000 * 5 : 5_000
+        )
 
         # Should we display internal topics of Kafka. The once starting with `__`
         # By default we do not display them as they are not usable from regular users perspective

data/lib/karafka/web/ui/models/processes.rb

@@ -18,9 +18,10 @@ module Karafka
             # @param state [State] current system state from which we can get processes metadata
             # @return [Array<Process>]
             def active(state)
-              processes = fetch_reports(state)
+              messages = fetch_reports(state)
+              messages = squash_processes_data(messages)
+              processes = messages.map(&:payload)
               evict_expired_processes(processes)
-              processes = squash_processes_data(processes)
               processes = sort_processes(processes)
 
               processes.map { |process_hash| Process.new(process_hash) }
@@ -32,7 +33,13 @@ module Karafka
             # @param state [State]
             # @return [Array<Hash>] array with deserialized processes reports
             def fetch_reports(state)
-              offsets = state[:processes]
+              processes = state[:processes]
+
+              # Short track when no processes not to run a read when nothing will be given
+              # This allows us to handle a case where we would load 10k of reports for nothing
+              return [] if processes.empty?
+
+              offsets = processes
                         .values
                         .map { |process| process[:offset] }
                         .sort
@@ -46,7 +53,7 @@ module Karafka
                 # was bypassed by state changes in the processes
                 10_000,
                 offsets.first || -1
-              ).map(&:payload)
+              )
             end
 
             # Collapses processes data and only keeps the most recent report for give process
@@ -55,7 +62,7 @@ module Karafka
             def squash_processes_data(processes)
               processes
                 .reverse
-                .uniq { |consumer| consumer[:process][:name] }
+                .uniq(&:key)
                 .reverse
             end
 

data/lib/karafka/web/ui/models/status.rb

@@ -134,46 +134,75 @@ module Karafka
             )
           end
 
-          # @return [Status::Step] Is the initial consumers state present in Kafka
+          # @return [Status::Step] Is the initial consumers state present in Kafka and that they
+          #   can be deserialized
           def initial_consumers_state
+            details = { issue_type: :presence }
+
             if replication.success?
-              @current_state ||= Models::ConsumersState.current
-              status = @current_state ? :success : :failure
+              begin
+                @current_state ||= Models::ConsumersState.current
+                status = @current_state ? :success : :failure
+              rescue JSON::ParserError
+                status = :failure
+                details[:issue_type] = :deserialization
+              end
             else
               status = :halted
             end
 
             Step.new(
               status,
-              nil
+              details
             )
           end
 
-          # @return [Status::Step] Is the initial consumers metrics record present in Kafka
+          # @return [Status::Step] Is the initial consumers metrics record present in Kafka and
+          #   that they can be deserialized
           def initial_consumers_metrics
+            details = { issue_type: :presence }
+
             if initial_consumers_state.success?
-              @current_metrics ||= Models::ConsumersMetrics.current
-              status = @current_metrics ? :success : :failure
+              begin
+                @current_metrics ||= Models::ConsumersMetrics.current
+                status = @current_metrics ? :success : :failure
+              rescue JSON::ParserError
+                status = :failure
+                details[:issue_type] = :deserialization
+              end
             else
               status = :halted
             end
 
             Step.new(
               status,
-              nil
+              details
             )
           end
 
-          # @return [Status::Step] Is there at least one active karafka server reporting to the
-          #   Web UI
-          def live_reporting
+          # @return [Status::Step] could we read and operate on the current processes data (if any)
+          def consumers_reports
             if initial_consumers_metrics.success?
               @processes ||= Models::Processes.active(@current_state)
-              status = @processes.empty? ? :failure : :success
+              status = :success
             else
               status = :halted
             end
 
+            Step.new(status, nil)
+          rescue JSON::ParserError
+            Step.new(:failure, nil)
+          end
+
+          # @return [Status::Step] Is there at least one active karafka server reporting to the
+          #   Web UI
+          def live_reporting
+            status = if consumers_reports.success?
+                       @processes.empty? ? :failure : :success
+                     else
+                       :halted
+                     end
+
             Step.new(
               status,
               nil

data/lib/karafka/web/ui/pro/views/consumers/_consumer.erb

@@ -14,7 +14,11 @@
           <% subscription_group.topics.each do |topic| %>
             <span class="badge bg-secondary badge-topic" title="Consumer group: <%= consumer_group.id %>">
               <%= topic.name %>:
-              <%= topic.partitions.map(&:id).join(',') %>
+              <% if topic.partitions.size > 10 %>
+                <%= "#{topic.partitions.first(10).map(&:id).join(',')}..." %>
+              <% else %>
+                <%= topic.partitions.map(&:id).join(',') %>
+              <% end %>
             </span>
           <% end %>
         <% end %>

data/lib/karafka/web/ui/pro/views/explorer/show.erb

@@ -49,7 +49,7 @@
               'explorer/messages/detail',
               locals: {
                 k: 'bytesize',
-                v: format_memory((@message.raw_payload&.bytesize || 0 / 1024.to_f).round(2))
+                v: format_memory(((@message.raw_payload&.bytesize || 0) / 1024.to_f).round(4))
               }
             )
           %>

data/lib/karafka/web/ui/public/javascripts/live_poll.js

@@ -116,6 +116,15 @@ function scheduleLivePoll() {
   }
 
   let ti = parseInt(localStorage.karafkaTimeInterval) || 5000;
+
+  // Ensure refresh frequency is not less than 1 second
+  // Prevent a case where local storage could be modified in such a way, that would cause issues
+  // due to too frequent refreshes
+  if (ti < 1000) {
+    localStorage.karafkaTimeInterval = 5000
+    ti = 5000
+  }
+
   livePollTimer = setTimeout(livePollCallback, ti);
 }
 

data/lib/karafka/web/ui/views/consumers/_consumer.erb

@@ -14,7 +14,11 @@
           <% subscription_group.topics.each do |topic| %>
             <span class="badge bg-secondary badge-topic" title="Consumer group: <%= consumer_group.id %>">
               <%= topic.name %>:
-              <%= topic.partitions.map(&:id).join(',') %>
+              <% if topic.partitions.size > 10 %>
+                <%= "#{topic.partitions.first(10).map(&:id).join(',')}..." %>
+              <% else %>
+                <%= topic.partitions.map(&:id).join(',') %>
+              <% end %>
             </span>
           <% end %>
         <% end %>

data/lib/karafka/web/ui/views/dashboard/index.erb

@@ -29,7 +29,7 @@
 
         <div class="tab-content">
           <div class="tab-pane show active" id="messages" role="tabpanel">
-            <% data = @aggregated_charts.with(:messages) %>
+            <% data = @aggregated_charts.with(:messages, :errors) %>
             <%== partial 'shared/chart', locals: { data: data, id: 'messages' } %>
           </div>
 

data/lib/karafka/web/ui/views/status/failures/_consumers_reports.erb

@@ -0,0 +1,11 @@
+<p>
+  At least one consumer report appears to be corrupted.
+</p>
+
+<p>
+  This issue typically arises when invalid messages have been sent to the Karafka consumers' reports topic or when the topic has been populated with data from a newer Karafka Web UI without updating it.
+</p>
+
+<p class="mb-0">
+  To resolve this, please first attempt to upgrade the Karafka Web UI. If the problem persists, execute <code>bundle exec karafka-web reset</code> to reset the Web UI.
+</p>

data/lib/karafka/web/ui/views/status/failures/_initial_consumers_metrics.erb

@@ -1,11 +1,25 @@
-<p>
-  The initial consumers metrics for the Web UI were not created.
-</p>
+<% if details[:issue_type] == :deserialization %>
+  <p>
+    The initial state of the consumers metrics appears to be corrupted.
+  </p>
 
-<p>
-  It means that the <code>bundle exec karafka-web migrate</code> was not executed or failed.
-</p>
+  <p>
+    This issue typically arises when invalid messages have been sent to the Karafka consumers' metrics topic or when the topic has been populated with data from a newer Karafka Web UI without updating it.
+  </p>
 
-<p class="mb-0">
-  To fix this, you need to ensure that the <code>bundle exec karafka-web migrate</code> runs successfully.
-</p>
+  <p class="mb-0">
+    To resolve this, please first attempt to upgrade the Karafka Web UI. If the problem persists, execute <code>bundle exec karafka-web reset</code> to reset the Web UI.
+  </p>
+<% else %>
+  <p>
+    The initial consumers metrics for the Web UI were not created.
+  </p>
+
+  <p>
+    It means that the <code>bundle exec karafka-web migrate</code> was not executed or failed.
+  </p>
+
+  <p class="mb-0">
+    To fix this, you need to ensure that the <code>bundle exec karafka-web migrate</code> runs successfully.
+  </p>
+<% end %>

data/lib/karafka/web/ui/views/status/failures/_initial_consumers_state.erb

@@ -1,11 +1,25 @@
-<p>
-  The initial consumers state for the Web UI was not created.
-</p>
+<% if details[:issue_type] == :deserialization %>
+  <p>
+    The initial state of the consumers appears to be corrupted.
+  </p>
 
-<p>
-  It means that the <code>bundle exec karafka-web install</code> was not executed or failed.
-</p>
+  <p>
+    This issue typically arises when invalid messages have been sent to the Karafka consumers' state topic or when the topic has been populated with data from a newer Karafka Web UI without updating it.
+  </p>
 
-<p class="mb-0">
-  To fix this, you need to ensure that the <code>bundle exec karafka-web migrate</code> runs successfully.
-</p>
+  <p class="mb-0">
+    To resolve this, please first attempt to upgrade the Karafka Web UI. If the problem persists, execute <code>bundle exec karafka-web reset</code> to reset the Web UI.
+  </p>
+<% else %>
+  <p>
+    The initial consumers state for the Web UI was not created.
+  </p>
+
+  <p>
+    It means that the <code>bundle exec karafka-web migrate</code> was not executed or failed.
+  </p>
+
+  <p class="mb-0">
+    To fix this, you need to ensure that the <code>bundle exec karafka-web migrate</code> runs successfully.
+  </p>
+<% end %>

data/lib/karafka/web/ui/views/status/show.erb

@@ -82,7 +82,7 @@
         partial(
           "status/#{@status.initial_consumers_state.to_s}",
           locals: {
-            title: 'Initial consumers state presence',
+            title: 'Initial consumers state',
             description: partial(
               'status/failures/initial_consumers_state',
               locals: {
@@ -97,7 +97,7 @@
         partial(
           "status/#{@status.initial_consumers_metrics.to_s}",
           locals: {
-            title: 'Initial consumers metrics presence',
+            title: 'Initial consumers metrics',
             description: partial(
               'status/failures/initial_consumers_metrics',
               locals: {
@@ -108,6 +108,21 @@
         )
       %>
 
+      <%==
+        partial(
+          "status/#{@status.consumers_reports.to_s}",
+          locals: {
+            title: 'Consumers reports',
+            description: partial(
+              'status/failures/consumers_reports',
+              locals: {
+                details: @status.consumers_reports.details
+              }
+            )
+          }
+        )
+      %>
+
       <%==
         partial(
           "status/#{@status.live_reporting.to_s}",

data/lib/karafka/web/version.rb

@@ -3,6 +3,6 @@
 module Karafka
   module Web
     # Current gem version
-    VERSION = '0.7.0'
+    VERSION = '0.7.2'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-web
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.7.2
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2023-09-14 00:00:00.000000000 Z
+date: 2023-09-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: erubi
@@ -422,6 +422,7 @@ files:
 - lib/karafka/web/ui/views/status/_success.erb
 - lib/karafka/web/ui/views/status/_warning.erb
 - lib/karafka/web/ui/views/status/failures/_connection.erb
+- lib/karafka/web/ui/views/status/failures/_consumers_reports.erb
 - lib/karafka/web/ui/views/status/failures/_consumers_reports_schema_state.erb
 - lib/karafka/web/ui/views/status/failures/_enabled.erb
 - lib/karafka/web/ui/views/status/failures/_initial_consumers_metrics.erb