karafka-web 0.9.2 → 0.10.0.beta1

data/lib/karafka/web/pro/ui/views/topics/_topic.erb

@@ -1,12 +1,9 @@
-<div class="col">
-  <div class="card" >
-    <div class="card-body p-2">
-      <p class="card-text mb-0 p-2">
-        <a href="<%= root_path('topics', topic.topic_name, 'config') %>">
-          <%= topic.topic_name %> /
-          <%= topic.partition_count %>
-        </a>
-      </p>
-    </div>
+<div class="topic-tile">
+  <div class="topic-tile-body">
+    <p class="topic-tile-text">
+      <a href="<%= root_path('topics', topic.topic_name, 'config') %>" class="topic-tile-link">
+        <%= topic.topic_name %> / <%= topic.partition_count %>
+      </a>
+    </p>
   </div>
 </div>

data/lib/karafka/web/pro/ui/views/topics/config.erb

@@ -1,29 +1,25 @@
-<%== view_title(@topic.topic_name, hr: false) %>
+<% view_title @topic.topic_name %>
 
 <%== partial 'topics/tabs' %>
 
-<div class="container mb-5">
-  <div class="row">
-    <div class="col-lg-12 table-responsive">
-      <table class="processes bg-white table table-hover table-bordered table-striped">
-        <thead>
-          <tr class="align-middle">
-            <th><%== sort_link(:name) %></th>
-            <th><%== sort_link(:value) %></th>
-            <th><%== sort_link(:default?) %></th>
-            <th><%== sort_link(:sensitive?) %></th>
-            <th><%== sort_link(:read_only?) %></th>
-          </tr>
-        </thead>
-        <tbody>
-          <%==
-            each_partial(
-              @configs,
-              'cluster/config'
-            )
-          %>
-        </tbody>
-      </table>
-    </div>
-  </div>
+<div class="data-table-wrapper">
+  <table class="data-table">
+    <thead>
+      <tr>
+        <th><%== sort_link(:name) %></th>
+        <th><%== sort_link(:value) %></th>
+        <th><%== sort_link(:default?) %></th>
+        <th><%== sort_link(:sensitive?) %></th>
+        <th><%== sort_link(:read_only?) %></th>
+      </tr>
+    </thead>
+    <tbody>
+      <%==
+        each_partial(
+          @configs,
+          'cluster/config'
+        )
+      %>
+    </tbody>
+  </table>
 </div>

data/lib/karafka/web/pro/ui/views/topics/distribution/_badges.erb

@@ -1,7 +1,12 @@
-<p class="text-end">
-  <span class="badge bg-primary">Total messages: <%= number_with_delimiter @aggregated.sum.round, ' ' %></span>
-  <span class="badge bg-primary">Std dev: <%= @aggregated.std_dev %></span>
-  <span class="badge bg-primary">Std dev relative: <%= @aggregated.std_dev_rel %>%</span>
+<p class="text-end mb-5">
+  <%==
+    value = number_with_delimiter @aggregated.sum.round, ' '
+    badge_primary("Total messages: #{value}")
+  %>
+
+  <%== badge_primary("Std dev: #{@aggregated.std_dev}") %>
+  <%== badge_primary("Std dev relative: #{@aggregated.std_dev_rel}%") %>
+
   <% partitions_range = [@active_partitions.first, @active_partitions.last].uniq.join(' to ') %>
-  <span class="badge bg-secondary">Partitions: <%= partitions_range %></span>
+  <%== badge_secondary("Partitions: #{partitions_range}") %>
 </p>

data/lib/karafka/web/pro/ui/views/topics/distribution/_chart.erb

@@ -1,2 +1,4 @@
 <% data = { 'Estimated count': @distribution.map { [_1.partition_id, _1.count] } }.to_json %>
-<%== partial 'shared/charts/bar', locals: { data: data, id: 'distribution' } %>
+<div class="mb-5">
+  <%== partial 'shared/charts/bar', locals: { data: data, id: 'distribution' } %>
+</div>

data/lib/karafka/web/pro/ui/views/topics/distribution/_limited.erb

@@ -1,4 +1,4 @@
-<div class="mb-4">
+<div class="mb-10">
   <%==
     alert_info(
       '

data/lib/karafka/web/pro/ui/views/topics/distribution.erb

@@ -1,47 +1,42 @@
-<%== view_title(@topic.topic_name, hr: false) %>
+<% view_title @topic.topic_name %>
 
 <%== partial 'topics/tabs' %>
 
-<div class="container">
-  <div class="row">
-    <div class="col-lg-12">
-      <% if @limited %>
-        <%== partial 'topics/distribution/limited' %>
-      <% end %>
+<% if @limited %>
+  <%== partial 'topics/distribution/limited' %>
+<% end %>
 
-      <% if @aggregated.sum.zero? %>
-        <%== partial 'topics/distribution/empty_partitions' %>
-      <% else %>
-        <% if @active_partitions.size >= 2 %>
-          <%== partial 'topics/distribution/chart' %>
-        <% end %>
+<% if @aggregated.sum.zero? %>
+  <%== partial 'topics/distribution/empty_partitions' %>
+<% else %>
+  <div class="col-span-12 mb-3">
+    <% if @active_partitions.size >= 2 %>
+      <%== partial 'topics/distribution/chart' %>
+    <% end %>
 
-        <div id="refreshable" class="mt-4">
-          <%== partial 'topics/distribution/badges' %>
-
-          <div class="table-responsive">
-            <table class="bg-white table table-hover table-bordered table-striped mt-4 mb-4">
-              <thead>
-                <tr class="align-middle">
-                  <th><%== sort_link(:partition_id) %></th>
-                  <th><%== sort_link('Estimated count', :count) %></th>
-                  <th><%== sort_link('Total share', :share) %></th>
-                  <th><%== sort_link(:diff) %></th>
-                </tr>
-              </thead>
-              <tbody>
-                <%==
-                  each_partial(
-                    @distribution,
-                    'topics/distribution/partition'
-                  )
-                %>
-              </tbody>
-            </table>
-          </div>
-        </div>
-      <% end %>
+    <div id="refreshable" class="mt-4">
+      <%== partial 'topics/distribution/badges' %>
 
+      <div class="data-table-wrapper">
+        <table class="data-table">
+          <thead>
+            <tr>
+              <th><%== sort_link(:partition_id) %></th>
+              <th><%== sort_link('Estimated count', :count) %></th>
+              <th><%== sort_link('Total share', :share) %></th>
+              <th><%== sort_link(:diff) %></th>
+            </tr>
+          </thead>
+          <tbody>
+            <%==
+              each_partial(
+                @distribution,
+                'topics/distribution/partition'
+              )
+            %>
+          </tbody>
+        </table>
+      </div>
     </div>
   </div>
-</div>
+<% end %>

data/lib/karafka/web/pro/ui/views/topics/index.erb

@@ -1,16 +1,14 @@
-<%== view_title('Topics', hr: true) %>
+<% view_title 'Topics' %>
 
-<div class="container">
-  <% if @topics.empty? %>
-    <%== partial 'explorer/no_topics' %>
-  <% else %>
-    <div class="row mb-5 row-cols-1 row-cols-md-4 g-4">
-      <%==
-        each_partial(
-          @topics,
-          'topics/topic'
-        )
-      %>
-    </div>
-  <% end %>
-</div>
+<% if @topics.empty? %>
+  <%== partial 'explorer/no_topics' %>
+<% else %>
+  <div class="topic-tiles">
+    <%==
+      each_partial(
+        @topics,
+        'topics/topic'
+      )
+    %>
+  </div>
+<% end %>

data/lib/karafka/web/pro/ui/views/topics/offsets.erb

@@ -0,0 +1,24 @@
+<% view_title @topic.topic_name %>
+
+<%== partial 'topics/tabs' %>
+
+<div class="data-table-wrapper">
+  <table class="data-table">
+    <thead>
+      <tr>
+        <th><%== sort_link(:partition_id) %></th>
+        <th><%== sort_link(:low) %></th>
+        <th><%== sort_link(:high) %></th>
+        <th><%== sort_link('Count', :diff) %></th>
+      </tr>
+    </thead>
+    <tbody>
+      <%==
+        each_partial(
+          @offsets,
+          'topics/partition_offsets'
+        )
+      %>
+    </tbody>
+  </table>
+</div>

data/lib/karafka/web/pro/ui/views/topics/replication.erb

@@ -1,28 +1,24 @@
-<%== view_title(@topic.topic_name, hr: false) %>
+<% view_title @topic.topic_name %>
 
 <%== partial 'topics/tabs' %>
 
-<div class="container mb-5">
-  <div class="row">
-    <div class="col-lg-12 table-responsive">
-      <table class="processes bg-white table table-hover table-bordered table-striped">
-        <thead>
-          <tr class="align-middle">
-            <th><%== sort_link(:partition_id) %></th>
-            <th><%== sort_link(:leader) %></th>
-            <th><%== sort_link(:replica_count) %></th>
-            <th><%== sort_link('In sync brokers', :in_sync_replica_brokers) %></th>
-          </tr>
-        </thead>
-        <tbody>
-          <%==
-            each_partial(
-              @partitions,
-              'topics/partition'
-            )
-          %>
-        </tbody>
-      </table>
-    </div>
-  </div>
+<div class="data-table-wrapper">
+  <table class="data-table">
+    <thead>
+      <tr>
+        <th><%== sort_link(:partition_id) %></th>
+        <th><%== sort_link(:leader) %></th>
+        <th><%== sort_link(:replica_count) %></th>
+        <th><%== sort_link('In sync brokers', :in_sync_replica_brokers) %></th>
+      </tr>
+    </thead>
+    <tbody>
+      <%==
+        each_partial(
+          @partitions,
+          'topics/partition'
+        )
+      %>
+    </tbody>
+  </table>
 </div>

data/lib/karafka/web/processing/consumers/aggregators/metrics.rb

@@ -11,7 +11,7 @@ module Karafka
           class Metrics < Base
             # Current schema version
             # This is used for detecting incompatible changes and writing migrations
-            SCHEMA_VERSION = '1.2.1'
+            SCHEMA_VERSION = '1.3.0'
 
             def initialize
               super

data/lib/karafka/web/processing/consumers/aggregators/state.rb

@@ -20,7 +20,7 @@ module Karafka
             # Current schema version
             # This can be used in the future for detecting incompatible changes and writing
             # migrations
-            SCHEMA_VERSION = '1.3.1'
+            SCHEMA_VERSION = '1.4.0'
 
             # @param schema_manager [Karafka::Web::Processing::Consumers::SchemaManager] schema
             #   manager that tracks the compatibility of schemas.

data/lib/karafka/web/processing/consumers/contracts/aggregated_stats.rb

@@ -11,6 +11,7 @@ module Karafka
             configure
 
             required(:batches) { |val| val.is_a?(Integer) && val >= 0 }
+            required(:jobs) { |val| val.is_a?(Integer) && val >= 0 }
             required(:messages) { |val| val.is_a?(Integer) && val >= 0 }
             required(:retries) { |val| val.is_a?(Integer) && val >= 0 }
             required(:dead) { |val| val.is_a?(Integer) && val >= 0 }

data/lib/karafka/web/tracking/consumers/contracts/report.rb

@@ -29,6 +29,7 @@ module Karafka
               required(:threads) { |val| val.is_a?(Integer) && val >= 0 }
               required(:workers) { |val| val.is_a?(Integer) && val.positive? }
               required(:tags) { |val| val.is_a?(Karafka::Core::Taggable::Tags) }
+              required(:execution_mode) { |val| val.is_a?(String) && !val.empty? }
 
               nested(:listeners) do
                 required(:active) { |val| val.is_a?(Integer) && val >= 0 }
@@ -60,6 +61,11 @@ module Karafka
               required(:utilization) { |val| val.is_a?(Numeric) && val >= 0 }
 
               nested(:total) do
+                # There can be jobs that run without new data batches like revocation, periodic,
+                # shutdown, etc. We want to track them. This is needed because in case of a
+                # setup where those are significant, user may have a false sense that nothing
+                # is happening in the system when no new messages are coming
+                required(:jobs) { |val| val.is_a?(Integer) && val >= 0 }
                 required(:batches) { |val| val.is_a?(Integer) && val >= 0 }
                 required(:messages) { |val| val.is_a?(Integer) && val >= 0 }
                 required(:errors) { |val| val.is_a?(Integer) && val >= 0 }

data/lib/karafka/web/tracking/consumers/listeners/connections.rb

@@ -7,11 +7,15 @@ module Karafka
         module Listeners
           # Listener for listening on connections related events like polling, etc
           class Connections < Base
-            # Set first poll time before we start fetching so we always have a poll time
-            # and we don't have to worry about it being always available
+            # Initializes the subscription group with defaults so it is always available
             # @param event [Karafka::Core::Monitoring::Event]
             def on_connection_listener_before_fetch_loop(event)
-              on_connection_listener_fetch_loop_received(event)
+              sg_id = event[:subscription_group].id
+
+              track do |sampler|
+                # This will initialize the hash upon first request
+                sampler.subscription_groups[sg_id]
+              end
             end
 
             # When fetch loop is done it means this subscription group is no longer active and we
@@ -36,9 +40,7 @@ module Karafka
               sg_id = event[:subscription_group].id
 
               track do |sampler|
-                sampler.subscription_groups[sg_id] = {
-                  polled_at: monotonic_now
-                }
+                sampler.subscription_groups[sg_id][:polled_at] = monotonic_now
               end
             end
           end

data/lib/karafka/web/tracking/consumers/listeners/processing.rb

@@ -51,6 +51,7 @@ module Karafka
               track do |sampler|
                 # We count batches and messages prior to the execution, so they are tracked even
                 # if error occurs, etc.
+                sampler.counters[:jobs] += 1
                 sampler.counters[:batches] += 1
                 sampler.counters[:messages] += messages_count
                 sampler.jobs[jid] = job_details
@@ -120,6 +121,7 @@ module Karafka
                   job_details = job_details(consumer, '#{action}')
 
                   track do |sampler|
+                    sampler.counters[:jobs] += 1
                     sampler.jobs[jid] = job_details
                   end
                 end

data/lib/karafka/web/tracking/consumers/listeners/tags.rb

@@ -47,7 +47,7 @@ module Karafka
               attempt = consumer.coordinator.pause_tracker.attempt
 
               if attempt > 1
-                consumer.tags.add(:attempt, "attempt:#{attempt}")
+                consumer.tags.add(:attempt, "attempt: #{attempt}")
               else
                 consumer.tags.delete(:attempt)
               end

data/lib/karafka/web/tracking/consumers/reporter.rb

@@ -6,13 +6,6 @@ module Karafka
       module Consumers
         # Reports the collected data about the process and sends it, so we can use it in the UI
         class Reporter < Tracking::Reporter
-          # Minimum number of messages to produce to produce them in sync mode
-          # This acts as a small back-off not to overload the system in case we would have
-          # extremely big number of errors happening
-          PRODUCE_SYNC_THRESHOLD = 25
-
-          private_constant :PRODUCE_SYNC_THRESHOLD
-
           # This mutex is shared between tracker and samplers so there is no case where metrics
           # would be collected same time tracker reports
           MUTEX = Mutex.new
@@ -134,7 +127,7 @@ module Karafka
           #   normal operations we should not have that many messages to dispatch and it should not
           #   slowdown any processing.
           def produce(messages)
-            if messages.count >= PRODUCE_SYNC_THRESHOLD
+            if messages.count >= sync_threshold
               ::Karafka::Web.producer.produce_many_sync(messages)
             else
               ::Karafka::Web.producer.produce_many_async(messages)
@@ -145,6 +138,11 @@ module Karafka
           rescue WaterDrop::Errors::ProducerClosedError
             nil
           end
+
+          # @return [Integer] min number of messages when we switch to sync flushing to slow things
+          def sync_threshold
+            @sync_threshold ||= ::Karafka::Web.config.tracking.consumers.sync_threshold
+          end
         end
       end
     end

data/lib/karafka/web/tracking/consumers/sampler.rb

@@ -15,10 +15,12 @@ module Karafka
           # Current schema version
           # This is used for detecting incompatible changes and not using outdated data during
           # upgrades
-          SCHEMA_VERSION = '1.3.0'
+          SCHEMA_VERSION = '1.4.0'
 
           # Counters that count events occurrences during the given window
           COUNTERS_BASE = {
+            # Number of processed jobs of any type
+            jobs: 0,
             # Number of processed batches
             batches: 0,
             # Number of processed messages
@@ -38,13 +40,21 @@ module Karafka
 
             @windows = Helpers::Ttls::Windows.new
             @counters = COUNTERS_BASE.dup
+
             @consumer_groups = Hash.new do |h, cg_id|
               h[cg_id] = {
                 id: cg_id,
                 subscription_groups: {}
               }
             end
-            @subscription_groups = {}
+
+            @subscription_groups = Hash.new do |h, sg_id|
+              h[sg_id] = {
+                id: sg_id,
+                polled_at: monotonic_now
+              }
+            end
+
             @errors = []
             @pauses = {}
             @jobs = {}
@@ -73,6 +83,7 @@ module Karafka
                 id: process_id,
                 started_at: started_at,
                 status: ::Karafka::App.config.internal.status.to_s,
+                execution_mode: ::Karafka::Server.execution_mode.to_s,
                 listeners: listeners,
                 workers: workers,
                 memory_usage: @memory_usage,
@@ -148,7 +159,7 @@ module Karafka
 
             # We divide by 1_000 to convert from milliseconds
             # We multiply by 100 to have it in % scale
-            totals.sum / 1_000 / workers / timefactor * 100
+            (totals.sum / 1_000 / workers / timefactor * 100).round(2)
           end
 
           # @return [Hash] number of active and standby listeners
@@ -294,12 +305,12 @@ module Karafka
             @consumer_groups.each_value do |cg_details|
               cg_details.each do
                 cg_details.fetch(:subscription_groups, {}).each do |sg_id, sg_details|
-                  # This should be always available, since we subscription group polled at time
+                  # This should be always available, since the subscription group polled at time
                   # is first initialized before we start polling, there should be no case where
                   # we have statistics about a given subscription group but we do not have the
                   # last polling time
                   polled_at = subscription_groups.fetch(sg_id).fetch(:polled_at)
-                  sg_details[:state][:poll_age] = monotonic_now - polled_at
+                  sg_details[:state][:poll_age] = (monotonic_now - polled_at).round(2)
                 end
               end
             end

data/lib/karafka/web/ui/app.rb

@@ -27,12 +27,21 @@ module Karafka
           end
 
           r.on 'consumers' do
+            %w[
+              performance
+              controls
+              commands
+            ].each do |path|
+              r.get path do |_process_id|
+                raise Errors::Ui::ProOnlyError
+              end
+            end
+
             r.get String, 'subscriptions' do |_process_id|
               raise Errors::Ui::ProOnlyError
             end
 
             r.get do
-              @breadcrumbs = false
               controller = Controllers::ConsumersController.new(params)
               controller.index
             end
@@ -108,6 +117,16 @@ module Karafka
             controller = Controllers::StatusController.new(params)
             controller.show
           end
+
+          r.get 'ux' do
+            controller = Controllers::UxController.new(params)
+            controller.show
+          end
+
+          r.get 'support' do
+            controller = Controllers::SupportController.new(params)
+            controller.show
+          end
         end
       end
     end

data/lib/karafka/web/ui/base.rb

@@ -7,7 +7,7 @@ module Karafka
       class Base < Roda
         include Helpers::PathsHelper
         include Helpers::ApplicationHelper
-        include Helpers::AlertsHelper
+        include Helpers::TailwindHelper
 
         # Details that need to be evaluated in the context of OSS or Pro web UI.
         # If those would be evaluated in the base, they would not be initialized as expected
@@ -17,7 +17,9 @@ module Karafka
             root: Karafka::Web.gem_root.join('lib/karafka/web/ui/public'),
             # Cache all static files for the end user for as long as possible
             # We can do it because we ship per version assets so they invalidate with gem bumps
-            headers: { 'Cache-Control' => 'max-age=31536000, immutable' }
+            headers: { 'Cache-Control' => 'max-age=31536000, immutable' },
+            gzip: true,
+            brotli: true
           )
           plugin :render_each
           plugin :partials
@@ -35,6 +37,9 @@ module Karafka
         plugin :hooks
         plugin :flash
         plugin :path
+        plugin :capture_erb
+        plugin :content_for
+        plugin :inject_erb
 
         # Based on
         # https://github.com/sidekiq/sidekiq/blob/ae6ca119/lib/sidekiq/web/application.rb#L8
@@ -48,7 +53,7 @@ module Karafka
           csp.manifest_src "'self'"
           csp.media_src "'self'"
           csp.object_src "'none'"
-          csp.script_src "'self' https: http: 'unsafe-inline'"
+          csp.script_src "'self' https: http:"
           csp.style_src "'self' https: http: 'unsafe-inline'"
           csp.worker_src "'self'"
           csp.base_uri "'self'"
@@ -60,12 +65,6 @@ module Karafka
           render_response(result)
         end
 
-        handle_block_result Controllers::Responses::Deny do
-          @error = true
-          response.status = 403
-          view 'shared/exceptions/not_allowed'
-        end
-
         # Redirect either to referer back or to the desired path
         handle_block_result Controllers::Responses::Redirect do |result|
           # Map redirect flashes (if any) to Roda flash messages
@@ -84,13 +83,18 @@ module Karafka
         plugin :error_handler, classes: [
           ::Rdkafka::RdkafkaError,
           Errors::Ui::NotFoundError,
-          Errors::Ui::ProOnlyError
+          Errors::Ui::ProOnlyError,
+          Errors::Ui::ForbiddenError
         ] do |e|
           @error = true
 
-          if e.is_a?(Errors::Ui::ProOnlyError)
+          case e
+          when Errors::Ui::ProOnlyError
             response.status = 402
             view 'shared/exceptions/pro_only'
+          when Errors::Ui::ForbiddenError
+            response.status = 403
+            view 'shared/exceptions/not_allowed'
           else
             response.status = 404
             view 'shared/exceptions/not_found'
@@ -128,15 +132,17 @@ module Karafka
         # the query data. Query data takes priority over request params.
         # @param query_data [Hash] query params we want to add to the current path
         path :current do |query_data = {}|
-          q = query_data
-              .transform_values(&:to_s)
-              .transform_keys(&:to_s)
-              .then { |candidates| request.params.merge(candidates) }
-              .select { |_, v| v }
-              .map { |k, v| "#{k}=#{CGI.escape(v.to_s)}" }
-              .join('&')
-
-          [request.path, q].compact.delete_if(&:empty?).join('?')
+          # Merge existing request parameters with new query data
+          merged_params = request.params.deep_merge(query_data)
+
+          # Flatten the merged parameters
+          flattened_params = flatten_params('', merged_params)
+
+          # Build the query string from the flattened parameters
+          query_string = URI.encode_www_form(flattened_params)
+
+          # Construct the full path with query string
+          [request.path, query_string].compact.join('?')
         end
 
         # Sets appropriate template variables based on the response object and renders the