karafka-web 0.7.10 → 0.8.0.rc1

data/lib/karafka/web/ui/pro/controllers/explorer.rb

@@ -26,11 +26,11 @@ module Karafka
                         .topics
                         .sort_by { |topic| topic[:topic_name] }
 
-              unless ::Karafka::Web.config.ui.show_internal_topics
+              unless ::Karafka::Web.config.ui.visibility.internal_topics
                 @topics.reject! { |topic| topic[:topic_name].start_with?('__') }
               end
 
-              respond
+              render
             end
 
             # Displays aggregated messages from (potentially) all partitions of a topic
@@ -45,7 +45,7 @@ module Karafka
             # @note We cannot use offset references here because each of the partitions may have
             #   completely different values
             def topic(topic_id)
-              @visibility_filter = ::Karafka::Web.config.ui.visibility_filter
+              @visibility_filter = ::Karafka::Web.config.ui.visibility.filter
 
               @topic_id = topic_id
               @partitions_count = Models::ClusterInfo.partitions_count(topic_id)
@@ -60,7 +60,7 @@ module Karafka
 
               paginate(@params.current_page, next_page)
 
-              respond
+              render
             end
 
             # Shows messages available in a given partition
@@ -68,7 +68,7 @@ module Karafka
             # @param topic_id [String]
             # @param partition_id [Integer]
             def partition(topic_id, partition_id)
-              @visibility_filter = ::Karafka::Web.config.ui.visibility_filter
+              @visibility_filter = ::Karafka::Web.config.ui.visibility.filter
               @topic_id = topic_id
               @partition_id = partition_id
               @watermark_offsets = Ui::Models::WatermarkOffsets.find(topic_id, partition_id)
@@ -84,7 +84,7 @@ module Karafka
                 @messages.map { |message| message.is_a?(Array) ? message.last : message.offset }
               )
 
-              respond
+              render
             end
 
             # Displays given message
@@ -94,7 +94,7 @@ module Karafka
             # @param offset [Integer] offset of the message we want to display
             # @param paginate [Boolean] do we want to have pagination
             def show(topic_id, partition_id, offset, paginate: true)
-              @visibility_filter = ::Karafka::Web.config.ui.visibility_filter
+              @visibility_filter = ::Karafka::Web.config.ui.visibility.filter
               @topic_id = topic_id
               @partition_id = partition_id
               @offset = offset
@@ -116,7 +116,7 @@ module Karafka
                 paginate(offset, watermark_offsets.low, watermark_offsets.high)
               end
 
-              respond
+              render
             end
 
             # Displays the most recent message on a topic/partition

data/lib/karafka/web/ui/pro/controllers/health.rb

@@ -18,12 +18,36 @@ module Karafka
         module Controllers
           # Health state controller
           class Health < Ui::Controllers::Base
+            self.sortable_attributes = %w[
+              id
+              lag_stored
+              lag_stored_d
+              committed_offset
+              committed_offset_fd
+              stored_offset
+              stored_offset_fd
+              hi_offset
+              hi_offset_fd
+              ls_offset
+              ls_offset_fd
+              fetch_state
+              poll_state
+              lso_risk_state
+              name
+              poll_state_ch
+            ].freeze
+
             # Displays the current system state
             def overview
               current_state = Models::ConsumersState.current!
               @stats = Models::Health.current(current_state)
 
-              respond
+              # Refine only on a per topic basis not to resort higher levels
+              @stats.each_value do |cg_details|
+                cg_details.each_value { |topic_details| refine(topic_details) }
+              end
+
+              render
             end
 
             # Displays details about offsets and their progression/statuses
@@ -31,7 +55,15 @@ module Karafka
               # Same data as overview but presented differently
               overview
 
-              respond
+              render
+            end
+
+            # Displays information related to time of changes of particular attributes
+            def changes
+              # Same data as overview but presented differently
+              overview
+
+              render
             end
           end
         end

data/lib/karafka/web/ui/pro/controllers/jobs.rb

@@ -18,6 +18,17 @@ module Karafka
         module Controllers
           # Displays list of active jobs
           class Jobs < Ui::Controllers::Jobs
+            self.sortable_attributes = %w[
+              name
+              topic
+              consumer
+              type
+              messages
+              first_offset
+              last_offset
+              committed_offset
+              updated_at
+            ].freeze
           end
         end
       end

data/lib/karafka/web/ui/pro/controllers/messages.rb

@@ -42,6 +42,43 @@ module Karafka
               )
             end
 
+            # Dispatches the message raw payload to the browser as a file
+            #
+            # @param topic_id [String]
+            # @param partition_id [Integer]
+            # @param offset [Integer] offset of the message we want to download
+            def download(topic_id, partition_id, offset)
+              message = Ui::Models::Message.find(topic_id, partition_id, offset)
+
+              # Check if downloads are allowed
+              return deny unless visibility_filter.download?(message)
+
+              file(
+                message.raw_payload,
+                "#{topic_id}_#{partition_id}_#{offset}_payload.msg"
+              )
+            end
+
+            # Dispatches the message payload first deserialized and then serialized to JSON
+            # It differs from the raw payload in cases where raw payload is compressed or binary
+            # or contains data that the Web UI user should not see that was altered on the Web UI
+            # with the visibility filter.
+            #
+            # @param topic_id [String]
+            # @param partition_id [Integer]
+            # @param offset [Integer] offset of the message we want to export
+            def export(topic_id, partition_id, offset)
+              message = Ui::Models::Message.find(topic_id, partition_id, offset)
+
+              # Check if exports are allowed
+              return deny unless visibility_filter.export?(message)
+
+              file(
+                message.payload.to_json,
+                "#{topic_id}_#{partition_id}_#{offset}_payload.json"
+              )
+            end
+
             private
 
             # @param message [Karafka::Messages::Message]
@@ -54,6 +91,11 @@ module Karafka
                 and received offset #{delivery.offset}.
               MSG
             end
+
+            # @return [Object] visibility filter. Either default or user-based
+            def visibility_filter
+              ::Karafka::Web.config.ui.visibility.filter
+            end
           end
         end
       end

data/lib/karafka/web/ui/pro/controllers/routing.rb

@@ -18,13 +18,22 @@ module Karafka
         module Controllers
           # Routing details - same as in OSS
           class Routing < Ui::Controllers::Routing
+            self.sortable_attributes = %w[
+              name
+              active?
+            ].freeze
+
             # Routing list
             def index
               detect_patterns_routes
 
               @routes = Karafka::App.routes
 
-              respond
+              @routes.each do |consumer_group|
+                refine(consumer_group.topics)
+              end
+
+              render
             end
 
             # Given route details
@@ -37,7 +46,7 @@ module Karafka
 
               @topic || raise(::Karafka::Web::Errors::Ui::NotFoundError, topic_id)
 
-              respond
+              render
             end
 
             private

data/lib/karafka/web/ui/pro/views/consumers/_breadcrumbs.erb

@@ -14,13 +14,19 @@
   <% if current_path.include?('/jobs') %>
     <li class="breadcrumb-item">
       <a href="<%= root_path('consumers', @process.id, 'jobs') %>">
-        Running jobs
+        Jobs
+      </a>
+    </li>
+
+    <li class="breadcrumb-item">
+      <a href="<%= root_path('consumers', @process.id, 'jobs') %>">
+        Running
       </a>
     </li>
   <% elsif current_path.include?('/subscriptions') %>
     <li class="breadcrumb-item">
       <a href="<%= root_path('consumers', @process.id, 'subscriptions') %>">
-        Active subscriptions
+        Subscriptions
       </a>
     </li>
   <% else %>

data/lib/karafka/web/ui/pro/views/consumers/_consumer.erb

@@ -10,18 +10,24 @@
 
     <p class="mt-0 mb-1">
       <% process.consumer_groups.each do |consumer_group| %>
+        <% sg_topics = Hash.new { |h, k| h[k] = [] } %>
+
         <% consumer_group.subscription_groups.each do |subscription_group| %>
           <% subscription_group.topics.each do |topic| %>
-            <span class="badge bg-secondary badge-topic" title="Consumer group: <%= consumer_group.id %>">
-              <%= topic.name %>:
-              <% if topic.partitions.size > 10 %>
-                <%= "#{topic.partitions.first(10).map(&:id).join(',')}..." %>
-              <% else %>
-                <%= topic.partitions.map(&:id).join(',') %>
-              <% end %>
-            </span>
+            <% sg_topics[topic.name] << topic.partitions.map(&:id) %>
           <% end %>
         <% end %>
+
+        <% sg_topics.each do |topic_name, partitions| %>
+          <span class="badge bg-secondary badge-topic" title="Consumer group: <%= consumer_group.id %>">
+            <%= topic_name %>:
+            <% if partitions.size > 10 %>
+              <%= "#{partitions.sort.first(10).join(',')}..." %>
+            <% else %>
+              <%= partitions.sort.join(',') %>
+            <% end %>
+          </span>
+        <% end %>
       <% end %>
     </p>
 

data/lib/karafka/web/ui/pro/views/consumers/_counters.erb

@@ -24,18 +24,20 @@
                 <div class="desc">Lag stored</div>
               </li>
               <li class="col-sm">
-                <a href="<%= root_path('jobs') %>">
+                <a href="<%= root_path('jobs/running') %>">
                   <div class="count mb-1">
                     <%= number_with_delimiter @counters.busy, ' ' %>
                   </div>
-                  <div class="desc">Busy</div>
+                  <div class="desc">Running</div>
                 </a>
               </li>
               <li class="col-sm">
-                <div class="count mb-1">
-                  <%= number_with_delimiter @counters.enqueued, ' ' %>
-                </div>
-                <div class="desc">Enqueued</div>
+                <a href="<%= root_path('jobs/pending') %>">
+                  <div class="count mb-1">
+                    <%= number_with_delimiter @counters.pending, ' ' %>
+                  </div>
+                  <div class="desc">Pending</div>
+                </a>
               </li>
               <li class="col-sm">
                 <a href="<%= root_path('errors') %>">

data/lib/karafka/web/ui/pro/views/consumers/consumer/_job.erb

@@ -16,6 +16,9 @@
   <td>
     <code>#<%= job.type %></code>
   </td>
+  <td>
+    <%= job.messages %>
+  </td>
   <td>
     <%== offset_with_label job.topic, job.partition, job.first_offset, explore: true %>
   </td>
@@ -26,6 +29,6 @@
     <%== offset_with_label job.topic, job.partition, job.committed_offset, explore: true %>
   </td>
   <td>
-    <%== relative_time job.started_at %>
+    <%== relative_time job.updated_at %>
   </td>
 </tr>

data/lib/karafka/web/ui/pro/views/consumers/consumer/_no_jobs.erb

@@ -2,7 +2,7 @@
   <div class="row">
     <div class="col-lg-12">
       <div class="alert alert-info" role="alert">
-        This process is not running any jobs at the moment.
+        This process has no <%= type %> jobs at the moment.
       </div>
     </div>
   </div>

data/lib/karafka/web/ui/pro/views/consumers/consumer/_partition.erb

@@ -26,9 +26,7 @@
     </span>
   </td>
   <td>
-    <span class="badge <%= kafka_state_bg(partition.poll_state) %> mt-1 mb-1">
-      <%= partition.poll_state %>
-    </span>
+    <%== poll_state_with_change_time_label(partition.poll_state, partition.poll_state_ch) %>
   </td>
   <td>
     <span class="badge bg-success <%= lso_risk_state_bg(partition) %> bg-opacity-100">

data/lib/karafka/web/ui/pro/views/consumers/consumer/_subscription_group.erb

@@ -14,6 +14,7 @@
           </span>
         </div>
       </div>
+
       <div class="card">
         <div class="card-body d-flex flex-column align-items-center justify-content-center p-2">
           Join state:&nbsp;
@@ -22,30 +23,46 @@
           </span>
         </div>
       </div>
+
       <div class="card">
         <div class="card-body d-flex flex-column align-items-center justify-content-center p-2">
           State change:&nbsp;
           <span class="badge bg-secondary mt-1 mb-1">
             <%==
               relative_time(
-                Time.at(@process.dispatched_at) - (subscription_group.stateage / 1_000)
+                Time.at(@process.dispatched_at) - (subscription_group.stateage / 1_000.0)
               )
             %>
           </span>
         </div>
       </div>
+
+      <div class="card">
+        <div class="card-body d-flex flex-column align-items-center justify-content-center p-2">
+          Last Poll:&nbsp;
+          <span class="badge bg-secondary mt-1 mb-1">
+            <%==
+              relative_time(
+                Time.at(@process.dispatched_at) - (subscription_group.poll_age / 1_000.0)
+              )
+            %>
+          </span>
+        </div>
+      </div>
+
       <div class="card">
         <div class="card-body d-flex flex-column align-items-center justify-content-center p-2">
           Last rebalance:&nbsp;
           <span class="badge bg-secondary mt-1 mb-1">
             <%==
               relative_time(
-                Time.at(@process.dispatched_at) - (subscription_group.rebalance_age / 1_000)
+                Time.at(@process.dispatched_at) - (subscription_group.rebalance_age / 1_000.0)
               )
             %>
           </span>
         </div>
       </div>
+
       <div class="card">
         <div class="card-body d-flex flex-column align-items-center justify-content-center p-2">
           Rebalance count:&nbsp;
@@ -68,7 +85,7 @@
   <div class="row mb-4">
     <div class="col-lg-12">
       <div class="alert alert-info" role="alert">
-        This process does not consume any messages from any topics of this consumer group.
+        This process does not consume any messages from any topics of this subscription group.
       </div>
     </div>
   </div>
@@ -86,14 +103,14 @@
               </th>
             <tr class="align-middle">
             </tr>
-              <th>Partition</th>
-              <th>Lag stored</th>
-              <th>Lag stored trend</th>
-              <th>Committed offset</th>
-              <th>Stored offset</th>
-              <th>Fetch state</th>
-              <th>Poll state</th>
-              <th>LSO state</th>
+              <th><%== sort_link('Partition', :id) %></th>
+              <th><%== sort_link(:lag_stored) %></th>
+              <th><%== sort_link('Lag stored trend', :lag_stored_d) %></th>
+              <th><%== sort_link(:committed_offset) %></th>
+              <th><%== sort_link(:stored_offset) %></th>
+              <th><%== sort_link(:fetch_state) %></th>
+              <th><%== sort_link(:poll_state) %></th>
+              <th><%== sort_link('LSO state', :lso_risk_state) %></th>
             </tr>
           </thead>
           <tbody>

data/lib/karafka/web/ui/pro/views/consumers/consumer/_tabs.erb

@@ -5,15 +5,22 @@
       <ul class="nav nav-tabs">
         <li class="nav-item">
           <a class="nav-link <%= nav_class(include: 'subscriptions') %>" href="<%= root_path('consumers', @process.id, 'subscriptions') %>">
-            Active subscriptions
+            Subscriptions
             (<%= @process.subscribed_partitions_count %>)
           </a>
         </li>
 
         <li class="nav-item">
-          <a class="nav-link <%= nav_class(include: 'jobs') %>" href="<%= root_path('consumers', @process.id, 'jobs') %>">
+          <a class="nav-link <%= nav_class(include: 'jobs/running') %>" href="<%= root_path('consumers', @process.id, 'jobs', 'running') %>">
             Running jobs
-            (<%= @process.jobs.count %>)
+            (<%= @process.jobs.running.count %>)
+          </a>
+        </li>
+
+        <li class="nav-item">
+          <a class="nav-link <%= nav_class(include: 'jobs/pending') %>" href="<%= root_path('consumers', @process.id, 'jobs', 'pending') %>">
+            Pending jobs
+            (<%= @process.jobs.pending.count %>)
           </a>
         </li>
 

data/lib/karafka/web/ui/pro/views/consumers/index.erb

@@ -12,12 +12,12 @@
         <table class="processes bg-white table table-hover table-bordered table-striped mb-0 align-middle">
           <thead>
             <tr class="align-middle">
-              <th>Name</th>
-              <th class="col-sm-2">Started</th>
+              <th><%== sort_link(:name) %></th>
+              <th class="col-sm-2"><%== sort_link('Started', :started_at, rev: true) %></th>
               <th class="col-sm-1">Memory</th>
               <th class="col-sm-1">Performance</th>
               <th class="col-sm-1">Load</th>
-              <th class="col-sm-1">Lag stored</th>
+              <th class="col-sm-1"><%== sort_link(:lag_stored) %></th>
             </tr>
           </thead>
           <tbody>

data/lib/karafka/web/ui/pro/views/consumers/pending_jobs.erb

@@ -0,0 +1,43 @@
+<%== view_title(@process.name) %>
+
+<% if @process.status == 'stopped' %>
+  <%== partial 'consumers/consumer/stopped' %>
+<% end %>
+
+<%== partial 'consumers/consumer/metrics' %>
+
+<%== partial 'consumers/consumer/tabs' %>
+
+<% if @pending_jobs.empty? %>
+  <%== partial 'consumers/consumer/no_jobs', locals: { type: 'pending' } %>
+<% else %>
+  <div class="container">
+    <div class="row mb-5">
+      <div class="col-sm-12">
+        <table class="processes bg-white table table-hover table-bordered table-striped mb-0 align-middle">
+          <thead>
+            <tr class="align-middle">
+              <th><%== sort_link(:topic) %></th>
+              <th><%== sort_link(:consumer) %></th>
+              <th><%== sort_link(:type) %></th>
+              <th><%== sort_link(:messages) %></th>
+              <th><%== sort_link(:first_offset) %></th>
+              <th><%== sort_link(:last_offset) %></th>
+              <th><%== sort_link(:committed_offset) %></th>
+              <th><%== sort_link('Scheduled at', :updated_at, rev: true) %></th>
+            </tr>
+          </thead>
+          <tbody>
+            <%==
+              render_each(
+                @pending_jobs,
+                'consumers/consumer/_job',
+                local: :job
+              )
+            %>
+          </tbody>
+        </table>
+      </div>
+    </div>
+  </div>
+<% end %>

data/lib/karafka/web/ui/pro/views/consumers/{jobs.erb → running_jobs.erb}

@@ -8,8 +8,8 @@
 
 <%== partial 'consumers/consumer/tabs' %>
 
-<% if @process.jobs.empty? %>
-  <%== partial 'consumers/consumer/no_jobs' %>
+<% if @running_jobs.empty? %>
+  <%== partial 'consumers/consumer/no_jobs', locals: { type: 'running' } %>
 <% else %>
   <div class="container">
     <div class="row mb-5">
@@ -17,19 +17,20 @@
         <table class="processes bg-white table table-hover table-bordered table-striped mb-0 align-middle">
           <thead>
             <tr class="align-middle">
-              <th>Topic</th>
-              <th>Consumer</th>
-              <th>Type</th>
-              <th>First offset</th>
-              <th>Last offset</th>
-              <th>Committed offset</th>
-              <th>Started at</th>
+              <th><%== sort_link(:topic) %></th>
+              <th><%== sort_link(:consumer) %></th>
+              <th><%== sort_link(:type) %></th>
+              <th><%== sort_link(:messages) %></th>
+              <th><%== sort_link(:first_offset) %></th>
+              <th><%== sort_link(:last_offset) %></th>
+              <th><%== sort_link(:committed_offset) %></th>
+              <th><%== sort_link('Started at', :updated_at, rev: true) %></th>
             </tr>
           </thead>
           <tbody>
             <%==
               render_each(
-                @process.jobs,
+                @running_jobs,
                 'consumers/consumer/_job',
                 local: :job
               )

data/lib/karafka/web/ui/pro/views/dashboard/index.erb

@@ -56,6 +56,7 @@
           <%== partial 'shared/tab_nav', locals: { title: 'Utilization', id: 'utilization', active: true } %>
           <%== partial 'shared/tab_nav', locals: { title: 'RSS', id: 'rss' } %>
           <%== partial 'shared/tab_nav', locals: { title: 'Concurrency', id: 'concurrency' } %>
+          <%== partial 'shared/tab_nav', locals: { title: 'Data transfers', id: 'data-transfers' } %>
         </ul>
 
         <div class="tab-content">
@@ -70,9 +71,14 @@
           </div>
 
           <div class="tab-pane show" id="concurrency" role="tabpanel">
-            <% data = @aggregated_charts.with(:processes, :workers, :listeners) %>
+            <% data = @aggregated_charts.with(:processes, :workers, :active_listeners, :standby_listeners) %>
             <%== partial 'shared/chart', locals: { data: data, id: 'concurrency' } %>
           </div>
+
+          <div class="tab-pane show" id="data-transfers" role="tabpanel">
+            <% data = @aggregated_charts.data_transfers %>
+            <%== partial 'shared/chart', locals: { data: data, id: 'data-transfers', label_type_y: 'memory' } %>
+          </div>
         </div>
       </div>
     </div>

data/lib/karafka/web/ui/pro/views/explorer/message/_message_actions.erb

@@ -0,0 +1,18 @@
+<%
+  republish_path = root_path('messages', @message.topic, @message.partition, @message.offset, 'republish')
+  surrounding_path = explorer_path(@message.topic, @message.partition, @message.offset, 'surrounding')
+%>
+
+<div class="row mb-0">
+  <div class="col-sm-12 text-end">
+    <a href="<%= surrounding_path %>" class="btn btn-secondary btn-sm float-end ms-1">
+      &#8651;
+      Surrounding
+    </a>
+
+    <form action="<%= republish_path %>" method="post" class="confirm-action float-end">
+      <%== csrf_tag(republish_path) %>
+      <input type="submit" value="&#10227; Republish" class="btn btn-primary btn-sm"/>
+    </form>
+  </div>
+</div>

data/lib/karafka/web/ui/pro/views/explorer/message/_metadata.erb

@@ -0,0 +1,43 @@
+<div class="row mb-5">
+  <div class="col-sm-12">
+    <table class="processes bg-white table table-hover table-bordered table-striped mb-0 align-middle">
+      <tbody>
+        <% @message.metadata.to_h.except(:received_at, :key, :headers).each do |k, v| %>
+          <%==
+            partial(
+              'explorer/messages/detail',
+              locals: {
+                k: k,
+                v: v
+              }
+            )
+          %>
+        <% end %>
+
+        <%==
+          partial(
+            'explorer/messages/detail',
+            locals: {
+              k: 'bytesize',
+              v: format_memory(((@message.raw_payload&.bytesize || 0) / 1024.to_f).round(4))
+            }
+          )
+        %>
+
+        <%==
+          partial(
+            'explorer/messages/key',
+            locals: { message: @message }
+          )
+        %>
+
+        <%==
+          partial(
+            'explorer/messages/headers',
+            locals: { message: @message }
+          )
+        %>
+      </tbody>
+    </table>
+  </div>
+</div>