karafka-web 0.1.2 → 0.2.0

data/lib/karafka/web/ui/models/status.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Models
+        # Model that represents the general status of the Web UI.
+        # We use this data to display a status page that helps with debugging on what is missing
+        # in the overall setup of the Web UI.
+        #
+        # People have various problems like too many partitions, not created topics, etc. and this
+        # data and view aims to help them with understanding the current status of the setup
+        class Status
+          # Status of a single step of setup
+          Step = Struct.new(:status, :details) do
+            # @return [Boolean] is the given step successfully configured and working
+            def success?
+              status == :success
+            end
+
+            # @return [String] stringified status
+            def to_s
+              status.to_s
+            end
+          end
+
+          # Initializes the status object and tries to connect to Kafka
+          def initialize
+            connect
+          end
+
+          # @return [Status::Step] were we able to connect to Kafka or not
+          def connection
+            Step.new(
+              @connected ? :success : :failure,
+              nil
+            )
+          end
+
+          # @return [Status::Step] do all the needed topics exist
+          def topics
+            if connection.success?
+              details = topics_details
+              status = details.all? { |_, detail| detail[:present] } ? :success : :failure
+            else
+              status = :halted
+              details = {}
+            end
+
+            Step.new(
+              status,
+              details
+            )
+          end
+
+          # @return [Status::Step] do we have all topics with expected number of partitions
+          def partitions
+            if topics.success?
+              status = :success
+              status = :failure if topics_details[topics_consumers_states][:partitions] != 1
+              status = :failure if topics_details[topics_consumers_reports][:partitions] != 1
+              details = topics_details
+            else
+              status = :halted
+              details = {}
+            end
+
+            Step.new(
+              status,
+              details
+            )
+          end
+
+          # @return [Status::Step] Is the initial state present in the setup or not
+          def initial_state
+            if partitions.success?
+              @current_state ||= Models::State.current
+              status = @current_state ? :success : :failure
+            else
+              status = :halted
+            end
+
+            Step.new(
+              status,
+              nil
+            )
+          end
+
+          # @return [Status::Step] Is there at least one active karafka server reporting to the
+          #   Web UI
+          def live_reporting
+            if initial_state.success?
+              @processes ||= Models::Processes.active(@current_state)
+              status = @processes.empty? ? :failure : :success
+            else
+              status = :halted
+            end
+
+            Step.new(
+              status,
+              nil
+            )
+          end
+
+          # @return [Status::Step] is there a subscription to our reports topic that is being
+          #   consumed actively.
+          def state_calculation
+            if live_reporting.success?
+              @subscriptions ||= Models::Health.current(@current_state).values.flat_map(&:keys)
+              status = @subscriptions.include?(topics_consumers_reports) ? :success : :failure
+            else
+              status = :halted
+            end
+
+            Step.new(
+              status,
+              nil
+            )
+          end
+
+          private
+
+          # @return [String] consumers states topic name
+          def topics_consumers_states
+            ::Karafka::Web.config.topics.consumers.states.to_s
+          end
+
+          # @return [String] consumers reports topic name
+          def topics_consumers_reports
+            ::Karafka::Web.config.topics.consumers.reports.to_s
+          end
+
+          # @return [String] errors topic name
+          def topics_errors
+            ::Karafka::Web.config.topics.errors
+          end
+
+          # @return [Hash] hash with topics with which we work details (even if don't exist)
+          def topics_details
+            topics = {
+              topics_consumers_states => { present: false, partitions: 0 },
+              topics_consumers_reports => { present: false, partitions: 0 },
+              topics_errors => { present: false, partitions: 0 }
+            }
+
+            @cluster_info.topics.each do |topic|
+              name = topic[:topic_name]
+
+              next unless topics.key?(name)
+
+              topics[name][:present] = true
+              topics[name][:partitions] = topic[:partition_count]
+            end
+
+            topics
+          end
+
+          # Tries connecting with the cluster and sets the connection state
+          def connect
+            @cluster_info = ::Karafka::Admin.cluster_info
+            @connected = true
+          rescue ::Rdkafka::RdkafkaError
+            @connected = false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/pro/app.rb

@@ -112,6 +112,11 @@ module Karafka
               controller = Controllers::Dlq.new(params)
               render_response controller.index
             end
+
+            r.get 'status' do
+              controller = Controllers::Status.new(params)
+              render_response controller.show
+            end
           end
         end
       end

data/lib/karafka/web/ui/pro/controllers/status.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Ui
+      module Pro
+        module Controllers
+          # Status details - same as in OSS
+          class Status < Ui::Controllers::Status
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/pro/views/consumers/_consumer.erb

@@ -8,16 +8,24 @@
       <%= process.name %>
     </a>
 
-    <br/>
+    <p class="mt-0 mb-1">
+      <% process.consumer_groups.each do |consumer_group| %>
+        <% consumer_group.topics.each do |topic| %>
+          <span class="badge bg-secondary badge-topic" title="Consumer group: <%= consumer_group.id %>">
+            <%= topic.name %>:
+            <%= topic.partitions.map(&:id).join(',') %>
+          </span>
+        <% end %>
+      <% end %>
+    </p>
 
-    <% process.consumer_groups.each do |consumer_group| %>
-      <% consumer_group.topics.each do |topic| %>
-        <span class="badge bg-secondary badge-topic" title="Consumer group: <%= consumer_group.id %>">
-          <%= topic.name %>:
-          <%= topic.partitions.map(&:id).join(',') %>
+    <p class="mt-0 mb-1">
+      <% process.tags.each do |tag| %>
+        <span class="badge bg-info badge-topic">
+          <%= tag %>
         </span>
       <% end %>
-    <% end %>
+    </p>
   </td>
 
   <td>

data/lib/karafka/web/ui/pro/views/consumers/consumer/_job.erb

@@ -1,6 +1,10 @@
 <tr>
   <td>
     <code><%= job.consumer %></code>
+
+    <% job.tags.each do |tag| %>
+      <span class="badge bg-info"><%= tag %></span>
+    <% end %>
   </td>
   <td>
     <span class="badge bg-secondary badge-topic" title="Consumer group: <%= job.consumer_group %>">

data/lib/karafka/web/ui/pro/views/consumers/consumer/_metrics.erb

@@ -27,6 +27,16 @@
               </li>
             </ul>
             </p>
+
+            <% unless @process.tags.empty? %>
+              <p class="mb-0 text-end">
+                <% @process.tags.each do |tag| %>
+                  <span class="badge bg-info badge-topic">
+                    <%= tag %>
+                  </span>
+                <% end %>
+              </p>
+            <% end %>
           </div>
         </div>
         <div class="card">

data/lib/karafka/web/ui/pro/views/jobs/_job.erb

@@ -12,6 +12,10 @@
   </td>
   <td>
     <code><%= job.consumer %></code>
+
+    <% job.tags.each do |tag| %>
+      <span class="badge bg-info"><%= tag %></span>
+    <% end %>
   </td>
   <td>
     <code>#<%= job.type %></code>

data/lib/karafka/web/ui/pro/views/shared/_navigation.erb

@@ -45,6 +45,11 @@
         Cluster
       </a>
     </li>
+    <li class="nav-item ms-3">
+      <a class="nav-link <%= nav_class(start_with: '/status') %>" href="<%= root_path('Status') %>">
+        Status
+      </a>
+    </li>
   </ul>
 </div>
 

data/lib/karafka/web/ui/views/consumers/_consumer.erb

@@ -8,16 +8,24 @@
       <%= process.name %>
     </a>
 
-    <br/>
+    <p class="mt-0 mb-1">
+      <% process.consumer_groups.each do |consumer_group| %>
+        <% consumer_group.topics.each do |topic| %>
+          <span class="badge bg-secondary badge-topic" title="Consumer group: <%= consumer_group.id %>">
+            <%= topic.name %>:
+            <%= topic.partitions.map(&:id).join(',') %>
+          </span>
+        <% end %>
+      <% end %>
+    </p>
 
-    <% process.consumer_groups.each do |consumer_group| %>
-      <% consumer_group.topics.each do |topic| %>
-        <span class="badge bg-secondary badge-topic" title="Consumer group: <%= consumer_group.id %>">
-          <%= topic.name %>:
-          <%= topic.partitions.map(&:id).join(',') %>
+    <p class="mt-0 mb-1">
+      <% process.tags.each do |tag| %>
+        <span class="badge bg-info badge-topic">
+          <%= tag %>
         </span>
       <% end %>
-    <% end %>
+    </p>
   </td>
 
   <td>

data/lib/karafka/web/ui/views/jobs/_job.erb

@@ -12,6 +12,10 @@
   </td>
   <td>
     <code><%= job.consumer %></code>
+
+    <% job.tags.each do |tag| %>
+      <span class="badge bg-info"><%= tag %></span>
+    <% end %>
   </td>
   <td>
     <code>#<%= job.type %></code>

data/lib/karafka/web/ui/views/shared/_navigation.erb

@@ -45,6 +45,11 @@
         Cluster
       </a>
     </li>
+    <li class="nav-item ms-3">
+      <a class="nav-link <%= nav_class(start_with: '/status') %>" href="<%= root_path('status') %>">
+        Status
+      </a>
+    </li>
   </ul>
 </div>
 

data/lib/karafka/web/ui/views/shared/exceptions/not_found.erb

@@ -20,6 +20,7 @@
         </p>
 
         <ul class="mb-5 text-start">
+          <li>You have visited the <a href="<%= root_path('status') %>">Status</a> page to troubleshoot any potential issues</li>
           <li>All the topics required by Karafka Web exist</li>
           <li>You have used <code>bundle exec karafka-web install</code> to initialize the Web UI</li>
           <li>You have a working connection with your Kafka cluster</li>
@@ -32,6 +33,7 @@
 
         <p>
           <a href="<%= root_path %>" class="btn btn-primary">Go Home</a>
+          <a href="<%= root_path('status') %>" class="btn btn-success">Status page</a>
         </p>
       </div>
     </div>

data/lib/karafka/web/ui/views/status/_breadcrumbs.erb

@@ -0,0 +1,5 @@
+<li class="breadcrumb-item">
+  <a href="<%= root_path('status') %>">
+    Web UI status details
+  </a>
+</li>

data/lib/karafka/web/ui/views/status/_failure.erb

@@ -0,0 +1,14 @@
+<div class="card border-danger mb-3">
+  <div class="card-header text-bg-danger">
+    <span>
+      <%= title %>
+    </span>
+
+    <span class="float-end">
+      <span class="badge  text-bg-light">Failure</span>
+    </span>
+  </div>
+  <div class="card-body">
+    <%== description %>
+  </div>
+</div>

data/lib/karafka/web/ui/views/status/_halted.erb

@@ -0,0 +1,11 @@
+<div class="card text-bg-secondary mb-3">
+  <div class="card-header">
+    <span>
+      <%= title %>
+    </span>
+
+    <span class="float-end">
+      <span class="badge  text-bg-light">Halted</span>
+    </span>
+  </div>
+</div>

data/lib/karafka/web/ui/views/status/_success.erb

@@ -0,0 +1,11 @@
+<div class="card text-bg-success mb-3">
+  <div class="card-header">
+    <span>
+      <%= title %>
+    </span>
+
+    <span class="float-end">
+      <span class="badge  text-bg-light">Successful</span>
+    </span>
+  </div>
+</div>

data/lib/karafka/web/ui/views/status/failures/_connection.erb

@@ -0,0 +1,7 @@
+<p>
+  Web UI was not able to establish a connection with the Kafka cluster.
+</p>
+
+<p class="mb-0">
+  Please make sure that Web UI can reach Kafka.
+</p>

data/lib/karafka/web/ui/views/status/failures/_initial_state.erb

@@ -0,0 +1,20 @@
+<p>
+  The initial state for the Web UI was not created.
+</p>
+
+<p>
+  It means that the <code>bundle exec karafka-web install</code> was not executed or failed.
+</p>
+
+<p>
+  To fix this, you can either:
+</p>
+
+<ul>
+  <li>Run <code>bundle exec karafka-web install</code></li>
+  <li>Run at least one Karafka consumer process</li>
+</ul>
+
+<p class="mb-0">
+  Any of the above, when successful, will bootstrap the initial state.
+</p>

data/lib/karafka/web/ui/views/status/failures/_live_reporting.erb

@@ -0,0 +1,7 @@
+<p>
+  There are no Karafka consumer processes actively reporting to the Web UI.
+</p>
+
+<p class="mb-0">
+  If you are sure you are running at least one <code>karafka server</code> instance, please make sure that it can report to the <code><%= Karafka::Web.config.topics.consumers.reports %></code> topic.
+</p>

data/lib/karafka/web/ui/views/status/failures/_partitions.erb

@@ -0,0 +1,19 @@
+<p>
+  Both
+  <code><%= Karafka::Web.config.topics.consumers.states %></code>
+  and
+  <code><%= Karafka::Web.config.topics.consumers.reports %></code>
+  topics need to be configured with <strong>exactly</strong> one partition.
+</p>
+
+<p>
+  Your current setup contains the following:
+</p>
+
+<ul class="mb-0">
+  <% details.each do |name, details| %>
+    <li>
+      <code><%= name %> </code> with <code><%= details[:partitions] %> </code> partitions.
+    </li>
+  <% end %>
+</ul>

data/lib/karafka/web/ui/views/status/failures/_state_calculation.erb

@@ -0,0 +1,8 @@
+<p>
+  None of the <code>karafka server</code> processes is subscribed to handle the
+  <code><%= Karafka::Web.config.topics.consumers.reports %></code> topic data aggregations.
+</p>
+
+<p class="mb-0">
+  If you are limiting the topics you consume using the <a href="https://karafka.io/docs/CLI/#limiting-consumer-groups-used-per-process" target="_blank">limiting API</a>, please include the Karafka Web consumer group and the <code><%= Karafka::Web.config.topics.consumers.reports %></code> topic.
+</p>

data/lib/karafka/web/ui/views/status/failures/_topics.erb

@@ -0,0 +1,20 @@
+<p>
+  Web UI requires the following topics to exist:
+</p>
+
+<ul>
+  <% details.each do |name, details| %>
+    <li>
+      <code><%= name %></code> - <%= details[:present] ? 'Exists' : 'Missing' %>
+    </li>
+  <% end %>
+</ul>
+
+<p>
+  Please ensure all those topics exist and are accessible to your Karafka user.
+</p>
+
+<p class="mb-0">
+  You can read more about their setup and configuration
+  <a href="https://karafka.io/docs/Web-UI-Getting-Started/" target="_blank">here</a>.
+</p>

data/lib/karafka/web/ui/views/status/show.erb

@@ -0,0 +1,94 @@
+<%== view_title('Web UI status details') %>
+
+<div class="container mb-5">
+  <div class="row">
+    <div class="col-lg-10 offset-md-1">
+
+      <%==
+        partial(
+          "status/#{@status.connection.to_s}",
+          locals: {
+            title: 'Connection to Kafka',
+            description: partial('status/failures/connection')
+          }
+        )
+      %>
+
+      <%==
+        partial(
+          "status/#{@status.topics.to_s}",
+          locals: {
+            title: 'Topics presence',
+            description: partial(
+              'status/failures/topics',
+              locals: {
+                details: @status.topics.details
+              }
+            )
+          }
+        )
+      %>
+
+      <%==
+        partial(
+          "status/#{@status.partitions.to_s}",
+          locals: {
+            title: 'Partitions count',
+            description: partial(
+              'status/failures/partitions',
+              locals: {
+                details: @status.partitions.details
+              }
+            )
+          }
+        )
+      %>
+
+      <%==
+        partial(
+          "status/#{@status.initial_state.to_s}",
+          locals: {
+            title: 'Initial state presence',
+            description: partial(
+              'status/failures/initial_state',
+              locals: {
+                details: @status.initial_state.details
+              }
+            )
+          }
+        )
+      %>
+
+      <%==
+        partial(
+          "status/#{@status.live_reporting.to_s}",
+          locals: {
+            title: 'Live reporting',
+            description: partial(
+              'status/failures/live_reporting',
+              locals: {
+                details: @status.live_reporting.details
+              }
+            )
+          }
+        )
+      %>
+
+      <%==
+        partial(
+          "status/#{@status.state_calculation.to_s}",
+          locals: {
+            title: 'State calculation subscription',
+            description: partial(
+              'status/failures/state_calculation',
+              locals: {
+                details: @status.state_calculation.details
+              }
+            )
+          }
+        )
+      %>
+
+    </div>
+  </div>
+</div>

data/lib/karafka/web/version.rb

@@ -3,6 +3,6 @@
 module Karafka
   module Web
     # Current gem version
-    VERSION = '0.1.2'
+    VERSION = '0.2.0'
   end
 end

data/lib/karafka/web.rb

@@ -27,12 +27,6 @@ module Karafka
         Config.config
       end
 
-      # Creates all the needed topics for the admin UI to work and populates initial (empty) set
-      # of data, so the UI will work even when no Karafka servers are started
-      def bootstrap!
-        Installer.new.bootstrap!
-      end
-
       # Activates all the needed routing and sets up listener, etc
       # This needs to run **after** the optional configuration of the web component
       def enable!