karafka-web 0.5.2 → 0.6.0

data/lib/karafka/web/ui/views/errors/_error.erb

@@ -9,14 +9,16 @@
 
   <tr>
     <td>
-      <% if error[:details].key?(:topic) %>
-        <%= error[:details][:topic] %>: <%= error[:details][:partition] %>
-      <% else %>
-        <%= error[:type] %>
-      <% end %>
+      <span class="badge bg-secondary badge-topic">
+        <% if error[:details].key?(:topic) %>
+          <%= error[:details][:topic] %>: <%= error[:details][:partition] %>
+        <% else %>
+          <%= error[:type] %>
+        <% end %>
+      </span>
     </td>
     <td>
-      <%== error[:process_name] %>
+      <%== error[:process][:name] %>
     </td>
     <td>
       <%= error[:error_class] %>:

data/lib/karafka/web/ui/views/errors/show.erb

@@ -1,12 +1,20 @@
 <%==
   type = @error_message.payload[:type]
   error_class = @error_message.payload[:error_class]
-  offset = @error_message.offset
 
-  view_title("#{type}: #{error_class} #{offset}", hr: true)
+  view_title("#{type}: #{error_class}")
 %>
 
 <div class="container">
+  <div class="row mb-4">
+    <div class="col-sm-12">
+      <h5 class="mb-2">
+        Metadata
+      </h5>
+      <hr/>
+    </div>
+  </div>
+
   <div class="row mb-5">
     <div class="col-sm-12 table-responsive">
       <table class="processes bg-white table table-hover table-bordered table-striped mb-0 align-middle">
@@ -27,4 +35,44 @@
       </table>
     </div>
   </div>
+
+  <div class="row mb-2">
+    <div class="col-sm-12">
+      <h5 class="mb-2">
+        Backtrace
+      </h5>
+      <hr/>
+    </div>
+  </div>
+
+  <div class="mb-4">
+    <%== partial 'shared/feature_pro' %>
+  </div>
+
+  <div class="row mb-4">
+    <div class="col-sm-12">
+      <div class="card">
+        <div class="card-body">
+          <pre class="m-0 p-0 blurred"><code class="wrapped json p-0 m-0">this is just an example backtrace
+please subscribe to our Pro offering to be able to view the real one
+gems/karafka-rdkafka/lib/rdkafka/consumer.rb:255:in `query_watermark_offsets'
+gems/karafka/lib/karafka/admin.rb:56:in `block in read_topic'
+gems/karafka/lib/karafka/admin.rb:184:in `with_consumer'
+gems/karafka/lib/karafka/admin.rb:55:in `read_topic'
+/mnt/software/Karafka/karafka-web/lib/karafka/web/processing/consumers/state.rb:19:in `current'
+/mnt/software/Karafka/karafka-web/lib/karafka/web/processing/consumers/aggregator.rb:45:in `state'
+/mnt/software/Karafka/karafka-web/lib/karafka/web/processing/consumers/aggregator.rb:38:in `to_json'
+gems/karafka/lib/karafka/processing/strategies/default.rb:136:in `block in handle_shutdown'
+gems/karafka-core/lib/karafka/core/monitoring/notifications.rb:118:in `measure_time_taken'
+gems/karafka-core/lib/karafka/core/monitoring/notifications.rb:94:in `instrument'
+gems/karafka-core/lib/karafka/core/monitoring/monitor.rb:34:in `instrument'
+gems/karafka/lib/karafka/processing/strategies/default.rb:135:in `handle_shutdown'
+gems/karafka/lib/karafka/base_consumer.rb:134:in `on_shutdown'
+gems/karafka/lib/karafka/processing/executor.rb:123:in `shutdown'
+gems/karafka/lib/karafka/processing/jobs/shutdown.rb:18:in `call'
+gems/karafka/lib/karafka/helpers/async.rb:28:in `block in async_call'</code></pre>
+        </div>
+      </div>
+    </div>
+  </div>
 </div>

data/lib/karafka/web/ui/views/shared/_feature_pro.erb

@@ -0,0 +1,4 @@
+<div class="alert alert-light">
+  This Web UI feature is available only to <a target="_blank" href="https://karafka.io/#become-pro">Pro</a>
+  users.
+</div>

data/lib/karafka/web/ui/views/shared/_pagination.erb

@@ -3,9 +3,15 @@
     <div class="row">
       <nav>
         <ul class="pagination justify-content-center">
+          <li class="page-item <%= 'disabled' if @current_page <= 1 %>">
+            <a class="page-link" href="<%= current_path(page: 1) %>">
+              <span>&laquo;</span>
+            </a>
+          </li>
+
           <li class="page-item <%= 'disabled' if @current_page <= 1 %>">
             <a class="page-link" href="<%= current_path(page: @current_page - 1) %>">
-              <span>&laquo; Prev</span>
+              <span>&lsaquo; Prev</span>
             </a>
           </li>
 
@@ -17,7 +23,7 @@
 
           <li class="page-item <%= 'disabled' unless @next_page %>">
             <a class="page-link" href="<%= current_path(page: @next_page) %>">
-              <span>Next &raquo;</span>
+              <span>Next &rsaquo;</span>
             </a>
           </li>
         </ul>

data/lib/karafka/web/ui/views/shared/exceptions/pro_only.erb

@@ -37,10 +37,6 @@
           <li>Architecture consultations</li>
         </ul>
 
-        <p class="mb-5">
-          If you were looking for a given process or other real-time information, the state might have changed, and the information you are looking for no longer exists.
-        </p>
-
         <p>
           <a href="<%= root_path %>" class="btn btn-primary">Go Home</a>
           <a href="https://karafka.io/#become-pro" class="btn btn-success" target="_blank">Become Pro!</a>

data/lib/karafka/web/version.rb

@@ -3,6 +3,6 @@
 module Karafka
   module Web
     # Current gem version
-    VERSION = '0.5.2'
+    VERSION = '0.6.0'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-web
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 0.6.0
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2023-05-22 00:00:00.000000000 Z
+date: 2023-06-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: erubi
@@ -57,7 +57,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.40
+        version: 2.1.4
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -67,7 +67,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.40
+        version: 2.1.4
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -77,7 +77,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.12
+        version: 2.0.13
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -87,7 +87,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.12
+        version: 2.0.13
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -97,14 +97,20 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.63'
+        version: '3.68'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.68'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.63'
+        version: '3.68'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.68'
 - !ruby/object:Gem::Dependency
   name: tilt
   requirement: !ruby/object:Gem::Requirement
@@ -171,7 +177,6 @@ files:
 - lib/karafka/web/processing/consumer.rb
 - lib/karafka/web/processing/consumers/aggregator.rb
 - lib/karafka/web/processing/consumers/state.rb
-- lib/karafka/web/tracking/base_contract.rb
 - lib/karafka/web/tracking/consumers/contracts/consumer_group.rb
 - lib/karafka/web/tracking/consumers/contracts/job.rb
 - lib/karafka/web/tracking/consumers/contracts/partition.rb
@@ -185,9 +190,17 @@ files:
 - lib/karafka/web/tracking/consumers/listeners/statistics.rb
 - lib/karafka/web/tracking/consumers/listeners/status.rb
 - lib/karafka/web/tracking/consumers/listeners/tags.rb
+- lib/karafka/web/tracking/consumers/reporter.rb
 - lib/karafka/web/tracking/consumers/sampler.rb
+- lib/karafka/web/tracking/contracts/base.rb
+- lib/karafka/web/tracking/contracts/error.rb
+- lib/karafka/web/tracking/helpers/error_info.rb
 - lib/karafka/web/tracking/memoized_shell.rb
-- lib/karafka/web/tracking/reporter.rb
+- lib/karafka/web/tracking/producers/listeners/base.rb
+- lib/karafka/web/tracking/producers/listeners/errors.rb
+- lib/karafka/web/tracking/producers/listeners/reporter.rb
+- lib/karafka/web/tracking/producers/reporter.rb
+- lib/karafka/web/tracking/producers/sampler.rb
 - lib/karafka/web/tracking/sampler.rb
 - lib/karafka/web/tracking/ttl_array.rb
 - lib/karafka/web/tracking/ttl_hash.rb
@@ -207,6 +220,7 @@ files:
 - lib/karafka/web/ui/lib/hash_proxy.rb
 - lib/karafka/web/ui/lib/paginate_array.rb
 - lib/karafka/web/ui/models/consumer_group.rb
+- lib/karafka/web/ui/models/counters.rb
 - lib/karafka/web/ui/models/health.rb
 - lib/karafka/web/ui/models/job.rb
 - lib/karafka/web/ui/models/message.rb
@@ -231,7 +245,6 @@ files:
 - lib/karafka/web/ui/pro/views/consumers/_breadcrumbs.erb
 - lib/karafka/web/ui/pro/views/consumers/_consumer.erb
 - lib/karafka/web/ui/pro/views/consumers/_counters.erb
-- lib/karafka/web/ui/pro/views/consumers/_summary.erb
 - lib/karafka/web/ui/pro/views/consumers/consumer/_consumer_group.erb
 - lib/karafka/web/ui/pro/views/consumers/consumer/_job.erb
 - lib/karafka/web/ui/pro/views/consumers/consumer/_metrics.erb
@@ -249,10 +262,7 @@ files:
 - lib/karafka/web/ui/pro/views/dlq/_topic.erb
 - lib/karafka/web/ui/pro/views/dlq/index.erb
 - lib/karafka/web/ui/pro/views/errors/_breadcrumbs.erb
-- lib/karafka/web/ui/pro/views/errors/_cleaned.erb
-- lib/karafka/web/ui/pro/views/errors/_detail.erb
 - lib/karafka/web/ui/pro/views/errors/_error.erb
-- lib/karafka/web/ui/pro/views/errors/_no_errors.erb
 - lib/karafka/web/ui/pro/views/errors/_partition_option.erb
 - lib/karafka/web/ui/pro/views/errors/index.erb
 - lib/karafka/web/ui/pro/views/errors/show.erb
@@ -273,7 +283,6 @@ files:
 - lib/karafka/web/ui/pro/views/health/_breadcrumbs.erb
 - lib/karafka/web/ui/pro/views/health/_partition.erb
 - lib/karafka/web/ui/pro/views/health/index.erb
-- lib/karafka/web/ui/pro/views/jobs/_breadcrumbs.erb
 - lib/karafka/web/ui/pro/views/jobs/_job.erb
 - lib/karafka/web/ui/pro/views/jobs/_no_jobs.erb
 - lib/karafka/web/ui/pro/views/jobs/index.erb
@@ -293,9 +302,9 @@ files:
 - lib/karafka/web/ui/views/cluster/_broker.erb
 - lib/karafka/web/ui/views/cluster/_partition.erb
 - lib/karafka/web/ui/views/cluster/index.erb
-- lib/karafka/web/ui/views/consumers/_breadcrumbs.erb
 - lib/karafka/web/ui/views/consumers/_consumer.erb
 - lib/karafka/web/ui/views/consumers/_counters.erb
+- lib/karafka/web/ui/views/consumers/_no_consumers.erb
 - lib/karafka/web/ui/views/consumers/_summary.erb
 - lib/karafka/web/ui/views/consumers/index.erb
 - lib/karafka/web/ui/views/errors/_breadcrumbs.erb
@@ -320,6 +329,7 @@ files:
 - lib/karafka/web/ui/views/shared/_become_pro.erb
 - lib/karafka/web/ui/views/shared/_brand.erb
 - lib/karafka/web/ui/views/shared/_content.erb
+- lib/karafka/web/ui/views/shared/_feature_pro.erb
 - lib/karafka/web/ui/views/shared/_header.erb
 - lib/karafka/web/ui/views/shared/_navigation.erb
 - lib/karafka/web/ui/views/shared/_pagination.erb

data/lib/karafka/web/tracking/base_contract.rb

@@ -1,31 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Web
-    module Tracking
-      # Base for all the metric related contracts
-      class BaseContract < ::Karafka::Core::Contractable::Contract
-        class << self
-          # This layer is not for users extensive feedback, thus we can easily use the minimum
-          # error messaging there is.
-          def configure
-            super do |config|
-              config.error_messages = YAML.safe_load(
-                File.read(
-                  File.join(Karafka::Web.gem_root, 'config', 'locales', 'errors.yml')
-                )
-              ).fetch('en').fetch('validations').fetch('web')
-            end
-          end
-        end
-
-        # @param data [Hash] data for validation
-        # @return [Boolean] true if all good
-        # @raise [Errors::ContractError] invalid report
-        def validate!(data)
-          super(data, Errors::Tracking::ContractError)
-        end
-      end
-    end
-  end
-end

data/lib/karafka/web/tracking/reporter.rb

@@ -1,144 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Web
-    module Tracking
-      # Reports the collected data about the process and sends it, so we can use it in the UI
-      class Reporter
-        include ::Karafka::Core::Helpers::Time
-        include ::Karafka::Helpers::Async
-
-        # Minimum number of messages to produce to produce then in sync mode
-        # This acts as a small back-off not to overload the system in case we would have extremely
-        # big number of errors happening
-        PRODUCE_SYNC_THRESHOLD = 25
-
-        private_constant :PRODUCE_SYNC_THRESHOLD
-
-        # This mutex is shared between tracker and samplers so there is no case where metrics
-        # would be collected same time tracker reports
-        MUTEX = Mutex.new
-
-        def initialize
-          # Move back so first report is dispatched fast to indicate, that the process is alive
-          @tracked_at = monotonic_now - 10_000
-          @consumer_contract = Consumers::Contracts::Report.new
-        end
-
-        # Dispatches the current state from sampler to appropriate topics
-        #
-        # @param forced [Boolean] should we report bypassing the time frequency or should we report
-        #   only in case we would not send the report for long enough time.
-        def report(forced: false)
-          MUTEX.synchronize do
-            # Start background thread only when needed
-            # This prevents us from starting it too early or for non-consumer processes where
-            # Karafka is being included
-            async_call unless @running
-
-            return unless report?(forced)
-
-            @tracked_at = monotonic_now
-
-            consumer_report = consumer_sampler.to_report
-
-            @consumer_contract.validate!(consumer_report)
-
-            process_name = consumer_report[:process][:name]
-
-            # Report consumers statuses
-            messages = [
-              {
-                topic: ::Karafka::Web.config.topics.consumers.reports,
-                payload: consumer_report.to_json,
-                key: process_name,
-                partition: 0
-              }
-            ]
-
-            # Report errors that occurred (if any)
-            messages += consumer_sampler.errors.map do |error|
-              {
-                topic: Karafka::Web.config.topics.errors,
-                payload: error.to_json,
-                # Always dispatch errors from the same process to the same partition
-                key: process_name
-              }
-            end
-
-            produce(messages)
-
-            # Clear the sampler so it tracks new state changes without previous once impacting
-            # the data
-            consumer_sampler.clear
-          end
-        # Since we run this in a background thread, there may be a case upon shutdown, where the
-        # producer is closed right before a potential dispatch. It is not worth dealing with this
-        # and we can just safely ignore this
-        rescue WaterDrop::Errors::ProducerClosedError
-          nil
-        end
-
-        # Reports bypassing frequency check. This can be used to report when state changes in the
-        # process drastically. For example when process is stopping, we want to indicate this as
-        # fast as possible in the UI, etc.
-        def report!
-          report(forced: true)
-        end
-
-        private
-
-        # Reports the process state once in a while
-        def call
-          @running = true
-
-          loop do
-            report
-
-            # We won't track more often anyhow but want to try frequently not to miss a window
-            # We need to convert the sleep interval into seconds for sleep
-            sleep(::Karafka::Web.config.tracking.interval / 1_000 / 10)
-          end
-        end
-
-        # @param forced [Boolean] is this report forced. Forced means that as long as we can flush
-        #   we will flush
-        # @return [Boolean] Should we report or is it not yet time to do so
-        def report?(forced)
-          # We never report in initializing phase because things are not yet fully configured
-          return false if ::Karafka::App.initializing?
-          # We never report in the initialized because server is not yet ready until Karafka is
-          # fully running and some of the things like listeners are not yet available
-          return false if ::Karafka::App.initialized?
-
-          return true if forced
-
-          (monotonic_now - @tracked_at) >= ::Karafka::Web.config.tracking.interval
-        end
-
-        # @return [Object] sampler for the metrics
-        def consumer_sampler
-          @consumer_sampler ||= ::Karafka::Web.config.tracking.consumers.sampler
-        end
-
-        # Produces messages to Kafka.
-        #
-        # @param messages [Array<Hash>]
-        #
-        # @note We pick either sync or async dependent on number of messages. The trick here is,
-        #   that we do not want to end up overloading the internal queue with messages in case
-        #   someone has a lot of errors from processing or other errors. Producing sync will wait
-        #   for the delivery, hence will slow things down a little bit. On the other hand during
-        #   normal operations we should not have that many messages to dispatch and it should not
-        #   slowdown any processing.
-        def produce(messages)
-          if messages.count >= PRODUCE_SYNC_THRESHOLD
-            ::Karafka.producer.produce_many_sync(messages)
-          else
-            ::Karafka.producer.produce_many_async(messages)
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/web/ui/pro/views/consumers/_summary.erb

@@ -1,81 +0,0 @@
-<div id="summary" class="container text-center mb-5">
-  <div class="row">
-
-    <div class="col-lg-2 offset-md-1">
-      <div class="card mb-0">
-        <div class="card-body">
-          <p class="card-text">
-            <div class="stat mb-1">
-              <%= @counters.processes %>
-            </div>
-            <div class="desc">
-              Processes
-            </div>
-          </p>
-        </div>
-      </div>
-    </div>
-
-    <div class="col-lg-2">
-      <div class="card mb-0">
-        <div class="card-body">
-          <p class="card-text">
-            <div class="stat mb-1">
-              <%= @counters.threads_count %>
-            </div>
-            <div class="desc">
-              Worker threads
-            </div>
-          </p>
-        </div>
-      </div>
-    </div>
-
-    <div class="col-lg-2">
-      <div class="card mb-0">
-        <div class="card-body">
-          <p class="card-text">
-            <div class="stat mb-1">
-              <%= @counters.listeners_count %>
-            </div>
-            <div class="desc">
-              Listeners
-            </div>
-          </p>
-        </div>
-      </div>
-    </div>
-
-    <div class="col-lg-2">
-      <div class="card mb-0">
-        <div class="card-body">
-          <p class="card-text">
-            <div class="stat mb-1">
-              <%= @counters.utilization.round(2) %>
-              %
-            </div>
-            <div class="desc">
-              Utilization
-            </div>
-          </p>
-        </div>
-      </div>
-    </div>
-
-    <div class="col-lg-2">
-      <div class="card mb-0">
-        <div class="card-body">
-          <p class="card-text">
-            <div class="stat mb-1">
-              <%= format_memory @counters.rss %>
-            </div>
-            <div class="desc">
-              RSS
-            </div>
-          </p>
-        </div>
-      </div>
-    </div>
-
-  </div>
-</div>

data/lib/karafka/web/ui/pro/views/errors/_cleaned.erb

@@ -1,3 +0,0 @@
-<div class="alert alert-info" role="alert">
-  This errors topic partition had all of its errored compacted and cleaned.
-</div>

data/lib/karafka/web/ui/pro/views/errors/_detail.erb

@@ -1,31 +0,0 @@
-<% if v.is_a?(Hash) %>
-  <% v.each do |k2, v2| %>
-    <tr>
-      <td>
-        <%= "#{k}.#{k2}" %>
-      </td>
-      <td>
-        <% if %w[sasl ssl].any? { |scope| k2.to_s.include?(scope) } %>
-          ***
-        <% elsif k2.to_s == 'tags' %>
-          <%== tags(v2) %>
-        <% else %>
-          <%= v2 %>
-        <% end %>
-      </td>
-    </tr>
-  <% end %>
-<% else %>
-  <tr>
-    <td>
-      <%= k %>
-    </td>
-    <td>
-      <% if k == :occurred_at %>
-        <%== relative_time v %>
-      <% else %>
-        <%= v %>
-      <% end %>
-    </td>
-  </tr>
-<% end %>

data/lib/karafka/web/ui/pro/views/errors/_no_errors.erb

@@ -1,3 +0,0 @@
-<div class="alert alert-info" role="alert">
-  There are no errors in this errors topic partition.
-</div>

data/lib/karafka/web/ui/pro/views/jobs/_breadcrumbs.erb

@@ -1,5 +0,0 @@
-<li class="breadcrumb-item">
-  <a href="<%= root_path('jobs') %>">
-    Running jobs
-  </a>
-</li>

data/lib/karafka/web/ui/views/consumers/_breadcrumbs.erb

@@ -1,27 +0,0 @@
-<% if @process %>
-  <li class="breadcrumb-item">
-    <a href="<%= root_path('consumers') %>">
-      Consumers
-    </a>
-  </li>
-
-  <li class="breadcrumb-item">
-    <a href="<%= root_path('consumers', @process.id, 'subscriptions') %>">
-      <%= @process.name %>
-    </a>
-  </li>
-
-  <% if current_path.include?('/jobs') %>
-    <li class="breadcrumb-item">
-      <a href="<%= root_path('consumers', @process.id, 'jobs') %>">
-        Running jobs
-      </a>
-    </li>
-  <% else %>
-    <li class="breadcrumb-item">
-      <a href="<%= root_path('consumers', @process.id, 'subscriptions') %>">
-        Active subscriptions
-      </a>
-    </li>
-  <% end %>
-<% end %>