karafka-web 0.2.2 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7619f1c461afdd16646dfe0d7fff92badad1448de8d40afcc41b42af4ac37dfa
-  data.tar.gz: c706ae9212b87947630be27d366d0d8a938b2307d8e2e969d7dbddad6488ab4b
+  metadata.gz: b03937826882f0a85bbd1718f2b09411148ec9647ab46d0d9a9701c278a07b66
+  data.tar.gz: 257bbf154f1d19aaaecd7150105ecd7233aa33f8bd42c4195a938c215061220b
 SHA512:
-  metadata.gz: 1ee9e33f8a8a358b45ff1e8ad707252eaecc0d3801194ce313e45c3cb0c54e6d8fadf905fac3dc809fbb276a12adc9d29a91285e0f63f6ba09d97803b1c4f198
-  data.tar.gz: 67a4d2902138aa73689b2c1ccd2f1a76c7d17993ff2584f29ab57a50b1893c01157239d37d7b125b07fce18556354bb88dfb6d87099e0d61cabb7bf5e8ddfaa6
+  metadata.gz: 1aed9e94d037cd9bdaa07e55f96281f83dc18ebe151ec4a7f936e1592ac9d530a45fbb45dbd7c5d843692ede58bbca5d1d81a63974e222b0649cd0aa6bd8a9b6
+  data.tar.gz: 4ce6deaac3737bb8d25952158ac2b022621187bb8b0d42f952261b831d664e21cf5e12ff15bbccc054927a0b9dd000f75361270adeb3cb6708df93144b848d08

data/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Karafka Web changelog
 
+## 0.2.4 (2023-03-14)
+- [Improvement] Paginate topics list in cluster info on every 100 partitions.
+- [Improvement] Provide current page in the pagination.
+- [Improvement] Report usage of Karafka Pro on the status page view.
+- [Fix] Add missing three months limit on errors storage.
+- [Maintenance] Exclude Karafka Web UI topics from declarative topics.
+
+## 0.2.3 (2023-03-04)
+- [Improvement] Snapshot current consumer tags upon consumer errors.
+- [Improvement] Optimize exception message extraction from errors.
+- [Improvement] Slightly change error reporting structure (backwards compatible) to collect process tags on errors and to align with other reports.
+
 ## 0.2.2 (2023-02-25)
 - [Fix] Fix status page reference in Pro.
 

data/Gemfile.lock

@@ -1,9 +1,9 @@
 PATH
   remote: .
   specs:
-    karafka-web (0.2.2)
+    karafka-web (0.2.4)
       erubi (~> 1.4)
-      karafka (>= 2.0.33, < 3.0.0)
+      karafka (>= 2.0.35, < 3.0.0)
       karafka-core (>= 2.0.12, < 3.0.0)
       roda (~> 3.63)
       tilt (~> 2.0)
@@ -17,7 +17,7 @@ GEM
       minitest (>= 5.1)
       tzinfo (~> 2.0)
     byebug (11.1.3)
-    concurrent-ruby (1.2.0)
+    concurrent-ruby (1.2.2)
     diff-lcs (1.5.0)
     docile (1.4.0)
     erubi (1.12.0)
@@ -26,7 +26,7 @@ GEM
     ffi (1.15.5)
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)
-    karafka (2.0.33)
+    karafka (2.0.35)
       karafka-core (>= 2.0.12, < 3.0.0)
       thor (>= 0.20)
       waterdrop (>= 2.4.10, < 3.0.0)
@@ -70,13 +70,14 @@ GEM
     tilt (2.1.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.4.10)
-      karafka-core (>= 2.0.9, < 3.0.0)
+    waterdrop (2.4.11)
+      karafka-core (>= 2.0.12, < 3.0.0)
       zeitwerk (~> 2.3)
     webrick (1.8.1)
     zeitwerk (2.6.7)
 
 PLATFORMS
+  arm64-darwin-21
   x86_64-linux
 
 DEPENDENCIES
@@ -88,4 +89,4 @@ DEPENDENCIES
   simplecov
 
 BUNDLED WITH
-   2.4.6
+   2.4.7

data/karafka-web.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |spec|
   spec.licenses    = %w[LGPL-3.0 Commercial]
 
   spec.add_dependency 'erubi', '~> 1.4'
-  spec.add_dependency 'karafka', '>= 2.0.33', '< 3.0.0'
+  spec.add_dependency 'karafka', '>= 2.0.35', '< 3.0.0'
   spec.add_dependency 'karafka-core', '>= 2.0.12', '< 3.0.0'
   spec.add_dependency 'roda', '~> 3.63'
   spec.add_dependency 'tilt', '~> 2.0'

data/lib/karafka/web/installer.rb

@@ -45,6 +45,7 @@ module Karafka
           consumer_group ::Karafka::Web.config.processing.consumer_group do
             # Topic we listen on to materialize the states
             topic ::Karafka::Web.config.topics.consumers.reports do
+              config(active: false)
               active ::Karafka::Web.config.processing.active
               # Since we materialize state in intervals, we can poll for half of this time without
               # impacting the reporting responsiveness
@@ -58,11 +59,13 @@ module Karafka
             # We define those two here without consumption, so Web understands how to deserialize
             # them when used / viewed
             topic ::Karafka::Web.config.topics.consumers.states do
+              config(active: false)
               active false
               deserializer web_deserializer
             end
 
             topic ::Karafka::Web.config.topics.errors do
+              config(active: false)
               active false
               deserializer web_deserializer
             end
@@ -126,7 +129,9 @@ module Karafka
           ::Karafka::Admin.create_topic(
             errors_topic,
             1,
-            replication_factor
+            replication_factor,
+            # Remove really old errors (older than 3 months just to preserve space)
+            { 'retention.ms': 3 * 31 * 24 * 60 * 60 * 1_000 }
           )
         end
       end

data/lib/karafka/web/tracking/consumers/listeners/errors.rb

@@ -28,7 +28,8 @@ module Karafka
                   error_message: error_message,
                   backtrace: backtrace,
                   details: details,
-                  occurred_at: float_now
+                  occurred_at: float_now,
+                  process: sampler.to_report[:process].slice(:name, :tags)
                 }
 
                 sampler.counters[:errors] += 1
@@ -55,6 +56,13 @@ module Karafka
 
             private
 
+            # @return [Object] sampler for the metrics
+            # @note We use this sampler to get basic process details that we want to assign
+            #   to the error
+            def consumer_sampler
+              @consumer_sampler ||= ::Karafka::Web.config.tracking.consumers.sampler
+            end
+
             # @param consumer [::Karafka::BaseConsumer]
             # @return [Hash] hash with consumer specific info for details of error
             def extract_consumer_info(consumer)
@@ -65,7 +73,8 @@ module Karafka
                 first_offset: consumer.messages.first.offset,
                 last_offset: consumer.messages.last.offset,
                 comitted_offset: consumer.coordinator.seek_offset - 1,
-                consumer: consumer.class.to_s
+                consumer: consumer.class.to_s,
+                tags: consumer.tags
               }
             end
 
@@ -74,12 +83,6 @@ module Karafka
             # @param error [StandardError] error that occurred
             # @return [Array<String, String, String>] array with error name, message and backtrace
             def extract_error_info(error)
-              error_message = error.message.to_s.dup
-              error_message.force_encoding('utf-8')
-              error_message.scrub!
-
-              backtrace = (error.backtrace || [])
-
               app_root = "#{::Karafka.root}/"
 
               gem_home = if ENV.key?('GEM_HOME')
@@ -90,15 +93,27 @@ module Karafka
 
               gem_home = "#{gem_home}/"
 
+              backtrace = error.backtrace || []
               backtrace.map! { |line| line.gsub(app_root, '') }
               backtrace.map! { |line| line.gsub(gem_home, '') }
 
               [
                 error.class.name,
-                error_message,
+                extract_exception_message(error),
                 backtrace.join("\n")
               ]
             end
+
+            # @param error [StandardError] error that occurred
+            # @return [String] formatted exception message
+            def extract_exception_message(error)
+              error_message = error.message.to_s[0, 10_000]
+              error_message.force_encoding('utf-8')
+              error_message.scrub! if error_message.respond_to?(:scrub!)
+              error_message
+            rescue StandardError
+              '!!! Error message extraction failed !!!'
+            end
           end
         end
       end

data/lib/karafka/web/tracking/consumers/sampler.rb

@@ -60,7 +60,7 @@ module Karafka
                 started_at: @started_at,
                 name: process_name,
                 status: ::Karafka::App.config.internal.status.to_s,
-                listeners: Karafka::Server.listeners.count,
+                listeners: listeners,
                 concurrency: concurrency,
                 memory_usage: memory_usage,
                 memory_total_usage: memory_total_usage,
@@ -76,7 +76,7 @@ module Karafka
                 waterdrop: ::WaterDrop::VERSION
               },
 
-              stats: Karafka::Server.jobs_queue.statistics.merge(
+              stats: jobs_queue_statistics.merge(
                 utilization: utilization
               ).merge(total: @counters),
 
@@ -112,6 +112,12 @@ module Karafka
             times[:total].sum / 1_000 / concurrency / timefactor * 100
           end
 
+          # @return [Integer] number of listeners
+          def listeners
+            # This can be zero before the server starts
+            Karafka::Server.listeners&.count.to_i
+          end
+
           # @return [String] Unique process name
           def process_name
             @process_name ||= "#{Socket.gethostname}:#{::Process.pid}:#{SecureRandom.hex(6)}"
@@ -142,6 +148,12 @@ module Karafka
             end
           end
 
+          # @return [Hash] job queue statistics
+          def jobs_queue_statistics
+            # We return empty stats in case jobs queue is not yet initialized
+            Karafka::Server.jobs_queue&.statistics || { busy: 0, enqueued: 0 }
+          end
+
           # Total memory used in the OS
           def memory_total_usage
             case RUBY_PLATFORM

data/lib/karafka/web/tracking/reporter.rb

@@ -44,12 +44,14 @@ module Karafka
 
             @consumer_contract.validate!(consumer_report)
 
+            process_name = consumer_report[:process][:name]
+
             # Report consumers statuses
             messages = [
               {
                 topic: ::Karafka::Web.config.topics.consumers.reports,
                 payload: consumer_report.to_json,
-                key: consumer_report[:process][:name],
+                key: process_name,
                 partition: 0
               }
             ]
@@ -58,12 +60,9 @@ module Karafka
             messages += consumer_sampler.errors.map do |error|
               {
                 topic: Karafka::Web.config.topics.errors,
-                # Inject process name into the error details for easier tracing
-                payload: error.merge(
-                  process_name: consumer_report[:process][:name]
-                ).to_json,
+                payload: error.to_json,
                 # Always dispatch errors from the same process to the same partition
-                key: consumer_report[:process][:name]
+                key: process_name
               }
             end
 

data/lib/karafka/web/ui/controllers/cluster.rb

@@ -10,13 +10,33 @@ module Karafka
           def index
             @cluster_info = Karafka::Admin.cluster_info
 
-            @topics = @cluster_info
-                      .topics
-                      .reject { |topic| topic[:topic_name] == '__consumer_offsets' }
-                      .sort_by { |topic| topic[:topic_name] }
+            partitions_total = []
+
+            displayable_topics(@cluster_info).each do |topic|
+              topic[:partitions].each do |partition|
+                partitions_total << partition.merge(topic: topic)
+              end
+            end
+
+            @partitions, @next_page = Ui::Lib::PaginateArray.new.call(
+              partitions_total,
+              @params.current_page
+            )
 
             respond
           end
+
+          private
+
+          # @param cluster_info [Rdkafka::Metadata] cluster metadata
+          # @return [Array<Hash>] array with topics to be displayed sorted in an alphabetical
+          #   order
+          def displayable_topics(cluster_info)
+            cluster_info
+              .topics
+              .reject { |topic| topic[:topic_name] == '__consumer_offsets' }
+              .sort_by { |topic| topic[:topic_name] }
+          end
         end
       end
     end

data/lib/karafka/web/ui/helpers/application_helper.rb

@@ -74,6 +74,16 @@ module Karafka
             bg
           end
 
+          # Renders tags one after another
+          #
+          # @param tags_array [Array<String>]
+          # @return [String] tags badges
+          def tags(tags_array)
+            tags_array
+              .map { |tag| %(<span class="badge bg-info">#{tag}</span>) }
+              .join(' ')
+          end
+
           # Takes a kafka report state and recommends background style color
           # @param state [String] state
           # @return [String] background style

data/lib/karafka/web/ui/models/status.rb

@@ -118,6 +118,22 @@ module Karafka
             )
           end
 
+          # @return [Status::Step] is Pro enabled with all of its features.
+          # @note It's not an error not to have it but we want to warn, that some of the features
+          #   may not work without Pro.
+          def pro_subscription
+            status = if state_calculation.success?
+                       ::Karafka.pro? ? :success : :warning
+                     else
+                       :halted
+                     end
+
+            Step.new(
+              status,
+              nil
+            )
+          end
+
           private
 
           # @return [String] consumers states topic name

data/lib/karafka/web/ui/pro/views/consumers/_consumer.erb

@@ -20,11 +20,7 @@
     </p>
 
     <p class="mt-0 mb-1">
-      <% process.tags.each do |tag| %>
-        <span class="badge bg-info badge-topic">
-          <%= tag %>
-        </span>
-      <% end %>
+      <%== tags(process.tags) %>
     </p>
   </td>
 

data/lib/karafka/web/ui/pro/views/consumers/consumer/_metrics.erb

@@ -30,11 +30,7 @@
 
             <% unless @process.tags.empty? %>
               <p class="mb-0 text-end">
-                <% @process.tags.each do |tag| %>
-                  <span class="badge bg-info badge-topic">
-                    <%= tag %>
-                  </span>
-                <% end %>
+                <%== tags(@process.tags) %>
               </p>
             <% end %>
           </div>

data/lib/karafka/web/ui/pro/views/errors/_detail.erb

@@ -7,6 +7,8 @@
       <td>
         <% if k2.to_s.include?('ssl') %>
           ***
+        <% elsif k2.to_s == 'tags' %>
+          <%== tags(v2) %>
         <% else %>
           <%= v2 %>
         <% end %>

data/lib/karafka/web/ui/pro/views/jobs/_job.erb

@@ -13,9 +13,7 @@
   <td>
     <code><%= job.consumer %></code>
 
-    <% job.tags.each do |tag| %>
-      <span class="badge bg-info"><%= tag %></span>
-    <% end %>
+    <%== tags(job.tags) %>
   </td>
   <td>
     <code>#<%= job.type %></code>

data/lib/karafka/web/ui/views/cluster/_partition.erb

@@ -1,3 +1,5 @@
+<% topic = partition[:topic] %>
+
 <% if topic[:topic_name] != '__consumer_offsets' %>
   <tr>
     <td>

data/lib/karafka/web/ui/views/cluster/index.erb

@@ -56,15 +56,12 @@
             </tr>
           </thead>
           <tbody>
-            <% @topics.each do |topic| %>
-              <%==
-                each_partial(
-                  topic[:partitions],
-                  'cluster/partition',
-                  locals: { topic: topic }
-                )
-              %>
-            <% end %>
+            <%==
+              each_partial(
+                @partitions,
+                'cluster/partition'
+              )
+            %>
           </tbody>
         </table>
     </div>

data/lib/karafka/web/ui/views/consumers/_consumer.erb

@@ -20,11 +20,7 @@
     </p>
 
     <p class="mt-0 mb-1">
-      <% process.tags.each do |tag| %>
-        <span class="badge bg-info badge-topic">
-          <%= tag %>
-        </span>
-      <% end %>
+      <%== tags(process.tags) %>
     </p>
   </td>
 

data/lib/karafka/web/ui/views/errors/_detail.erb

@@ -7,6 +7,8 @@
       <td>
         <% if k2.to_s.include?('ssl') %>
           ***
+        <% elsif k2.to_s == 'tags' %>
+          <%== tags(v2) %>
         <% else %>
           <%= v2 %>
         <% end %>

data/lib/karafka/web/ui/views/jobs/_job.erb

@@ -13,9 +13,7 @@
   <td>
     <code><%= job.consumer %></code>
 
-    <% job.tags.each do |tag| %>
-      <span class="badge bg-info"><%= tag %></span>
-    <% end %>
+    <%== tags(job.tags) %>
   </td>
   <td>
     <code>#<%= job.type %></code>

data/lib/karafka/web/ui/views/shared/_pagination.erb

@@ -9,6 +9,12 @@
             </a>
           </li>
 
+          <li class="page-item active disabled">
+            <a class="page-link" href="<%= current_path(page: @current_page) %>">
+              <span><%= @current_page %></span>
+            </a>
+          </li>
+
           <li class="page-item <%= 'disabled' unless @next_page %>">
             <a class="page-link" href="<%= current_path(page: @next_page) %>">
               <span>Next &raquo;</span>

data/lib/karafka/web/ui/views/status/_warning.erb

@@ -0,0 +1,14 @@
+<div class="card border-warning mb-3">
+  <div class="card-header text-bg-warning">
+    <span>
+      <%= title %>
+    </span>
+
+    <span class="float-end">
+      <span class="badge  text-bg-light">Warning</span>
+    </span>
+  </div>
+  <div class="card-body">
+    <%== description %>
+  </div>
+</div>

data/lib/karafka/web/ui/views/status/show.erb

@@ -89,6 +89,21 @@
         )
       %>
 
+      <%==
+        partial(
+          "status/#{@status.pro_subscription.to_s}",
+          locals: {
+            title: 'Karafka Pro subscription',
+            description: partial(
+              'status/warnings/pro_subscription',
+              locals: {
+                details: @status.pro_subscription.details
+              }
+            )
+          }
+        )
+      %>
+
     </div>
   </div>
 </div>

data/lib/karafka/web/ui/views/status/warnings/_pro_subscription.erb

@@ -0,0 +1,9 @@
+<p>
+  You are using the OSS version of the Karafka ecosystem, and some of the Karafka ecosystem features are unavailable in the OSS version.
+</p>
+
+<p class="mb-0">
+  Please help us make the Karafka ecosystem better by subscribing to our
+  <a target="_blank" href="https://karafka.io/#become-pro">Pro</a>
+  offering.
+</p>

data/lib/karafka/web/version.rb

@@ -3,6 +3,6 @@
 module Karafka
   module Web
     # Current gem version
-    VERSION = '0.2.2'
+    VERSION = '0.2.4'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-web
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.4
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2023-02-25 00:00:00.000000000 Z
+date: 2023-03-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: erubi
@@ -57,7 +57,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.33
+        version: 2.0.35
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -67,7 +67,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.33
+        version: 2.0.35
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -314,6 +314,7 @@ files:
 - lib/karafka/web/ui/views/status/_failure.erb
 - lib/karafka/web/ui/views/status/_halted.erb
 - lib/karafka/web/ui/views/status/_success.erb
+- lib/karafka/web/ui/views/status/_warning.erb
 - lib/karafka/web/ui/views/status/failures/_connection.erb
 - lib/karafka/web/ui/views/status/failures/_initial_state.erb
 - lib/karafka/web/ui/views/status/failures/_live_reporting.erb
@@ -321,6 +322,7 @@ files:
 - lib/karafka/web/ui/views/status/failures/_state_calculation.erb
 - lib/karafka/web/ui/views/status/failures/_topics.erb
 - lib/karafka/web/ui/views/status/show.erb
+- lib/karafka/web/ui/views/status/warnings/_pro_subscription.erb
 - lib/karafka/web/version.rb
 homepage: https://karafka.io
 licenses: