karafka-web 0.1.3 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3a70fc5cb35d0e6e60a348e3efe91113b54d0c62310b7c3af4e7584aa70cfb70
-  data.tar.gz: 43a96b7e7a4622721dbc6ba144852335aecbfe907049bc47ab563ea07f40eb54
+  metadata.gz: 020c1e94ca0603044d2260ebe796a6509432b7ec100aadc7505a6de85e5cf341
+  data.tar.gz: 71f6a22eb9bccbaa9bb9f4005d79667b83d486e13f59d966f3ccdaf049aff0e6
 SHA512:
-  metadata.gz: db0b153569ca93f1164a851216e0f224acc56e6a7533a35a86a44f6f2f8bffc1630e5ccf7c2e53a14455206401c45b59b1728eb266c59360be8f952794d3011b
-  data.tar.gz: 794c77edb27f7f9a397c3de891b146f9020653b84bad46e1f1b537a1d547ffc793211e7bd5ffb0f426c0b8fa9b6359230bb3fe3ee8fa5ec668cb0e1d39782d5a
+  metadata.gz: c480170adb20bcf51abe1be667343c198a09dbeb5280443cbdcbfe60f917a44ba0cdb8b8d3dd558c4a102299c9e131f902958f312f11f392329b3bae440e57b7
+  data.tar.gz: 16ca1295589f06cf7ebd7804353a6ccaf08d77d7cce1ef7046a763dd5b8f2f0abf248c4e73431032dfae19ca9250eedfa1729e2983b063eaa85dbd16d3ce0eb8

data/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Karafka Web changelog
 
+## 0.2.1 (2023-02-24)
+- [Fix] Fix format incompatibility between 0.1.x and 0.2.x data formats. This will allow for the 0.2 Web UI to work with 0.1.x reporting.
+
+## 0.2.0 (2023-02-24)
+- **[Feature]** Introduce ability to tag `Karafka::Process` to display process-centric tags in the Web UI.
+- **[Feature]** Introduce ability to tag consumer instances to display consumption-centric tags in the Web UI.
+- **[Feature]** Introduce a /status page that can validate the setup and tell what is missing (#1318)
+- [Improvement] Allow for disabling the consumer subscription from Web for multi-tenant Web UI usage (#1331)
+- [Improvement] Make sure that states and reports are always dispatched to the partition `0`. This should prevent UI from not fully working when someone accidentally creates more partitions than expected.
+- [Fix] Fix a bug where bootstrapping would create two initial states.
+- [Fix] Fix a case, where errors listener would try to force encoding on a frozen error message.
+
 ## 0.1.3 (2023-02-14)
 - Skip topics creation if web topics already exist (do not raise error)
 - Support ability to provide replication factor in the install command

data/Gemfile

@@ -8,6 +8,7 @@ gemspec
 
 group :test do
   gem 'byebug'
+  gem 'factory_bot'
   gem 'rspec'
   gem 'simplecov'
 end

data/Gemfile.lock

@@ -1,41 +1,51 @@
 PATH
   remote: .
   specs:
-    karafka-web (0.1.3)
+    karafka-web (0.2.1)
       erubi (~> 1.4)
-      karafka (>= 2.0.29, < 3.0.0)
-      karafka-core (>= 2.0.10, < 3.0.0)
+      karafka (>= 2.0.33, < 3.0.0)
+      karafka-core (>= 2.0.12, < 3.0.0)
       roda (~> 3.63)
       tilt (~> 2.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
+    activesupport (7.0.4.2)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
     byebug (11.1.3)
     concurrent-ruby (1.2.0)
     diff-lcs (1.5.0)
     docile (1.4.0)
     erubi (1.12.0)
+    factory_bot (6.2.1)
+      activesupport (>= 5.0.0)
     ffi (1.15.5)
-    karafka (2.0.30)
-      karafka-core (>= 2.0.9, < 3.0.0)
+    i18n (1.12.0)
+      concurrent-ruby (~> 1.0)
+    karafka (2.0.33)
+      karafka-core (>= 2.0.12, < 3.0.0)
       thor (>= 0.20)
       waterdrop (>= 2.4.10, < 3.0.0)
       zeitwerk (~> 2.3)
-    karafka-core (2.0.10)
+    karafka-core (2.0.12)
       concurrent-ruby (>= 1.1)
-      karafka-rdkafka (>= 0.12)
-    karafka-rdkafka (0.12.0)
+      karafka-rdkafka (>= 0.12.1)
+    karafka-rdkafka (0.12.1)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
     mini_portile2 (2.8.1)
+    minitest (5.17.0)
     rack (3.0.4.1)
     rackup (0.2.3)
       rack (>= 3.0.0.beta1)
       webrick
     rake (13.0.6)
-    roda (3.64.0)
+    roda (3.65.0)
       rack
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
@@ -57,7 +67,9 @@ GEM
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
     thor (1.2.1)
-    tilt (2.0.11)
+    tilt (2.1.0)
+    tzinfo (2.0.6)
+      concurrent-ruby (~> 1.0)
     waterdrop (2.4.10)
       karafka-core (>= 2.0.9, < 3.0.0)
       zeitwerk (~> 2.3)
@@ -65,11 +77,11 @@ GEM
     zeitwerk (2.6.7)
 
 PLATFORMS
-  arm64-darwin-21
   x86_64-linux
 
 DEPENDENCIES
   byebug
+  factory_bot
   karafka-web!
   rackup (~> 0.2)
   rspec

data/karafka-web.gemspec

@@ -17,8 +17,8 @@ Gem::Specification.new do |spec|
   spec.licenses    = %w[LGPL-3.0 Commercial]
 
   spec.add_dependency 'erubi', '~> 1.4'
-  spec.add_dependency 'karafka', '>= 2.0.29', '< 3.0.0'
-  spec.add_dependency 'karafka-core', '>= 2.0.10', '< 3.0.0'
+  spec.add_dependency 'karafka', '>= 2.0.33', '< 3.0.0'
+  spec.add_dependency 'karafka-core', '>= 2.0.12', '< 3.0.0'
   spec.add_dependency 'roda', '~> 3.63'
   spec.add_dependency 'tilt', '~> 2.0'
 

data/lib/karafka/web/config.rb

@@ -56,6 +56,11 @@ module Karafka
 
       # States processing related settings
       setting :processing do
+        # Should we actively process reports
+        # This can be disabled in case of using a multi-tenant approach where only one of the
+        # apps should materialize the state
+        setting :active, default: true
+
         # What should be the consumer group name for web consumer
         setting :consumer_group, default: 'karafka_web'
 

data/lib/karafka/web/installer.rb

@@ -45,6 +45,7 @@ module Karafka
           consumer_group ::Karafka::Web.config.processing.consumer_group do
             # Topic we listen on to materialize the states
             topic ::Karafka::Web.config.topics.consumers.reports do
+              active ::Karafka::Web.config.processing.active
               # Since we materialize state in intervals, we can poll for half of this time without
               # impacting the reporting responsiveness
               max_wait_time ::Karafka::Web.config.processing.interval / 2
@@ -128,8 +129,6 @@ module Karafka
             replication_factor
           )
         end
-
-        bootstrap_state!
       end
 
       # Creates the initial state record with all values being empty

data/lib/karafka/web/processing/consumer.rb

@@ -57,7 +57,8 @@ module Karafka
             topic: Karafka::Web.config.topics.consumers.states,
             payload: @consumers_aggregator.to_json,
             # This will ensure that the consumer states are compacted
-            key: Karafka::Web.config.topics.consumers.states
+            key: Karafka::Web.config.topics.consumers.states,
+            partition: 0
           )
         end
       end

data/lib/karafka/web/tracking/consumers/contracts/job.rb

@@ -18,6 +18,7 @@ module Karafka
             required(:last_offset) { |val| val.is_a?(Integer) && val >= 0 }
             required(:comitted_offset) { |val| val.is_a?(Integer) }
             required(:type) { |val| %w[consume revoked shutdown].include?(val) }
+            required(:tags) { |val| val.is_a?(Karafka::Core::Taggable::Tags) }
           end
         end
       end

data/lib/karafka/web/tracking/consumers/contracts/report.rb

@@ -27,6 +27,7 @@ module Karafka
               required(:status) { |val| ::Karafka::Status::STATES.key?(val.to_sym) }
               required(:listeners) { |val| val.is_a?(Integer) && val >= 0 }
               required(:concurrency) { |val| val.is_a?(Integer) && val.positive? }
+              required(:tags) { |val| val.is_a?(Karafka::Core::Taggable::Tags) }
 
               required(:cpu_usage) do |val|
                 val.is_a?(Array) &&

data/lib/karafka/web/tracking/consumers/listeners/errors.rb

@@ -74,7 +74,7 @@ module Karafka
             # @param error [StandardError] error that occurred
             # @return [Array<String, String, String>] array with error name, message and backtrace
             def extract_error_info(error)
-              error_message = error.message.to_s
+              error_message = error.message.to_s.dup
               error_message.force_encoding('utf-8')
               error_message.scrub!
 

data/lib/karafka/web/tracking/consumers/listeners/processing.rb

@@ -146,7 +146,8 @@ module Karafka
                 comitted_offset: consumer.coordinator.seek_offset - 1,
                 consumer: consumer.class.to_s,
                 consumer_group: consumer.topic.consumer_group.id,
-                type: type
+                type: type,
+                tags: consumer.tags
               }
             end
           end

data/lib/karafka/web/tracking/consumers/sampler.rb

@@ -11,13 +11,18 @@ module Karafka
 
           attr_reader :counters, :consumer_groups, :errors, :times, :pauses, :jobs
 
+          # Current schema version
+          # This can be used in the future for detecting incompatible changes and writing
+          # migrations
+          SCHEMA_VERSION = '1.0.1'
+
           # 60 seconds window for time tracked window-based metrics
           TIMES_TTL = 60
 
           # Times ttl in ms
           TIMES_TTL_MS = TIMES_TTL * 1_000
 
-          private_constant :TIMES_TTL, :TIMES_TTL_MS
+          private_constant :TIMES_TTL, :TIMES_TTL_MS, :SCHEMA_VERSION
 
           def initialize
             @counters = {
@@ -47,7 +52,7 @@ module Karafka
           # @return [Hash] report hash with all the details about consumer operations
           def to_report
             {
-              schema_version: '1.0.0',
+              schema_version: SCHEMA_VERSION,
               type: 'consumer',
               dispatched_at: float_now,
 
@@ -61,7 +66,8 @@ module Karafka
                 memory_total_usage: memory_total_usage,
                 memory_size: memory_size,
                 cpu_count: cpu_count,
-                cpu_usage: cpu_usage
+                cpu_usage: cpu_usage,
+                tags: Karafka::Process.tags
               },
 
               versions: {

data/lib/karafka/web/tracking/reporter.rb

@@ -49,7 +49,8 @@ module Karafka
               {
                 topic: ::Karafka::Web.config.topics.consumers.reports,
                 payload: consumer_report.to_json,
-                key: consumer_report[:process][:name]
+                key: consumer_report[:process][:name],
+                partition: 0
               }
             ]
 

data/lib/karafka/web/ui/app.rb

@@ -71,6 +71,11 @@ module Karafka
               render_response controller.index
             end
           end
+
+          r.get 'status' do
+            controller = Controllers::Status.new(params)
+            render_response controller.show
+          end
         end
       end
     end

data/lib/karafka/web/ui/controllers/status.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Controllers
+        # View that helps understand the status of the Web UI
+        # Many people reported problems understanding the requirements or misconfigured things.
+        # While all of the things are documented, people are lazy. Hence we provide a status
+        # page where we check that everything is as expected and if not, we can provide some
+        # helpful instructions on how to fix the issues.
+        class Status < Base
+          # Displays the Web UI setup status
+          def show
+            @status = Models::Status.new
+
+            respond
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/models/job.rb

@@ -6,6 +6,10 @@ module Karafka
       module Models
         # Single job data representation model
         class Job < Lib::HashProxy
+          # @return [Array<String>] tags of this consuming job / consumer
+          def tags
+            @hash[:tags] || []
+          end
         end
       end
     end

data/lib/karafka/web/ui/models/process.rb

@@ -40,6 +40,13 @@ module Karafka
               .sort_by(&:started_at)
           end
 
+          # @return [Array<String>] tags of this consuming process
+          # @note We use direct reference here and a fallback for users that come from the versions
+          #   that did not report tags
+          def tags
+            @hash[:process][:tags] || []
+          end
+
           # @return [Integer] collective lag on this process
           def lag_stored
             consumer_groups

data/lib/karafka/web/ui/models/status.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Models
+        # Model that represents the general status of the Web UI.
+        # We use this data to display a status page that helps with debugging on what is missing
+        # in the overall setup of the Web UI.
+        #
+        # People have various problems like too many partitions, not created topics, etc. and this
+        # data and view aims to help them with understanding the current status of the setup
+        class Status
+          # Status of a single step of setup
+          Step = Struct.new(:status, :details) do
+            # @return [Boolean] is the given step successfully configured and working
+            def success?
+              status == :success
+            end
+
+            # @return [String] stringified status
+            def to_s
+              status.to_s
+            end
+          end
+
+          # Initializes the status object and tries to connect to Kafka
+          def initialize
+            connect
+          end
+
+          # @return [Status::Step] were we able to connect to Kafka or not
+          def connection
+            Step.new(
+              @connected ? :success : :failure,
+              nil
+            )
+          end
+
+          # @return [Status::Step] do all the needed topics exist
+          def topics
+            if connection.success?
+              details = topics_details
+              status = details.all? { |_, detail| detail[:present] } ? :success : :failure
+            else
+              status = :halted
+              details = {}
+            end
+
+            Step.new(
+              status,
+              details
+            )
+          end
+
+          # @return [Status::Step] do we have all topics with expected number of partitions
+          def partitions
+            if topics.success?
+              status = :success
+              status = :failure if topics_details[topics_consumers_states][:partitions] != 1
+              status = :failure if topics_details[topics_consumers_reports][:partitions] != 1
+              details = topics_details
+            else
+              status = :halted
+              details = {}
+            end
+
+            Step.new(
+              status,
+              details
+            )
+          end
+
+          # @return [Status::Step] Is the initial state present in the setup or not
+          def initial_state
+            if partitions.success?
+              @current_state ||= Models::State.current
+              status = @current_state ? :success : :failure
+            else
+              status = :halted
+            end
+
+            Step.new(
+              status,
+              nil
+            )
+          end
+
+          # @return [Status::Step] Is there at least one active karafka server reporting to the
+          #   Web UI
+          def live_reporting
+            if initial_state.success?
+              @processes ||= Models::Processes.active(@current_state)
+              status = @processes.empty? ? :failure : :success
+            else
+              status = :halted
+            end
+
+            Step.new(
+              status,
+              nil
+            )
+          end
+
+          # @return [Status::Step] is there a subscription to our reports topic that is being
+          #   consumed actively.
+          def state_calculation
+            if live_reporting.success?
+              @subscriptions ||= Models::Health.current(@current_state).values.flat_map(&:keys)
+              status = @subscriptions.include?(topics_consumers_reports) ? :success : :failure
+            else
+              status = :halted
+            end
+
+            Step.new(
+              status,
+              nil
+            )
+          end
+
+          private
+
+          # @return [String] consumers states topic name
+          def topics_consumers_states
+            ::Karafka::Web.config.topics.consumers.states.to_s
+          end
+
+          # @return [String] consumers reports topic name
+          def topics_consumers_reports
+            ::Karafka::Web.config.topics.consumers.reports.to_s
+          end
+
+          # @return [String] errors topic name
+          def topics_errors
+            ::Karafka::Web.config.topics.errors
+          end
+
+          # @return [Hash] hash with topics with which we work details (even if don't exist)
+          def topics_details
+            topics = {
+              topics_consumers_states => { present: false, partitions: 0 },
+              topics_consumers_reports => { present: false, partitions: 0 },
+              topics_errors => { present: false, partitions: 0 }
+            }
+
+            @cluster_info.topics.each do |topic|
+              name = topic[:topic_name]
+
+              next unless topics.key?(name)
+
+              topics[name][:present] = true
+              topics[name][:partitions] = topic[:partition_count]
+            end
+
+            topics
+          end
+
+          # Tries connecting with the cluster and sets the connection state
+          def connect
+            @cluster_info = ::Karafka::Admin.cluster_info
+            @connected = true
+          rescue ::Rdkafka::RdkafkaError
+            @connected = false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/pro/app.rb

@@ -112,6 +112,11 @@ module Karafka
               controller = Controllers::Dlq.new(params)
               render_response controller.index
             end
+
+            r.get 'status' do
+              controller = Controllers::Status.new(params)
+              render_response controller.show
+            end
           end
         end
       end

data/lib/karafka/web/ui/pro/controllers/status.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Ui
+      module Pro
+        module Controllers
+          # Status details - same as in OSS
+          class Status < Ui::Controllers::Status
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/pro/views/consumers/_consumer.erb

@@ -8,16 +8,24 @@
       <%= process.name %>
     </a>
 
-    <br/>
+    <p class="mt-0 mb-1">
+      <% process.consumer_groups.each do |consumer_group| %>
+        <% consumer_group.topics.each do |topic| %>
+          <span class="badge bg-secondary badge-topic" title="Consumer group: <%= consumer_group.id %>">
+            <%= topic.name %>:
+            <%= topic.partitions.map(&:id).join(',') %>
+          </span>
+        <% end %>
+      <% end %>
+    </p>
 
-    <% process.consumer_groups.each do |consumer_group| %>
-      <% consumer_group.topics.each do |topic| %>
-        <span class="badge bg-secondary badge-topic" title="Consumer group: <%= consumer_group.id %>">
-          <%= topic.name %>:
-          <%= topic.partitions.map(&:id).join(',') %>
+    <p class="mt-0 mb-1">
+      <% process.tags.each do |tag| %>
+        <span class="badge bg-info badge-topic">
+          <%= tag %>
         </span>
       <% end %>
-    <% end %>
+    </p>
   </td>
 
   <td>

data/lib/karafka/web/ui/pro/views/consumers/consumer/_job.erb

@@ -1,6 +1,10 @@
 <tr>
   <td>
     <code><%= job.consumer %></code>
+
+    <% job.tags.each do |tag| %>
+      <span class="badge bg-info"><%= tag %></span>
+    <% end %>
   </td>
   <td>
     <span class="badge bg-secondary badge-topic" title="Consumer group: <%= job.consumer_group %>">

data/lib/karafka/web/ui/pro/views/consumers/consumer/_metrics.erb

@@ -27,6 +27,16 @@
               </li>
             </ul>
             </p>
+
+            <% unless @process.tags.empty? %>
+              <p class="mb-0 text-end">
+                <% @process.tags.each do |tag| %>
+                  <span class="badge bg-info badge-topic">
+                    <%= tag %>
+                  </span>
+                <% end %>
+              </p>
+            <% end %>
           </div>
         </div>
         <div class="card">

data/lib/karafka/web/ui/pro/views/jobs/_job.erb

@@ -12,6 +12,10 @@
   </td>
   <td>
     <code><%= job.consumer %></code>
+
+    <% job.tags.each do |tag| %>
+      <span class="badge bg-info"><%= tag %></span>
+    <% end %>
   </td>
   <td>
     <code>#<%= job.type %></code>

data/lib/karafka/web/ui/pro/views/shared/_navigation.erb

@@ -45,6 +45,11 @@
         Cluster
       </a>
     </li>
+    <li class="nav-item ms-3">
+      <a class="nav-link <%= nav_class(start_with: '/status') %>" href="<%= root_path('Status') %>">
+        Status
+      </a>
+    </li>
   </ul>
 </div>
 

data/lib/karafka/web/ui/views/consumers/_consumer.erb

@@ -8,16 +8,24 @@
       <%= process.name %>
     </a>
 
-    <br/>
+    <p class="mt-0 mb-1">
+      <% process.consumer_groups.each do |consumer_group| %>
+        <% consumer_group.topics.each do |topic| %>
+          <span class="badge bg-secondary badge-topic" title="Consumer group: <%= consumer_group.id %>">
+            <%= topic.name %>:
+            <%= topic.partitions.map(&:id).join(',') %>
+          </span>
+        <% end %>
+      <% end %>
+    </p>
 
-    <% process.consumer_groups.each do |consumer_group| %>
-      <% consumer_group.topics.each do |topic| %>
-        <span class="badge bg-secondary badge-topic" title="Consumer group: <%= consumer_group.id %>">
-          <%= topic.name %>:
-          <%= topic.partitions.map(&:id).join(',') %>
+    <p class="mt-0 mb-1">
+      <% process.tags.each do |tag| %>
+        <span class="badge bg-info badge-topic">
+          <%= tag %>
         </span>
       <% end %>
-    <% end %>
+    </p>
   </td>
 
   <td>