karafka-web 0.1.2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1ffbcae8c8e094bdd72518fa41694bb9b6b658c713010ca09ff9f3ca8f857666
-  data.tar.gz: 4e9d38e683197bb003afb3732e333a19d603e2b8fd3e8ff4fa7b8d0d1cc5f3c3
+  metadata.gz: eb17bd9d46e6fa0d0125bc665bff74a9d4277be394f1bf29223b6d49ef09e054
+  data.tar.gz: 2f1da7d4fa79116a67ae3300863b56c3ac2215d80dccba89520e33e78c3b1a84
 SHA512:
-  metadata.gz: ba7f91646a8062b56380afe9ccde45802c18c12d5a7581fce4b99db24480b730aa0f9743af825dd24006725424887f63cf5a2dbf17d32c541d7a4df83f823c34
-  data.tar.gz: 19084a9683d9d8023fbef859a428d418a375c2b44d588ad9d065de6e2a89f913cc505258836f717c2e5a095e3a0d76b15d0d2d6c23df95682d41e042ec1d16f8
+  metadata.gz: cc9f331aa0b9276c1b3c2d6d4f400b9d8600da174123d472b1699554e0cd95de14ff60b7c4effbb0c4f86a423f24ff27f5facc282ff8215863c146a4b1c0bb41
+  data.tar.gz: 1f638b2ca877b1b34612654a19f0fb8d04ab88e5dc6b560a7090e74a6d1202da0257d4f941f8f182b6b874d8a099db30772a1ab9517b32ebae76161d9314797a

data/CHANGELOG.md

@@ -1,8 +1,24 @@
 # Karafka Web changelog
 
+## 0.2.0 (2023-02-24)
+- **[Feature]** Introduce ability to tag `Karafka::Process` to display process-centric tags in the Web UI.
+- **[Feature]** Introduce ability to tag consumer instances to display consumption-centric tags in the Web UI.
+- **[Feature]** Introduce a /status page that can validate the setup and tell what is missing (#1318)
+- [Improvement] Allow for disabling the consumer subscription from Web for multi-tenant Web UI usage (#1331)
+- [Improvement] Make sure that states and reports are always dispatched to the partition `0`. This should prevent UI from not fully working when someone accidentally creates more partitions than expected.
+- [Fix] Fix a bug where bootstrapping would create two initial states.
+- [Fix] Fix a case, where errors listener would try to force encoding on a frozen error message.
+
+## 0.1.3 (2023-02-14)
+- Skip topics creation if web topics already exist (do not raise error)
+- Support ability to provide replication factor in the install command
+- Provide ability to reset the state with a `reset` command. It will remove and re-create the topics.
+- Provide ability to uninstall the web via the CLI `uninstall` command
+- Remove the `Karafka::Web.bootstrap!` method as the install should happen via `bundle exec karafka-web install`
+
 ## 0.1.2 (2023-02-10)
 - Provide more comprehensive info when lag stored and stored offset are not available.
-- Setup rspec scaffold
+- Setup rspec scaffold.
 
 ## 0.1.1 (2023-01-30)
 - Rename `Karafka::Web.bootstrap_topics!` to `Karafka::Web.bootstrap!` and expand it with the zero state injection.

data/Gemfile

@@ -8,6 +8,7 @@ gemspec
 
 group :test do
   gem 'byebug'
+  gem 'factory_bot'
   gem 'rspec'
   gem 'simplecov'
 end

data/Gemfile.lock

@@ -1,41 +1,51 @@
 PATH
   remote: .
   specs:
-    karafka-web (0.1.2)
+    karafka-web (0.2.0)
       erubi (~> 1.4)
-      karafka (>= 2.0.29, < 3.0.0)
-      karafka-core (>= 2.0.10, < 3.0.0)
+      karafka (>= 2.0.33, < 3.0.0)
+      karafka-core (>= 2.0.12, < 3.0.0)
       roda (~> 3.63)
       tilt (~> 2.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
+    activesupport (7.0.4.2)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
     byebug (11.1.3)
     concurrent-ruby (1.2.0)
     diff-lcs (1.5.0)
     docile (1.4.0)
     erubi (1.12.0)
+    factory_bot (6.2.1)
+      activesupport (>= 5.0.0)
     ffi (1.15.5)
-    karafka (2.0.30)
-      karafka-core (>= 2.0.9, < 3.0.0)
+    i18n (1.12.0)
+      concurrent-ruby (~> 1.0)
+    karafka (2.0.33)
+      karafka-core (>= 2.0.12, < 3.0.0)
       thor (>= 0.20)
       waterdrop (>= 2.4.10, < 3.0.0)
       zeitwerk (~> 2.3)
-    karafka-core (2.0.10)
+    karafka-core (2.0.12)
       concurrent-ruby (>= 1.1)
-      karafka-rdkafka (>= 0.12)
-    karafka-rdkafka (0.12.0)
+      karafka-rdkafka (>= 0.12.1)
+    karafka-rdkafka (0.12.1)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
     mini_portile2 (2.8.1)
+    minitest (5.17.0)
     rack (3.0.4.1)
     rackup (0.2.3)
       rack (>= 3.0.0.beta1)
       webrick
     rake (13.0.6)
-    roda (3.64.0)
+    roda (3.65.0)
       rack
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
@@ -57,7 +67,9 @@ GEM
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
     thor (1.2.1)
-    tilt (2.0.11)
+    tilt (2.1.0)
+    tzinfo (2.0.6)
+      concurrent-ruby (~> 1.0)
     waterdrop (2.4.10)
       karafka-core (>= 2.0.9, < 3.0.0)
       zeitwerk (~> 2.3)
@@ -65,11 +77,11 @@ GEM
     zeitwerk (2.6.7)
 
 PLATFORMS
-  arm64-darwin-21
   x86_64-linux
 
 DEPENDENCIES
   byebug
+  factory_bot
   karafka-web!
   rackup (~> 0.2)
   rspec

data/bin/karafka-web

@@ -4,30 +4,5 @@ require 'karafka'
 require 'karafka/web'
 
 ::Karafka::Cli::Base.load
-include ::Karafka::Helpers::Colorize
 
-case ARGV[0]
-when 'install'
-  puts
-  puts 'Installing Karafka Web UI...'
-  puts
-  puts 'Creating necessary topics and populating state data...'
-  Karafka::Web.bootstrap!
-  puts 'Updating the Karafka boot file...'
-
-  enabler = 'Karafka::Web.enable!'
-
-  if File.read(Karafka.boot_file).include?(enabler)
-    puts "Web UI #{green('already')} installed."
-  else
-    File.open(Karafka.boot_file, 'a') do |f|
-      f << "\n#{enabler}\n"
-    end
-  end
-
-  puts
-  puts("Installation #{green('completed')}. Have fun!")
-  puts
-else
-  raise NotImplementedError, "#{ARGV[0]} is not supported"
-end
+::Karafka::Web::Cli.start

data/karafka-web.gemspec

@@ -17,8 +17,8 @@ Gem::Specification.new do |spec|
   spec.licenses    = %w[LGPL-3.0 Commercial]
 
   spec.add_dependency 'erubi', '~> 1.4'
-  spec.add_dependency 'karafka', '>= 2.0.29', '< 3.0.0'
-  spec.add_dependency 'karafka-core', '>= 2.0.10', '< 3.0.0'
+  spec.add_dependency 'karafka', '>= 2.0.33', '< 3.0.0'
+  spec.add_dependency 'karafka-core', '>= 2.0.12', '< 3.0.0'
   spec.add_dependency 'roda', '~> 3.63'
   spec.add_dependency 'tilt', '~> 2.0'
 

data/lib/karafka/web/cli.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    # Karafka itself depends on Thor, so we can use it
+    class Cli < Thor
+      include ::Karafka::Helpers::Colorize
+
+      # Code that is needed in the `karafka.rb` to connect Web UI to Karafka
+      ENABLER_CODE = 'Karafka::Web.enable!'
+
+      private_constant :ENABLER_CODE
+
+      package_name 'Karafka Web'
+
+      desc 'install', 'Installs the Web UI'
+      method_option(
+        :replication_factor,
+        desc: 'Replication factor for created topics',
+        default: 1,
+        type: :numeric
+      )
+      # Installs Karafka Web
+      def install
+        puts
+        puts 'Installing Karafka Web UI...'
+        puts
+        puts 'Creating necessary topics and populating state data...'
+
+        Karafka::Web::Installer.new.bootstrap!(replication_factor: options[:replication_factor])
+
+        puts 'Updating the Karafka boot file...'
+
+        if File.read(Karafka.boot_file).include?(ENABLER_CODE)
+          puts "Web UI #{green('already')} installed."
+        else
+          File.open(Karafka.boot_file, 'a') do |f|
+            f << "\n#{ENABLER_CODE}\n"
+          end
+        end
+
+        puts
+        puts("Installation #{green('completed')}. Have fun!")
+        puts
+      end
+
+      desc 'reset', 'Resets the Web UI by removing all the Web topics and creating them again'
+      # Resets Karafka Web
+      def reset
+        puts
+        puts 'Resetting Karafka Web UI...'
+        Karafka::Web::Installer.new.reset!
+        puts
+        puts("Resetting #{green('completed')}. Have fun!")
+        puts
+      end
+
+      desc 'uninstall', 'Removes all the Web UI topics and the enabled code'
+      # Uninstalls Karafka Web
+      def uninstall
+        puts
+        puts 'Uninstalling Karafka Web UI...'
+        Karafka::Web::Installer.new.uninstall!
+
+        puts 'Updating the Karafka boot file...'
+
+        karafka_rb = File.readlines(Karafka.boot_file)
+        if karafka_rb.any? { |line| line.include?(ENABLER_CODE) }
+          karafka_rb.delete_if { |line| line.include?(ENABLER_CODE) }
+
+          File.write(Karafka.boot_file, karafka_rb.join)
+        end
+
+        puts
+        puts("Uninstalling #{green('completed')}. Goodbye!")
+        puts
+      end
+    end
+  end
+end

data/lib/karafka/web/config.rb

@@ -56,6 +56,11 @@ module Karafka
 
       # States processing related settings
       setting :processing do
+        # Should we actively process reports
+        # This can be disabled in case of using a multi-tenant approach where only one of the
+        # apps should materialize the state
+        setting :active, default: true
+
         # What should be the consumer group name for web consumer
         setting :consumer_group, default: 'karafka_web'
 

data/lib/karafka/web/installer.rb

@@ -13,6 +13,30 @@ module Karafka
         bootstrap_state!
       end
 
+      # Removes all the Karafka topics and creates them again with the same replication factor
+      def reset!
+        states_topic = ::Karafka::Web.config.topics.consumers.states
+        replication_factor = ::Karafka::Admin
+                             .cluster_info
+                             .topics
+                             .find { |topic| topic[:topic_name] == states_topic }
+                             .fetch(:partitions)
+                             .first
+                             .fetch(:replica_count)
+
+        uninstall!
+        bootstrap!(replication_factor: replication_factor)
+      end
+
+      # Removes all the Karafka Web topics
+      def uninstall!
+        [
+          ::Karafka::Web.config.topics.consumers.states,
+          ::Karafka::Web.config.topics.consumers.reports,
+          ::Karafka::Web.config.topics.errors
+        ].each { |topic_name| ::Karafka::Admin.delete_topic(topic_name) }
+      end
+
       # Adds the extra needed consumer group, topics and routes for Web UI to be able to operate
       def enable!
         ::Karafka::App.routes.draw do
@@ -21,6 +45,7 @@ module Karafka
           consumer_group ::Karafka::Web.config.processing.consumer_group do
             # Topic we listen on to materialize the states
             topic ::Karafka::Web.config.topics.consumers.reports do
+              active ::Karafka::Web.config.processing.active
               # Since we materialize state in intervals, we can poll for half of this time without
               # impacting the reporting responsiveness
               max_wait_time ::Karafka::Web.config.processing.interval / 2
@@ -61,38 +86,49 @@ module Karafka
       #
       # @param replication_factor [Integer]
       def bootstrap_topics!(replication_factor = 1)
-        # This topic needs to have one partition
-        ::Karafka::Admin.create_topic(
-          ::Karafka::Web.config.topics.consumers.states,
-          1,
-          replication_factor,
-          # We care only about the most recent state, previous are irrelevant
-          { 'cleanup.policy': 'compact' }
-        )
+        existing_topics = ::Karafka::Admin.cluster_info.topics.map { |topic| topic[:topic_name] }
 
-        # This topic needs to have one partition
-        ::Karafka::Admin.create_topic(
-          ::Karafka::Web.config.topics.consumers.reports,
-          1,
-          replication_factor,
-          # We do not need to to store this data for longer than 7 days as this data is only used
-          # to materialize the end states
-          # On the other hand we do not want to have it really short-living because in case of a
-          # consumer crash, we may want to use this info to catch up and backfill the state
-          { 'retention.ms': 7 * 24 * 60 * 60 * 1_000 }
-        )
+        consumers_states_topic = ::Karafka::Web.config.topics.consumers.states
+        consumers_reports_topic = ::Karafka::Web.config.topics.consumers.reports
+        errors_topic = ::Karafka::Web.config.topics.errors
 
-        # All the errors will be dispatched here
-        # This topic can have multiple partitions but we go with one by default. A single Ruby
-        # process should not crash that often and if there is an expectation of a higher volume
-        # of errors, this can be changed by the end user
-        ::Karafka::Admin.create_topic(
-          ::Karafka::Web.config.topics.errors,
-          1,
-          replication_factor
-        )
+        # Create only if needed
+        unless existing_topics.include?(consumers_states_topic)
+          # This topic needs to have one partition
+          ::Karafka::Admin.create_topic(
+            consumers_states_topic,
+            1,
+            replication_factor,
+            # We care only about the most recent state, previous are irrelevant
+            { 'cleanup.policy': 'compact' }
+          )
+        end
 
-        bootstrap_state!
+        unless existing_topics.include?(consumers_reports_topic)
+          # This topic needs to have one partition
+          ::Karafka::Admin.create_topic(
+            consumers_reports_topic,
+            1,
+            replication_factor,
+            # We do not need to to store this data for longer than 7 days as this data is only used
+            # to materialize the end states
+            # On the other hand we do not want to have it really short-living because in case of a
+            # consumer crash, we may want to use this info to catch up and backfill the state
+            { 'retention.ms': 7 * 24 * 60 * 60 * 1_000 }
+          )
+        end
+
+        unless existing_topics.include?(errors_topic)
+          # All the errors will be dispatched here
+          # This topic can have multiple partitions but we go with one by default. A single Ruby
+          # process should not crash that often and if there is an expectation of a higher volume
+          # of errors, this can be changed by the end user
+          ::Karafka::Admin.create_topic(
+            errors_topic,
+            1,
+            replication_factor
+          )
+        end
       end
 
       # Creates the initial state record with all values being empty

data/lib/karafka/web/processing/consumer.rb

@@ -57,7 +57,8 @@ module Karafka
             topic: Karafka::Web.config.topics.consumers.states,
             payload: @consumers_aggregator.to_json,
             # This will ensure that the consumer states are compacted
-            key: Karafka::Web.config.topics.consumers.states
+            key: Karafka::Web.config.topics.consumers.states,
+            partition: 0
           )
         end
       end

data/lib/karafka/web/tracking/consumers/contracts/job.rb

@@ -18,6 +18,7 @@ module Karafka
             required(:last_offset) { |val| val.is_a?(Integer) && val >= 0 }
             required(:comitted_offset) { |val| val.is_a?(Integer) }
             required(:type) { |val| %w[consume revoked shutdown].include?(val) }
+            required(:tags) { |val| val.is_a?(Karafka::Core::Taggable::Tags) }
           end
         end
       end

data/lib/karafka/web/tracking/consumers/contracts/report.rb

@@ -27,6 +27,7 @@ module Karafka
               required(:status) { |val| ::Karafka::Status::STATES.key?(val.to_sym) }
               required(:listeners) { |val| val.is_a?(Integer) && val >= 0 }
               required(:concurrency) { |val| val.is_a?(Integer) && val.positive? }
+              required(:tags) { |val| val.is_a?(Karafka::Core::Taggable::Tags) }
 
               required(:cpu_usage) do |val|
                 val.is_a?(Array) &&

data/lib/karafka/web/tracking/consumers/listeners/errors.rb

@@ -74,7 +74,7 @@ module Karafka
             # @param error [StandardError] error that occurred
             # @return [Array<String, String, String>] array with error name, message and backtrace
             def extract_error_info(error)
-              error_message = error.message.to_s
+              error_message = error.message.to_s.dup
               error_message.force_encoding('utf-8')
               error_message.scrub!
 

data/lib/karafka/web/tracking/consumers/listeners/processing.rb

@@ -146,7 +146,8 @@ module Karafka
                 comitted_offset: consumer.coordinator.seek_offset - 1,
                 consumer: consumer.class.to_s,
                 consumer_group: consumer.topic.consumer_group.id,
-                type: type
+                type: type,
+                tags: consumer.tags
               }
             end
           end

data/lib/karafka/web/tracking/consumers/sampler.rb

@@ -11,13 +11,18 @@ module Karafka
 
           attr_reader :counters, :consumer_groups, :errors, :times, :pauses, :jobs
 
+          # Current schema version
+          # This can be used in the future for detecting incompatible changes and writing
+          # migrations
+          SCHEMA_VERSION = '1.0.1'
+
           # 60 seconds window for time tracked window-based metrics
           TIMES_TTL = 60
 
           # Times ttl in ms
           TIMES_TTL_MS = TIMES_TTL * 1_000
 
-          private_constant :TIMES_TTL, :TIMES_TTL_MS
+          private_constant :TIMES_TTL, :TIMES_TTL_MS, :SCHEMA_VERSION
 
           def initialize
             @counters = {
@@ -47,7 +52,7 @@ module Karafka
           # @return [Hash] report hash with all the details about consumer operations
           def to_report
             {
-              schema_version: '1.0.0',
+              schema_version: SCHEMA_VERSION,
               type: 'consumer',
               dispatched_at: float_now,
 
@@ -61,7 +66,8 @@ module Karafka
                 memory_total_usage: memory_total_usage,
                 memory_size: memory_size,
                 cpu_count: cpu_count,
-                cpu_usage: cpu_usage
+                cpu_usage: cpu_usage,
+                tags: Karafka::Process.tags
               },
 
               versions: {

data/lib/karafka/web/tracking/reporter.rb

@@ -49,7 +49,8 @@ module Karafka
               {
                 topic: ::Karafka::Web.config.topics.consumers.reports,
                 payload: consumer_report.to_json,
-                key: consumer_report[:process][:name]
+                key: consumer_report[:process][:name],
+                partition: 0
               }
             ]
 

data/lib/karafka/web/ui/app.rb

@@ -71,6 +71,11 @@ module Karafka
               render_response controller.index
             end
           end
+
+          r.get 'status' do
+            controller = Controllers::Status.new(params)
+            render_response controller.show
+          end
         end
       end
     end

data/lib/karafka/web/ui/controllers/status.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Controllers
+        # View that helps understand the status of the Web UI
+        # Many people reported problems understanding the requirements or misconfigured things.
+        # While all of the things are documented, people are lazy. Hence we provide a status
+        # page where we check that everything is as expected and if not, we can provide some
+        # helpful instructions on how to fix the issues.
+        class Status < Base
+          # Displays the Web UI setup status
+          def show
+            @status = Models::Status.new
+
+            respond
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/models/process.rb

@@ -40,6 +40,13 @@ module Karafka
               .sort_by(&:started_at)
           end
 
+          # @return [Array<String>] tags of this consuming process
+          # @note We use direct reference here and a fallback for users that come from the versions
+          #   that did not report tags
+          def tags
+            @hash[:process][:tags] || []
+          end
+
           # @return [Integer] collective lag on this process
           def lag_stored
             consumer_groups