karafka-web 0.1.1 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4883680e156d1147dddceb19b1a8ed0977c1930a4dbdb5bffab7adf836d2a95f
-  data.tar.gz: 21cc44c9390cc2b098bd6261cb999f5143a560d5d774dda786c6b294e865e5f2
+  metadata.gz: 3a70fc5cb35d0e6e60a348e3efe91113b54d0c62310b7c3af4e7584aa70cfb70
+  data.tar.gz: 43a96b7e7a4622721dbc6ba144852335aecbfe907049bc47ab563ea07f40eb54
 SHA512:
-  metadata.gz: 59e16f47b0085db2a0e4b91ec76c8a3b6d51dbdf806840e1ac9179777f587340fba046a8c4f33df879e519e34d0737be201a736d595fbf330c9e2f5e2e4eb1fd
-  data.tar.gz: a3896ace699760b73c2674bb4155d9255309b371e52485e9ebc3757ab18ac5fa1708f266831b463d112b0c3630886114a560cecdc0f505950a8bd58d266efbcf
+  metadata.gz: db0b153569ca93f1164a851216e0f224acc56e6a7533a35a86a44f6f2f8bffc1630e5ccf7c2e53a14455206401c45b59b1728eb266c59360be8f952794d3011b
+  data.tar.gz: 794c77edb27f7f9a397c3de891b146f9020653b84bad46e1f1b537a1d547ffc793211e7bd5ffb0f426c0b8fa9b6359230bb3fe3ee8fa5ec668cb0e1d39782d5a

data/.github/workflows/ci.yml

@@ -13,6 +13,41 @@ env:
   BUNDLE_JOBS: 4
 
 jobs:
+  specs:
+    runs-on: ubuntu-latest
+    needs: diffend
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby:
+          - '3.2'
+          - '3.1'
+          - '3.0'
+          - '2.7'
+        include:
+          - ruby: '3.2'
+            coverage: 'true'
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install package dependencies
+        run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{matrix.ruby}}
+      - name: Install latest bundler
+        run: |
+          gem install bundler --no-document
+          bundle config set without 'tools benchmarks docs'
+      - name: Bundle install
+        run: |
+          bundle config set without development
+          bundle install --jobs 4 --retry 3
+      - name: Run all tests
+        env:
+          GITHUB_COVERAGE: ${{matrix.coverage}}
+        run: bundle exec rspec
+
   diffend:
     runs-on: ubuntu-latest
     strategy:

data/.rspec

@@ -0,0 +1 @@
+--require spec_helper

data/.ruby-version

@@ -1 +1 @@
-3.2.0
+3.2.1

data/CHANGELOG.md

@@ -1,9 +1,20 @@
 # Karafka Web changelog
 
+## 0.1.3 (2023-02-14)
+- Skip topics creation if web topics already exist (do not raise error)
+- Support ability to provide replication factor in the install command
+- Provide ability to reset the state with a `reset` command. It will remove and re-create the topics.
+- Provide ability to uninstall the web via the CLI `uninstall` command
+- Remove the `Karafka::Web.bootstrap!` method as the install should happen via `bundle exec karafka-web install`
+
+## 0.1.2 (2023-02-10)
+- Provide more comprehensive info when lag stored and stored offset are not available.
+- Setup rspec scaffold.
+
 ## 0.1.1 (2023-01-30)
 - Rename `Karafka::Web.bootstrap_topics!` to `Karafka::Web.bootstrap!` and expand it with the zero state injection.
 - Require Karafka `2.0.28` due to some instrumentation fixes.
 - Provide an auto-installer under the `bundle exec karafka-web install` command.
 
 ## 0.1.0
-- Initial code of the Web and Web Pro
+- Initial code of the Web and Web Pro.

data/Gemfile

@@ -5,3 +5,9 @@ source 'https://rubygems.org'
 plugin 'diffend'
 
 gemspec
+
+group :test do
+  gem 'byebug'
+  gem 'rspec'
+  gem 'simplecov'
+end

data/Gemfile.lock

@@ -1,24 +1,28 @@
 PATH
   remote: .
   specs:
-    karafka-web (0.1.1)
+    karafka-web (0.1.3)
       erubi (~> 1.4)
       karafka (>= 2.0.29, < 3.0.0)
+      karafka-core (>= 2.0.10, < 3.0.0)
       roda (~> 3.63)
       tilt (~> 2.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
+    byebug (11.1.3)
     concurrent-ruby (1.2.0)
+    diff-lcs (1.5.0)
+    docile (1.4.0)
     erubi (1.12.0)
     ffi (1.15.5)
-    karafka (2.0.29)
+    karafka (2.0.30)
       karafka-core (>= 2.0.9, < 3.0.0)
       thor (>= 0.20)
       waterdrop (>= 2.4.10, < 3.0.0)
       zeitwerk (~> 2.3)
-    karafka-core (2.0.9)
+    karafka-core (2.0.10)
       concurrent-ruby (>= 1.1)
       karafka-rdkafka (>= 0.12)
     karafka-rdkafka (0.12.0)
@@ -33,20 +37,43 @@ GEM
     rake (13.0.6)
     roda (3.64.0)
       rack
+    rspec (3.12.0)
+      rspec-core (~> 3.12.0)
+      rspec-expectations (~> 3.12.0)
+      rspec-mocks (~> 3.12.0)
+    rspec-core (3.12.1)
+      rspec-support (~> 3.12.0)
+    rspec-expectations (3.12.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.12.0)
+    rspec-mocks (3.12.3)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.12.0)
+    rspec-support (3.12.0)
+    simplecov (0.22.0)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.4)
     thor (1.2.1)
     tilt (2.0.11)
     waterdrop (2.4.10)
       karafka-core (>= 2.0.9, < 3.0.0)
       zeitwerk (~> 2.3)
     webrick (1.8.1)
-    zeitwerk (2.6.6)
+    zeitwerk (2.6.7)
 
 PLATFORMS
+  arm64-darwin-21
   x86_64-linux
 
 DEPENDENCIES
+  byebug
   karafka-web!
   rackup (~> 0.2)
+  rspec
+  simplecov
 
 BUNDLED WITH
-   2.4.5
+   2.4.6

data/bin/karafka-web

@@ -4,30 +4,5 @@ require 'karafka'
 require 'karafka/web'
 
 ::Karafka::Cli::Base.load
-include ::Karafka::Helpers::Colorize
 
-case ARGV[0]
-when 'install'
-  puts
-  puts 'Installing Karafka Web UI...'
-  puts
-  puts 'Creating necessary topics and populating state data...'
-  Karafka::Web.bootstrap!
-  puts 'Updating the Karafka boot file...'
-
-  enabler = 'Karafka::Web.enable!'
-
-  if File.read(Karafka.boot_file).include?(enabler)
-    puts "Web UI #{green('already')} installed."
-  else
-    File.open(Karafka.boot_file, 'a') do |f|
-      f << "\n#{enabler}\n"
-    end
-  end
-
-  puts
-  puts("Installation #{green('completed')}. Have fun!")
-  puts
-else
-  raise NotImplementedError, "#{ARGV[0]} is not supported"
-end
+::Karafka::Web::Cli.start

data/karafka-web.gemspec

@@ -18,6 +18,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'erubi', '~> 1.4'
   spec.add_dependency 'karafka', '>= 2.0.29', '< 3.0.0'
+  spec.add_dependency 'karafka-core', '>= 2.0.10', '< 3.0.0'
   spec.add_dependency 'roda', '~> 3.63'
   spec.add_dependency 'tilt', '~> 2.0'
 

data/lib/karafka/web/cli.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    # Karafka itself depends on Thor, so we can use it
+    class Cli < Thor
+      include ::Karafka::Helpers::Colorize
+
+      # Code that is needed in the `karafka.rb` to connect Web UI to Karafka
+      ENABLER_CODE = 'Karafka::Web.enable!'
+
+      private_constant :ENABLER_CODE
+
+      package_name 'Karafka Web'
+
+      desc 'install', 'Installs the Web UI'
+      method_option(
+        :replication_factor,
+        desc: 'Replication factor for created topics',
+        default: 1,
+        type: :numeric
+      )
+      # Installs Karafka Web
+      def install
+        puts
+        puts 'Installing Karafka Web UI...'
+        puts
+        puts 'Creating necessary topics and populating state data...'
+
+        Karafka::Web::Installer.new.bootstrap!(replication_factor: options[:replication_factor])
+
+        puts 'Updating the Karafka boot file...'
+
+        if File.read(Karafka.boot_file).include?(ENABLER_CODE)
+          puts "Web UI #{green('already')} installed."
+        else
+          File.open(Karafka.boot_file, 'a') do |f|
+            f << "\n#{ENABLER_CODE}\n"
+          end
+        end
+
+        puts
+        puts("Installation #{green('completed')}. Have fun!")
+        puts
+      end
+
+      desc 'reset', 'Resets the Web UI by removing all the Web topics and creating them again'
+      # Resets Karafka Web
+      def reset
+        puts
+        puts 'Resetting Karafka Web UI...'
+        Karafka::Web::Installer.new.reset!
+        puts
+        puts("Resetting #{green('completed')}. Have fun!")
+        puts
+      end
+
+      desc 'uninstall', 'Removes all the Web UI topics and the enabled code'
+      # Uninstalls Karafka Web
+      def uninstall
+        puts
+        puts 'Uninstalling Karafka Web UI...'
+        Karafka::Web::Installer.new.uninstall!
+
+        puts 'Updating the Karafka boot file...'
+
+        karafka_rb = File.readlines(Karafka.boot_file)
+        if karafka_rb.any? { |line| line.include?(ENABLER_CODE) }
+          karafka_rb.delete_if { |line| line.include?(ENABLER_CODE) }
+
+          File.write(Karafka.boot_file, karafka_rb.join)
+        end
+
+        puts
+        puts("Uninstalling #{green('completed')}. Goodbye!")
+        puts
+      end
+    end
+  end
+end

data/lib/karafka/web/installer.rb

@@ -13,6 +13,30 @@ module Karafka
         bootstrap_state!
       end
 
+      # Removes all the Karafka topics and creates them again with the same replication factor
+      def reset!
+        states_topic = ::Karafka::Web.config.topics.consumers.states
+        replication_factor = ::Karafka::Admin
+                             .cluster_info
+                             .topics
+                             .find { |topic| topic[:topic_name] == states_topic }
+                             .fetch(:partitions)
+                             .first
+                             .fetch(:replica_count)
+
+        uninstall!
+        bootstrap!(replication_factor: replication_factor)
+      end
+
+      # Removes all the Karafka Web topics
+      def uninstall!
+        [
+          ::Karafka::Web.config.topics.consumers.states,
+          ::Karafka::Web.config.topics.consumers.reports,
+          ::Karafka::Web.config.topics.errors
+        ].each { |topic_name| ::Karafka::Admin.delete_topic(topic_name) }
+      end
+
       # Adds the extra needed consumer group, topics and routes for Web UI to be able to operate
       def enable!
         ::Karafka::App.routes.draw do
@@ -61,36 +85,49 @@ module Karafka
       #
       # @param replication_factor [Integer]
       def bootstrap_topics!(replication_factor = 1)
-        # This topic needs to have one partition
-        ::Karafka::Admin.create_topic(
-          ::Karafka::Web.config.topics.consumers.states,
-          1,
-          replication_factor,
-          # We care only about the most recent state, previous are irrelevant
-          { 'cleanup.policy': 'compact' }
-        )
+        existing_topics = ::Karafka::Admin.cluster_info.topics.map { |topic| topic[:topic_name] }
 
-        # This topic needs to have one partition
-        ::Karafka::Admin.create_topic(
-          ::Karafka::Web.config.topics.consumers.reports,
-          1,
-          replication_factor,
-          # We do not need to to store this data for longer than 7 days as this data is only used
-          # to materialize the end states
-          # On the other hand we do not want to have it really short-living because in case of a
-          # consumer crash, we may want to use this info to catch up and backfill the state
-          { 'retention.ms': 7 * 24 * 60 * 60 * 1_000 }
-        )
+        consumers_states_topic = ::Karafka::Web.config.topics.consumers.states
+        consumers_reports_topic = ::Karafka::Web.config.topics.consumers.reports
+        errors_topic = ::Karafka::Web.config.topics.errors
 
-        # All the errors will be dispatched here
-        # This topic can have multiple partitions but we go with one by default. A single Ruby
-        # process should not crash that often and if there is an expectation of a higher volume
-        # of errors, this can be changed by the end user
-        ::Karafka::Admin.create_topic(
-          ::Karafka::Web.config.topics.errors,
-          1,
-          replication_factor
-        )
+        # Create only if needed
+        unless existing_topics.include?(consumers_states_topic)
+          # This topic needs to have one partition
+          ::Karafka::Admin.create_topic(
+            consumers_states_topic,
+            1,
+            replication_factor,
+            # We care only about the most recent state, previous are irrelevant
+            { 'cleanup.policy': 'compact' }
+          )
+        end
+
+        unless existing_topics.include?(consumers_reports_topic)
+          # This topic needs to have one partition
+          ::Karafka::Admin.create_topic(
+            consumers_reports_topic,
+            1,
+            replication_factor,
+            # We do not need to to store this data for longer than 7 days as this data is only used
+            # to materialize the end states
+            # On the other hand we do not want to have it really short-living because in case of a
+            # consumer crash, we may want to use this info to catch up and backfill the state
+            { 'retention.ms': 7 * 24 * 60 * 60 * 1_000 }
+          )
+        end
+
+        unless existing_topics.include?(errors_topic)
+          # All the errors will be dispatched here
+          # This topic can have multiple partitions but we go with one by default. A single Ruby
+          # process should not crash that often and if there is an expectation of a higher volume
+          # of errors, this can be changed by the end user
+          ::Karafka::Admin.create_topic(
+            errors_topic,
+            1,
+            replication_factor
+          )
+        end
 
         bootstrap_state!
       end

data/lib/karafka/web/ui/pro/views/health/_partition.erb

@@ -5,7 +5,15 @@
   <td>
     <%= partition_id %>
   </td>
-  <td><%= details[:lag_stored] %></td>
+  <td>
+    <% if details[:lag_stored].to_i < 0 %>
+      <span class="badge bg-secondary" title="Not available until first offset commit">
+        N/A
+      </span>
+    <% else %>
+      <%= details[:lag_stored] %>
+    <% end %>
+  </td>
   <td>
     <span class="badge <%= lag_trend_bg(details[:lag_stored_d]) %>">
       <%= details[:lag_stored_d] %>
@@ -15,7 +23,13 @@
     <%= details[:committed_offset] %>
   </td>
   <td>
-    <%= details[:stored_offset] %>
+    <% if details[:stored_offset].to_i < 0 %>
+      <span class="badge bg-secondary" title="Not available until first offset commit">
+        N/A
+      </span>
+    <% else %>
+      <%= details[:stored_offset] %>
+    <% end %>
   </td>
   <td>
     <span class="badge <%= kafka_state_bg(details[:fetch_state]) %> mt-1 mb-1">

data/lib/karafka/web/version.rb

@@ -3,6 +3,6 @@
 module Karafka
   module Web
     # Current gem version
-    VERSION = '0.1.1'
+    VERSION = '0.1.3'
   end
 end

data/lib/karafka/web.rb

@@ -27,12 +27,6 @@ module Karafka
         Config.config
       end
 
-      # Creates all the needed topics for the admin UI to work and populates initial (empty) set
-      # of data, so the UI will work even when no Karafka servers are started
-      def bootstrap!
-        Installer.new.bootstrap!
-      end
-
       # Activates all the needed routing and sets up listener, etc
       # This needs to run **after** the optional configuration of the web component
       def enable!

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-web
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.3
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2023-01-30 00:00:00.000000000 Z
+date: 2023-02-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: erubi
@@ -71,6 +71,26 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
+- !ruby/object:Gem::Dependency
+  name: karafka-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.10
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.10
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
 - !ruby/object:Gem::Dependency
   name: roda
   requirement: !ruby/object:Gem::Requirement
@@ -128,6 +148,7 @@ files:
 - ".github/ISSUE_TEMPLATE/feature_request.md"
 - ".github/workflows/ci.yml"
 - ".gitignore"
+- ".rspec"
 - ".ruby-gemset"
 - ".ruby-version"
 - CHANGELOG.md
@@ -142,6 +163,7 @@ files:
 - karafka-web.gemspec
 - lib/karafka/web.rb
 - lib/karafka/web/app.rb
+- lib/karafka/web/cli.rb
 - lib/karafka/web/config.rb
 - lib/karafka/web/deserializer.rb
 - lib/karafka/web/errors.rb
@@ -321,7 +343,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.1
+rubygems_version: 3.4.6
 signing_key: 
 specification_version: 4
 summary: Karafka ecosystem Web UI interface