karafka-web 0.6.3 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 12e92e2fc2ceb4c30ab825881074948d32c6041d02b882f074a8d2e94ccae7a4
-  data.tar.gz: 3cc07b90ac8037f9509cf7c42f12fa1ce22e1fdca8d67109ed2aef19faa6c4c4
+  metadata.gz: 651916e28aaa3964d58370954dc66d15afb727f5ca2b4e3f2d8909c41420f329
+  data.tar.gz: c4134a5c8a95a59fe9cd31954333d0663d242291a63718b256b4a95d03a865c0
 SHA512:
-  metadata.gz: 11baf2c9e72a216c8cec688922c76dc5d66eee5ac52e0225b0379b490bc5f5b556c9545482d51b7c88eb8ba9a85eec8b74642e98ce8c4c836d44828412638944
-  data.tar.gz: '010109c8fe106e8ecfa27e7ab75e657868ef01e6c9d5677074d70017a193a22475a8ab2c0938ba545d76602d03564f697b99d0802ae02ff75e7e5aed940a0d54'
+  metadata.gz: 87f1a55595f44d1233a0a88433bf8d245dee7f11e2a1d0d78574eebba3a0471d47588ae7d4a6666674c22d0b35e84042113b37f083900d82a919339192443555
+  data.tar.gz: b618135edf9acdc3130e213fcf8d70d71975f1b813335252d9ef7b94a2f495cd653b891e9628f7bfc63ba8fcdc02dd9c28139508780d4fbbdd5e715e27150641

data/.github/workflows/ci.yml

@@ -28,32 +28,41 @@ jobs:
           - ruby: '3.2'
             coverage: 'true'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4
       - name: Install package dependencies
         run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
+
+      - name: Start Kafka with docker-compose
+        run: |
+          docker-compose up -d
+
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{matrix.ruby}}
+
       - name: Install latest bundler
         run: |
           gem install bundler --no-document
           bundle config set without 'tools benchmarks docs'
+
       - name: Bundle install
         run: |
           bundle config set without development
           bundle install --jobs 4 --retry 3
+
       - name: Run all tests
         env:
           GITHUB_COVERAGE: ${{matrix.coverage}}
-        run: bundle exec rspec
+          KARAFKA_PRO_LICENSE_TOKEN: ${{ secrets.KARAFKA_PRO_LICENSE_TOKEN }}
+        run: bin/rspecs
 
   diffend:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4
         with:
           fetch-depth: 0
 
@@ -74,7 +83,7 @@ jobs:
     strategy:
       fail-fast: false
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4
         with:
           fetch-depth: 0
       - name: Run Coditsu

data/CHANGELOG.md

@@ -1,5 +1,119 @@
 # Karafka Web changelog
 
+## 0.7.0 (2023-09-14)
+- **[Feature]** Introduce graphs.
+- **[Feature]** Introduce historical metrics storage.
+- **[Feature]** Introduce per-topic data exploration in the Explorer.
+- **[Feature]** Introduce per-topic and per-partition most recent message view with live reload.
+- **[Feature]** Introduce a new per-process inspection view called "Details" ti display all process real-time aggregated data.
+- **[Feature]** Introduce `bundle exec karafka-web migrate` that can be used to bootstrap the proper topics and initial data in environments where Karafka Web-UI should be used but is missing the initial setup.
+- **[Feature]** Replace `decrypt` with a pluggable API for deciding which topics data to display.
+- **[Feature]** Make sure, that the karafka server process that is materializing UI states is not processing any data having unsupported (newer) schemas. This state will be also visible in the status page.
+- **[Feature]** Provide ability to reproduce a given message to the same topic partition with all the details from the per message explorer view.
+- **[Feature]** Provide "surrounding" navigation link that allows to view the given message in the context of its surrounding. Useful for debugging of failures where the batch context may be relevant.
+- **[Feature]** Allow for time based lookups per topic partition.
+- **[Feature]** Introduce Offsets Health inspection view for frozen LSO lookups with `lso_threshold` configuration option.
+- [Improvement] Support pattern subscriptions details in the routing view both by displaying the pattern as well as expanded routing details.
+- [Improvement] Collect total number of threads per process for the process details view.
+- [Improvement] Normalize naming of metrics to better reflect what they do (in reports and in the Web UI).
+- [Improvement] Link error reported first and last offset to the explorer.
+- [Improvement] Expand routing details to compensate for nested values in declarative topics.
+- [Improvement] Include last rebalance age in the health view per consumer group.
+- [Improvement] Provide previous / next navigation when viewing particular messages in the explorer.
+- [Improvement] Provide previous / next navigation when viewing particular errors.
+- [Improvement] Link all explorable offsets to the explorer.
+- [Improvement] Extend status page checks to ensure, that it detects a case when Web-UI is not part of `karafka.rb` but still referenced in routes.
+- [Improvement] Extend status page checks to ensure, that it detects a case where there is no initial consumers metrics in Kafka topic.
+- [Improvement] Report Rails version when viewing status page (if Rails used).
+- [Improvement] List Web UI topics names on the status page in the info section.
+- [Improvement] Start versioning the materialized states schemas.
+- [Improvement] Drastically improve the consumers view performance.
+- [Improvement] Ship versioned assets to prevent invalid assets loading due to cache.
+- [Improvement] Use `Cache-Control` to cache all the static assets.
+- [Improvement] Link `counters` counter to jobs page.
+- [Improvement] Include a sticky footer with the most important links and copyrights.
+- [Improvement] Store lag in counters for performance improvement and historical metrics.
+- [Improvement] Introduce in-memory cluster state cached to improve performance.
+- [Improvement] Switch to offset based pagination instead of per-page pagination.
+- [Improvement] Avoid double-reading of watermark offsets for explorer and errors display.
+- [Improvement] When no params needed for a page, do not include empty params.
+- [Improvement] Do not include page when page is 1 in the url.
+- [Improvement] Align descriptions for the status page, to reflect that state check happens for consumers initial state.
+- [Improvement] Report bytesize of raw payload when viewing message in the explorer.
+- [Improvement] Use zlib compression for Karafka Web UI topics reports (all). Reduces space needed from 50 to 91%.
+- [Improvement] Rename lag to lag stored in counters to reflect what it does.
+- [Improvement] Collect both stored lag and lag.
+- [Improvement] Introduce states and metrics schema validation.
+- [Improvement] Prevent locking in sampler for time of OS data aggregation.
+- [Improvement] Collect and report number of messages in particular jobs.
+- [Improvement] Limit segment size for Web topics to ensure, that Web-UI does not drain resources.
+- [Improvement] Introduce cookie based sessions management for future usage.
+- [Improvement] Introduce config validation.
+- [Improvement] Provide flash messages support.
+- [Improvement] Use replication factor of two by default (if not overridden) for Web UI topics when there is more than one broker.
+- [Improvement] Show a warning when replication factor of 1 is used for Web UI topics in production.
+- [Improvement] Collect extra additional metrics useful for hanging transactions detection.
+- [Improvement] Reorganize how the Health view looks.
+- [Improvement] Hide all private Kafka topics by default in the explorer. Configurable with `show_internal_topics` config setting.
+- [Fix] Return 402 status instead of 500 on Pro features that are not available in OSS.
+- [Fix] Fix a case where errors would not be visible without Rails due to the `String#first` usage.
+- [Fix] Fix a case where live-poll would be disabled but would still update data.
+- [Fix] Fix a case where states materializing consumer would update state too often.
+- [Fix] Fix a bug when rapid non-initialized shutdown could mess up the metrics.
+- [Fix] Fix a case where upon multiple rebalances, part of the states materialization could be lost.
+- [Fix] Make sure, that the flushing interval computation division happens with float.
+- [Fix] Fix a case where app client id change could force web-ui to recompute the metrics.
+- [Fix] Make sure, that when re-using same Karafka Web-UI topics as a different up, all states and reports are not recomputed back.
+- [Fix] Fix headers size inconsistency between Health and Routing.
+- [Fix] Fix invalid padding on status page.
+- [Fix] Fix a case where root mounted Karafka Web-UI would not work.
+- [Fix] Fix a case where upon hitting a too high page of consumers we would inform that no consumers are reporting instead of information that this page does not contain any reporting.
+- [Refactor] Limit usage of UI models for data intense computation to speed up states materialization under load.
+- [Refactor] Reorganize pagination engine to support offset based pagination.
+- [Refactor] Use Roda `custom_block_results` plugin for controllers results handling.
+- [Maintenance] Require `karafka` `2.2.0` due to fixes in the Iterator API and routing API extensions.
+
+### Upgrade Notes
+
+This is a **major** release that brings many things to the table.
+
+#### Configuration
+
+Karafka Web UI now relies on Roda session management. Please configure the `ui.sessions.secret` key with a secret value string of at least 64 characters:
+
+```ruby
+# Configure it BEFORE enabling
+Karafka::Web.setup do |config|
+  # REPLACE THIS with your own value. You can use `SecureRandom.hex(64)` to generate it
+  # You may want to set it per ENV
+  config.ui.sessions.secret = 'REPLACE ME! b94b2215cc66371f2c34b7d0c0df1a010f83ca45 REPLACE ME!'
+end
+
+Karafka::Web.enable!
+```
+
+#### Deployment
+
+Because of the reporting schema update and new web-ui topics introduction, it is recommended to:
+
+1. Upgrade the codebase based on the below details.
+2. **Stop** the consumer materializing Web-UI. Unless you are running a Web-UI dedicated consumer as recommended [here](https://karafka.io/docs/Web-UI-Development-vs-Production/), you will have to stop all the consumers. This is **crucial** because of schema changes. `karafka-web` `0.7.0` introduces the detection of schema changes, so this step should not be needed in the future.
+3. Run a migration command: `bundle exec karafka-web migrate` that will create missing states and missing topics. You **need** to run it for each of the environments where you use Karafka Web UI.
+4. Deploy **all** the Karafka consumer processes (`karafka server`).
+5. Deploy the Web update to your web server and check that everything is OK by visiting the status page.
+
+Please note that if you decide to use the updated Web UI with not updated consumers, you may hit a 500 error, or offset-related data may not be displayed correctly.
+
+#### Code and API changes
+
+1. `bundle exec karafka-web install` is now a single-purpose command that should run **only** when installing the Web-UI for the first time.
+2. For creating needed topics and states per environment and during upgrades, please use the newly introduced non-destructive `bundle exec karafka-web migrate`. It will assess changes required and will apply only those.
+3. Is no longer`ui.decrypt` has been replaced with `ui.visibility_filter` API. This API by default also does not decrypt data. To change this behavior, please implement your visibility filter as presented in our documentation.
+4. Karafka Web UI `0.7.0` introduces an in-memory topics cache for some views. This means that rapid topics changes (repartitions/new topics) may be visible up to 5 minutes after those changes.
+3. `ui.decrypt` setting has been replaced with `ui.visibility_filter` API. This API by default also does not decrypt data. To change this behavior, please implement your visibility filter as presented in our documentation.
+4. Karafka Web-UI `0.7.0` introduces an in-memory topics cache for some views. This means that rapid topics changes (repartitions/new topics) may be visible up to 5 minutes after those changes.
+5. Karafka Web UI requires now a new topic called `karafka_consumers_metrics`. If you use strict topic creation and ACL policies, please make sure it exists and that Karafka can both read and write to it.
+
 ## 0.6.3 (2023-07-22)
 - [Fix] Remove files from 0.7.0 accidentally added to the release.
 
@@ -37,7 +151,7 @@
 - [Refactor] Remove not used and redundant partials.
 - [Maintenance] Require `karafka` `2.1.4` due to fixes in metrics usage for workless flows.
 
-### Upgrade notes
+### Upgrade Notes
 
 Because of the reporting schema update, it is recommended to:
 
@@ -85,7 +199,7 @@ Please make sure **not** to do it for the default `Karafka.producer` because it
 - [Fix] Fix misspelling of word `committed`.
 - [Fix] Shutdown and revocation jobs statistics extraction crashes when idle initialized without messages (#53)
 
-### Upgrade notes
+### Upgrade Notes
 
 Because of the reporting schema change, it is recommended to:
 
@@ -106,7 +220,7 @@ Please note that if you decide to use the updated Web UI with not updated consum
 - [Fix] Add missing support for using multiple subscription groups within a single consumer group.
 - [Fix] Mask SASL credentials in topic routing view (#46)
 
-### Upgrade notes
+### Upgrade Notes
 
 Because of the reporting schema change, it is recommended to:
 
@@ -120,7 +234,7 @@ Please note that if you decide to use the updated Web UI with not updated consum
 - [Fix] Fix display of compacted messages placeholders for offsets lower than low watermark.
 - [Fix] Fix invalid pagination per page count.
 
-### Upgrade notes
+### Upgrade Notes
 
 If upgrading from `0.3.0`, nothing.
 
@@ -145,7 +259,7 @@ If upgrading from lower, please follow `0.3.0` upgrade procedure.
 - [Maintenance] Remove compatibility fallbacks for job and process tags (#1342)
 - [Maintenance] Extract base sampler for tracking and web.
 
-### Upgrade notes
+### Upgrade Notes
 
 Because of the removal of compatibility fallbacks for some metrics fetches, it is recommended to:
 

data/Gemfile

@@ -9,6 +9,7 @@ gemspec
 group :test do
   gem 'byebug'
   gem 'factory_bot'
+  gem 'rack-test'
   gem 'rspec'
   gem 'simplecov'
 end

data/Gemfile.lock

@@ -1,17 +1,17 @@
 PATH
   remote: .
   specs:
-    karafka-web (0.6.3)
+    karafka-web (0.7.0)
       erubi (~> 1.4)
-      karafka (>= 2.1.4, < 3.0.0)
-      karafka-core (>= 2.0.13, < 3.0.0)
-      roda (~> 3.68, >= 3.68)
+      karafka (>= 2.2.3, < 3.0.0)
+      karafka-core (>= 2.2.2, < 3.0.0)
+      roda (~> 3.68, >= 3.69)
       tilt (~> 2.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (7.0.4.3)
+    activesupport (7.0.8)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -21,31 +21,33 @@ GEM
     diff-lcs (1.5.0)
     docile (1.4.0)
     erubi (1.12.0)
-    factory_bot (6.2.1)
+    factory_bot (6.3.0)
       activesupport (>= 5.0.0)
     ffi (1.15.5)
-    i18n (1.13.0)
+    i18n (1.14.1)
       concurrent-ruby (~> 1.0)
-    karafka (2.1.4)
-      karafka-core (>= 2.0.13, < 3.0.0)
+    karafka (2.2.3)
+      karafka-core (>= 2.2.2, < 2.3.0)
       thor (>= 0.20)
-      waterdrop (>= 2.5.3, < 3.0.0)
+      waterdrop (>= 2.6.6, < 3.0.0)
       zeitwerk (~> 2.3)
-    karafka-core (2.0.13)
+    karafka-core (2.2.2)
       concurrent-ruby (>= 1.1)
-      karafka-rdkafka (>= 0.12.3)
-    karafka-rdkafka (0.12.3)
+      karafka-rdkafka (>= 0.13.1, < 0.14.0)
+    karafka-rdkafka (0.13.4)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    mini_portile2 (2.8.2)
-    minitest (5.18.0)
-    rack (3.0.7)
+    mini_portile2 (2.8.4)
+    minitest (5.20.0)
+    rack (3.0.8)
+    rack-test (2.1.0)
+      rack (>= 1.3)
     rackup (0.2.3)
       rack (>= 3.0.0.beta1)
       webrick
     rake (13.0.6)
-    roda (3.68.0)
+    roda (3.71.0)
       rack
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
@@ -56,10 +58,10 @@ GEM
     rspec-expectations (3.12.3)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
-    rspec-mocks (3.12.5)
+    rspec-mocks (3.12.6)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
-    rspec-support (3.12.0)
+    rspec-support (3.12.1)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
@@ -67,14 +69,14 @@ GEM
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
     thor (1.2.2)
-    tilt (2.1.0)
+    tilt (2.2.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.5.3)
-      karafka-core (>= 2.0.13, < 3.0.0)
+    waterdrop (2.6.7)
+      karafka-core (>= 2.1.1, < 3.0.0)
       zeitwerk (~> 2.3)
     webrick (1.8.1)
-    zeitwerk (2.6.8)
+    zeitwerk (2.6.11)
 
 PLATFORMS
   x86_64-linux
@@ -83,9 +85,10 @@ DEPENDENCIES
   byebug
   factory_bot
   karafka-web!
+  rack-test
   rackup (~> 0.2)
   rspec
   simplecov
 
 BUNDLED WITH
-   2.4.12
+   2.4.19

data/README.md

@@ -18,6 +18,8 @@ Karafka Web UI documentation is part of the Karafka framework documentation and
 
 The Enhanced Web UI, aside from all the features from the OSS version, also offers additional features and capabilities not available in the free version, making it a better option for those looking for more robust monitoring and management capabilities for their Karafka applications. Some of the key benefits of the Enhanced Web UI version include the following:
 
+- Real-time and historical processing and utilization metrics.
+- Real-time topics lag awareness.
 - Enhanced consumers utilization metrics providing much better insights into processes resources utilization.
 - Consumer process inspection to quickly analyze the state of a given consuming process.
 - Consumer jobs inspection to view currently running jobs on a per-process basis.

data/bin/rspecs

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -e
+
+SPECS_TYPE=regular bundle exec rspec --exclude-pattern "**/pro/**/*_spec.rb"
+SPECS_TYPE=pro bundle exec rspec --pattern "**/pro/**/*_spec.rb"

data/certs/cert_chain.pem

@@ -1,26 +1,26 @@
 -----BEGIN CERTIFICATE-----
 MIIEcDCCAtigAwIBAgIBATANBgkqhkiG9w0BAQsFADA/MRAwDgYDVQQDDAdjb250
 YWN0MRcwFQYKCZImiZPyLGQBGRYHa2FyYWZrYTESMBAGCgmSJomT8ixkARkWAmlv
-MB4XDTIyMDgxOTE3MjEzN1oXDTIzMDgxOTE3MjEzN1owPzEQMA4GA1UEAwwHY29u
+MB4XDTIzMDgyMTA3MjU1NFoXDTI0MDgyMDA3MjU1NFowPzEQMA4GA1UEAwwHY29u
 dGFjdDEXMBUGCgmSJomT8ixkARkWB2thcmFma2ExEjAQBgoJkiaJk/IsZAEZFgJp
-bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAODzeO3L6lxdATzMHKNW
-jFA/GGunoPuylO/BMzy8RiQHh7VIvysAKs0tHhTx3g2D0STDpF+hcQcPELFikiT2
-F+1wOHj/SsrK7VKqfA8+gq04hKc5sQoX2Egf9k3V0YJ3eZ6R/koHkQ8A0TVt0w6F
-ZQckoV4MqnEAx0g/FZN3mnHTlJ3VFLSBqJEIe+S6FZMl92mSv+hTrlUG8VaYxSfN
-lTCvnKk284F6QZq5XIENLRmcDd/3aPBLnLwNnyMyhB+6gK8cUO+CFlDO5tjo/aBA
-rUnl++wGG0JooF1ed0v+evOn9KoMBG6rHewcf79qJbVOscbD8qSAmo+sCXtcFryr
-KRMTB8gNbowJkFRJDEe8tfRy11u1fYzFg/qNO82FJd62rKAw2wN0C29yCeQOPRb1
-Cw9Y4ZwK9VFNEcV9L+3pHTHn2XfuZHtDaG198VweiF6raFO4yiEYccodH/USP0L5
-cbcCFtmu/4HDSxL1ByQXO84A0ybJuk3/+aPUSXe9C9U8fwIDAQABo3cwdTAJBgNV
-HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQUSlcEakb7gfn/5E2WY6z73BF/
-iZkwHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
-bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEA1aS+E7RXJ1w9g9mJ
-G0NzFxe64OEuENosNlvYQCbRKGCXAU1qqelYkBQHseRgRKxLICrnypRo9IEobyHa
-vDnJ4r7Tsb34dleqQW2zY/obG+cia3Ym2JsegXWF7dDOzCXJ4FN8MFoT2jHlqLLw
-yrap0YO5zx0GSQ0Dwy8h2n2v2vanMEeCx7iNm3ERgR5WuN5sjzWoz2A/JLEEcK0C
-EnAGKCWAd1fuG8IemDjT1edsd5FyYR4bIX0m+99oDuFZyPiiIbalmyYiSBBp59Yb
-Q0P8zeBi4OfwCZNcxqz0KONmw9JLNv6DgyEAH5xe/4JzhMEgvIRiPj0pHfA7oqQF
-KUNqvD1KlxbEC+bZfE5IZhnqYLdld/Ksqd22FI1RBhiS1Ejfsj99LVIm9cBuZEY2
-Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
-MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
+bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAOuZpyQKEwsTG9plLat7
+8bUaNuNBEnouTsNMr6X+XTgvyrAxTuocdsyP1sNCjdS1B8RiiDH1/Nt9qpvlBWon
+sdJ1SYhaWNVfqiYStTDnCx3PRMmHRdD4KqUWKpN6VpZ1O/Zu+9Mw0COmvXgZuuO9
+wMSJkXRo6dTCfMedLAIxjMeBIxtoLR2e6Jm6MR8+8WYYVWrO9kSOOt5eKQLBY7aK
+b/Dc40EcJKPg3Z30Pia1M9ZyRlb6SOj6SKpHRqc7vbVQxjEw6Jjal1lZ49m3YZMd
+ArMAs9lQZNdSw5/UX6HWWURLowg6k10RnhTUtYyzO9BFev0JFJftHnmuk8vtb+SD
+5VPmjFXg2VOcw0B7FtG75Vackk8QKfgVe3nSPhVpew2CSPlbJzH80wChbr19+e3+
+YGr1tOiaJrL6c+PNmb0F31NXMKpj/r+n15HwlTMRxQrzFcgjBlxf2XFGnPQXHhBm
+kp1OFnEq4GG9sON4glRldkwzi/f/fGcZmo5fm3d+0ZdNgwIDAQABo3cwdTAJBgNV
+HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQUPVH5+dLA80A1kJ2Uz5iGwfOa
+1+swHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
+bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEAnpa0jcN7JzREHMTQ
+bfZ+xcvlrzuROMY6A3zIZmQgbnoZZNuX4cMRrT1p1HuwXpxdpHPw7dDjYqWw3+1h
+3mXLeMuk7amjQpYoSWU/OIZMhIsARra22UN8qkkUlUj3AwTaChVKN/bPJOM2DzfU
+kz9vUgLeYYFfQbZqeI6SsM7ltilRV4W8D9yNUQQvOxCFxtLOetJ00fC/E7zMUzbK
+IBwYFQYsbI6XQzgAIPW6nGSYKgRhkfpmquXSNKZRIQ4V6bFrufa+DzD0bt2ZA3ah
+fMmJguyb5L2Gf1zpDXzFSPMG7YQFLzwYz1zZZvOU7/UCpQsHpID/YxqDp4+Dgb+Y
+qma0whX8UG/gXFV2pYWpYOfpatvahwi+A1TwPQsuZwkkhi1OyF1At3RY+hjSXyav
+AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
+msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
 -----END CERTIFICATE-----

data/docker-compose.yml

@@ -0,0 +1,22 @@
+version: '2'
+services:
+  zookeeper:
+    container_name: karafka_web_21_zookeeper
+    image: wurstmeister/zookeeper
+    restart: on-failure
+    ports:
+      - '2181:2181'
+
+  kafka:
+    container_name: karafka_web_21_kafka
+    image: wurstmeister/kafka
+    ports:
+      - '9092:9092'
+    environment:
+      KAFKA_ADVERTISED_HOST_NAME: localhost
+      KAFKA_ADVERTISED_PORT: 9092
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    restart: on-failure

data/karafka-web.gemspec

@@ -17,9 +17,9 @@ Gem::Specification.new do |spec|
   spec.licenses    = %w[LGPL-3.0 Commercial]
 
   spec.add_dependency 'erubi', '~> 1.4'
-  spec.add_dependency 'karafka', '>= 2.1.4', '< 3.0.0'
-  spec.add_dependency 'karafka-core', '>= 2.0.13', '< 3.0.0'
-  spec.add_dependency 'roda', '~> 3.68', '>= 3.68'
+  spec.add_dependency 'karafka', '>= 2.2.3', '< 3.0.0'
+  spec.add_dependency 'karafka-core', '>= 2.2.2', '< 3.0.0'
+  spec.add_dependency 'roda', '~> 3.68', '>= 3.69'
   spec.add_dependency 'tilt', '~> 2.0'
 
   spec.add_development_dependency 'rackup', '~> 0.2'

data/lib/karafka/web/app.rb

@@ -8,8 +8,12 @@ module Karafka
         # @param env [Hash] Rack env
         # @param block [Proc] Rack block
         def call(env, &block)
-          handler = Karafka.pro? ? Ui::Pro::App : Ui::App
-          handler.call(env, &block)
+          engine.call(env, &block)
+        end
+
+        # @return [Class] regular or pro Web engine
+        def engine
+          ::Karafka.pro? ? Ui::Pro::App : Ui::App
         end
       end
     end

data/lib/karafka/web/cli.rb

@@ -6,74 +6,78 @@ module Karafka
     class Cli < Thor
       include ::Karafka::Helpers::Colorize
 
-      # Code that is needed in the `karafka.rb` to connect Web UI to Karafka
-      ENABLER_CODE = 'Karafka::Web.enable!'
-
-      private_constant :ENABLER_CODE
-
       package_name 'Karafka Web'
 
       desc 'install', 'Installs the Web UI'
       method_option(
         :replication_factor,
         desc: 'Replication factor for created topics',
-        default: 1,
+        default: false,
+        check_default_type: false,
         type: :numeric
       )
-      # Installs Karafka Web
+      # Installs Karafka Web. Creates all needed topics, populates the data and adds the needed
+      # code to `karafka.rb`.
       def install
-        puts
-        puts 'Installing Karafka Web UI...'
-        puts
-        puts 'Creating necessary topics and populating state data...'
-
-        Karafka::Web::Installer.new.bootstrap!(replication_factor: options[:replication_factor])
-
-        puts 'Updating the Karafka boot file...'
-
-        if File.read(Karafka.boot_file).include?(ENABLER_CODE)
-          puts "Web UI #{green('already')} installed."
-        else
-          File.open(Karafka.boot_file, 'a') do |f|
-            f << "\n#{ENABLER_CODE}\n"
-          end
-        end
+        Karafka::Web::Installer.new.install(
+          replication_factor: compute_replication_factor(options[:replication_factor])
+        )
+      end
 
-        puts
-        puts("Installation #{green('completed')}. Have fun!")
-        puts
+      desc 'migrate', 'Creates necessary topics if not present and populates state data'
+      method_option(
+        :replication_factor,
+        desc: 'Replication factor for created topics',
+        default: false,
+        check_default_type: false,
+        type: :numeric
+      )
+      # Creates new topics (if any) and populates missing data.
+      # It does **not** remove topics and will not populate data if it is already there.
+      #
+      # Useful in two scenarios:
+      #   1. When setting up Web-UI in a new environment, so the Web-UI has the proper initial
+      #      state.
+      #   2. When upgrading Web-UI in-between versions that would require extra topics and/or extra
+      #      states populated.
+      def migrate
+        Karafka::Web::Installer.new.migrate(
+          replication_factor: compute_replication_factor(options[:replication_factor])
+        )
       end
 
       desc 'reset', 'Resets the Web UI by removing all the Web topics and creating them again'
-      # Resets Karafka Web
+      method_option(
+        :replication_factor,
+        desc: 'Replication factor for created topics',
+        default: false,
+        check_default_type: false,
+        type: :numeric
+      )
+      # Resets Karafka Web. Removes the topics, creates them again and populates the initial state
+      # again. This is useful in case the Web-UI metrics or anything else got corrupted.
       def reset
-        puts
-        puts 'Resetting Karafka Web UI...'
-        Karafka::Web::Installer.new.reset!
-        puts
-        puts("Resetting #{green('completed')}. Have fun!")
-        puts
+        Karafka::Web::Installer.new.reset(
+          replication_factor: compute_replication_factor(options[:replication_factor])
+        )
       end
 
       desc 'uninstall', 'Removes all the Web UI topics and the enabled code'
       # Uninstalls Karafka Web
       def uninstall
-        puts
-        puts 'Uninstalling Karafka Web UI...'
-        Karafka::Web::Installer.new.uninstall!
-
-        puts 'Updating the Karafka boot file...'
-
-        karafka_rb = File.readlines(Karafka.boot_file)
-        if karafka_rb.any? { |line| line.include?(ENABLER_CODE) }
-          karafka_rb.delete_if { |line| line.include?(ENABLER_CODE) }
+        Karafka::Web::Installer.new.uninstall
+      end
 
-          File.write(Karafka.boot_file, karafka_rb.join)
-        end
+      private
 
-        puts
-        puts("Uninstalling #{green('completed')}. Goodbye!")
-        puts
+      # Takes the CLI user provided replication factor but if not present, uses the brokers count
+      # to decide. For non-dev clusters (with one broker) we usually want to have replication of
+      # two, just to have some redundancy.
+      # @param cli_replication_factor [Integer, false] user requested replication factor or false
+      #   if we are supposed to compute the factor automatically
+      # @return [Integer] replication factor for Karafka Web UI topics
+      def compute_replication_factor(cli_replication_factor)
+        cli_replication_factor || Ui::Models::ClusterInfo.fetch.brokers.size > 1 ? 2 : 1
       end
     end
   end