karafka-web 0.9.1 → 0.10.0.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ce075ec71d155adbf35706c5bfa9beab07a707970e53ad9b0f7e6b2fc2196976
-  data.tar.gz: 747b28ece60db10c34cabfcaa70edcc8d7c0aaa63440c6b3dba6b108a35a0435
+  metadata.gz: d03bd99016dad674c04a734df15c1dd42c52d869ea7c5e1c5158f1b2ad5d31f9
+  data.tar.gz: 830c8e38a82bee5876519a5517d321ac1c6f9bb0f197bab0644bb65ae8ba7cba
 SHA512:
-  metadata.gz: 39d4e856b439c83acc929cf17432c52abbbf4c2a063897b0595ed360321fc0cafb62527e5d99f35710d9d731accbd12aaca9a8c0a6c97ead1d5302df1fd400dd
-  data.tar.gz: fc8eb8d769bc3dd9edeca90fedb05c83ce382c562f26b261290a352f04f09ffc029bdb21627bc6d305924b5673ed8465631d6a60d58c486575b24055f7255c9e
+  metadata.gz: cce9fba1e6f3f3503a8893d74711c9de78e7ff54b3f7c07a6e8add2aa7d8adf3b61411a051f08c0c5b31550cb1533b2a80c6b903e4ba0e42a022895160146616
+  data.tar.gz: 2e823dd6b69bc5be01f9c9b330b4691af8e278d1a306fde6c9d762eb6d282a0b2defefeb192aa705b17dddc5ba8cda11ea3bbb5128ea9ecacf9d2e9ad89620bc

data/.github/workflows/ci.yml

@@ -22,6 +22,7 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
+          - '3.4.0-preview1'
           - '3.3'
           - '3.2'
           - '3.1'
@@ -101,3 +102,32 @@ jobs:
           CODITSU_API_KEY: ${{ secrets.CODITSU_API_KEY }}
           CODITSU_API_SECRET: ${{ secrets.CODITSU_API_SECRET }}
         run: \curl -sSL https://api.coditsu.io/run/ci | bash
+
+
+  assets:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '17'
+
+      - name: Cache node modules
+        uses: actions/cache@v4
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+
+      - name: Install dependencies
+        run: npm install
+
+      - name: Build development assets
+        run: ./bin/build_assets
+
+      - name: Build release assets
+        run: KARAFKA_RELEASE=true ./bin/build_assets

data/.gitignore

@@ -38,8 +38,10 @@ db/*.sqlite3
 /capybara-*.html
 /gems
 /specifications
+/node_modules
 rerun.txt
 pickle-email-*.html
+tailwind.min.css
 
 # If you find yourself ignoring temporary files generated by your text editor
 # or operating system, you probably want to add a global ignore instead:

data/.ruby-version

@@ -1 +1 @@
-3.3.1
+3.3.4

data/CHANGELOG.md

@@ -1,4 +1,88 @@
-# Karafka Web changelog
+# Karafka Web Changelog
+
+## 0.10.0 (Unreleased)
+- **[Breaking]** Rename and reorganize visibility filter to policies engine since it is not only about visibility.
+- **[Feature]** Replace Bootstrap with with tailwind + DaisyUI.
+- **[Feature]** Redesign the UI and move navigation to the left to make space for future features.
+- **[Feature]** Support per request policies for inspection and operations limitation.
+- **[Feature]** Provide Search capabilities in the Explorer (Pro).
+- **[Feature]** Provide dark mode.
+- [Enhancement] Provide topics watermarks inspection page (Pro).
+- [Enhancement] Use Turbo to improve usability.
+- [Enhancement] Round poll age reporting to precision of 2 reducing the payload size.
+- [Enhancement] Round utilization reporting to precision of 2 reducing the payload size.
+- [Enhancement] Validate states materialization lag in the status view.
+- [Enhancement] Promote topics data pace to OSS.
+- [Enhancement] Rename and normalize dashboard tabs.
+- [Enhancement] Enable live data polling on the first visit so it does not have to be enabled manually.
+- [Enhancement] Allow disabling ability to republish messages via policies.
+- [Enhancement] Display raw numerical timestamp alongside message time.
+- [Enhancement] Support `/topics` root redirect.
+- [Enhancement] Prevent explorer from displaying too big payloads (bigger than 1MB by default)
+- [Enhancement] Include deserialization object allocation stats.
+- [Enhancement] Improve how charts with many topics work.
+- [Enhancement] Count and display executed jobs independently from processed batches.
+- [Enhancement] Prevent karafka-web from being configured before karafka is configured.
+- [Enhancement] Use `ostruct` from RubyGems in testing.
+- [Enhancement] Indicate in the status reporting whether Karafka is OSS or Pro.
+- [Enhancement] Ship JS and CSS assets using Brotli and Gzip when possible.
+- [Enhancement] Introduce a `/ux` page to ease with styling improvements and components management.
+- [Enhancement] disallow usage of `<script>` blocks to prevent XSS.
+- [Enhancement] Display full subscription group information in the Routing view, including injectable defaults.
+- [Enhancement] Report Karafka consumer server execution mode.
+- [Enhancement] Expose `sync_threshold` consumer tracking config to allow aligning of error-intense applications.
+- [Refactor] Optimize subscription group data tracking flow.
+- [Refactor] Namespace migrations so migrations related to each topic data are in an independent directory.
+- [Refactor] Use errors for deny flow so request denials can occur from the inspection layer.
+- [Maintenance] Require `karafka` `2.4.7` due to fixes and API changes.
+- [Fix] Disallow quiet and stop commands for swarm workers.
+- [Fix] Disallow quiet and stop commands for embedded workers.
+- [Fix] Fix invalid deserialization metadata display in the per-message Explorer view.
+- [Fix] Fix a case where started page refresh would update content despite limiters being in place.
+- [Fix] Ruby 3.4.0 preview1 - No such file or directory.
+- [Fix] Fix the live poll button state flickering when disabled.
+- [Fix] Pace computation does not compensate for partial data reported.
+- [Fix] DLQ parent topics get classified as DLQ in the Web.
+- [Fix] Add missing space in the attempt label.
+- [Fix] Fix lack of highlight of "Consumers" navigation when in the "Commands" tab.
+- [Fix] Fix not working page reporting in breadcrumbs.
+- [Fix] Fix invalid redirect when trying to view particular errors partition time location.
+- [Fix] Fix several UI inconsistencies.
+- [Fix] License identifier `LGPL-3.0` is deprecated for SPDX (#2177).
+- [Fix] Do not include prettifying the payload for visibility in the resource computation cost.
+
+### Upgrade Notes
+
+This is a **major** release that brings many things to the table.
+
+This version of the Karafka Web UI requires Karafka `>= 2.4.7`. You can either upgrade both or upgrade Karafka first and then the Web UI. Karafka `2.4.7` is also compatible with Web UI `0.9.1`; thus, you can upgrade one at a time.
+
+#### Configuration
+
+Visibility Filters have been reorganized into messages policies.
+
+Please read the [Policies API](https://karafka.io/docs/Pro-Web-UI-Policies/) documentation and convert your visibility filters to policies.
+
+Your existing message-related visibility filter policies should now be assigned to a new configuration:
+
+```ruby
+Karafka::Web.setup do |config|
+  config.ui.policies.messages = MyCustomRequestsPolicy.new
+end
+```
+
+#### Deployment
+
+Because of the reporting schema update, it is recommended to:
+
+0. Make sure you have upgraded to `0.9.1` before and that it was fully deployed.
+1. Test the upgrade on a staging or dev environment.
+3. The Web UI interface may throw 500 errors during the upgrade because of schema incompatibility (until Puma is deployed and all consumers redeployed). This will have no long-term effects and can be ignored.
+4. `Karafka::Web::Errors::Processing::IncompatibleSchemaError` **is expected**. It is part of the Karafka Web UI zero-downtime deployment strategy. This error allows the Web UI materialization consumer to back off and wait for it to be replaced with a new one.
+5. Perform a rolling deployment (or a regular one) and replace all consumer processes.
+6. Update the Web UI Puma.
+7. **No** CLI command execution is required.
+8. Enjoy.
 
 ## 0.9.1 (2024-05-03)
 - [Fix] OSS `lag_stored` for not-subscribed consumers causes Web UI to crash.

data/Gemfile

@@ -9,6 +9,7 @@ gemspec
 group :test do
   gem 'byebug'
   gem 'factory_bot'
+  gem 'ostruct'
   gem 'rack-test'
   gem 'rspec'
   gem 'simplecov'

data/Gemfile.lock

@@ -1,9 +1,9 @@
 PATH
   remote: .
   specs:
-    karafka-web (0.9.1)
+    karafka-web (0.10.0.beta1)
       erubi (~> 1.4)
-      karafka (>= 2.4.0, < 2.5.0)
+      karafka (>= 2.4.7, < 2.5.0)
       karafka-core (>= 2.4.0, < 2.5.0)
       roda (~> 3.68, >= 3.69)
       tilt (~> 2.0)
@@ -11,7 +11,7 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (7.1.3.2)
+    activesupport (7.1.3.4)
       base64
       bigdecimal
       concurrent-ruby (~> 1.0, >= 1.0.2)
@@ -22,40 +22,43 @@ GEM
       mutex_m
       tzinfo (~> 2.0)
     base64 (0.2.0)
-    bigdecimal (3.1.7)
+    bigdecimal (3.1.8)
     byebug (11.1.3)
-    concurrent-ruby (1.2.3)
+    concurrent-ruby (1.3.3)
     connection_pool (2.4.1)
     diff-lcs (1.5.1)
     docile (1.4.0)
     drb (2.2.1)
-    erubi (1.12.0)
+    erubi (1.13.0)
     factory_bot (6.4.6)
       activesupport (>= 5.0.0)
-    ffi (1.16.3)
-    i18n (1.14.4)
+    ffi (1.17.0)
+    ffi (1.17.0-x86_64-linux-gnu)
+    i18n (1.14.5)
       concurrent-ruby (~> 1.0)
-    karafka (2.4.0)
-      karafka-core (>= 2.4.0, < 2.5.0)
-      waterdrop (>= 2.7.0, < 3.0.0)
+    karafka (2.4.7)
+      base64 (~> 0.2)
+      karafka-core (>= 2.4.3, < 2.5.0)
+      waterdrop (>= 2.7.3, < 3.0.0)
       zeitwerk (~> 2.3)
-    karafka-core (2.4.0)
-      karafka-rdkafka (>= 0.15.0, < 0.16.0)
-    karafka-rdkafka (0.15.0)
+    karafka-core (2.4.4)
+      karafka-rdkafka (>= 0.15.0, < 0.18.0)
+    karafka-rdkafka (0.17.1)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    mini_portile2 (2.8.6)
-    minitest (5.22.3)
+    mini_portile2 (2.8.7)
+    minitest (5.24.0)
     mutex_m (0.2.0)
-    rack (3.0.10)
+    ostruct (0.6.0)
+    rack (3.1.4)
     rack-test (2.1.0)
       rack (>= 1.3)
     rackup (0.2.3)
       rack (>= 3.0.0.beta1)
       webrick
     rake (13.2.1)
-    roda (3.79.0)
+    roda (3.81.0)
       rack
     rspec (3.13.0)
       rspec-core (~> 3.13.0)
@@ -63,10 +66,10 @@ GEM
       rspec-mocks (~> 3.13.0)
     rspec-core (3.13.0)
       rspec-support (~> 3.13.0)
-    rspec-expectations (3.13.0)
+    rspec-expectations (3.13.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-mocks (3.13.0)
+    rspec-mocks (3.13.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
     rspec-support (3.13.1)
@@ -76,14 +79,15 @@ GEM
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
-    tilt (2.3.0)
+    tilt (2.4.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.7.0)
-      karafka-core (>= 2.4.0, < 3.0.0)
+    waterdrop (2.7.4)
+      karafka-core (>= 2.4.3, < 3.0.0)
+      karafka-rdkafka (>= 0.15.1)
       zeitwerk (~> 2.3)
     webrick (1.8.1)
-    zeitwerk (2.6.13)
+    zeitwerk (2.6.17)
 
 PLATFORMS
   ruby
@@ -93,10 +97,11 @@ DEPENDENCIES
   byebug
   factory_bot
   karafka-web!
+  ostruct
   rack-test
   rackup (~> 0.2)
   rspec
   simplecov
 
 BUNDLED WITH
-   2.5.9
+   2.5.14

data/LICENSE

@@ -1,7 +1,7 @@
 Copyright (c) Maciej Mensfeld
 
 Karafka Web is part of Karafka and it is an Open Source project licensed under the terms of
-the LGPLv3 license. Please see <https://github.com/karafka/karafka/blob/master/LGPL>
+the LGPLv3 license. Please see <https://github.com/karafka/karafka/blob/master/LICENSE-LGPL>
 for license text.
 
 Karafka and Karafka Web have also commercial-friendly license, commercial support and commercial components.

data/bin/build_assets

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+rm -f lib/karafka/web/ui/public/javascripts/application.min.*
+rm -f lib/karafka/web/ui/public/stylesheets/application.min.*
+rm -f lib/karafka/web/ui/public/stylesheets/libs/highlight_light.min.css.*
+rm -f lib/karafka/web/ui/public/stylesheets/libs/highlight_dark.min.css.*
+
+set -e
+
+npx postcss \
+  lib/karafka/web/ui/public/stylesheets/libs/tailwind.css \
+  -o lib/karafka/web/ui/public/stylesheets/libs/tailwind.min.css
+
+npx gulp
+
+# We remove the intermediate tailwind file because its already bundled with gulp
+rm lib/karafka/web/ui/public/stylesheets/libs/tailwind.min.css
+
+compress_file() {
+  local input_file="$1"
+  local output_dir
+  output_dir=$(dirname "$input_file")
+  local base_name
+  base_name=$(basename "$input_file")
+
+  # Check if input file exists
+  if [ ! -f "$input_file" ]; then
+    echo "Input file not found: $input_file"
+    return 1
+  fi
+
+  # Compress with Brotli
+  brotli -k -q 11 "$input_file" -o "$output_dir/$base_name.br"
+  if [ $? -ne 0 ]; then
+    echo "Brotli compression failed"
+    return 1
+  fi
+
+  # Compress with Gzip
+  gzip -k -9 "$input_file" -c > "$output_dir/$base_name.gz"
+  if [ $? -ne 0 ]; then
+    echo "Gzip compression failed"
+    return 1
+  fi
+}
+
+compress_file "lib/karafka/web/ui/public/stylesheets/application.min.css"
+compress_file "lib/karafka/web/ui/public/javascripts/application.min.js"
+
+compress_file "lib/karafka/web/ui/public/stylesheets/libs/highlight_light.min.css"
+compress_file "lib/karafka/web/ui/public/stylesheets/libs/highlight_dark.min.css"

data/bin/release

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+# Compiles assets in the release mode and creates the web gem
+
+KARAFKA_RELEASE=true ./bin/build_assets
+gem build karafka-web.gemspec

data/config/locales/pro_errors.yml

@@ -9,3 +9,21 @@ en:
       commanding.pause_timeout_format: needs to be an integer bigger than 0
       key_must_be_a_symbol: All keys under the kafka settings scope need to be symbols
       commanding.kafka_format: needs to be a filled hash
+      ui.search.matchers_must_have_name_and_call: 'must respond to #name and its instance to #call'
+      ui.search.matchers_format: must be an array with matchers
+      ui.search.matchers_name_must_be_valid: all matchers names must be non-empty strings
+      ui.search.limits_format: all limits need to be integers bigger than 0
+      ui.search.timeout_format: must be at least 1 ms
+      ui.policies.messages_format: cannot be nil
+      ui.policies.requests_format: cannot be nil
+
+    search_form:
+      missing: needs to be present
+      timestamp_key_must_be_large_enough: 'must be a Kafka message timestamp with ms precision'
+      matcher_format: must match the existing matchers names
+      limit_format: must be one of the predefined limits
+      phrase_format: must be a non-empty string
+      offset_type_format: must be latest, offset or a timestamp
+      offset_format: needs to be an integer bigger than 0
+      partitions_format: needs to include "all" or partitions ids
+      timestamp_format: must be a Kafka message timestamp with ms precision

data/docker-compose.yml

@@ -3,7 +3,7 @@ version: '2'
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:7.6.1
+    image: confluentinc/cp-kafka:7.7.0
 
     ports:
       - 9092:9092

data/gulpfile.js

@@ -0,0 +1,73 @@
+const gulp = require('gulp');
+const concat = require('gulp-concat');
+const uglify = require('gulp-uglify');
+const through2 = require('through2');
+const path = require('path');
+const cleanCSS = require('gulp-clean-css');
+const sourcemaps = require('gulp-sourcemaps');
+
+// Check if we are in development mode
+const isDev = process.env.KARAFKA_RELEASE !== 'true';
+
+// Define JavaScript source files
+const jsFiles = [
+  'lib/karafka/web/ui/public/javascripts/libs/**/*.js',
+  'lib/karafka/web/ui/public/javascripts/charts/**/*.js',
+  'lib/karafka/web/ui/public/javascripts/components/**/*.js',
+  'lib/karafka/web/ui/public/javascripts/application.js'
+];
+
+// Define CSS source files
+const cssFiles = [
+  'lib/karafka/web/ui/public/stylesheets/libs/datepicker.min.css',
+  'lib/karafka/web/ui/public/stylesheets/libs/tailwind.min.css',
+  'lib/karafka/web/ui/public/stylesheets/application.css'
+];
+
+// Custom transform stream to add file location comments for JavaScript
+function addFileLocation() {
+  return through2.obj(function (file, enc, cb) {
+    if (file.isBuffer()) {
+      const fileLocationComment = `/*! Source: ${path.relative(__dirname, file.path)} */\n\n`;
+      const contents = Buffer.concat([Buffer.from(fileLocationComment), file.contents]);
+      file.contents = contents;
+    }
+    cb(null, file);
+  });
+}
+
+// JavaScript task
+gulp.task('scripts', function() {
+  return gulp.src(jsFiles)
+    .pipe(isDev ? sourcemaps.init() : through2.obj()) // Initialize sourcemaps in dev
+    .pipe(addFileLocation()) // Add file location comments
+    .pipe(concat('application.min.js'))
+    .pipe(uglify({
+      output: {
+        comments: function(node, comment) {
+          const text = comment.value;
+          const type = comment.type;
+          if (type == "comment2") {
+            // Preserve comments starting with `/*!` or `/**`
+            return /@preserve|@license|@cc_on|^\!/.test(text);
+          }
+          return false;
+        }
+      }
+    }))
+    .pipe(isDev ? sourcemaps.write('.') : through2.obj()) // Write sourcemaps in dev
+    .pipe(gulp.dest('lib/karafka/web/ui/public/javascripts'));
+});
+
+// CSS task
+gulp.task('styles', function() {
+  return gulp.src(cssFiles)
+    .pipe(isDev ? sourcemaps.init() : through2.obj()) // Initialize sourcemaps in dev
+    .pipe(concat('application.min.css'))
+    .pipe(cleanCSS({ level: 2 })) // Minify CSS
+    .pipe(isDev ? sourcemaps.write('.') : through2.obj()) // Write sourcemaps in dev
+    .pipe(gulp.dest('lib/karafka/web/ui/public/stylesheets'));
+});
+
+// Define default task to run both scripts and styles tasks
+gulp.task('default', gulp.series('scripts', 'styles'));

data/karafka-web.gemspec

@@ -14,10 +14,10 @@ Gem::Specification.new do |spec|
   spec.homepage    = 'https://karafka.io'
   spec.summary     = 'Karafka ecosystem Web UI interface'
   spec.description = 'Karafka ecosystem plug-and-play Web UI'
-  spec.licenses    = %w[LGPL-3.0 Commercial]
+  spec.licenses    = %w[LGPL-3.0-only Commercial]
 
   spec.add_dependency 'erubi', '~> 1.4'
-  spec.add_dependency 'karafka', '>= 2.4.0', '< 2.5.0'
+  spec.add_dependency 'karafka', '>= 2.4.7', '< 2.5.0'
   spec.add_dependency 'karafka-core', '>= 2.4.0', '< 2.5.0'
   spec.add_dependency 'roda', '~> 3.68', '>= 3.69'
   spec.add_dependency 'tilt', '~> 2.0'

data/lib/karafka/web/config.rb

@@ -76,6 +76,11 @@ module Karafka
           # Reports the metrics collected in the consumer sampler
           setting :reporter, default: Tracking::Consumers::Reporter.new
 
+          # Minimum number of messages to produce to produce them in sync mode
+          # This acts as a small back-off not to overload the system in case we would have
+          # extremely big number of errors and reports happening
+          setting :sync_threshold, default: 50
+
           # Samples for fetching and storing metrics samples about the consumer process
           setting :sampler, default: Tracking::Consumers::Sampler.new
 
@@ -139,11 +144,6 @@ module Karafka
         )
 
         setting :visibility do
-          # Allows to manage visibility of payload, headers and message key in the UI
-          # In some cases you may want to limit what is being displayed due to the type of data you
-          # are dealing with
-          setting :filter, default: Ui::Models::VisibilityFilter.new
-
           # Should we display internal topics of Kafka. The once starting with `__`
           # By default we do not display them as they are not usable from regular users perspective
           setting :internal_topics, default: false
@@ -162,17 +162,16 @@ module Karafka
         # transaction that will cause given consumer group to halt processing and wait
         setting :lso_threshold, default: 5 * 60 * 1_000
 
-        # Allows to manage visibility of payload, headers and message key in the UI
-        # In some cases you may want to limit what is being displayed due to the type of data you
-        # are dealing with
-        setting :visibility_filter, default: Ui::Models::VisibilityFilter.new
-
         # Consider any topic matching those names as a DLQ topic for the DLQ view
         # Web UI uses auto DLQ discovery based on routing but this may not be fully operable when
         # using a multi-app setup. This config allows to add extra topics if needed without having
         # to explicitly define routing
         setting :dlq_patterns, default: [/(dlq)|(dead_letter)/i]
 
+        # Maximum in-memory size of payload that we will render. Anything bigger than this by
+        # default will not be displayed not to hang the browser. 512KB of serialized data is a lot.
+        setting :max_visible_payload_size, default: 524_288
+
         # Specific kafka settings that are tuned to operate within the Web UI interface.
         #
         # Please do not change them unless you know what you are doing as their misconfiguration

data/lib/karafka/web/contracts/base.rb

@@ -10,6 +10,8 @@ module Karafka
           # This layer is not for users extensive feedback, thus we can easily use the minimum
           # error messaging there is.
           def configure
+            return super if block_given?
+
             super do |config|
               config.error_messages = YAML.safe_load(
                 File.read(

data/lib/karafka/web/contracts/config.rb

@@ -34,6 +34,7 @@ module Karafka
             required(:reporter) { |val| !val.nil? }
             required(:sampler) { |val| !val.nil? }
             required(:listeners) { |val| val.is_a?(Array) }
+            required(:sync_threshold) { |val| val.is_a?(Integer) && val.positive? }
           end
 
           nested(:producers) do
@@ -57,6 +58,7 @@ module Karafka
 
           required(:cache) { |val| !val.nil? }
           required(:per_page) { |val| val.is_a?(Integer) && val >= 1 && val <= 100 }
+          required(:max_visible_payload_size) { |val| val.is_a?(Integer) && val >= 1 }
 
           required(:dlq_patterns) do |val|
             val.is_a?(Array) &&
@@ -64,7 +66,6 @@ module Karafka
           end
 
           nested(:visibility) do
-            required(:filter) { |val| !val.nil? }
             required(:internal_topics) { |val| [true, false].include?(val) }
             required(:active_topics_cluster_lags_only) { |val| [true, false].include?(val) }
           end

data/lib/karafka/web/errors.rb

@@ -11,6 +11,15 @@ module Karafka
       # This should never happen and if you see this, please open an issue.
       ContractError = Class.new(BaseError)
 
+      # Raised when you try to configure Web UI but Karafka was not yet configured and is not in
+      # a state to accept Web UI integration.
+      #
+      # If you are seeing this error, it means you tried to configure and setup the Web UI before
+      # you configured Karafka. In case of a "split-setup" where you divided your `karafka.rb`
+      # into separate files, likely you are requiring the web-ui component prior to the one that
+      # configures Karafka
+      KarafkaNotInitializedError = Class.new(BaseError)
+
       # Errors specific to management
       module Management
         # Similar to processing error with the same name, it is raised when a critical
@@ -62,6 +71,9 @@ module Karafka
 
         # Raised whe a given feature is available for Pro but not pro used
         ProOnlyError = Class.new(BaseError)
+
+        # Raised when we want to stop the flow and render 403
+        ForbiddenError = Class.new(BaseError)
       end
     end
   end

data/lib/karafka/web/inflector.rb

@@ -6,7 +6,7 @@ module Karafka
     # to how Rails does that.
     class Inflector < Zeitwerk::GemInflector
       # Checks if given path is a migration one
-      MIGRATION_ABSPATH_REGEXP = /migrations\/[0-9]+_(.*)/
+      MIGRATION_ABSPATH_REGEXP = /migrations\/[a-z_]+\/[0-9]+_(.*)/
 
       # Checks if it is a migration file
       MIGRATION_BASENAME_REGEXP = /\A[0-9]+_(.*)/

data/lib/karafka/web/management/actions/enable.rb

@@ -10,6 +10,8 @@ module Karafka
         class Enable < Base
           # Enables routing consumer group and subscribes Web-UI listeners
           def call
+            ensure_karafka_initialized!
+
             # Prevent double enabling
             return if ::Karafka::Web.config.enabled
 
@@ -27,6 +29,15 @@ module Karafka
 
           private
 
+          # We should not allow for enabling of Karafka Web when Karafka is not configured.
+          # Karafka needs to be loaded and configured before Web can be configured because Web is
+          # using Karafka configuration
+          def ensure_karafka_initialized!
+            return unless Karafka::App.config.internal.status.initializing?
+
+            raise Web::Errors::KarafkaNotInitializedError, 'Please initialize Karafka first'
+          end
+
           # Enables tracking if it was not explicitly disabled by the user
           def setup_tracking_activity
             return unless ::Karafka::Web.config.tracking.active.nil?

data/lib/karafka/web/management/migrations/consumers_metrics/0_set_initial.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      module Migrations
+        # Consumers metrics topic data related migrations
+        module ConsumersMetrics
+          # Initial migration that sets the consumers metrics initial first state.
+          # This is the basic of metrics as they were when they were introduced.
+          class SetInitial < Base
+            # Always migrate from empty up
+            self.versions_until = '0.0.1'
+            self.type = :consumers_metrics
+
+            # @param state [Hash] initial empty state
+            def migrate(state)
+              state.merge!(
+                aggregated: {
+                  days: [],
+                  hours: [],
+                  minutes: [],
+                  seconds: []
+                },
+                consumer_groups: {
+                  days: [],
+                  hours: [],
+                  minutes: [],
+                  seconds: []
+                },
+                dispatched_at: float_now
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+end