karafka-web 0.9.1 → 0.10.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ce075ec71d155adbf35706c5bfa9beab07a707970e53ad9b0f7e6b2fc2196976
-  data.tar.gz: 747b28ece60db10c34cabfcaa70edcc8d7c0aaa63440c6b3dba6b108a35a0435
+  metadata.gz: 40794ee9c8b084cf75de0403b904134cbc6a762dbfa338bb9ce574b56e46ac36
+  data.tar.gz: dedefe087c54e118abb90e819b36b29aa253c4351aa18bd8a78d01e2c096ce79
 SHA512:
-  metadata.gz: 39d4e856b439c83acc929cf17432c52abbbf4c2a063897b0595ed360321fc0cafb62527e5d99f35710d9d731accbd12aaca9a8c0a6c97ead1d5302df1fd400dd
-  data.tar.gz: fc8eb8d769bc3dd9edeca90fedb05c83ce382c562f26b261290a352f04f09ffc029bdb21627bc6d305924b5673ed8465631d6a60d58c486575b24055f7255c9e
+  metadata.gz: 88b9b6ed8082b0484dcd32a20b2cee2b77d4df762b65d0487325e03ef375e1e7ac621d612f81b3f4a6870aa4352e07286979863d656161ca7d8d880e5709d108
+  data.tar.gz: f9350f68d872c76d289933fbf2548b692179bae30fb2c0fccf923cdefca7131bbf3e6bf362e21a442c424fc4b43472e8334115b90c32449e80ef45a3518c3228

data/.github/workflows/ci.yml

@@ -22,6 +22,7 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
+          - '3.4.0-preview1'
           - '3.3'
           - '3.2'
           - '3.1'
@@ -101,3 +102,32 @@ jobs:
           CODITSU_API_KEY: ${{ secrets.CODITSU_API_KEY }}
           CODITSU_API_SECRET: ${{ secrets.CODITSU_API_SECRET }}
         run: \curl -sSL https://api.coditsu.io/run/ci | bash
+
+
+  assets:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '17'
+
+      - name: Cache node modules
+        uses: actions/cache@v4
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+
+      - name: Install dependencies
+        run: npm install
+
+      - name: Build development assets
+        run: ./bin/build_assets
+
+      - name: Build release assets
+        run: KARAFKA_RELEASE=true ./bin/build_assets

data/.gitignore

@@ -38,8 +38,10 @@ db/*.sqlite3
 /capybara-*.html
 /gems
 /specifications
+/node_modules
 rerun.txt
 pickle-email-*.html
+tailwind.min.css
 
 # If you find yourself ignoring temporary files generated by your text editor
 # or operating system, you probably want to add a global ignore instead:

data/.ruby-version

@@ -1 +1 @@
-3.3.1
+3.3.4

data/CHANGELOG.md

@@ -1,4 +1,101 @@
-# Karafka Web changelog
+# Karafka Web Changelog
+
+## 0.10.1 (Unreleased)
+- **[Feature]** Support Recurring Tasks management (Pro).
+- [Enhancement] Optimize command buttons so they occupy less space.
+- [Enhancement] Improve tables headers capitalization.
+- [Enhancement] Prevent live-polling when user hovers over actionable links to mitigate race conditions.
+- [Fix] Fix partial lack of tables hover in daily mode.
+- [Fix] Fix lack of tables hover in dark mode.
+- [Fix] Normalize various tables types styling.
+- [Fix] Fix ranges selectors position on wide screens.
+
+## 0.10.0 (2024-08-19)
+- **[Breaking]** Rename and reorganize visibility filter to policies engine since it is not only about visibility.
+- **[Feature]** Replace Bootstrap with with tailwind + DaisyUI.
+- **[Feature]** Redesign the UI and move navigation to the left to make space for future features.
+- **[Feature]** Support per request policies for inspection and operations limitation.
+- **[Feature]** Provide Search capabilities in the Explorer (Pro).
+- **[Feature]** Provide dark mode.
+- [Enhancement] Set `enable.partition.eof` to `false` for Web UI consumer group as it is not needed.
+- [Enhancement] Allow for configuration of extra `kafka` scope options for the Web UI consumer group.
+- [Enhancement] Support Karafka `#eofed` consumer action.
+- [Enhancement] Provide topics watermarks inspection page (Pro).
+- [Enhancement] Use Turbo to improve usability.
+- [Enhancement] Round poll age reporting to precision of 2 reducing the payload size.
+- [Enhancement] Round utilization reporting to precision of 2 reducing the payload size.
+- [Enhancement] Validate states materialization lag in the status view.
+- [Enhancement] Promote topics data pace to OSS.
+- [Enhancement] Rename and normalize dashboard tabs.
+- [Enhancement] Enable live data polling on the first visit so it does not have to be enabled manually.
+- [Enhancement] Allow disabling ability to republish messages via policies.
+- [Enhancement] Display raw numerical timestamp alongside message time.
+- [Enhancement] Support `/topics` root redirect.
+- [Enhancement] Prevent explorer from displaying too big payloads (bigger than 1MB by default)
+- [Enhancement] Include deserialization object allocation stats.
+- [Enhancement] Improve how charts with many topics work.
+- [Enhancement] Count and display executed jobs independently from processed batches.
+- [Enhancement] Prevent karafka-web from being configured before karafka is configured.
+- [Enhancement] Use `ostruct` from RubyGems in testing.
+- [Enhancement] Indicate in the status reporting whether Karafka is OSS or Pro.
+- [Enhancement] Ship JS and CSS assets using Brotli and Gzip when possible.
+- [Enhancement] Introduce a `/ux` page to ease with styling improvements and components management.
+- [Enhancement] disallow usage of `<script>` blocks to prevent XSS.
+- [Enhancement] Display full subscription group information in the Routing view, including injectable defaults.
+- [Enhancement] Report Karafka consumer server execution mode.
+- [Enhancement] Expose `sync_threshold` consumer tracking config to allow aligning of error-intense applications.
+- [Refactor] Optimize subscription group data tracking flow.
+- [Refactor] Namespace migrations so migrations related to each topic data are in an independent directory.
+- [Refactor] Use errors for deny flow so request denials can occur from the inspection layer.
+- [Maintenance] Require `karafka` `2.4.7` due to fixes and API changes.
+- [Fix] Disallow quiet and stop commands for swarm workers.
+- [Fix] Disallow quiet and stop commands for embedded workers.
+- [Fix] Fix invalid deserialization metadata display in the per-message Explorer view.
+- [Fix] Fix a case where started page refresh would update content despite limiters being in place.
+- [Fix] Ruby 3.4.0 preview1 - No such file or directory.
+- [Fix] Fix the live poll button state flickering when disabled.
+- [Fix] Pace computation does not compensate for partial data reported.
+- [Fix] DLQ parent topics get classified as DLQ in the Web.
+- [Fix] Add missing space in the attempt label.
+- [Fix] Fix lack of highlight of "Consumers" navigation when in the "Commands" tab.
+- [Fix] Fix not working page reporting in breadcrumbs.
+- [Fix] Fix invalid redirect when trying to view particular errors partition time location.
+- [Fix] Fix several UI inconsistencies.
+- [Fix] License identifier `LGPL-3.0` is deprecated for SPDX (#2177).
+- [Fix] Do not include prettifying the payload for visibility in the resource computation cost.
+
+### Upgrade Notes
+
+This is a **major** release that brings many things to the table.
+
+This version of the Karafka Web UI requires Karafka `>= 2.4.7`. You can either upgrade both or upgrade Karafka first and then the Web UI. Karafka `2.4.7` is also compatible with Web UI `0.9.1`; thus, you can upgrade one at a time.
+
+#### Configuration
+
+Visibility Filters have been reorganized into messages policies.
+
+Please read the [Policies API](https://karafka.io/docs/Pro-Web-UI-Policies/) documentation and convert your visibility filters to policies.
+
+Your existing message-related visibility filter policies should now be assigned to a new configuration:
+
+```ruby
+Karafka::Web.setup do |config|
+  config.ui.policies.messages = MyCustomRequestsPolicy.new
+end
+```
+
+#### Deployment
+
+Because of the reporting schema update, it is recommended to:
+
+0. Make sure you have upgraded to `0.9.1` before and that it was fully deployed.
+1. Test the upgrade on a staging or dev environment.
+3. The Web UI interface may throw 500 errors during the upgrade because of schema incompatibility (until Puma is deployed and all consumers redeployed). This will have no long-term effects and can be ignored.
+4. `Karafka::Web::Errors::Processing::IncompatibleSchemaError` **is expected**. It is part of the Karafka Web UI zero-downtime deployment strategy. This error allows the Web UI materialization consumer to back off and wait for it to be replaced with a new one.
+5. Perform a rolling deployment (or a regular one) and replace all consumer processes.
+6. Update the Web UI Puma.
+7. **No** CLI command execution is required.
+8. Enjoy.
 
 ## 0.9.1 (2024-05-03)
 - [Fix] OSS `lag_stored` for not-subscribed consumers causes Web UI to crash.

data/Gemfile

@@ -9,6 +9,8 @@ gemspec
 group :test do
   gem 'byebug'
   gem 'factory_bot'
+  gem 'fugit'
+  gem 'ostruct'
   gem 'rack-test'
   gem 'rspec'
   gem 'simplecov'

data/Gemfile.lock

@@ -1,9 +1,9 @@
 PATH
   remote: .
   specs:
-    karafka-web (0.9.1)
+    karafka-web (0.10.1)
       erubi (~> 1.4)
-      karafka (>= 2.4.0, < 2.5.0)
+      karafka (>= 2.4.9, < 2.5.0)
       karafka-core (>= 2.4.0, < 2.5.0)
       roda (~> 3.68, >= 3.69)
       tilt (~> 2.0)
@@ -11,7 +11,7 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (7.1.3.2)
+    activesupport (7.1.3.4)
       base64
       bigdecimal
       concurrent-ruby (~> 1.0, >= 1.0.2)
@@ -22,40 +22,49 @@ GEM
       mutex_m
       tzinfo (~> 2.0)
     base64 (0.2.0)
-    bigdecimal (3.1.7)
+    bigdecimal (3.1.8)
     byebug (11.1.3)
-    concurrent-ruby (1.2.3)
+    concurrent-ruby (1.3.3)
     connection_pool (2.4.1)
     diff-lcs (1.5.1)
     docile (1.4.0)
     drb (2.2.1)
-    erubi (1.12.0)
+    erubi (1.13.0)
+    et-orbi (1.2.11)
+      tzinfo
     factory_bot (6.4.6)
       activesupport (>= 5.0.0)
-    ffi (1.16.3)
-    i18n (1.14.4)
+    ffi (1.17.0)
+    fugit (1.11.1)
+      et-orbi (~> 1, >= 1.2.11)
+      raabro (~> 1.4)
+    i18n (1.14.5)
       concurrent-ruby (~> 1.0)
-    karafka (2.4.0)
-      karafka-core (>= 2.4.0, < 2.5.0)
-      waterdrop (>= 2.7.0, < 3.0.0)
+    karafka (2.4.9)
+      base64 (~> 0.2)
+      karafka-core (>= 2.4.3, < 2.5.0)
+      karafka-rdkafka (>= 0.17.2)
+      waterdrop (>= 2.7.3, < 3.0.0)
       zeitwerk (~> 2.3)
-    karafka-core (2.4.0)
-      karafka-rdkafka (>= 0.15.0, < 0.16.0)
-    karafka-rdkafka (0.15.0)
+    karafka-core (2.4.4)
+      karafka-rdkafka (>= 0.15.0, < 0.18.0)
+    karafka-rdkafka (0.17.3)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    mini_portile2 (2.8.6)
-    minitest (5.22.3)
+    mini_portile2 (2.8.7)
+    minitest (5.24.0)
     mutex_m (0.2.0)
-    rack (3.0.10)
+    ostruct (0.6.0)
+    raabro (1.4.0)
+    rack (3.1.4)
     rack-test (2.1.0)
       rack (>= 1.3)
     rackup (0.2.3)
       rack (>= 3.0.0.beta1)
       webrick
     rake (13.2.1)
-    roda (3.79.0)
+    roda (3.81.0)
       rack
     rspec (3.13.0)
       rspec-core (~> 3.13.0)
@@ -63,10 +72,10 @@ GEM
       rspec-mocks (~> 3.13.0)
     rspec-core (3.13.0)
       rspec-support (~> 3.13.0)
-    rspec-expectations (3.13.0)
+    rspec-expectations (3.13.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-mocks (3.13.0)
+    rspec-mocks (3.13.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
     rspec-support (3.13.1)
@@ -76,14 +85,15 @@ GEM
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
-    tilt (2.3.0)
+    tilt (2.4.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.7.0)
-      karafka-core (>= 2.4.0, < 3.0.0)
+    waterdrop (2.7.4)
+      karafka-core (>= 2.4.3, < 3.0.0)
+      karafka-rdkafka (>= 0.15.1)
       zeitwerk (~> 2.3)
     webrick (1.8.1)
-    zeitwerk (2.6.13)
+    zeitwerk (2.6.17)
 
 PLATFORMS
   ruby
@@ -92,11 +102,13 @@ PLATFORMS
 DEPENDENCIES
   byebug
   factory_bot
+  fugit
   karafka-web!
+  ostruct
   rack-test
   rackup (~> 0.2)
   rspec
   simplecov
 
 BUNDLED WITH
-   2.5.9
+   2.5.14

data/LICENSE

@@ -1,7 +1,7 @@
 Copyright (c) Maciej Mensfeld
 
 Karafka Web is part of Karafka and it is an Open Source project licensed under the terms of
-the LGPLv3 license. Please see <https://github.com/karafka/karafka/blob/master/LGPL>
+the LGPLv3 license. Please see <https://github.com/karafka/karafka/blob/master/LICENSE-LGPL>
 for license text.
 
 Karafka and Karafka Web have also commercial-friendly license, commercial support and commercial components.

data/bin/build_assets

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+rm -f lib/karafka/web/ui/public/javascripts/application.min.*
+rm -f lib/karafka/web/ui/public/stylesheets/application.min.*
+rm -f lib/karafka/web/ui/public/stylesheets/libs/highlight_light.min.css.*
+rm -f lib/karafka/web/ui/public/stylesheets/libs/highlight_dark.min.css.*
+
+set -e
+
+npx postcss \
+  lib/karafka/web/ui/public/stylesheets/libs/tailwind.css \
+  -o lib/karafka/web/ui/public/stylesheets/libs/tailwind.min.css
+
+npx gulp
+
+# We remove the intermediate tailwind file because its already bundled with gulp
+rm lib/karafka/web/ui/public/stylesheets/libs/tailwind.min.css
+
+compress_file() {
+  local input_file="$1"
+  local output_dir
+  output_dir=$(dirname "$input_file")
+  local base_name
+  base_name=$(basename "$input_file")
+
+  # Check if input file exists
+  if [ ! -f "$input_file" ]; then
+    echo "Input file not found: $input_file"
+    return 1
+  fi
+
+  # Compress with Brotli
+  brotli -k -q 11 "$input_file" -o "$output_dir/$base_name.br"
+  if [ $? -ne 0 ]; then
+    echo "Brotli compression failed"
+    return 1
+  fi
+
+  # Compress with Gzip
+  gzip -k -9 "$input_file" -c > "$output_dir/$base_name.gz"
+  if [ $? -ne 0 ]; then
+    echo "Gzip compression failed"
+    return 1
+  fi
+}
+
+compress_file "lib/karafka/web/ui/public/stylesheets/application.min.css"
+compress_file "lib/karafka/web/ui/public/javascripts/application.min.js"
+
+compress_file "lib/karafka/web/ui/public/stylesheets/libs/highlight_light.min.css"
+compress_file "lib/karafka/web/ui/public/stylesheets/libs/highlight_dark.min.css"

data/bin/release

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+# Compiles assets in the release mode and creates the web gem
+
+KARAFKA_RELEASE=true ./bin/build_assets
+gem build karafka-web.gemspec

data/certs/cert.pem

@@ -0,0 +1,26 @@
+-----BEGIN CERTIFICATE-----
+MIIEcDCCAtigAwIBAgIBATANBgkqhkiG9w0BAQsFADA/MRAwDgYDVQQDDAdjb250
+YWN0MRcwFQYKCZImiZPyLGQBGRYHa2FyYWZrYTESMBAGCgmSJomT8ixkARkWAmlv
+MB4XDTI0MDgyMzEwMTkyMFoXDTQ5MDgxNzEwMTkyMFowPzEQMA4GA1UEAwwHY29u
+dGFjdDEXMBUGCgmSJomT8ixkARkWB2thcmFma2ExEjAQBgoJkiaJk/IsZAEZFgJp
+bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAKjLhLjQqUlNayxkXnO+
+PsmCDs/KFIzhrsYMfLZRZNaWmzV3ujljMOdDjd4snM2X06C41iVdQPWjpe3j8vVe
+ZXEWR/twSbOP6Eeg8WVH2wCOo0x5i7yhVn4UBLH4JpfEMCbemVcWQ9ry9OMg4WpH
+Uu4dRwxFV7hzCz3p0QfNLRI4miAxnGWcnlD98IJRjBAksTuR1Llj0vbOrDGsL9ZT
+JeXP2gdRLd8SqzAFJEWrbeTBCBU7gfSh3oMg5SVDLjaqf7Kz5wC/8bDZydzanOxB
+T6CDXPsCnllmvTNx2ei2T5rGYJOzJeNTmJLLK6hJWUlAvaQSvCwZRvFJ0tVGLEoS
+flqSr6uGyyl1eMUsNmsH4BqPEYcAV6P2PKTv2vUR8AP0raDvZ3xL1TKvfRb8xRpo
+vPopCGlY5XBWEc6QERHfVLTIVsjnls2/Ujj4h8/TSfqqYnaHKefIMLbuD/tquMjD
+iWQsW2qStBV0T+U7FijKxVfrfqZP7GxQmDAc9o1iiyAa3QIDAQABo3cwdTAJBgNV
+HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQU3O4dTXmvE7YpAkszGzR9DdL9
+sbEwHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
+bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEAVKTfoLXn7mqdSxIR
+eqxcR6Huudg1jes81s1+X0uiRTR3hxxKZ3Y82cPsee9zYWyBrN8TA4KA0WILTru7
+Ygxvzha0SRPsSiaKLmgOJ+61ebI4+bOORzIJLpD6GxCxu1r7MI4+0r1u1xe0EWi8
+agkVo1k4Vi8cKMLm6Gl9b3wG9zQBw6fcgKwmpjKiNnOLP+OytzUANrIUJjoq6oal
+TC+f/Uc0TLaRqUaW/bejxzDWWHoM3SU6aoLPuerglzp9zZVzihXwx3jPLUVKDFpF
+Rl2lcBDxlpYGueGo0/oNzGJAAy6js8jhtHC9+19PD53vk7wHtFTZ/0ugDQYnwQ+x
+oml2fAAuVWpTBCgOVFe6XCQpMKopzoxQ1PjKztW2KYxgJdIBX87SnL3aWuBQmhRd
+i9zWxov0mr44TWegTVeypcWGd/0nxu1+QHVNHJrpqlPBRvwQsUm7fwmRInGpcaB8
+ap8wNYvryYzrzvzUxIVFBVM5PacgkFqRmolCa8I7tdKQN+R1
+-----END CERTIFICATE-----

data/config/locales/pro_errors.yml

@@ -9,3 +9,21 @@ en:
       commanding.pause_timeout_format: needs to be an integer bigger than 0
       key_must_be_a_symbol: All keys under the kafka settings scope need to be symbols
       commanding.kafka_format: needs to be a filled hash
+      ui.search.matchers_must_have_name_and_call: 'must respond to #name and its instance to #call'
+      ui.search.matchers_format: must be an array with matchers
+      ui.search.matchers_name_must_be_valid: all matchers names must be non-empty strings
+      ui.search.limits_format: all limits need to be integers bigger than 0
+      ui.search.timeout_format: must be at least 1 ms
+      ui.policies.messages_format: cannot be nil
+      ui.policies.requests_format: cannot be nil
+
+    search_form:
+      missing: needs to be present
+      timestamp_key_must_be_large_enough: 'must be a Kafka message timestamp with ms precision'
+      matcher_format: must match the existing matchers names
+      limit_format: must be one of the predefined limits
+      phrase_format: must be a non-empty string
+      offset_type_format: must be latest, offset or a timestamp
+      offset_format: needs to be an integer bigger than 0
+      partitions_format: needs to include "all" or partitions ids
+      timestamp_format: must be a Kafka message timestamp with ms precision

data/docker-compose.yml

@@ -3,7 +3,7 @@ version: '2'
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:7.6.1
+    image: confluentinc/cp-kafka:7.7.0
 
     ports:
       - 9092:9092

data/gulpfile.js

@@ -0,0 +1,73 @@
+const gulp = require('gulp');
+const concat = require('gulp-concat');
+const uglify = require('gulp-uglify');
+const through2 = require('through2');
+const path = require('path');
+const cleanCSS = require('gulp-clean-css');
+const sourcemaps = require('gulp-sourcemaps');
+
+// Check if we are in development mode
+const isDev = process.env.KARAFKA_RELEASE !== 'true';
+
+// Define JavaScript source files
+const jsFiles = [
+  'lib/karafka/web/ui/public/javascripts/libs/**/*.js',
+  'lib/karafka/web/ui/public/javascripts/charts/**/*.js',
+  'lib/karafka/web/ui/public/javascripts/components/**/*.js',
+  'lib/karafka/web/ui/public/javascripts/application.js'
+];
+
+// Define CSS source files
+const cssFiles = [
+  'lib/karafka/web/ui/public/stylesheets/libs/datepicker.min.css',
+  'lib/karafka/web/ui/public/stylesheets/libs/tailwind.min.css',
+  'lib/karafka/web/ui/public/stylesheets/application.css'
+];
+
+// Custom transform stream to add file location comments for JavaScript
+function addFileLocation() {
+  return through2.obj(function (file, enc, cb) {
+    if (file.isBuffer()) {
+      const fileLocationComment = `/*! Source: ${path.relative(__dirname, file.path)} */\n\n`;
+      const contents = Buffer.concat([Buffer.from(fileLocationComment), file.contents]);
+      file.contents = contents;
+    }
+    cb(null, file);
+  });
+}
+
+// JavaScript task
+gulp.task('scripts', function() {
+  return gulp.src(jsFiles)
+    .pipe(isDev ? sourcemaps.init() : through2.obj()) // Initialize sourcemaps in dev
+    .pipe(addFileLocation()) // Add file location comments
+    .pipe(concat('application.min.js'))
+    .pipe(uglify({
+      output: {
+        comments: function(node, comment) {
+          const text = comment.value;
+          const type = comment.type;
+          if (type == "comment2") {
+            // Preserve comments starting with `/*!` or `/**`
+            return /@preserve|@license|@cc_on|^\!/.test(text);
+          }
+          return false;
+        }
+      }
+    }))
+    .pipe(isDev ? sourcemaps.write('.') : through2.obj()) // Write sourcemaps in dev
+    .pipe(gulp.dest('lib/karafka/web/ui/public/javascripts'));
+});
+
+// CSS task
+gulp.task('styles', function() {
+  return gulp.src(cssFiles)
+    .pipe(isDev ? sourcemaps.init() : through2.obj()) // Initialize sourcemaps in dev
+    .pipe(concat('application.min.css'))
+    .pipe(cleanCSS({ level: 2 })) // Minify CSS
+    .pipe(isDev ? sourcemaps.write('.') : through2.obj()) // Write sourcemaps in dev
+    .pipe(gulp.dest('lib/karafka/web/ui/public/stylesheets'));
+});
+
+// Define default task to run both scripts and styles tasks
+gulp.task('default', gulp.series('scripts', 'styles'));

data/karafka-web.gemspec

@@ -14,10 +14,10 @@ Gem::Specification.new do |spec|
   spec.homepage    = 'https://karafka.io'
   spec.summary     = 'Karafka ecosystem Web UI interface'
   spec.description = 'Karafka ecosystem plug-and-play Web UI'
-  spec.licenses    = %w[LGPL-3.0 Commercial]
+  spec.licenses    = %w[LGPL-3.0-only Commercial]
 
   spec.add_dependency 'erubi', '~> 1.4'
-  spec.add_dependency 'karafka', '>= 2.4.0', '< 2.5.0'
+  spec.add_dependency 'karafka', '>= 2.4.9', '< 2.5.0'
   spec.add_dependency 'karafka-core', '>= 2.4.0', '< 2.5.0'
   spec.add_dependency 'roda', '~> 3.68', '>= 3.69'
   spec.add_dependency 'tilt', '~> 2.0'
@@ -30,7 +30,7 @@ Gem::Specification.new do |spec|
     spec.signing_key = File.expand_path('~/.ssh/gem-private_key.pem')
   end
 
-  spec.cert_chain    = %w[certs/cert_chain.pem]
+  spec.cert_chain    = %w[certs/cert.pem]
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
   spec.executables   = %w[karafka-web]
   spec.require_paths = %w[lib]

data/lib/karafka/web/config.rb

@@ -76,6 +76,11 @@ module Karafka
           # Reports the metrics collected in the consumer sampler
           setting :reporter, default: Tracking::Consumers::Reporter.new
 
+          # Minimum number of messages to produce to produce them in sync mode
+          # This acts as a small back-off not to overload the system in case we would have
+          # extremely big number of errors and reports happening
+          setting :sync_threshold, default: 50
+
           # Samples for fetching and storing metrics samples about the consumer process
           setting :sampler, default: Tracking::Consumers::Sampler.new
 
@@ -118,6 +123,15 @@ module Karafka
         # By default we flush the states twice as often as the data reporting.
         # This will allow us to have closer to real-time reporting.
         setting :interval, default: 2_500
+
+        # Extra kafka level settings that we merge to the defaults when building the Web consumer
+        # group. User may want different things than we in regard to operations, thus effectively
+        # crippling responsiveness or stability of reporting.
+        setting :kafka, default: {
+          # We do not use at the moment the `#eofed` flag for anything, thus there is no point in
+          # having it set to true if user users it.
+          'enable.partition.eof': false
+        }.freeze
       end
 
       setting :ui do
@@ -139,11 +153,6 @@ module Karafka
         )
 
         setting :visibility do
-          # Allows to manage visibility of payload, headers and message key in the UI
-          # In some cases you may want to limit what is being displayed due to the type of data you
-          # are dealing with
-          setting :filter, default: Ui::Models::VisibilityFilter.new
-
           # Should we display internal topics of Kafka. The once starting with `__`
           # By default we do not display them as they are not usable from regular users perspective
           setting :internal_topics, default: false
@@ -162,17 +171,16 @@ module Karafka
         # transaction that will cause given consumer group to halt processing and wait
         setting :lso_threshold, default: 5 * 60 * 1_000
 
-        # Allows to manage visibility of payload, headers and message key in the UI
-        # In some cases you may want to limit what is being displayed due to the type of data you
-        # are dealing with
-        setting :visibility_filter, default: Ui::Models::VisibilityFilter.new
-
         # Consider any topic matching those names as a DLQ topic for the DLQ view
         # Web UI uses auto DLQ discovery based on routing but this may not be fully operable when
         # using a multi-app setup. This config allows to add extra topics if needed without having
         # to explicitly define routing
         setting :dlq_patterns, default: [/(dlq)|(dead_letter)/i]
 
+        # Maximum in-memory size of payload that we will render. Anything bigger than this by
+        # default will not be displayed not to hang the browser. 512KB of serialized data is a lot.
+        setting :max_visible_payload_size, default: 524_288
+
         # Specific kafka settings that are tuned to operate within the Web UI interface.
         #
         # Please do not change them unless you know what you are doing as their misconfiguration

data/lib/karafka/web/contracts/base.rb

@@ -10,6 +10,8 @@ module Karafka
           # This layer is not for users extensive feedback, thus we can easily use the minimum
           # error messaging there is.
           def configure
+            return super if block_given?
+
             super do |config|
               config.error_messages = YAML.safe_load(
                 File.read(

data/lib/karafka/web/contracts/config.rb

@@ -34,6 +34,7 @@ module Karafka
             required(:reporter) { |val| !val.nil? }
             required(:sampler) { |val| !val.nil? }
             required(:listeners) { |val| val.is_a?(Array) }
+            required(:sync_threshold) { |val| val.is_a?(Integer) && val.positive? }
           end
 
           nested(:producers) do
@@ -47,6 +48,9 @@ module Karafka
           required(:active) { |val| [true, false].include?(val) }
           # Do not update data more often not to overload and not to generate too much data
           required(:interval) { |val| val.is_a?(Integer) && val >= 1_000 }
+
+          # Extra Kafka setup for our processing consumer
+          required(:kafka) { |val| val.is_a?(Hash) }
         end
 
         nested(:ui) do
@@ -57,6 +61,8 @@ module Karafka
 
           required(:cache) { |val| !val.nil? }
           required(:per_page) { |val| val.is_a?(Integer) && val >= 1 && val <= 100 }
+          required(:max_visible_payload_size) { |val| val.is_a?(Integer) && val >= 1 }
+          required(:kafka) { |val| val.is_a?(Hash) }
 
           required(:dlq_patterns) do |val|
             val.is_a?(Array) &&
@@ -64,7 +70,6 @@ module Karafka
           end
 
           nested(:visibility) do
-            required(:filter) { |val| !val.nil? }
             required(:internal_topics) { |val| [true, false].include?(val) }
             required(:active_topics_cluster_lags_only) { |val| [true, false].include?(val) }
           end