karafka-web 0.11.5 → 0.11.6

data/bin/rspecs_parallel

@@ -0,0 +1,111 @@
+#!/usr/bin/env bash
+
+# Runs all specs in parallel using timing-based load balancing
+# Uses bin/balance_specs to distribute files across workers
+#
+# Environment variables:
+#   REGULAR_WORKERS - number of workers for regular specs (default: 4)
+#   PRO_WORKERS     - number of workers for pro specs (default: 4)
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# Number of workers per spec type
+REGULAR_WORKERS="${REGULAR_WORKERS:-4}"
+PRO_WORKERS="${PRO_WORKERS:-4}"
+
+echo "Running specs in parallel mode..."
+echo "  Regular workers: $REGULAR_WORKERS"
+echo "  Pro workers: $PRO_WORKERS"
+
+# Create temp directory for logs
+TMPDIR=$(mktemp -d)
+trap "rm -rf $TMPDIR" EXIT
+
+# Get balanced file lists from the balancer (JSON output parsed with jq)
+echo ""
+echo "=== Balancing Regular Specs ==="
+REGULAR_JSON=$("$SCRIPT_DIR/balance_specs" regular "$REGULAR_WORKERS")
+readarray -t REGULAR_GROUPS < <(echo "$REGULAR_JSON" | jq -r '.[] | join(" ")')
+
+echo ""
+echo "=== Balancing Pro Specs ==="
+PRO_JSON=$("$SCRIPT_DIR/balance_specs" pro "$PRO_WORKERS")
+readarray -t PRO_GROUPS < <(echo "$PRO_JSON" | jq -r '.[] | join(" ")')
+
+echo ""
+echo "Starting parallel execution..."
+
+# Launch regular spec workers
+PIDS=()
+for i in $(seq 0 $((REGULAR_WORKERS - 1))); do
+  if [ -n "${REGULAR_GROUPS[$i]}" ]; then
+    PARALLEL=true PARALLEL_GROUP_ID="regular_$i" SPECS_TYPE=regular \
+      bundle exec rspec ${REGULAR_GROUPS[$i]} --format progress \
+      > "$TMPDIR/regular_$i.log" 2>&1 &
+    PIDS+=($!)
+    echo "  Started regular worker $((i + 1)) (PID ${PIDS[-1]}): $(echo ${REGULAR_GROUPS[$i]} | wc -w) files"
+  fi
+done
+
+# Launch pro spec workers
+for i in $(seq 0 $((PRO_WORKERS - 1))); do
+  if [ -n "${PRO_GROUPS[$i]}" ]; then
+    PARALLEL=true PARALLEL_GROUP_ID="pro_$i" SPECS_TYPE=pro \
+      bundle exec rspec ${PRO_GROUPS[$i]} --format progress \
+      > "$TMPDIR/pro_$i.log" 2>&1 &
+    PIDS+=($!)
+    echo "  Started pro worker $((i + 1)) (PID ${PIDS[-1]}): $(echo ${PRO_GROUPS[$i]} | wc -w) files"
+  fi
+done
+
+echo ""
+echo "Waiting for ${#PIDS[@]} workers to complete..."
+
+# Wait for all processes and collect exit codes
+FAILED=0
+for idx in "${!PIDS[@]}"; do
+  PID="${PIDS[$idx]}"
+  if ! wait "$PID"; then
+    FAILED=1
+  fi
+done
+
+# Show all logs
+echo ""
+echo "=========================================="
+echo "                 RESULTS                  "
+echo "=========================================="
+
+for i in $(seq 0 $((REGULAR_WORKERS - 1))); do
+  if [ -f "$TMPDIR/regular_$i.log" ]; then
+    echo ""
+    echo "=== Regular Worker $((i + 1)) ==="
+    cat "$TMPDIR/regular_$i.log"
+  fi
+done
+
+for i in $(seq 0 $((PRO_WORKERS - 1))); do
+  if [ -f "$TMPDIR/pro_$i.log" ]; then
+    echo ""
+    echo "=== Pro Worker $((i + 1)) ==="
+    cat "$TMPDIR/pro_$i.log"
+  fi
+done
+
+if [ $FAILED -eq 1 ]; then
+  echo ""
+  echo "=========================================="
+  echo "         SOME SPECS FAILED!              "
+  echo "=========================================="
+  exit 1
+fi
+
+echo ""
+echo "=========================================="
+echo "          ALL SPECS PASSED!               "
+echo "=========================================="
+
+# Run coverage check
+"$SCRIPT_DIR/check_coverage"

data/docker-compose.yml

@@ -1,7 +1,7 @@
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:8.1.0
+    image: confluentinc/cp-kafka:8.1.1
 
     ports:
       - 9092:9092

data/karafka-web.gemspec

@@ -1,48 +1,48 @@
 # frozen_string_literal: true
 
-lib = File.expand_path('lib', __dir__)
+lib = File.expand_path("lib", __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
-require 'karafka/web/version'
+require "karafka/web/version"
 
 Gem::Specification.new do |spec|
-  spec.name        = 'karafka-web'
-  spec.version     = ::Karafka::Web::VERSION
-  spec.platform    = Gem::Platform::RUBY
-  spec.authors     = ['Maciej Mensfeld']
-  spec.email       = %w[contact@karafka.io]
-  spec.homepage    = 'https://karafka.io'
-  spec.summary     = 'Karafka ecosystem Web UI interface'
-  spec.description = 'Karafka ecosystem plug-and-play Web UI'
-  spec.licenses    = %w[LGPL-3.0-only Commercial]
-
-  spec.add_dependency 'erubi', '~> 1.4'
-  spec.add_dependency 'karafka', '>= 2.5.2', '< 2.6.0'
-  spec.add_dependency 'karafka-core', '>= 2.5.0', '< 2.6.0'
-  spec.add_dependency 'roda', '~> 3.69', '>= 3.69'
-  spec.add_dependency 'tilt', '~> 2.0'
-
-  spec.add_development_dependency 'rackup', '~> 0.2'
-
-  spec.required_ruby_version = '>= 3.2.0'
-
-  spec.executables   = %w[karafka-web]
+  spec.name = "karafka-web"
+  spec.version = ::Karafka::Web::VERSION
+  spec.platform = Gem::Platform::RUBY
+  spec.authors = ["Maciej Mensfeld"]
+  spec.email = %w[contact@karafka.io]
+  spec.homepage = "https://karafka.io"
+  spec.summary = "Karafka ecosystem Web UI interface"
+  spec.description = "Karafka ecosystem plug-and-play Web UI"
+  spec.licenses = %w[LGPL-3.0-only Commercial]
+
+  spec.add_dependency "erubi", "~> 1.4"
+  spec.add_dependency "karafka", ">= 2.5.2", "< 2.6.0"
+  spec.add_dependency "karafka-core", ">= 2.5.0", "< 2.6.0"
+  spec.add_dependency "roda", "~> 3.69", ">= 3.69"
+  spec.add_dependency "tilt", "~> 2.0"
+
+  spec.add_development_dependency "rackup", "~> 0.2"
+
+  spec.required_ruby_version = ">= 3.2.0"
+
+  spec.executables = %w[karafka-web]
   spec.require_paths = %w[lib]
 
   spec.files = `git ls-files -z`
-               .split("\x0")
-               .reject { |f| f.match(%r{^(spec)/}) }
-               .reject { |f| f.include?('.github')  }
-               .reject { |f| f.start_with?('.')  }
-               .reject { |f| f.end_with?('.map')  }
+    .split("\x0")
+    .reject { |f| f.match(%r{^(spec)/}) }
+    .reject { |f| f.include?(".github") }
+    .reject { |f| f.start_with?(".") }
+    .reject { |f| f.end_with?(".map") }
 
   spec.metadata = {
-    'funding_uri' => 'https://karafka.io/#become-pro',
-    'homepage_uri' => 'https://karafka.io',
-    'changelog_uri' => 'https://karafka.io/docs/Changelog-Karafka-Web-UI',
-    'bug_tracker_uri' => 'https://github.com/karafka/karafka-web/issues',
-    'source_code_uri' => 'https://github.com/karafka/karafka-web',
-    'documentation_uri' => 'https://karafka.io/docs',
-    'rubygems_mfa_required' => 'true'
+    "funding_uri" => "https://karafka.io/#become-pro",
+    "homepage_uri" => "https://karafka.io",
+    "changelog_uri" => "https://karafka.io/docs/Changelog-Karafka-Web-UI",
+    "bug_tracker_uri" => "https://github.com/karafka/karafka-web/issues",
+    "source_code_uri" => "https://github.com/karafka/karafka-web",
+    "documentation_uri" => "https://karafka.io/docs",
+    "rubygems_mfa_required" => "true"
   }
 end

data/lib/karafka/web/cli/base.rb

@@ -13,7 +13,7 @@ module Karafka
             ObjectSpace
               .each_object(Class)
               .select { |klass| klass.superclass == Karafka::Web::Cli::Base }
-              .reject { |klass| klass.to_s.end_with?('::Base') }
+              .reject { |klass| klass.to_s.end_with?("::Base") }
               .sort_by(&:name)
           end
         end
@@ -27,7 +27,7 @@ module Karafka
         #   if we are supposed to compute the factor automatically
         # @return [Integer] replication factor for Karafka Web UI topics
         def compute_replication_factor(cli_replication_factor)
-          cli_replication_factor || (Ui::Models::ClusterInfo.fetch.brokers.size > 1 ? 2 : 1)
+          cli_replication_factor || ((Ui::Models::ClusterInfo.fetch.brokers.size > 1) ? 2 : 1)
         end
       end
     end

data/lib/karafka/web/cli/help.rb

@@ -5,14 +5,14 @@ module Karafka
     class Cli
       # Displays help
       class Help < Base
-        desc 'Describes available commands'
+        desc "Describes available commands"
 
         # Print available commands
         def call
           # Find the longest command for alignment purposes
           max_command_length = self.class.commands.map { |command| command.name.size }.max
 
-          puts 'Karafka Web UI commands:'
+          puts "Karafka Web UI commands:"
 
           # Print each command formatted with its description
           self.class.commands.each do |command|

data/lib/karafka/web/cli/install.rb

@@ -5,13 +5,13 @@ module Karafka
     class Cli
       # Installs Web UI
       class Install < Base
-        desc 'Installs the Web UI'
+        desc "Installs the Web UI"
 
         option(
           :replication_factor,
-          'Replication factor for created topics',
+          "Replication factor for created topics",
           Integer,
-          ['--replication_factor [FACTOR]']
+          ["--replication_factor [FACTOR]"]
         )
 
         # Installs Karafka Web. Creates all needed topics, populates the data and adds the needed

data/lib/karafka/web/cli/migrate.rb

@@ -5,13 +5,13 @@ module Karafka
     class Cli
       # Migrates the Web UI topics and states if needed
       class Migrate < Base
-        desc 'Runs necessary migrations of Web UI topics and states'
+        desc "Runs necessary migrations of Web UI topics and states"
 
         option(
           :replication_factor,
-          'Replication factor for created topics',
+          "Replication factor for created topics",
           Integer,
-          ['--replication_factor [FACTOR]']
+          ["--replication_factor [FACTOR]"]
         )
 
         # Creates new topics (if any) and populates missing data.

data/lib/karafka/web/cli/reset.rb

@@ -5,13 +5,13 @@ module Karafka
     class Cli
       # Resets the Web UI
       class Reset < Base
-        desc 'Resets the Web UI by removing all the Web topics and creating them again'
+        desc "Resets the Web UI by removing all the Web topics and creating them again"
 
         option(
           :replication_factor,
-          'Replication factor for created topics',
+          "Replication factor for created topics",
           Integer,
-          ['--replication_factor [FACTOR]']
+          ["--replication_factor [FACTOR]"]
         )
 
         # Resets Karafka Web. Removes the topics, creates them again and populates the initial

data/lib/karafka/web/cli/uninstall.rb

@@ -5,7 +5,7 @@ module Karafka
     class Cli
       # Uninstalls the Web UI
       class Uninstall < Base
-        desc 'Removes all the Web UI topics and the enabled code'
+        desc "Removes all the Web UI topics and the enabled code"
 
         # Uninstalls Karafka Web
         def call

data/lib/karafka/web/config.rb

@@ -15,14 +15,22 @@ module Karafka
       # This is used both in the processing for eviction and in the UI
       setting :ttl, default: 30_000
 
-      # Producer for the Web UI. By default it is a `Karafka.producer`, however it may be
-      # overwritten if we want to use a separate instance in case of heavy usage of the
-      # transactional producer as a default. In cases like this, Karafka may not be able to report
-      # data because it uses this producer and it may be locked because of the transaction in a
-      # user space.
+      # Producer for the Web UI.
+      #
+      # By default it uses a low-intensity variant of `Karafka.producer` with reduced
+      # acknowledgment requirements (acks: 0) since Web UI reporting is not mission-critical
+      # and serves primarily analytical purposes. This minimizes overhead by using
+      # fire-and-forget semantics where occasional message loss is acceptable.
+      #
+      # For idempotent or transactional producers, the default producer is used unchanged
+      # since acks settings cannot be altered for these producer types.
+      #
+      # This setting may be overwritten if you want to use a completely separate producer
+      # instance, for example in case of heavy usage of the transactional producer as a default
+      # where Karafka may not be able to report data because it is locked in a user transaction.
       setting(
         :producer,
-        constructor: -> { ::Karafka.producer },
+        constructor: -> { Producer.new },
         lazy: true
       )
 
@@ -30,18 +38,18 @@ module Karafka
       # Karafka Web UI uses the Admin API for many operations, but there are few
       # (like states materialization) where a distinct consumer group is needed. In cases like that
       # this group id will be used
-      setting :group_id, default: 'karafka_web'
+      setting :group_id, default: "karafka_web"
 
       # Topics naming - used for processing and UI
       setting :topics do
         # All the errors encountered will be dispatched to this topic for inspection
         setting :errors do
-          setting :name, default: 'karafka_errors'
+          setting :name, default: "karafka_errors"
 
           # Remove really old errors (older than 3 months just to preserve space)
           setting :config, default: {
-            'cleanup.policy': 'delete',
-            'retention.ms': 3 * 31 * 24 * 60 * 60 * 1_000 # 3 months
+            "cleanup.policy": "delete",
+            "retention.ms": 3 * 31 * 24 * 60 * 60 * 1_000 # 3 months
           }
         end
 
@@ -50,7 +58,7 @@ module Karafka
           # information sent from each consumer process.
           setting :reports do
             # Name of the topic
-            setting :name, default: 'karafka_consumers_reports'
+            setting :name, default: "karafka_consumers_reports"
 
             # We do not need to to store this data for longer than 1 day as this data is only
             # used to materialize the end states
@@ -61,36 +69,36 @@ module Karafka
             # In case its not consumed because no processes are running, it also usually means
             # there's no data to consume because no karafka servers report
             setting :config, default: {
-              'cleanup.policy': 'delete',
-              'retention.ms': 24 * 60 * 60 * 1_000 # 1 day
+              "cleanup.policy": "delete",
+              "retention.ms": 24 * 60 * 60 * 1_000 # 1 day
             }
           end
 
           # Topic for storing states aggregated info
           setting :states do
-            setting :name, default: 'karafka_consumers_states'
+            setting :name, default: "karafka_consumers_states"
 
             # We care only about the most recent state, previous are irrelevant. So we can
             # easily compact after one minute. We do not use this beyond the most recent
             # collective state, hence it all can easily go away. We also limit the segment
             # size to at most 100MB not to use more space ever.
             setting :config, default: {
-              'cleanup.policy': 'compact',
-              'retention.ms': 60 * 60 * 1_000,
-              'segment.ms': 24 * 60 * 60 * 1_000, # 1 day
-              'segment.bytes': 104_857_600 # 100MB
+              "cleanup.policy": "compact",
+              "retention.ms": 60 * 60 * 1_000,
+              "segment.ms": 24 * 60 * 60 * 1_000, # 1 day
+              "segment.bytes": 104_857_600 # 100MB
             }
           end
 
           # Topic for storing consumers historical metrics info
           setting :metrics do
-            setting :name, default: 'karafka_consumers_metrics'
+            setting :name, default: "karafka_consumers_metrics"
 
             setting :config, default: {
-              'cleanup.policy': 'compact',
-              'retention.ms': 24 * 60 * 60 * 1_000, # 1 day
-              'segment.ms': 24 * 60 * 60 * 1_000, # 1 day
-              'segment.bytes': 104_857_600 # 100MB
+              "cleanup.policy": "compact",
+              "retention.ms": 24 * 60 * 60 * 1_000, # 1 day
+              "segment.ms": 24 * 60 * 60 * 1_000, # 1 day
+              "segment.bytes": 104_857_600 # 100MB
             }
           end
 
@@ -99,13 +107,13 @@ module Karafka
           # transition from one to another is smooth. Otherwise upgrade would require changes
           # to topics (migration) which may be more complex
           setting :commands do
-            setting :name, default: 'karafka_consumers_commands'
+            setting :name, default: "karafka_consumers_commands"
 
             setting :config, default: {
-              'cleanup.policy': 'delete',
-              'retention.ms': 7 * 24 * 60 * 60 * 1_000, # 7 days
-              'segment.ms': 24 * 60 * 60 * 1_000, # 1 day
-              'segment.bytes': 104_857_600 # 100MB
+              "cleanup.policy": "delete",
+              "retention.ms": 7 * 24 * 60 * 60 * 1_000, # 7 days
+              "segment.ms": 24 * 60 * 60 * 1_000, # 1 day
+              "segment.bytes": 104_857_600 # 100MB
             }
           end
         end
@@ -199,7 +207,7 @@ module Karafka
         setting :kafka, default: {
           # We do not use at the moment the `#eofed` flag for anything, thus there is no point in
           # having it set to true if user users it.
-          'enable.partition.eof': false
+          "enable.partition.eof": false
         }.freeze
       end
 
@@ -208,13 +216,13 @@ module Karafka
         # Should be set per ENV.
         setting :sessions do
           # Cookie key name
-          setting :key, default: '_karafka_session'
+          setting :key, default: "_karafka_session"
 
           # Rack middleware session env key. We use independent key from "rack.session" here to
           # prevent our data from leaking to the main app (when mounted) and the other way around.
           # This also prevents us from overloading the session object with extra data that could
           # cause it to go beyond a cookie limit.
-          setting :env_key, default: 'karafka.session'
+          setting :env_key, default: "karafka.session"
 
           # Secret for the session cookie
           setting :secret, default: SecureRandom.hex(32)
@@ -299,7 +307,7 @@ module Karafka
           # that might have been committed via transactional producer, we would wait for 1 second
           # to get needed data. If you are experiencing timeouts or other issues with the Web IU
           # interface, you can increase this.
-          'fetch.wait.max.ms': 100
+          "fetch.wait.max.ms": 100
         }
       end
     end

data/lib/karafka/web/contracts/base.rb

@@ -14,8 +14,8 @@ module Karafka
 
             super do |config|
               config.error_messages = YAML.safe_load_file(
-                File.join(Karafka::Web.gem_root, 'config', 'locales', 'errors.yml')
-              ).fetch('en').fetch('validations').fetch('web')
+                File.join(Karafka::Web.gem_root, "config", "locales", "errors.yml")
+              ).fetch("en").fetch("validations").fetch("web")
             end
           end
         end

data/lib/karafka/web/deserializer.rb

@@ -13,11 +13,11 @@ module Karafka
       # @param message [::Karafka::Messages::Message]
       # @return [Object] deserialized data
       def call(message)
-        raw_payload = if message.headers.key?('zlib')
-                        Zlib::Inflate.inflate(message.raw_payload)
-                      else
-                        message.raw_payload
-                      end
+        raw_payload = if message.headers.key?("zlib")
+          Zlib::Inflate.inflate(message.raw_payload)
+        else
+          message.raw_payload
+        end
 
         ::JSON.parse(
           raw_payload,

data/lib/karafka/web/installer.rb

@@ -14,20 +14,20 @@ module Karafka
       def install(replication_factor: 1)
         enable!
         puts
-        puts 'Installing Karafka Web UI...'
+        puts "Installing Karafka Web UI..."
         puts
         Management::Actions::ExtendBootFile.new.call
         puts
-        puts 'Creating necessary topics and populating state data...'
+        puts "Creating necessary topics and populating state data..."
         puts
         Management::Actions::CreateTopics.new.call(replication_factor)
         wait_for_topics
         Management::Actions::CreateInitialStates.new.call
         puts
-        puts 'Running data migrations...'
+        puts "Running data migrations..."
         Management::Actions::MigrateStatesData.new.call
         puts
-        puts("Installation #{green('completed')}. Have fun!")
+        puts("Installation #{green("completed")}. Have fun!")
         puts
       end
 
@@ -38,16 +38,16 @@ module Karafka
       def migrate(replication_factor: 1)
         enable!
         puts
-        puts 'Creating necessary topics and populating state data...'
+        puts "Creating necessary topics and populating state data..."
         puts
         Management::Actions::CreateTopics.new.call(replication_factor)
         wait_for_topics
         Management::Actions::CreateInitialStates.new.call
         puts
-        puts 'Running data migrations...'
+        puts "Running data migrations..."
         Management::Actions::MigrateStatesData.new.call
         puts
-        puts("Migration #{green('completed')}. Have fun!")
+        puts("Migration #{green("completed")}. Have fun!")
         puts
       end
 
@@ -56,7 +56,7 @@ module Karafka
       def reset(replication_factor: 1)
         enable!
         puts
-        puts 'Resetting Karafka Web UI...'
+        puts "Resetting Karafka Web UI..."
         puts
         Management::Actions::DeleteTopics.new.call
         puts
@@ -64,10 +64,10 @@ module Karafka
         wait_for_topics
         Management::Actions::CreateInitialStates.new.call
         puts
-        puts 'Running data migrations...'
+        puts "Running data migrations..."
         Management::Actions::MigrateStatesData.new.call
         puts
-        puts("Resetting #{green('completed')}. Have fun!")
+        puts("Resetting #{green("completed")}. Have fun!")
         puts
       end
 
@@ -75,12 +75,12 @@ module Karafka
       def uninstall
         enable!
         puts
-        puts 'Uninstalling Karafka Web UI...'
+        puts "Uninstalling Karafka Web UI..."
         puts
         Management::Actions::DeleteTopics.new.call
         Management::Actions::CleanBootFile.new.call
         puts
-        puts("Uninstalling #{green('completed')}. Goodbye!")
+        puts("Uninstalling #{green("completed")}. Goodbye!")
         puts
       end
 
@@ -97,7 +97,7 @@ module Karafka
       # bit of time just in case.
       def wait_for_topics
         puts
-        print 'Waiting for the topics to synchronize in the cluster'
+        print "Waiting for the topics to synchronize in the cluster"
         wait(5)
         puts
       end
@@ -107,7 +107,7 @@ module Karafka
       def wait(time_in_seconds)
         time_in_seconds.times do
           sleep(1)
-          print '.'
+          print "."
         end
 
         print "\n"

data/lib/karafka/web/management/actions/base.rb

@@ -14,20 +14,20 @@ module Karafka
 
           # @return [String] green colored word "successfully"
           def successfully
-            green('successfully')
+            green("successfully")
           end
 
           # @return [String] green colored word "already"
           def already
-            green('already')
+            green("already")
           end
 
           # @return [Array<String>] topics available in the cluster
           def existing_topics_names
             @existing_topics_names ||= ::Karafka::Admin
-                                       .cluster_info
-                                       .topics
-                                       .map { |topic| topic[:topic_name] }
+              .cluster_info
+              .topics
+              .map { |topic| topic[:topic_name] }
           end
         end
       end

data/lib/karafka/web/management/actions/clean_boot_file.rb

@@ -16,14 +16,14 @@ module Karafka
             karafka_rb = File.readlines(Karafka.boot_file)
 
             if karafka_rb.any? { |line| line.include?(ENABLER_CODE) }
-              puts 'Updating the Karafka boot file...'
+              puts "Updating the Karafka boot file..."
               karafka_rb.delete_if { |line| line.include?(ENABLER_CODE) }
 
               File.write(Karafka.boot_file, karafka_rb.join)
               puts "Karafka boot file #{successfully} updated."
-              puts 'Make sure to remove configuration and other customizations as well.'
+              puts "Make sure to remove configuration and other customizations as well."
             else
-              puts 'Karafka Web UI components not found in the boot file.'
+              puts "Karafka Web UI components not found in the boot file."
             end
           end
         end