karafka-web 0.9.2 → 0.10.0.beta1

data/lib/karafka/web/management/migrations/consumers_metrics/0_set_initial.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      module Migrations
+        # Consumers metrics topic data related migrations
+        module ConsumersMetrics
+          # Initial migration that sets the consumers metrics initial first state.
+          # This is the basic of metrics as they were when they were introduced.
+          class SetInitial < Base
+            # Always migrate from empty up
+            self.versions_until = '0.0.1'
+            self.type = :consumers_metrics
+
+            # @param state [Hash] initial empty state
+            def migrate(state)
+              state.merge!(
+                aggregated: {
+                  days: [],
+                  hours: [],
+                  minutes: [],
+                  seconds: []
+                },
+                consumer_groups: {
+                  days: [],
+                  hours: [],
+                  minutes: [],
+                  seconds: []
+                },
+                dispatched_at: float_now
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/management/migrations/consumers_metrics/1699543515_fill_missing_received_and_sent_bytes.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      module Migrations
+        module ConsumersMetrics
+          # Adds bytes_sent and bytes_received to all the aggregated metrics samples, so we have
+          # charts that do not have to fill gaps or check anything
+          class FillMissingReceivedAndSentBytes < Base
+            self.versions_until = '1.1.0'
+            self.type = :consumers_metrics
+
+            # @param state [Hash] metrics state
+            def migrate(state)
+              state[:aggregated].each_value do |metrics|
+                metrics.each do |metric|
+                  metric.last[:bytes_sent] = 0
+                  metric.last[:bytes_received] = 0
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/management/migrations/consumers_metrics/1700234522_introduce_waiting.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      module Migrations
+        module ConsumersMetrics
+          # Introduce waiting in consumers metrics to complement busy and enqueued for jobs metrics
+          class IntroduceWaiting < Base
+            self.versions_until = '1.1.1'
+            self.type = :consumers_metrics
+
+            # @param state [Hash]
+            def migrate(state)
+              state[:aggregated].each_value do |metrics|
+                metrics.each do |metric|
+                  metric.last[:waiting] = 0
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/management/migrations/consumers_metrics/1700234522_remove_processing.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      module Migrations
+        module ConsumersMetrics
+          # Moves unused "processing" that was used instead of "busy" in older versions
+          class RemoveProcessing < Base
+            self.versions_until = '1.1.1'
+            self.type = :consumers_metrics
+
+            # @param state [Hash]
+            def migrate(state)
+              state[:aggregated].each_value do |metrics|
+                metrics.each do |metric|
+                  metric.last.delete(:processing)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/management/migrations/consumers_metrics/1704722380_split_listeners_into_active_and_paused.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      module Migrations
+        module ConsumersMetrics
+          # Since we have introduced notion of pause listeners, we need to reflect this in the
+          # UI, so the scaling changes are visible
+          class SplitListenersIntoActiveAndPaused < Base
+            self.versions_until = '1.1.2'
+            self.type = :consumers_metrics
+
+            # @param state [Hash]
+            def migrate(state)
+              state[:aggregated].each_value do |metrics|
+                metrics.each do |metric|
+                  listeners = if metric.last.key?(:listeners)
+                                metric.last[:listeners].to_i
+                              elsif metric.last.key?(:listeners_count)
+                                metric.last[:listeners_count].to_i
+                              else
+                                0
+                              end
+
+                  metric.last[:listeners] = {
+                    active: listeners,
+                    standby: 0
+                  }
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/management/migrations/consumers_metrics/1706607960_introduce_lag_total.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      module Migrations
+        module ConsumersMetrics
+          # Moves to using lag total as a normalization for both lags
+          class IntroduceLagTotal < Base
+            self.versions_until = '1.2.0'
+            self.type = :consumers_metrics
+
+            # @param state [Hash]
+            def migrate(state)
+              state[:aggregated].each_value do |metrics|
+                metrics.each do |metric|
+                  metric.last[:lag_total] = metric.last[:lag_stored]
+                  metric.last.delete(:lag_stored)
+                  metric.last.delete(:lag)
+                end
+              end
+
+              state[:consumer_groups].each_value do |metrics|
+                metrics.each do |metric_group|
+                  metric_group.last.each_value do |metric|
+                    metric.each_value do |sample|
+                      sample[:lag_total] = sample[:lag_stored]
+                      sample.delete(:lag_stored)
+                      sample.delete(:lag)
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/management/migrations/consumers_metrics/1706611396_rename_lag_total_to_lag_hybrid.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      module Migrations
+        module ConsumersMetrics
+          # Renames total lag to hybrid to better represent what it is
+          class RenameLagTotalToLagHybrid < Base
+            self.versions_until = '1.2.1'
+            self.type = :consumers_metrics
+
+            # @param state [Hash]
+            def migrate(state)
+              state[:aggregated].each_value do |metrics|
+                metrics.each do |metric|
+                  metric.last[:lag_hybrid] = metric.last[:lag_total] || 0
+                  metric.last.delete(:lag_total)
+                end
+              end
+
+              state[:consumer_groups].each_value do |metrics|
+                metrics.each do |metric_group|
+                  metric_group.last.each_value do |metric|
+                    metric.each_value do |sample|
+                      sample[:lag_hybrid] = sample[:lag_total]
+                      sample.delete(:lag_total)
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/management/migrations/consumers_metrics/1716218393_populate_jobs_metrics.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      module Migrations
+        module ConsumersMetrics
+          # Adds the jobs metric matching batches for aggregated metrics results
+          class PopulateJobsMetrics < Base
+            self.versions_until = '1.3.0'
+            self.type = :consumers_metrics
+
+            # @param state [Hash]
+            def migrate(state)
+              state[:aggregated].each_value do |metrics|
+                metrics.each do |metric|
+                  metric.last[:jobs] = metric.last[:batches] || 0
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/management/migrations/consumers_states/0_set_initial.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      module Migrations
+        # Consumers states topic data related migrations
+        module ConsumersStates
+          # Initial migration that sets the consumers state initial first state.
+          # This is the basic of state as they were when they were introduced.
+          class SetInitial < Base
+            # Run this only on the first setup
+            self.versions_until = '0.0.1'
+            self.type = :consumers_states
+
+            # @param state [Hash]
+            def migrate(state)
+              state.merge!(
+                processes: {},
+                stats: {
+                  batches: 0,
+                  messages: 0,
+                  retries: 0,
+                  dead: 0,
+                  busy: 0,
+                  enqueued: 0,
+                  processing: 0,
+                  workers: 0,
+                  processes: 0,
+                  rss: 0,
+                  listeners: 0,
+                  utilization: 0,
+                  errors: 0,
+                  lag_stored: 0,
+                  lag: 0
+                },
+                schema_state: 'accepted',
+                dispatched_at: float_now
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/management/migrations/consumers_states/1699543515_fill_missing_received_and_sent_bytes.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      module Migrations
+        module ConsumersStates
+          # Similar to filling in consumers metrics, we initialize this with zeros so it is always
+          # present as expected
+          class FillMissingReceivedAndSentBytes < Base
+            # Network metrics were introduced with schema 1.2.0
+            self.versions_until = '1.2.0'
+            self.type = :consumers_states
+
+            # @param state [Hash]
+            def migrate(state)
+              state[:stats][:bytes_sent] = 0
+              state[:stats][:bytes_received] = 0
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/management/migrations/consumers_states/1700234522_introduce_waiting.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      module Migrations
+        module ConsumersStates
+          # Introduce waiting in consumers metrics to complement busy and enqueued for jobs stats
+          class IntroduceWaiting < Base
+            self.versions_until = '1.2.1'
+            self.type = :consumers_states
+
+            # @param state [Hash]
+            def migrate(state)
+              state[:stats][:waiting] = 0
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/management/migrations/consumers_states/1700234522_remove_processing.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      module Migrations
+        module ConsumersStates
+          # Moves unused "processing" that was used instead of "busy" in older versions
+          class RemoveProcessing < Base
+            self.versions_until = '1.2.1'
+            self.type = :consumers_states
+
+            # @param state [Hash]
+            def migrate(state)
+              state[:stats].delete(:processing)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/management/migrations/consumers_states/1704722380_split_listeners_into_active_and_paused.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      module Migrations
+        module ConsumersStates
+          # Since we have introduced notion of pause listeners, we need to reflect this in the
+          # UI, so the scaling changes are visible
+          class SplitListenersIntoActiveAndPaused < Base
+            self.versions_until = '1.2.2'
+            self.type = :consumers_states
+
+            # @param state [Hash]
+            def migrate(state)
+              listeners = if state[:stats].key?(:listeners)
+                            state[:stats][:listeners].to_i
+                          elsif state[:stats].key?(:listeners_count)
+                            state[:stats][:listeners_count].to_i
+                          else
+                            0
+                          end
+
+              state[:stats][:listeners] = {
+                active: listeners,
+                standby: 0
+              }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/management/migrations/consumers_states/1706607960_introduce_lag_total.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      module Migrations
+        module ConsumersStates
+          # Moves to using lag total as a normalization for both lags
+          class IntroduceLagTotal < Base
+            self.versions_until = '1.3.0'
+            self.type = :consumers_states
+
+            # @param state [Hash]
+            def migrate(state)
+              state[:stats][:lag_total] = state[:stats][:lag_stored]
+              state[:stats].delete(:lag)
+              state[:stats].delete(:lag_stored)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/management/migrations/consumers_states/1706611396_rename_lag_total_to_lag_hybrid.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      module Migrations
+        module ConsumersStates
+          # Renames total lag to hybrid to better represent what it is
+          class RenameLagTotalToLagHybrid < Base
+            self.versions_until = '1.3.1'
+            self.type = :consumers_states
+
+            # @param state [Hash]
+            def migrate(state)
+              state[:stats][:lag_hybrid] = state[:stats][:lag_total] || 0
+              state[:stats].delete(:lag_total)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/management/migrations/consumers_states/1716218393_add_jobs_counter.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      module Migrations
+        module ConsumersStates
+          # Adds the jobs counter matching batches
+          class AddJobsCounter < Base
+            self.versions_until = '1.4.0'
+            self.type = :consumers_states
+
+            # @param state [Hash]
+            def migrate(state)
+              # If missing we copy-paste from batches as the closest matching value.
+              # Since batch always has a job, this is a good starting point
+              state[:stats][:jobs] = state[:stats][:batches]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/management/migrator.rb

@@ -52,7 +52,7 @@ module Karafka
         # any states we manage. We can only move forward, so attempt to migrate for example from
         # 1.0.0 to 0.9.0 should be considered and error.
         def ensure_migrable!
-          if consumers_state[:schema_version] > State::SCHEMA_VERSION
+          if consumers_states[:schema_version] > State::SCHEMA_VERSION
             raise(
               Errors::Management::IncompatibleSchemaError,
               'consumers state newer than supported'
@@ -90,21 +90,21 @@ module Karafka
 
         # Publishes all the states migrated records
         def publish
-          consumers_state[:schema_version] = State::SCHEMA_VERSION
+          consumers_states[:schema_version] = State::SCHEMA_VERSION
           consumers_metrics[:schema_version] = Metrics::SCHEMA_VERSION
 
           # Migrator may run in the context of the processing consumer prior to any states
           # fetching related to processing. We use sync to make sure, that the following
           # processing related states fetched fetch the new states
           Processing::Publisher.publish!(
-            consumers_state,
+            consumers_states,
             consumers_metrics
           )
         end
 
         # @return [Hash] current consumers states most recent state
-        def consumers_state
-          @consumers_state ||= Processing::Consumers::State.current!
+        def consumers_states
+          @consumers_states ||= Processing::Consumers::State.current!
         end
 
         # @return [Hash] current consumers metrics most recent state

data/lib/karafka/web/pro/commanding/commands/base.rb

@@ -25,6 +25,14 @@ module Karafka
             def process_id
               @process_id ||= ::Karafka::Web.config.tracking.consumers.sampler.process_id
             end
+
+            # @return [Boolean] Is given process to which a command was sent operating in an
+            #   standalone mode. We need to know this, because some commands are prohibited from
+            #   being executed in the embedded or swarm processes since there the signaling is
+            #   handled differently (either via the main process or supervisor).
+            def standalone?
+              Karafka::Server.execution_mode == :standalone
+            end
           end
         end
       end

data/lib/karafka/web/pro/commanding/commands/quiet.rb

@@ -18,9 +18,12 @@ module Karafka
         # Namespace for commands the process can react to
         module Commands
           # Sends a signal to quiet the consumer
+          # @note Does not work in an embedded mode because we do not own the Ruby process.
           class Quiet < Base
-            # Performs the command
+            # Performs the command if not in embedded mode
             def call
+              return unless standalone?
+
               ::Process.kill('TSTP', ::Process.pid)
             end
           end

data/lib/karafka/web/pro/commanding/commands/stop.rb

@@ -18,9 +18,12 @@ module Karafka
         # Namespace for commands the process can react to
         module Commands
           # Sends a signal to stop the process
+          # @note Does not work in an embedded mode because we do not own the Ruby process.
           class Stop < Base
-            # Performs the command
+            # Performs the command if not in embedded mode
             def call
+              return unless standalone?
+
               ::Process.kill('QUIT', ::Process.pid)
             end
           end

data/lib/karafka/web/pro/loader.rb

@@ -25,6 +25,12 @@ module Karafka
             config.instance_eval do
               setting(:commanding, default: Commanding::Config.config)
             end
+
+            # Expand UI config with extra search capabilities settings
+            config.ui.instance_eval do
+              setting(:search, default: Ui::Lib::Search::Config.config)
+              setting(:policies, default: Ui::Lib::Policies::Config.config)
+            end
           end
 
           # Runs post setup features configuration operations
@@ -32,6 +38,8 @@ module Karafka
           # @param config [Karafka::Core::Configurable::Node]
           def post_setup_all(config)
             Commanding.post_setup(config)
+            Ui::Lib::Search.post_setup(config)
+            Ui::Lib::Policies.post_setup(config)
           end
         end
       end