karafka-web 0.1.0

data/lib/karafka/web/ui/models/process.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Models
+        # Single consumer process representation
+        class Process < Lib::HashProxy
+          class << self
+            # Looks for a given process based on its id
+            # @param state [State] state of the system based on which we will do the lookup
+            # @param process_id [String] id of the process we are looking for
+            # @return [Process] selected process or error raised
+            # @raise [::Karafka::Web::Errors::Ui::NotFoundError] raised if process not found
+            def find(state, process_id)
+              found_process = Processes.active(state).find { |process| process.id == process_id }
+              found_process || raise(::Karafka::Web::Errors::Ui::NotFoundError, process_id)
+            end
+          end
+
+          # @return [String] process id without the name and ip
+          def id
+            @id ||= name.split(':').last
+          end
+
+          # @return [Array<ConsumerGroup>] consumer groups to which this process is subscribed in
+          #   an alphabetical order
+          def consumer_groups
+            super
+              .values
+              .map { |cg_hash| ConsumerGroup.new(cg_hash) }
+              .sort_by(&:id)
+          end
+
+          # Jobs sorted from longest running to youngest
+          # @return [Array<Job>] current jobs of this process
+          def jobs
+            super
+              .map { |job| Job.new(job) }
+              .sort_by(&:started_at)
+          end
+
+          # @return [Integer] collective lag on this process
+          def lag_stored
+            consumer_groups
+              .flat_map(&:topics)
+              .flat_map(&:partitions)
+              .map(&:lag_stored)
+              .delete_if(&:negative?)
+              .sum
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/models/processes.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Models
+        # Represents the active processes data
+        # @note Active also includes processes stopped recently. We use it to provide better
+        #   visibility via UI.
+        module Processes
+          class << self
+            include ::Karafka::Core::Helpers::Time
+
+            # Returns the active processes in an array and alongside of that the current state of
+            # the system. We use those together in the UI and it would be expensive to pick it up
+            # while we've already had it.
+            #
+            # @param state [State] current system state from which we can get processes metadata
+            # @return [Array<Process>]
+            def active(state)
+              processes = fetch_reports(state)
+              evict_expired_processes(processes)
+              processes = squash_processes_data(processes)
+              processes = sort_processes(processes)
+
+              processes.map { |process_hash| Process.new(process_hash) }
+            end
+
+            private
+
+            # Fetches the relevant processes reports from the reports topic
+            # @param state [State]
+            # @return [Array<Hash>] array with deserialized processes reports
+            def fetch_reports(state)
+              offsets = state[:processes]
+                        .values
+                        .map { |process| process[:offset] }
+                        .sort
+
+              ::Karafka::Admin.read_topic(
+                ::Karafka::Web.config.topics.consumers.reports,
+                0,
+                # We set 10k here because we start from the latest offset of the reports, hence
+                # we will never get this much. Do do not know however exactly how many reports
+                # we may get as for some processes we may get few if the reporting interval
+                # was bypassed by state changes in the processes
+                10_000,
+                offsets.first || -1
+              ).map(&:payload)
+            end
+
+            # Collapses processes data and only keeps the most recent report for give process
+            # @param processes [Array<Hash>]
+            # @return [Array<Hash>] unique processes data
+            def squash_processes_data(processes)
+              processes
+                .reverse
+                .uniq { |consumer| consumer[:process][:name] }
+                .reverse
+            end
+
+            # Removes processes that are no longer active. They may still be here because the state
+            # may have a small lag but we want to compensate for it that way.
+            # @param processes [Array<Hash>]
+            # @return [Array<Hash>] only active processes data
+            def evict_expired_processes(processes)
+              max_ttl = ::Karafka::Web.config.ttl / 1_000
+              now = float_now
+
+              processes.delete_if do |details|
+                now - details[:dispatched_at] > max_ttl
+              end
+            end
+
+            # Ensures that we always return processes sorted by their name
+            # @param processes [Array<Hash>]
+            # @return [Array<Hash>] sorted processes data
+            def sort_processes(processes)
+              processes.sort_by { |consumer| consumer[:process][:name] }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/models/state.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Models
+        # Represents the current consumer processes aggregated state
+        # This state is the core of Karafka reporting. It holds the most important aggregated data
+        # as well as pointers to states of particular consumers and their details.
+        class State < Lib::HashProxy
+          extend ::Karafka::Core::Helpers::Time
+
+          class << self
+            # @return [State, false] current aggregated state or false if not found
+            # @note Current state may contain expired data, for example of processes that were
+            #   forcefully killed, etc. We clean this prior to returning the state.
+            def current
+              state = fetch
+
+              return false unless state
+
+              state = state.payload
+              evict_expired_processes(state)
+              sort_processes(state)
+
+              new(state)
+            end
+
+            # @return [State] current aggregated state
+            # @raise [::Karafka::Web::Errors::Ui::NotFoundError] raised when there is no current
+            #   state. Probably because `karafka server` was never executed
+            def current!
+              current || raise(::Karafka::Web::Errors::Ui::NotFoundError)
+            end
+
+            private
+
+            # @return [::Karafka::Messages::Message, nil] most recent state or nil if none
+            def fetch
+              ::Karafka::Admin.read_topic(
+                Karafka::Web.config.topics.consumers.states,
+                0,
+                1
+              ).last
+            end
+
+            # Evicts (removes) details about processes that are beyond our TTL on liveliness
+            # @param state_hash [Hash] raw message state hash
+            def evict_expired_processes(state_hash)
+              max_ttl = ::Karafka::Web.config.ttl / 1_000
+
+              state_hash[:processes].delete_if do |_, details|
+                float_now - details[:dispatched_at] > max_ttl
+              end
+            end
+
+            # Sorts the processes based on their unique ids, so they are always in order
+            # @param state_hash [Hash] raw message state hash
+            def sort_processes(state_hash)
+              state_hash[:processes] = state_hash[:processes].to_a.sort_by(&:first).to_h
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/models/topic.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Models
+        # Single topic data representation model
+        class Topic < Lib::HashProxy
+          # @return [Array<Partition>] All topic partitions data
+          def partitions
+            super.values.map do |partition_hash|
+              Partition.new(partition_hash)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/pro/app.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Ui
+      # Pro Web UI components
+      module Pro
+        # Main Roda Web App that servers all the metrics and stats
+        class App < Ui::Base
+          opts[:root] = Karafka::Web.gem_root.join('lib/karafka/web/ui/pro')
+
+          instance_exec(&CONTEXT_DETAILS)
+
+          plugin :render, escape: true, engine: 'erb', allowed_paths: [
+            Karafka::Web.gem_root.join('lib/karafka/web/ui/pro/views'),
+            Karafka::Web.gem_root.join('lib/karafka/web/ui/views')
+          ]
+
+          plugin :additional_view_directories, [
+            Karafka::Web.gem_root.join('lib/karafka/web/ui/views')
+          ]
+
+          route do |r|
+            r.root { r.redirect root_path('consumers') }
+
+            @current_page = params.current_page
+
+            r.on 'consumers' do
+              controller = Controllers::Consumers.new(params)
+
+              r.get String, 'jobs' do |process_id|
+                render_response controller.jobs(process_id)
+              end
+
+              r.get String, 'subscriptions' do |process_id|
+                render_response controller.subscriptions(process_id)
+              end
+
+              r.get do
+                @breadcrumbs = false
+                render_response controller.index
+              end
+            end
+
+            r.get 'jobs' do
+              controller = Controllers::Jobs.new(params)
+              render_response controller.index
+            end
+
+            r.on 'routing' do
+              controller = Controllers::Routing.new(params)
+
+              r.get String do |topic_id|
+                render_response controller.show(topic_id)
+              end
+
+              r.get do
+                render_response controller.index
+              end
+            end
+
+            r.on 'explorer' do
+              controller = Controllers::Explorer.new(params)
+
+              r.get String, Integer, Integer do |topic_id, partition_id, offset|
+                render_response controller.show(topic_id, partition_id, offset)
+              end
+
+              r.get String, Integer do |topic_id, partition_id|
+                render_response controller.partition(topic_id, partition_id)
+              end
+
+              r.get do
+                render_response controller.index
+              end
+            end
+
+            r.get 'health' do
+              controller = Controllers::Health.new(params)
+              render_response controller.index
+            end
+
+            r.get 'cluster' do
+              controller = Controllers::Cluster.new(params)
+              render_response controller.index
+            end
+
+            r.on 'errors' do
+              controller = Controllers::Errors.new(params)
+
+              r.get Integer do |partition_id|
+                render_response controller.index(partition_id)
+              end
+
+              r.get Integer, Integer do |partition_id, offset|
+                render_response controller.show(partition_id, offset)
+              end
+            end
+
+            r.get 'dlq' do
+              controller = Controllers::Dlq.new(params)
+              render_response controller.index
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/pro/controllers/cluster.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Pro
+        # Namespace for Pro controllers
+        module Controllers
+          # Cluster controller
+          class Cluster < Ui::Controllers::Cluster
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/pro/controllers/consumers.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Ui
+      module Pro
+        module Controllers
+          # Controller for displaying consumers states and details about them
+          class Consumers < Ui::Controllers::Base
+            # Consumers list
+            def index
+              @current_state = Models::State.current!
+              processes_total = Models::Processes.active(@current_state)
+              @counters = Lib::HashProxy.new(@current_state[:stats])
+              @processes, @next_page = Lib::PaginateArray.new.call(
+                processes_total,
+                @params.current_page
+              )
+
+              respond
+            end
+
+            # @param process_id [String] id of the process we're interested in
+            def jobs(process_id)
+              current_state = Models::State.current!
+              @process = Models::Process.find(current_state, process_id)
+
+              respond
+            end
+
+            # @param process_id [String] id of the process we're interested in
+            def subscriptions(process_id)
+              current_state = Models::State.current!
+              @process = Models::Process.find(current_state, process_id)
+
+              respond
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/pro/controllers/dlq.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Ui
+      module Pro
+        module Controllers
+          # DLQ topics overview
+          class Dlq < Ui::Controllers::Base
+            # Lists DLQ topics
+            def index
+              topics = Karafka::App.consumer_groups.flat_map(&:topics).flat_map(&:to_a)
+
+              dlq_topic_names = topics
+                                .map { |source_topic| source_topic.dead_letter_queue.topic }
+                                .uniq
+                                .compact
+                                .select(&:itself)
+
+              @dlq_topics = Karafka::Admin
+                            .cluster_info
+                            .topics
+                            .select { |topic| dlq_topic_names.include?(topic[:topic_name]) }
+                            .sort_by { |topic| topic[:topic_name] }
+
+              respond
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/pro/controllers/errors.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Ui
+      module Pro
+        module Controllers
+          # Errors details controller
+          class Errors < Ui::Controllers::Base
+            # @param partition_id [Integer] id of the partition of errors we are interested in
+            def index(partition_id)
+              errors_topic = ::Karafka::Web.config.topics.errors
+              @partition_id = partition_id
+              @previous_page, @error_messages, @next_page, @partitions_count = \
+                Models::Message.page(
+                  errors_topic,
+                  @partition_id,
+                  @params.current_page
+                )
+
+              respond
+            end
+
+            # Shows given error details
+            #
+            # @param partition_id [Integer]
+            # @param offset [Integer]
+            def show(partition_id, offset)
+              errors_topic = ::Karafka::Web.config.topics.errors
+
+              @partition_id = partition_id
+              @offset = offset
+              @error_message = Models::Message.find(
+                errors_topic,
+                @partition_id,
+                @offset
+              )
+
+              respond
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/pro/controllers/explorer.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Ui
+      module Pro
+        module Controllers
+          # Data explorer controller
+          class Explorer < Ui::Controllers::Base
+            # Lists all the topics we can explore
+            def index
+              @topics = Karafka::Admin
+                        .cluster_info
+                        .topics
+                        .reject { |topic| topic[:topic_name] == '__consumer_offsets' }
+                        .sort_by { |topic| topic[:topic_name] }
+
+              respond
+            end
+
+            # Shows messages available in a given partition
+            #
+            # @param topic_id [String]
+            # @param partition_id [Integer]
+            def partition(topic_id, partition_id)
+              @topic_id = topic_id
+              @partition_id = partition_id
+              @previous_page, @messages, @next_page, @partitions_count = Ui::Models::Message.page(
+                @topic_id,
+                @partition_id,
+                @params.current_page
+              )
+
+              respond
+            end
+
+            # Displays given message
+            #
+            # @param topic_id [String]
+            # @param partition_id [Integer]
+            # @param offset [Integer] offset of the message we want to display
+            def show(topic_id, partition_id, offset)
+              @topic_id = topic_id
+              @partition_id = partition_id
+              @offset = offset
+              @message = Ui::Models::Message.find(@topic_id, @partition_id, @offset)
+              @payload_error = false
+
+              @decrypt = if ::Karafka::App.config.encryption.active
+                           ::Karafka::Web.config.ui.decrypt
+                         else
+                           true
+                         end
+
+              begin
+                @pretty_payload = JSON.pretty_generate(@message.payload)
+              rescue StandardError => e
+                @payload_error = e
+              end
+
+              respond
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/pro/controllers/health.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Ui
+      module Pro
+        module Controllers
+          # Health state controller
+          class Health < Ui::Controllers::Base
+            # Displays the current system state
+            def index
+              current_state = Models::State.current!
+              @stats = Models::Health.current(current_state)
+
+              respond
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/pro/controllers/jobs.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Ui
+      module Pro
+        module Controllers
+          # Displays list of active jobs
+          class Jobs < Ui::Controllers::Jobs
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/pro/controllers/routing.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Ui
+      module Pro
+        module Controllers
+          # Routing details - same as in OSS
+          class Routing < Ui::Controllers::Routing
+          end
+        end
+      end
+    end
+  end
+end