karafka-web 0.1.0

data/lib/karafka/web/ui/controllers/cluster.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Controllers
+        # Selects cluster info and topics basic info
+        class Cluster < Base
+          # List cluster info data
+          def index
+            @cluster_info = Karafka::Admin.cluster_info
+
+            @topics = @cluster_info
+                      .topics
+                      .reject { |topic| topic[:topic_name] == '__consumer_offsets' }
+                      .sort_by { |topic| topic[:topic_name] }
+
+            respond
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/controllers/consumers.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Controllers
+        # Consumers (consuming processes - `karafka server`) processes display consumer
+        class Consumers < Base
+          # List page with consumers
+          # @note For now we load all and paginate over the squashed data.
+          def index
+            @current_state = Models::State.current!
+            processes_total = Models::Processes.active(@current_state)
+
+            @counters = Lib::HashProxy.new(@current_state[:stats])
+            @processes, @next_page = Lib::PaginateArray.new.call(
+              processes_total,
+              @params.current_page
+            )
+
+            respond
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/controllers/errors.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Controllers
+        # Errors displaying controller
+        # It supports only scenarios with a single partition for errors
+        # If you have high load of errors, consider going Pro
+        class Errors < Base
+          # Lists first page of the errors
+          def index
+            @previous_page, @error_messages, @next_page, = Models::Message.page(
+              errors_topic,
+              0,
+              @params.current_page
+            )
+
+            respond
+          end
+
+          # @param offset [Integer] given error message offset
+          def show(offset)
+            @error_message = Models::Message.find(
+              errors_topic,
+              0,
+              offset
+            )
+
+            respond
+          end
+
+          private
+
+          # @return [String] errors topic
+          def errors_topic
+            ::Karafka::Web.config.topics.errors
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/controllers/jobs.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Controllers
+        # Active jobs (work) reporting controller
+        class Jobs < Base
+          # Lists jobs
+          def index
+            current_state = Models::State.current!
+            processes = Models::Processes.active(current_state)
+
+            # Aggregate jobs and inject the process info into them for better reporting
+            jobs_total = processes.flat_map do |process|
+              process.jobs.map do |job|
+                job.to_h[:process] = process
+                job
+              end
+            end
+
+            @jobs, @next_page = Ui::Lib::PaginateArray.new.call(
+              jobs_total,
+              @params.current_page
+            )
+
+            respond
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/controllers/requests/params.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Controllers
+        # Namespace for request related components
+        module Requests
+          # Internal representation of params with sane sanitization
+          class Params
+            # @param request_params [Hash] raw hash with params
+            def initialize(request_params)
+              @request_params = request_params
+            end
+
+            # @return [Integer] current page for paginated views
+            # @note It does basic sanitization
+            def current_page
+              @current_page ||= begin
+                page = @request_params['page'].to_i
+
+                page.positive? ? page : 1
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/controllers/responses/data.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Controllers
+        # Response related components
+        module Responses
+          # Response data object. It is used to transfer attributes assigned in controllers into
+          # views
+          # It acts as a simplification / transport layer for assigned attributes
+          class Data
+            attr_reader :path, :attributes
+
+            # @param path [String] render path
+            # @param attributes [Hash] attributes assigned in the controller
+            def initialize(path, attributes)
+              @path = path
+              @attributes = attributes
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/controllers/routing.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Controllers
+        # Routing presentation controller
+        class Routing < Base
+          # Routing list
+          def index
+            @routes = Karafka::App.routes
+
+            respond
+          end
+
+          # Given route details
+          #
+          # @param topic_id [String] topic id
+          def show(topic_id)
+            @topic = Karafka::Routing::Router.find_by(id: topic_id)
+
+            @topic || raise(::Karafka::Web::Errors::Ui::NotFoundError, topic_id)
+
+            respond
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/helpers/application_helper.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+# @see https://github.com/mperham/sidekiq/blob/main/lib/sidekiq/web/helpers.rb
+module Karafka
+  module Web
+    module Ui
+      # Namespace for helpers used by the Web UI
+      module Helpers
+        # Main application helper
+        module ApplicationHelper
+          # Generates a full path with the root path out of the provided arguments
+          #
+          # @param args [Array<String, Numeric>] arguments that will make the path
+          # @return [String] path from the root
+          #
+          # @note This needs to be done that way with the `#root_path` because the web UI can be
+          #   mounted in a sub-path and we need to make sure our all paths are relative to "our"
+          #   root, not the root of the app in which it was mounted.
+          def root_path(*args)
+            "#{env.fetch('SCRIPT_NAME')}/#{args.join('/')}"
+          end
+
+          # Adds active class to the current location in the nav if needed
+          # @param location [Hash]
+          def nav_class(location)
+            comparator, value = location.to_a.first
+
+            local_location = request.path.gsub(env.fetch('SCRIPT_NAME'), '')
+            local_location.public_send(:"#{comparator}?", value) ? 'active' : ''
+          end
+
+          # Converts object into a string and for objects that would anyhow return their
+          # stringified instance value, it replaces it with the class name instead.
+          # Useful for deserializers, etc presentation.
+          #
+          # @param object [Object]
+          # @return [String]
+          def object_value_to_s(object)
+            object.to_s.include?('#<') ? object.class.to_s : object.to_s
+          end
+
+          # Renders per scope breadcrumbs
+          def render_breadcrumbs
+            scope = request.path.gsub(root_path, '').split('/')[0]
+
+            render "#{scope}/_breadcrumbs"
+          end
+
+          # Takes a status and recommends background style color
+          #
+          # @param status [String] status
+          # @return [String] background style
+          def status_bg(status)
+            case status
+            when 'initialized' then 'bg-success'
+            when 'running' then 'bg-success'
+            when 'quieting' then 'bg-warning'
+            when 'quiet' then 'bg-warning text-dark'
+            when 'stopping' then 'bg-warning text-dark'
+            when 'stopped' then 'bg-danger'
+            else
+              raise ::Karafka::Errors::UnsupportedCaseError, status
+            end
+          end
+
+          # Takes the lag trend and gives it appropriate background style color for badge
+          #
+          # @param trend [Numeric] lag trend
+          # @return [String] bg classes
+          def lag_trend_bg(trend)
+            bg = 'bg-success' if trend.negative?
+            bg ||= 'bg-warning text-dark' if trend.positive?
+            bg ||= 'bg-secondary'
+            bg
+          end
+
+          # Takes a kafka report state and recommends background style color
+          # @param state [String] state
+          # @return [String] background style
+          def kafka_state_bg(state)
+            case state
+            when 'up' then 'bg-success text-white'
+            when 'active' then 'bg-success text-white'
+            when 'steady' then 'bg-success text-white'
+            else
+              'bg-warning text-dark'
+            end
+          end
+
+          # @param mem_kb [Integer] memory used in KB
+          # @return [String] formatted memory usage
+          def format_memory(mem_kb)
+            return '0' if !mem_kb || mem_kb.zero?
+
+            if mem_kb < 10_240
+              "#{number_with_delimiter(mem_kb)} KB"
+            elsif mem_kb < 1_000_000
+              "#{number_with_delimiter((mem_kb / 1024.0).to_i)} MB"
+            else
+              "#{number_with_delimiter((mem_kb / (1024.0 * 1024.0)).round(1))} GB"
+            end
+          end
+
+          # Converts number to a more friendly delimiter based version
+          # @param number [Numeric]
+          # @return [String] number with delimiter
+          def number_with_delimiter(number)
+            return '' unless number
+
+            parts = number.to_s.to_str.split('.')
+            parts[0].gsub!(/(\d)(?=(\d\d\d)+(?!\d))/, '\1,')
+            parts.join('.')
+          end
+
+          # @param time [Float] UTC time float
+          # @return [String] relative time tag for timeago.js
+          def relative_time(time)
+            stamp = Time.at(time).getutc.iso8601
+            %(<time class="ltr" dir="ltr" title="#{stamp}" datetime="#{stamp}">#{time}</time>)
+          end
+
+          # Returns the view title html code
+          #
+          # @param title [String] page title
+          # @param hr [Boolean] should we add the hr tag at the end
+          # @return [String] title html
+          def view_title(title, hr: false)
+            <<-HTML
+              <div class="container mb-5">
+                <div class="row">
+                  <h3>
+                    #{title}
+                  </h3>
+                </div>
+
+                #{hr ? '<hr/>' : ''}
+              </div>
+            HTML
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/lib/hash_proxy.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      # Non info related extra components used in the UI
+      module Lib
+        # Proxy for hashes we use across UI.
+        # Often we have nested values we want to extract or just values we want to reference and
+        # this object drastically simplifies that.
+        #
+        # It is mostly used for flat hashes.
+        #
+        # It is in a way similar to openstruct but has abilities to dive deep into objects
+        class HashProxy
+          # @param hash [Hash] hash we want to convert to a proxy
+          def initialize(hash)
+            @hash = hash
+          end
+
+          # @param key [Object] hash key
+          # @return [Object] key content or nil if missing
+          def [](key)
+            @hash[key]
+          end
+
+          # @return [Original hash]
+          def to_h
+            @hash
+          end
+
+          # @param method_name [String] method name
+          # @param args [Object] all the args of the method
+          # @param block [Proc] block for the method
+          def method_missing(method_name, *args, &block)
+            return super unless args.empty? && block.nil?
+
+            result = deep_find(@hash, method_name.to_sym)
+            result.nil? ? super : result
+          end
+
+          # @param method_name [String] method name
+          # @param include_private [Boolean]
+          def respond_to_missing?(method_name, include_private = false)
+            result = deep_find(@hash, method_name.to_sym)
+            result.nil? ? super : true
+          end
+
+          private
+
+          # @param obj [Object] local scope of iterating
+          # @param key [Symbol, String] key we are looking for
+          def deep_find(obj, key)
+            if obj.respond_to?(:key?) && obj.key?(key)
+              obj[key]
+            elsif obj.respond_to?(:each)
+              r = nil
+              obj.find { |*a| r = deep_find(a.last, key) }
+              r
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/lib/paginate_array.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Lib
+        # A simple wrapper for paginating array related data structures
+        class PaginateArray
+          # @param array [Array] array we want to paginate
+          # @param current_page [Integer] page we want to be on
+          # @return [Array<Array, <Integer, nil>>] Array with two elements: first is the array with
+          #   data of the given page and second is the next page number of nil in case there is
+          #   no next page (end of data)
+          def call(array, current_page)
+            slices = array.each_slice(per_page).to_a
+
+            current_data = slices[current_page - 1] || []
+
+            if slices.count >= current_page - 1 && current_data.size >= per_page
+              next_page = current_page + 1
+            else
+              next_page = nil
+            end
+
+            [current_data, next_page]
+          end
+
+          private
+
+          # @return [Integer] how many elements should we display in the UI
+          def per_page
+            ::Karafka::Web.config.ui.per_page
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/models/consumer_group.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      # Namespace for models representing pieces of data about Karafka setup
+      module Models
+        # Representation of data of a Karafka consumer group
+        class ConsumerGroup < Lib::HashProxy
+          # @return [Array<Topic>] Data of topics belonging to this consumer group
+          def topics
+            super.values.map do |topic_hash|
+              Topic.new(topic_hash)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/models/health.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Models
+        # Aggregated health data statistics representation
+        class Health
+          class << self
+            # @param state [State] current system state
+            # @return [Hash] has with aggregated statistics
+            def current(state)
+              stats = {}
+
+              processes = Processes.active(state)
+
+              processes.each do |process|
+                process.consumer_groups.each do |details|
+                  name = details.id
+
+                  stats[name] ||= {}
+
+                  details.topics.each do |details2|
+                    t_name = details2.name
+
+                    stats[name][t_name] ||= {}
+                    details2.partitions.each do |partition|
+                      partition_id = partition.id
+                      stats[name][t_name] ||= {}
+                      stats[name][t_name][partition_id] = partition.to_h
+                      stats[name][t_name][partition_id][:process] = process
+                    end
+                  end
+                end
+              end
+
+              stats
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/models/job.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Models
+        # Single job data representation model
+        class Job < Lib::HashProxy
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/models/message.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Models
+        # A proxy between `::Karafka::Messages::Message` and web UI
+        # We work with the Karafka messages but use this model to wrap the work needed.
+        class Message
+          class << self
+            # Looks for a message from a given topic partition
+            #
+            # @param topic_id [String]
+            # @param partition_id [Integer]
+            # @param offset [Integer]
+            # @raise [::Karafka::Web::Errors::Ui::NotFoundError] when not found
+            def find(topic_id, partition_id, offset)
+              message = Karafka::Admin.read_topic(
+                topic_id,
+                partition_id,
+                1,
+                offset
+              ).first
+
+              return message if message
+
+              raise(
+                ::Karafka::Web::Errors::Ui::NotFoundError,
+                [topic_id, partition_id, offset].join(', ')
+              )
+            end
+
+            # Fetches requested page of Kafka messages.
+            #
+            # @param topic_id [String]
+            # @param partition_id [Integer]
+            # @param page [Integer]
+            # @return [Array] We return both page data as well as all the details needed to build
+            #   the pagination details.
+            def page(topic_id, partition_id, page)
+              # Establish the leading offset
+              lead = Karafka::Admin.read_topic(topic_id, partition_id, 1).first
+
+              partitions_count = fetch_partition_count(topic_id)
+
+              # If there is not even one message, we need to early exit
+              return [false, [], false, partitions_count] unless lead
+
+              # We add plus one because we compute previous offset from which we want to start and
+              # not previous page leading offset
+              previous_offset = lead.offset - (per_page * page) + 1
+
+              if previous_offset.negative?
+                count = per_page + previous_offset
+                previous_page = page < 2 ? false : page - 1
+                next_page = false
+                previous_offset = 0
+              else
+                previous_page = page < 2 ? false : page - 1
+                next_page = page + 1
+                count = per_page
+              end
+
+              [
+                previous_page,
+                read_topic(topic_id, partition_id, count, previous_offset).reverse,
+                next_page,
+                partitions_count
+              ]
+            end
+
+            private
+
+            # @param args [Object] anything required by the admin `#read_topic`
+            # @return [Array<Karafka::Messages::Message>] topic partition messages
+            def read_topic(*args)
+              ::Karafka::Admin.read_topic(*args)
+            end
+
+            # @param topic_id [String] id of the topic
+            # @return [Integer] number of partitions this topic has
+            def fetch_partition_count(topic_id)
+              ::Karafka::Admin
+                .cluster_info
+                .topics
+                .find { |topic| topic[:topic_name] == topic_id }
+                .fetch(:partition_count)
+            end
+
+            # @return [Integer] elements per page
+            def per_page
+              ::Karafka::Web.config.ui.per_page
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/models/partition.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Models
+        # Single topic partition data representation model
+        class Partition < Lib::HashProxy
+        end
+      end
+    end
+  end
+end