kafka_command 0.0.1

data/app/assets/stylesheets/kafka_command/application.css

@@ -0,0 +1,27 @@
+/*
+ * This is a manifest file that'll be compiled into application.css, which will include all the files
+ * listed below.
+ *
+ * Any CSS and SCSS file within this directory, lib/assets/stylesheets, or any plugin's
+ * vendor/assets/stylesheets directory can be referenced here using a relative path.
+ *
+ * You're free to add application-wide styles to this file and they'll appear at the bottom of the
+ * compiled file so the styles you add here take precedence over styles defined in any other CSS/SCSS
+ * files in this directory. Styles in this file should be added after the last require_* statement.
+ * It is generally better to create a new file per style scope.
+ *
+ *= require_tree .
+ *= require_self
+ */
+
+.is-borderless {
+  border-width: 0px !important;
+}
+
+.kafka-logo {
+  margin-left: 0 !important;
+}
+
+.page-title {
+  margin-top: 1rem;
+}

data/app/assets/stylesheets/kafka_command/clusters.css

@@ -0,0 +1,8 @@
+.cluster-card {
+  margin-bottom: 1.5rem;
+}
+
+.delete-cluster {
+/*  justify-content: flex-end; */
+  padding: 0.5rem;
+}

data/app/assets/stylesheets/kafka_command/topics.css

@@ -0,0 +1,3 @@
+.config-table {
+  margin-bottom: 2.5rem !important;
+}

data/app/channels/application_cable/channel.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module ApplicationCable
+  class Channel < ActionCable::Channel::Base
+  end
+end

data/app/channels/application_cable/connection.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module ApplicationCable
+  class Connection < ActionCable::Connection::Base
+  end
+end

data/app/controllers/kafka_command/application_controller.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module KafkaCommand
+  class ApplicationController < ActionController::Base
+    protect_from_forgery unless: -> { request.format.json? }
+
+    rescue_from Kafka::ConnectionError, with: :kafka_connection_error
+    rescue_from Kafka::ClusterAuthorizationFailed, with: :kafka_authorization_error
+    rescue_from UnsupportedApiError, with: :unsupported_api_error
+
+    before_action do
+      unless KafkaCommand.config.valid?
+        flash[:error] = KafkaCommand.config.errors.join("\n")
+        render 'kafka_command/configuration_error'
+      end
+    end
+
+    protected
+
+      def unsupported_api_error(exception)
+        render_error(exception.message, status: 422, flash: { error: exception.message })
+      end
+
+      def record_not_found
+        render_error('Not Found', status: :not_found)
+      end
+
+      def kafka_connection_error
+        error_msg = 'Could not connect to Kafka with the specified brokers'
+        render_error(
+          error_msg,
+          status: 500,
+          flash: { error: error_msg }
+        )
+      end
+
+      def kafka_authorization_error
+        error_msg = 'You are not authorized to perform that action'
+        render_error(
+          error_msg,
+          status: 401,
+          flash: { error: error_msg }
+        )
+      end
+
+      def serialize_json(data, **kwargs)
+        if data.is_a?(ActiveRecord::Relation) || data.is_a?(Array)
+          return {
+            data: data.map { |d| d.as_json(**kwargs) }
+          }
+        end
+
+        data.as_json(**kwargs)
+      end
+
+      def render_success(data, status: :ok, redirection_path: nil, flash: {}, **kwargs)
+        respond_to do |format|
+          format.html do
+            redirect_to redirection_path, flash: flash if redirection_path
+          end
+
+          format.json do
+            if status == :no_content || status.to_s.to_i == 204
+              head :no_content
+            else
+              render_json(data, status: status, **kwargs)
+            end
+          end
+        end
+      end
+
+      def render_error(data, status: :unprocessible_entity, flash: {})
+        respond_to do |format|
+          format.html do
+            redirect_back(fallback_location: root_path, flash: flash) && (return) if flash.present?
+
+            case status
+            when :not_found, 404
+              render json: '404 Not Found', status: status, layout: false
+            else
+              render json: '500 Internal Server Error', status: status, layout: false
+            end
+          end
+          format.json { render_json_errors(data, status: status) }
+        end
+      end
+
+      def render_json(data, status:, **kwargs)
+        render json: serialize_json(data, **kwargs), status: status
+      end
+
+      def render_json_errors(errors, status: :unprocessible_entity)
+        render json: errors, status: status
+      end
+  end
+end

data/app/controllers/kafka_command/brokers_controller.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require_dependency 'kafka_command/application_controller'
+
+module KafkaCommand
+  class BrokersController < ApplicationController
+    # GET /clusters/:cluster_id/brokers
+    def index
+      @cluster = Cluster.find(params[:cluster_id])
+      @brokers = @cluster.brokers
+      render_success(@brokers)
+    end
+
+    # GET /clusters/:cluster_id/brokers/:id
+    def show
+      cluster = Cluster.find(params[:cluster_id])
+      @broker = cluster.brokers.find { |b| b.node_id == params[:id].to_i }
+
+      if @broker.present?
+        render_success(@broker)
+      else
+        record_not_found
+      end
+    end
+  end
+end

data/app/controllers/kafka_command/clusters_controller.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require_dependency 'kafka_command/application_controller'
+
+module KafkaCommand
+  class ClustersController < ApplicationController
+    # GET /clusters
+    def index
+      @clusters = Cluster.all
+
+      flash[:search] = params[:name]
+
+      if params[:name].present?
+        @clusters = @clusters.select do |c|
+          regex = /#{params[:name]}/i
+          c.name.match?(regex)
+        end
+      end
+
+      redirection_path = new_cluster_path if Cluster.none?
+      render_success(@clusters, redirection_path: redirection_path, flash: flash.to_hash)
+    end
+
+    # GET /clusters/:id
+    def show
+      @cluster = Cluster.find(params[:id])
+
+      if @cluster
+        render_success(@cluster)
+      else
+        record_not_found
+      end
+    end
+
+    private
+      # leave for config validation
+
+      def cluster_params
+        params.permit(*cluster_params_keys)
+      end
+
+      def cluster_params_keys
+        [:name, :description, :version]
+      end
+  end
+end

data/app/controllers/kafka_command/consumer_groups_controller.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require_dependency 'kafka_command/application_controller'
+
+module KafkaCommand
+  class ConsumerGroupsController < ApplicationController
+    # GET /clusters/:cluster_id/consumer_groups
+    def index
+      @cluster = Cluster.find(params[:cluster_id])
+      @groups = @cluster.groups
+
+      flash[:search] = params[:group_id]
+
+      if params[:group_id].present?
+        @groups = @groups.select do |g|
+          regex = /#{params[:group_id]}/i
+          g.group_id.match?(regex)
+        end
+      end
+
+      render_success(@groups)
+    end
+
+    # GET /alusters/:cluster_id/consumer_groups/:id
+    def show
+      @cluster = Cluster.find(params[:cluster_id])
+      @group = @cluster.groups.find { |g| g.group_id == params[:id] }
+
+      if @group.nil?
+        render_error('Consumer group not found', status: 404)
+        return
+      end
+
+      @current_topic =
+        if params[:topic]
+          @group.consumed_topics.find { |t| t.name == params[:topic] }
+        else
+          @group.consumed_topics.first
+        end
+
+      render_success(@group)
+    end
+  end
+end

data/app/controllers/kafka_command/topics_controller.rb

@@ -0,0 +1,187 @@
+# frozen_string_literal: true
+
+require_dependency 'kafka_command/application_controller'
+
+module KafkaCommand
+  class TopicsController < ApplicationController
+    rescue_from Kafka::InvalidPartitions, with: :invalid_partitions
+    rescue_from Kafka::InvalidReplicationFactor, with: :invalid_replication_factor
+    rescue_from Kafka::InvalidTopic, with: :invalid_topic_name
+    rescue_from Kafka::TopicAlreadyExists, with: :topic_already_exists
+    rescue_from Kafka::UnknownError, with: :unknown_error
+    rescue_from Kafka::InvalidRequest, with: :unknown_error
+    rescue_from Kafka::InvalidConfig, with: :unknown_error
+    rescue_from Kafka::TopicAuthorizationFailed, with: :kafka_authorization_error
+    rescue_from KafkaCommand::Topic::DeletionError, with: :topic_deletion_error
+
+    # GET /clusters/:cluster_id/topics
+    def index
+      @cluster = Cluster.find(params[:cluster_id])
+      @topics = @cluster.topics
+
+      flash[:search] = params[:name]
+
+      if params[:name].present?
+        @topics = @topics.select do |t|
+          regex = /#{params[:name]}/i
+          t.name.match?(regex)
+        end
+      end
+
+      render_success(@topics)
+    end
+
+    # GET /clusters/:cluster_id/topics/:id
+    def show
+      @cluster = Cluster.find(params[:cluster_id])
+      @topic = @cluster.topics.find { |t| t.name == params[:id] }
+
+      if @topic.nil?
+        render_error('Topic not found', status: 404)
+      else
+        render_success(@topic, include_config: true)
+      end
+    end
+
+    # GET /clusters/:cluster_id/topics/new
+    def new
+      @cluster = Cluster.find(params[:cluster_id])
+    end
+
+    # GET /clusters/:cluster_id/topics/edit
+    def edit
+      @cluster = Cluster.find(params[:cluster_id])
+      @topic = @cluster.topics.find { |t| t.name == params[:id] }
+      @redirect_path = params[:redirect_path]
+    end
+
+    # POST /clusters/:cluster_id/topics
+    def create
+      @cluster = Cluster.find(params[:cluster_id])
+      @topic = @cluster.create_topic(
+        params[:name],
+        num_partitions: params[:num_partitions].to_i,
+        replication_factor: params[:replication_factor].to_i,
+        config: build_config
+      )
+
+      render_success(
+        @topic,
+        status: :created,
+        redirection_path: cluster_topics_path,
+        flash: { success: 'Topic created' },
+        include_config: true
+      )
+    end
+
+    # PATCH/PUT /clusters/:cluster_id/topics/:id
+    def update
+      cluster = Cluster.find(params[:cluster_id])
+      @topic = cluster.topics.find { |t| t.name == params[:id] }
+
+      render_error('Topic not found', status: 404) && (return) if @topic.nil?
+      if params[:num_partitions]
+        @topic.set_partitions!(params[:num_partitions].to_i) unless params[:num_partitions].to_i == @topic.partitions.count
+      end
+
+      if build_config.present?
+        @topic.set_configs!(
+          max_message_bytes: params[:max_message_bytes],
+          retention_ms: params[:retention_ms],
+          retention_bytes: params[:retention_bytes]
+        )
+      end
+
+      render_success(
+        @topic,
+        status: :ok,
+        redirection_path: params[:redirect_path] || cluster_topics_path,
+        flash: { success: 'Topic updated' }
+      )
+    end
+
+    # DELETE /clusters/:cluster_id/topics/:id
+    def destroy
+      @cluster = Cluster.find(params[:cluster_id])
+      @topic = @cluster.topics.find { |t| t.name == params[:id] }
+
+      if @topic.nil?
+        render_error('Topic not found', status: 404)
+      else
+        @topic.destroy
+        render_success(
+          @topic,
+          status: :no_content,
+          redirection_path: cluster_topics_path,
+          flash: { success: "Topic \"#{@topic.name}\" is marked for deletion. <strong>Note: This will have no impact if delete.topic.enable is not set to true.</strong>".html_safe }
+        )
+      end
+    end
+
+    private
+
+      def build_config
+        {}.tap do |config|
+          config['max.message.bytes'] = params[:max_message_bytes] if params[:max_message_bytes]
+          config['retention.ms']      = params[:retention_ms]      if params[:retention_ms]
+          config['retention.bytes']   = params[:retention_bytes]   if params[:retention_bytes]
+        end
+      end
+
+      def invalid_partitions
+        error_msg = 'Num partitions must be > 0 or > current number of partitions'
+
+        render_error(
+          error_msg,
+          status: 422,
+          flash: { error: error_msg }
+        )
+      end
+
+      def invalid_replication_factor
+        error_msg = 'Replication factor must be > 0 and < total number of brokers'
+
+        render_error(
+          error_msg,
+          status: 422,
+          flash: { error: error_msg }
+        )
+      end
+
+      def invalid_topic_name
+        error_msg = 'Topic must have a name'
+        render_error(
+          error_msg,
+          status: 422,
+          flash: { error: error_msg }
+        )
+      end
+
+      def topic_already_exists
+        error_msg = 'Topic already exists'
+        render_error(
+          error_msg,
+          status: 422,
+          flash: { error: error_msg }
+        )
+      end
+
+      def unknown_error
+        error_msg = 'An unknown error occurred with the request to Kafka. Check any request parameters.'
+
+        render_error(
+          error_msg,
+          status: 422,
+          flash: { error: error_msg }
+        )
+      end
+
+      def topic_deletion_error(exception)
+        render_error(
+          exception.message,
+          status: 422,
+          flash: { error: exception.message }
+        )
+      end
+  end
+end

data/app/helpers/kafka_command/application_helper.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module KafkaCommand
+  module ApplicationHelper
+    def format_flash_errors
+      return '' if flash[:error].blank?
+      return flash[:error] if flash[:error].is_a?(String)
+
+      '<ul>'.tap do |str|
+        flash[:error].each do |k, v|
+          str << "<li><span class='has-text-weight-bold'>#{k.humanize}:</span> #{v.join('. ')}</li>"
+        end
+      end.html_safe
+    end
+
+    def topic_path(topic)
+      "#{cluster_path(@cluster)}/topics/#{CGI.escape(topic.name)}"
+    end
+
+    def consumer_groups_path(group)
+      "#{cluster_path(@cluster)}/consumer_groups/#{CGI.escape(group.group_id)}"
+    end
+
+    def trim_name(name)
+      return name if name.length < 25
+      name[0..22] + '...'
+    end
+  end
+end

data/app/helpers/kafka_command/consumer_group_helper.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module KafkaCommand
+  module ConsumerGroupHelper
+    def status_color(group)
+      if group.empty?
+        'has-text-info'
+      elsif group.stable?
+        'has-text-success'
+      end
+    end
+  end
+end

data/app/jobs/application_job.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module KafkaCommand
+  class ApplicationJob < ActiveJob::Base
+  end
+end

data/app/mailers/application_mailer.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module KafkaCommand
+  class ApplicationMailer < ActionMailer::Base
+    default from: 'from@example.com'
+    layout 'mailer'
+  end
+end

data/app/models/kafka_command/broker.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+
+module KafkaCommand
+  class Broker
+    extend Forwardable
+    attr_reader :broker
+    def_delegators :@broker, :port, :host, :node_id, :fetch_metadata, :fetch_offsets
+    alias_method :kafka_broker_id, :node_id
+    alias_method :hostname, :host
+
+
+    def initialize(broker)
+      @broker = broker
+    end
+
+    def host_with_port
+      "#{host}:#{port}"
+    end
+
+    def as_json(*)
+      {
+        id: node_id,
+        host: host_with_port
+      }
+    end
+
+    # needs to be the group coordinator to work
+    def offsets_for(group, topic)
+      offsets = @broker.fetch_offsets(
+        group_id: group.group_id,
+        topics: { topic.name => topic.partitions.map(&:partition_id) }
+      ).topics[topic.name]
+
+      offsets.keys.each { |partition_id| offsets[partition_id] = offsets[partition_id].offset }
+      offsets
+    end
+
+    def connected?
+      @broker.api_versions # simple request to check connections
+      true
+    rescue Kafka::ConnectionError
+      false
+    end
+  end
+end

data/app/models/kafka_command/client.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+
+module KafkaCommand
+  class Client
+    extend Forwardable
+
+    CLUSTER_METHOD_DELGATIONS = %i(
+      broker_pool
+      delete_topic
+      alter_topic
+      describe_topic
+      create_partitions_for
+      resolve_offset
+      resolve_offsets
+      describe_group
+      supports_api?
+    ).freeze
+
+    attr_reader :cluster, :client
+    def_delegators :@client, :create_topic
+    def_delegators :@cluster, *CLUSTER_METHOD_DELGATIONS
+
+    def initialize(brokers:, **kwargs)
+      @client  = Kafka.new(brokers, **kwargs)
+      @cluster = @client.cluster
+    end
+
+    def refresh!
+      refresh_brokers!
+      refresh_topics!
+      refresh_groups!
+    end
+
+    def topics
+      @topics ||= initialize_topics
+    end
+
+    def brokers
+      @brokers ||= initialize_brokers
+    end
+
+    def groups
+      @groups ||= initialize_groups
+    end
+
+    def refresh_groups!
+      @groups = initialize_groups
+    end
+
+    def refresh_topics!
+      @topics = initialize_topics
+    end
+
+    def refresh_brokers!
+      @brokers = initialize_brokers
+    end
+
+    def fetch_metadata(topics: nil)
+      brokers.sample.fetch_metadata(topics: topics)
+    end
+
+    def get_group_coordinator(group_id:)
+      broker = @cluster.get_group_coordinator(group_id: group_id)
+      Broker.new(broker)
+    end
+
+    def find_topic(topic_name)
+      topics.find { |t| t.name == topic_name }
+    end
+
+    def connect_to_broker(host:, port:, broker_id:)
+      Broker.new(broker_pool.connect(host, port, node_id: broker_id))
+    end
+
+    private
+
+      def initialize_brokers
+        cluster_info = @cluster.refresh_metadata!
+
+        cluster_info.brokers.map do |broker|
+          connect_to_broker(
+            host: broker.host,
+            port: broker.port,
+            broker_id: broker.node_id
+          )
+        end
+      end
+
+      def initialize_topics
+        # returns information about each topic
+        # i.e isr, leader, partitions
+        fetch_metadata.topics.map { |tm| Topic.new(tm, self) }
+      end
+
+      def initialize_groups
+        group_ids = @cluster.list_groups
+        group_ids.map { |id| ConsumerGroup.new(id, self) }
+      end
+  end
+end