kafka_command 0.0.1

data/config/initializers/kafka.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require 'core_extensions/kafka/broker/attr_readers'
+require 'core_extensions/kafka/broker_pool/attr_readers'
+require 'core_extensions/kafka/client/attr_readers'
+require 'core_extensions/kafka/cluster/attr_readers'
+require 'core_extensions/kafka/protocol/metadata_response/partition_metadata/attr_readers'
+
+Kafka::Broker.include CoreExtensions::Kafka::Broker::AttrReaders
+Kafka::BrokerPool.include CoreExtensions::Kafka::BrokerPool::AttrReaders
+Kafka::Client.include CoreExtensions::Kafka::Client::AttrReaders
+Kafka::Cluster.include CoreExtensions::Kafka::Cluster::AttrReaders
+Kafka::Protocol::MetadataResponse::PartitionMetadata.include CoreExtensions::Kafka::Protocol::MetadataResponse::PartitionMetadata::AttrReaders

data/config/initializers/kafka_command.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require 'yaml'
+
+config_file_path = "#{Rails.root}/config/kafka_command.yml"
+
+if File.exists?(config_file_path)
+  KafkaCommand::Configuration.load!(config_file_path)
+else
+  puts 'kafka_command.yml not found. KafkaCommand not configured via a yml file.'
+end

data/config/routes.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+KafkaCommand::Engine.routes.draw do
+  root 'clusters#index'
+
+  resources :clusters, only: [:index, :show] do
+    resources :brokers, only: [:index, :show]
+    resources :topics, id: /([^\/])+?/, format: /html|json/
+    resources :consumer_groups, only: [:index, :show], id: /([^\/])+?/, format: /html|json/
+  end
+end

data/docker-compose.yml

@@ -0,0 +1,18 @@
+version: '3'
+services:
+  zookeeper:
+    image: wurstmeister/zookeeper
+    ports:
+      - "2181:2181"
+  kafka:
+    depends_on:
+      - zookeeper
+    image: wurstmeister/kafka:2.12-2.1.0
+    ports:
+      - "9092:9092"
+    environment:
+      KAFKA_ADVERTISED_HOST_NAME: localhost
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_DELETE_TOPIC_ENABLE: 'true'
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock

data/kafka_command.gemspec

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+$:.push File.expand_path('lib', __dir__)
+
+# Maintain your gem's version:
+require 'kafka_command/version'
+
+# Describe your gem and declare its dependencies:
+Gem::Specification.new do |s|
+  s.name        = 'kafka_command'
+  s.version     = KafkaCommand::VERSION
+  s.authors     = ['jasondoc3']
+  s.email       = ['jasondoc3@gmail.com']
+  s.homepage    = 'https://github.com/jasondoc3/kafka_command'
+  s.summary     = 'A simple Kafka management UI.'
+  s.description = 'A simple Kafka management UI.'
+  s.license     = 'MIT'
+
+  s.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  s.bindir        = 'exe'
+  s.require_paths = ['lib']
+  s.test_files = Dir['spec/**/*']
+
+  s.add_dependency 'rails', '>= 4'
+  s.add_dependency 'ruby-kafka', '> 0.6.3'
+  s.add_dependency 'rails-ujs'
+end

data/lib/core_extensions/kafka/broker/attr_readers.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module CoreExtensions
+  module Kafka
+    module Broker
+      module AttrReaders
+        attr_reader :host, :port, :node_id
+      end
+    end
+  end
+end

data/lib/core_extensions/kafka/broker_pool/attr_readers.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module CoreExtensions
+  module Kafka
+    module BrokerPool
+      module AttrReaders
+        attr_reader :brokers
+      end
+    end
+  end
+end

data/lib/core_extensions/kafka/client/attr_readers.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module CoreExtensions
+  module Kafka
+    module Client
+      module AttrReaders
+        attr_reader :cluster
+      end
+    end
+  end
+end

data/lib/core_extensions/kafka/cluster/attr_readers.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module CoreExtensions
+  module Kafka
+    module Cluster
+      module AttrReaders
+        attr_reader :broker_pool
+      end
+    end
+  end
+end

data/lib/core_extensions/kafka/protocol/metadata_response/partition_metadata/attr_readers.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module CoreExtensions
+  module Kafka
+    module Protocol
+      class MetadataResponse
+        class PartitionMetadata
+          module AttrReaders
+            attr_reader :isr, :replicas
+          end
+        end
+      end
+    end
+  end
+end

data/lib/kafka_command/configuration.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+module KafkaCommand
+  def self.config=(config_hash)
+    @config ||= Configuration.new(config_hash)
+  end
+
+  def self.config
+    @config
+  end
+
+  class ConfigurationError < StandardError; end
+  class Configuration
+    HOST_REGEX = /[^\:]+:[0-9]{1,5}/
+    attr_reader :config, :clusters, :errors
+
+    CLUSTER_KEYS = %w(
+      protocol
+      description
+      version
+      seed_brokers
+      ssl_ca_cert
+      ssl_ca_cert_file_path
+      ssl_client_cert
+      ssl_client_cert_file_path
+      ssl_client_cert_key
+      ssl_client_cert_key_file_path
+      sasl_scram_username
+      sasl_scram_password
+      socket_timeout
+      connect_timeout
+    )
+
+    def initialize(config_hash)
+      @config = config_hash[ENV['RAILS_ENV']]
+      @clusters = config['clusters'] if config.present?
+      @errors = []
+    end
+
+    def valid?
+      @errors = []
+
+      if config.blank?
+        errors << 'No config specified for environment'
+        return false
+      end
+
+      validate!
+      errors.none?
+    end
+
+    def invalid?
+      !valid?
+    end
+
+    def self.parse_yaml(file_path)
+      YAML.load(ERB.new(File.read(file_path)).result(binding))
+    end
+
+    def self.load!(file_path)
+      KafkaCommand.config = parse_yaml(file_path)
+
+      if KafkaCommand.config.invalid?
+        puts "KafkaCommand improperly configured. #{KafkaCommand.config.errors}"
+      end
+    end
+
+    private
+
+      def validate!
+        validate_clusters
+      rescue => e
+        errors << 'Kafka Command is configured incorrectly'
+      end
+
+      def validate_clusters
+        if clusters.blank?
+          errors << 'Cluster must be provided'
+          return
+        end
+
+        clusters.each do |_, cluster_hash|
+          validate_cluster(cluster_hash)
+        end
+      end
+
+      def validate_cluster(cluster)
+        cluster.keys.each do |key|
+          unless CLUSTER_KEYS.include?(key)
+            errors << "Invalid cluster option, #{key}"
+            return
+          end
+        end
+
+        seed_brokers = cluster['seed_brokers']
+        seed_brokers = seed_brokers.split(',') if seed_brokers.is_a?(String)
+
+        if seed_brokers&.compact.blank?
+          errors << 'Must specify a list of seed brokers'
+          return
+        end
+
+        seed_brokers.each(&method(:validate_broker))
+        validate_ssl(cluster)
+        validate_sasl(cluster)
+      end
+
+      def validate_broker(broker)
+        unless broker&.match?(HOST_REGEX)
+          errors << 'Broker must be a valid host/port combination'
+        end
+      end
+
+      def validate_ssl(cluster)
+        ca_cert         = certificate_authority(cluster)
+        client_cert     = client_certificate(cluster)
+        client_cert_key = client_certificate_key(cluster)
+
+        if ca_cert
+          if client_cert && !client_cert_key
+            errors << 'Initialized with `ssl_client_cert` but no `ssl_client_cert_key`. Please provide both.'
+          elsif !client_cert && client_cert_key
+            errors << 'Initialized with `ssl_client_cert_key`, but no `ssl_client_cert`. Please provide both.'
+          end
+        elsif client_cert || client_cert_key
+          errors << 'Cannot provide client certificate/key without a certificate authority'
+        end
+      end
+
+      def validate_sasl(cluster)
+        if cluster['sasl_scram_username'].present? && cluster['sasl_scram_password'].blank?
+          errors << 'Initialized with `sasl_scram_username` but no `sasl_scram_password`. Please provide both.'
+        elsif cluster['sasl_scram_username'].blank? && cluster['sasl_scram_password'].present?
+          errors << 'Initialized with `sasl_scram_password` but no `sasl_scram_username`. Please provide both.'
+        end
+      end
+
+      def certificate_authority(cluster)
+        cluster['ssl_ca_cert'] || cluster['ssl_ca_cert_file_path']
+      end
+
+      def client_certificate(cluster)
+        cluster['ssl_client_cert'] || cluster['ssl_client_cert_file_path']
+      end
+
+      def client_certificate_key(cluster)
+        cluster['ssl_client_cert_key'] || cluster['ssl_client_cert_key_file_path']
+      end
+  end
+end

data/lib/kafka_command/engine.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module KafkaCommand
+  class Engine < ::Rails::Engine
+    isolate_namespace KafkaCommand
+
+    initializer 'kafka_command.assets.precompile' do |app|
+      app.config.assets.precompile += %w( kafka_command/* )
+    end
+  end
+end

data/lib/kafka_command/errors.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module KafkaCommand
+  class UnsupportedApiError < StandardError
+  end
+end

data/lib/kafka_command/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module KafkaCommand
+  VERSION = '0.0.1'
+end

data/lib/kafka_command.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require 'kafka_command/engine'
+require 'kafka'
+require 'kafka_command/configuration'
+require 'kafka_command/errors'
+
+if defined?(Rails) && Rails::VERSION::MAJOR < 5
+  require 'rails-ujs'
+end
+
+module KafkaCommand
+end

data/spec/dummy/Rakefile

@@ -0,0 +1,6 @@
+# Add your own tasks in files placed in lib/tasks ending in .rake,
+# for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
+
+require_relative 'config/application'
+
+Rails.application.load_tasks

data/spec/dummy/app/assets/config/manifest.js

@@ -0,0 +1,4 @@
+//= link_tree ../images
+//= link_directory ../javascripts .js
+//= link_directory ../stylesheets .css
+//= link kafka_command_manifest.js

data/spec/dummy/app/assets/javascripts/application.js

@@ -0,0 +1,15 @@
+// This is a manifest file that'll be compiled into application.js, which will include all the files
+// listed below.
+//
+// Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts,
+// or any plugin's vendor/assets/javascripts directory can be referenced here using a relative path.
+//
+// It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the
+// compiled file. JavaScript code in this file should be added after the last require_* statement.
+//
+// Read Sprockets README (https://github.com/rails/sprockets#sprockets-directives) for details
+// about supported directives.
+//
+//= require rails-ujs
+//= require activestorage
+//= require_tree .

data/spec/dummy/app/assets/javascripts/cable.js

@@ -0,0 +1,13 @@
+// Action Cable provides the framework to deal with WebSockets in Rails.
+// You can generate new channels where WebSocket features live using the `rails generate channel` command.
+//
+//= require action_cable
+//= require_self
+//= require_tree ./channels
+
+(function() {
+  this.App || (this.App = {});
+
+  App.cable = ActionCable.createConsumer();
+
+}).call(this);

data/spec/dummy/app/assets/stylesheets/application.css

@@ -0,0 +1,15 @@
+/*
+ * This is a manifest file that'll be compiled into application.css, which will include all the files
+ * listed below.
+ *
+ * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
+ * or any plugin's vendor/assets/stylesheets directory can be referenced here using a relative path.
+ *
+ * You're free to add application-wide styles to this file and they'll appear at the bottom of the
+ * compiled file so the styles you add here take precedence over styles defined in any other CSS/SCSS
+ * files in this directory. Styles in this file should be added after the last require_* statement.
+ * It is generally better to create a new file per style scope.
+ *
+ *= require_tree .
+ *= require_self
+ */

data/spec/dummy/app/channels/application_cable/channel.rb

@@ -0,0 +1,4 @@
+module ApplicationCable
+  class Channel < ActionCable::Channel::Base
+  end
+end

data/spec/dummy/app/channels/application_cable/connection.rb

@@ -0,0 +1,4 @@
+module ApplicationCable
+  class Connection < ActionCable::Connection::Base
+  end
+end

data/spec/dummy/app/controllers/application_controller.rb

@@ -0,0 +1,2 @@
+class ApplicationController < ActionController::Base
+end

data/spec/dummy/app/helpers/application_helper.rb

@@ -0,0 +1,2 @@
+module ApplicationHelper
+end

data/spec/dummy/app/jobs/application_job.rb

@@ -0,0 +1,2 @@
+class ApplicationJob < ActiveJob::Base
+end

data/spec/dummy/app/mailers/application_mailer.rb

@@ -0,0 +1,4 @@
+class ApplicationMailer < ActionMailer::Base
+  default from: 'from@example.com'
+  layout 'mailer'
+end

data/spec/dummy/app/models/application_record.rb

@@ -0,0 +1,3 @@
+class ApplicationRecord < ActiveRecord::Base
+  self.abstract_class = true
+end

data/spec/dummy/app/views/layouts/application.html.erb

@@ -0,0 +1,15 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>Dummy</title>
+    <%= csrf_meta_tags %>
+    <%= csp_meta_tag %>
+
+    <%= stylesheet_link_tag    'application', media: 'all' %>
+    <%= javascript_include_tag 'application' %>
+  </head>
+
+  <body>
+    <%= yield %>
+  </body>
+</html>

data/spec/dummy/app/views/layouts/mailer.html.erb

@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+    <style>
+      /* Email styles need to be inline */
+    </style>
+  </head>
+
+  <body>
+    <%= yield %>
+  </body>
+</html>

data/spec/dummy/app/views/layouts/mailer.text.erb

@@ -0,0 +1 @@
+<%= yield %>

data/spec/dummy/bin/bundle

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__)
+load Gem.bin_path('bundler', 'bundle')

data/spec/dummy/bin/rails

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+APP_PATH = File.expand_path('../config/application', __dir__)
+require_relative '../config/boot'
+require 'rails/commands'

data/spec/dummy/bin/rake

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require_relative '../config/boot'
+require 'rake'
+Rake.application.run

data/spec/dummy/bin/setup

@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby
+require 'fileutils'
+include FileUtils
+
+# path to your application root.
+APP_ROOT = File.expand_path('..', __dir__)
+
+def system!(*args)
+  system(*args) || abort("\n== Command #{args} failed ==")
+end
+
+chdir APP_ROOT do
+  # This script is a starting point to setup your application.
+  # Add necessary setup steps to this file.
+
+  puts '== Installing dependencies =='
+  system! 'gem install bundler --conservative'
+  system('bundle check') || system!('bundle install')
+
+  # Install JavaScript dependencies if using Yarn
+  # system('bin/yarn')
+
+  # puts "\n== Copying sample files =="
+  # unless File.exist?('config/database.yml')
+  #   cp 'config/database.yml.sample', 'config/database.yml'
+  # end
+
+  puts "\n== Preparing database =="
+  system! 'bin/rails db:setup'
+
+  puts "\n== Removing old logs and tempfiles =="
+  system! 'bin/rails log:clear tmp:clear'
+
+  puts "\n== Restarting application server =="
+  system! 'bin/rails restart'
+end

data/spec/dummy/bin/update

@@ -0,0 +1,31 @@
+#!/usr/bin/env ruby
+require 'fileutils'
+include FileUtils
+
+# path to your application root.
+APP_ROOT = File.expand_path('..', __dir__)
+
+def system!(*args)
+  system(*args) || abort("\n== Command #{args} failed ==")
+end
+
+chdir APP_ROOT do
+  # This script is a way to update your development environment automatically.
+  # Add necessary update steps to this file.
+
+  puts '== Installing dependencies =='
+  system! 'gem install bundler --conservative'
+  system('bundle check') || system!('bundle install')
+
+  # Install JavaScript dependencies if using Yarn
+  # system('bin/yarn')
+
+  puts "\n== Updating database =="
+  system! 'bin/rails db:migrate'
+
+  puts "\n== Removing old logs and tempfiles =="
+  system! 'bin/rails log:clear tmp:clear'
+
+  puts "\n== Restarting application server =="
+  system! 'bin/rails restart'
+end

data/spec/dummy/bin/yarn

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.expand_path('..', __dir__)
+Dir.chdir(APP_ROOT) do
+  begin
+    exec "yarnpkg", *ARGV
+  rescue Errno::ENOENT
+    $stderr.puts "Yarn executable was not detected in the system."
+    $stderr.puts "Download Yarn at https://yarnpkg.com/en/docs/install"
+    exit 1
+  end
+end

data/spec/dummy/config/application.rb

@@ -0,0 +1,19 @@
+require_relative 'boot'
+
+require 'rails/all'
+
+Bundler.require(*Rails.groups)
+require "kafka_command"
+
+module Dummy
+  class Application < Rails::Application
+    # Initialize configuration defaults for originally generated Rails version.
+    config.load_defaults 5.2
+
+    # Settings in config/environments/* take precedence over those specified here.
+    # Application configuration can go into files in config/initializers
+    # -- all .rb files in that directory are automatically loaded after loading
+    # the framework and any gems in your application.
+  end
+end
+

data/spec/dummy/config/boot.rb

@@ -0,0 +1,5 @@
+# Set up gems listed in the Gemfile.
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../../Gemfile', __dir__)
+
+require 'bundler/setup' if File.exist?(ENV['BUNDLE_GEMFILE'])
+$LOAD_PATH.unshift File.expand_path('../../../lib', __dir__)

data/spec/dummy/config/cable.yml

@@ -0,0 +1,10 @@
+development:
+  adapter: async
+
+test:
+  adapter: async
+
+production:
+  adapter: redis
+  url: <%= ENV.fetch("REDIS_URL") { "redis://localhost:6379/1" } %>
+  channel_prefix: dummy_production

data/spec/dummy/config/database.yml

@@ -0,0 +1,25 @@
+# SQLite version 3.x
+#   gem install sqlite3
+#
+#   Ensure the SQLite 3 gem is defined in your Gemfile
+#   gem 'sqlite3'
+#
+default: &default
+  adapter: sqlite3
+  pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %>
+  timeout: 5000
+
+development:
+  <<: *default
+  database: db/development.sqlite3
+
+# Warning: The database defined as "test" will be erased and
+# re-generated from your development database when you run "rake".
+# Do not set this db to the same as development or production.
+test:
+  <<: *default
+  database: db/test.sqlite3
+
+production:
+  <<: *default
+  database: db/production.sqlite3

data/spec/dummy/config/environment.rb

@@ -0,0 +1,5 @@
+# Load the Rails application.
+require_relative 'application'
+
+# Initialize the Rails application.
+Rails.application.initialize!