kafka_command 0.0.1

data/app/models/kafka_command/cluster.rb

@@ -0,0 +1,172 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+
+module KafkaCommand
+  class Cluster
+    extend Forwardable
+    DEFAULT_PROTOCOL = 'PLAINTEXT'
+
+    attr_reader :client,
+      :name,
+      :description,
+      :protocol,
+      :sasl_scram_username,
+      :sasl_scram_password,
+      :version,
+      :connect_timeout,
+      :socket_timeout
+
+    alias_method :id, :name
+
+    def_delegators :@client, :topics, :groups, :brokers
+
+    def initialize(name:, seed_brokers:, description: nil, protocol: DEFAULT_PROTOCOL,
+                   sasl_scram_username: nil, sasl_scram_password: nil, ssl_ca_cert_file_path: nil,
+                   ssl_client_cert_file_path: nil, ssl_client_cert_key_file_path: nil, version: nil,
+                   ssl_ca_cert: nil, ssl_client_cert: nil, ssl_client_cert_key: nil, connect_timeout: nil,
+                   socket_timeout: nil
+                  )
+      @name = name
+      @seed_brokers = seed_brokers
+      @description = description
+      @sasl_scram_username = sasl_scram_username
+      @sasl_scram_password = sasl_scram_password
+      @ssl_ca_cert = ssl_ca_cert
+      @ssl_ca_cert_file_path = ssl_ca_cert_file_path
+      @ssl_client_cert = ssl_client_cert
+      @ssl_client_cert_file_path = ssl_client_cert_file_path
+      @ssl_client_cert_key = ssl_client_cert_key
+      @ssl_client_cert_key_file_path = ssl_client_cert_key_file_path
+      @version = version
+      @connect_timeout = connect_timeout
+      @socket_timeout = socket_timeout
+      @client = initialize_client
+    end
+
+    def connected?
+      # Tried using all?(&:connected?) here, but was getting some weird behavior with the views
+      brokers.map(&:connected?).all?
+    end
+
+    def create_topic(name, **kwargs)
+      client.create_topic(name, **kwargs)
+
+      # Give the cluster time to know about the topic
+      3.times do
+        client.refresh_topics!
+        topic = topics.find { |t| t.name == name }
+        return topic if topic
+      end
+    end
+
+    def to_human
+      name.humanize.capitalize
+    end
+
+    def to_s
+      name
+    end
+
+    def ssl?
+      get_ssl_ca_cert.present?
+    end
+
+    def sasl?
+      sasl_scram_username.present? && sasl_scram_password.present?
+    end
+
+    def ==(other)
+      name == other.name
+    end
+
+    def self.find(cluster_name)
+      all.find { |c| c.name == cluster_name }
+    end
+
+    def self.none?
+      all.none?
+    end
+
+    def self.count
+      all.count
+    end
+
+    def self.all
+      KafkaCommand.config.clusters.map do |name, cluster_info|
+        new(
+          name: name,
+          seed_brokers: cluster_info['seed_brokers'],
+          protocol: cluster_info['protocol'],
+          description: cluster_info['description'],
+          connect_timeout: cluster_info['connect_timeout'],
+          socket_timeout: cluster_info['socket_timeout'],
+          sasl_scram_username: cluster_info['sasl_scram_username'],
+          sasl_scram_password: cluster_info['sasl_scram_password'],
+          ssl_ca_cert: cluster_info['ssl_ca_cert'],
+          ssl_ca_cert_file_path: cluster_info['ssl_ca_cert_file_path'],
+          ssl_client_cert: cluster_info['ssl_client_cert'],
+          ssl_client_cert_file_path: cluster_info['ssl_client_cert_file_path'],
+          ssl_client_cert_key: cluster_info['ssl_client_cert_key'],
+          ssl_client_cert_key_file_path: cluster_info['ssl_client_cert_key_file_path'],
+        )
+      end
+    end
+
+    private
+
+      def get_ssl_ca_cert
+        return @ssl_ca_cert if @ssl_ca_cert
+
+        if @ssl_ca_cert_file_path && File.exists?(@ssl_ca_cert_file_path)
+          File.read(@ssl_ca_cert_file_path).strip
+        end
+      end
+
+      def get_ssl_client_cert
+        return @ssl_client_cert if @ssl_client_cert
+
+        if @ssl_client_cert_file_path && File.exists?(@ssl_client_cert_file_path)
+          File.read(@ssl_client_cert_file_path).strip
+        end
+      end
+
+      def get_ssl_client_cert_key
+        return @ssl_client_cert_key if @ssl_client_cert_key
+
+        if @ssl_client_cert_key_file_path && File.exists?(@ssl_client_cert_key_file_path)
+          File.read(@ssl_client_cert_key_file_path).strip
+        end
+      end
+
+      def seed_brokers
+        return @seed_brokers.split(',') if @seed_brokers.is_a?(String)
+        @seed_brokers
+      end
+
+      def initialize_client
+        @client ||= begin
+          client_kwargs = {
+            brokers: seed_brokers,
+            client_id: name
+          }
+
+          if sasl?
+            client_kwargs[:sasl_scram_username] = sasl_scram_username
+            client_kwargs[:sasl_scram_password] = sasl_scram_password
+            client_kwargs[:sasl_scram_mechanism] = 'sha256'
+            client_kwargs[:ssl_ca_cert] = get_ssl_ca_cert
+          elsif ssl?
+            client_kwargs[:ssl_ca_cert] = get_ssl_ca_cert
+            client_kwargs[:ssl_client_cert] = get_ssl_client_cert
+            client_kwargs[:ssl_client_cert_key] = get_ssl_client_cert_key
+          end
+
+          client_kwargs[:connect_timeout] = connect_timeout if connect_timeout
+          client_kwargs[:socket_timeout] = socket_timeout if socket_timeout
+
+          Client.new(**client_kwargs)
+        end
+      end
+  end
+end

data/app/models/kafka_command/consumer_group.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+module KafkaCommand
+  class ConsumerGroup
+    attr_reader :group_id
+
+    def initialize(group_id, client)
+      @client   = client
+      @group_id = group_id
+    end
+
+    def refresh!
+      clear_group_metadata!
+    end
+
+    def stable?
+      state.match?(/stable/i)
+    end
+
+    def empty?
+      state.match?(/empty/i) || members.none?
+    end
+
+    def partitions_for(topic_name)
+      topic = @client.find_topic(topic_name)
+      partition_lag = lag_for(topic.name)
+
+      topic.partitions.map do |p|
+        ConsumerGroupPartition.new(
+          lag: partition_lag[p.partition_id][:lag],
+          offset: partition_lag[p.partition_id][:offset],
+          group_id: @group_id,
+          topic_name: topic.name,
+          partition_id: p.partition_id
+        )
+      end
+    end
+
+    def total_lag_for(topic_name)
+      lag_for(topic_name).values.map { |lag_hash| lag_hash[:lag] || 0 }.reduce(:+)
+    end
+
+    def as_json(*)
+      topics_json = consumed_topics.map do |topic|
+        {
+          name: topic.name,
+          partitions: partitions_for(topic.name).map(&:as_json)
+        }
+      end
+
+      {
+        group_id: @group_id,
+        state: state,
+        topics: topics_json
+      }
+    end
+
+    def consumed_topics
+      topic_names = members.flat_map(&:topic_names).uniq
+
+      @client.topics.select do |t|
+        topic_names.include?(t.name)
+      end
+    end
+
+    def coordinator
+      @coordinator ||= @client.get_group_coordinator(group_id: @group_id)
+    end
+
+    def group_metadata
+      @group_metadata ||= initialize_group_metadata
+    end
+
+    def state
+      group_metadata.state
+    end
+
+    def members
+      group_metadata.members.map do |member|
+        GroupMember.new(member)
+      end
+    end
+
+    private
+
+      def lag_for(topic_name)
+        topic = @client.find_topic(topic_name)
+        topic_offsets = topic.offsets
+        group_offsets = offsets_for(topic_name)
+
+        lag_hash = compute_lag(topic_offsets, group_offsets)
+        lag_hash.each_with_object({}) do |(partition_id, lag), return_hash|
+          return_hash[partition_id] = {
+            offset: group_offsets[partition_id],
+            lag: lag
+          }
+        end
+      end
+
+      def offsets_for(topic_name)
+        topic = @client.find_topic(topic_name)
+
+        offsets = coordinator.fetch_offsets(
+          group_id: @group_id,
+          topics: { topic.name => topic.partitions.map(&:partition_id) }
+        ).topics[topic.name]
+
+        offsets.keys.each do |partition_id|
+          if offsets[partition_id].offset == -1
+            offsets[partition_id] = nil
+          else
+            offsets[partition_id] = offsets[partition_id].offset
+          end
+        end
+
+        offsets
+      end
+
+      def compute_lag(topic_offsets, group_offsets)
+        topic_offsets.each_with_object({}) do |(partition_id, latest_offset), lag_hash|
+          lag =
+            if group_offsets[partition_id].nil?
+              nil
+            elsif group_offsets[partition_id] >= latest_offset
+              0
+            else
+              latest_offset - group_offsets[partition_id]
+            end
+
+          lag_hash[partition_id] = lag
+        end
+      end
+
+      def initialize_group_metadata
+        @client.describe_group(@group_id)
+      end
+
+      def clear_group_metadata!
+        @group_metadata = nil
+      end
+  end
+end

data/app/models/kafka_command/consumer_group_partition.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module KafkaCommand
+  class ConsumerGroupPartition
+    attr_reader :lag, :topic_name, :offset, :partition_id, :group_id
+
+    def initialize(lag:, topic_name:, offset:, group_id:, partition_id:)
+      @group_id     = group_id
+      @lag          = lag
+      @topic_name   = topic_name
+      @offset       = offset
+      @partition_id = partition_id
+    end
+
+    def as_json(*)
+      {
+        lag: @lag,
+        offset: @offset,
+        partition_id: @partition_id
+      }
+    end
+  end
+end

data/app/models/kafka_command/group_member.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module KafkaCommand
+  class GroupMember
+    attr_reader :member_id, :client_host, :client_id, :topic_assignment
+
+    def initialize(member_metadata)
+      @member_id = member_metadata.member_id
+      @client_host = member_metadata.client_host
+      @client_id = member_metadata.client_id
+      @topic_assignment = member_metadata.member_assignment.topics
+    end
+
+    def topic_names
+      @topic_assignment.keys
+    end
+  end
+end

data/app/models/kafka_command/partition.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+
+module KafkaCommand
+  class Partition
+    extend Forwardable
+    attr_reader :topic
+    def_delegators :@partition_metadata, :isr, :leader, :partition_id, :replicas
+
+    def initialize(partition_metadata, topic)
+      @topic = topic
+      @partition_metadata = partition_metadata
+    end
+
+    def highwater_mark_offset
+      @topic.offset_for(self)
+    end
+    alias offset highwater_mark_offset
+
+    def as_json(*)
+      {
+        isr: isr,
+        leader: leader,
+        id: partition_id,
+        highwater_mark_offset: offset
+      }
+    end
+
+    # TODO
+    #
+    # implement describe to retrieve important configs
+    # def describe
+    # end
+  end
+end

data/app/models/kafka_command/topic.rb

@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+module KafkaCommand
+  class Topic
+    attr_reader :name,
+                :partitions,
+                :replication_factor,
+                :max_message_bytes,
+                :retention_ms,
+                :retention_bytes
+
+    alias_method :id, :name
+
+    API_TIMEOUT = 10
+    CONSUMER_OFFSET_TOPIC = '__consumer_offsets'
+    DEFAULT_MAX_MESSAGE_BYTES = 1000012
+    DEFAULT_RETENTION_MS = 604800000
+    DEFAULT_RETENTION_BYTES = -1
+    TOPIC_CONFIGS = %w[
+      max.message.bytes
+      retention.bytes
+      retention.ms
+    ].freeze
+
+    class DeletionError < StandardError; end
+
+    def initialize(topic_metadata, client)
+      @client         = client
+      @topic_metadata = topic_metadata
+      initialize_from_metadata
+    end
+
+    def brokers_spread
+      ((replication_factor.to_f / @client.brokers.count.to_f) * 100).round
+    end
+
+    def destroy
+      if name == CONSUMER_OFFSET_TOPIC
+        raise DeletionError, "Cannot delete the #{CONSUMER_OFFSET_TOPIC} topic"
+      end
+
+      @client.delete_topic(@name, timeout: API_TIMEOUT)
+    end
+
+    def set_configs!(max_message_bytes: nil, retention_ms: nil, retention_bytes: nil)
+      config = {}
+      config['max.message.bytes'] = max_message_bytes if max_message_bytes
+      config['retention.ms']      = retention_ms      if retention_ms
+      config['retention.bytes']   = retention_bytes   if retention_bytes
+
+      @client.alter_topic(@name, config) unless config.empty?
+      refresh!
+    end
+
+    def set_partitions!(num_partitions)
+      unless @client.supports_api?(Kafka::Protocol::CREATE_PARTITIONS_API)
+        raise UnsupportedApiError, 'This version of Kafka does not support the create partitions API.'
+      end
+
+      @client.create_partitions_for(
+        @name,
+        num_partitions: num_partitions,
+        timeout: API_TIMEOUT
+      )
+
+      refresh!
+    end
+
+    def refresh!
+      @topic_metadata = @client.fetch_metadata(topics: [@name]).topics.first
+      initialize_from_metadata
+    end
+
+    def offset_for(partition)
+      tries = 0
+      @client.resolve_offset(@name, partition.partition_id, :latest)
+    rescue Kafka::UnknownTopicOrPartition
+      raise if tries >= 3
+      tries += 1
+      retry
+    end
+
+    def offsets(partition_ids = nil)
+      partition_ids ||= @partitions.map(&:partition_id)
+      @client.resolve_offsets(@name, partition_ids, :latest)
+    end
+
+    def offset_sum
+      offsets.values.reduce(:+)
+    end
+
+    def consumer_offset_topic?
+      name == CONSUMER_OFFSET_TOPIC
+    end
+
+    def groups
+      @client.groups.select do |g|
+        g.consumed_topics.include?(self)
+      end
+    end
+
+    # Needs arguments to be compatible with rails as_json calls
+    def as_json(include_config: false, **kwargs)
+      json = {
+        name: @name,
+        replication_factor: @replication_factor,
+        partitions: @partitions.sort_by(&:partition_id).map(&:as_json)
+      }
+
+      if include_config
+        json[:config] = {
+          max_message_bytes: max_message_bytes,
+          retention_ms: retention_ms,
+          retention_bytes: retention_bytes
+        }
+      end
+
+      json
+    end
+
+    def ==(other)
+      @name == other.name
+    end
+
+    def topic_configs
+      @topic_configs ||= describe
+    end
+
+    def retention_ms
+      topic_configs['retention.ms'].to_i
+    end
+
+    def retention_bytes
+      topic_configs['retention.bytes'].to_i
+    end
+
+    def max_message_bytes
+      topic_configs['max.message.bytes'].to_i
+    end
+
+    private
+
+      def describe
+        @client.describe_topic(@name, TOPIC_CONFIGS)
+      end
+
+      def initialize_from_metadata
+        @name               = @topic_metadata.topic_name
+        @partitions         = @topic_metadata.partitions.map { |pm| Partition.new(pm, self) }
+        @replication_factor = @partitions.map(&:isr).map(&:length).max
+      end
+  end
+end

data/app/views/kafka_command/brokers/index.html.erb

@@ -0,0 +1,38 @@
+<%= render partial: 'kafka_command/shared/title', locals: { title: @cluster.to_human, subtitle: 'Brokers' } %>
+
+<div class="level"a>
+  <div class="level-left">
+    <div class="level-item" style="margin-left: 0.2rem">
+      <p class="subtitle is-5">
+        <strong><%= @brokers.count %></strong> brokers
+      </p>
+    </div>
+  </div>
+</div>
+
+<table class="table is-striped is-fullwidth is-bordered">
+  <thead>
+    <th>Id</th>
+    <th>Host</th>
+    <th>Connection Status</th>
+  </thead>
+  <tbody>
+    <% @brokers.each do |broker| %>
+      <tr>
+        <td><%= broker.kafka_broker_id %></td>
+        <td><%= broker.host %></td>
+        <td>
+          <% if broker.connected? %>
+            <span class="icon has-text-success">
+              <i class="fas fa-check-circle"></i>
+            </span>
+          <% else %>
+            <span class="icon has-text-danger">
+              <i class="fas fa-times-circle"></i>
+            </span>
+          <% end %>
+        </td>
+      </tr>
+    <% end %>
+  </tbody>
+</table>

data/app/views/kafka_command/clusters/_tabs.html.erb

@@ -0,0 +1,9 @@
+<%= render partial: 'kafka_command/shared/title', locals: { title: @cluster.to_human, subtitle: @cluster.description } %>
+
+<div class="tabs">
+  <ul>
+    <li class="<%= tab_class('topics') %>"><%= link_to 'Topics', cluster_topics_path(@cluster) %></li>
+    <li class="<%= tab_class('consumer_groups') %>"><%= link_to 'Consumer Groups', cluster_consumer_groups_path(@cluster) %></li>
+    <li class="<%= tab_class('brokers') %>"><%= link_to 'Brokers', cluster_brokers_path(@cluster) %></li>
+  </ul>
+</div>

data/app/views/kafka_command/clusters/index.html.erb

@@ -0,0 +1,54 @@
+<%= render partial: 'kafka_command/shared/title', locals: { title: 'Clusters', subtitle: 'Manage multiple clusters' } %>
+
+<div class="level"a>
+  <%=
+    render(
+      partial: 'kafka_command/shared/search_bar',
+      locals: {
+        resources: @clusters,
+        resource_name: 'cluster',
+        resource_path: clusters_path,
+        filter_property: :name
+      }
+    )
+  %>
+</div>
+
+<table class="table is-striped is-fullwidth is-bordered">
+  <thead>
+    <th>Name</th>
+    <th>Description</th>
+    <th>Brokers</th>
+    <th>Protocol</th>
+    <th>Connection Status</th>
+  </thead>
+  <tbody>
+    <% @clusters.each do |cluster| %>
+      <tr>
+        <td><%= link_to cluster.name, cluster_topics_path(cluster) %></td>
+        <td><%= cluster.description %></td>
+        <td><%= cluster.brokers.count %></td>
+        <td>
+          <% if cluster.ssl? %>
+            SSL
+          <% elsif cluster.sasl? %>
+            SASL/SCRAM
+          <% else %>
+            PLAINTEXT
+          <% end %>
+        </td>
+        <td>
+          <% if cluster.connected? %>
+            <span class="icon has-text-success">
+              <i class="fas fa-check-circle"></i>
+            </span>
+          <% else %>
+            <span class="icon has-text-danger">
+              <i class="fas fa-times-circle"></i>
+            </span>
+          <% end %>
+        </td>
+      </tr>
+    <% end %>
+  </tbody>
+</table>