kafka_command 0.0.1

data/spec/requests/json/topics_spec.rb

@@ -0,0 +1,274 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe 'Topics API', type: :request do
+  let(:cluster) { KafkaCommand::Cluster.all.first }
+  let(:topic_name) { "test-#{SecureRandom.hex(12)}" }
+  let(:num_partitions) { 5 }
+  let(:replication_factor) { 1 }
+  let(:uri_base) { '/clusters' }
+  let(:create_topic_kwargs) do
+    {
+      num_partitions: num_partitions,
+      replication_factor: replication_factor
+    }
+  end
+
+  before { create_topic(topic_name, **create_topic_kwargs) }
+
+  describe 'listing all topics' do
+    let(:topic_two_name) { "test-#{SecureRandom.hex(12)}" }
+
+    before { create_topic(topic_two_name) }
+
+    it 'lists' do
+      get "#{uri_base}/#{cluster.id}/topics.json"
+      expect(response.status).to eq(200)
+      expect(json['data']).to be_an_instance_of(Array)
+      expect(json['data'].map { |d| d['name'] }).to include(topic_name)
+      expect(json['data'].map { |d| d['name'] }).to include(topic_two_name)
+    end
+
+    context 'filtering' do
+      it 'filters by name' do
+        get "#{uri_base}/#{cluster.id}/topics.json?name=#{topic_name}"
+        expect(response.status).to eq(200)
+        expect(json['data']).to be_an_instance_of(Array)
+        expect(json['data'].map { |d| d['name'] }).to include(topic_name)
+        expect(json['data'].map { |d| d['name'] }).to_not include(topic_two_name)
+      end
+
+      it 'filters by name' do
+        get "#{uri_base}/#{cluster.id}/topics.json?name=unknown"
+        expect(response.status).to eq(200)
+        expect(json['data']).to be_an_instance_of(Array)
+        expect(json['data']).to be_empty
+      end
+    end
+  end
+
+  describe 'showing a topic' do
+    context 'topic exists' do
+      it 'shows' do
+        get "#{uri_base}/#{cluster.id}/topics/#{topic_name}.json"
+        expect(response.status).to eq(200)
+        expect(json['name']).to eq(topic_name)
+        expect(json['partitions']).to be_an_instance_of(Array)
+        expect(json['partitions'].count).to eq(num_partitions)
+        expect(json['replication_factor']).to eq(replication_factor)
+      end
+    end
+
+    context 'topic does not exist' do
+      it 'returns 404' do
+        get "#{uri_base}/#{cluster.id}/topics/doesnotexist.json"
+        expect(response.status).to eq(404)
+        expect(response.body).to eq('Topic not found')
+      end
+    end
+  end
+
+  describe 'creating a topic' do
+    let(:topic_two_name) { "test-#{SecureRandom.hex(12)}" }
+    let(:new_num_partitions) { num_partitions }
+    let(:new_replication_factor) { replication_factor }
+    let(:retention_ms) { 1024 }
+    let(:retention_bytes) { 10000 }
+    let(:max_message_bytes) { 10000 }
+    let(:create_topic_params) do
+      {
+        name: topic_two_name,
+        replication_factor: new_replication_factor,
+        num_partitions: new_num_partitions,
+        retention_bytes: retention_bytes,
+        retention_ms: retention_ms,
+        max_message_bytes: max_message_bytes
+      }
+    end
+
+    it 'creates' do
+      expect do
+        post "#{uri_base}/#{cluster.id}/topics.json", params: create_topic_params
+        expect(response.status).to eq(201)
+        expect(json['name']).to eq(topic_two_name)
+        expect(json['partitions']).to be_an_instance_of(Array)
+        expect(json['partitions'].count).to eq(num_partitions)
+        expect(json['replication_factor']).to eq(replication_factor)
+        expect(json['config']['retention_ms']).to eq(retention_ms)
+        expect(json['config']['retention_bytes']).to eq(retention_bytes)
+        expect(json['config']['max_message_bytes']).to eq(max_message_bytes)
+      end.to change { cluster.client.refresh_topics!; cluster.topics.count }.by(1)
+    end
+
+    context 'invalid parameters' do
+      describe 'no name' do
+        let(:topic_two_name) { '' }
+
+        it 'returns 422' do
+          expect do
+            post "#{uri_base}/#{cluster.id}/topics.json", params: create_topic_params
+            expect(response.status).to eq(422)
+            expect(response.body).to eq('Topic must have a name')
+          end.to change { cluster.client.refresh_topics!; cluster.topics.count }.by(0)
+        end
+      end
+
+      describe 'topic already exists' do
+        let(:topic_two_name) { topic_name }
+
+        it 'returns 422' do
+          expect do
+            post "#{uri_base}/#{cluster.id}/topics.json", params: create_topic_params
+            expect(response.status).to eq(422)
+            expect(response.body).to eq('Topic already exists')
+          end.to change { cluster.topics.count }.by(0)
+        end
+      end
+
+      describe 'invalid max message bytes' do
+        let(:max_message_bytes) { -1 }
+
+        it 'returns 422' do
+          expect do
+            post "#{uri_base}/#{cluster.id}/topics.json", params: create_topic_params
+            expect(response.status).to eq(422)
+            expect(response.body).to eq('An unknown error occurred with the request to Kafka. Check any request parameters.')
+          end.to change { cluster.topics.count }.by(0)
+        end
+      end
+
+      describe 'invalid partitions' do
+        let(:new_num_partitions) { -1 }
+
+        it 'returns 422' do
+          expect do
+            post "#{uri_base}/#{cluster.id}/topics.json", params: create_topic_params
+            expect(response.status).to eq(422)
+            expect(response.body).to eq('Num partitions must be > 0 or > current number of partitions')
+          end.to change { cluster.topics.count }.by(0)
+        end
+      end
+
+      describe 'invalid replication factor' do
+        let(:error_message) do
+          'Replication factor must be > 0 and < total number of brokers'
+        end
+
+        describe 'when 0' do
+          let(:new_replication_factor) { 0 }
+
+          it 'returns 422' do
+            expect do
+              post "#{uri_base}/#{cluster.id}/topics.json", params: create_topic_params
+              expect(response.status).to eq(422)
+              expect(response.body).to eq(error_message)
+            end.to change { cluster.topics.count }.by(0)
+          end
+        end
+
+        describe 'when > number of brokers' do
+          let(:new_replication_factor) { cluster.brokers.count + 1 }
+
+          it 'returns 422' do
+            expect do
+              post "#{uri_base}/#{cluster.id}/topics.json", params: create_topic_params
+              expect(response.status).to eq(422)
+              expect(response.body).to eq(error_message)
+            end.to change { cluster.topics.count }.by(0)
+          end
+        end
+      end
+    end
+  end
+
+  describe 'updating a topic' do
+    let(:topic) { cluster.topics.find { |t| t.name == topic_name } }
+    let(:new_num_partitions) { num_partitions + 1 }
+    let(:max_message_bytes) { 1024 }
+    let(:retention_ms) { 1024 }
+    let(:retention_bytes) { 1024 }
+    let(:update_topic_params) do
+      {
+        num_partitions: new_num_partitions,
+        retention_bytes: retention_bytes,
+        max_message_bytes: max_message_bytes,
+        retention_ms: retention_ms
+      }
+    end
+
+    context 'topic exists' do
+      it 'updates the topic' do
+        patch "#{uri_base}/#{cluster.id}/topics/#{topic_name}.json", params: update_topic_params
+        expect(response.status).to eq(200)
+        expect(topic.partitions.count).to eq(update_topic_params[:num_partitions])
+        expect(topic.retention_ms).to eq(retention_ms)
+        expect(topic.retention_bytes).to eq(retention_bytes)
+        expect(topic.max_message_bytes).to eq(max_message_bytes)
+      end
+
+      context 'invalid parameters' do
+        describe 'invalid num partitions' do
+          let(:new_num_partitions) { num_partitions - 1 }
+
+          it 'returns 422' do
+            patch "#{uri_base}/#{cluster.id}/topics/#{topic_name}.json", params: update_topic_params
+            expect(response.status).to eq(422)
+            expect(response.body).to eq('Num partitions must be > 0 or > current number of partitions')
+          end
+        end
+
+        describe 'invalid max message bytes' do
+          let(:max_message_bytes) { -1 }
+
+          it 'returns 422' do
+            patch "#{uri_base}/#{cluster.id}/topics/#{topic_name}.json", params: update_topic_params
+            expect(response.status).to eq(422)
+            expect(response.body).to eq('An unknown error occurred with the request to Kafka. Check any request parameters.')
+          end
+        end
+      end
+    end
+
+    context 'topic does not exist' do
+      it 'returns 404' do
+        patch "#{uri_base}/#{cluster.id}/topics/nonexistent.json", params: update_topic_params
+        expect(response.status).to eq(404)
+      end
+    end
+  end
+
+  describe 'destroying a topic' do
+    context 'topic exists' do
+      it 'destroys' do
+        expect do
+          delete "#{uri_base}/#{cluster.id}/topics/#{topic_name}.json"
+          expect(response.status).to eq(204)
+        end.to change { cluster.client.refresh_topics!; cluster.topics.count }.by(-1)
+      end
+
+      context 'consumer offsets topic' do
+        before do
+          allow_any_instance_of(KafkaCommand::Topic).to receive(:name).and_return(KafkaCommand::Topic::CONSUMER_OFFSET_TOPIC)
+        end
+
+        it 'returns 422' do
+          delete "#{uri_base}/#{cluster.id}/topics/#{KafkaCommand::Topic::CONSUMER_OFFSET_TOPIC}.json"
+          expect(response.status).to eq(422)
+        end
+      end
+    end
+
+    context 'topic does not exist' do
+      before { delete_topic(topic_name) }
+
+      it 'returns 404' do
+        expect do
+          delete "#{uri_base}/#{cluster.id}/topics/doesnotexist.json"
+          expect(response.status).to eq(404)
+          expect(response.body).to eq('Topic not found')
+        end.to change { cluster.topics.count }.by(0)
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+# This file was generated by the `rails generate rspec:install` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# The generated `.rspec` file contains `--require spec_helper` which will cause
+# this file to always be loaded, without a need to explicitly require it in any
+# files.
+#
+# Given that it is always loaded, you are encouraged to keep this file as
+# light-weight as possible. Requiring heavyweight dependencies from this file
+# will add to the boot time of your test suite on EVERY test run, even for an
+# individual file that may not need all of that loaded. Instead, consider making
+# a separate helper file that requires the additional dependencies and performs
+# the additional setup, and require it from the spec files that actually need
+# it.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+#
+# Makes loading files into out tests without loading rails much easier
+$LOAD_PATH.unshift(File.expand_path('.'))
+
+require 'pry'
+require 'json'
+require 'dotenv'
+
+Dotenv.load('.env.test.local', '.env.test')
+
+require 'support/kafka_helper'
+
+RSpec.configure do |config|
+  # rspec-expectations config goes here. You can use an alternate
+  # assertion/expectation library such as wrong or the stdlib/minitest
+  # assertions if you prefer.
+  config.expect_with :rspec do |expectations|
+    # This option will default to `true` in RSpec 4. It makes the `description`
+    # and `failure_message` of custom matchers include text for helper methods
+    # defined using `chain`, e.g.:
+    #     be_bigger_than(2).and_smaller_than(4).description
+    #     # => "be bigger than 2 and smaller than 4"
+    # ...rather than:
+    #     # => "be bigger than 2"
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  # rspec-mocks config goes here. You can use an alternate test double
+  # library (such as bogus or mocha) by changing the `mock_with` option here.
+  config.mock_with :rspec do |mocks|
+    # Prevents you from mocking or stubbing a method that does not exist on
+    # a real object. This is generally recommended, and will default to
+    # `true` in RSpec 4.
+    mocks.verify_partial_doubles = true
+  end
+
+  # This option will default to `:apply_to_host_groups` in RSpec 4 (and will
+  # have no way to turn it off -- the option exists only for backwards
+  # compatibility in RSpec 3). It causes shared context metadata to be
+  # inherited by the metadata hash of host groups and examples, rather than
+  # triggering implicit auto-inclusion in groups with matching metadata.
+  config.shared_context_metadata_behavior = :apply_to_host_groups
+
+  # The settings below are suggested to provide a good initial experience
+  # with RSpec, but feel free to customize to your heart's content.
+
+  # This allows you to limit a spec run to individual examples or groups
+  # you care about by tagging them with `:focus` metadata. When nothing
+  # is tagged with `:focus`, all examples get run. RSpec also provides
+  # aliases for `it`, `describe`, and `context` that include `:focus`
+  # metadata: `fit`, `fdescribe` and `fcontext`, respectively.
+  config.filter_run_when_matching :focus
+
+  # Allows RSpec to persist some state between runs in order to support
+  # the `--only-failures` and `--next-failure` CLI options. We recommend
+  # you configure your source control system to ignore this file.
+  config.example_status_persistence_file_path = 'spec/examples.txt'
+
+  # Limits the available syntax to the non-monkey patched syntax that is
+  # recommended. For more details, see:
+  #   - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/
+  #   - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
+  #   - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode
+  config.disable_monkey_patching!
+
+  # Many RSpec users commonly either run the entire suite or an individual
+  # file, and it's useful to allow more verbose output when running an
+  # individual spec file.
+  if config.files_to_run.one?
+    # Use the documentation formatter for detailed output,
+    # unless a formatter has already been configured
+    # (e.g. via a command-line flag).
+    config.default_formatter = 'doc'
+  end
+
+  # Print the 10 slowest examples and example groups at the
+  # end of the spec run, to help surface which specs are running
+  # particularly slow.
+  # config.profile_examples = 10
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = :random
+
+  # Seed global randomization in this process using the `--seed` CLI option.
+  # Setting this allows you to use `--seed` to deterministically reproduce
+  # test failures related to randomization by passing the same `--seed` value
+  # as the one that triggered the failure.
+  Kernel.srand config.seed
+end

data/spec/support/factory_bot.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+RSpec.configure do |config|
+  config.include FactoryBot::Syntax::Methods
+end

data/spec/support/json_helper.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Requests
+  module JsonHelper
+    def json
+      JSON.parse(response.body)
+    end
+  end
+end
+
+RSpec.configure do |config|
+  config.include(Requests::JsonHelper)
+end

data/spec/support/kafka_helper.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require 'fast_helper'
+require 'securerandom'
+require 'kafka'
+require 'lib/kafka_command/configuration'
+require 'lib/kafka_command/errors'
+require 'config/initializers/kafka'
+require 'app/models/kafka_command/client'
+require 'app/models/kafka_command/broker'
+require 'app/models/kafka_command/cluster'
+require 'app/models/kafka_command/topic'
+require 'app/models/kafka_command/consumer_group'
+require 'app/models/kafka_command/partition'
+require 'app/models/kafka_command/consumer_group_partition'
+require 'app/models/kafka_command/group_member'
+
+$LOAD_PATH.unshift(File.expand_path('.'))
+ENV['RAILS_ENV'] = 'test'
+
+KafkaCommand::Configuration.load!('spec/dummy/config/kafka_command.yml')
+
+begin
+  KafkaCommand::Cluster.all.first.topics
+rescue => e
+  puts "#{e.class}. An online kafka cluster is required to run the specs."
+  exit(0)
+end
+
+module KafkaHelpers
+  def kafka_command_cluster
+    KafkaCommand::Cluster.all.find { |c| c.name == 'test_cluster' }
+  end
+
+  def kafka_client
+    Kafka.new(ENV['SEED_BROKERS'].split(','))
+  end
+
+  def create_topic(topic_name, **kwargs)
+    kafka_client.create_topic(topic_name, **kwargs)
+    sleep_if_necessary
+  end
+
+  def delete_topic(topic_name)
+    kafka_client.delete_topic(topic_name)
+    sleep_if_necessary
+  end
+
+  def list_topic_names
+    kafka_client.topics
+  end
+
+  def deliver_message(msg, **kwargs)
+    kafka_client.deliver_message(msg, **kwargs)
+  end
+
+  def partitions_for(topic_name)
+    kafka_client.partitions_for(topic_name)
+  end
+
+  def create_partitions_for(topic_name, **kwargs)
+    kafka_client.create_partitions_for(topic_name, **kwargs)
+    sleep_if_necessary
+  end
+
+  def topic_exists?(topic_name)
+    return false if topic_name.nil? || topic_name.empty?
+    list_topic_names.include?(topic_name)
+  end
+
+  def run_consumer_group(topic_name, group_id, num_messages_to_consume: 0)
+    deliver_message('test', topic: topic_name)
+    consumer = kafka_client.consumer(group_id: group_id)
+    consumer.subscribe(topic_name)
+
+    message_counter = 0
+    num_messages_to_consume += 1
+    consumer.each_message do |msg|
+      yield if block_given?
+      message_counter += 1
+      consumer.stop if message_counter >= num_messages_to_consume
+    end
+  end
+
+  # Sleep if more than one broker is in the cluster to fix flaky tests
+  def sleep_if_necessary
+    sleep(0.5) if ENV['SEED_BROKERS'].split(',').count > 1
+  end
+end
+
+RSpec.configure do |config|
+  config.include(KafkaHelpers)
+end