kafka_command 0.0.1

data/spec/models/kafka_command/cluster_spec.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+RSpec.describe KafkaCommand::Cluster do
+  let(:cluster)    { described_class.all.first }
+  let(:topic_name) { SecureRandom.hex(12) }
+
+  describe '#new' do
+    let(:brokers) { ENV['SEED_BROKERS'].split(',') }
+
+    context 'plaintext' do
+      it 'creates a kafka client with brokers and client id' do
+        expect(KafkaCommand::Client).to receive(:new).with(
+          hash_including(
+            brokers: brokers,
+            client_id: 'test_cluster'
+          )
+        ).at_least(:once)
+
+        cluster
+
+        expect(cluster.ssl?).to eq(false)
+        expect(cluster.sasl?).to eq(false)
+      end
+    end
+
+    context 'sasl config' do
+      before do
+        allow(KafkaCommand).to receive(:config).and_return(
+          KafkaCommand::Configuration.new(KafkaCommand::Configuration.parse_yaml('spec/fixtures/files/kafka_command_sasl.yml'))
+        )
+      end
+
+      it 'creates a kafka client with brokers, client_id, and sasl params' do
+        expect(KafkaCommand::Client).to receive(:new).with(
+          brokers: brokers,
+          client_id: 'sasl_test_cluster',
+          sasl_scram_username: 'test',
+          sasl_scram_password: 'test',
+          sasl_scram_mechanism: 'sha256',
+          ssl_ca_cert: 'test_ca_cert'
+        )
+
+        cluster
+
+        expect(cluster.sasl?).to eq(true)
+      end
+    end
+
+    context 'ssl' do
+      before do
+        allow(KafkaCommand).to receive(:config).and_return(
+          KafkaCommand::Configuration.new(KafkaCommand::Configuration.parse_yaml('spec/fixtures/files/kafka_command_ssl.yml'))
+        )
+      end
+
+      it 'creates a kafka client with brokers, client_id, and ssl params' do
+        expect(KafkaCommand::Client).to receive(:new).with(
+          brokers: brokers,
+          client_id: 'ssl_test_cluster',
+          ssl_client_cert: 'test_client_cert',
+          ssl_client_cert_key: 'test_client_cert_key',
+          ssl_ca_cert: 'test_ca_cert'
+        )
+
+        cluster
+
+        expect(cluster.ssl?).to eq(true)
+        expect(cluster.sasl?).to eq(false)
+      end
+    end
+  end
+
+  describe '#create_topic' do
+    let(:kwargs) do
+      { replication_factor: 1, num_partitions: 1 }
+    end
+
+    it 'calls KafkaCommand::Client#create_topic' do
+      expect_any_instance_of(KafkaCommand::Client).to receive(:create_topic).with(topic_name, **kwargs)
+      cluster.create_topic(topic_name, **kwargs)
+    end
+
+    it 'creates a topic' do
+      topic_count = cluster.topics.count
+      topic = cluster.create_topic(topic_name, **kwargs)
+      expect(topic).to be_an_instance_of(KafkaCommand::Topic)
+      cluster.client.refresh_topics!
+      expect(cluster.topics.count).to eq(topic_count + 1)
+    end
+  end
+
+  describe '#to_s' do
+    it 'returns the name' do
+      expect(cluster.to_s).to eq(cluster.name)
+    end
+  end
+
+  describe '.all' do
+    it "returns a list of #{described_class}" do
+      described_class.all.each do |c|
+        expect(c).to be_an_instance_of(described_class)
+      end
+    end
+  end
+
+  describe '.find' do
+    context 'cluster exists' do
+      it 'finds the cluster' do
+        expect(described_class.find(cluster.name)).to eq(cluster)
+      end
+    end
+
+    context 'cluster does not exist' do
+      it 'returns nil' do
+        expect(described_class.find('doesnotexists')).to be_nil
+      end
+    end
+  end
+
+  context 'forwarding' do
+    describe '#topics' do
+      before { create_topic(topic_name) }
+
+      it 'fowards the call to the KafkaCommand::Client' do
+        expect(cluster.client).to receive(:topics).once
+        cluster.topics
+      end
+
+      it 'returns a list of topics' do
+        expect(cluster.topics.first).to be_an_instance_of(KafkaCommand::Topic)
+        expect(cluster.topics.map(&:name)).to include(topic_name)
+      end
+    end
+
+    describe '#brokers' do
+      it 'fowards the call to the KafkaCommand::Client' do
+        expect(cluster.client).to receive(:brokers).once
+        cluster.brokers
+      end
+
+      it 'returns a list of brokers' do
+        expect(cluster.brokers.first).to be_an_instance_of(KafkaCommand::Broker)
+      end
+    end
+
+    describe '#groups' do
+      before { create_topic(topic_name) }
+      let(:group_id) { SecureRandom.hex(12) }
+
+      it 'fowards to the call to the KafkaCommand::Client' do
+        expect(cluster.client).to receive(:groups).once
+        cluster.groups
+      end
+
+      it 'returns a list of groups' do
+        run_consumer_group(topic_name, group_id) do
+          expect(cluster.groups.first).to be_an_instance_of(KafkaCommand::ConsumerGroup)
+          expect(cluster.groups.map(&:group_id)).to include(group_id)
+        end
+      end
+    end
+  end
+end

data/spec/models/kafka_command/consumer_group_partition_spec.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+RSpec.describe KafkaCommand::ConsumerGroupPartition do
+  let(:group_id)     { "group-#{SecureRandom.hex(12)}" }
+  let(:topic_name)   { "test-#{SecureRandom.hex(12)}" }
+  let(:lag)          { 1 }
+  let(:offset)       { 200 }
+  let(:partition_id) { 1 }
+
+  let(:consumer_group_partition_) do
+    described_class.new(
+      group_id: group_id,
+      lag: lag,
+      offset: offset,
+      topic_name: topic_name,
+      partition_id: partition_id
+    )
+  end
+
+  describe '#new' do
+    it 'creates a new Kafka::ConsumerGroupPartition with attributes' do
+      expect(consumer_group_partition_.lag).to eq(lag)
+      expect(consumer_group_partition_.group_id).to eq(group_id)
+      expect(consumer_group_partition_.topic_name).to eq(topic_name)
+      expect(consumer_group_partition_.offset).to eq(offset)
+      expect(consumer_group_partition_.partition_id).to eq(partition_id)
+    end
+  end
+
+  describe '#as_json' do
+    let(:expected_json) do
+      {
+        lag: lag,
+        offset: offset,
+        partition_id: partition_id
+      }
+    end
+
+    it 'returns the expected payload' do
+      expect(consumer_group_partition_.as_json).to eq(expected_json)
+    end
+  end
+end

data/spec/models/kafka_command/consumer_group_spec.rb

@@ -0,0 +1,236 @@
+# frozen_string_literal: true
+
+RSpec.describe KafkaCommand::ConsumerGroup do
+  let(:group_id) { "test-group-#{SecureRandom.hex(24)}" }
+  let(:topic_name) { "test-topic-#{SecureRandom.hex(24)}" }
+  let(:num_partitions) { 5 }
+  let(:group) do
+    KafkaCommand::Cluster.all.first.groups.find { |g| g.group_id == group_id }
+  end
+
+  before { create_topic(topic_name, num_partitions: num_partitions) }
+
+  describe '#new' do
+    describe 'running' do
+      it 'initializes a stable Kafka::ConsumerGroup with group_id, members, and state' do
+        run_consumer_group(topic_name, group_id) do
+          expect(group.group_id).to eq(group_id)
+          expect(group.members).to_not be_empty
+          expect(group.members.sample).to be_an_instance_of(KafkaCommand::GroupMember)
+          expect(group.state).to eq('Stable')
+        end
+      end
+    end
+
+    describe 'dormant' do
+      before { run_consumer_group(topic_name, group_id) }
+
+      it 'initializes an empty Kafka::ConsumerGroup with group_id, members, and state' do
+        expect(group.group_id).to eq(group_id)
+        expect(group.members).to be_empty
+        expect(group.state).to eq('Empty')
+      end
+    end
+  end
+
+  describe '#refresh!' do
+    it 'refreshes the group metadata' do
+      run_consumer_group(topic_name, group_id) do
+        expect(group.state).to eq('Stable')
+        expect(group.members).to_not be_empty
+      end
+
+      group.refresh!
+      expect(group.members).to be_empty
+      expect(group.state).to eq('Empty')
+    end
+  end
+
+  describe '#stable?' do
+    describe 'running' do
+      it 'returns true' do
+        run_consumer_group(topic_name, group_id) do
+          expect(group.stable?).to eq(true)
+        end
+      end
+    end
+
+    describe 'dormant' do
+      before { run_consumer_group(topic_name, group_id) }
+
+      it 'returns false' do
+        expect(group.stable?).to eq(false)
+      end
+    end
+  end
+
+  describe '#stable?' do
+    describe 'running' do
+      it 'returns true' do
+        run_consumer_group(topic_name, group_id) do
+          expect(group.stable?).to eq(true)
+        end
+      end
+    end
+
+    describe 'dormant' do
+      before { run_consumer_group(topic_name, group_id) }
+
+      it 'returns false' do
+        expect(group.stable?).to eq(false)
+      end
+    end
+  end
+
+  describe '#empty?' do
+    describe 'running' do
+      it 'returns false' do
+        run_consumer_group(topic_name, group_id) do
+          expect(group.empty?).to eq(false)
+        end
+      end
+    end
+
+    describe 'dormant' do
+      before { run_consumer_group(topic_name, group_id) }
+
+      it 'returns true' do
+        expect(group.empty?).to eq(true)
+      end
+    end
+  end
+
+  describe '#partitions_for' do
+    let(:num_partitions) { 3 }
+    let(:partitions) { group.partitions_for(topic_name) }
+    let(:total_lag)  { partitions.map(&:lag).compact.reduce(:+) }
+
+    it 'returns ConsumerGroupPartitions' do
+      run_consumer_group(topic_name, group_id) do
+        expect(partitions.count).to eq(num_partitions)
+        expect(partitions.sample).to be_an_instance_of(KafkaCommand::ConsumerGroupPartition)
+        expect(partitions.sample.group_id).to eq(group_id)
+        expect(partitions.sample.topic_name).to eq(topic_name)
+      end
+    end
+
+    context 'some partitions no offsets' do
+      before { run_consumer_group(topic_name, group_id) }
+
+      it 'has "nil" lag' do
+        expect(partitions.map(&:lag)).to include(nil)
+      end
+    end
+
+    context 'lag' do
+      let(:partition_0) do
+        partitions.find { |p| p.partition_id == 0 }
+      end
+
+      let(:partition_1) do
+        partitions.find { |p| p.partition_id == 1 }
+      end
+
+      let(:partition_2) do
+        partitions.find { |p| p.partition_id == 2 }
+      end
+
+      before do
+        deliver_message('test', topic: topic_name, partition: 0)
+        deliver_message('test1', topic: topic_name,  partition: 1)
+        deliver_message('test2', topic: topic_name, partition: 2)
+        run_consumer_group(topic_name, group_id, num_messages_to_consume: 3)
+      end
+
+      describe 'no lag' do
+        it 'has no lag' do
+          expect(partition_0.lag).to eq(0)
+          expect(partition_1.lag).to eq(0)
+          expect(partition_2.lag).to eq(0)
+        end
+      end
+
+      describe 'some lag' do
+        before do
+          2.times { deliver_message('test', topic: topic_name, partition: 0) }
+          3.times { deliver_message('test1', topic: topic_name,  partition: 1) }
+        end
+
+        it 'has lag' do
+          expect(total_lag).to eq(5)
+          expect(partition_0.lag).to eq(2)
+          expect(partition_1.lag).to eq(3)
+          expect(partition_2.lag).to eq(0)
+        end
+      end
+    end
+  end
+
+  describe '#consumed_topics' do
+    context 'running' do
+      it 'returns a list of topics' do
+        run_consumer_group(topic_name, group_id) do
+          expect(group.consumed_topics.map(&:name)).to include(topic_name)
+        end
+      end
+    end
+
+    context 'dormant' do
+      before { run_consumer_group(topic_name, group_id) }
+
+      it 'returns an empty list' do
+        expect(group.consumed_topics).to be_empty
+      end
+    end
+  end
+
+  describe '#as_json' do
+    let(:num_partitions) { 3 }
+    let(:partitions) { group.partitions_for(topic_name) }
+
+
+    before do
+      deliver_message('test', topic: topic_name, partition: 0)
+      deliver_message('test1', topic: topic_name,  partition: 1)
+      deliver_message('test2', topic: topic_name, partition: 2)
+      run_consumer_group(topic_name, group_id, num_messages_to_consume: 3)
+      2.times { deliver_message('test', topic: topic_name, partition: 0) }
+      3.times { deliver_message('test1', topic: topic_name,  partition: 1) }
+    end
+
+    context 'running' do
+      let(:expected_json) do
+        {
+          group_id: group_id,
+          topics: [
+            {
+              name: topic_name,
+              partitions: partitions.map(&:as_json)
+            }
+          ],
+          state: 'Stable'
+        }
+      end
+
+      it 'returns the expected payload' do
+        run_consumer_group(topic_name, group_id) do
+          expect(group.as_json).to eq(expected_json)
+        end
+      end
+    end
+
+    context 'dormant' do
+      let(:expected_json) do
+        {
+          group_id: group_id,
+          topics: [],
+          state: 'Empty'
+        }
+      end
+
+      it 'returns the expected payload' do
+        expect(group.as_json).to eq(expected_json)
+      end
+    end
+  end
+end

data/spec/models/kafka_command/partition_spec.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+RSpec.describe KafkaCommand::Partition do
+  let(:topic_name) { "test-#{SecureRandom.hex(12)}" }
+  subject do
+    KafkaCommand::Cluster.all.first.topics.find { |t| t.name == topic_name }.partitions.sample
+  end
+
+  before { create_topic(topic_name) }
+
+  describe '#new' do
+    it 'contains a reference to a topic wrapper' do
+      expect(subject.topic).to be_an_instance_of(KafkaCommand::Topic)
+    end
+  end
+
+  describe '#highwater_mark_offset' do
+    let(:num_messages) { 2 }
+
+    before do
+      num_messages.times { deliver_message('test', topic: topic_name) }
+    end
+
+    it 'retrieves the latest offset' do
+      expect(subject.highwater_mark_offset).to eq(num_messages)
+    end
+  end
+
+  describe '#as_json' do
+    let(:expected_json) do
+      {
+        isr: subject.isr,
+        leader: subject.leader,
+        id: subject.partition_id,
+        highwater_mark_offset: subject.highwater_mark_offset
+      }.to_json
+    end
+
+    it 'equals the expected json' do
+      expect(subject.as_json.to_json).to eq(expected_json)
+    end
+  end
+
+  context 'forwarding' do
+    let(:partition_metadata) do
+      subject.instance_variable_get(:@partition_metadata)
+    end
+
+    describe '#isr' do
+      it 'forwards isr to the partition metadata' do
+        expect(partition_metadata).to receive(:isr)
+        subject.isr
+      end
+
+      it 'returns the isr' do
+        expect(subject.isr).to be_an_instance_of(Array)
+        expect(subject.isr.first).to be_an_instance_of(Integer)
+      end
+    end
+
+    describe '#leader' do
+      it 'forwards leader to the partition metadata' do
+        expect(partition_metadata).to receive(:leader)
+        subject.leader
+      end
+
+      it 'returns the leader broker' do
+        expect(subject.leader).to be_an_instance_of(Integer)
+      end
+    end
+
+    describe '#partition_id' do
+      it 'forwards leader to the partition metadata' do
+        expect(partition_metadata).to receive(:leader)
+        subject.leader
+      end
+
+      it 'returns the partition id' do
+        expect(subject.partition_id).to be > -1
+      end
+    end
+
+    describe '#replicas' do
+      it 'forwards replicas to the partition metadata' do
+        expect(partition_metadata).to receive(:replicas)
+        subject.replicas
+      end
+
+      it 'returns the replica node ids' do
+        expect(subject.replicas).to be_an_instance_of(Array)
+        expect(subject.replicas.first).to be_an_instance_of(Integer)
+      end
+    end
+  end
+end