codeclimate-poseidon 0.0.8

data/spec/unit/message_to_send_spec.rb

@@ -0,0 +1,10 @@
+require 'spec_helper'
+
+RSpec.describe MessageToSend do
+  it "provides access to topic,value,key" do
+    mts = MessageToSend.new("hello_topic", "Hello World", "key")
+    expect(mts.topic).to eq("hello_topic")
+    expect(mts.value).to eq("Hello World")
+    expect(mts.key).to eq("key")
+  end
+end

data/spec/unit/messages_for_broker_spec.rb

@@ -0,0 +1,54 @@
+require 'spec_helper'
+
+RSpec.describe MessagesForBroker do
+  context "twos message one to broker 0, partition 0, another to partition 1" do
+    before(:each) do
+      @messages = [ Message.new(:topic => "topic1",:value => "hi0"),
+                    Message.new(:topic => "topic1",:value => "hi1")]
+
+      @compression_config = double('compression_config',
+                                 :compression_codec_for_topic => nil)
+
+      @mfb = MessagesForBroker.new(0)
+      @mfb.add(@messages[0], 0)
+      @mfb.add(@messages[1], 1)
+    end
+
+    it "provides the messages" do
+      expect(@mfb.messages.to_set).to eq(@messages.to_set)
+    end
+
+    it "is has a broker_id of 0" do
+      expect(@mfb.broker_id).to eq(0)
+    end
+
+    it "builds the protocol object correctly" do
+      protocol_object = @mfb.build_protocol_objects(@compression_config)
+
+      messages_for_topics = [
+        MessagesForTopic.new("topic1",
+          [
+            MessagesForPartition.new(0, MessageSet.new([@messages[0]])),
+            MessagesForPartition.new(1, MessageSet.new([@messages[1]])),
+          ])
+      ]
+      expect(protocol_object).to eq(messages_for_topics)
+    end
+
+    context "and topic is compressed" do
+      it "builds the protocol object correctly" do
+        allow(@compression_config).to receive_messages(:compression_codec_for_topic => Compression::GzipCodec)
+        protocol_object = @mfb.build_protocol_objects(@compression_config)
+
+        messages_for_topics = [
+          MessagesForTopic.new("topic1",
+            [
+              MessagesForPartition.new(0, MessageSet.new([@messages[0]]).compress(Compression::GzipCodec)),
+              MessagesForPartition.new(1, MessageSet.new([@messages[1]]).compress(Compression::GzipCodec)),
+            ])
+        ]
+        expect(protocol_object).to eq(messages_for_topics)
+      end
+    end
+  end
+end

data/spec/unit/messages_to_send_batch_spec.rb

@@ -0,0 +1,25 @@
+require 'spec_helper'
+
+RSpec.describe MessagesToSendBatch do
+  context "messages sent to two different brokers" do
+    before(:each) do
+      message_conductor = double('message_conductor')
+      allow(message_conductor).to receive(:destination).and_return([0,0],[1,1])
+
+      @messages = [
+        Message.new(:topic => "topic1", :value => "hi"),
+        Message.new(:topic => "topic1", :value => "hi")
+      ]
+      @batch = MessagesToSendBatch.new(@messages, message_conductor)
+    end
+
+    it "returns a couple messages brokers" do
+      expect(@batch.messages_for_brokers.size).to eq(2)
+    end
+
+    it "has all messages in the returned message brokers" do
+      messages = @batch.messages_for_brokers.map(&:messages).flatten
+      expect(messages.to_set).to eq(@messages.to_set)
+    end
+  end
+end

data/spec/unit/messages_to_send_spec.rb

@@ -0,0 +1,63 @@
+require 'spec_helper'
+
+RSpec.describe MessagesToSend do
+  before(:each) do
+    @messages = []
+    @messages << Message.new(:topic => "test1", :value => "hi")
+    @messages << Message.new(:topic => "test2", :value => "hi")
+    @messages << Message.new(:topic => "test2", :value => "hi")
+
+
+    @cluster_metadata = double('cluster_metdata').as_null_object
+    @mts = MessagesToSend.new(@messages, @cluster_metadata)
+  end
+
+  describe "needing metadata" do
+    it "returns set of topics" do
+      expect(@mts.topic_set).to eq(Set.new(["test1","test2"]))
+    end
+
+    it "asks ClusterMetadata about having metadata" do
+      allow(@cluster_metadata).to receive(:have_metadata_for_topics?).and_return(true)
+
+      expect(@mts.needs_metadata?).to eq(false)
+    end
+  end
+
+  describe "sending" do
+    before(:each) do
+      @mfb = double('mfb', :messages => @messages)
+      @messages_for_brokers = [@mfb]
+
+      @mtsb = double('messages_to_send_batch').as_null_object
+      allow(@mtsb).to receive(:messages_for_brokers).and_return(@messages_for_brokers)
+
+      allow(MessagesToSendBatch).to receive(:new).and_return(@mtsb)
+    end
+
+    context "is successful" do
+      before(:each) do
+        @mts.messages_for_brokers(nil).each do |mfb|
+          @mts.successfully_sent(mfb.messages)
+        end
+      end
+
+      it "successfully sends all" do
+        expect(@mts.pending_messages?).to eq(false)
+      end
+    end
+
+    context "is not successful" do
+      before(:each) do
+        @mts.messages_for_brokers(nil).each do |mfb|
+        end
+      end
+
+      it "does not send all" do
+        @mts.messages_for_brokers(nil).each do |mfb|
+        end
+        expect(@mts.pending_messages?).to eq(true)
+      end
+    end
+  end
+end

data/spec/unit/partition_consumer_spec.rb

@@ -0,0 +1,142 @@
+require 'spec_helper'
+
+RSpec.describe PartitionConsumer do
+  before(:each) do
+    @connection = double('connection')
+    allow(Connection).to receive(:new).and_return(@connection)
+
+    offset = Protocol::Offset.new(100)
+    partition_offsets = [Protocol::PartitionOffset.new(0, 0, [offset])]
+    @offset_response = [Protocol::TopicOffsetResponse.new("test_topic", partition_offsets)]
+    allow(@connection).to receive(:offset).and_return(@offset_response)
+  end
+
+  describe "creation" do
+    context "when passed unknown options" do
+      it "raises an ArgumentError" do
+        expect { PartitionConsumer.new("test_client", "localhost", 9092, "test_topic", 0, :earliest_offset, :unknown => true) }.to raise_error(ArgumentError)
+      end
+    end
+
+    context "when passed an unknown offset" do
+      it "raises an ArgumentError" do
+        expect { PartitionConsumer.new("test_client", "localhost", 9092, "test_topic", 0, :coolest_offset) }.to raise_error(ArgumentError)
+      end
+    end
+  end
+
+  describe "next offset" do
+    context "when offset is not set" do
+      it "resolves offset if it's not set" do
+        expect(@connection).to receive(:offset).and_return(@offset_response)
+        pc = PartitionConsumer.new("test_client", "localhost", 9092, "test_topic",
+                                   0, :earliest_offset)
+
+        pc.next_offset
+      end
+
+      it "returns resolved offset" do
+        pc = PartitionConsumer.new("test_client", "localhost", 9092, "test_topic",
+                                   0, :earliest_offset)
+        expect(pc.next_offset).to eq(100)
+      end
+    end
+
+    context "when offset is set" do
+      it "does not resolve it" do
+        pc = PartitionConsumer.new("test_client", "localhost", 9092, "test_topic",
+                                   0, 200)
+        pc.next_offset
+      end
+    end
+
+    context "when call returns an error" do
+      it "is raised" do
+        allow(@offset_response.first.partition_offsets.first).to receive(:error).and_return(2)
+        pc = PartitionConsumer.new("test_client", "localhost", 9092, "test_topic",
+                                   0, :earliest_offset)
+
+        expect { pc.next_offset }.to raise_error(Errors::InvalidMessage)
+      end
+    end
+
+    context "when no offset exists" do
+      it "sets offset to 0" do
+        pc = PartitionConsumer.new("test_client", "localhost", 9092, "test_topic",
+                                   0, :earliest_offset)
+
+        allow(@offset_response.first.partition_offsets.first).to receive(:offsets).and_return([])
+        expect(pc.next_offset).to eq(0)
+      end
+    end
+
+    context "when offset negative" do
+      it "resolves offset to one " do
+        pc = PartitionConsumer.new("test_client", "localhost", 9092, "test_topic",
+                                   0, -10)
+        expect(pc.next_offset).to eq(90)
+      end
+    end
+  end
+
+  describe "fetching messages" do
+    before(:each) do
+      message_set = MessageSet.new
+      message_set << Message.new(:value => "value", :key => "key", :offset => 90)
+      partition_fetch_response = Protocol::PartitionFetchResponse.new(0, 0, 100, message_set)
+      topic_fetch_response = Protocol::TopicFetchResponse.new('test_topic',
+                                                    [partition_fetch_response])
+      @response = Protocol::FetchResponse.new(double('common'), [topic_fetch_response])
+
+      allow(@connection).to receive(:fetch).and_return(@response)
+      @pc = PartitionConsumer.new("test_client", "localhost", 9092, "test_topic", 0, :earliest_offset)
+    end
+
+    it "returns FetchedMessage objects" do
+      expect(@pc.fetch.first.class).to eq(FetchedMessage)
+    end
+
+    it "uses object defaults" do
+      expect(@connection).to receive(:fetch).with(10_000, 1, anything)
+      @pc.fetch
+    end
+
+    context "when options are passed" do
+      it "overrides object defaults" do
+        expect(@connection).to receive(:fetch).with(20_000, 1, anything)
+        @pc = PartitionConsumer.new("test_client", "localhost", 9092, "test_topic", 0, :earliest_offset, :max_wait_ms => 20_000)
+
+        @pc.fetch
+      end
+    end
+
+    context "when negative offset beyond beginning of partition is passed" do
+      it "starts from the earliest offset" do
+        @pc = PartitionConsumer.new("test_client", "localhost", 9092, "test_topic", 0, -10000)
+        pfr = @response.topic_fetch_responses.first.partition_fetch_responses.first
+        allow(pfr).to receive(:error).and_return(1, 1, 0)
+
+        @pc.fetch
+      end
+    end
+
+    context "when call returns an error" do
+      it "is raised" do
+        pfr = @response.topic_fetch_responses.first.partition_fetch_responses.first
+        allow(pfr).to receive(:error).and_return(2)
+
+        expect { @pc.fetch }.to raise_error(Errors::InvalidMessage)
+      end
+    end
+
+    it "sets the highwater mark" do
+      @pc.fetch
+      expect(@pc.highwater_mark).to eq(100)
+    end
+
+    it "sets the latest offset" do
+      @pc.fetch
+      expect(@pc.next_offset).to eq(91)
+    end
+  end
+end

data/spec/unit/producer_compression_config_spec.rb

@@ -0,0 +1,42 @@
+require 'spec_helper'
+
+RSpec.describe ProducerCompressionConfig do
+  describe "creation" do
+    it "raises ArgumentError when codec is unknown" do
+      expect { ProducerCompressionConfig.new(:ripple, nil) }.to raise_error(ArgumentError)
+    end
+  end
+
+  context "no codec set" do
+    it "compresses no topics" do
+      pcc = ProducerCompressionConfig.new(nil,nil)
+      expect(pcc.compression_codec_for_topic("test")).to eq(false)
+    end
+  end
+
+  describe "none compression codec" do
+    it "compresses no topics" do
+      pcc = ProducerCompressionConfig.new(:none,nil)
+      expect(pcc.compression_codec_for_topic("test")).to eq(false)
+    end
+  end
+
+  describe "compression codec no topics specified" do
+    it "compresses any topic" do
+      pcc = ProducerCompressionConfig.new(:gzip,nil)
+      expect(pcc.compression_codec_for_topic("test")).to eq(Compression::GzipCodec)
+    end
+  end
+
+  describe "compression codec set, but only compress 'compressed' topic" do
+    it "compresses 'compressed' topic" do
+      pcc = ProducerCompressionConfig.new(:gzip, ["compressed"])
+      expect(pcc.compression_codec_for_topic("compressed")).to eq(Compression::GzipCodec)
+    end
+
+    it "does not compresses 'test' topic" do
+      pcc = ProducerCompressionConfig.new(:gzip, ["compressed"])
+      expect(pcc.compression_codec_for_topic("test")).to eq(false)
+    end
+  end
+end

data/spec/unit/producer_spec.rb

@@ -0,0 +1,51 @@
+require 'spec_helper'
+
+RSpec.describe Producer do
+  it "requires brokers and client_id" do
+    expect { Producer.new }.to raise_error
+  end
+
+  it "raises ArgumentError on unknown arguments" do
+    expect { Producer.new([],"client_id", :unknown => true) }.to raise_error(ArgumentError)
+  end
+
+  it "raises ArgumentError unless brokers is an enumerable" do
+    expect { Producer.new("host:port","client_id") }.to raise_error(ArgumentError)
+  end
+
+  it "raises ProducerShutdown if we try to send to a shutdown producer" do
+    p = Producer.new(["host:port"],"client_id")
+    p.close
+    expect { p.send_messages([]) }.to raise_error(Errors::ProducerShutdownError)
+  end
+
+  it "accepts all options" do
+    expect { Producer.new([],"client_id", Producer::OPTION_DEFAULTS.dup) }.not_to raise_error
+  end
+
+  it "accepts socket_timeout_ms option" do
+    expect { Producer.new([],"client_id", socket_timeout_ms: 10_000) }.not_to raise_error
+  end
+
+  describe "sending messages" do
+    before(:each) do
+      @sync_producer = double('sync_producer').as_null_object
+      allow(SyncProducer).to receive(:new).and_return(@sync_producer)
+
+      @producer = Producer.new([], "client_id", :type => :sync)
+    end
+
+    it "turns MessagesToSend into Message objects" do
+      expect(@sync_producer).to receive(:send_messages).with(an_instance_of(Array)) do |array|
+        array.each { |obj| expect(obj).to be_an_instance_of(Message) }
+      end
+
+      m = MessageToSend.new("topic", "value")
+      @producer.send_messages([m])
+    end
+
+    it "raises an ArgumentError if you try to send a single message" do
+      expect { @producer.send_messages(MessageToSend.new("topic", "value")) }.to raise_error(ArgumentError)
+    end
+  end
+end

data/spec/unit/protocol/request_buffer_spec.rb

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+
+require 'spec_helper'
+include Protocol
+
+RSpec.describe RequestBuffer do
+  subject(:buffer) { Poseidon::Protocol::RequestBuffer.new }
+
+  it 'appends UTF-8 strings' do
+    expect do
+      str = 'hello ümlaut'
+      buffer.append(str)
+      buffer.append(str.force_encoding(Encoding::BINARY))
+    end.to_not raise_error
+  end
+end

data/spec/unit/protocol_spec.rb

@@ -0,0 +1,54 @@
+require 'spec_helper'
+include Protocol
+RSpec.describe RequestCommon do
+  it "roundtrips" do
+    rc = RequestCommon.new(0,1,2,"client_id")
+
+    req_buffer = RequestBuffer.new
+    rc.write(req_buffer)
+
+    resp_buffer = ResponseBuffer.new(req_buffer.to_s)
+    rc_roundtrip = RequestCommon.read(resp_buffer)
+
+    expect(rc).to eq(rc_roundtrip)
+  end
+end
+
+RSpec.describe MetadataRequest do
+  it "roundtrips" do
+    rc = RequestCommon.new(0,1,2,"client_id")
+    mr = MetadataRequest.new(rc, ["topic1","topic2"])
+
+    req_buffer = RequestBuffer.new
+    mr.write(req_buffer)
+
+    resp_buffer = ResponseBuffer.new(req_buffer.to_s)
+    mr_roundtrip = MetadataRequest.read(resp_buffer)
+
+    expect(mr).to eq(mr_roundtrip)
+  end
+end
+
+RSpec.describe "objects with errors" do
+  it "returns objects that have errors" do
+    message_set = MessageSet.new
+    message_set << Message.new(:value => "value", :key => "key")
+    partition_fetch_response = PartitionFetchResponse.new(0, 5, 100, message_set)
+    topic_fetch_response = TopicFetchResponse.new('test_topic',
+                                                  [partition_fetch_response])
+    response = FetchResponse.new(double('common'), [topic_fetch_response])
+
+    expect(response.objects_with_errors).to eq([partition_fetch_response])
+  end
+
+  it "raises error when asked" do
+    message_set = MessageSet.new
+    message_set << Message.new(:value => "value", :key => "key")
+    partition_fetch_response = PartitionFetchResponse.new(0, 5, 100, message_set)
+    topic_fetch_response = TopicFetchResponse.new('test_topic',
+                                                  [partition_fetch_response])
+    response = FetchResponse.new(double('common'), [topic_fetch_response])
+
+    expect { response.raise_error_if_one_exists }.to raise_error
+  end
+end