poseidon 0.0.1

data/spec/unit/message_conductor_spec.rb

@@ -0,0 +1,147 @@
+require 'spec_helper'
+
+include Protocol
+describe MessageConductor do
+  context "two avialable partitions" do
+    before(:each) do
+      partitions = [
+        PartitionMetadata.new(nil, 0, 1, [1,2], [1,2]),
+        PartitionMetadata.new(nil, 1, 2, [2,1], [2,1])
+      ]
+      topics = [TopicMetadata.new(TopicMetadataStruct.new(nil, "test", partitions))]
+      brokers = [Broker.new(1, "host1", 1), Broker.new(2, "host2", 2)]
+
+      @mr = MetadataResponse.new(nil, brokers, topics)
+
+      @cm = ClusterMetadata.new
+      @cm.update(@mr)
+    end
+
+    context "no custom partitioner" do
+      before(:each) do
+        @mc = MessageConductor.new(@cm, nil)
+      end
+
+      context "for unkeyed messages" do
+        it "round robins which partition the message should go to" do
+          [0,1,0,1].each do |destination|
+            expect(@mc.destination("test").first).to eq(destination)
+          end
+        end
+
+        context "unknown topic" do
+          it "returns -1 for broker and partition" do
+            expect(@mc.destination("no_exist")).to eq([-1,-1])
+          end
+        end
+      end
+
+      context "keyed message" do
+        it "sends the same keys to the same destinations" do
+          keys = 1000.times.map { rand(500).to_s }
+          key_destinations = {}
+
+          keys.sort_by { rand }.each do |k|
+            partition,broker = @mc.destination("test", k)
+
+            key_destinations[k] ||= []
+            key_destinations[k].push([partition,broker])
+          end
+
+          expect(key_destinations.values.all? { |destinations| destinations.uniq.size == 1 }).to eq(true)
+        end
+      end
+    end
+
+    context "custom partitioner" do
+      before(:each) do
+        partitioner = Proc.new { |key, count| key.split("_").first.to_i % count }
+        @mc = MessageConductor.new(@cm, partitioner)
+      end
+
+      it "obeys custom partitioner" do
+        expect(@mc.destination("test", "2_hello").first).to eq(0)
+        expect(@mc.destination("test", "3_hello").first).to eq(1)
+      end
+    end
+
+    context "broken partitioner" do
+      before(:each) do
+        partitioner = Proc.new { |key, count| count + 1 }
+        @mc = MessageConductor.new(@cm, partitioner)
+      end
+
+      it "raises InvalidPartitionError" do
+        expect{@mc.destination("test", "2_hello").first}.to raise_error(Errors::InvalidPartitionError)
+      end
+    end
+  end
+
+  context "two partitions, one is unavailable" do
+    before(:each) do
+      partitions = [
+        Protocol::PartitionMetadata.new(nil, 0, 1, [1,2], [1,2]),
+        Protocol::PartitionMetadata.new(nil, 1, nil, [2,1], [2,1])
+      ]
+      topics = [TopicMetadata.new(TopicMetadataStruct.new(nil, "test", partitions))]
+      brokers = [Broker.new(1, "host1", 1), Broker.new(2, "host2", 2)]
+
+      @mr = MetadataResponse.new(nil, brokers, topics)
+
+      @cm = ClusterMetadata.new
+      @cm.update(@mr)
+
+      @mc = MessageConductor.new(@cm, nil)
+    end
+
+    context "keyless message" do
+      it "is never sent to an unavailable partition" do
+        10.times do |destination|
+          expect(@mc.destination("test").first).to eq(0)
+        end
+      end
+    end
+
+    context "keyed message" do
+      it "is sent to unavailable partition" do
+        destinations = Set.new
+        100.times do |key|
+          destinations << @mc.destination("test",key.to_s).first
+        end
+        expect(destinations).to eq(Set.new([0,1]))
+      end
+    end
+  end
+
+  context "no available partitions" do
+    before(:each) do
+      partitions = [
+        Protocol::PartitionMetadata.new(nil, 0, nil, [1,2], [1,2]),
+        Protocol::PartitionMetadata.new(nil, 1, nil, [2,1], [2,1])
+      ]
+      topics = [TopicMetadata.new(TopicMetadataStruct.new(nil, "test", partitions))]
+      brokers = [Broker.new(1, "host1", 1), Broker.new(2, "host2", 2)]
+
+      @mr = MetadataResponse.new(nil, brokers, topics)
+
+      @cm = ClusterMetadata.new
+      @cm.update(@mr)
+
+      @mc = MessageConductor.new(@cm, nil)
+    end
+
+    context "keyless message" do
+      it "return -1 for broker and partition" do
+        expect(@mc.destination("test").first).to eq(-1)
+      end
+    end
+
+    context "keyed message" do
+      it "returns a valid partition and -1 for broker" do
+        partition_id, broker_id = @mc.destination("test", "key")
+        expect(partition_id).to_not eq(-1)
+        expect(broker_id).to eq(-1)
+      end
+    end
+  end
+end

data/spec/unit/message_set_spec.rb

@@ -0,0 +1,42 @@
+require 'spec_helper'
+
+describe MessageSet do
+  describe "converting to a compressed message" do
+    before(:each) do
+      ms = MessageSet.new([Message.new(:value => "I will be compressed", :topic => "test")])
+
+      @compressed_message_set = ms.compress(Compression::GzipCodec)
+    end
+
+    it "contains a compressed message" do
+      expect(@compressed_message_set.messages.first.compressed?).to eq(true)
+    end
+
+    it "can be decompressed and reconstituted" do
+      expect(@compressed_message_set.flatten.first.value).to eq("I will be compressed")
+    end
+  end
+
+  describe "adding messages" do
+    it "adds the message to the struct" do
+      m = Message.new(:value => "sup", :topic => "topic")
+      ms = MessageSet.new
+      ms << m
+      expect(ms.struct.messages.first).to eq(m)
+    end
+  end
+
+  describe "encoding" do
+    it "round trips" do
+      m = Message.new(:value => "sup", :key => "keyz", :topic => "hello")
+      ms = MessageSet.new
+      ms << m
+
+      request_buffer = Protocol::RequestBuffer.new
+      ms.write(request_buffer)
+
+      response_buffer = Protocol::ResponseBuffer.new(request_buffer.to_s)
+      expect(MessageSet.read(response_buffer)).to eq(ms)
+    end
+  end
+end

data/spec/unit/message_spec.rb

@@ -0,0 +1,112 @@
+require 'spec_helper'
+
+describe Message do
+  describe "when constructing a new message" do
+    it 'raises an ArgumentError on unknown options' do
+      expect { Message.new(:cow => "dog") }.to raise_error(ArgumentError)
+    end
+
+    it 'handles options correctly' do
+      m = Message.new(:value => "value",
+                      :key => "key",
+                      :attributes => 1,
+                      :topic => "topic")
+
+      expect(m.value).to eq("value")
+      expect(m.key).to eq("key")
+      expect(m.compressed?).to eq(true)
+      expect(m.topic).to eq("topic")
+    end
+  end
+
+  describe "checksum" do
+    context "is incorrect" do
+      before(:each) do
+        m = Message.new(:value => "value",
+                        :key => "key",
+                        :topic => "topic")
+
+        req_buf = Protocol::RequestBuffer.new
+        m.write(req_buf)
+
+        @s = req_buf.to_s
+        @s[-1] = "q" # break checksum
+      end
+
+      it "knows it" do
+        expect { Message.read(Protocol::ResponseBuffer.new(@s)) }.to raise_error(Errors::ChecksumError)
+      end
+    end
+
+    context 'is correct' do
+      before(:each) do
+        m = Message.new(:value => "value",
+                        :key => "key",
+                        :topic => "topic")
+
+        req_buf = Protocol::RequestBuffer.new
+        m.write(req_buf)
+
+        @s = req_buf.to_s
+      end
+
+      it "raises no error" do
+        expect { Message.read(Protocol::ResponseBuffer.new(@s)) }.to_not raise_error
+      end
+    end
+  end
+
+  describe "truncated message" do
+    before(:each) do
+      m = Message.new(:value => "value",
+                      :key => "key",
+                      :topic => "topic")
+
+      req_buf = Protocol::RequestBuffer.new
+      m.write(req_buf)
+
+      @s = req_buf.to_s
+    end
+
+    it "reading returns nil" do
+      expect(Message.read(Protocol::ResponseBuffer.new(@s[0..-4]))).to eq(nil)
+    end
+  end
+
+  context "invalid utf8 string for value" do
+    it "builds the payload without error" do
+      s = "asdf\xffasdf"
+      m = Message.new(:value => s,
+                      :key => "key",
+                      :topic => "topic")
+
+      req_buf = Protocol::RequestBuffer.new
+      expect {
+        m.write(req_buf)
+      }.to_not raise_error
+    end
+  end
+
+  context "frozen string for value" do
+    it "builds the payload without error" do
+      s = "asdffasdf".freeze
+      m = Message.new(:value => s,
+                      :key => "key",
+                      :topic => "topic")
+
+      req_buf = Protocol::RequestBuffer.new
+      expect {
+        m.write(req_buf)
+      }.to_not raise_error
+    end
+  end
+
+  it "decompresses a compressed value"
+
+  it "raises an error if you try to decompress an uncompressed value"
+
+  describe "#write" do
+    it 'writes a MessageWithOffsetStruct to the request buffer' do
+    end
+  end
+end

data/spec/unit/message_to_send_spec.rb

@@ -0,0 +1,10 @@
+require 'spec_helper'
+
+describe MessageToSend do
+  it "provides access to topic,value,key" do
+    mts = MessageToSend.new("hello_topic", "Hello World", "key")
+    expect(mts.topic).to eq("hello_topic")
+    expect(mts.value).to eq("Hello World")
+    expect(mts.key).to eq("key")
+  end
+end

data/spec/unit/messages_for_broker_spec.rb

@@ -0,0 +1,54 @@
+require 'spec_helper'
+
+describe MessagesForBroker do
+  context "twos message one to broker 0, partition 0, another to partition 1" do
+    before(:each) do
+      @messages = [ Message.new(:topic => "topic1",:value => "hi0"),
+                    Message.new(:topic => "topic1",:value => "hi1")]
+
+      @compression_config = stub('compression_config',
+                                 :compression_codec_for_topic => nil)
+
+      @mfb = MessagesForBroker.new(0)
+      @mfb.add(@messages[0], 0)
+      @mfb.add(@messages[1], 1)
+    end
+
+    it "provides the messages" do
+      expect(@mfb.messages.to_set).to eq(@messages.to_set)
+    end
+
+    it "is has a broker_id of 0" do
+      expect(@mfb.broker_id).to eq(0)
+    end
+
+    it "builds the protocol object correctly" do
+      protocol_object = @mfb.build_protocol_objects(@compression_config)
+
+      messages_for_topics = [
+        MessagesForTopic.new("topic1",
+          [
+            MessagesForPartition.new(0, MessageSet.new([@messages[0]])),
+            MessagesForPartition.new(1, MessageSet.new([@messages[1]])),
+          ])
+      ]
+      expect(protocol_object).to eq(messages_for_topics)
+    end
+
+    context "and topic is compressed" do
+      it "builds the protocol object correctly" do
+        @compression_config.stub!(:compression_codec_for_topic => Compression::GzipCodec)
+        protocol_object = @mfb.build_protocol_objects(@compression_config)
+
+        messages_for_topics = [
+          MessagesForTopic.new("topic1",
+            [
+              MessagesForPartition.new(0, MessageSet.new([@messages[0]]).compress(Compression::GzipCodec)),
+              MessagesForPartition.new(1, MessageSet.new([@messages[1]]).compress(Compression::GzipCodec)),
+            ])
+        ]
+        expect(protocol_object).to eq(messages_for_topics)
+      end
+    end
+  end
+end

data/spec/unit/messages_to_send_batch_spec.rb

@@ -0,0 +1,25 @@
+require 'spec_helper'
+
+describe MessagesToSendBatch do
+  context "messages sent to two different brokers" do
+    before(:each) do
+      message_conductor = stub('message_conductor')
+      message_conductor.stub!(:destination).and_return([0,0],[1,1])
+
+      @messages = [
+        Message.new(:topic => "topic1", :value => "hi"),
+        Message.new(:topic => "topic1", :value => "hi")
+      ]
+      @batch = MessagesToSendBatch.new(@messages, message_conductor)
+    end
+
+    it "returns a couple messages brokers" do
+      expect(@batch.messages_for_brokers.size).to eq(2)
+    end
+
+    it "has all messages in the returned message brokers" do
+      messages = @batch.messages_for_brokers.map(&:messages).flatten
+      expect(messages.to_set).to eq(@messages.to_set) 
+    end
+  end
+end

data/spec/unit/messages_to_send_spec.rb

@@ -0,0 +1,63 @@
+require 'spec_helper'
+
+describe MessagesToSend do
+  before(:each) do
+    @messages = []
+    @messages << Message.new(:topic => "test1", :value => "hi")
+    @messages << Message.new(:topic => "test2", :value => "hi")
+    @messages << Message.new(:topic => "test2", :value => "hi")
+
+
+    @cluster_metadata = stub('cluster_metdata').as_null_object
+    @mts = MessagesToSend.new(@messages, @cluster_metadata)
+  end
+
+  describe "needing metadata" do
+    it "returns set of topics" do
+      expect(@mts.topic_set).to eq(Set.new(["test1","test2"]))
+    end
+
+    it "asks ClusterMetadata about having metadata" do
+      @cluster_metadata.stub!(:have_metadata_for_topics?).and_return(true)
+
+      expect(@mts.needs_metadata?).to eq(false)
+    end
+  end 
+
+  describe "sending" do
+    before(:each) do
+      @mfb = stub('mfb', :messages => @messages)
+      @messages_for_brokers = [@mfb]
+
+      @mtsb = stub('messages_to_send_batch').as_null_object
+      @mtsb.stub!(:messages_for_brokers).and_return(@messages_for_brokers)
+
+      MessagesToSendBatch.stub!(:new).and_return(@mtsb)
+    end
+
+    context "is successful" do
+      before(:each) do
+        @mts.messages_for_brokers(nil).each do |mfb|
+          @mts.successfully_sent(mfb)
+        end
+      end
+
+      it "successfully sends all" do
+        expect(@mts.all_sent?).to eq(true)
+      end
+    end
+
+    context "is not successful" do
+      before(:each) do
+        @mts.messages_for_brokers(nil).each do |mfb|
+        end
+      end
+
+      it "does not send all" do
+        @mts.messages_for_brokers(nil).each do |mfb|
+        end
+        expect(@mts.all_sent?).to eq(false)
+      end
+    end
+  end
+end

data/spec/unit/partition_consumer_spec.rb

@@ -0,0 +1,124 @@
+require 'spec_helper'
+
+describe PartitionConsumer do
+  before(:each) do
+    @connection = stub('connection')
+    Connection.stub!(:new).and_return(@connection)
+
+    offset = Protocol::Offset.new(100)
+    partition_offsets = [Protocol::PartitionOffset.new(0, 0, [offset])]
+    @offset_response = [Protocol::TopicOffsetResponse.new("test_topic", partition_offsets)]
+    @connection.stub(:offset).and_return(@offset_response)
+  end
+
+  describe "creation" do
+    context "when passed unknown options" do
+      it "raises an ArgumentError" do
+        expect { PartitionConsumer.new("test_client", "localhost", 9092, "test_topic", 0,-2, :unknown => true) }.to raise_error(ArgumentError)
+      end
+    end
+
+    context "when passed an unknown offset" do
+      it "raises an ArgumentError" do
+        expect { PartitionConsumer.new("test_client", "localhost", 9092, "test_topic", 0,:coolest_offset) }.to raise_error(ArgumentError)
+      end
+    end
+  end
+
+  describe "next offset" do
+    context "when offset is not set" do
+      it "resolves offset if it's not set" do
+        @connection.should_receive(:offset).and_return(@offset_response)
+        pc = PartitionConsumer.new("test_client", "localhost", 9092, "test_topic",
+                                   0, -2)
+
+        pc.next_offset
+      end
+
+      it "returns resolved offset" do
+        pc = PartitionConsumer.new("test_client", "localhost", 9092, "test_topic",
+                                   0, -2)
+        expect(pc.next_offset).to eq(100)
+      end
+    end
+
+    context "when offset is set" do
+      it "does not resolve it" do
+        pc = PartitionConsumer.new("test_client", "localhost", 9092, "test_topic",
+                                   0, 200)
+        pc.next_offset
+      end
+    end
+
+    context "when call returns an error" do
+      it "is raised" do
+        @offset_response.first.partition_offsets.first.stub!(:error).and_return(2)
+        pc = PartitionConsumer.new("test_client", "localhost", 9092, "test_topic",
+                                   0, -2)
+
+        expect { pc.next_offset }.to raise_error(Errors::InvalidMessage)
+      end
+    end
+
+    context "when no offset exists" do
+      it "sets offset to 0" do
+        pc = PartitionConsumer.new("test_client", "localhost", 9092, "test_topic",
+                                   0, -2)
+
+        @offset_response.first.partition_offsets.first.stub!(:offsets).and_return([])
+        expect(pc.next_offset).to eq(0)
+      end
+    end
+  end
+
+  describe "fetching messages" do
+    before(:each) do
+      message_set = MessageSet.new
+      message_set << Message.new(:value => "value", :key => "key", :offset => 90)
+      partition_fetch_response = Protocol::PartitionFetchResponse.new(0, 0, 100, message_set)
+      topic_fetch_response = Protocol::TopicFetchResponse.new('test_topic',
+                                                    [partition_fetch_response])
+      @response = Protocol::FetchResponse.new(stub('common'), [topic_fetch_response])
+
+      @connection.stub(:fetch).and_return(@response)
+      @pc = PartitionConsumer.new("test_client", "localhost", 9092, "test_topic", 0, -2)
+    end
+
+    it "returns FetchedMessage objects" do
+      expect(@pc.fetch.first.class).to eq(FetchedMessage)
+    end
+
+    it "uses object defaults" do
+      @connection.should_receive(:fetch).with(10_000, 0, anything)
+      @pc.fetch
+    end
+
+    context "when options are passed" do
+      it "overrides object defaults" do
+        @connection.should_receive(:fetch).with(20_000, 0, anything)
+        @pc = PartitionConsumer.new("test_client", "localhost", 9092, "test_topic", 0, -2, :max_wait_ms => 20_000)
+
+        @pc.fetch
+      end
+    end
+
+    context "when call returns an error" do
+      it "is raised" do
+        pfr = @response.topic_fetch_responses.first.partition_fetch_responses.first
+        pfr.stub!(:error).and_return(2)
+
+        expect { @pc.fetch }.to raise_error(Errors::InvalidMessage)
+      end
+    end
+
+    it "sets the highwater mark" do
+      @pc.fetch
+      expect(@pc.highwater_mark).to eq(100)
+    end
+
+    it "sets the latest offset" do
+      @pc.fetch
+      expect(@pc.next_offset).to eq(91)
+    end
+  end
+end

data/spec/unit/producer_compression_config_spec.rb

@@ -0,0 +1,35 @@
+require 'spec_helper'
+
+describe ProducerCompressionConfig do
+  describe "creation" do
+    it "raises ArgumentError when codec is unknown" do
+      expect { ProducerCompressionConfig.new(:ripple, nil) }.to raise_error(ArgumentError)
+    end
+  end
+
+  context "no codec set" do
+    it "compresses no topics" do
+      pcc = ProducerCompressionConfig.new(nil,nil)
+      expect(pcc.compression_codec_for_topic("test")).to eq(false)
+    end
+  end
+
+  describe "compression codec no topics specified" do
+    it "compresses any topic" do
+      pcc = ProducerCompressionConfig.new(:gzip,nil)
+      expect(pcc.compression_codec_for_topic("test")).to eq(Compression::GzipCodec)
+    end
+  end
+
+  describe "compression codec set, but only compress 'compressed' topic" do
+    it "compresses 'compressed' topic" do
+      pcc = ProducerCompressionConfig.new(:gzip, ["compressed"])
+      expect(pcc.compression_codec_for_topic("compressed")).to eq(Compression::GzipCodec)
+    end
+
+    it "does not compresses 'test' topic" do
+      pcc = ProducerCompressionConfig.new(:gzip, ["compressed"])
+      expect(pcc.compression_codec_for_topic("test")).to eq(false)
+    end
+  end
+end

data/spec/unit/producer_spec.rb

@@ -0,0 +1,45 @@
+require 'spec_helper'
+
+describe Producer do
+  it "requires brokers and client_id" do
+    expect { Producer.new }.to raise_error
+  end
+
+  it "raises ArgumentError on unknown arguments" do
+    expect { Producer.new([],"client_id", :unknown => true) }.to raise_error(ArgumentError)
+  end
+
+  it "raises ArgumentError unless brokers is an enumerable" do
+    expect { Producer.new("host:port","client_id") }.to raise_error(ArgumentError)
+  end
+
+  it "raises ProducerShutdown if we try to send to a shutdown producer" do
+    p = Producer.new(["host:port"],"client_id")
+    p.shutdown
+    expect { p.send_messages([]) }.to raise_error(Errors::ProducerShutdownError)
+  end
+
+  it "accepts all options" do
+    expect { Producer.new([],"client_id", Producer::OPTION_DEFAULTS.dup) }.not_to raise_error
+  end
+
+  describe "sending messages" do
+    before(:each) do
+      @sync_producer = double('sync_producer').as_null_object
+      SyncProducer.stub!(:new).and_return(@sync_producer)
+
+      @producer = Producer.new([], "client_id", :type => :sync)
+    end
+
+    it "turns MessagesToSend into Message objects" do
+      @sync_producer.should_receive(:send_messages).with([an_instance_of(Message)])
+
+      m = MessageToSend.new("topic", "value")
+      @producer.send_messages([m])
+    end
+
+    it "raises an ArgumentError if you try to send a single message" do
+      expect { @producer.send_messages(MessageToSend.new("topic", "value")) }.to raise_error(ArgumentError)
+    end
+  end
+end