codeclimate-poseidon 0.0.8

data/spec/unit/cluster_metadata_spec.rb

@@ -0,0 +1,46 @@
+require 'spec_helper'
+
+include Protocol
+RSpec.describe ClusterMetadata do
+  describe "populated" do
+    before(:each) do
+      partitions = [
+        PartitionMetadata.new(0, 1, 1, [1,2], [1,2]),
+        PartitionMetadata.new(0, 2, 2, [2,1], [2,1])
+      ]
+      topics = [TopicMetadata.new(TopicMetadataStruct.new(0, "test", partitions))]
+
+      brokers = [Broker.new(1, "host1", 1), Broker.new(2, "host2", 2)]
+
+      @mr = MetadataResponse.new(nil, brokers, topics)
+
+      @cm = ClusterMetadata.new
+      @cm.update(@mr)
+    end
+
+    it "knows when it has metadata for a set of topics" do
+      have_metadata = @cm.have_metadata_for_topics?(Set.new(["test"]))
+      expect(have_metadata).to eq(true)
+    end
+
+    it "knows when it doesn't have metadata for a topic" do
+      have_metadata = @cm.have_metadata_for_topics?(Set.new(["test", "no_data"]))
+      expect(have_metadata).to eq(false)
+    end
+
+    it "provides topic metadata for a set of topics" do
+      topic_metadata = @cm.metadata_for_topics(Set.new(["test"]))
+      expect(topic_metadata).to eq({ "test" => @mr.topics.first })
+    end
+
+    it "provides broker information" do
+      broker = @cm.broker(1)
+      expect(broker).to eq(@mr.brokers.first)
+    end
+
+    it "provides the lead broker for a partition" do
+      expect(@cm.lead_broker_for_partition("test",1).id).to eq(1)
+      expect(@cm.lead_broker_for_partition("test",2).id).to eq(2)
+    end
+  end
+end

data/spec/unit/compression/gzip_codec_spec.rb

@@ -0,0 +1,34 @@
+# encoding: utf-8
+require 'spec_helper'
+
+RSpec.describe Poseidon::Compression::GzipCodec do
+
+  let :data do
+    %({"a":"val1"}\n{"a":"val2"}\n{"a":"val3"})
+  end
+
+  it "should have an ID" do
+    expect(described_class.codec_id).to eq(1)
+  end
+
+  it "should compress" do
+    compressed = described_class.compress(data)
+    expect(compressed.size).to eq(41)
+    expect(compressed.encoding).to eq(Encoding::BINARY)
+  end
+
+  it "should decompress" do
+    original = described_class.decompress(described_class.compress(data))
+    expect(original).to eq(data)
+    expect(original.encoding).to eq(Encoding::UTF_8)
+  end
+
+  it "should decompress unicode messages" do
+    str = "\x1F\x8B\b\x00\x00\x00\x00\x00\x00\x00c`\x80\x03\xE3I\x91\xD3|\x19\x18\xFE\x03\x01\x90\xA7Z\xAD\x94\xA8d\xA5\x14Z\x92XP\xEC\xE9\xE3\xE1\xEB\x12Y\xEE\xE8\x98\x16\xA4\xA4\xA3\x94\x04\x14~6}\xE9\xB39k\x94j\xA1Z\x19A\xDAm\f\xD9\xEF\x10\xD0\x1E\x8C\xA6\x1D\x00\x96\x98\x1E\xB9~\x00\x00\x00".force_encoding(Encoding::BINARY)
+    buf = Protocol::ResponseBuffer.new(described_class.decompress(str))
+    msg = MessageSet.read_without_size(buf).flatten
+    expect(msg.size).to eq(2)
+    expect(msg[0].value).to eq(%({"a":"UtapsILHMDYwAAfR","b":"日本"}))
+  end
+
+end

data/spec/unit/compression/snappy_codec_spec.rb

@@ -0,0 +1,49 @@
+# encoding: utf-8
+require 'spec_helper'
+
+RSpec.describe Poseidon::Compression::SnappyCodec do
+
+  let :data do
+    %({"a":"val1"}\n{"a":"val2"}\n{"a":"val3"})
+  end
+
+  it "should have an ID" do
+    expect(described_class.codec_id).to eq(2)
+  end
+
+  it "should compress" do
+    compressed = described_class.compress(data)
+    expect(compressed.size).to eq(34)
+    expect(compressed.encoding).to eq(Encoding::BINARY)
+  end
+
+  it "should decompress" do
+    original = described_class.decompress(described_class.compress(data))
+    expect(original).to eq(data)
+  end
+
+  it "should decompress streams" do
+    str = "\x82SNAPPY\x00\x00\x00\x00\x01\x00\x00\x00\x01\x00\x00\x00\x1E#\x00\x00\x19\x01\\\x17\x8B\xA7x\xB9\x00\x00\xFF\xFF\xFF\xFF\x00\x00\x00\tPLAINDATA".force_encoding(Encoding::BINARY)
+    buf = Protocol::ResponseBuffer.new(described_class.decompress(str))
+    msg = MessageSet.read_without_size(buf).flatten
+    expect(msg.size).to eq(1)
+    expect(msg[0].value).to eq("PLAINDATA")
+  end
+
+  it "should decompress bulk streams" do
+    str = "\x82SNAPPY\x00\x00\x00\x00\x01\x00\x00\x00\x01\x00\x00\x00\xCA\xE8\x04\x00\x00\x19\x01\xA0L`\x9E\xD4(\x00\x00\xFF\xFF\xFF\xFF\x00\x00\x00>{\"a\":\"UtaaKYLHMCwiAA-l\",\"bF\x17\x00Dm\",\"c\":1389795881}\rW \x01\x00\x00\x00Ln\x14\x98\xA8zX\x00\x00nVX\x00\x00o^X\x00\x00\x02\x01X\b3\xF1\e~\xB0\x00\x00pVX\x00\x00q^X\x00\x00\x03\x01X\b.\xE5\x82~X\x00\x00tVX\x00\x00u^X\x00\x00\x04\x01X\b o\xCE~\b\x01\x00vVX\x00\x00w^X\x00\x00\x05\x01X\f\t\xD8)(z`\x01\x00xVX\x00\x00y^X\x00\x00\x06\x01X\f@\bf\xA6zX\x00\x00zVX\x00\x000BX\x00".force_encoding(Encoding::BINARY)
+    buf = Protocol::ResponseBuffer.new(described_class.decompress(str))
+    msg = MessageSet.read_without_size(buf).flatten
+    expect(msg.size).to eq(7)
+    expect(msg[0].value).to eq(%({"a":"UtaaKYLHMCwiAA-l","b":"UtaaKYLHMCwiAA-m","c":1389795881}))
+  end
+
+  it "should decompress unicode messages" do
+    str = "\x82SNAPPY\x00\x00\x00\x00\x01\x00\x00\x00\x01\x00\x00\x00:?\x00\x00\x19\x01\xCC3\xBA?\x91\xFA\x00\x00\xFF\xFF\xFF\xFF\x00\x00\x00%{\"a\":\"UtaitILHMDAAAAfU\",\"b\":\"\xE6\x97\xA5\xE6\x9C\xAC\"}".force_encoding(Encoding::BINARY)
+    buf = Protocol::ResponseBuffer.new(described_class.decompress(str))
+    msg = MessageSet.read_without_size(buf).flatten
+    expect(msg.size).to eq(1)
+    expect(msg[0].value).to eq(%({"a":"UtaitILHMDAAAAfU","b":"日本"}))
+  end
+
+end

data/spec/unit/compression_spec.rb

@@ -0,0 +1,17 @@
+require 'spec_helper'
+
+RSpec.describe Compression do
+  it 'returns GzipCompessor for codec_id of 1' do
+    codec = Compression.find_codec(1)
+    expect(codec).to eq(Compression::GzipCodec)
+  end
+
+  it 'returns SnappyCompessor for codec_id of 2' do
+    codec = Compression.find_codec(2)
+    expect(codec).to eq(Compression::SnappyCodec)
+  end
+
+  it 'raises UnrecognizedCompressionCodec for codec_id of 3' do
+    expect { Compression.find_codec(3) }.to raise_error(Compression::UnrecognizedCompressionCodec)
+  end
+end

data/spec/unit/connection_spec.rb

@@ -0,0 +1,4 @@
+require 'spec_helper'
+
+RSpec.describe Connection do
+end

data/spec/unit/fetched_message_spec.rb

@@ -0,0 +1,11 @@
+require 'spec_helper'
+
+RSpec.describe FetchedMessage do
+  it "provides access to topic,value,key,offset" do
+    mts = FetchedMessage.new("hello_topic", "Hello World", "key", 0)
+    expect(mts.topic).to eq("hello_topic")
+    expect(mts.value).to eq("Hello World")
+    expect(mts.key).to eq("key")
+    expect(mts.offset).to eq(0)
+  end
+end

data/spec/unit/message_conductor_spec.rb

@@ -0,0 +1,164 @@
+require 'spec_helper'
+
+include Protocol
+RSpec.describe MessageConductor do
+  context "two available partitions" do
+    before(:each) do
+      partitions = [
+        # These are intentionally not ordered by partition_id.
+        # [:error, :id, :leader, :replicas, :isr]
+        PartitionMetadata.new(0, 1, 2, [2,1], [2,1]),
+        PartitionMetadata.new(0, 0, 1, [1,2], [1,2])
+      ]
+      topics = [TopicMetadata.new(TopicMetadataStruct.new(0, "test", partitions))]
+      brokers = [Broker.new(1, "host1", 1), Broker.new(2, "host2", 2)]
+
+      @mr = MetadataResponse.new(0, brokers, topics)
+
+      @cm = ClusterMetadata.new
+      @cm.update(@mr)
+    end
+
+    context "no custom partitioner" do
+      before(:each) do
+        @mc = MessageConductor.new(@cm, nil)
+      end
+
+      context "for unkeyed messages" do
+        it "round robins which partition the message should go to" do
+          destinations = 4.times.map do
+            @mc.destination("test").first
+          end
+
+          first = [destinations[0], destinations[2]]
+          second = [destinations[1], destinations[3]]
+          expect([first.uniq, second.uniq].sort).to eq([[0],[1]])
+        end
+
+        context "unknown topic" do
+          it "returns -1 for broker and partition" do
+            expect(@mc.destination("no_exist")).to eq([-1,-1])
+          end
+        end
+      end
+
+      context "keyed message" do
+        it "sends the same keys to the same destinations" do
+          keys = 1000.times.map { rand(500).to_s }
+          key_destinations = {}
+
+          keys.sort_by { rand }.each do |k|
+            partition,broker = @mc.destination("test", k)
+
+            key_destinations[k] ||= []
+            key_destinations[k].push([partition,broker])
+          end
+
+          expect(key_destinations.values.all? { |destinations| destinations.uniq.size == 1 }).to eq(true)
+        end
+      end
+    end
+
+    context "custom partitioner" do
+      before(:each) do
+        partitioner = Proc.new { |key, count| key.split("_").first.to_i % count }
+        @mc = MessageConductor.new(@cm, partitioner)
+      end
+
+      it "obeys custom partitioner" do
+        expect(@mc.destination("test", "2_hello").first).to eq(0)
+        expect(@mc.destination("test", "3_hello").first).to eq(1)
+      end
+    end
+
+    context "partitioner always sends to partition 1" do
+      before(:each) do
+        partitioner = Proc.new { 1 }
+        @mc = MessageConductor.new(@cm, partitioner)
+      end
+
+      it "sends to partition 1 on broker 2" do
+        expect(@mc.destination("test", "2_hello")).to eq([1,2])
+      end
+    end
+
+    context "broken partitioner" do
+      before(:each) do
+        partitioner = Proc.new { |key, count| count + 1 }
+        @mc = MessageConductor.new(@cm, partitioner)
+      end
+
+      it "raises InvalidPartitionError" do
+        expect{@mc.destination("test", "2_hello").first}.to raise_error(Errors::InvalidPartitionError)
+      end
+    end
+  end
+
+  context "two partitions, one is unavailable" do
+    before(:each) do
+      partitions = [
+        Protocol::PartitionMetadata.new(0, 0, 1, [1,2], [1,2]),
+        Protocol::PartitionMetadata.new(0, 1, -1, [2,1], [2,1])
+      ]
+      topics = [TopicMetadata.new(TopicMetadataStruct.new(0, "test", partitions))]
+      brokers = [Broker.new(1, "host1", 1), Broker.new(2, "host2", 2)]
+
+      @mr = MetadataResponse.new(0, brokers, topics)
+
+      @cm = ClusterMetadata.new
+      @cm.update(@mr)
+
+      @mc = MessageConductor.new(@cm, nil)
+    end
+
+    context "keyless message" do
+      it "is never sent to an unavailable partition" do
+        10.times do |destination|
+          expect(@mc.destination("test").first).to eq(0)
+        end
+      end
+    end
+
+    context "keyed message" do
+      it "is sent to unavailable partition" do
+        destinations = Set.new
+        100.times do |key|
+          destinations << @mc.destination("test",key.to_s).first
+        end
+        expect(destinations).to eq(Set.new([0,1]))
+      end
+    end
+  end
+
+  context "no available partitions" do
+    before(:each) do
+      partitions = [
+        Protocol::PartitionMetadata.new(0, 0, -1, [1,2], [1,2]),
+        Protocol::PartitionMetadata.new(0, 1, -1, [2,1], [2,1])
+      ]
+      topics = [TopicMetadata.new(TopicMetadataStruct.new(0, "test", partitions))]
+      brokers = [Broker.new(1, "host1", 1), Broker.new(2, "host2", 2)]
+
+      @mr = MetadataResponse.new(0, brokers, topics)
+
+      @cm = ClusterMetadata.new
+      @cm.update(@mr)
+
+      @mc = MessageConductor.new(@cm, nil)
+    end
+
+    context "keyless message" do
+      it "return -1 for broker and partition" do
+        expect(@mc.destination("test")).to eq([-1,-1])
+      end
+    end
+
+    context "keyed message" do
+      it "returns a valid partition and -1 for broker" do
+        partition_id, broker_id = @mc.destination("test", "key")
+        expect(partition_id).to_not eq(-1)
+        expect(broker_id).to eq(-1)
+      end
+    end
+  end
+end

data/spec/unit/message_set_spec.rb

@@ -0,0 +1,42 @@
+require 'spec_helper'
+
+RSpec.describe MessageSet do
+  describe "converting to a compressed message" do
+    before(:each) do
+      ms = MessageSet.new([Message.new(:value => "I will be compressed", :topic => "test")])
+
+      @compressed_message_set = ms.compress(Compression::GzipCodec)
+    end
+
+    it "contains a compressed message" do
+      expect(@compressed_message_set.messages.first.compressed?).to eq(true)
+    end
+
+    it "can be decompressed and reconstituted" do
+      expect(@compressed_message_set.flatten.first.value).to eq("I will be compressed")
+    end
+  end
+
+  describe "adding messages" do
+    it "adds the message to the struct" do
+      m = Message.new(:value => "sup", :topic => "topic")
+      ms = MessageSet.new
+      ms << m
+      expect(ms.struct.messages.first).to eq(m)
+    end
+  end
+
+  describe "encoding" do
+    it "round trips" do
+      m = Message.new(:value => "sup", :key => "keyz", :topic => "hello")
+      ms = MessageSet.new
+      ms << m
+
+      request_buffer = Protocol::RequestBuffer.new
+      ms.write(request_buffer)
+
+      response_buffer = Protocol::ResponseBuffer.new(request_buffer.to_s)
+      expect(MessageSet.read(response_buffer)).to eq(ms)
+    end
+  end
+end

data/spec/unit/message_spec.rb

@@ -0,0 +1,129 @@
+# encoding: UTF-8
+require 'spec_helper'
+
+RSpec.describe Message do
+  describe "when constructing a new message" do
+    it 'raises an ArgumentError on unknown options' do
+      expect { Message.new(:cow => "dog") }.to raise_error(ArgumentError)
+    end
+
+    it 'handles options correctly' do
+      m = Message.new(:value => "value",
+                      :key => "key",
+                      :attributes => 1,
+                      :topic => "topic")
+
+      expect(m.value).to eq("value")
+      expect(m.key).to eq("key")
+      expect(m.compressed?).to eq(true)
+      expect(m.topic).to eq("topic")
+    end
+  end
+
+  describe "checksum" do
+    context "is incorrect" do
+      before(:each) do
+        m = Message.new(:value => "value",
+                        :key => "key",
+                        :topic => "topic")
+
+        req_buf = Protocol::RequestBuffer.new
+        m.write(req_buf)
+
+        @s = req_buf.to_s
+        @s[-1] = "q" # break checksum
+      end
+
+      it "knows it" do
+        expect { Message.read(Protocol::ResponseBuffer.new(@s)) }.to raise_error(Errors::ChecksumError)
+      end
+    end
+
+    context 'is correct' do
+      before(:each) do
+        m = Message.new(:value => "value",
+                        :key => "key",
+                        :topic => "topic")
+
+        req_buf = Protocol::RequestBuffer.new
+        m.write(req_buf)
+
+        @s = req_buf.to_s
+      end
+
+      it "raises no error" do
+        expect { Message.read(Protocol::ResponseBuffer.new(@s)) }.to_not raise_error
+      end
+    end
+  end
+
+  describe "truncated message" do
+    before(:each) do
+      m = Message.new(:value => "value",
+                      :key => "key",
+                      :topic => "topic")
+
+      req_buf = Protocol::RequestBuffer.new
+      m.write(req_buf)
+
+      @s = req_buf.to_s
+    end
+
+    it "reading returns nil" do
+      expect(Message.read(Protocol::ResponseBuffer.new(@s[0..-4]))).to eq(nil)
+    end
+  end
+
+  context "invalid utf8 string for value" do
+    it "builds the payload without error" do
+      s = "asdf\xffasdf"
+      m = Message.new(:value => s,
+                      :key => "key",
+                      :topic => "topic")
+
+      req_buf = Protocol::RequestBuffer.new
+      expect {
+        m.write(req_buf)
+      }.to_not raise_error
+    end
+  end
+
+  context "utf8 string with multibyte characters" do
+    it "roundtrips correctly" do
+      s = "the µ is two bytes"
+      m = Message.new(:value => s,
+                      :key => "key",
+                      :topic => "topic")
+
+      req_buf = Protocol::RequestBuffer.new
+      m.write(req_buf)
+
+      resp_buf = Protocol::ResponseBuffer.new(req_buf.to_s)
+
+      expect(Message.read(resp_buf).value).to eq(s.force_encoding(Encoding::BINARY))
+    end
+  end
+
+  context "frozen string for value" do
+    it "builds the payload without error" do
+      s = "asdffasdf".freeze
+      m = Message.new(:value => s,
+                      :key => "key",
+                      :topic => "topic")
+
+      req_buf = Protocol::RequestBuffer.new
+      expect {
+        m.write(req_buf)
+      }.to_not raise_error
+    end
+  end
+
+  it "decompresses a compressed value"
+
+  it "raises an error if you try to decompress an uncompressed value"
+
+  describe "#write" do
+    it 'writes a MessageWithOffsetStruct to the request buffer' do
+    end
+  end
+end