leffen-kafka-rb 0.0.15

data/spec/encoder_spec.rb

@@ -0,0 +1,251 @@
+# encoding: utf-8
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe Encoder do
+  def check_message(bytes, message)
+    encoded = [message.magic].pack("C") + [message.calculate_checksum].pack("N") + message.payload
+    encoded = [encoded.length].pack("N") + encoded
+    bytes.should == encoded
+  end
+
+  describe "Message Encoding" do
+    it "should encode a message" do
+      message = Kafka::Message.new("alejandro")
+      check_message(described_class.message(message), message)
+    end
+
+    it "should encode an empty message" do
+      message = Kafka::Message.new
+      check_message(described_class.message(message), message)
+    end
+
+    it "should encode strings containing non-ASCII characters" do
+      message = Kafka::Message.new("ümlaut")
+      encoded = described_class.message(message)
+      message = Kafka::Message.parse_from(encoded).messages.first
+      if RUBY_VERSION[0,3] == "1.8" # Use old iconv on Ruby 1.8 for encoding
+        #ic = Iconv.new('UTF-8//IGNORE', 'UTF-8')
+        #ic.iconv(message.payload).should eql("ümlaut")
+        message.payload.should eql("ümlaut")
+      else
+        message.payload.force_encoding(Encoding::UTF_8).should eql("ümlaut")
+      end
+    end
+
+    it "should encode strings containing non-ASCII characters" do
+      message = Kafka::Message.new("\214")
+      encoded = described_class.message(message)
+      message = Kafka::Message.parse_from(encoded).messages.first
+      if RUBY_VERSION[0,3] == "1.8"
+        message.payload.should eql("\214")
+      else
+        message.payload.force_encoding(Encoding::UTF_8).should eql("\214")
+      end
+    end
+  end
+
+  describe :compression do
+    before do
+      @message = Kafka::Message.new "foo"
+    end
+
+    it "should default to no compression" do
+      msg = "foo"
+      checksum = Zlib.crc32 msg
+      magic = 0
+      msg_size = 5 + msg.size
+      raw = [msg_size, magic, checksum, msg].pack "NCNa#{msg.size}"
+
+      Encoder.message(@message).should == raw
+    end
+
+    it "should support GZip compression" do
+      buffer = StringIO.new
+      gz = Zlib::GzipWriter.new buffer, nil, nil
+      gz.write "foo"
+      gz.close
+      buffer.rewind
+      msg = buffer.string
+      checksum = Zlib.crc32 msg
+      magic = 1
+      attrs = 1
+      msg_size = 6 + msg.size
+      raw = [msg_size, magic, attrs, checksum, msg].pack "NCCNa#{msg.size}"
+      Encoder.message(@message, 1).should == raw
+    end
+
+    if Object.const_defined? "Snappy"
+      it "should support Snappy compression" do
+        buffer = StringIO.new
+        Snappy::Writer.new buffer do |w|
+          w << "foo"
+        end
+        buffer.rewind
+        msg = buffer.string
+        checksum = Zlib.crc32 msg
+        magic = 1
+        attrs = 2
+        msg_size = 6 + msg.size
+        raw = [msg_size, magic, attrs, checksum, msg].pack "NCCNa#{msg.size}"
+
+        Encoder.message(@message, 2).should == raw
+      end
+    end
+  end
+
+  describe "produce" do
+    it "should binary encode an empty request" do
+      bytes = described_class.produce("test", 0, [])
+      bytes.length.should eql(20)
+      bytes.should eql("\000\000\000\020\000\000\000\004test\000\000\000\000\000\000\000\000")
+    end
+
+    it "should binary encode a request with a message, using a specific wire format" do
+      message = Kafka::Message.new("ale")
+      bytes = described_class.produce("test", 3, message)
+      data_size  = bytes[0, 4].unpack("N").shift
+      request_id = bytes[4, 2].unpack("n").shift
+      topic_length = bytes[6, 2].unpack("n").shift
+      topic = bytes[8, 4]
+      partition = bytes[12, 4].unpack("N").shift
+      messages_length = bytes[16, 4].unpack("N").shift
+      messages = bytes[20, messages_length]
+
+      bytes.length.should eql(32)
+      data_size.should eql(28)
+      request_id.should eql(0)
+      topic_length.should eql(4)
+      topic.should eql("test")
+      partition.should eql(3)
+      messages_length.should eql(12)
+    end
+  end
+
+  describe "message_set" do
+    it "should compress messages into a message set" do
+      message_one = Kafka::Message.new "foo"
+      message_two = Kafka::Message.new "bar"
+      bytes = described_class.message_set [message_one, message_two], Kafka::Message::GZIP_COMPRESSION
+
+      messages = Kafka::Message.parse_from bytes
+      messages.should be_a Kafka::Message::MessageSet
+      messages.messages.size.should == 2
+
+      messages.messages[0].should be_a Kafka::Message
+      messages.messages[0].payload.should == "foo"
+      messages.messages[1].should be_a Kafka::Message
+      messages.messages[1].payload.should == "bar"
+    end
+  end
+
+  describe "multiproduce" do
+    it "encodes an empty request" do
+      bytes = described_class.multiproduce([])
+      bytes.length.should == 8
+      bytes.should == "\x00\x00\x00\x04\x00\x03\x00\x00"
+    end
+
+    it "encodes a request with a single topic/partition" do
+      message = Kafka::Message.new("ale")
+      bytes = described_class.multiproduce(Kafka::ProducerRequest.new("test", message))
+
+      req_length = bytes[0, 4].unpack("N").shift
+      req_type = bytes[4, 2].unpack("n").shift
+      tp_count = bytes[6, 2].unpack("n").shift
+
+      req_type.should == Kafka::RequestType::MULTIPRODUCE
+      tp_count.should == 1
+
+      topic_length = bytes[8, 2].unpack("n").shift
+      topic = bytes[10, 4]
+      partition = bytes[14, 4].unpack("N").shift
+      messages_length = bytes[18, 4].unpack("N").shift
+      messages_data = bytes[22, messages_length]
+
+      topic_length.should == 4
+      topic.should == "test"
+      partition.should == 0
+      messages_length.should == 12
+      check_message(messages_data, message)
+    end
+
+    it "encodes a request with a single topic/partition but multiple messages" do
+      messages = [Kafka::Message.new("ale"), Kafka::Message.new("beer")]
+      bytes = described_class.multiproduce(Kafka::ProducerRequest.new("test", messages))
+
+      req_length = bytes[0, 4].unpack("N").shift
+      req_type = bytes[4, 2].unpack("n").shift
+      tp_count = bytes[6, 2].unpack("n").shift
+
+      req_type.should == Kafka::RequestType::MULTIPRODUCE
+      tp_count.should == 1
+
+      topic_length = bytes[8, 2].unpack("n").shift
+      topic = bytes[10, 4]
+      partition = bytes[14, 4].unpack("N").shift
+      messages_length = bytes[18, 4].unpack("N").shift
+      messages_data = bytes[22, messages_length]
+
+      topic_length.should == 4
+      topic.should == "test"
+      partition.should == 0
+      messages_length.should == 25
+      check_message(messages_data[0, 12], messages[0])
+      check_message(messages_data[12, 13], messages[1])
+    end
+
+    it "encodes a request with multiple topic/partitions" do
+      messages = [Kafka::Message.new("ale"), Kafka::Message.new("beer")]
+      bytes = described_class.multiproduce([
+          Kafka::ProducerRequest.new("test", messages[0]),
+          Kafka::ProducerRequest.new("topic", messages[1], :partition => 1),
+        ])
+
+      req_length = bytes[0, 4].unpack("N").shift
+      req_type = bytes[4, 2].unpack("n").shift
+      tp_count = bytes[6, 2].unpack("n").shift
+
+      req_type.should == Kafka::RequestType::MULTIPRODUCE
+      tp_count.should == 2
+
+      topic_length = bytes[8, 2].unpack("n").shift
+      topic = bytes[10, 4]
+      partition = bytes[14, 4].unpack("N").shift
+      messages_length = bytes[18, 4].unpack("N").shift
+      messages_data = bytes[22, 12]
+
+      topic_length.should == 4
+      topic.should == "test"
+      partition.should == 0
+      messages_length.should == 12
+      check_message(messages_data[0, 12], messages[0])
+
+      topic_length = bytes[34, 2].unpack("n").shift
+      topic = bytes[36, 5]
+      partition = bytes[41, 4].unpack("N").shift
+      messages_length = bytes[45, 4].unpack("N").shift
+      messages_data = bytes[49, 13]
+
+      topic_length.should == 5
+      topic.should == "topic"
+      partition.should == 1
+      messages_length.should == 13
+      check_message(messages_data[0, 13], messages[1])
+    end
+  end
+end

data/spec/io_spec.rb

@@ -0,0 +1,88 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+# 
+#    http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+require File.dirname(__FILE__) + '/spec_helper'
+
+class IOTest
+  include Kafka::IO
+end
+
+describe IO do
+
+  before(:each) do
+    @mocked_socket = mock(TCPSocket)
+    TCPSocket.stub!(:new).and_return(@mocked_socket) # don't use a real socket
+    @io = IOTest.new
+    @io.connect("somehost", 9093)
+  end
+
+  describe "default methods" do
+    it "has a socket, a host and a port" do
+      [:socket, :host, :port].each do |m|
+        @io.should respond_to(m.to_sym)
+      end
+    end
+
+    it "raises an exception if no host and port is specified" do
+      lambda {
+        io = IOTest.new
+        io.connect
+      }.should raise_error(ArgumentError)
+    end
+    
+    it "should remember the port and host on connect" do
+      @io.connect("somehost", 9093)
+      @io.host.should eql("somehost")
+      @io.port.should eql(9093)
+    end
+
+    it "should write to a socket" do
+      data = "some data"
+      @mocked_socket.should_receive(:write).with(data).and_return(9)
+      @io.write(data).should eql(9)
+    end
+
+    it "should read from a socket" do
+      length = 200
+      @mocked_socket.should_receive(:read).with(length).and_return("foo")
+      @io.read(length)
+    end
+
+    it "should disconnect on a timeout when reading from a socket (to aviod protocol desync state)" do
+      length = 200
+      @mocked_socket.should_receive(:read).with(length).and_raise(Errno::EAGAIN)
+      @io.should_receive(:disconnect)
+      lambda { @io.read(length) }.should raise_error(Kafka::SocketError)
+    end
+
+    it "should disconnect" do
+      @io.should respond_to(:disconnect)
+      @mocked_socket.should_receive(:close).and_return(nil)
+      @io.disconnect
+    end
+
+    it "should reconnect" do
+      TCPSocket.should_receive(:new)
+      @io.reconnect
+    end
+
+    it "should disconnect on a broken pipe error" do
+      [Errno::ECONNABORTED, Errno::EPIPE, Errno::ECONNRESET].each do |error|
+        @mocked_socket.should_receive(:write).exactly(:once).and_raise(error)
+        @mocked_socket.should_receive(:close).exactly(:once).and_return(nil)
+        lambda { @io.write("some data to send") }.should raise_error(Kafka::SocketError)
+      end
+    end
+  end
+end

data/spec/kafka_spec.rb

@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+# 
+#    http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe Kafka do
+  before(:each) do
+  end
+end

data/spec/message_spec.rb

@@ -0,0 +1,227 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+# 
+#    http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe Message do
+
+  def pack_v1_message bytes, attributes
+    [6 + bytes.length, 1, attributes, Zlib.crc32(bytes), bytes].pack "NCCNa*"
+  end
+
+  before(:each) do
+    @message = Message.new
+  end
+
+  describe "Kafka Message" do
+    it "should have a default magic number" do
+      Message::MAGIC_IDENTIFIER_DEFAULT.should eql(0)
+    end
+
+    it "should have a magic field, a checksum and a payload" do
+      [:magic, :checksum, :payload].each do |field|
+        @message.should respond_to(field.to_sym)
+      end
+    end
+
+    it "should set a default value of zero" do
+      @message.magic.should eql(Kafka::Message::MAGIC_IDENTIFIER_DEFAULT)
+    end
+
+    it "should allow to set a custom magic number" do
+      @message = Message.new("ale", 1)
+      @message.magic.should eql(1)
+    end
+
+    it "should have an empty payload by default" do
+      @message.payload.should == ""
+    end
+
+    it "should calculate the checksum (crc32 of a given message)" do
+      @message.payload = "ale"
+      @message.calculate_checksum.should eql(1120192889)
+      @message.payload = "alejandro"
+      @message.calculate_checksum.should eql(2865078607)
+    end
+
+    it "should say if the message is valid using the crc32 signature" do
+      @message.payload  = "alejandro"
+      @message.checksum = 2865078607
+      @message.valid?.should eql(true)
+      @message.checksum = 0
+      @message.valid?.should eql(false)
+      @message = Message.new("alejandro", 0, 66666666) # 66666666 is a funny checksum
+      @message.valid?.should eql(false)
+    end
+  end
+
+  describe "parsing" do
+    it "should parse a version-0 message from bytes" do
+      bytes = [8, 0, 1120192889, 'ale'].pack('NCNa*')
+      message = Kafka::Message.parse_from(bytes).messages.first
+      message.valid?.should eql(true)
+      message.magic.should eql(0)
+      message.checksum.should eql(1120192889)
+      message.payload.should eql("ale")
+    end
+
+    it "should parse a version-1 message from bytes" do
+      bytes = [12, 1, 0, 755095536, 'martin'].pack('NCCNa*')
+      message = Kafka::Message.parse_from(bytes).messages.first
+      message.should be_valid
+      message.magic.should == 1
+      message.checksum.should == 755095536
+      message.payload.should == 'martin'
+    end
+
+    it "should raise an error if the magic number is not recognised" do
+      bytes = [12, 2, 0, 755095536, 'martin'].pack('NCCNa*') # 2 = some future format that's not yet invented
+      lambda {
+        Kafka::Message.parse_from(bytes)
+      }.should raise_error(RuntimeError, /Unsupported Kafka message version/)
+    end
+
+    it "should skip an incomplete message at the end of the response" do
+      bytes = [8, 0, 1120192889, 'ale'].pack('NCNa*')
+      bytes += [8].pack('N') # incomplete message (only length, rest is truncated)
+      message_set = Message.parse_from(bytes)
+      message_set.messages.size.should == 1
+      message_set.size.should == 12 # bytes consumed
+    end
+
+    it "should skip an incomplete message at the end of the response which has the same length as an empty message" do
+      bytes = [8, 0, 1120192889, 'ale'].pack('NCNa*')
+      bytes += [8, 0, 1120192889].pack('NCN') # incomplete message (payload is missing)
+      message_set = Message.parse_from(bytes)
+      message_set.messages.size.should == 1
+      message_set.size.should == 12 # bytes consumed
+    end
+
+    it "should read empty messages correctly" do
+      # empty message
+      bytes = [5, 0, 0, ''].pack('NCNa*')
+      messages = Message.parse_from(bytes).messages
+      messages.size.should == 1
+      messages.first.payload.should == ''
+    end
+
+    it "should parse a gzip-compressed message" do
+      compressed = 'H4sIAG0LI1AAA2NgYBBkZBB/9XN7YlJRYnJiCogCAH9lueQVAAAA'.unpack('m*').shift
+      bytes = [45, 1, 1, 1303540914, compressed].pack('NCCNa*')
+      message = Message.parse_from(bytes).messages.first
+      message.should be_valid
+      message.payload.should == 'abracadabra'
+    end
+
+    if Object.const_defined? "Snappy"
+      it "should parse a snappy-compressed message" do
+        cleartext = "abracadabra"
+        bytes = pack_v1_message cleartext, 0
+        compressed = Snappy.deflate(bytes)
+        bytes = pack_v1_message compressed, 2
+        message = Message.parse_from(bytes).messages.first
+        message.should be_valid
+        message.payload.should == cleartext
+      end
+
+      it "should recursively parse nested snappy compressed messages" do
+        uncompressed = pack_v1_message('abracadabra', 0)
+        uncompressed << pack_v1_message('foobar', 0)
+        compressed = pack_v1_message(Snappy.deflate(uncompressed), 2)
+        messages = Message.parse_from(compressed).messages
+        messages.map(&:payload).should == ['abracadabra', 'foobar']
+        messages.map(&:valid?).should == [true, true]
+      end
+
+      it "should support a mixture of snappy compressed and uncompressed messages" do
+        bytes = pack_v1_message(Snappy.deflate(pack_v1_message("compressed", 0)), 2)
+        bytes << pack_v1_message('uncompressed', 0)
+        messages = Message.parse_from(bytes).messages
+        messages.map(&:payload).should == ["compressed", "uncompressed"]
+        messages.map(&:valid?).should == [true, true]
+      end
+    end
+
+    it "should recursively parse nested gzip compressed messages" do
+      uncompressed = [17, 1, 0, 401275319, 'abracadabra'].pack('NCCNa*')
+      uncompressed << [12, 1, 0, 2666930069, 'foobar'].pack('NCCNa*')
+      compressed_io = StringIO.new('')
+      Zlib::GzipWriter.new(compressed_io).tap{|gzip| gzip << uncompressed; gzip.close }
+      compressed = compressed_io.string
+      bytes = [compressed.size + 6, 1, 1, Zlib.crc32(compressed), compressed].pack('NCCNa*')
+      messages = Message.parse_from(bytes).messages
+      messages.map(&:payload).should == ['abracadabra', 'foobar']
+      messages.map(&:valid?).should == [true, true]
+    end
+
+    it "should support a mixture of gzip compressed and uncompressed messages" do
+      compressed = 'H4sIAG0LI1AAA2NgYBBkZBB/9XN7YlJRYnJiCogCAH9lueQVAAAA'.unpack('m*').shift
+      bytes = [45, 1, 1, 1303540914, compressed].pack('NCCNa*')
+      bytes << [11, 1, 0, 907060870, 'hello'].pack('NCCNa*')
+      messages = Message.parse_from(bytes).messages
+      messages.map(&:payload).should == ['abracadabra', 'hello']
+      messages.map(&:valid?).should == [true, true]
+    end
+
+    it "should raise an error if the compression codec is not supported" do
+      bytes = [6, 1, 3, 0, ''].pack('NCCNa*') # 3 = some unknown future compression codec
+      lambda {
+        Kafka::Message.parse_from(bytes)
+      }.should raise_error(RuntimeError, /Unsupported Kafka compression codec/)
+    end
+  end
+
+  describe "#ensure_snappy!" do
+    let(:message) { Kafka::Message.new }
+    before { Kafka::Message.instance_variable_set :@snappy, nil }
+
+    subject { message.ensure_snappy! { 42 } }
+
+    if Object.const_defined? "Snappy"
+      context "when snappy is available" do
+        before { Object.stub! :const_defined? => true }
+        it { should == 42 }
+      end
+    end
+
+    context "when snappy is not available" do
+      before { Object.stub! :const_defined? => false }
+
+      it "raises an error" do
+        expect { message.ensure_snappy! { 42 } }.to raise_error
+      end
+    end
+  end
+
+  describe ".ensure_snappy!" do
+    before { Kafka::Message.instance_variable_set :@snappy, nil }
+
+    subject { Kafka::Message.ensure_snappy! { 42 } }
+
+    if Object.const_defined? "Snappy"
+      context "when snappy is available" do
+        before { Object.stub! :const_defined? => true }
+        it { should == 42 }
+      end
+    end
+
+    context "when snappy is not available" do
+      before { Object.stub! :const_defined? => false }
+
+      it "raises an error" do
+        expect { Kafka::Message.ensure_snappy! { 42 } }.to raise_error
+      end
+    end
+  end
+end