poseidon 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3d5bb4f86848c948a0395678c366213e43a69129
-  data.tar.gz: f9ef623ef716cfe2197c55cb50031c41eb45f785
+  metadata.gz: 4c4852d5b615cd1fe50528b0136602756087d0db
+  data.tar.gz: 2172c5cd70c8047cb0b0bdd171d5f9ed5cb82b73
 SHA512:
-  metadata.gz: ff293ca0f11064c40e3411a57ab8df729013524cf2afb9c2dd018153457f34e200bd8fdfd6ee32aaf4a7af3fa2ce6e19447f9a70275c8bda8332285180e2b509
-  data.tar.gz: 4c4ab22ed0aa8d84cdfce68051788c0cdfb5fb60d468c64740285af4bda8b217473c11e1974aaa4c4220f4fe6ca7542e8c9a71de5e081a8d2e33aa83323bb91f
+  metadata.gz: 3949ab96ce033255e31bc588db0f0895c6a72a91d9bfb56137995f0d5f52bc467dfa660359d440fddca89fe64fd3f6af7a2b28f2476698ba35319f999358a721
+  data.tar.gz: 2348e329e244a3b4a777e8635143d33ec591c10ee411f40aa69a629487643585e8b303c13f120d51e85a2dde13f5dea5b0cb3290b152e619d3fa07dc6f4769d4

data/CHANGES.md

@@ -1,3 +1,8 @@
+# 0.0.4
+
+* Don't truncate UTF8 Messages [GH-18]
+* Gracefully handle truncated fetch reponses [GH-19]
+
 # 0.0.3
 
 * Better distribute messages across partitions.

data/lib/poseidon/connection.rb

@@ -115,7 +115,7 @@ module Poseidon
     def send_request(request)
       buffer = Protocol::RequestBuffer.new
       request.write(buffer)
-      @socket.write([buffer.to_s.size].pack("N") + buffer.to_s)
+      @socket.write([buffer.to_s.bytesize].pack("N") + buffer.to_s)
     rescue Errno::EPIPE, Errno::ECONNRESET
       @socket = nil
       raise ConnectionFailedError

data/lib/poseidon/partition_consumer.rb

@@ -82,9 +82,13 @@ module Poseidon
     #
     # @api public
     def fetch(options = {})
-      fetch_max_wait = options[:max_wait] || max_wait_ms
-      fetch_max_bytes = options[:max_bytes] || max_bytes
-      fetch_min_bytes = options[:min_bytes] || min_bytes
+      fetch_max_wait = options.delete(:max_wait) || max_wait_ms
+      fetch_max_bytes = options.delete(:max_bytes) || max_bytes
+      fetch_min_bytes = options.delete(:min_bytes) || min_bytes
+
+      if options.keys.any?
+        raise ArgumentError, "Unknown options: #{options.keys.inspect}"
+      end
 
       topic_fetches = build_topic_fetch_request(fetch_max_bytes)
       fetch_response = @connection.fetch(fetch_max_wait, fetch_min_bytes, topic_fetches)

data/lib/poseidon/protocol/protocol_struct.rb

@@ -158,6 +158,11 @@ module Poseidon
 
       def read(buffer)
         if self.class.prepend_size?
+          if !have_header?(buffer)
+            @truncated = true
+            return
+          end
+
           @size = buffer.int32
 
           if self.class.prepend_crc32?
@@ -171,7 +176,6 @@ module Poseidon
             expected_bytes_remaining = @size
           end
 
-
           if self.class.truncatable? && expected_bytes_remaining > buffer.bytes_remaining
             @truncated = true
             return
@@ -190,6 +194,20 @@ module Poseidon
         end
       end
 
+      def have_header?(buffer)
+        if self.class.truncatable?
+          if self.class.prepend_crc32?
+            header_bytes = 8
+          else
+            header_bytes = 4
+          end
+
+          return buffer.bytes_remaining >= header_bytes
+        else
+          return true
+        end
+      end
+
       def read_member(buffer, member)
         case type = type_map[member]
         when Array

data/lib/poseidon/protocol/request_buffer.rb

@@ -40,7 +40,7 @@ module Poseidon
         if string.nil?
           int16(-1)
         else
-          int16(string.size)
+          int16(string.bytesize)
           append(string)
         end
       end
@@ -49,13 +49,13 @@ module Poseidon
         if string.nil?
           int32(-1)
         else
-          int32(string.size)
+          int32(string.bytesize)
           append(string)
         end
       end
 
       def prepend_crc32
-        checksum_pos = @s.size
+        checksum_pos = @s.bytesize
         @s += " "
         yield
         @s[checksum_pos] = [Zlib::crc32(@s[(checksum_pos+1)..-1])].pack("N")
@@ -63,10 +63,10 @@ module Poseidon
       end
 
       def prepend_size
-        size_pos = @s.size
+        size_pos = @s.bytesize
         @s += " "
         yield
-        @s[size_pos] = [(@s.size-1) - size_pos].pack("N")
+        @s[size_pos] = [(@s.bytesize-1) - size_pos].pack("N")
         nil
       end
 

data/lib/poseidon/protocol/response_buffer.rb

@@ -57,11 +57,11 @@ module Poseidon
       end
 
       def bytes_remaining
-        @s.size - @pos
+        @s.bytesize - @pos
       end
 
       def eof?
-        @pos == @s.size
+        @pos == @s.bytesize
       end
 
       def to_s

data/lib/poseidon/version.rb

@@ -1,4 +1,4 @@
 module Poseidon
   # Unstable! API May Change!
-  VERSION = "0.0.3"
+  VERSION = "0.0.4"
 end

data/spec/integration/simple/truncated_messages_spec.rb

@@ -0,0 +1,44 @@
+require 'integration/simple/spec_helper'
+
+describe "truncated messages" do
+  before(:all) do
+    @s1 = "a" * 335
+    @s2 = "b" * 338
+
+    @producer = Producer.new(["localhost:9092"],
+                             "test_client",
+                             :type => :sync)
+                             
+    @producer.send_messages([Message.new(:topic => 'test_max_bytes', :value => @s1), Message.new(:topic => 'test_max_bytes', :value => @s2)])
+  end
+
+  it "correctly handles max_byte lengths smallert than a message" do
+    0.upto(360) do |n|
+      consumer = PartitionConsumer.new("test_consumer", "localhost", 9092,
+                                       "test_max_bytes", 0, :earliest_offset)
+      expect(consumer.fetch(:max_bytes => n)).to eq([])
+    end
+  end
+
+  it "correctly handles max_byte lengths that should return a single message" do
+    361.upto(724) do |n|
+      consumer = PartitionConsumer.new("test_consumer", "localhost", 9092,
+                                       "test_max_bytes", 0, :earliest_offset)
+
+      messages = consumer.fetch(:max_bytes => n) 
+      expect(messages.size).to eq(1)
+      expect(messages.first.value).to eq(@s1)
+    end
+  end
+
+  it "correctly handles max_byte lengths that should return two messages" do
+    725.upto(1000) do |n|
+      consumer = PartitionConsumer.new("test_consumer", "localhost", 9092,
+                                       "test_max_bytes", 0, :earliest_offset)
+
+      messages = consumer.fetch(:max_bytes => n) 
+      expect(messages.size).to eq(2)
+      expect(messages.map(&:value)).to eq([@s1, @s2])
+    end
+  end
+end

data/spec/unit/message_conductor_spec.rb

@@ -24,9 +24,13 @@ describe MessageConductor do
 
       context "for unkeyed messages" do
         it "round robins which partition the message should go to" do
-          [0,1,0,1].each do |destination|
-            expect(@mc.destination("test").first).to eq(destination)
+          destinations = 4.times.map do
+            @mc.destination("test").first
           end
+
+          first = [destinations[0], destinations[2]]
+          second = [destinations[1], destinations[3]]
+          expect([first.uniq, second.uniq].sort).to eq([[0],[1]])
         end
 
         context "unknown topic" do

data/spec/unit/message_spec.rb

@@ -1,3 +1,4 @@
+# encoding: UTF-8
 require 'spec_helper'
 
 describe Message do
@@ -87,6 +88,22 @@ describe Message do
     end
   end
 
+  context "utf8 string with multibyte characters" do
+    it "roundtrips correctly" do
+      s = "the µ is two bytes"
+      m = Message.new(:value => s,
+                      :key => "key",
+                      :topic => "topic")
+
+      req_buf = Protocol::RequestBuffer.new
+      m.write(req_buf)
+
+      resp_buf = Protocol::ResponseBuffer.new(req_buf.to_s)
+
+      expect(Message.read(resp_buf).value).to eq(s.force_encoding("ASCII-8BIT"))
+    end
+  end
+
   context "frozen string for value" do
     it "builds the payload without error" do
       s = "asdffasdf".freeze

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: poseidon
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Bob Potter
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-09-25 00:00:00.000000000 Z
+date: 2013-09-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -121,6 +121,7 @@ files:
 - spec/integration/simple/multiple_brokers_spec.rb
 - spec/integration/simple/simple_producer_and_consumer_spec.rb
 - spec/integration/simple/spec_helper.rb
+- spec/integration/simple/truncated_messages_spec.rb
 - spec/integration/simple/unavailable_broker_spec.rb
 - spec/spec_helper.rb
 - spec/test_cluster.rb
@@ -176,6 +177,7 @@ test_files:
 - spec/integration/simple/multiple_brokers_spec.rb
 - spec/integration/simple/simple_producer_and_consumer_spec.rb
 - spec/integration/simple/spec_helper.rb
+- spec/integration/simple/truncated_messages_spec.rb
 - spec/integration/simple/unavailable_broker_spec.rb
 - spec/spec_helper.rb
 - spec/test_cluster.rb