kafka-rb 0.0.5 → 0.0.6

data/README.md

@@ -44,6 +44,7 @@ sudo gem install kafka-rb
     consumer = Kafka::Consumer.new
     messages = consumer.consume
 
+
 ### Consuming messages using a block loop
 
     require 'kafka'

data/Rakefile

@@ -2,7 +2,7 @@ require 'rubygems'
 require 'rake/gempackagetask'
 require 'rubygems/specification'
 require 'date'
-require 'spec/rake/spectask'
+require 'rspec/core/rake_task'
 
 GEM = 'kafka-rb'
 GEM_NAME = 'Kafka Client'
@@ -33,9 +33,9 @@ end
 task :default => :spec
 
 desc "Run specs"
-Spec::Rake::SpecTask.new do |t|
-  t.spec_files = FileList['spec/**/*_spec.rb']
-  t.spec_opts = %w(-fs --color)
+RSpec::Core::RakeTask.new do |t|
+  t.pattern = FileList['spec/**/*_spec.rb']
+  t.rspec_opts = %w(-fs --color)
 end
 
 Rake::GemPackageTask.new(spec) do |pkg|
@@ -55,7 +55,7 @@ task :make_spec do
 end
 
 desc "Run all examples with RCov"
-Spec::Rake::SpecTask.new(:rcov) do |t|
-  t.spec_files = FileList['spec/**/*_spec.rb']
+RSpec::Core::RakeTask.new(:rcov) do |t|
+  t.pattern = FileList['spec/**/*_spec.rb']
   t.rcov = true
 end

data/lib/kafka/consumer.rb

@@ -71,9 +71,11 @@ module Kafka
       processed = 0
       length = data.length - 4
       while(processed <= length) do
-        message_size = data[processed, 4].unpack("N").shift
-        messages << Kafka::Message.parse_from(data[processed, message_size + 4])
-        processed += 4 + message_size
+        message_size = data[processed, 4].unpack("N").shift + 4
+        message_data = data[processed, message_size]
+        break unless message_data.size == message_size
+        messages << Kafka::Message.parse_from(message_data)
+        processed += message_size
       end
       self.offset += processed
       messages

data/lib/kafka/producer.rb

@@ -16,7 +16,12 @@ module Kafka
     end
 
     def encode(message)
-      [message.magic].pack("C") + [message.calculate_checksum].pack("N") + message.payload.to_s
+      if RUBY_VERSION[0,3] == "1.8"  # Use old iconv on Ruby 1.8 for encoding
+        ic = Iconv.new('UTF-8//IGNORE', 'UTF-8')
+        [message.magic].pack("C") + [message.calculate_checksum].pack("N") + ic.iconv(message.payload.to_s)
+      else
+        [message.magic].pack("C") + [message.calculate_checksum].pack("N") + message.payload.to_s.force_encoding(Encoding::ASCII_8BIT)
+      end
     end
 
     def encode_request(topic, partition, messages)

data/lib/kafka.rb

@@ -1,5 +1,8 @@
 require 'socket'
 require 'zlib'
+if RUBY_VERSION[0,3] == "1.8"
+  require 'iconv'
+end
 
 require File.join(File.dirname(__FILE__), "kafka", "io")
 require File.join(File.dirname(__FILE__), "kafka", "request_type")

data/spec/consumer_spec.rb

@@ -64,7 +64,7 @@ describe Consumer do
     end
 
     it "should read the response data" do
-      bytes = [12].pack("N") + [0].pack("C") + [1120192889].pack("N") + "ale"
+      bytes = [8].pack("N") + [0].pack("C") + [1120192889].pack("N") + "ale"
       @mocked_socket.should_receive(:read).exactly(:twice).and_return(bytes)
       @consumer.read_data_response.should eql(bytes[2, bytes.length])
     end
@@ -78,7 +78,7 @@ describe Consumer do
     end
 
     it "should parse a message set from bytes" do
-      bytes = [12].pack("N") + [0].pack("C") + [1120192889].pack("N") + "ale"
+      bytes = [8].pack("N") + [0].pack("C") + [1120192889].pack("N") + "ale"
       message = @consumer.parse_message_set_from(bytes).first
       message.payload.should eql("ale")
       message.checksum.should eql(1120192889)
@@ -86,6 +86,30 @@ describe Consumer do
       message.valid?.should eql(true)
     end
 
+    it "should skip an incomplete message at the end of the response" do
+      bytes = [8].pack("N") + [0].pack("C") + [1120192889].pack("N") + "ale"
+      # incomplete message
+      bytes += [8].pack("N")
+      messages = @consumer.parse_message_set_from(bytes)
+      messages.size.should eql(1)
+    end
+    
+    it "should skip an incomplete message at the end of the response which has the same length as an empty message" do
+      bytes = [8].pack("N") + [0].pack("C") + [1120192889].pack("N") + "ale"
+      # incomplete message because payload is missing
+      bytes += [8].pack("N") + [0].pack("C") + [1120192889].pack("N")
+      messages = @consumer.parse_message_set_from(bytes)
+      messages.size.should eql(1)
+    end
+    
+    it "should read empty messages correctly" do
+      # empty message
+      bytes = [5].pack("N") + [0].pack("C") + [0].pack("N") + ""
+      messages = @consumer.parse_message_set_from(bytes)
+      messages.size.should eql(1)
+      messages.first.payload.should eql("")
+    end
+
     it "should consume messages" do
       @consumer.should_receive(:send_consume_request).and_return(true)
       @consumer.should_receive(:read_data_response).and_return("")

data/spec/producer_spec.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 require File.dirname(__FILE__) + '/spec_helper'
 
 describe Producer do
@@ -44,6 +45,18 @@ describe Producer do
         full_message = [message.magic].pack("C") + [message.calculate_checksum].pack("N") + message.payload.to_s
         @producer.encode(message).should eql(full_message)
       end
+
+      it "should encode strings containing non-ASCII characters" do
+        message = Kafka::Message.new("ümlaut")
+        encoded = @producer.encode(message)
+        data = [encoded.size].pack("N") + encoded
+        if RUBY_VERSION[0,3] == "1.8" # Use old iconv on Ruby 1.8 for encoding
+          ic = Iconv.new('UTF-8//IGNORE', 'UTF-8')
+          ic.iconv(Kafka::Message.parse_from(data).payload).should eql("ümlaut")
+        else
+          Kafka::Message.parse_from(data).payload.force_encoding(Encoding::ASCII_8BIT).should eql("ümlaut")
+        end
+      end
     end
 
     describe "Request Encoding" do

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: kafka-rb
 version: !ruby/object:Gem::Version 
-  hash: 21
+  hash: 19
   prerelease: false
   segments: 
   - 0
   - 0
-  - 5
-  version: 0.0.5
+  - 6
+  version: 0.0.6
 platform: ruby
 authors: 
 - Alejandro Crosa
@@ -15,7 +15,7 @@ autorequire: kafka-rb
 bindir: bin
 cert_chain: []
 
-date: 2011-01-14 00:00:00 -08:00
+date: 2011-01-13 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency