kafka-rb 0.0.10 → 0.0.11

data/Rakefile

@@ -14,14 +14,14 @@
 # limitations under the License.
 
 require 'rubygems'
-require 'rake/gempackagetask'
+require 'rubygems/package_task'
 require 'rubygems/specification'
 require 'date'
 require 'rspec/core/rake_task'
 
 spec = Gem::Specification.new do |s|
   s.name = %q{kafka-rb}
-  s.version = "0.0.10"
+  s.version = "0.0.11"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Alejandro Crosa", "Stefan Mees", "Tim Lossen", "Liam Stewart"]
@@ -29,10 +29,10 @@ spec = Gem::Specification.new do |s|
   s.date = Time.now.strftime("%Y-%m-%d")
   s.description = %q{kafka-rb allows you to produce and consume messages using the Kafka distributed publish/subscribe messaging service.}
   s.extra_rdoc_files = ["LICENSE"]
-  s.files = ["LICENSE", "README.md", "Rakefile", "lib/kafka", "lib/kafka/batch.rb", "lib/kafka/consumer.rb", "lib/kafka/io.rb", "lib/kafka/message.rb", "lib/kafka/producer.rb", "lib/kafka/request_type.rb", "lib/kafka/error_codes.rb", "lib/kafka.rb", "spec/batch_spec.rb", "spec/consumer_spec.rb", "spec/io_spec.rb", "spec/kafka_spec.rb", "spec/message_spec.rb", "spec/producer_spec.rb", "spec/spec_helper.rb"]
+  s.files = ["LICENSE", "README.md", "Rakefile"] + Dir.glob("lib/**/*.rb")
+  s.test_files = Dir.glob("spec/**/*.rb")
   s.homepage = %q{http://github.com/acrosa/kafka-rb}
   s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.3.7}
   s.summary = %q{A Ruby client for the Kafka distributed publish/subscribe messaging service}
 
   if s.respond_to? :specification_version then

data/lib/kafka/encoder.rb

@@ -0,0 +1,61 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Kafka
+  module Encoder
+    def self.message(message)
+      payload = \
+        if RUBY_VERSION[0,3] == "1.8"  # Use old iconv on Ruby 1.8 for encoding
+          Iconv.new('UTF-8//IGNORE', 'UTF-8').iconv(message.payload.to_s)
+        else
+          message.payload.to_s.force_encoding(Encoding::ASCII_8BIT)
+        end
+      data = [message.magic].pack("C") + [message.calculate_checksum].pack("N") + payload
+
+      [data.length].pack("N") + data
+    end
+
+    def self.message_block(topic, partition, messages)
+      message_set = Array(messages).collect { |message|
+        self.message(message)
+      }.join("")
+
+      topic     = [topic.length].pack("n") + topic
+      partition = [partition].pack("N")
+      messages  = [message_set.length].pack("N") + message_set
+
+      return topic + partition + messages
+    end
+
+    def self.produce(topic, partition, messages)
+      request = [RequestType::PRODUCE].pack("n")
+      data = request + self.message_block(topic, partition, messages)
+
+      return [data.length].pack("N") + data
+    end
+
+    def self.multiproduce(producer_requests)
+      part_set = Array(producer_requests).map { |req|
+        self.message_block(req.topic, req.partition, req.messages)
+      }
+
+      request = [RequestType::MULTIPRODUCE].pack("n")
+      parts = [part_set.length].pack("n") + part_set.join("")
+      data = request + parts
+
+      return [data.length].pack("N") + data
+    end
+  end
+end

data/lib/kafka/multi_producer.rb

@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+module Kafka
+  class MultiProducer
+    include Kafka::IO
+
+    def initialize(options={})
+      self.host = options[:host] || HOST
+      self.port = options[:port] || PORT
+      self.connect(self.host, self.port)
+    end
+
+    def send(topic, messages, options={})
+      partition = options[:partition] || 0
+      self.write(Encoder.produce(topic, partition, messages))
+    end
+
+    def multi_send(producer_requests)
+      self.write(Encoder.multiproduce(producer_requests))
+    end
+  end
+end

data/lib/kafka/producer_request.rb

@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Kafka
+  class ProducerRequest
+    attr_accessor :topic, :messages, :partition
+
+    def initialize(topic, messages, options={})
+      self.topic = topic
+      self.partition = options[:partition] || 0
+      self.messages = Array(messages)
+    end
+  end
+end

data/spec/encoder_spec.rb

@@ -0,0 +1,173 @@
+# encoding: utf-8
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe Encoder do
+  def check_message(bytes, message)
+    encoded = [message.magic].pack("C") + [message.calculate_checksum].pack("N") + message.payload
+    encoded = [encoded.length].pack("N") + encoded
+    bytes.should == encoded
+  end
+
+  describe "Message Encoding" do
+    it "should encode a message" do
+      message = Kafka::Message.new("alejandro")
+      check_message(described_class.message(message), message)
+    end
+
+    it "should encode an empty message" do
+      message = Kafka::Message.new
+      check_message(described_class.message(message), message)
+    end
+
+    it "should encode strings containing non-ASCII characters" do
+      message = Kafka::Message.new("ümlaut")
+      encoded = described_class.message(message)
+      message = Kafka::Message.parse_from(encoded).messages.first
+      if RUBY_VERSION[0,3] == "1.8" # Use old iconv on Ruby 1.8 for encoding
+        ic = Iconv.new('UTF-8//IGNORE', 'UTF-8')
+        ic.iconv(message.payload).should eql("ümlaut")
+      else
+        message.payload.force_encoding(Encoding::UTF_8).should eql("ümlaut")
+      end
+    end
+  end
+
+  describe "produce" do
+    it "should binary encode an empty request" do
+      bytes = described_class.produce("test", 0, [])
+      bytes.length.should eql(20)
+      bytes.should eql("\000\000\000\020\000\000\000\004test\000\000\000\000\000\000\000\000")
+    end
+
+    it "should binary encode a request with a message, using a specific wire format" do
+      message = Kafka::Message.new("ale")
+      bytes = described_class.produce("test", 3, message)
+      data_size  = bytes[0, 4].unpack("N").shift
+      request_id = bytes[4, 2].unpack("n").shift
+      topic_length = bytes[6, 2].unpack("n").shift
+      topic = bytes[8, 4]
+      partition = bytes[12, 4].unpack("N").shift
+      messages_length = bytes[16, 4].unpack("N").shift
+      messages = bytes[20, messages_length]
+
+      bytes.length.should eql(32)
+      data_size.should eql(28)
+      request_id.should eql(0)
+      topic_length.should eql(4)
+      topic.should eql("test")
+      partition.should eql(3)
+      messages_length.should eql(12)
+    end
+  end
+
+  describe "multiproduce" do
+    it "encodes an empty request" do
+      bytes = described_class.multiproduce([])
+      bytes.length.should == 8
+      bytes.should == "\x00\x00\x00\x04\x00\x03\x00\x00"
+    end
+
+    it "encodes a request with a single topic/partition" do
+      message = Kafka::Message.new("ale")
+      bytes = described_class.multiproduce(Kafka::ProducerRequest.new("test", message))
+
+      req_length = bytes[0, 4].unpack("N").shift
+      req_type = bytes[4, 2].unpack("n").shift
+      tp_count = bytes[6, 2].unpack("n").shift
+
+      req_type.should == Kafka::RequestType::MULTIPRODUCE
+      tp_count.should == 1
+
+      topic_length = bytes[8, 2].unpack("n").shift
+      topic = bytes[10, 4]
+      partition = bytes[14, 4].unpack("N").shift
+      messages_length = bytes[18, 4].unpack("N").shift
+      messages_data = bytes[22, messages_length]
+
+      topic_length.should == 4
+      topic.should == "test"
+      partition.should == 0
+      messages_length.should == 12
+      check_message(messages_data, message)
+    end
+
+    it "encodes a request with a single topic/partition but multiple messages" do
+      messages = [Kafka::Message.new("ale"), Kafka::Message.new("beer")]
+      bytes = described_class.multiproduce(Kafka::ProducerRequest.new("test", messages))
+
+      req_length = bytes[0, 4].unpack("N").shift
+      req_type = bytes[4, 2].unpack("n").shift
+      tp_count = bytes[6, 2].unpack("n").shift
+
+      req_type.should == Kafka::RequestType::MULTIPRODUCE
+      tp_count.should == 1
+
+      topic_length = bytes[8, 2].unpack("n").shift
+      topic = bytes[10, 4]
+      partition = bytes[14, 4].unpack("N").shift
+      messages_length = bytes[18, 4].unpack("N").shift
+      messages_data = bytes[22, messages_length]
+
+      topic_length.should == 4
+      topic.should == "test"
+      partition.should == 0
+      messages_length.should == 25
+      check_message(messages_data[0, 12], messages[0])
+      check_message(messages_data[12, 13], messages[1])
+    end
+
+    it "encodes a request with multiple topic/partitions" do
+      messages = [Kafka::Message.new("ale"), Kafka::Message.new("beer")]
+      bytes = described_class.multiproduce([
+          Kafka::ProducerRequest.new("test", messages[0]),
+          Kafka::ProducerRequest.new("topic", messages[1], partition: 1),
+        ])
+
+      req_length = bytes[0, 4].unpack("N").shift
+      req_type = bytes[4, 2].unpack("n").shift
+      tp_count = bytes[6, 2].unpack("n").shift
+
+      req_type.should == Kafka::RequestType::MULTIPRODUCE
+      tp_count.should == 2
+
+      topic_length = bytes[8, 2].unpack("n").shift
+      topic = bytes[10, 4]
+      partition = bytes[14, 4].unpack("N").shift
+      messages_length = bytes[18, 4].unpack("N").shift
+      messages_data = bytes[22, 12]
+
+      topic_length.should == 4
+      topic.should == "test"
+      partition.should == 0
+      messages_length.should == 12
+      check_message(messages_data[0, 12], messages[0])
+
+      topic_length = bytes[34, 2].unpack("n").shift
+      topic = bytes[36, 5]
+      partition = bytes[41, 4].unpack("N").shift
+      messages_length = bytes[45, 4].unpack("N").shift
+      messages_data = bytes[49, 13]
+
+      topic_length.should == 5
+      topic.should == "topic"
+      partition.should == 1
+      messages_length.should == 13
+      check_message(messages_data[0, 13], messages[1])
+    end
+  end
+end

data/spec/multi_producer_spec.rb

@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe MultiProducer do
+  before(:each) do
+    @mocked_socket = mock(TCPSocket)
+    TCPSocket.stub!(:new).and_return(@mocked_socket) # don't use a real socket
+  end
+
+  describe "Kafka Producer" do
+    it "should set default host and port if none is specified" do
+      subject.host.should eql("localhost")
+      subject.port.should eql(9092)
+    end
+
+    it "sends single messages" do
+      message = Kafka::Message.new("ale")
+      encoded = Kafka::Encoder.produce("test", 0, message)
+
+      subject.should_receive(:write).with(encoded).and_return(encoded.length)
+      subject.send("test", message, partition: 0).should == encoded.length
+    end
+
+    it "sends multiple messages" do
+      messages = [Kafka::Message.new("ale"), Kafka::Message.new("beer")]
+      reqs = [
+        Kafka::ProducerRequest.new("topic", messages[0]),
+        Kafka::ProducerRequest.new("topic", messages[1]),
+      ]
+      encoded = Encoder.multiproduce(reqs)
+
+      subject.should_receive(:write).with(encoded).and_return(encoded.length)
+      subject.multi_send(reqs).should == encoded.length
+    end
+  end
+end

data/spec/producer_request_spec.rb

@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe ProducerRequest do
+  let(:message) { Kafka::Message.new }
+  let(:req) { described_class.new("topic", message) }
+
+  it "has a topic" do
+    req.topic = "topic"
+  end
+
+  it "has a set of messages" do
+    req.messages.should == [message]
+  end
+
+  it "has a default partition" do
+    req.partition.should == 0
+  end
+
+  it "can use a user-specified partition" do
+    req = described_class.new("topic", message, partition: 42)
+    req.partition.should == 42
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: kafka-rb
 version: !ruby/object:Gem::Version
-  version: 0.0.10
+  version: 0.0.11
   prerelease: 
 platform: ruby
 authors:
@@ -12,7 +12,7 @@ authors:
 autorequire: kafka-rb
 bindir: bin
 cert_chain: []
-date: 2012-09-11 00:00:00.000000000 Z
+date: 2012-09-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -43,17 +43,23 @@ files:
 - Rakefile
 - lib/kafka/batch.rb
 - lib/kafka/consumer.rb
+- lib/kafka/encoder.rb
+- lib/kafka/error_codes.rb
 - lib/kafka/io.rb
 - lib/kafka/message.rb
+- lib/kafka/multi_producer.rb
 - lib/kafka/producer.rb
+- lib/kafka/producer_request.rb
 - lib/kafka/request_type.rb
-- lib/kafka/error_codes.rb
 - lib/kafka.rb
 - spec/batch_spec.rb
 - spec/consumer_spec.rb
+- spec/encoder_spec.rb
 - spec/io_spec.rb
 - spec/kafka_spec.rb
 - spec/message_spec.rb
+- spec/multi_producer_spec.rb
+- spec/producer_request_spec.rb
 - spec/producer_spec.rb
 - spec/spec_helper.rb
 homepage: http://github.com/acrosa/kafka-rb
@@ -80,4 +86,14 @@ rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: A Ruby client for the Kafka distributed publish/subscribe messaging service
-test_files: []
+test_files:
+- spec/batch_spec.rb
+- spec/consumer_spec.rb
+- spec/encoder_spec.rb
+- spec/io_spec.rb
+- spec/kafka_spec.rb
+- spec/message_spec.rb
+- spec/multi_producer_spec.rb
+- spec/producer_request_spec.rb
+- spec/producer_spec.rb
+- spec/spec_helper.rb