kafka-rb 0.0.2 → 0.0.5

data/README.md

@@ -13,40 +13,49 @@ sudo gem install kafka-rb
 
 ### Sending a simple message
 
-require 'kafka-rb'
+    require 'kafka'
+    producer = Kafka::Producer.new
+    message = Kafka::Message.new("some random message content")
+    producer.send(message)
 
-producer = Kafka::Producer.new
+### Sending a sequence of messages
 
-message = Kafka::Message.new("some random message content")
+    require 'kafka'
+    producer = Kafka::Producer.new
+    message1 = Kafka::Message.new("some random message content")
+    message2 = Kafka::Message.new("some more content")
+    producer.send([message1, message2])
 
-producer.send(message)
+### Batching a bunch of messages using the block syntax
 
-### sending a sequence of messages
+    require 'kafka'
+    producer = Kafka::Producer.new
+    producer.batch do |messages|
+        puts "Batching a send of multiple messages.."
+        messages << Kafka::Message.new("first message to send")
+        messages << Kafka::Message.new("second message to send")
+    end
 
-require 'kafka-rb'
-
-producer = Kafka::Producer.new
-
-message1 = Kafka::Message.new("some random message content")
-
-message2 = Kafka::Message.new("some more content")
-
-producer.send([message1, message2])
-
-### batching a bunch of messages using the block syntax
+* they will be sent all at once, after the block execution
 
-require 'kafka-rb'
+### Consuming messages one by one
 
-producer = Kafka::Producer.new
+    require 'kafka'
+    consumer = Kafka::Consumer.new
+    messages = consumer.consume
 
-producer.batch do |messages|
+### Consuming messages using a block loop
 
-  puts "Batching a send of multiple messages.."
+    require 'kafka'
+    consumer = Kafka::Consumer.new
+    consumer.loop do |messages|
+        puts "Received"
+        puts messages
+    end
 
-  messages << Kafka::Message.new("first message to send")
 
-  messages << Kafka::Message.new("second message to send")
+Contact for questions
 
-end
+alejandrocrosa at(@) gmail.com
 
-* they will be sent all at once, after the block execution
+http://twitter.com/alejandrocrosa

data/Rakefile

@@ -5,13 +5,13 @@ require 'date'
 require 'spec/rake/spectask'
 
 GEM = 'kafka-rb'
-GEM_NAME = 'Kafka Client Producer'
-GEM_VERSION = '0.0.2'
+GEM_NAME = 'Kafka Client'
+GEM_VERSION = '0.0.5'
 AUTHORS = ['Alejandro Crosa']
 EMAIL = "alejandrocrosa@gmail.com"
 HOMEPAGE = "http://github.com/acrosa/kafka-rb"
 SUMMARY = "A Ruby client for the Kafka distributed publish/subscribe messaging service"
-DESCRIPTION = "kafka-rb allows you to produce messages to the Kafka distributed publish/subscribe messaging service."
+DESCRIPTION = "kafka-rb allows you to produce and consume messages using the Kafka distributed publish/subscribe messaging service."
 
 spec = Gem::Specification.new do |s|
   s.name = GEM

data/lib/kafka/consumer.rb

@@ -0,0 +1,82 @@
+module Kafka
+  class Consumer
+
+    include Kafka::IO
+
+    CONSUME_REQUEST_TYPE = Kafka::RequestType::FETCH
+    MAX_SIZE = 1048576 # 1 MB
+    DEFAULT_POLLING_INTERVAL = 2 # 2 seconds
+
+    attr_accessor :topic, :partition, :offset, :max_size, :request_type, :polling
+
+    def initialize(options = {})
+      self.topic        = options[:topic]        || "test"
+      self.partition    = options[:partition]    || 0
+      self.host         = options[:host]         || "localhost"
+      self.port         = options[:port]         || 9092
+      self.offset       = options[:offset]       || 0
+      self.max_size     = options[:max_size]     || MAX_SIZE
+      self.request_type = options[:request_type] || CONSUME_REQUEST_TYPE
+      self.polling      = options[:polling]      || DEFAULT_POLLING_INTERVAL
+      self.connect(self.host, self.port)
+    end
+
+    # REQUEST TYPE ID + TOPIC LENGTH + TOPIC + PARTITION + OFFSET + MAX SIZE
+    def request_size
+      2 + 2 + topic.length + 4 + 8 + 4
+    end
+
+    def encode_request_size
+      [self.request_size].pack("N")
+    end
+
+    def encode_request(request_type, topic, partition, offset, max_size)
+      request_type = [request_type].pack("n")
+      topic        = [topic.length].pack('n') + topic
+      partition    = [partition].pack("N")
+      offset       = [offset].pack("Q").reverse # DIY 64bit big endian integer
+      max_size     = [max_size].pack("N")
+
+      request_type + topic + partition + offset + max_size
+    end
+
+    def consume
+      self.send_consume_request         # request data
+      data = self.read_data_response    # read data response
+      self.parse_message_set_from(data) # parse message set
+    end
+
+    def loop(&block)
+      messages = []
+      while(true) do
+        messages = self.consume
+        block.call(messages) if messages && !messages.empty?
+        sleep(self.polling)
+      end
+    end
+
+    def read_data_response
+      data_length = self.socket.read(4).unpack("N").shift # read length
+      data = self.socket.read(data_length)                # read message set
+      data[2, data.length]                                # we start with a 2 byte offset
+    end
+
+    def send_consume_request
+      self.write(self.encode_request_size) # write request_size
+      self.write(self.encode_request(self.request_type, self.topic, self.partition, self.offset, self.max_size)) # write request
+    end
+
+    def parse_message_set_from(data)
+      messages = []
+      processed = 0
+      length = data.length - 4
+      while(processed <= length) do
+        message_size = data[processed, 4].unpack("N").shift
+        messages << Kafka::Message.parse_from(data[processed, message_size + 4])
+        processed += 4 + message_size
+      end
+      self.offset += processed
+      messages
+    end
+  end
+end

data/lib/kafka/io.rb

@@ -26,5 +26,14 @@ module Kafka
       self.reconnect
       self.socket.write(data) # retry
     end
+
+    def read(length)
+      begin
+        self.socket.read(length)
+      rescue Errno::EAGAIN
+        self.disconnect
+        raise Errno::EAGAIN, "Timeout reading from the socket"
+      end
+    end
   end
 end

data/lib/kafka/message.rb

@@ -5,13 +5,15 @@ module Kafka
   # 4 byte CRC32 of the payload
   # N - 5 byte payload
   class Message
+
     MAGIC_IDENTIFIER_DEFAULT = 0
+
     attr_accessor :magic, :checksum, :payload
 
-    def initialize(payload = nil, magic = MAGIC_IDENTIFIER_DEFAULT)
-      self.magic = magic
-      self.payload = payload
-      self.checksum = self.calculate_checksum
+    def initialize(payload = nil, magic = MAGIC_IDENTIFIER_DEFAULT, checksum = nil)
+      self.magic    = magic
+      self.payload  = payload
+      self.checksum = checksum || self.calculate_checksum
     end
 
     def calculate_checksum
@@ -21,5 +23,13 @@ module Kafka
     def valid?
       self.checksum == Zlib.crc32(self.payload)
     end
+
+    def self.parse_from(binary)
+      size     = binary[0, 4].unpack("N").shift.to_i
+      magic    = binary[4, 1].unpack("C").shift
+      checksum = binary[5, 4].unpack("N").shift
+      payload  = binary[9, size] # 5 = 1 + 4 is Magic + Checksum
+      return Kafka::Message.new(payload, magic, checksum)
+    end
   end
-end
+end

data/lib/kafka/producer.rb

@@ -3,15 +3,15 @@ module Kafka
 
     include Kafka::IO
 
-    PRODUCE_REQUEST_ID = 0
+    PRODUCE_REQUEST_ID = Kafka::RequestType::PRODUCE
 
     attr_accessor :topic, :partition
 
     def initialize(options = {})
-      self.topic = options[:topic] || "test"
-      self.partition = options[:partition] || 0
-      self.host = options[:host] || "localhost"
-      self.port = options[:port] || 9092
+      self.topic     = options[:topic]      || "test"
+      self.partition = options[:partition]  || 0
+      self.host      = options[:host]       || "localhost"
+      self.port      = options[:port]       || 9092
       self.connect(self.host, self.port)
     end
 

data/lib/kafka/request_type.rb

@@ -0,0 +1,9 @@
+module Kafka
+  module RequestType
+    PRODUCE      = 0
+    FETCH        = 1
+    MULTIFETCH   = 2
+    MULTIPRODUCE = 3
+    OFFSETS      = 4
+  end
+end

data/lib/kafka.rb

@@ -1,9 +1,12 @@
 require 'socket'
 require 'zlib'
+
 require File.join(File.dirname(__FILE__), "kafka", "io")
+require File.join(File.dirname(__FILE__), "kafka", "request_type")
 require File.join(File.dirname(__FILE__), "kafka", "batch")
 require File.join(File.dirname(__FILE__), "kafka", "message")
 require File.join(File.dirname(__FILE__), "kafka", "producer")
+require File.join(File.dirname(__FILE__), "kafka", "consumer")
 
 module Kafka
 end

data/spec/consumer_spec.rb

@@ -0,0 +1,120 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe Consumer do
+
+  before(:each) do
+    @mocked_socket = mock(TCPSocket)
+    TCPSocket.stub!(:new).and_return(@mocked_socket) # don't use a real socket
+    @consumer = Consumer.new
+  end
+
+  describe "Kafka Consumer" do
+
+    it "should have a CONSUME_REQUEST_TYPE" do
+      Consumer::CONSUME_REQUEST_TYPE.should eql(1)
+      @consumer.should respond_to(:request_type)
+    end
+
+    it "should have a topic and a partition" do
+      @consumer.should respond_to(:topic)
+      @consumer.should respond_to(:partition)
+    end
+
+    it "should have a polling option, and a default value" do
+      Consumer::DEFAULT_POLLING_INTERVAL.should eql(2)
+      @consumer.should respond_to(:polling)
+      @consumer.polling.should eql(2)
+    end
+
+    it "should set a topic and partition on initialize" do
+      @consumer = Consumer.new({ :host => "localhost", :port => 9092, :topic => "testing" })
+      @consumer.topic.should eql("testing")
+      @consumer.partition.should eql(0)
+      @consumer = Consumer.new({ :topic => "testing", :partition => 3 })
+      @consumer.partition.should eql(3)
+    end
+
+    it "should set default host and port if none is specified" do
+      @consumer = Consumer.new
+      @consumer.host.should eql("localhost")
+      @consumer.port.should eql(9092)
+    end
+
+    it "should have a default offset, and be able to set it" do
+      @consumer.offset.should eql(0)
+      @consumer = Consumer.new({ :offset => 1111 })
+      @consumer.offset.should eql(1111)
+    end
+
+    it "should have a max size" do
+      Consumer::MAX_SIZE.should eql(1048576)
+      @consumer.max_size.should eql(1048576)
+    end
+
+    it "should return the size of the request" do
+      @consumer.request_size.should eql(24)
+      @consumer.topic = "someothertopicname"
+      @consumer.request_size.should eql(38)
+      @consumer.encode_request_size.should eql([@consumer.request_size].pack("N"))
+    end
+
+    it "should encode a request to consume" do
+      bytes = [Kafka::Consumer::CONSUME_REQUEST_TYPE].pack("n") + ["test".length].pack("n") + "test" + [0].pack("N") + [0].pack("L_") + [Kafka::Consumer::MAX_SIZE].pack("N")
+      @consumer.encode_request(Kafka::Consumer::CONSUME_REQUEST_TYPE, "test", 0, 0, Kafka::Consumer::MAX_SIZE).should eql(bytes)
+    end
+
+    it "should read the response data" do
+      bytes = [12].pack("N") + [0].pack("C") + [1120192889].pack("N") + "ale"
+      @mocked_socket.should_receive(:read).exactly(:twice).and_return(bytes)
+      @consumer.read_data_response.should eql(bytes[2, bytes.length])
+    end
+
+    it "should send a consumer request" do
+      @consumer.stub!(:encode_request_size).and_return(666)
+      @consumer.stub!(:encode_request).and_return("someencodedrequest")
+      @consumer.should_receive(:write).with("someencodedrequest").exactly(:once).and_return(true)
+      @consumer.should_receive(:write).with(666).exactly(:once).and_return(true)
+      @consumer.send_consume_request.should eql(true)
+    end
+
+    it "should parse a message set from bytes" do
+      bytes = [12].pack("N") + [0].pack("C") + [1120192889].pack("N") + "ale"
+      message = @consumer.parse_message_set_from(bytes).first
+      message.payload.should eql("ale")
+      message.checksum.should eql(1120192889)
+      message.magic.should eql(0)
+      message.valid?.should eql(true)
+    end
+
+    it "should consume messages" do
+      @consumer.should_receive(:send_consume_request).and_return(true)
+      @consumer.should_receive(:read_data_response).and_return("")
+      @consumer.consume.should eql([])
+    end
+
+    it "should loop and execute a block with the consumed messages" do
+      @consumer.stub!(:consume).and_return([mock(Kafka::Message)])
+      messages = []
+      messages.should_receive(:<<).exactly(:once).and_return([])
+      @consumer.loop do |message|
+        messages << message
+        break # we don't wanna loop forever on the test
+      end
+    end
+
+    it "should loop (every N seconds, configurable on polling attribute), and execute a block with the consumed messages" do
+      @consumer = Consumer.new({ :polling => 1 })
+      @consumer.stub!(:consume).and_return([mock(Kafka::Message)])
+      messages = []
+      messages.should_receive(:<<).exactly(:twice).and_return([])
+      executed_times = 0
+      @consumer.loop do |message|
+        messages << message
+        executed_times += 1
+        break if executed_times >= 2 # we don't wanna loop forever on the test, only 2 seconds
+      end
+
+      executed_times.should eql(2)
+    end
+  end
+end

data/spec/io_spec.rb

@@ -39,6 +39,19 @@ describe IO do
       @io.write(data).should eql(9)
     end
 
+    it "should read from a socket" do
+      length = 200
+      @mocked_socket.should_receive(:read).with(length).and_return(nil)
+      @io.read(length)
+    end
+
+    it "should disconnect on a timeout when reading from a socket (to aviod protocol desync state)" do
+      length = 200
+      @mocked_socket.should_receive(:read).with(length).and_raise(Errno::EAGAIN)
+      @io.should_receive(:disconnect)
+      lambda { @io.read(length) }.should raise_error(Errno::EAGAIN)
+    end
+
     it "should disconnect" do
       @io.should respond_to(:disconnect)
       @mocked_socket.should_receive(:close).and_return(nil)

data/spec/message_spec.rb

@@ -39,6 +39,17 @@ describe Message do
       @message.valid?.should eql(true)
       @message.checksum = 0
       @message.valid?.should eql(false)
+      @message = Message.new("alejandro", 0, 66666666) # 66666666 is a funny checksum
+      @message.valid?.should eql(false)
+    end
+
+    it "should parse a message from bytes" do
+      bytes = [12].pack("N") + [0].pack("C") + [1120192889].pack("N") + "ale"
+      message = Kafka::Message.parse_from(bytes)
+      message.valid?.should eql(true)
+      message.magic.should eql(0)
+      message.checksum.should eql(1120192889)
+      message.payload.should eql("ale")
     end
   end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: kafka-rb
 version: !ruby/object:Gem::Version 
-  hash: 27
+  hash: 21
   prerelease: false
   segments: 
   - 0
   - 0
-  - 2
-  version: 0.0.2
+  - 5
+  version: 0.0.5
 platform: ruby
 authors: 
 - Alejandro Crosa
@@ -15,7 +15,7 @@ autorequire: kafka-rb
 bindir: bin
 cert_chain: []
 
-date: 2011-01-11 00:00:00 -08:00
+date: 2011-01-14 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -32,7 +32,7 @@ dependencies:
         version: "0"
   type: :development
   version_requirements: *id001
-description: kafka-rb allows you to produce messages to the Kafka distributed publish/subscribe messaging service.
+description: kafka-rb allows you to produce and consume messages using the Kafka distributed publish/subscribe messaging service.
 email: alejandrocrosa@gmail.com
 executables: []
 
@@ -45,12 +45,14 @@ files:
 - README.md
 - Rakefile
 - lib/kafka/batch.rb
+- lib/kafka/consumer.rb
 - lib/kafka/io.rb
 - lib/kafka/message.rb
 - lib/kafka/producer.rb
+- lib/kafka/request_type.rb
 - lib/kafka.rb
-- lib/test.rb
 - spec/batch_spec.rb
+- spec/consumer_spec.rb
 - spec/io_spec.rb
 - spec/kafka_spec.rb
 - spec/message_spec.rb

data/lib/test.rb

@@ -1,32 +0,0 @@
-$KCODE = 'UTF-8'
-
-require 'zlib'
-
-PRODUCE_REQUEST_ID = 0
-
-def encode_message(message)
-    # <MAGIC_BYTE: char> <CRC32: int> <PAYLOAD: bytes>
-    data = [0].pack("C").to_s + [Zlib.crc32(message)].pack('N').to_s + message
-    # print ("CHECKSUM " + Zlib.crc32(message).to_s)
-    # print ("MES " + data.length.to_s)
-    return data
-end
-# encode_message("ale")
-
-def encode_produce_request(topic, partition, messages)
-    encoded = messages.collect { |m| encode_message(m) }
-    message_set = encoded.collect { |e| puts "Message size #{e.length}"; [e.length].pack("N") + e }.join("")
-
-    puts "MESSAGE"
-    puts message_set.inspect
-
-    data = [PRODUCE_REQUEST_ID].pack("n") + \
-           [topic.length].pack("n") + topic.to_s + \
-           [partition].pack("N") + \
-           [message_set.length].pack("N") + message_set
-    puts "DATA " + message_set.length.to_s
-    return [data.length].pack("N") + data
-end
-
-socket = TCPSocket.open("localhost", 9092)
-socket.write encode_produce_request("test", 0, ["ale"])