kafka-rb 0.0.8 → 0.0.9

data/README.md

@@ -10,7 +10,7 @@ You can obtain a copy and instructions on how to setup kafka at http://incubator
 
 ## Installation
 
-    sudo gem install wooga-kafka-rb
+    sudo gem install kafka-rb
 
 (should work fine with JRuby, Ruby 1.8 and 1.9)
 

data/Rakefile

@@ -21,7 +21,7 @@ require 'rspec/core/rake_task'
 
 spec = Gem::Specification.new do |s|
   s.name = %q{kafka-rb}
-  s.version = "0.0.8"
+  s.version = "0.0.9"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Alejandro Crosa", "Stefan Mees", "Tim Lossen"]
@@ -30,7 +30,7 @@ spec = Gem::Specification.new do |s|
   s.description = %q{kafka-rb allows you to produce and consume messages using the Kafka distributed publish/subscribe messaging service.}
   s.extra_rdoc_files = ["LICENSE"]
   s.files = ["LICENSE", "README.md", "Rakefile", "lib/kafka", "lib/kafka/batch.rb", "lib/kafka/consumer.rb", "lib/kafka/io.rb", "lib/kafka/message.rb", "lib/kafka/producer.rb", "lib/kafka/request_type.rb", "lib/kafka/error_codes.rb", "lib/kafka.rb", "spec/batch_spec.rb", "spec/consumer_spec.rb", "spec/io_spec.rb", "spec/kafka_spec.rb", "spec/message_spec.rb", "spec/producer_spec.rb", "spec/spec_helper.rb"]
-  s.homepage = %q{http://github.com/wooga/kafka-rb}
+  s.homepage = %q{http://github.com/acrosa/kafka-rb}
   s.require_paths = ["lib"]
   s.rubygems_version = %q{1.3.7}
   s.summary = %q{A Ruby client for the Kafka distributed publish/subscribe messaging service}
@@ -49,7 +49,7 @@ spec = Gem::Specification.new do |s|
   end
 end
 
-Rake::GemPackageTask.new(spec) do |pkg|
+Gem::PackageTask.new(spec) do |pkg|
   pkg.gem_spec = spec
 end
 

data/lib/kafka/consumer.rb

@@ -29,8 +29,8 @@ module Kafka
     def initialize(options = {})
       self.topic        = options[:topic]     || "test"
       self.partition    = options[:partition] || 0
-      self.host         = options[:host]      || "localhost"
-      self.port         = options[:port]      || 9092
+      self.host         = options[:host]      || HOST
+      self.port         = options[:port]      || PORT
       self.offset       = options[:offset]
       self.max_size     = options[:max_size]  || MAX_SIZE
       self.polling      = options[:polling]   || DEFAULT_POLLING_INTERVAL
@@ -49,8 +49,9 @@ module Kafka
     def consume
       self.offset ||= fetch_latest_offset
       send_consume_request
-      data = read_data_response
-      parse_message_set_from(data)
+      message_set = Kafka::Message.parse_from(read_data_response)
+      self.offset += message_set.size
+      message_set.messages
     rescue SocketError
       nil
     end
@@ -94,21 +95,5 @@ module Kafka
       max_size     = [max_size].pack("N")
       request_type + topic + partition + offset + max_size
     end
-
-    def parse_message_set_from(data)
-      messages = []
-      processed = 0
-      length = data.length - 4
-      while (processed <= length) do
-        message_size = data[processed, 4].unpack("N").shift + 4
-        message_data = data[processed, message_size]
-        break unless message_data.size == message_size
-        messages << Kafka::Message.parse_from(message_data)
-        processed += message_size
-      end
-      self.offset += processed
-      messages
-    end
-
   end
 end

data/lib/kafka/io.rb

@@ -16,6 +16,9 @@ module Kafka
   module IO
     attr_accessor :socket, :host, :port
 
+    HOST = "localhost"
+    PORT = 9092
+
     def connect(host, port)
       raise ArgumentError, "No host or port specified" unless host && port
       self.host = host

data/lib/kafka/message.rb

@@ -14,19 +14,35 @@
 # limitations under the License.
 module Kafka
 
-  # A message. The format of an N byte message is the following:
-  # 1 byte "magic" identifier to allow format changes
-  # 4 byte CRC32 of the payload
-  # N - 5 byte payload
+  # A message. The format of a message is as follows:
+  #
+  # 4 byte big-endian int: length of message in bytes (including the rest of
+  #                        the header, but excluding the length field itself)
+  # 1 byte: "magic" identifier (format version number)
+  #
+  # If the magic byte == 0, there is one more header field:
+  #
+  # 4 byte big-endian int: CRC32 checksum of the payload
+  #
+  # If the magic byte == 1, there are two more header fields:
+  #
+  # 1 byte: "attributes" (flags for compression, codec etc)
+  # 4 byte big-endian int: CRC32 checksum of the payload
+  #
+  # All following bytes are the payload.
   class Message
 
     MAGIC_IDENTIFIER_DEFAULT = 0
+    BASIC_MESSAGE_HEADER = 'NC'.freeze
+    VERSION_0_HEADER = 'N'.freeze
+    VERSION_1_HEADER = 'CN'.freeze
+    COMPRESSION_CODEC_MASK = 0x03
 
     attr_accessor :magic, :checksum, :payload
 
     def initialize(payload = nil, magic = MAGIC_IDENTIFIER_DEFAULT, checksum = nil)
       self.magic    = magic
-      self.payload  = payload
+      self.payload  = payload || ""
       self.checksum = checksum || self.calculate_checksum
     end
 
@@ -35,15 +51,67 @@ module Kafka
     end
 
     def valid?
-      self.checksum == Zlib.crc32(self.payload)
+      self.checksum == calculate_checksum
     end
 
-    def self.parse_from(binary)
-      size     = binary[0, 4].unpack("N").shift.to_i
-      magic    = binary[4, 1].unpack("C").shift
-      checksum = binary[5, 4].unpack("N").shift
-      payload  = binary[9, size] # 5 = 1 + 4 is Magic + Checksum
-      return Kafka::Message.new(payload, magic, checksum)
+    # Takes a byte string containing one or more messages; returns a MessageSet
+    # with the messages parsed from the string, and the number of bytes
+    # consumed from the string.
+    def self.parse_from(data)
+      messages = []
+      bytes_processed = 0
+
+      while bytes_processed <= data.length - 5 # 5 = size of BASIC_MESSAGE_HEADER
+        message_size, magic = data[bytes_processed, 5].unpack(BASIC_MESSAGE_HEADER)
+        break if bytes_processed + message_size + 4 > data.length # message is truncated
+
+        case magic
+        when 0
+          # |  0  |  1  |  2  |  3  |  4  |  5  |  6  |  7  |  8  |  9      ...
+          # |                       |     |                       |
+          # |      message_size     |magic|        checksum       | payload ...
+          payload_size = message_size - 5 # 5 = sizeof(magic) + sizeof(checksum)
+          checksum = data[bytes_processed + 5, 4].unpack(VERSION_0_HEADER).shift
+          payload  = data[bytes_processed + 9, payload_size]
+          messages << Kafka::Message.new(payload, magic, checksum)
+
+        when 1
+          # |  0  |  1  |  2  |  3  |  4  |  5  |  6  |  7  |  8  |  9  | 10      ...
+          # |                       |     |     |                       |
+          # |         size          |magic|attrs|        checksum       | payload ...
+          payload_size = message_size - 6 # 6 = sizeof(magic) + sizeof(attrs) + sizeof(checksum)
+          attributes, checksum = data[bytes_processed + 5, 5].unpack(VERSION_1_HEADER)
+          payload = data[bytes_processed + 10, payload_size]
+
+          case attributes & COMPRESSION_CODEC_MASK
+          when 0 # a single uncompressed message
+            messages << Kafka::Message.new(payload, magic, checksum)
+          when 1 # a gzip-compressed message set -- parse recursively
+            uncompressed = Zlib::GzipReader.new(StringIO.new(payload)).read
+            message_set = parse_from(uncompressed)
+            raise 'malformed compressed message' if message_set.size != uncompressed.size
+            messages.concat(message_set.messages)
+          else
+            # https://cwiki.apache.org/confluence/display/KAFKA/Compression
+            # claims that 2 is for Snappy compression, but Kafka's Scala client
+            # implementation doesn't seem to support it yet, so I don't have
+            # a reference implementation to test against.
+            raise "Unsupported Kafka compression codec: #{attributes & COMPRESSION_CODEC_MASK}"
+          end
+
+        else
+          raise "Unsupported Kafka message version: magic number #{magic}"
+        end
+
+        bytes_processed += message_size + 4 # 4 = sizeof(message_size)
+      end
+
+      MessageSet.new(bytes_processed, messages)
     end
   end
+
+  # Encapsulates a list of Kafka messages (as Kafka::Message objects in the
+  # +messages+ attribute) and their total serialized size in bytes (the +size+
+  # attribute).
+  class MessageSet < Struct.new(:size, :messages); end
 end

data/lib/kafka/producer.rb

@@ -17,45 +17,18 @@ module Kafka
 
     include Kafka::IO
 
-    PRODUCE_REQUEST_ID = Kafka::RequestType::PRODUCE
-
     attr_accessor :topic, :partition
 
     def initialize(options = {})
       self.topic     = options[:topic]      || "test"
       self.partition = options[:partition]  || 0
-      self.host      = options[:host]       || "localhost"
-      self.port      = options[:port]       || 9092
+      self.host      = options[:host]       || HOST
+      self.port      = options[:port]       || PORT
       self.connect(self.host, self.port)
     end
 
-    def encode(message)
-      if RUBY_VERSION[0,3] == "1.8"  # Use old iconv on Ruby 1.8 for encoding
-        ic = Iconv.new('UTF-8//IGNORE', 'UTF-8')
-        [message.magic].pack("C") + [message.calculate_checksum].pack("N") + ic.iconv(message.payload.to_s)
-      else
-        [message.magic].pack("C") + [message.calculate_checksum].pack("N") + message.payload.to_s.force_encoding(Encoding::ASCII_8BIT)
-      end
-    end
-
-    def encode_request(topic, partition, messages)
-      message_set = Array(messages).collect { |message|
-        encoded_message = self.encode(message)
-        [encoded_message.length].pack("N") + encoded_message
-      }.join("")
-
-      request   = [PRODUCE_REQUEST_ID].pack("n")
-      topic     = [topic.length].pack("n") + topic
-      partition = [partition].pack("N")
-      messages  = [message_set.length].pack("N") + message_set
-
-      data = request + topic + partition + messages
-
-      return [data.length].pack("N") + data
-    end
-
     def send(messages)
-      self.write(self.encode_request(self.topic, self.partition, messages))
+      self.write(Encoder.produce(self.topic, self.partition, messages))
     end
 
     def batch(&block)

data/lib/kafka.rb

@@ -20,10 +20,13 @@ end
 
 require File.join(File.dirname(__FILE__), "kafka", "io")
 require File.join(File.dirname(__FILE__), "kafka", "request_type")
+require File.join(File.dirname(__FILE__), "kafka", "encoder")
 require File.join(File.dirname(__FILE__), "kafka", "error_codes")
 require File.join(File.dirname(__FILE__), "kafka", "batch")
 require File.join(File.dirname(__FILE__), "kafka", "message")
+require File.join(File.dirname(__FILE__), "kafka", "multi_producer")
 require File.join(File.dirname(__FILE__), "kafka", "producer")
+require File.join(File.dirname(__FILE__), "kafka", "producer_request")
 require File.join(File.dirname(__FILE__), "kafka", "consumer")
 
 module Kafka

data/spec/consumer_spec.rb

@@ -25,7 +25,7 @@ describe Consumer do
   describe "Kafka Consumer" do
 
     it "should have a Kafka::RequestType::FETCH" do
-      Consumer::Kafka::RequestType::FETCH.should eql(1)
+      Kafka::RequestType::FETCH.should eql(1)
       @consumer.should respond_to(:request_type)
     end
 
@@ -91,39 +91,6 @@ describe Consumer do
       @consumer.send_consume_request.should eql(true)
     end
 
-    it "should parse a message set from bytes" do
-      bytes = [8].pack("N") + [0].pack("C") + [1120192889].pack("N") + "ale"
-      message = @consumer.parse_message_set_from(bytes).first
-      message.payload.should eql("ale")
-      message.checksum.should eql(1120192889)
-      message.magic.should eql(0)
-      message.valid?.should eql(true)
-    end
-
-    it "should skip an incomplete message at the end of the response" do
-      bytes = [8].pack("N") + [0].pack("C") + [1120192889].pack("N") + "ale"
-      # incomplete message
-      bytes += [8].pack("N")
-      messages = @consumer.parse_message_set_from(bytes)
-      messages.size.should eql(1)
-    end
-
-    it "should skip an incomplete message at the end of the response which has the same length as an empty message" do
-      bytes = [8].pack("N") + [0].pack("C") + [1120192889].pack("N") + "ale"
-      # incomplete message because payload is missing
-      bytes += [8].pack("N") + [0].pack("C") + [1120192889].pack("N")
-      messages = @consumer.parse_message_set_from(bytes)
-      messages.size.should eql(1)
-    end
-
-    it "should read empty messages correctly" do
-      # empty message
-      bytes = [5].pack("N") + [0].pack("C") + [0].pack("N") + ""
-      messages = @consumer.parse_message_set_from(bytes)
-      messages.size.should eql(1)
-      messages.first.payload.should eql("")
-    end
-
     it "should consume messages" do
       @consumer.should_receive(:send_consume_request).and_return(true)
       @consumer.should_receive(:read_data_response).and_return("")

data/spec/message_spec.rb

@@ -40,6 +40,10 @@ describe Message do
       @message.magic.should eql(1)
     end
 
+    it "should have an empty payload by default" do
+      @message.payload.should == ""
+    end
+
     it "should calculate the checksum (crc32 of a given message)" do
       @message.payload = "ale"
       @message.calculate_checksum.should eql(1120192889)
@@ -56,14 +60,92 @@ describe Message do
       @message = Message.new("alejandro", 0, 66666666) # 66666666 is a funny checksum
       @message.valid?.should eql(false)
     end
+  end
 
-    it "should parse a message from bytes" do
-      bytes = [12].pack("N") + [0].pack("C") + [1120192889].pack("N") + "ale"
-      message = Kafka::Message.parse_from(bytes)
+  describe "parsing" do
+    it "should parse a version-0 message from bytes" do
+      bytes = [8, 0, 1120192889, 'ale'].pack('NCNa*')
+      message = Kafka::Message.parse_from(bytes).messages.first
       message.valid?.should eql(true)
       message.magic.should eql(0)
       message.checksum.should eql(1120192889)
       message.payload.should eql("ale")
     end
+
+    it "should parse a version-1 message from bytes" do
+      bytes = [12, 1, 0, 755095536, 'martin'].pack('NCCNa*')
+      message = Kafka::Message.parse_from(bytes).messages.first
+      message.should be_valid
+      message.magic.should == 1
+      message.checksum.should == 755095536
+      message.payload.should == 'martin'
+    end
+
+    it "should raise an error if the magic number is not recognised" do
+      bytes = [12, 2, 0, 755095536, 'martin'].pack('NCCNa*') # 2 = some future format that's not yet invented
+      lambda {
+        Kafka::Message.parse_from(bytes)
+      }.should raise_error(RuntimeError, /Unsupported Kafka message version/)
+    end
+
+    it "should skip an incomplete message at the end of the response" do
+      bytes = [8, 0, 1120192889, 'ale'].pack('NCNa*')
+      bytes += [8].pack('N') # incomplete message (only length, rest is truncated)
+      message_set = Message.parse_from(bytes)
+      message_set.messages.size.should == 1
+      message_set.size.should == 12 # bytes consumed
+    end
+
+    it "should skip an incomplete message at the end of the response which has the same length as an empty message" do
+      bytes = [8, 0, 1120192889, 'ale'].pack('NCNa*')
+      bytes += [8, 0, 1120192889].pack('NCN') # incomplete message (payload is missing)
+      message_set = Message.parse_from(bytes)
+      message_set.messages.size.should == 1
+      message_set.size.should == 12 # bytes consumed
+    end
+
+    it "should read empty messages correctly" do
+      # empty message
+      bytes = [5, 0, 0, ''].pack('NCNa*')
+      messages = Message.parse_from(bytes).messages
+      messages.size.should == 1
+      messages.first.payload.should == ''
+    end
+
+    it "should parse a gzip-compressed message" do
+      compressed = 'H4sIAG0LI1AAA2NgYBBkZBB/9XN7YlJRYnJiCogCAH9lueQVAAAA'.unpack('m*').shift
+      bytes = [45, 1, 1, 1303540914, compressed].pack('NCCNa*')
+      message = Message.parse_from(bytes).messages.first
+      message.should be_valid
+      message.payload.should == 'abracadabra'
+    end
+
+    it "should recursively parse nested compressed messages" do
+      uncompressed = [17, 1, 0, 401275319, 'abracadabra'].pack('NCCNa*')
+      uncompressed << [12, 1, 0, 2666930069, 'foobar'].pack('NCCNa*')
+      compressed_io = StringIO.new('')
+      Zlib::GzipWriter.new(compressed_io).tap{|gzip| gzip << uncompressed; gzip.close }
+      compressed = compressed_io.string
+      bytes = [compressed.size + 6, 1, 1, Zlib.crc32(compressed), compressed].pack('NCCNa*')
+      messages = Message.parse_from(bytes).messages
+      messages.map(&:payload).should == ['abracadabra', 'foobar']
+      messages.map(&:valid?).should == [true, true]
+    end
+
+    it "should support a mixture of compressed and uncompressed messages" do
+      compressed = 'H4sIAG0LI1AAA2NgYBBkZBB/9XN7YlJRYnJiCogCAH9lueQVAAAA'.unpack('m*').shift
+      bytes = [45, 1, 1, 1303540914, compressed].pack('NCCNa*')
+      bytes << [11, 1, 0, 907060870, 'hello'].pack('NCCNa*')
+      messages = Message.parse_from(bytes).messages
+      messages.map(&:payload).should == ['abracadabra', 'hello']
+      messages.map(&:valid?).should == [true, true]
+    end
+
+    it "should raise an error if the compression codec is not supported" do
+      bytes = [6, 1, 2, 0, ''].pack('NCCNa*') # 2 = Snappy codec
+      lambda {
+        Kafka::Message.parse_from(bytes)
+      }.should raise_error(RuntimeError, /Unsupported Kafka compression codec/)
+    end
   end
 end

data/spec/producer_spec.rb

@@ -25,10 +25,6 @@ describe Producer do
   end
 
   describe "Kafka Producer" do
-    it "should have a PRODUCE_REQUEST_ID" do
-      Producer::PRODUCE_REQUEST_ID.should eql(0)
-    end
-
     it "should have a topic and a partition" do
       @producer.should respond_to(:topic)
       @producer.should respond_to(:partition)
@@ -47,60 +43,6 @@ describe Producer do
       @producer.host.should eql("localhost")
       @producer.port.should eql(9092)
     end
-
-    describe "Message Encoding" do
-      it "should encode a message" do
-        message = Kafka::Message.new("alejandro")
-        full_message = [message.magic].pack("C") + [message.calculate_checksum].pack("N") + message.payload
-        @producer.encode(message).should eql(full_message)
-      end
-      
-      it "should encode an empty message" do
-        message = Kafka::Message.new()
-        full_message = [message.magic].pack("C") + [message.calculate_checksum].pack("N") + message.payload.to_s
-        @producer.encode(message).should eql(full_message)
-      end
-
-      it "should encode strings containing non-ASCII characters" do
-        message = Kafka::Message.new("ümlaut")
-        encoded = @producer.encode(message)
-        data = [encoded.size].pack("N") + encoded
-        if RUBY_VERSION[0,3] == "1.8" # Use old iconv on Ruby 1.8 for encoding
-          ic = Iconv.new('UTF-8//IGNORE', 'UTF-8')
-          ic.iconv(Kafka::Message.parse_from(data).payload).should eql("ümlaut")
-        else
-          Kafka::Message.parse_from(data).payload.force_encoding(Encoding::UTF_8).should eql("ümlaut")
-        end
-      end
-    end
-
-    describe "Request Encoding" do
-      it "should binary encode an empty request" do
-        bytes = @producer.encode_request("test", 0, [])
-        bytes.length.should eql(20)
-        bytes.should eql("\000\000\000\020\000\000\000\004test\000\000\000\000\000\000\000\000")
-      end
-
-      it "should binary encode a request with a message, using a specific wire format" do
-        message = Kafka::Message.new("ale")
-        bytes = @producer.encode_request("test", 3, message)
-        data_size  = bytes[0, 4].unpack("N").shift
-        request_id = bytes[4, 2].unpack("n").shift
-        topic_length = bytes[6, 2].unpack("n").shift
-        topic = bytes[8, 4]
-        partition = bytes[12, 4].unpack("N").shift
-        messages_length = bytes[16, 4].unpack("N").shift
-        messages = bytes[20, messages_length]
-
-        bytes.length.should eql(32)
-        data_size.should eql(28)
-        request_id.should eql(0)
-        topic_length.should eql(4)
-        topic.should eql("test")
-        partition.should eql(3)
-        messages_length.should eql(12)
-      end
-    end
   end
 
   it "should send messages" do
@@ -120,4 +62,4 @@ describe Producer do
       end
     end
   end
-end
+end

metadata

@@ -1,47 +1,42 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: kafka-rb
-version: !ruby/object:Gem::Version 
-  hash: 15
+version: !ruby/object:Gem::Version
+  version: 0.0.9
   prerelease: 
-  segments: 
-  - 0
-  - 0
-  - 8
-  version: 0.0.8
 platform: ruby
-authors: 
+authors:
 - Alejandro Crosa
 - Stefan Mees
 - Tim Lossen
 autorequire: kafka-rb
 bindir: bin
 cert_chain: []
-
-date: 2012-01-14 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2012-09-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: rspec
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  version_requirements: *id001
-description: kafka-rb allows you to produce and consume messages using the Kafka distributed publish/subscribe messaging service.
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: kafka-rb allows you to produce and consume messages using the Kafka distributed
+  publish/subscribe messaging service.
 email: 
 executables: []
-
 extensions: []
-
-extra_rdoc_files: 
+extra_rdoc_files:
 - LICENSE
-files: 
+files:
 - LICENSE
 - README.md
 - Rakefile
@@ -60,38 +55,28 @@ files:
 - spec/message_spec.rb
 - spec/producer_spec.rb
 - spec/spec_helper.rb
-homepage: http://github.com/wooga/kafka-rb
+homepage: http://github.com/acrosa/kafka-rb
 licenses: []
-
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.8.11
+rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: A Ruby client for the Kafka distributed publish/subscribe messaging service
 test_files: []
-