em-kafka 0.0.1

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    NzM5ZGZjMjg0YzI4MTg4NGZiYzk2NTM2Y2Q3ODI5ZTFmYzQwMmMzMg==
+  data.tar.gz: !binary |-
+    ZjA2NDFhYWU1NDY0NTIzNmQ2MWY2YjJmZDI2YmViODAyZjNiYjBmMg==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    MWI4N2FiODA4ZjZhZDQ2YmJhZWUzZDY2NjA1YjY0NTRjN2Y2YjBiZGFiNGRk
+    ZDFmZDU2NzdlY2U0YzFmNWEzYjlhYjI0MTVkODE2YWQzOWIxOGZjN2U3NDFj
+    ZmM5ODMxZDE4OTU4ZmZmODBkODU1ZjhmNWI0ZDFjZmQ5YTgzMmU=
+  data.tar.gz: !binary |-
+    ZjQ1ZDlmNmI5MDljNTUwNGI5M2JhYzJiNmIzMjJkMGMyNTg1ODNiMDZjMjBj
+    MmQ1M2M3NjM1MjNlYmNhODU3NzhmMjc2YTUzNTg0NGU4NjA0ZjQ2ZWJmYTI3
+    NjhhNGM2OThjOGYxZWFhMDg5NWNmMGI4NzZiNjUwZWI2ODY2ZmE=

data/.gitignore

@@ -0,0 +1 @@
+.rvmrc

data/.rspec

@@ -0,0 +1 @@
+--color

data/Gemfile

@@ -0,0 +1,2 @@
+source "http://rubygems.org"
+gemspec

data/Gemfile.lock

@@ -0,0 +1,28 @@
+PATH
+  remote: .
+  specs:
+    em-kafka (0.0.1)
+      eventmachine (>= 1.0.0.beta.4)
+      yajl-ruby (>= 0.8.2)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    diff-lcs (1.1.3)
+    eventmachine (1.0.0.beta.4)
+    rspec (2.6.0)
+      rspec-core (~> 2.6.0)
+      rspec-expectations (~> 2.6.0)
+      rspec-mocks (~> 2.6.0)
+    rspec-core (2.6.4)
+    rspec-expectations (2.6.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.6.0)
+    yajl-ruby (1.1.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  em-kafka!
+  rspec (~> 2.6.0)

data/LICENSE

@@ -0,0 +1,22 @@
+(The MIT-License)
+
+Copyright (c) 2011 GroupMe
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,42 @@
+# EM-Kafka
+
+EventMachine driver for [Kafka](http://incubator.apache.org/kafka/index.html).
+
+## Producer
+
+When using Ruby objects, the payload is encoded to JSON
+
+    producer = EM::Kafka::Producer.new("kafka://topic@localhost:9092/0")
+    producer.deliver(:foo => "bar") # payload is {foo:"bar"}
+
+## Consumer
+
+    consumer = EM::Kafka::Consumer.new("kafka://topic@localhost:9092/0")
+    consumer.consume do |message|
+      puts message.payload
+    end    
+
+## Messages
+
+Messages are composed of:
+
+* a payload
+* a magic id (defaults to 0)
+
+Change the magic id when the payload format changes:
+
+    EM::Kafka::Message.new("payload", 2)
+    
+Pass messages when you want to be specific:
+
+    message_1 = EM::Kafka::Message.new("payload_1", 2)
+    message_2 = EM::Kafka::Message.new("payload_2", 2)
+    producer.deliver([message_1, message_2])
+  
+    
+## Credits
+
+Heavily influenced by / borrowed from:
+
+* kafka-rb (Alejandro Crosa)
+* em-hiredis (Martyn Loughran)

data/Rakefile

@@ -0,0 +1,4 @@
+require 'bundler/gem_tasks'
+require "rspec/core/rake_task"
+RSpec::Core::RakeTask.new(:core)
+task :default => :core

data/bin/consume

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+require_relative "../lib/em-kafka"
+
+topic = ARGV[0] || "test"
+
+EM.run do
+  trap("TERM") { EM.stop; exit; }
+  consumer = EM::Kafka::Consumer.new(:topic => topic)
+  puts "consuming topic #{consumer.topic}"
+  consumer.consume do |message|
+    puts "payload: #{message.payload}"
+  end
+end
+  

data/bin/produce

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+require_relative "../lib/em-kafka"
+
+topic = ARGV[0] || "test"
+puts "Producing topic '#{topic}'"
+
+EM.run do
+  trap("TERM") { EM.stop; exit; }
+  producer = EM::Kafka::Producer.new(:topic => topic)
+  puts "topic is #{producer.topic}"
+  
+  EM.add_periodic_timer(0.25) {
+    message = EM::Kafka::Message.new("hello-#{Time.now.to_i}")
+    producer.deliver(message)
+    puts "Sending #{message.payload}"
+  }
+end

data/em-kafka.gemspec

@@ -0,0 +1,21 @@
+# -*- mode: ruby; encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "em-kafka/version"
+
+Gem::Specification.new do |s|
+  s.name        = "em-kafka"
+  s.version     = EventMachine::Kafka::VERSION
+  s.authors     = ["Brandon Keene"]
+  s.email       = ["bkeene@gmail.com"]
+  s.homepage    = ""
+  s.summary     = %q{EventMachine Kafka driver}
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency "eventmachine-le", ">= 1.1.5"
+  s.add_dependency "yajl-ruby",    ">= 0.8.2"
+  s.add_development_dependency "rspec", "~> 2.6.0"
+end

data/lib/em-kafka/client.rb

@@ -0,0 +1,32 @@
+module EventMachine
+  module Kafka
+    class Client
+      def initialize(host, port)
+        @host = host || 'localhost'
+        @port = port || 9092
+        @callback = nil
+      end
+
+      def send_data(data)
+        connect if @connection.nil? || @connection.disconnected?
+        @connection.send_data(data)
+      end
+
+      def on_data(&block)
+        @callback = block
+      end
+
+      def connect
+        @connection = EM.connect(@host, @port, EM::Kafka::Connection)
+        @connection.on(:message) do |message|
+          @callback.call(message) if @callback
+        end
+        @connection
+      end
+
+      def close_connection
+        @connection.close_connection_after_writing
+      end
+    end
+  end
+end

data/lib/em-kafka/connection.rb

@@ -0,0 +1,29 @@
+module EventMachine::Kafka
+  class Connection < EM::Connection
+    include EventMachine::Kafka::EventEmitter
+
+    def initialize(*args)
+      super
+      @disconnected = false
+    end
+
+    def disconnected?
+      @disconnected
+    end
+
+    def connection_completed
+      EventMachine::Kafka.logger.info("Connected to Kafka")
+      emit(:connected)
+    end
+
+    def receive_data(data)
+      emit(:message, data)
+    end
+
+    def unbind
+      @disconnected = true
+      EventMachine::Kafka.logger.info("Disconnected from Kafka")
+      emit(:closed)
+    end
+  end
+end

data/lib/em-kafka/consumer.rb

@@ -0,0 +1,49 @@
+module EventMachine
+  module Kafka
+    class Consumer
+      require_relative "consumer_request"
+      require_relative "parser"
+
+      attr_accessor :topic,
+                    :partition,
+                    :offset,
+                    :max_size,
+                    :request_type,
+                    :polling,
+                    :client,
+                    :host,
+                    :port
+
+      def initialize(uri, options = {})
+        uri = URI(uri)
+        self.host = uri.host
+        self.port = uri.port
+        self.topic = uri.user
+        self.partition = uri.path[1..-1].to_i
+
+        self.offset       = options[:offset]       || 0
+        self.max_size     = options[:max_size]     || EM::Kafka::MESSAGE_MAX_SIZE
+        self.request_type = options[:request_type] || EM::Kafka::REQUEST_FETCH
+        self.polling      = options[:polling]      || EM::Kafka::CONSUMER_POLLING_INTERVAL
+        self.client = EM::Kafka::Client.new(host, port)
+        client.connect
+      end
+
+      def consume(&block)
+        raise ArgumentError.new("block required") unless block_given?
+        parser = EM::Kafka::Parser.new(offset, &block)
+        parser.on_offset_update { |i| self.offset = i }
+        client.on_data { |binary| parser.on_data(binary) }
+        EM.add_periodic_timer(polling) { request_consume }
+      end
+
+      private
+
+      def request_consume
+        request = EM::Kafka::ConsumerRequest.new(request_type, topic, partition, offset, max_size)
+        client.send_data(request.encode_size)
+        client.send_data(request.encode)
+      end
+    end
+  end
+end

data/lib/em-kafka/consumer_request.rb

@@ -0,0 +1,23 @@
+module EventMachine
+  module Kafka
+    class ConsumerRequest
+      def initialize(type, topic, partition, offset, max_size)
+        @type, @topic, @partition, @offset, @max_size =
+          type, topic, partition, offset, max_size
+      end
+
+      def encode_size
+        [2 + 2 + @topic.length + 4 + 8 + 4].pack("N")
+      end
+
+      def encode
+        [@type].pack("n") +
+        [@topic.length].pack("n") +
+        @topic +
+        [@partition].pack("N") +
+        [@offset].pack("Q").reverse + # DIY 64bit big endian integer
+        [@max_size].pack("N")
+      end
+    end
+  end
+end

data/lib/em-kafka/event_emitter.rb

@@ -0,0 +1,29 @@
+module EventMachine::Kafka
+  module EventEmitter
+    def on(event, &listener)
+      _listeners[event] << listener
+    end
+
+    def emit(event, *args)
+      _listeners[event].each { |l| l.call(*args) }
+    end
+
+    def remove_listener(event, &listener)
+      _listeners[event].delete(listener)
+    end
+
+    def remove_all_listeners(event)
+      _listeners.delete(event)
+    end
+
+    def listeners(event)
+      _listeners[event]
+    end
+
+    private
+
+    def _listeners
+      @_listeners ||= Hash.new { |h,k| h[k] = [] }
+    end
+  end
+end

data/lib/em-kafka/message.rb

@@ -0,0 +1,34 @@
+module EventMachine
+  module Kafka
+    # 1 byte "magic" identifier to allow format changes
+    # 4 byte CRC32 of the payload
+    # N - 5 byte payload
+    class Message
+      require "zlib"
+      attr_accessor :magic, :checksum, :payload, :size
+
+      def initialize(payload, magic = 0, checksum = nil, size = nil)
+        self.payload  = payload
+        self.magic    = magic
+        self.checksum = checksum || Zlib.crc32(payload)
+      end
+
+      def valid?
+        checksum == Zlib.crc32(payload)
+      end
+
+      def encode
+        [magic, checksum].pack("CN") +
+        payload.to_s.force_encoding(Encoding::ASCII_8BIT)
+      end
+
+      def self.decode(size, binary)
+        return unless binary
+        magic    = binary[4].unpack("C").shift
+        checksum = binary[5..9].unpack("N").shift
+        payload  = binary[9..-1]
+        new(payload, magic, checksum)
+      end
+    end
+  end
+end

data/lib/em-kafka/parser.rb

@@ -0,0 +1,66 @@
+module EventMachine
+  module Kafka
+    class Parser
+      attr_accessor :offset
+
+      def initialize(offset = 0, &block)
+        self.offset = offset
+        @block = block
+        reset
+      end
+
+      def on_data(binary)
+        if @complete
+          parsed_size = binary[0, 4].unpack("N").shift
+
+          if (parsed_size - 2) > 0
+            @size = parsed_size
+          else
+            # empty response
+            return
+          end
+        end
+
+        @buffer << binary
+
+        received_data = @buffer.size + binary.size
+        if received_data >= @size
+          parse(@buffer[6, @size]) # account for 4 byte size and 2 byte junk
+        else
+          @complete = false
+        end
+      end
+
+      def on_offset_update(&callback)
+        @on_offset_update_callback = callback
+      end
+
+      private
+
+      def parse(frame)
+        i = 0
+        while i <= frame.length do
+          break unless message_size = frame[i, 4].unpack("N").first
+          message_data = frame[i, message_size + 4]
+          message = Kafka::Message.decode(message_size, message_data)
+          i += message_size + 4
+          @block.call(message)
+        end
+
+        advance_offset(i)
+        reset
+      end
+
+      def reset
+        @size = 0
+        @complete = true
+        @buffer = ""
+      end
+
+      def advance_offset(i)
+        self.offset += i
+        @on_offset_update_callback.call(offset) if @on_offset_update_callback
+      end
+    end
+  end
+end

data/lib/em-kafka/producer.rb

@@ -0,0 +1,26 @@
+module EventMachine
+  module Kafka
+    class Producer
+      require_relative "producer_request"
+      attr_accessor :host, :port, :topic, :partition, :client
+
+      def initialize(uri)
+        uri = URI(uri)
+        self.host = uri.host
+        self.port = uri.port
+        self.topic = uri.user
+        self.partition = uri.path[1..-1].to_i
+
+        raise ArgumentError("topic required") unless topic
+
+        self.client = EM::Kafka::Client.new(host, port)
+        client.connect
+      end
+
+      def deliver(message)
+        request = EM::Kafka::ProducerRequest.new(topic, partition, message)
+        client.send_data(request.encode)
+      end
+    end
+  end
+end

data/lib/em-kafka/producer_request.rb

@@ -0,0 +1,38 @@
+module EventMachine
+  module Kafka
+    class ProducerRequest
+      def initialize(topic, partition, messages)
+        @topic, @partition, @messages = topic, partition, messages
+      end
+
+      def encode
+        data = "\x00\x00" +
+               [@topic.length].pack("n") +
+               @topic +
+               [@partition].pack("N") +
+               encode_messages(@messages)
+
+        [data.length].pack("N") + data
+      end
+
+      private
+
+      def encode_messages(messages)
+        messages = [messages].flatten
+        messages = messages.map do |m|
+          if m.is_a?(EM::Kafka::Message)
+            m
+          else
+            EM::Kafka::Message.new(Yajl::Encoder.encode(m))
+          end
+        end
+
+        message_set = messages.map { |m|
+          data = m.encode
+          [data.length].pack("N") + data
+        }.join("")
+        [message_set.length].pack("N") + message_set
+      end
+    end
+  end
+end

data/lib/em-kafka/version.rb

@@ -0,0 +1,5 @@
+module EventMachine
+  module Kafka
+    VERSION = "0.0.1"
+  end
+end

data/lib/em-kafka.rb

@@ -0,0 +1,48 @@
+require "eventmachine"
+require "logger"
+require "uri"
+require "yajl"
+
+require_relative "em-kafka/event_emitter"
+require_relative "em-kafka/connection"
+require_relative "em-kafka/client"
+require_relative "em-kafka/message"
+require_relative "em-kafka/producer"
+require_relative "em-kafka/consumer"
+
+module EventMachine
+  module Kafka
+    MESSAGE_MAX_SIZE = 1048576 # 1 MB
+    CONSUMER_POLLING_INTERVAL = 2 # 2 seconds
+
+    REQUEST_PRODUCE      = 0
+    REQUEST_FETCH        = 1
+    REQUEST_MULTIFETCH   = 2
+    REQUEST_MULTIPRODUCE = 3
+    REQUEST_OFFSETS      = 4
+
+    ERROR_NO_ERROR                = 0
+    ERROR_OFFSET_OUT_OF_RANGE     = 1
+    ERROR_INVALID_MESSAGE_CODE    = 2
+    ERROR_WRONG_PARTITION_CODE    = 3
+    ERROR_INVALID_RETCH_SIZE_CODE = 4
+
+    ERROR_DESCRIPTIONS = {
+      ERROR_NO_ERROR                => 'No error',
+      ERROR_INVALID_MESSAGE_CODE    => 'Offset out of range',
+      ERROR_INVALID_MESSAGE_CODE    => 'Invalid message code',
+      ERROR_WRONG_PARTITION_CODE    => 'Wrong partition code',
+      ERROR_INVALID_RETCH_SIZE_CODE => 'Invalid retch size code'
+    }
+
+    class << self
+      def logger
+        @logger ||= Logger.new(STDOUT)
+      end
+
+      def logger=(new_logger)
+        @logger = new_logger
+      end
+    end
+  end
+end

data/spec/em-kafka/consumer_request_spec.rb

@@ -0,0 +1,32 @@
+require 'spec_helper'
+
+describe EM::Kafka::ConsumerRequest do
+  before do
+    @request = EM::Kafka::ConsumerRequest.new(
+      EM::Kafka::REQUEST_FETCH,
+      "topic",
+      0,
+      100,
+      EM::Kafka::MESSAGE_MAX_SIZE
+    )
+  end
+
+  describe "#encode" do
+    it "returns binary" do
+      data = [EM::Kafka::REQUEST_FETCH].pack("n") +
+             ["topic".length].pack('n') +
+             "topic" +
+             [0].pack("N") +
+             [100].pack("Q").reverse + # DIY 64bit big endian integer
+             [EM::Kafka::MESSAGE_MAX_SIZE].pack("N")
+
+      @request.encode.should == data
+    end
+  end
+
+  describe "#encode_size" do
+    it "returns packed 2 + 2 + @topic.length + 4 + 8 + 4" do
+      @request.encode_size.should == "\x00\x00\x00\x19"
+    end
+  end
+end

data/spec/em-kafka/consumer_spec.rb

@@ -0,0 +1,24 @@
+require 'spec_helper'
+
+describe EM::Kafka::Consumer do
+  before do
+    @client = mock("Client", :connect => true)
+    EM::Kafka::Client.should_receive(:new).and_return(@client)
+  end
+
+  it "should set a topic and partition on initialize" do
+    consumer = EM::Kafka::Consumer.new("kafka://testing@localhost:9092/3")
+    consumer.host.should == "localhost"
+    consumer.port.should == 9092
+    consumer.topic.should == "testing"
+    consumer.partition.should == 3
+  end
+
+  it "should set default partition to 0" do
+    consumer = EM::Kafka::Consumer.new("kafka://testing@localhost:9092")
+    consumer.host.should == "localhost"
+    consumer.port.should == 9092
+    consumer.topic.should == "testing"
+    consumer.partition.should == 0
+  end
+end

data/spec/em-kafka/message_spec.rb

@@ -0,0 +1,30 @@
+require 'spec_helper'
+
+describe EM::Kafka::Message do
+  describe "#encode" do
+    it "turns Message into data" do
+      message = EM::Kafka::Message.new("ale")
+      message.payload.should == "ale"
+      message.checksum.should == 1120192889
+      message.magic.should == 0
+
+      message.encode.should == [0].pack("C") +
+                               [1120192889].pack("N") +
+                                "ale".force_encoding(Encoding::ASCII_8BIT)
+    end
+  end
+
+  describe ".decode" do
+    it "turns data into a Message" do
+      data = [12].pack("N") +
+             [0].pack("C") +
+             [1120192889].pack("N") + "ale"
+
+      message = EM::Kafka::Message.decode(data.size, data)
+      message.should be_valid
+      message.payload.should == "ale"
+      message.checksum.should == 1120192889
+      message.magic.should == 0
+    end
+  end
+end

data/spec/em-kafka/parser_spec.rb

@@ -0,0 +1,86 @@
+require 'spec_helper'
+
+describe EM::Kafka::Parser do
+  describe "parse" do
+    it "parses messages from newline boundaries across packets" do
+      messages = []
+      parser = EM::Kafka::Parser.new do |message|
+        messages << message
+      end
+
+      message_1 = EM::Kafka::Message.new("foo").encode
+      message_2 = EM::Kafka::Message.new("barizzle").encode
+      message_3 = EM::Kafka::Message.new("langlang").encode
+
+      binary = [51].pack("N") +
+               [0, 0].pack("CC") + # 2 byte offset
+               [message_1.size].pack("N") +
+               message_1 +
+               [message_2.size].pack("N") +
+               message_2 +
+               [message_3.size].pack("N") +
+               message_3
+
+      frame_1 = binary[0..11]
+      frame_2 = binary[12..-1]
+
+      parser.on_data(frame_1)
+      parser.on_data(frame_2)
+
+      messages[0].payload.should == "foo"
+      messages[0].should be_valid
+      messages[1].payload.should == "barizzle"
+      messages[1].should be_valid
+      messages[2].payload.should == "langlang"
+      messages[2].should be_valid
+
+      empty_frame = [2, 0, 0].pack("NCC")
+      parser.on_data(empty_frame)
+
+      message_4 = EM::Kafka::Message.new("after empty").encode
+
+      binary = [26].pack("N") +
+               [0, 0].pack("CC") + # 2 byte offset
+               [message_4.size].pack("N") +
+               message_4
+
+      frame_3 = binary
+      parser.on_data(frame_3)
+
+      messages[3].payload.should == "after empty"
+      messages[3].should be_valid
+    end
+  end
+
+  describe "on_offset_update" do
+    it "returns the proper offset" do
+      offset = 0
+      messages = []
+      parser = EM::Kafka::Parser.new do |message|
+        messages << message
+      end
+      parser.on_offset_update {|new_offset| offset = new_offset }
+
+      message_1 = EM::Kafka::Message.new("foo").encode
+      message_2 = EM::Kafka::Message.new("barizzle").encode
+      message_3 = EM::Kafka::Message.new("langlang").encode
+
+      binary = [51].pack("N") +
+               [0, 0].pack("CC") + # 2 byte offset
+               [message_1.size].pack("N") +
+               message_1 +
+               [message_2.size].pack("N") +
+               message_2 +
+               [message_3.size].pack("N") +
+               message_3
+
+      frame_1 = binary[0..11]
+      frame_2 = binary[12..-1]
+
+      parser.on_data(frame_1)
+      parser.on_data(frame_2)
+
+      offset.should == message_1.size + message_2.size + message_3.size + 4 + 4 + 4
+    end
+  end
+end

data/spec/em-kafka/producer_request_spec.rb

@@ -0,0 +1,41 @@
+require 'spec_helper'
+
+describe EM::Kafka::ProducerRequest do
+  describe "#encode" do
+    it "binary encodes an empty request" do
+      bytes = EM::Kafka::ProducerRequest.new("test", 0, []).encode
+      bytes.length.should eql(20)
+      bytes.should eql("\000\000\000\020\000\000\000\004test\000\000\000\000\000\000\000\000")
+    end
+
+    it "should binary encode a request with a message, using a specific wire format" do
+      request = EM::Kafka::ProducerRequest.new("test", 3, EM::Kafka::Message.new("ale"))
+      bytes = request.encode
+
+      data_size  = bytes[0, 4].unpack("N").shift
+      request_id = bytes[4, 2].unpack("n").shift
+      topic_length = bytes[6, 2].unpack("n").shift
+      topic = bytes[8, 4]
+      partition = bytes[12, 4].unpack("N").shift
+      messages_length = bytes[16, 4].unpack("N").shift
+      messages = bytes[20, messages_length]
+
+      bytes.length.should eql(32)
+      data_size.should eql(28)
+      request_id.should eql(0)
+      topic_length.should eql(4)
+      topic.should eql("test")
+      partition.should eql(3)
+      messages_length.should eql(12)
+    end
+
+    it "encodes ruby objects to JSON and inflates message" do
+      message = EM::Kafka::Message.new(Yajl::Encoder.encode(key: "value"))
+      request_with_message = EM::Kafka::ProducerRequest.new("test", 3, message)
+      request_with_message.encode.size.should == 44
+
+      request = EM::Kafka::ProducerRequest.new("test", 3, key: "value")
+      request.encode.size.should == 44
+    end
+  end
+end

data/spec/em-kafka/producer_spec.rb

@@ -0,0 +1,34 @@
+require "spec_helper"
+
+describe EM::Kafka::Producer do
+  before do
+    @client = mock("Client", :connect => true)
+    EM::Kafka::Client.should_receive(:new).and_return(@client)
+  end
+
+  it "should set a topic and partition on initialize" do
+    producer = EM::Kafka::Producer.new("kafka://testing@localhost:9092/3")
+    producer.host.should == "localhost"
+    producer.port.should == 9092
+    producer.topic.should == "testing"
+    producer.partition.should == 3
+  end
+
+  it "should set default partition to 0" do
+    producer = EM::Kafka::Producer.new("kafka://testing@localhost:9092")
+    producer.host.should == "localhost"
+    producer.port.should == 9092
+    producer.topic.should == "testing"
+    producer.partition.should == 0
+  end
+
+  it "should send messages" do
+    producer = EM::Kafka::Producer.new("kafka://testing@localhost:9092/3")
+    message = EM::Kafka::Message.new("hello world")
+    request = EM::Kafka::ProducerRequest.new("testing", 3, message)
+
+    @client.should_receive(:send_data).with(request.encode)
+
+    producer.deliver(message)
+  end
+end

data/spec/em-kafka_spec.rb

@@ -0,0 +1,11 @@
+require "spec_helper"
+
+describe EventMachine::Kafka do
+  describe ".logger" do
+    it "sets logger" do
+      new_logger = Logger.new(STDOUT)
+      EM::Kafka.logger = new_logger
+      EM::Kafka.logger.should == new_logger
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,10 @@
+require "bundler/setup"
+Bundler.require :default, :development
+
+require 'em-kafka'
+
+RSpec.configure do |config|
+  config.before(:each) do
+    EM::Kafka.logger = Logger.new("/dev/null")
+  end
+end

metadata

@@ -0,0 +1,117 @@
+--- !ruby/object:Gem::Specification
+name: em-kafka
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Brandon Keene
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-08-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: eventmachine-le
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.1.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.1.5
+- !ruby/object:Gem::Dependency
+  name: yajl-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.8.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.8.2
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.6.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.6.0
+description: 
+email:
+- bkeene@gmail.com
+executables:
+- consume
+- produce
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- bin/consume
+- bin/produce
+- em-kafka.gemspec
+- lib/em-kafka.rb
+- lib/em-kafka/client.rb
+- lib/em-kafka/connection.rb
+- lib/em-kafka/consumer.rb
+- lib/em-kafka/consumer_request.rb
+- lib/em-kafka/event_emitter.rb
+- lib/em-kafka/message.rb
+- lib/em-kafka/parser.rb
+- lib/em-kafka/producer.rb
+- lib/em-kafka/producer_request.rb
+- lib/em-kafka/version.rb
+- spec/em-kafka/consumer_request_spec.rb
+- spec/em-kafka/consumer_spec.rb
+- spec/em-kafka/message_spec.rb
+- spec/em-kafka/parser_spec.rb
+- spec/em-kafka/producer_request_spec.rb
+- spec/em-kafka/producer_spec.rb
+- spec/em-kafka_spec.rb
+- spec/spec_helper.rb
+homepage: ''
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: EventMachine Kafka driver
+test_files: []
+has_rdoc: