leffen-kafka-rb 0.0.15

data/lib/kafka/multi_producer.rb

@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+module Kafka
+  class MultiProducer
+    include Kafka::IO
+
+    def initialize(options={})
+      self.host = options[:host] || HOST
+      self.port = options[:port] || PORT
+      self.compression = options[:compression] || Message::NO_COMPRESSION
+      self.connect(self.host, self.port)
+    end
+
+    def push(topic, messages, options={})
+      partition = options[:partition] || 0
+      self.write(Encoder.produce(topic, partition, messages, compression))
+    end
+
+    def multi_push(producer_requests)
+      self.write(Encoder.multiproduce(producer_requests, compression))
+    end
+  end
+end

data/lib/kafka/producer.rb

@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+# 
+#    http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+module Kafka
+  class Producer
+
+    include Kafka::IO
+
+    attr_accessor :topic, :partition
+
+    def initialize(options = {})
+      self.topic       = options[:topic]      || "test"
+      self.partition   = options[:partition]  || 0
+      self.host        = options[:host]       || HOST
+      self.port        = options[:port]       || PORT
+      self.compression = options[:compression] || Message::NO_COMPRESSION
+      self.connect(self.host, self.port)
+    end
+
+    def push(messages)
+      self.write(Encoder.produce(self.topic, self.partition, messages, compression))
+    end
+
+    def batch(&block)
+      batch = Kafka::Batch.new
+      block.call( batch )
+      push(batch.messages)
+      batch.messages.clear
+    end
+  end
+end

data/lib/kafka/producer_request.rb

@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Kafka
+  class ProducerRequest
+    attr_accessor :topic, :messages, :partition
+
+    def initialize(topic, messages, options={})
+      self.topic = topic
+      self.partition = options[:partition] || 0
+      self.messages = Array(messages)
+    end
+  end
+end

data/lib/kafka/request_type.rb

@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+# 
+#    http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+module Kafka
+  module RequestType
+    PRODUCE      = 0
+    FETCH        = 1
+    MULTIFETCH   = 2
+    MULTIPRODUCE = 3
+    OFFSETS      = 4
+  end
+end

data/lib/leffen-kafka.rb

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require File.join(File.dirname(__FILE__), "kafka")

data/spec/batch_spec.rb

@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+# 
+#    http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe Batch do
+
+  before(:each) do
+    @batch = Batch.new
+  end
+
+  describe "batch messages" do
+    it "holds all messages to be sent" do
+      @batch.should respond_to(:messages)
+      @batch.messages.class.should eql(Array)
+    end
+
+    it "supports queueing/adding messages to be send" do
+      @batch.messages << mock(Kafka::Message.new("one"))
+      @batch.messages << mock(Kafka::Message.new("two"))
+      @batch.messages.length.should eql(2)
+    end
+  end
+end

data/spec/cli_spec.rb

@@ -0,0 +1,133 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+require File.dirname(__FILE__) + '/spec_helper'
+require 'kafka/cli'
+
+describe CLI do
+
+  before(:each) do
+    CLI.instance_variable_set("@config", {})
+    CLI.stub(:puts)
+  end
+
+  describe "should read from env" do
+    describe "kafka host" do
+      it "should read KAFKA_HOST from env" do
+        CLI.read_env("KAFKA_HOST" => "google.com")
+        CLI.config[:host].should == "google.com"
+      end
+
+      it "kafka port" do
+        CLI.read_env("KAFKA_PORT" => "1234")
+        CLI.config[:port].should == 1234
+      end
+
+      it "kafka topic" do
+        CLI.read_env("KAFKA_TOPIC" => "news")
+        CLI.config[:topic].should == "news"
+      end
+
+      it "kafka compression" do
+        CLI.read_env("KAFKA_COMPRESSION" => "no")
+        CLI.config[:compression].should == Message::NO_COMPRESSION
+
+        CLI.read_env("KAFKA_COMPRESSION" => "gzip")
+        CLI.config[:compression].should == Message::GZIP_COMPRESSION
+
+        CLI.read_env("KAFKA_COMPRESSION" => "snappy")
+        CLI.config[:compression].should == Message::SNAPPY_COMPRESSION
+      end
+    end
+  end
+
+  describe "should read from command line" do
+    it "kafka host" do
+      CLI.parse_args(%w(--host google.com))
+      CLI.config[:host].should == "google.com"
+
+      CLI.parse_args(%w(-h google.com))
+      CLI.config[:host].should == "google.com"
+    end
+
+    it "kafka port" do
+      CLI.parse_args(%w(--port 1234))
+      CLI.config[:port].should == 1234
+
+      CLI.parse_args(%w(-p 1234))
+      CLI.config[:port].should == 1234
+    end
+
+    it "kafka topic" do
+      CLI.parse_args(%w(--topic news))
+      CLI.config[:topic].should == "news"
+
+      CLI.parse_args(%w(-t news))
+      CLI.config[:topic].should == "news"
+    end
+
+    it "kafka compression" do
+      CLI.stub(:publish? => true)
+
+      CLI.parse_args(%w(--compression no))
+      CLI.config[:compression].should == Message::NO_COMPRESSION
+      CLI.parse_args(%w(-c no))
+      CLI.config[:compression].should == Message::NO_COMPRESSION
+
+      CLI.parse_args(%w(--compression gzip))
+      CLI.config[:compression].should == Message::GZIP_COMPRESSION
+      CLI.parse_args(%w(-c gzip))
+      CLI.config[:compression].should == Message::GZIP_COMPRESSION
+
+      CLI.parse_args(%w(--compression snappy))
+      CLI.config[:compression].should == Message::SNAPPY_COMPRESSION
+      CLI.parse_args(%w(-c snappy))
+      CLI.config[:compression].should == Message::SNAPPY_COMPRESSION
+    end
+
+    it "message" do
+      CLI.stub(:publish? => true)
+      CLI.parse_args(%w(--message YEAH))
+      CLI.config[:message].should == "YEAH"
+
+      CLI.parse_args(%w(-m YEAH))
+      CLI.config[:message].should == "YEAH"
+    end
+
+  end
+
+  describe "config validation" do
+    it "should assign a default port" do
+      CLI.stub(:exit)
+      CLI.stub(:puts)
+      CLI.validate_config
+      CLI.config[:port].should == Kafka::IO::PORT
+    end
+  end
+
+  it "should assign a default host" do
+    CLI.stub(:exit)
+    CLI.validate_config
+    CLI.config[:host].should == Kafka::IO::HOST
+  end
+
+
+  it "read compression method" do
+    CLI.string_to_compression("no").should == Message::NO_COMPRESSION
+    CLI.string_to_compression("gzip").should == Message::GZIP_COMPRESSION
+    CLI.string_to_compression("snappy").should == Message::SNAPPY_COMPRESSION
+    lambda { CLI.push(:string_to_compression,nil) }.should raise_error
+  end
+
+end

data/spec/consumer_spec.rb

@@ -0,0 +1,146 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+# 
+#    http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe Consumer do
+
+  before(:each) do
+    @mocked_socket = mock(TCPSocket)
+    TCPSocket.stub!(:new).and_return(@mocked_socket) # don't use a real socket
+    @consumer = Consumer.new(:offset => 0)
+  end
+
+  describe "Kafka Consumer" do
+
+    it "should have a Kafka::RequestType::FETCH" do
+      Kafka::RequestType::FETCH.should eql(1)
+      @consumer.should respond_to(:request_type)
+    end
+
+    it "should have a topic and a partition" do
+      @consumer.should respond_to(:topic)
+      @consumer.should respond_to(:partition)
+    end
+
+    it "should have a polling option, and a default value" do
+      Consumer::DEFAULT_POLLING_INTERVAL.should eql(2)
+      @consumer.should respond_to(:polling)
+      @consumer.polling.should eql(2)
+    end
+
+    it "should set a topic and partition on initialize" do
+      @consumer = Consumer.new({ :host => "localhost", :port => 9092, :topic => "testing" })
+      @consumer.topic.should eql("testing")
+      @consumer.partition.should eql(0)
+      @consumer = Consumer.new({ :topic => "testing", :partition => 3 })
+      @consumer.partition.should eql(3)
+    end
+
+    it "should set default host and port if none is specified" do
+      @consumer = Consumer.new
+      @consumer.host.should eql("localhost")
+      @consumer.port.should eql(9092)
+    end
+
+    it "should not have a default offset but be able to set it" do
+      @consumer = Consumer.new
+      @consumer.offset.should be_nil
+      @consumer = Consumer.new({ :offset => 1111 })
+      @consumer.offset.should eql(1111)
+    end
+
+    it "should have a max size" do
+      Consumer::MAX_SIZE.should eql(1048576)
+      @consumer.max_size.should eql(1048576)
+    end
+
+    it "should return the size of the request" do
+      @consumer.topic = "someothertopicname"
+      @consumer.encoded_request_size.should eql([38].pack("N"))
+    end
+
+    it "should encode a request to consume" do
+      bytes = [Kafka::RequestType::FETCH].pack("n") + ["test".length].pack("n") + "test" + [0].pack("N") + [0].pack("q").reverse + [Kafka::Consumer::MAX_SIZE].pack("N")
+      @consumer.encode_request(Kafka::RequestType::FETCH, "test", 0, 0, Kafka::Consumer::MAX_SIZE).should eql(bytes)
+    end
+
+    it "should read the response data" do
+      bytes = [0].pack("n") + [1120192889].pack("N") + "ale"
+      @mocked_socket.should_receive(:read).and_return([9].pack("N"))
+      @mocked_socket.should_receive(:read).with(9).and_return(bytes)
+      @consumer.read_data_response.should eql(bytes[2,7])
+    end
+
+    it "should send a consumer request" do
+      @consumer.stub!(:encoded_request_size).and_return(666)
+      @consumer.stub!(:encode_request).and_return("someencodedrequest")
+      @consumer.should_receive(:write).with("someencodedrequest").exactly(:once).and_return(true)
+      @consumer.should_receive(:write).with(666).exactly(:once).and_return(true)
+      @consumer.send_consume_request.should eql(true)
+    end
+
+    it "should consume messages" do
+      @consumer.should_receive(:send_consume_request).and_return(true)
+      @consumer.should_receive(:read_data_response).and_return("")
+      @consumer.consume.should eql([])
+    end
+
+    it "should loop and execute a block with the consumed messages" do
+      @consumer.stub!(:consume).and_return([mock(Kafka::Message)])
+      messages = []
+      messages.should_receive(:<<).exactly(:once).and_return([])
+      @consumer.loop do |message|
+        messages << message
+        break # we don't wanna loop forever on the test
+      end
+    end
+
+    it "should loop (every N seconds, configurable on polling attribute), and execute a block with the consumed messages" do
+      @consumer = Consumer.new({ :polling => 1 })
+      @consumer.stub!(:consume).and_return([mock(Kafka::Message)])
+      messages = []
+      messages.should_receive(:<<).exactly(:twice).and_return([])
+      executed_times = 0
+      @consumer.loop do |message|
+        messages << message
+        executed_times += 1
+        break if executed_times >= 2 # we don't wanna loop forever on the test, only 2 seconds
+      end
+
+      executed_times.should eql(2)
+    end
+
+    it "should fetch initial offset if no offset is given" do
+      @consumer = Consumer.new
+      @consumer.should_receive(:fetch_latest_offset).exactly(:once).and_return(1000)
+      @consumer.should_receive(:send_consume_request).and_return(true)
+      @consumer.should_receive(:read_data_response).and_return("")
+      @consumer.consume
+      @consumer.offset.should eql(1000)
+    end
+
+    it "should encode an offset request" do
+      bytes = [Kafka::RequestType::OFFSETS].pack("n") + ["test".length].pack("n") + "test" + [0].pack("N") + [-1].pack("q").reverse + [Kafka::Consumer::MAX_OFFSETS].pack("N")
+      @consumer.encode_request(Kafka::RequestType::OFFSETS, "test", 0, -1, Kafka::Consumer::MAX_OFFSETS).should eql(bytes)
+    end
+
+    it "should parse an offsets response" do
+      bytes = [0].pack("n") + [1].pack('N') + [21346].pack('q').reverse
+      @mocked_socket.should_receive(:read).and_return([14].pack("N"))
+      @mocked_socket.should_receive(:read).and_return(bytes)
+      @consumer.read_offsets_response.should eql(21346)
+    end
+  end
+end