poseidon 0.0.1

data/spec/integration/simple/simple_producer_and_consumer_spec.rb

@@ -0,0 +1,97 @@
+require 'integration/simple/spec_helper'
+
+describe "simple producer and consumer" do
+
+  describe "writing and consuming one topic" do
+    it "fetches produced messages" do
+      @producer = Producer.new(["localhost:9092"],
+                               "test_client",
+                               :type => :sync)
+
+
+      messages = [MessageToSend.new("topic_simple_producer_and_consumer", "Hello World")]
+      expect(@producer.send_messages(messages)).to eq(true)
+
+      @consumer = PartitionConsumer.new("test_consumer", "localhost", 9092,
+                                        "topic_simple_producer_and_consumer", 0, -2)
+      messages = @consumer.fetch
+      expect(messages.last.value).to eq("Hello World")
+
+      @producer.shutdown
+    end
+
+    it "fetches only messages since the last offset" do
+      @producer = Producer.new(["localhost:9092"],
+                               "test_client",
+                               :type => :sync,
+                               :required_acks => 1)
+
+      @consumer = PartitionConsumer.new("test_consumer", "localhost", 9092,
+                                        "topic_simple_producer_and_consumer", 0, -1)
+
+      # Read up to the end of the current messages
+      @consumer.fetch
+
+      # First Batch
+      messages = [MessageToSend.new("topic_simple_producer_and_consumer", "Hello World")]
+      expect(@producer.send_messages(messages)).to eq(true)
+
+      messages = @consumer.fetch
+      expect(messages.last.value).to eq("Hello World")
+
+      # Second Batch
+      messages = [MessageToSend.new("topic_simple_producer_and_consumer", "Hello World Again")]
+      expect(@producer.send_messages(messages)).to eq(true)
+
+      messages = @consumer.fetch
+      expect(messages.map(&:value)).to eq(["Hello World Again"])
+
+      # Empty Batch
+      messages = @consumer.fetch
+      expect(messages.empty?).to eq(true)
+    end
+
+    it "fetches larger messages with a larger max bytes size" do
+      @producer = Producer.new(["localhost:9092"],
+                                "test_client",
+                                :type => :sync,
+                                :required_acks => 1)
+
+      @consumer = PartitionConsumer.new("test_consumer", "localhost", 9092,
+                                        "topic_simple_producer_and_consumer", 0, -2)
+      
+      messages = []
+      2000.times do
+        messages << MessageToSend.new("topic_simple_producer_and_consumer",'KcjNyFBtqfSbpwjjcGKckMKLUCWz83IVcp21C8FQzs8JJKKTTrc4OLxSjLpYc5z7fsncX59te2cBn0sWDRaYmRuZyttRMLMHvXrM5o3QReKPIYUKzVCFahC4cb3Ivcbb5ZuS98Ohnb7Io42Bz9FucXwwGkQyFhJwyn3nD3BYs5r8TZM8Q76CGR2kTH1rjnFeB7J3hrRKukztxCrDY3smrQE1bbVR80IF3yWlhzkdfv3cpfwnD0TKadtt21sFJANFmORAJ0HKs6Z2262hcBQyF7WcWypC2RoLWVgKVQxbouVUP7yV6YYOAQEevYrl9sOB0Yi6h1mS8fTBUmRTmWLqyl8KzwbnbQvmCvgnX26F5JEzIoXsVaoDT2ks5eep9RyE1zm5yPtbYVmd2Sz7t5ru0wj6YiAmbF7Xgiw2l4VpNOxG0Ec6rFxXRXs0bahyBd2YtxpGyZBeruIK1RAN4n0t97xVXgZG5CGoVhL1oRDxw2pTbwEO1cvwHiiYXpXSqaxF7G9kiiPsQt24Vu7chXrJT7Xqv4RIg1aOT5Os5JVlISaJCmx8ZLtbC3OjAdGtF1ZkDuUeQHHohqeKh0qBJjw7Rv1oSDwcM0MRazjF36jijpYg26Qml9lSEnGYIFLQWHVDWKqqhl2GIntjxDXn1IyI')
+      end
+      expect(@producer.send_messages(messages)).to eq(true)
+      
+      messages = @consumer.fetch
+      expect(messages.length).to be > 2 
+      
+      @consumer = PartitionConsumer.new("test_consumer", "localhost", 9092,
+                                        "topic_simple_producer_and_consumer", 0, -2)
+      messages = @consumer.fetch(:max_bytes => 1400000)
+      expect(messages.length).to be > 2
+    end 
+  end
+
+  describe "broker that becomes unavailable" do
+    it "fails the fetch" do
+      @producer = Producer.new(["localhost:9092"],
+                               "test_client",
+                               :type => :sync)
+
+
+      messages = [MessageToSend.new("topic_simple_producer_and_consumer", "Hello World")]
+      expect(@producer.send_messages(messages)).to eq(true)
+
+      @consumer = PartitionConsumer.new("test_consumer", "localhost", 9092,
+                                        "topic_simple_producer_and_consumer", 0, -2)
+
+      $tc.broker.without_process do
+        expect { @consumer.fetch }.to raise_error(Connection::ConnectionFailedError)
+      end
+    end
+  end
+end

data/spec/integration/simple/spec_helper.rb

@@ -0,0 +1,17 @@
+require 'spec_helper'
+
+require 'test_cluster'
+
+RSpec.configure do |config|
+  config.before(:suite) do
+    JavaRunner.remove_tmp
+    JavaRunner.set_kafka_path!
+    $tc = TestCluster.new
+    $tc.start
+    sleep 5
+  end
+
+  config.after(:suite) do
+    $tc.stop
+  end
+end

data/spec/integration/simple/unavailable_broker_spec.rb

@@ -0,0 +1,77 @@
+require 'integration/simple/spec_helper'
+
+describe "unavailable broker scenarios:" do
+  context "producer with a dead broker in bootstrap list" do
+    before(:each) do
+      @p = Producer.new(["localhost:9091","localhost:9092"], "test")
+    end
+
+    it "succesfully sends a message" do
+      expect(@p.send_messages([MessageToSend.new("test", "hello")])).to eq(true)
+
+      pc = PartitionConsumer.new("test_consumer", "localhost",
+                                 9092, "test", 0, -2)
+
+      messages = pc.fetch
+      expect(messages.last.value).to eq("hello")
+    end
+  end
+
+  context "producer with required_acks set to 1" do
+    before(:each) do
+      @p = Producer.new(["localhost:9092"], "test", :required_acks => 1)
+    end
+
+    context "broker stops running" do
+      it "fails to send" do
+        expect(@p.send_messages([MessageToSend.new("test", "hello")])).to eq(true)
+
+        $tc.broker.without_process do
+          expect(@p.send_messages([MessageToSend.new("test", "hello")])).to eq(false)
+        end
+      end
+    end
+
+    context "broker stops running but starts again" do
+      it "sends succesfully once broker returns" do
+        expect(@p.send_messages([MessageToSend.new("test", "hello")])).to eq(true)
+
+        $tc.broker.without_process do
+          expect(@p.send_messages([MessageToSend.new("test", "hello")])).to eq(false)
+        end
+
+        expect(@p.send_messages([MessageToSend.new("test", "hello")])).to eq(true)
+      end
+    end
+  end
+
+  context "producer with required_acks set to 0" do
+    before(:each) do
+      @p = Producer.new(["localhost:9092"], "test", :required_acks => 0)
+    end
+
+    context "broker stops running" do
+      it "fails to send" do
+        expect(@p.send_messages([MessageToSend.new("test", "hello_a")])).to eq(true)
+
+        $tc.broker.without_process do
+          @p.send_messages([MessageToSend.new("test", "hello_b")])
+          expect(@p.send_messages([MessageToSend.new("test", "hello_b")])).to eq(false)
+        end
+      end
+    end
+
+    context "broker stops running but starts again" do
+      it "sends succesfully once broker returns" do
+        expect(@p.send_messages([MessageToSend.new("test", "hello")])).to eq(true)
+
+        $tc.broker.without_process do
+          @p.send_messages([MessageToSend.new("test", "hello")])
+          expect(@p.send_messages([MessageToSend.new("test", "hello")])).to eq(false)
+        end
+
+        expect(@p.send_messages([MessageToSend.new("test", "hello")])).to eq(true)
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,32 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# Require this file using `require "spec_helper"` to ensure that it is only
+# loaded once.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = 'random'
+
+  config.expect_with :rspec do |c|
+    c.syntax = :expect
+  end
+end
+
+POSEIDON_PATH = File.absolute_path(File.dirname(__FILE__) + "/../")
+
+require 'simplecov'
+SimpleCov.start
+
+require 'poseidon'
+include Poseidon
+
+require 'coveralls'
+Coveralls.wear!

data/spec/test_cluster.rb

@@ -0,0 +1,205 @@
+require 'daemon_controller'
+
+class TestCluster
+  attr_reader :broker, :zookeeper
+  def initialize
+    @zookeeper = ZookeeperRunner.new
+    @broker = BrokerRunner.new(0, 9092)
+  end
+
+  def start
+    @zookeeper.start
+    @broker.start
+  end
+
+  def stop
+    @zookeeper.stop
+    @broker.stop
+  end
+end
+
+class JavaRunner
+  def self.remove_tmp
+    FileUtils.rm_rf("#{POSEIDON_PATH}/tmp")
+  end
+
+  def self.set_kafka_path!
+    if ENV['KAFKA_PATH']
+      JavaRunner.kafka_path = ENV['KAFKA_PATH']
+    else
+      puts "******To run integration specs you must set KAFKA_PATH to kafka src directory. See README*****"
+      exit
+    end
+  end
+
+  def self.kafka_path=(kafka_path)
+    @kafka_path = kafka_path
+  end
+
+  def self.kafka_path
+    @kafka_path
+  end
+
+  attr_reader :pid
+  def initialize(id, start_cmd, port, properties = {})
+    @id = id
+    @properties = properties
+    @pid = nil
+    @start_cmd = start_cmd
+    @port = port
+  end
+
+  def start
+    write_properties
+    run
+  end
+
+  def stop
+    daemon_controller.stop
+  end
+
+  def without_process
+    stop
+    begin
+      yield
+    ensure
+      start
+      sleep 5
+    end
+  end
+
+  private
+
+  def daemon_controller
+    @dc ||= DaemonController.new(
+      :identifier => @id,
+      :start_command => "#{@start_cmd} #{config_path} >>#{log_path} 2>&1 & echo $! > #{pid_path}",
+      :ping_command => [:tcp, '127.0.0.1', @port],
+      :pid_file => pid_path,
+      :log_file => log_path,
+      :start_timeout => 25
+    )
+  end
+
+  def run
+    FileUtils.mkdir_p(log_dir)
+    FileUtils.mkdir_p(pid_dir)
+    daemon_controller.start
+  end
+
+  def write_properties
+    FileUtils.mkdir_p(config_dir)
+    File.open(config_path, "w+") do |f|
+      @properties.each do |k,v|
+        f.puts "#{k}=#{v}"
+      end
+    end
+  end
+
+  def pid_path
+    "#{pid_dir}/#{@id}.pid"
+  end
+
+  def pid_dir
+    "#{file_path}/pid"
+  end
+
+  def log_path
+    "#{log_dir}/#{@id}.log"
+  end
+
+  def log_dir
+    "#{file_path}/log"
+  end
+
+  def config_path
+    "#{config_dir}/#{@id}.properties"
+  end
+
+  def config_dir
+    "#{file_path}/config"
+  end
+
+  def file_path
+    POSEIDON_PATH + "/tmp/"
+  end
+end
+
+class BrokerRunner
+  DEFAULT_PROPERTIES = {
+    "broker.id" => 0,
+    "port" => 9092,
+    "num.network.threads" => 2,
+    "num.io.threads" => 2,
+    "socket.send.buffer.bytes" => 1048576,
+    "socket.receive.buffer.bytes" => 1048576,
+    "socket.request.max.bytes" => 104857600,
+    "log.dir" => "#{POSEIDON_PATH}/tmp/kafka-logs",
+    "num.partitions" => 1,
+    "log.flush.interval.messages" => 10000,
+    "log.flush.interval.ms" => 1000,
+    "log.retention.hours" => 168,
+    "log.segment.bytes" => 536870912,
+    "log.cleanup.interval.mins" => 1,
+    "zookeeper.connect" => "localhost:2181",
+    "zookeeper.connection.timeout.ms" => 1000000,
+    "kafka.metrics.polling.interval.secs" => 5,
+    "kafka.metrics.reporters" => "kafka.metrics.KafkaCSVMetricsReporter",
+    "kafka.csv.metrics.dir" => "#{POSEIDON_PATH}/tmp/kafka_metrics",
+    "kafka.csv.metrics.reporter.enabled" => "false",
+  }
+
+  def initialize(id, port, partition_count = 1)
+    @id   = id
+    @port = port
+    @jr = JavaRunner.new("broker_#{id}", 
+                         "#{POSEIDON_PATH}/spec/bin/kafka-run-class.sh kafka.Kafka", 
+                         port,
+                         DEFAULT_PROPERTIES.merge(
+                           "broker.id" => id,
+                           "port" => port,
+                           "log.dir" => "#{POSEIDON_PATH}/tmp/kafka-logs_#{id}",
+                           "num.partitions" => partition_count
+                         ))
+  end
+
+  def pid
+    @jr.pid
+  end
+
+  def start
+    @jr.start
+  end
+
+  def stop
+    @jr.stop
+  end
+
+  def without_process
+    @jr.without_process { yield }
+  end
+end
+
+
+class ZookeeperRunner
+  def initialize
+    @jr = JavaRunner.new("zookeeper",
+                         "#{POSEIDON_PATH}/spec/bin/kafka-run-class.sh org.apache.zookeeper.server.quorum.QuorumPeerMain", 
+                         2181,
+                         :dataDir => "#{POSEIDON_PATH}/tmp/zookeeper",
+                         :clientPort => 2181,
+                         :maxClientCnxns => 0)
+  end
+
+  def pid
+    @jr.pid
+  end
+  
+  def start
+    @jr.start
+  end
+
+  def stop
+    @jr.stop
+  end
+end

data/spec/unit/broker_pool_spec.rb

@@ -0,0 +1,77 @@
+require 'spec_helper'
+
+describe BrokerPool do
+  context "empty broker list" do
+    it "raises UnknownBroker error when trying to produce data" do
+      expect { BrokerPool.new("test_client", []).execute_api_call(0, :produce) }.to raise_error(BrokerPool::UnknownBroker)
+    end
+  end
+
+  describe "fetching metadata" do
+    context "no seed brokers" do
+      it "raises Error" do
+        @broker_pool = BrokerPool.new("test_client", [])
+        expect { @broker_pool.fetch_metadata(Set.new) }.to raise_error(Errors::UnableToFetchMetadata)
+      end
+    end
+
+    context "2 seed brokers" do
+      before(:each) do
+        @broker_pool = BrokerPool.new("test_client", ["first:9092","second:9092"])
+        @broker_1 = double('Poseidon::Connection_1', :topic_metadata => nil)
+        @broker_2 = double('Poseidon::Connection_2', :topic_metadata => double('topic_metadata').as_null_object)
+        Connection.stub!(:new).and_return(@broker_1, @broker_2)
+      end
+
+      context ", first doesn't have metadata" do
+        it "asks the second" do
+          @broker_2.should_receive(:topic_metadata)
+
+          @broker_pool.fetch_metadata(Set.new)
+        end
+      end
+    end
+  end
+
+  context "which knowns about two brokers" do
+    before(:each) do
+      @broker_pool = BrokerPool.new("test_client", [])
+      @broker_pool.update_known_brokers({0 => { :host => "localhost", :port => 9092 }, 1 => {:host => "localhost", :port => 9093 }})
+    end
+
+    describe "when executing a call" do
+
+      it "creates a connection for the correct broker" do
+        c = stub('conn').as_null_object
+        expected_args = ["localhost", 9092, "test_client"]
+
+        Connection.should_receive(:new).with(*expected_args).and_return(c)
+        @broker_pool.execute_api_call(0, :produce)
+      end
+
+      it "it does so on the correct broker" do
+        c = stub('conn').as_null_object
+        Connection.stub(:new).and_return(c)
+
+        c.should_receive(:produce)
+        @broker_pool.execute_api_call(0, :produce)
+      end
+    end
+
+    describe "when executing two calls" do
+      it "reuses the connection" do
+        c = stub('conn').as_null_object
+
+        Connection.should_receive(:new).once.and_return(c)
+        @broker_pool.execute_api_call(0, :produce)
+        @broker_pool.execute_api_call(0, :produce)
+      end
+    end
+
+    describe "executing a call for an unknown broker" do
+      it "raises UnknownBroker" do
+        expect { @broker_pool.execute_api_call(2, :produce) }.to raise_error(BrokerPool::UnknownBroker)
+      end
+    end
+  end
+end

data/spec/unit/cluster_metadata_spec.rb

@@ -0,0 +1,41 @@
+require 'spec_helper'
+
+include Protocol
+describe ClusterMetadata do
+  describe "populated" do
+    before(:each) do
+      partitions = [
+        PartitionMetadata.new(1, 1, [1,2], [1,2], nil),
+        PartitionMetadata.new(2, 2, [2,1], [2,1], nil)
+      ]
+      topics = [TopicMetadata.new(TopicMetadataStruct.new(nil, "test", partitions))]
+
+      brokers = [Broker.new(1, "host1", 1), Broker.new(2, "host2", 2)]
+
+      @mr = MetadataResponse.new(nil, brokers, topics)
+
+      @cm = ClusterMetadata.new
+      @cm.update(@mr)
+    end
+
+    it "knows when it has metadata for a set of topics" do
+      have_metadata = @cm.have_metadata_for_topics?(Set.new(["test"]))
+      expect(have_metadata).to eq(true)
+    end
+
+    it "knows when it doesn't have metadata for a topic" do
+      have_metadata = @cm.have_metadata_for_topics?(Set.new(["test", "no_data"]))
+      expect(have_metadata).to eq(false)
+    end
+
+    it "provides topic metadata for a set of topics" do
+      topic_metadata = @cm.metadata_for_topics(Set.new(["test"]))
+      expect(topic_metadata).to eq({ "test" => @mr.topics.first })
+    end
+
+    it "provides broker information" do
+      broker = @cm.broker(1)
+      expect(broker).to eq(@mr.brokers.first)
+    end
+  end
+end

data/spec/unit/compression_spec.rb

@@ -0,0 +1,17 @@
+require 'spec_helper'
+
+describe Compression do
+  it 'returns GzipCompessor for codec_id of 1' do
+    codec = Compression.find_codec(1)
+    expect(codec).to eq(Compression::GzipCodec)
+  end
+
+  it 'returns SnappyCompessor for codec_id of 2' do
+    codec = Compression.find_codec(2)
+    expect(codec).to eq(Compression::SnappyCodec)
+  end
+
+  it 'raises UnrecognizedCompressionCodec for codec_id of 3' do
+    expect { Compression.find_codec(3) }.to raise_error(Compression::UnrecognizedCompressionCodec)
+  end
+end

data/spec/unit/connection_spec.rb

@@ -0,0 +1,4 @@
+require 'spec_helper'
+
+describe Connection do
+end

data/spec/unit/fetched_message_spec.rb

@@ -0,0 +1,11 @@
+require 'spec_helper'
+
+describe FetchedMessage do
+  it "provides access to topic,value,key,offset" do
+    mts = FetchedMessage.new("hello_topic", "Hello World", "key", 0)
+    expect(mts.topic).to eq("hello_topic")
+    expect(mts.value).to eq("Hello World")
+    expect(mts.key).to eq("key")
+    expect(mts.offset).to eq(0)
+  end
+end