codeclimate-poseidon 0.0.8

data/spec/integration/simple/spec_helper.rb

@@ -0,0 +1,16 @@
+require 'spec_helper'
+
+require 'test_cluster'
+
+RSpec.shared_context "a single broker cluster" do
+  before(:each) do
+    JavaRunner.remove_tmp
+    JavaRunner.set_kafka_path!
+    $tc = TestCluster.new
+    $tc.start
+  end
+
+  after(:each) do
+    $tc.stop
+  end
+end

data/spec/integration/simple/truncated_messages_spec.rb

@@ -0,0 +1,46 @@
+require 'integration/simple/spec_helper'
+
+RSpec.describe "truncated messages", :type => :request do
+  include_context "a single broker cluster"
+
+  before(:each) do
+    @s1 = "a" * 335
+    @s2 = "b" * 338
+
+    @producer = Producer.new(["localhost:9092"],
+                             "test_client",
+                             :type => :sync)
+
+    @producer.send_messages([Message.new(:topic => 'test_max_bytes', :value => @s1), Message.new(:topic => 'test_max_bytes', :value => @s2)])
+  end
+
+  it "correctly handles max_byte lengths smallert than a message" do
+    0.upto(360) do |n|
+      consumer = PartitionConsumer.new("test_consumer", "localhost", 9092,
+                                       "test_max_bytes", 0, :earliest_offset)
+      expect(consumer.fetch(:max_bytes => n)).to eq([])
+    end
+  end
+
+  it "correctly handles max_byte lengths that should return a single message" do
+    361.upto(724) do |n|
+      consumer = PartitionConsumer.new("test_consumer", "localhost", 9092,
+                                       "test_max_bytes", 0, :earliest_offset)
+
+      messages = consumer.fetch(:max_bytes => n)
+      expect(messages.size).to eq(1)
+      expect(messages.first.value).to eq(@s1)
+    end
+  end
+
+  it "correctly handles max_byte lengths that should return two messages" do
+    725.upto(1000) do |n|
+      consumer = PartitionConsumer.new("test_consumer", "localhost", 9092,
+                                       "test_max_bytes", 0, :earliest_offset)
+
+      messages = consumer.fetch(:max_bytes => n)
+      expect(messages.size).to eq(2)
+      expect(messages.map(&:value)).to eq([@s1, @s2])
+    end
+  end
+end

data/spec/integration/simple/unavailable_broker_spec.rb

@@ -0,0 +1,72 @@
+require 'integration/simple/spec_helper'
+
+RSpec.describe "unavailable broker scenarios:", :type => :request do
+  include_context "a single broker cluster"
+
+  context "producer with a dead broker in bootstrap list" do
+    before(:each) do
+      @p = Producer.new(["localhost:9091","localhost:9092"], "test")
+    end
+
+    it "succesfully sends a message" do
+      expect(@p.send_messages([MessageToSend.new("test", "hello")])).to eq(true)
+
+      pc = PartitionConsumer.new("test_consumer", "localhost",
+                                 9092, "test", 0, -2)
+
+      messages = pc.fetch
+      expect(messages.last.value).to eq("hello")
+    end
+  end
+
+  context "producer with required_acks set to 1" do
+    before(:each) do
+      @p = Producer.new(["localhost:9092"], "test", :required_acks => 1)
+    end
+
+    context "broker stops running" do
+      it "fails to send" do
+        expect(@p.send_messages([MessageToSend.new("test", "hello")])).to eq(true)
+
+        $tc.broker.without_process do
+          expect {
+            @p.send_messages([MessageToSend.new("test", "hello")])
+          }.to raise_error(Poseidon::Errors::UnableToFetchMetadata)
+        end
+      end
+    end
+
+    context "broker stops running but starts again" do
+      it "sends succesfully once broker returns" do
+        expect(@p.send_messages([MessageToSend.new("test", "hello")])).to eq(true)
+
+        $tc.broker.without_process do
+          expect {
+            @p.send_messages([MessageToSend.new("test", "hello")])
+          }.to raise_error(Poseidon::Errors::UnableToFetchMetadata)
+        end
+
+        expect(@p.send_messages([MessageToSend.new("test", "hello")])).to eq(true)
+      end
+    end
+  end
+
+  context "producer with required_acks set to 0" do
+    before(:each) do
+      @p = Producer.new(["localhost:9092"], "test", :required_acks => 0)
+    end
+
+    context "broker stops running" do
+      it "fails to send" do
+        expect(@p.send_messages([MessageToSend.new("test", "hello_a")])).to eq(true)
+
+        $tc.broker.without_process do
+          @p.send_messages([MessageToSend.new("test", "hello_b")])
+          expect {
+            @p.send_messages([MessageToSend.new("test", "hello_b")])
+          }.to raise_error(Poseidon::Errors::UnableToFetchMetadata)
+        end
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,32 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# Require this file using `require "spec_helper"` to ensure that it is only
+# loaded once.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = 'random'
+
+  config.disable_monkey_patching!
+end
+
+POSEIDON_PATH = File.absolute_path(File.dirname(__FILE__) + "/../")
+
+require 'logger'
+SPEC_LOGGER = Logger.new(File.join(POSEIDON_PATH, "log", "spec.log"))
+
+require 'simplecov'
+SimpleCov.start
+
+require 'poseidon'
+include Poseidon
+
+require 'coveralls'
+Coveralls.wear!
+
+require 'timecop'
+Timecop.safe_mode = true

data/spec/test_cluster.rb

@@ -0,0 +1,211 @@
+class TestCluster
+  attr_reader :broker, :zookeeper
+  def initialize
+    @zookeeper = ZookeeperRunner.new
+    @broker = BrokerRunner.new(0, 9092)
+  end
+
+  def start
+    @zookeeper.start
+    @broker.start
+    sleep 5
+  end
+
+  def stop
+    # The broker will end up in a state where it ignores SIGTERM
+    # if zookeeper is stopped before the broker.
+    @broker.stop
+    sleep 5
+
+    @zookeeper.stop
+  end
+end
+
+class JavaRunner
+  def self.remove_tmp
+    FileUtils.rm_rf("#{POSEIDON_PATH}/tmp")
+  end
+
+  def self.set_kafka_path!
+    if ENV['KAFKA_PATH']
+      JavaRunner.kafka_path = ENV['KAFKA_PATH']
+    else
+      puts "******To run integration specs you must set KAFKA_PATH to kafka src directory. See README*****"
+      exit
+    end
+  end
+
+  def self.kafka_path=(kafka_path)
+    @kafka_path = kafka_path
+  end
+
+  def self.kafka_path
+    @kafka_path
+  end
+
+  def initialize(id, start_cmd, pid_cmd, kill_signal, properties = {})
+    @id = id
+    @properties = properties
+    @start_cmd = start_cmd
+    @pid_cmd = pid_cmd
+    @kill_signal = kill_signal
+    @stopped = false
+  end
+
+  def start
+    write_properties
+    run
+  end
+
+  def stop
+    if !@stopped
+      killed_at = Time.now
+      loop do
+        if (pid = `#{@pid_cmd}`.to_i) == 0
+          SPEC_LOGGER.info "Killed."
+          break
+        end
+
+        if Time.now - killed_at > 30
+          raise "Failed to kill process!"
+        end
+
+        SPEC_LOGGER.info "Sending #{@kill_signal} To #{pid}"
+        SPEC_LOGGER.info "(#{@start_cmd})"
+        `kill -#{@kill_signal} #{pid}`
+
+        sleep 5
+      end
+      @stopped = true
+    end
+  end
+
+  def without_process
+    stop
+    sleep 5
+    begin
+      yield
+    ensure
+      start
+      sleep 5
+    end
+  end
+
+  private
+
+  def run
+    FileUtils.mkdir_p(log_dir)
+    `LOG_DIR=#{log_dir} #{@start_cmd} #{config_path}`
+    @stopped = false
+  end
+
+  def write_properties
+    FileUtils.mkdir_p(config_dir)
+    File.open(config_path, "w+") do |f|
+      @properties.each do |k,v|
+        f.puts "#{k}=#{v}"
+      end
+    end
+  end
+
+  def log_dir
+    "#{file_path}/log"
+  end
+
+  def config_path
+    "#{config_dir}/#{@id}.properties"
+  end
+
+  def config_dir
+    "#{file_path}/config"
+  end
+
+  def file_path
+    POSEIDON_PATH + "/tmp/"
+  end
+end
+
+class BrokerRunner
+  DEFAULT_PROPERTIES = {
+    "broker.id" => 0,
+    "port" => 9092,
+    "num.network.threads" => 2,
+    "num.io.threads" => 2,
+    "socket.send.buffer.bytes" => 1048576,
+    "socket.receive.buffer.bytes" => 1048576,
+    "socket.request.max.bytes" => 104857600,
+    "log.dir" => "#{POSEIDON_PATH}/tmp/kafka-logs",
+    "num.partitions" => 1,
+    "log.flush.interval.messages" => 10000,
+    "log.flush.interval.ms" => 1000,
+    "log.retention.hours" => 168,
+    "log.segment.bytes" => 536870912,
+    #"log.cleanup.interval.mins" => 1,
+    "zookeeper.connect" => "localhost:2181",
+    "zookeeper.connection.timeout.ms" => 1000000,
+    #"kafka.metrics.polling.interval.secs" => 5,
+    #"kafka.metrics.reporters" => "kafka.metrics.KafkaCSVMetricsReporter",
+    #"kafka.csv.metrics.dir" => "#{POSEIDON_PATH}/tmp/kafka_metrics",
+    #"kafka.csv.metrics.reporter.enabled" => "false",
+    "auto.create.topics.enable" => "true",
+
+    # Trigger rebalances often to catch edge cases.
+    "auto.leader.rebalance.enable" => "true",
+    "leader.imbalance.check.interval.seconds" => 5
+  }
+
+  attr_reader :id
+
+  def initialize(id, port, partition_count = 1, replication_factor = 1, properties = {})
+    @id   = id
+    @port = port
+    @jr = JavaRunner.new("broker_#{id}",
+                         "#{ENV['KAFKA_PATH']}/bin/kafka-run-class.sh -daemon -name broker_#{id} kafka.Kafka",
+                         "ps ax | grep -i 'kafka\.Kafka' | grep java | grep broker_#{id} | grep -v grep | awk '{print $1}'",
+                         "SIGTERM",
+                         DEFAULT_PROPERTIES.merge(
+                           "broker.id" => id,
+                           "port" => port,
+                           "log.dir" => "#{POSEIDON_PATH}/tmp/kafka-logs_#{id}",
+                           "default.replication.factor" => replication_factor,
+                           "num.partitions" => partition_count
+                         ).merge(properties))
+  end
+
+  def start
+    @jr.start
+  end
+
+  def stop
+    @jr.stop
+  end
+
+  def without_process
+    @jr.without_process { yield }
+  end
+end
+
+
+class ZookeeperRunner
+  def initialize
+    @jr = JavaRunner.new("zookeeper",
+                         "#{ENV['KAFKA_PATH']}/bin/zookeeper-server-start.sh -daemon",
+                         "ps ax | grep -i 'zookeeper' | grep -v grep | awk '{print $1}'",
+                         "SIGKILL",
+                         :dataDir => "#{POSEIDON_PATH}/tmp/zookeeper",
+                         :clientPort => 2181,
+                         :maxClientCnxns => 0)
+  end
+
+  def pid
+    @jr.pid
+  end
+
+  def start
+    @jr.start
+  end
+
+  def stop
+    @jr.stop
+  end
+end

data/spec/unit/broker_pool_spec.rb

@@ -0,0 +1,98 @@
+require 'spec_helper'
+
+RSpec.describe BrokerPool do
+  context "empty broker list" do
+    it "raises UnknownBroker error when trying to produce data" do
+      expect { BrokerPool.new("test_client", [], 10_000).execute_api_call(0, :produce) }.to raise_error(BrokerPool::UnknownBroker)
+    end
+  end
+
+  describe "fetching metadata" do
+    context "single broker" do
+      it "initializes connection properly" do
+        @broker_pool = BrokerPool.new("test_client", ["localhost:9092"], 2_000)
+        @broker = double('Poseidon::Connection', :topic_metadata => nil)
+
+        expected_args = ["localhost", "9092", "test_client", 2_000]
+        connection = double('conn').as_null_object
+
+        expect(Connection).to receive(:new).with(*expected_args).and_return(connection)
+
+        @broker_pool.fetch_metadata(Set.new)
+      end
+    end
+
+    context "no seed brokers" do
+      it "raises Error" do
+        @broker_pool = BrokerPool.new("test_client", [], 10_000)
+        expect { @broker_pool.fetch_metadata(Set.new) }.to raise_error(Errors::UnableToFetchMetadata)
+      end
+    end
+
+    context "2 seed brokers" do
+      before(:each) do
+        @broker_pool = BrokerPool.new("test_client", ["first:9092","second:9092"], 10_000)
+        @broker_1 = double('Poseidon::Connection_1', :topic_metadata => nil, :close => nil)
+        @broker_2 = double('Poseidon::Connection_2', :topic_metadata => double('topic_metadata').as_null_object, :close => nil)
+        allow(Connection).to receive(:new).and_return(@broker_1, @broker_2)
+      end
+
+      context ", first doesn't have metadata" do
+        it "asks the second" do
+          expect(@broker_2).to receive(:topic_metadata)
+
+          @broker_pool.fetch_metadata(Set.new)
+        end
+      end
+
+      it "cleans up its connections" do
+        expect(@broker_1).to receive(:close)
+        expect(@broker_2).to receive(:close)
+
+        @broker_pool.fetch_metadata(Set.new)
+      end
+    end
+  end
+
+  context "which knowns about two brokers" do
+    before(:each) do
+      @broker_pool = BrokerPool.new("test_client", [], 10_000)
+      @broker_pool.update_known_brokers({0 => { :host => "localhost", :port => 9092 }, 1 => {:host => "localhost", :port => 9093 }})
+    end
+
+    describe "when executing a call" do
+
+      it "creates a connection for the correct broker" do
+        c = double('conn').as_null_object
+        expected_args = ["localhost", 9092, "test_client", 10_000]
+
+        expect(Connection).to receive(:new).with(*expected_args).and_return(c)
+        @broker_pool.execute_api_call(0, :produce)
+      end
+
+      it "it does so on the correct broker" do
+        c = double('conn').as_null_object
+        allow(Connection).to receive(:new).and_return(c)
+
+        expect(c).to receive(:produce)
+        @broker_pool.execute_api_call(0, :produce)
+      end
+    end
+
+    describe "when executing two calls" do
+      it "reuses the connection" do
+        c = double('conn').as_null_object
+
+        expect(Connection).to receive(:new).once.and_return(c)
+        @broker_pool.execute_api_call(0, :produce)
+        @broker_pool.execute_api_call(0, :produce)
+      end
+    end
+
+    describe "executing a call for an unknown broker" do
+      it "raises UnknownBroker" do
+        expect { @broker_pool.execute_api_call(2, :produce) }.to raise_error(BrokerPool::UnknownBroker)
+      end
+    end
+  end
+end