codeclimate-poseidon 0.0.8

data/spec/integration/multiple_brokers/metadata_failures_spec.rb

@@ -0,0 +1,144 @@
+require 'integration/multiple_brokers/spec_helper'
+
+# Created because you can't use a block form for receive with and_call_original
+# https://github.com/rspec/rspec-mocks/issues/774
+RSpec::Matchers.define :a_broker_id_of do |id|
+  match { |actual| actual.broker_id == id }
+end
+
+RSpec::Matchers.define :a_broker_id_not do |id|
+  match { |actual| actual.broker_id != id }
+end
+
+RSpec::Matchers.define :needing_metadata do |val|
+  match { |actual| actual.send(:needs_metadata?) == val }
+end
+
+RSpec.describe "handling failures", :type => :request do
+  include_context "a multiple broker cluster"
+
+  describe "metadata failures" do
+    before(:each) do
+      @messages_to_send = [
+        MessageToSend.new("topic1", "hello"),
+        MessageToSend.new("topic2", "hello")
+      ]
+    end
+
+    describe "unable to connect to brokers" do
+      before(:each) do
+        @p = Producer.new(["localhost:1092","localhost:1093","localhost:1094"], "producer")
+      end
+
+      it "triggers callback failures for both topics" do
+        expect {
+          @p.send_messages(@messages_to_send)
+        }.to raise_error(Poseidon::Errors::UnableToFetchMetadata)
+      end
+    end
+  end
+
+  describe "unknown topic" do
+    it "receives error callback" do
+      pending "need a way to turn off auto-topic creation just for this test"
+      @p = Producer.new(["localhost:9092","localhost:9093","localhost:9094"], "producer")
+
+      expect {
+        @p.send_messages([MessageToSend.new("imnothere", "hello")])
+      }.to raise_error(Poseidon::Errors::UnableToFetchMetadata)
+    end
+  end
+
+  describe "leader node loss" do
+    let(:topic) { "testing" }
+    let(:kafka_partitions) { 1 }
+
+    it "is still able to send messages" do
+      @p = Producer.new(["localhost:9092","localhost:9093","localhost:9094"], "producer", :required_acks => 1)
+
+      # Send one to force topic creation
+      expect(@p.send_messages([MessageToSend.new(topic, "hello")])).to be_truthy
+
+      @producer = @p.instance_variable_get(:@producer)
+      @cluster_metadata = @producer.instance_variable_get(:@cluster_metadata)
+      topic_metadata = @cluster_metadata.metadata_for_topics([topic])[topic]
+
+      expect(topic_metadata.available_partitions.length).to be(kafka_partitions)
+
+      # Now, lets kill the topic leader
+      leader = topic_metadata.available_partitions.first.leader
+      broker = $tc.brokers[topic_metadata.available_partitions.first.leader]
+      expect(broker.id).to be(leader)
+
+      broker.without_process do
+        expect(@cluster_metadata.metadata_for_topics([topic])[topic].available_partitions.first.leader).to be(leader)
+        expect(@producer.send(:refresh_interval_elapsed?)).to be_falsy
+
+        # Setup expectations that the consumer updates its info
+        expect(@producer).to receive(:ensure_metadata_available_for_topics).with(needing_metadata(false)).ordered.and_call_original
+        expect(@producer).to receive(:send_to_broker).with(a_broker_id_of(leader)).ordered.and_call_original
+
+        expect(@producer).to receive(:reset_metadata).ordered.and_call_original
+        expect(@producer).to receive(:ensure_metadata_available_for_topics).ordered.and_call_original
+        expect(@producer).to receive(:send_to_broker).with(a_broker_id_not(leader)).ordered.and_call_original
+
+        expect(@p.send_messages([MessageToSend.new(topic, "hello")])).to be_truthy
+      end
+    end
+  end
+
+  describe "partition replica loss" do
+    let(:topic) { "testing" }
+    let(:kafka_partitions) { 1 }
+
+    it "refreshes metadata correctly" do
+      @p = Producer.new(["localhost:9092","localhost:9093","localhost:9094"], "producer", :required_acks => 1)
+
+      # Send one to force topic creation
+      expect(@p.send_messages([MessageToSend.new(topic, "hello")])).to be_truthy
+
+      @producer = @p.instance_variable_get(:@producer)
+      @cluster_metadata = @producer.instance_variable_get(:@cluster_metadata)
+      topic_metadata = @cluster_metadata.metadata_for_topics([topic])[topic]
+
+      expect(topic_metadata.available_partitions.length).to be(kafka_partitions)
+      expect(topic_metadata.available_partitions.first.error).to be(0)
+
+      # Now, lets kill the topic leader
+      partition_metadata = topic_metadata.available_partitions.first
+      expect(partition_metadata.replicas.length).to be(2)
+      leader = partition_metadata.leader
+      replica = (partition_metadata.replicas - [leader]).first
+
+      broker = $tc.brokers[replica]
+      expect(broker.id).to_not be(partition_metadata.leader)
+      expect(broker.id).to be(replica)
+
+      broker.without_process do
+        expect(@cluster_metadata.metadata_for_topics([topic])[topic].available_partitions.first.replicas).to include(replica)
+        expect(@producer.send(:refresh_interval_elapsed?)).to be_falsy
+
+        Timecop.travel(@producer.metadata_refresh_interval_ms) do
+          expect(@producer.send(:refresh_interval_elapsed?)).to be_truthy
+
+          # Setup expectations that the consumer updates its info
+          expect(@producer).to receive(:refresh_metadata).with(Set.new([topic])).ordered.and_call_original
+          expect(@producer).to receive(:ensure_metadata_available_for_topics).with(needing_metadata(false)).ordered.and_call_original
+          expect(@producer).to receive(:send_to_broker).with(a_broker_id_of(partition_metadata.leader)).ordered.and_call_original
+
+          # Make sure we don't error out
+          expect(@producer).to_not receive(:reset_metadata)
+
+          expect(@p.send_messages([MessageToSend.new(topic, "hello")])).to be_truthy
+
+          # Check the valid metadata
+          updated_topic_metadata = @cluster_metadata.metadata_for_topics([topic])[topic]
+          expect(updated_topic_metadata.available_partitions.length).to be(kafka_partitions)
+          expect(updated_topic_metadata.available_partitions.first.leader).to be(leader)
+          expect(updated_topic_metadata.available_partitions.first.replicas).to eq([leader])
+          expect(updated_topic_metadata.available_partitions.first.error).to be(9)
+        end
+      end
+    end
+  end
+end

data/spec/integration/multiple_brokers/rebalance_spec.rb

@@ -0,0 +1,69 @@
+require 'integration/multiple_brokers/spec_helper'
+
+RSpec.describe "producer handles rebalancing", :type => :request do
+  include_context "a multiple broker cluster"
+
+  before(:each) do
+    # autocreate the topic by asking for information about it
+    @c = Connection.new("localhost", 9093, "metadata_fetcher", 10_000)
+    @c.topic_metadata(["failure_spec"])
+    sleep 1
+  end
+
+  def current_leadership_mapping(c)
+    metadata = c.topic_metadata(["failure_spec"])
+    topic_metadata = metadata.topics.find { |t| t.name == "failure_spec" }
+    (0..2).map { |p| topic_metadata.partition_leader(p) }
+  end
+
+  it "produces a bunch of messages and consumes all without error" do
+    @p = Producer.new(["localhost:9092","localhost:9093","localhost:9094"], "test",
+                     :required_acks => -1)
+
+    1.upto(25) do |n|
+      @p.send_messages([MessageToSend.new("failure_spec", n.to_s)])
+    end
+
+    # The goal here is to have the producer attempt to send messages
+    # to a broker which is no longer the leader for the partition.
+    #
+    # We accomplish this by turning off a broker which causes leadership
+    # to failover. Then we turn that broker back on and begin sending
+    # messages. While sending messages, the kafka cluster should rebalance
+    # the partitions causing leadership to switch back to the original
+    # broker in the midst of messages being sent.
+    #
+    # We compare leadership before and after the message sending period
+    # to make sure we were successful.
+    $tc.stop_first_broker
+    sleep 30
+    SPEC_LOGGER.info "Pre start #{current_leadership_mapping(@c).inspect}"
+    $tc.start_first_broker
+
+    pre_send_leadership = current_leadership_mapping(@c)
+    SPEC_LOGGER.info "Pre send #{pre_send_leadership.inspect}"
+    26.upto(50) do |n|
+      sleep 0.5
+      @p.send_messages([MessageToSend.new("failure_spec", n.to_s)])
+    end
+    post_send_leadership = current_leadership_mapping(@c)
+    SPEC_LOGGER.info "Post send #{post_send_leadership.inspect}"
+
+    expect(pre_send_leadership).to_not eq(post_send_leadership)
+
+    messages = []
+    0.upto(2) do |partition|
+      consumer = PartitionConsumer.consumer_for_partition("consumer_failure_spect",
+                                                          ["localhost:9092","localhost:9093","localhost:9094"],
+                                                          "failure_spec",
+                                                          partition,
+                                                          :earliest_offset)
+      while (fetched = consumer.fetch).any?
+        messages.push(*fetched)
+      end
+    end
+
+    expect(messages.size).to eq(50)
+    expect(messages.map { |m| m.value.to_i }.sort).to eq((1..50).to_a)
+  end
+end

data/spec/integration/multiple_brokers/round_robin_spec.rb

@@ -0,0 +1,41 @@
+require 'integration/multiple_brokers/spec_helper'
+
+RSpec.describe "round robin sending", :type => :request do
+  include_context "a multiple broker cluster"
+
+  describe "with small message batches" do
+    it "evenly distributes messages across brokers" do
+      c = Connection.new("localhost", 9092, "metadata_fetcher", 10_000)
+      md = c.topic_metadata(["test"])
+      sleep 1
+      md = c.topic_metadata(["test"])
+
+      test_topic = md.topics.first
+
+      consumers = test_topic.send(:partitions).map do |partition|
+        leader_id = partition.leader
+        broker = md.brokers.find { |b| b.id == leader_id }
+        PartitionConsumer.new("test_consumer_#{partition.id}", broker.host,
+                              broker.port, "test", partition.id, -1)
+      end
+
+      # Update offsets to current position before adding test messages
+      consumers.each do |c|
+        c.fetch
+      end
+
+      @p = Producer.new(["localhost:9092","localhost:9093","localhost:9094"], "test",
+                       :required_acks => 1)
+      24.times do
+        @p.send_messages([MessageToSend.new("test", "hello")])
+      end
+
+      sleep 5
+
+      consumers.each do |c|
+        messages = c.fetch
+        expect(messages.size).to eq(8)
+      end
+    end
+  end
+end

data/spec/integration/multiple_brokers/spec_helper.rb

@@ -0,0 +1,60 @@
+require 'spec_helper'
+
+require 'test_cluster'
+
+class ThreeBrokerCluster
+  attr_reader :brokers, :zookeeper
+
+  def initialize(partitions, properties = {})
+    @zookeeper = ZookeeperRunner.new
+    @brokers = (9092..9094).map { |port| BrokerRunner.new(port - 9092, port,
+                                                          partitions,
+                                                          2,
+                                                          properties) }
+  end
+
+  def start
+    @zookeeper.start
+    @brokers.each(&:start)
+    sleep 5
+  end
+
+  def stop
+    SPEC_LOGGER.info "Stopping three broker cluster"
+    SPEC_LOGGER.info "Stopping brokers"
+    @brokers.each(&:stop)
+    sleep 5
+
+    SPEC_LOGGER.info "Stopping ZK"
+    @zookeeper.stop
+    sleep 5
+  end
+
+  def stop_first_broker
+    SPEC_LOGGER.info "Stopping first broker"
+    @brokers.first.stop
+    sleep 5
+  end
+
+  def start_first_broker
+    SPEC_LOGGER.info "Starting first broker"
+    @brokers.first.start
+  end
+end
+
+RSpec.shared_context "a multiple broker cluster" do
+  let(:kafka_partitions) { 3 }
+
+  before(:each) do
+    JavaRunner.remove_tmp
+    JavaRunner.set_kafka_path!
+    $tc = ThreeBrokerCluster.new(kafka_partitions)
+    $tc.start
+    SPEC_LOGGER.info "Waiting on cluster"
+    sleep 10 # wait for cluster to come up
+  end
+
+  after(:each) do
+    $tc.stop if $tc
+  end
+end

data/spec/integration/simple/compression_spec.rb

@@ -0,0 +1,23 @@
+require 'integration/simple/spec_helper'
+
+RSpec.describe "compression", :type => :request do
+  include_context "a single broker cluster"
+
+  it "roundtrips" do
+    i = rand(1000)
+
+    @consumer = PartitionConsumer.new("test_consumer", "localhost", 9092,
+                                      "test12", 0, -2)
+
+    @producer = Producer.new(["localhost:9092"],
+                             "test_client",
+                             :type => :sync,
+                             :compression_codec => :gzip)
+    messages = [MessageToSend.new("test12", "Hello World: #{i}")]
+
+    expect(@producer.send_messages(messages)).to eq(true)
+    sleep 1
+    messages = @consumer.fetch
+    expect(messages.last.value).to eq("Hello World: #{i}")
+  end
+end

data/spec/integration/simple/connection_spec.rb

@@ -0,0 +1,35 @@
+require 'integration/simple/spec_helper'
+
+include Protocol
+RSpec.describe Connection, :type => :request do
+  include_context "a single broker cluster"
+
+  before(:each) do
+    @connection = Connection.new("localhost", 9092, "test", 10_000)
+  end
+
+  it 'sends and parses topic metadata requests' do
+    @connection.topic_metadata(["test2"])
+  end
+
+  it 'sends and parsers produce requests' do
+    message = MessageStruct.new(0, 0, nil, "hello")
+    message_with_offset = MessageWithOffsetStruct.new(0, message)
+    message_set = MessageSetStruct.new([message_with_offset])
+    messages_for_partitions = [MessagesForPartition.new(0,message_set)]
+    messages_for_topics = [MessagesForTopic.new("test2",messages_for_partitions)]
+    @connection.produce(1, 10_000, messages_for_topics)
+  end
+
+  it 'sends and parsers fetch requests' do
+    partition_fetches = [PartitionFetch.new(0,0,1024*1024)]
+    topic_fetches = [TopicFetch.new("test2", partition_fetches)]
+    @connection.fetch(1000, 0, topic_fetches)
+  end
+
+  it 'sends and parsers offset requests' do
+    partition_offset_requests = [PartitionOffsetRequest.new(0,-1,1000)]
+    offset_topic_requests = [TopicOffsetRequest.new("test2", partition_offset_requests)]
+    @connection.offset(offset_topic_requests)
+  end
+end

data/spec/integration/simple/multiple_brokers_spec.rb

@@ -0,0 +1,10 @@
+require 'integration/simple/spec_helper'
+
+RSpec.describe "three brokers in cluster", :type => :request do
+  include_context "a single broker cluster"
+
+  describe "sending batches of 1 message" do
+    it "sends messages to all brokers" do
+    end
+  end
+end

data/spec/integration/simple/simple_producer_and_consumer_spec.rb

@@ -0,0 +1,121 @@
+require 'integration/simple/spec_helper'
+
+RSpec.describe "simple producer and consumer", :type => :request do
+  include_context "a single broker cluster"
+
+  describe "writing and consuming one topic" do
+    it "fetches produced messages" do
+      @producer = Producer.new(["localhost:9092"],
+                               "test_client",
+                               :type => :sync)
+
+
+      messages = [MessageToSend.new("topic_simple_producer_and_consumer", "Hello World")]
+      expect(@producer.send_messages(messages)).to eq(true)
+
+      @consumer = PartitionConsumer.new("test_consumer", "localhost", 9092,
+                                        "topic_simple_producer_and_consumer", 0, -2)
+      messages = @consumer.fetch
+      expect(messages.last.value).to eq("Hello World")
+
+      @producer.close
+    end
+
+    it "fetches only messages since the last offset" do
+      @producer = Producer.new(["localhost:9092"],
+                               "test_client",
+                               :type => :sync,
+                               :required_acks => 1)
+
+      @consumer = PartitionConsumer.new("test_consumer", "localhost", 9092,
+                                        "topic_simple_producer_and_consumer", 0, -1)
+
+      # Read up to the end of the current messages (if there are any)
+      begin
+        @consumer.fetch
+      rescue Errors::UnknownTopicOrPartition
+      end
+
+      # First Batch
+      messages = [MessageToSend.new("topic_simple_producer_and_consumer", "Hello World")]
+      expect(@producer.send_messages(messages)).to eq(true)
+
+      messages = @consumer.fetch
+      expect(messages.last.value).to eq("Hello World")
+
+      # Second Batch
+      messages = [MessageToSend.new("topic_simple_producer_and_consumer", "Hello World Again")]
+      expect(@producer.send_messages(messages)).to eq(true)
+
+      messages = @consumer.fetch
+      expect(messages.map(&:value)).to eq(["Hello World Again"])
+
+      # Empty Batch
+      messages = @consumer.fetch
+      expect(messages.empty?).to eq(true)
+    end
+
+    it "waits for messages" do
+      # Create topic
+      @c = Connection.new("localhost", 9092, "metadata_fetcher", 10_000)
+      @c.topic_metadata(["simple_wait_test"])
+
+      sleep 5
+      @consumer = PartitionConsumer.new("test_consumer", "localhost", 9092,
+                                        "simple_wait_test", 0, :earliest_offset,
+                                        :max_wait_ms => 2500)
+
+      require 'benchmark'
+      n = Benchmark.realtime do
+        @consumer.fetch
+      end
+      expect(n).to be_within(0.25).of(2.5)
+    end
+
+    # Not sure what's going on here, will revisit.
+=begin
+    it "fetches larger messages with a larger max bytes size" do
+      @producer = Producer.new(["localhost:9092"],
+                                "test_client",
+                                :type => :sync,
+                                :required_acks => 1)
+
+      @consumer = PartitionConsumer.new("test_consumer", "localhost", 9092,
+                                        "topic_simple_producer_and_consumer", 0, -2)
+
+      messages = []
+      2000.times do
+        messages << MessageToSend.new("topic_simple_producer_and_consumer",'KcjNyFBtqfSbpwjjcGKckMKLUCWz83IVcp21C8FQzs8JJKKTTrc4OLxSjLpYc5z7fsncX59te2cBn0sWDRaYmRuZyttRMLMHvXrM5o3QReKPIYUKzVCFahC4cb3Ivcbb5ZuS98Ohnb7Io42Bz9FucXwwGkQyFhJwyn3nD3BYs5r8TZM8Q76CGR2kTH1rjnFeB7J3hrRKukztxCrDY3smrQE1bbVR80IF3yWlhzkdfv3cpfwnD0TKadtt21sFJANFmORAJ0HKs6Z2262hcBQyF7WcWypC2RoLWVgKVQxbouVUP7yV6YYOAQEevYrl9sOB0Yi6h1mS8fTBUmRTmWLqyl8KzwbnbQvmCvgnX26F5JEzIoXsVaoDT2ks5eep9RyE1zm5yPtbYVmd2Sz7t5ru0wj6YiAmbF7Xgiw2l4VpNOxG0Ec6rFxXRXs0bahyBd2YtxpGyZBeruIK1RAN4n0t97xVXgZG5CGoVhL1oRDxw2pTbwEO1cvwHiiYXpXSqaxF7G9kiiPsQt24Vu7chXrJT7Xqv4RIg1aOT5Os5JVlISaJCmx8ZLtbC3OjAdGtF1ZkDuUeQHHohqeKh0qBJjw7Rv1oSDwcM0MRazjF36jijpYg26Qml9lSEnGYIFLQWHVDWKqqhl2GIntjxDXn1IyI')
+      end
+      expect(@producer.send_messages(messages)).to eq(true)
+
+      messages = @consumer.fetch
+      expect(messages.length).to be > 2
+
+      @consumer = PartitionConsumer.new("test_consumer", "localhost", 9092,
+                                        "topic_simple_producer_and_consumer", 0, -2)
+      messages = @consumer.fetch(:max_bytes => 1400000)
+      expect(messages.length).to be > 2
+    end
+=end
+  end
+
+  describe "broker that becomes unavailable" do
+    it "fails the fetch" do
+      @producer = Producer.new(["localhost:9092"],
+                               "test_client",
+                               :type => :sync)
+
+
+      messages = [MessageToSend.new("topic_simple_producer_and_consumer", "Hello World")]
+      expect(@producer.send_messages(messages)).to eq(true)
+
+      @consumer = PartitionConsumer.new("test_consumer", "localhost", 9092,
+                                        "topic_simple_producer_and_consumer", 0, -2)
+
+      $tc.broker.without_process do
+        expect { @consumer.fetch }.to raise_error(Connection::ConnectionFailedError)
+      end
+    end
+  end
+end