poseidon 0.0.4 → 0.0.5.pre1

data/lib/poseidon/producer.rb

@@ -125,9 +125,12 @@ module Poseidon
     # @option options [Integer] :required_acks (0)
     #   The number of acks required per request.
     #
-    # @option options [Integer] :request_timeout_ms (1500)
+    # @option options [Integer] :ack_timeout_ms (1500)
     #   How long the producer waits for acks.
     #
+    # @option options [Integer] :socket_timeout_ms] (10000)
+    #   How long the producer socket waits for any reply from server.
+    #
     # @api public
     def initialize(brokers, client_id, options = {})
       options = options.dup

data/lib/poseidon/protocol/request_buffer.rb

@@ -7,12 +7,10 @@ module Poseidon
     # (https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-ProtocolPrimitiveTypes) 
     class RequestBuffer
       def initialize
-        @s = ''.encode("ASCII-8BIT")
+        @s = ''
       end
 
       def append(string)
-        string = string.dup
-        string.force_encoding("ASCII-8BIT")
         @s << string
         nil
       end
@@ -55,23 +53,33 @@ module Poseidon
       end
 
       def prepend_crc32
+        ensure_ascii
         checksum_pos = @s.bytesize
         @s += " "
         yield
+        ensure_ascii
         @s[checksum_pos] = [Zlib::crc32(@s[(checksum_pos+1)..-1])].pack("N")
         nil
       end
 
       def prepend_size
+        ensure_ascii
         size_pos = @s.bytesize
         @s += " "
         yield
+        ensure_ascii
         @s[size_pos] = [(@s.bytesize-1) - size_pos].pack("N")
         nil
       end
 
       def to_s
-        @s
+        ensure_ascii
+      end
+
+      private
+
+      def ensure_ascii
+        @s.force_encoding("ASCII-8BIT")
       end
     end
   end

data/lib/poseidon/sync_producer.rb

@@ -21,10 +21,11 @@ module Poseidon
       :retry_backoff_ms => 100,
       :required_acks => 0,
       :ack_timeout_ms => 1500,
+      :socket_timeout_ms => 10_000
     }
 
     attr_reader :client_id, :retry_backoff_ms, :max_send_retries,
-      :metadata_refresh_interval_ms, :required_acks, :ack_timeout_ms
+      :metadata_refresh_interval_ms, :required_acks, :ack_timeout_ms, :socket_timeout_ms
     def initialize(client_id, seed_brokers, options = {})
       @client_id = client_id
 
@@ -32,7 +33,7 @@ module Poseidon
 
       @cluster_metadata   = ClusterMetadata.new
       @message_conductor  = MessageConductor.new(@cluster_metadata, @partitioner)
-      @broker_pool        = BrokerPool.new(client_id, seed_brokers)
+      @broker_pool        = BrokerPool.new(client_id, seed_brokers, socket_timeout_ms)
     end
 
     def send_messages(messages)
@@ -40,22 +41,20 @@ module Poseidon
 
       messages_to_send = MessagesToSend.new(messages, @cluster_metadata)
 
-      (@max_send_retries+1).times do
-        if messages_to_send.needs_metadata? || refresh_interval_elapsed?
-          refreshed_metadata = refresh_metadata(messages_to_send.topic_set)
-          if !refreshed_metadata
-            # If we can't refresh metadata we have to give up.
-            break
-          end
-        end
+      if refresh_interval_elapsed?
+        refresh_metadata(messages_to_send.topic_set)
+      end
+
+      ensure_metadata_available_for_topics(messages_to_send)
 
+      (@max_send_retries+1).times do
         messages_to_send.messages_for_brokers(@message_conductor).each do |messages_for_broker|
-          if send_to_broker(messages_for_broker)
-            messages_to_send.successfully_sent(messages_for_broker)
+          if sent = send_to_broker(messages_for_broker)
+            messages_to_send.successfully_sent(sent)
           end
         end
 
-        if messages_to_send.all_sent? || @max_send_retries == 0
+        if !messages_to_send.pending_messages? || @max_send_retries == 0
           break
         else
           Kernel.sleep retry_backoff_ms / 1000.0
@@ -63,7 +62,11 @@ module Poseidon
         end
       end
 
-      messages_to_send.all_sent?
+      if messages_to_send.pending_messages?
+        raise "Failed to send all messages: #{messages_to_send.messages} remaining"
+      else
+        true
+      end
     end
 
     def shutdown
@@ -71,8 +74,27 @@ module Poseidon
     end
     
     private
+
+    def ensure_metadata_available_for_topics(messages_to_send)
+      return if !messages_to_send.needs_metadata?
+
+      Poseidon.logger.debug { "Fetching metadata for #{messages_to_send.topic_set}. (Attempt 1)" }
+      refresh_metadata(messages_to_send.topic_set)
+      return if !messages_to_send.needs_metadata?
+
+      2.times do |n|
+        sleep 5
+
+        Poseidon.logger.debug { "Fetching metadata for #{messages_to_send.topic_set}. (Attempt #{n+2})" }
+        refresh_metadata(messages_to_send.topic_set)
+        return if !messages_to_send.needs_metadata?
+      end
+      raise Errors::UnableToFetchMetadata
+    end
+
     def handle_options(options)
       @ack_timeout_ms    = handle_option(options, :ack_timeout_ms)
+      @socket_timeout_ms = handle_option(options, :socket_timeout_ms)
       @retry_backoff_ms  = handle_option(options, :retry_backoff_ms)
 
       @metadata_refresh_interval_ms = 
@@ -95,23 +117,34 @@ module Poseidon
     end
 
     def refresh_interval_elapsed?
-      (Time.now - @cluster_metadata.last_refreshed_at) > metadata_refresh_interval_ms
+      @cluster_metadata.last_refreshed_at.nil? ||
+        (Time.now - @cluster_metadata.last_refreshed_at) > metadata_refresh_interval_ms
     end
 
     def refresh_metadata(topics)
-      @cluster_metadata.update(@broker_pool.fetch_metadata(topics))
+      topics_to_refresh = topics.dup
+
+      @cluster_metadata.topics.each do |topic|
+        topics_to_refresh.add(topic)
+      end
+
+      @cluster_metadata.update(@broker_pool.fetch_metadata(topics_to_refresh))
       @broker_pool.update_known_brokers(@cluster_metadata.brokers)
-      true
-    rescue Errors::UnableToFetchMetadata
-      false
     end
 
     def send_to_broker(messages_for_broker)
       return false if messages_for_broker.broker_id == -1
       to_send = messages_for_broker.build_protocol_objects(@compression_config)
-      @broker_pool.execute_api_call(messages_for_broker.broker_id, :produce, 
-                                    required_acks, ack_timeout_ms,
-                                    to_send)
+
+      Poseidon.logger.debug { "Sending messages to broker #{messages_for_broker.broker_id}" }
+      response = @broker_pool.execute_api_call(messages_for_broker.broker_id, :produce,
+                                              required_acks, ack_timeout_ms,
+                                              to_send)
+      if required_acks == 0
+        messages_for_broker.messages
+      else
+        messages_for_broker.successfully_sent(response)
+      end
     rescue Connection::ConnectionFailedError
       false
     end

data/lib/poseidon/topic_metadata.rb

@@ -26,10 +26,6 @@ module Poseidon
       nil
     end
 
-    def partitions
-      struct.partitions
-    end
-
     def name
       struct.name
     end
@@ -38,6 +34,10 @@ module Poseidon
       eql?(o)
     end
 
+    def exists?
+      struct.error == 0
+    end
+
     def eql?(o)
       struct.eql?(o.struct)
     end
@@ -54,21 +54,36 @@ module Poseidon
       @partition_count ||= struct.partitions.count
     end
 
-    def available_partition_leader_ids
-      @available_partition_leader_ids ||= struct.partitions.select(&:leader)
+    def available_partitions
+      @available_partitions ||= struct.partitions.select do |partition|
+        partition.error == 0 && partition.leader != -1
+      end
     end
 
     def available_partition_count
-      @available_partition_count ||= available_partition_leader_ids.count
+      available_partitions.count
     end
 
     def partition_leader(partition_id)
-      partition = struct.partitions.find { |p| p.id == partition_id }
+      partition = partitions_by_id[partition_id]
       if partition
         partition.leader
       else
         nil
       end
     end
+
+    def to_s
+      struct.partitions.map { |p| p.inspect }.join("\n")
+    end
+
+    private
+    def partitions_by_id
+      @partitions_by_id ||= Hash[partitions.map { |p| [p.id, p] }]
+    end
+
+    def partitions
+      struct.partitions
+    end
   end
 end

data/lib/poseidon/version.rb

@@ -1,4 +1,4 @@
 module Poseidon
   # Unstable! API May Change!
-  VERSION = "0.0.4"
+  VERSION = "0.0.5.pre1"
 end

data/poseidon.gemspec

@@ -12,14 +12,14 @@ Gem::Specification.new do |gem|
   gem.summary       = %q{Poseidon is a producer and consumer implementation for Kafka >= 0.8}
   gem.homepage      = "https://github.com/bpot/poseidon"
   gem.licenses      = ["MIT"]
+  gem.required_ruby_version = '>= 1.9.3'
 
   gem.files         = `git ls-files`.split($/)
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
-  gem.add_development_dependency(%q<rspec>)
+  gem.add_development_dependency(%q<rspec>, '~> 2.13.0')
   gem.add_development_dependency(%q<yard>)
   gem.add_development_dependency(%q<simplecov>)
-  gem.add_development_dependency(%q<daemon_controller>)
 end

data/spec/integration/multiple_brokers/consumer_spec.rb

@@ -3,7 +3,7 @@ require 'integration/multiple_brokers/spec_helper'
 describe "consuming with multiple brokers" do
   before(:each) do
     # autocreate the topic by asking for information about it
-    c = Connection.new("localhost", 9092, "metadata_fetcher")
+    c = Connection.new("localhost", 9092, "metadata_fetcher", 10_000)
     md = c.topic_metadata(["test"])
     sleep 1
   end

data/spec/integration/multiple_brokers/metadata_failures_spec.rb

@@ -0,0 +1,35 @@
+require 'integration/multiple_brokers/spec_helper'
+
+describe "handling failures" do
+  describe "metadata failures" do
+    before(:each) do
+      @messages_to_send = [
+        MessageToSend.new("topic1", "hello"),
+        MessageToSend.new("topic2", "hello")
+      ]
+    end
+
+    describe "unable to connect to brokers" do
+      before(:each) do
+        @p = Producer.new(["localhost:1092","localhost:1093","localhost:1094"], "producer")
+      end
+
+      it "triggers callback failures for both topics" do
+        expect {
+          @p.send_messages(@messages_to_send)
+        }.to raise_error(Poseidon::Errors::UnableToFetchMetadata)
+      end
+    end
+  end
+
+  describe "unknown topic" do
+    it "receives error callback" do
+      pending "need a way to turn off auto-topic creation just for this test"
+      @p = Producer.new(["localhost:9092","localhost:9093","localhost:9094"], "producer")
+
+      expect {
+        @p.send_messages([MessageToSend.new("imnothere", "hello")])
+      }.to raise_error(Poseidon::Errors::UnableToFetchMetadata)
+    end
+  end
+end

data/spec/integration/multiple_brokers/rebalance_spec.rb

@@ -0,0 +1,67 @@
+require 'integration/multiple_brokers/spec_helper'
+
+describe "producer handles rebalancing" do
+  before(:each) do
+    # autocreate the topic by asking for information about it
+    @c = Connection.new("localhost", 9093, "metadata_fetcher", 10_000)
+    @c.topic_metadata(["failure_spec"])
+    sleep 1
+  end
+
+  def current_leadership_mapping(c)
+    metadata = c.topic_metadata(["failure_spec"])
+    topic_metadata = metadata.topics.find { |t| t.name == "failure_spec" }
+    (0..2).map { |p| topic_metadata.partition_leader(p) }
+  end
+
+  it "produces a bunch of messages and consumes all without error" do
+    @p = Producer.new(["localhost:9092","localhost:9093","localhost:9094"], "test",
+                     :required_acks => -1)
+
+    1.upto(25) do |n|
+      @p.send_messages([MessageToSend.new("failure_spec", n.to_s)])
+    end
+
+    # The goal here is to have the producer attempt to send messages
+    # to a broker which is no longer the leader for the partition.
+    #
+    # We accomplish this by turning off a broker which causes leadership
+    # to failover. Then we turn that broker back on and begin sending
+    # messages. While sending messages, the kafka cluster should rebalance
+    # the partitions causing leadership to switch back to the original
+    # broker in the midst of messages being sent.
+    #
+    # We compare leadership before and after the message sending period
+    # to make sure we were successful.
+    $tc.stop_first_broker
+    sleep 30
+    SPEC_LOGGER.info "Pre start #{current_leadership_mapping(@c).inspect}"
+    $tc.start_first_broker
+
+    pre_send_leadership = current_leadership_mapping(@c)
+    SPEC_LOGGER.info "Pre send #{pre_send_leadership.inspect}"
+    26.upto(50) do |n|
+      sleep 0.5
+      @p.send_messages([MessageToSend.new("failure_spec", n.to_s)])
+    end
+    post_send_leadership = current_leadership_mapping(@c)
+    SPEC_LOGGER.info "Post send #{post_send_leadership.inspect}"
+
+    expect(pre_send_leadership).to_not eq(post_send_leadership)
+
+    messages = []
+    0.upto(2) do |partition|
+      consumer = PartitionConsumer.consumer_for_partition("consumer_failure_spect",
+                                                          ["localhost:9092","localhost:9093","localhost:9094"],
+                                                          "failure_spec",
+                                                          partition,
+                                                          :earliest_offset)
+      while (fetched = consumer.fetch).any?
+        messages.push(*fetched)
+      end
+    end
+
+    expect(messages.size).to eq(50)
+    expect(messages.map { |m| m.value.to_i }.sort).to eq((1..50).to_a)
+  end
+end

data/spec/integration/multiple_brokers/round_robin_spec.rb

@@ -3,14 +3,14 @@ require 'integration/multiple_brokers/spec_helper'
 describe "round robin sending" do
   describe "with small message batches" do
     it "evenly distributes messages across brokers" do
-      c = Connection.new("localhost", 9092, "metadata_fetcher")
+      c = Connection.new("localhost", 9092, "metadata_fetcher", 10_000)
       md = c.topic_metadata(["test"])
       sleep 1
       md = c.topic_metadata(["test"])
 
       test_topic = md.topics.first
 
-      consumers = test_topic.partitions.map do |partition|
+      consumers = test_topic.send(:partitions).map do |partition|
         leader_id = partition.leader
         broker = md.brokers.find { |b| b.id == leader_id }
         PartitionConsumer.new("test_consumer_#{partition.id}", broker.host,
@@ -22,14 +22,14 @@ describe "round robin sending" do
         c.fetch
       end
 
-
       @p = Producer.new(["localhost:9092","localhost:9093","localhost:9094"], "test",
                        :required_acks => 1)
-
       24.times do
         @p.send_messages([MessageToSend.new("test", "hello")])
       end
 
+      sleep 5
+
       consumers.each do |c|
         messages = c.fetch
         expect(messages.size).to eq(8)

data/spec/integration/multiple_brokers/spec_helper.rb

@@ -3,32 +3,54 @@ require 'spec_helper'
 require 'test_cluster'
 
 class ThreeBrokerCluster
-  def initialize
+  def initialize(properties = {})
     @zookeeper = ZookeeperRunner.new
-    @brokers = (9092..9094).map { |port| BrokerRunner.new(port - 9092, port, 3) }
+    @brokers = (9092..9094).map { |port| BrokerRunner.new(port - 9092, port,
+                                                          3,
+                                                          2,
+                                                          properties) }
   end
 
   def start
     @zookeeper.start
     @brokers.each(&:start)
+    sleep 5
   end
 
   def stop
-    @zookeeper.stop
+    SPEC_LOGGER.info "Stopping three broker cluster"
+    SPEC_LOGGER.info "Stopping brokers"
     @brokers.each(&:stop)
+    sleep 5
+
+    SPEC_LOGGER.info "Stopping ZK"
+    @zookeeper.stop
+    sleep 5
+  end
+
+  def stop_first_broker
+    SPEC_LOGGER.info "Stopping first broker"
+    @brokers.first.stop
+    sleep 5
+  end
+
+  def start_first_broker
+    SPEC_LOGGER.info "Starting first broker"
+    @brokers.first.start
   end
 end
 
 RSpec.configure do |config|
-  config.before(:suite) do
+  config.before(:each) do
     JavaRunner.remove_tmp
     JavaRunner.set_kafka_path!
     $tc = ThreeBrokerCluster.new
     $tc.start
-    sleep 5 # wait for cluster to come up
+    SPEC_LOGGER.info "Waiting on cluster"
+    sleep 10 # wait for cluster to come up
   end
 
-  config.after(:suite) do
-    $tc.stop
+  config.after(:each) do
+    $tc.stop if $tc
   end
 end