poseidon 0.0.4 → 0.0.5.pre1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4c4852d5b615cd1fe50528b0136602756087d0db
-  data.tar.gz: 2172c5cd70c8047cb0b0bdd171d5f9ed5cb82b73
+  metadata.gz: 6b96a319eb67baf9aa7081d2c2800cf9d24eb672
+  data.tar.gz: fb213c7eb1e31de507c3160bdd6fc6a4fd04084c
 SHA512:
-  metadata.gz: 3949ab96ce033255e31bc588db0f0895c6a72a91d9bfb56137995f0d5f52bc467dfa660359d440fddca89fe64fd3f6af7a2b28f2476698ba35319f999358a721
-  data.tar.gz: 2348e329e244a3b4a777e8635143d33ec591c10ee411f40aa69a629487643585e8b303c13f120d51e85a2dde13f5dea5b0cb3290b152e619d3fa07dc6f4769d4
+  metadata.gz: 1f6daff2a688fafccd154005ef4a3c99c1cb0b3b4cadaf3776695b49607b25f4e2b58863341aa3a15bc47fdcfa40cfa16de0860c3c75d68a28b237185c94c6e7
+  data.tar.gz: 0861e6e8539a62174f01ea7bf9cd3d8c6bec9b77b38488465484e08a670d38713bd8c61636b6496818052f85cf9003eb2ec40e11ae553d28d8ae5b2b22bcb69f

data/.gitignore

@@ -17,3 +17,5 @@ test/version_tmp
 tmp
 *.log
 *.log.*
+tags
+.rvmrc

data/.travis.yml

@@ -2,6 +2,7 @@ laguage: ruby
 rvm:
   - 1.9.3
   - 2.0.0
+  - 2.1.2
   - ruby-head
   - jruby-19mode
   - jruby-head
@@ -9,4 +10,5 @@ rvm:
 matrix:
   allow_failures:
     - rvm: ruby-head
+    - rvm: jruby-head
     - rvm: rbx-19mode

data/CHANGES.md

@@ -1,3 +1,7 @@
+# 0.0.5 (Unreleased)
+
+* Fix serious bug where we would send messages to the wrong partition [GH-36]. (Thanks @sclasen and @jorgeortiz85 for tracking this down.)
+
 # 0.0.4
 
 * Don't truncate UTF8 Messages [GH-18]

data/README.md

@@ -13,7 +13,7 @@ Poseidon is a Kafka client. Poseidon only supports the 0.8 API and above.
 
 ### Installing a Kafka broker locally
 
-Follow the [instructions](https://cwiki.apache.org/KAFKA/kafka-08-quick-start.html) on the Kafka wiki to build Kafka 0.8 and get a test broker up and running.
+Follow the [instructions](http://kafka.apache.org/documentation.html#quickstart) on the Kafka wiki to build Kafka 0.8 and get a test broker up and running.
 
 ### Sending messages to Kafka
 
@@ -60,15 +60,10 @@ This gem follows [SemVer](http://semver.org). In particular, the public API shou
 
 ## Integration Tests
 
-In order to run integration tests you must specify a `KAFKA_PATH` environment variable which points to a built Kafka installation.  There are more detailed [instructions](https://cwiki.apache.org/KAFKA/kafka-08-quick-start.html) on the Kafka wiki, but the following should allow you to run integration tests.
+In order to run integration tests you must specify a `KAFKA_PATH` environment variable which points to a built Kafka installation.  To build Kafka locally follow the [instructions](http://kafka.apache.org/documentation.html#quickstart) provided by the project.
 
-    # cd ~/src/
-    # git clone https://git-wip-us.apache.org/repos/asf/kafka.git
-    # git checkout -b 0.8 remotes/origin/0.8
-    # ./sbt update
-    # ./sbt package
-    # ./sbt assembly-package-dependency
     # cd ~/src/poseidon/
-    # KAFKA_PATH=~/src/kafka rake spec:integration:simple
+    # bundle
+    # KAFKA_PATH=~/src/kafka bundle exec rake spec:all # run all unit and integration specs
 
 The poseidon test suite will take care of spinning up and down the broker(s) needed for the integration tests.

data/Rakefile

@@ -7,11 +7,14 @@ end
 
 RSpec::Core::RakeTask.new('spec:integration:simple') do |t|
   t.pattern = 'spec/integration/simple/*_spec.rb'
+  t.rspec_opts = ["--fail-fast", "-f d"]
 end
 
 RSpec::Core::RakeTask.new('spec:integration:multiple_brokers') do |t|
   t.pattern = 'spec/integration/multiple_brokers/*_spec.rb'
+  t.rspec_opts = ["--fail-fast", "-f d"]
 end
 
 task :spec => 'spec:unit'
+task 'spec:all' => ['spec:unit', 'spec:integration:simple', 'spec:integration:multiple_brokers']
 task :default => 'spec:unit'

data/lib/poseidon.rb

@@ -3,30 +3,7 @@ require 'socket'
 require 'zlib'
 require 'thread'
 require 'set'
-
-# Public API
-require "poseidon/message_to_send"
-require "poseidon/producer"
-require "poseidon/fetched_message"
-require "poseidon/partition_consumer"
-
-# Poseidon!
-require "poseidon/message"
-require "poseidon/message_set"
-require "poseidon/topic_metadata"
-require "poseidon/protocol"
-
-require "poseidon/broker_pool"
-require "poseidon/cluster_metadata"
-require "poseidon/compression"
-require "poseidon/connection"
-require "poseidon/message_conductor"
-require "poseidon/messages_for_broker"
-require "poseidon/messages_to_send"
-require "poseidon/messages_to_send_batch"
-require "poseidon/producer_compression_config"
-require "poseidon/sync_producer"
-require "poseidon/version"
+require 'logger'
 
 # Top level Poseidon namespace
 #
@@ -99,4 +76,44 @@ module Poseidon
     # object that has been #shutdown
     class ProducerShutdownError < StandardError; end
   end
+
+  def self.logger
+    @logger ||= null_logger
+  end
+
+  def self.logger=(logger)
+    @logger = logger
+  end
+
+  private
+  def self.null_logger
+    devnull = RUBY_PLATFORM =~ /w32/ ? 'nul' : '/dev/null'
+    l = Logger.new(devnull)
+    l.level = Logger::INFO
+    l
+  end
 end
+
+# Public API
+require "poseidon/message_to_send"
+require "poseidon/producer"
+require "poseidon/fetched_message"
+require "poseidon/partition_consumer"
+
+# Poseidon!
+require "poseidon/message"
+require "poseidon/message_set"
+require "poseidon/topic_metadata"
+require "poseidon/protocol"
+
+require "poseidon/broker_pool"
+require "poseidon/cluster_metadata"
+require "poseidon/compression"
+require "poseidon/connection"
+require "poseidon/message_conductor"
+require "poseidon/messages_for_broker"
+require "poseidon/messages_to_send"
+require "poseidon/messages_to_send_batch"
+require "poseidon/producer_compression_config"
+require "poseidon/sync_producer"
+require "poseidon/version"

data/lib/poseidon/broker_pool.rb

@@ -6,16 +6,18 @@ module Poseidon
     class UnknownBroker < StandardError; end
 
     # @param [String] client_id
-    def initialize(client_id, seed_brokers)
+    def initialize(client_id, seed_brokers, socket_timeout_ms)
       @connections = {}
       @brokers = {}
       @client_id = client_id
       @seed_brokers = seed_brokers
+      @socket_timeout_ms = socket_timeout_ms
     end
 
     def fetch_metadata(topics)
       @seed_brokers.each do |broker|
         if metadata = fetch_metadata_from_broker(broker, topics)
+          Poseidon.logger.debug { "Fetched metadata\n" + metadata.to_s }
           return metadata
         end
       end
@@ -51,10 +53,12 @@ module Poseidon
     private
     def fetch_metadata_from_broker(broker, topics)
       host, port = broker.split(":")
-      c = Connection.new(host, port, @client_id)
+      c = Connection.new(host, port, @client_id, @socket_timeout_ms)
       c.topic_metadata(topics)
     rescue Connection::ConnectionFailedError
       return nil
+    ensure
+      c && c.close
     end
 
     def connection(broker_id)
@@ -66,7 +70,7 @@ module Poseidon
       if info.nil?
         raise UnknownBroker
       end
-      Connection.new(info[:host], info[:port], @client_id)
+      Connection.new(info[:host], info[:port], @client_id, @socket_timeout_ms)
     end
   end
 end

data/lib/poseidon/cluster_metadata.rb

@@ -57,10 +57,26 @@ module Poseidon
       end
     end
 
+    def topics
+      @topic_metadata.keys
+    end
+
+    def to_s
+      out = ""
+      @topic_metadata.each do |topic, metadata|
+        out << "Topic: #{topic}"
+        out << "-------------------------"
+        out << metadata.to_s
+      end
+      out
+    end
+
     private
     def update_topics(topics)
       topics.each do |topic|
-        @topic_metadata[topic.name] = topic
+        if topic.exists?
+          @topic_metadata[topic.name] = topic
+        end
       end
     end
 

data/lib/poseidon/connection.rb

@@ -6,6 +6,7 @@ module Poseidon
     include Protocol
 
     class ConnectionFailedError < StandardError; end
+    class TimeoutException < Exception; end
 
     API_VERSION = 0
     REPLICA_ID = -1 # Replica id is always -1 for non-brokers
@@ -17,18 +18,19 @@ module Poseidon
     # @param [String] host Host to connect to
     # @param [Integer] port Port broker listens on
     # @param [String] client_id Unique across processes?
-    def initialize(host, port, client_id)
+    def initialize(host, port, client_id, socket_timeout_ms)
       @host = host
       @port = port
 
       @client_id = client_id
+      @socket_timeout_ms = socket_timeout_ms
     end
 
     # Close broker connection
     def close
       @socket && @socket.close
     end
-  
+
     # Execute a produce call
     #
     # @param [Integer] required_acks
@@ -93,32 +95,48 @@ module Poseidon
         begin
           @socket = TCPSocket.new(@host, @port)
         rescue SystemCallError
-          raise ConnectionFailedError
+          raise_connection_failed_error
         end
       end
     end
 
     def read_response(response_class)
-      r = @socket.read(4)
+      r = ensure_read_or_timeout(4)
       if r.nil?
-        raise ConnectionFailedError
+        raise_connection_failed_error
       end
       n = r.unpack("N").first
-      s = @socket.read(n)
+      s = ensure_read_or_timeout(n)
       buffer = Protocol::ResponseBuffer.new(s)
       response_class.read(buffer)
-    rescue Errno::ECONNRESET
+    rescue Errno::ECONNRESET, SocketError, TimeoutException
       @socket = nil
-      raise ConnectionFailedError
+      raise_connection_failed_error
+    end
+
+    def ensure_read_or_timeout(maxlen)
+      if IO.select([@socket], nil, nil, @socket_timeout_ms / 1000.0)
+         @socket.read(maxlen)
+      else
+         raise TimeoutException.new
+      end
     end
 
     def send_request(request)
       buffer = Protocol::RequestBuffer.new
       request.write(buffer)
-      @socket.write([buffer.to_s.bytesize].pack("N") + buffer.to_s)
-    rescue Errno::EPIPE, Errno::ECONNRESET
+      ensure_write_or_timeout([buffer.to_s.bytesize].pack("N") + buffer.to_s)
+    rescue Errno::EPIPE, Errno::ECONNRESET, TimeoutException
       @socket = nil
-      raise ConnectionFailedError
+      raise_connection_failed_error
+    end
+
+    def ensure_write_or_timeout(data)
+      if IO.select(nil, [@socket], nil, @socket_timeout_ms / 1000.0)
+        @socket.write(data)
+      else
+        raise TimeoutException.new
+      end
     end
 
     def request_common(request_type)
@@ -134,5 +152,9 @@ module Poseidon
       @correlation_id ||= 0
       @correlation_id  += 1
     end
+
+    def raise_connection_failed_error
+      raise ConnectionFailedError, "Failed to connect to #{@host}:#{@port}"
+    end
   end
 end

data/lib/poseidon/message_conductor.rb

@@ -31,7 +31,7 @@ module Poseidon
       topic_metadata = topic_metadatas[topic]
       if topic_metadata && topic_metadata.leader_available?
         partition_id = determine_partition(topic_metadata, key)
-        broker_id    = topic_metadata.partitions[partition_id].leader || NO_BROKER
+        broker_id    = topic_metadata.partition_leader(partition_id) || NO_BROKER
       else
         partition_id  = NO_PARTITION
         broker_id     = NO_BROKER
@@ -73,7 +73,7 @@ module Poseidon
       partition_count = topic_metadata.available_partition_count
 
       if partition_count > 0
-        next_partition_counter % partition_count
+        topic_metadata.available_partitions[next_partition_counter % partition_count].id
       else
         NO_PARTITION
       end

data/lib/poseidon/messages_for_broker.rb

@@ -35,5 +35,22 @@ module Poseidon
         Protocol::MessagesForTopic.new(topic, messages_for_partitions)
       end
     end
+
+    # We can always retry these errors because they mean none of the kafka brokers persisted the message
+    ALWAYS_RETRYABLE = [Poseidon::Errors::LeaderNotAvailable, Poseidon::Errors::NotLeaderForPartition]
+
+    def successfully_sent(producer_response)
+      failed = []
+      producer_response.topic_response.each do |topic_response|
+        topic_response.partitions.each do |partition|
+          if ALWAYS_RETRYABLE.include?(partition.error_class)
+            Poseidon.logger.debug { "Received #{partition.error_class} when attempting to send messages to #{topic_response.topic} on #{partition.partition}" }
+            failed.push(*@topics[topic_response.topic][partition.partition])
+          end
+        end
+      end
+
+      return @messages - failed
+    end
   end
 end

data/lib/poseidon/messages_to_send.rb

@@ -30,12 +30,12 @@ module Poseidon
       MessagesToSendBatch.new(@messages, message_conductor).messages_for_brokers
     end
 
-    def successfully_sent(messages_for_broker)
-      @messages -= messages_for_broker.messages
+    def successfully_sent(messages_sent)
+      @messages -= messages_sent
     end
 
-    def all_sent?
-      !@messages.any?
+    def pending_messages?
+      @messages.any?
     end
 
     private

data/lib/poseidon/partition_consumer.rb

@@ -14,18 +14,22 @@ module Poseidon
 
     attr_reader :offset
 
+    attr_reader :topic
+
     # Returns a consumer pointing at the lead broker for the partition.
     #
     # Eventually this will be replaced by higher level consumer functionality,
     # this is a stop-gap.
     #
     def self.consumer_for_partition(client_id, seed_brokers, topic, partition, offset, options = {})
-      broker_pool = BrokerPool.new(client_id, seed_brokers)
+      broker_pool = BrokerPool.new(client_id, seed_brokers, options[:socket_timeout_ms] || 10_000)
 
       cluster_metadata = ClusterMetadata.new
       cluster_metadata.update(broker_pool.fetch_metadata([topic]))
 
       broker = cluster_metadata.lead_broker_for_partition(topic, partition)
+      broker_pool.shutdown
+
       new(client_id, broker.host, broker.port, topic, partition, offset, options)
     end
 
@@ -38,35 +42,43 @@ module Poseidon
     # @param [String] topic Topic to read from
     # @param [Integer] partition Partitions are zero indexed.
     # @param [Integer,Symbol] offset 
-    #   Offset to start reading from.
+    #   Offset to start reading from. A negative offset can also be passed.
     #   There are a couple special offsets which can be passed as symbols:
-    #     :earliest_offset Start reading from the first offset the server has. 
-    #     :latest_offset   Start reading from the latest offset the server has. 
+    #     :earliest_offset       Start reading from the first offset the server has.
+    #     :latest_offset         Start reading from the latest offset the server has.
     #
     # @param [Hash] options
     #   Theses options can all be overridden in each individual fetch command.
     #
     # @option options [:max_bytes] Maximum number of bytes to fetch
     #   Default: 1048576 (1MB)
+    #
     # @option options [:max_wait_ms] 
     #   How long to block until the server sends us data.
+    #   NOTE: This is only enforced if min_bytes is > 0.
     #   Default: 100 (100ms)
+    #
     # @option options [:min_bytes] Smallest amount of data the server should send us.
-    #   Default: 0 (Send us data as soon as it is ready)
+    #   Default: 1 (Send us data as soon as it is ready)
+    #
+    # @option options [:socket_timeout_ms]
+    #   How long to wait for reply from server. Should be higher than max_wait_ms.
+    #   Default: 10000 (10s)
     #
     # @api public
     def initialize(client_id, host, port, topic, partition, offset, options = {})
       @host = host
       @port = port
 
-      @connection = Connection.new(host, port, client_id)
+      handle_options(options)
+
+      @connection = Connection.new(host, port, client_id, @socket_timeout_ms)
       @topic = topic
       @partition = partition
       if Symbol === offset
         raise ArgumentError, "Unknown special offset type: #{offset}" unless [:earliest_offset, :latest_offset].include?(offset)
       end
       @offset = offset
-      handle_options(options)
     end
 
     # Fetch messages from the broker.
@@ -75,14 +87,16 @@ module Poseidon
     #
     # @option options [:max_bytes]
     #   Maximum number of bytes to fetch
+    #
     # @option options [:max_wait_ms]
     #   How long to block until the server sends us data.
+    #
     # @option options [:min_bytes] 
     #   Smallest amount of data the server should send us.
     #
     # @api public
     def fetch(options = {})
-      fetch_max_wait = options.delete(:max_wait) || max_wait_ms
+      fetch_max_wait = options.delete(:max_wait_ms) || max_wait_ms
       fetch_max_bytes = options.delete(:max_bytes) || max_bytes
       fetch_min_bytes = options.delete(:min_bytes) || min_bytes
 
@@ -94,7 +108,14 @@ module Poseidon
       fetch_response = @connection.fetch(fetch_max_wait, fetch_min_bytes, topic_fetches)
       topic_response = fetch_response.topic_fetch_responses.first 
       partition_response = topic_response.partition_fetch_responses.first
-      if partition_response.error != Errors::NO_ERROR_CODE
+
+      unless partition_response.error == Errors::NO_ERROR_CODE
+        if @offset < 0 &&
+          Errors::ERROR_CODES[partition_response.error] == Errors::OffsetOutOfRange
+          @offset = :earliest_offset
+          return fetch(options)
+        end
+
         raise Errors::ERROR_CODES[partition_response.error]
       else
         @highwater_mark = partition_response.highwater_mark_offset
@@ -116,11 +137,27 @@ module Poseidon
       @offset
     end
 
+    # Close the connection to the kafka broker
+    #
+    # @return [Nil]
+    #
+    # @api public
+    def close
+      @connection.close
+      nil
+    end
+
     private
     def handle_options(options)
-      @max_bytes    = options.delete(:max_bytes) || 1024*1024
-      @min_bytes    = options.delete(:min_bytes) || 0
-      @max_wait_ms  = options.delete(:max_wait_ms) || 10_000
+      @max_bytes         = options.delete(:max_bytes) || 1024*1024
+      @min_bytes         = options.delete(:min_bytes) || 1
+      @max_wait_ms       = options.delete(:max_wait_ms) || 10_000
+      @socket_timeout_ms = options.delete(:socket_timeout_ms) || @max_wait_ms + 10_000
+
+      if @socket_timeout_ms < @max_wait_ms
+        raise ArgumentError, "Setting socket_timeout_ms should be higher than max_wait_ms"
+      end
+
       if options.keys.any?
         raise ArgumentError, "Unknown options: #{options.keys.inspect}"
       end
@@ -141,40 +178,46 @@ module Poseidon
     def resolve_offset_if_necessary
       return unless Symbol === @offset || @offset < 0
 
-      if @offset == :earliest_offset
-        @offset = -2
-      elsif @offset == :latest_offset
-        @offset = -1
+      protocol_offset = case @offset
+      when :earliest_offset
+        -2
+      when :latest_offset
+        -1
+      else
+        -1
       end
 
-      topic_offset_responses = @connection.offset(build_topic_offset_request)
+      topic_offset_responses = @connection.offset(build_topic_offset_request(protocol_offset))
       partition_offsets = topic_offset_responses.first.partition_offsets
       if partition_offsets.first.error != Errors::NO_ERROR_CODE
         raise Errors::ERROR_CODES[partition_offsets.first.error]
       end
 
       offset_struct = partition_offsets.first.offsets.first
-      if offset_struct.nil?
-        @offset = 0
+
+      @offset = if offset_struct.nil?
+        0
+      elsif @offset.kind_of?(Fixnum) && @offset < 0
+        offset_struct.offset + @offset
       else
-        @offset = offset_struct.offset
+        offset_struct.offset
       end
     end
 
-    def build_topic_offset_request
+    def build_topic_offset_request(protocol_offset)
       partition_offset_request = Protocol::PartitionOffsetRequest.new(
         @partition,
-        @offset,
+        protocol_offset,
         max_number_of_offsets = 1)
         
-      [Protocol::TopicOffsetRequest.new(@topic, [partition_offset_request])]
+      [Protocol::TopicOffsetRequest.new(topic, [partition_offset_request])]
     end
 
     def build_topic_fetch_request(max_bytes)
       partition_fetches = [Protocol::PartitionFetch.new(@partition,
                                                         next_offset,
                                                         max_bytes)]
-      topic_fetches = [Protocol::TopicFetch.new(@topic, partition_fetches)]
+      topic_fetches = [Protocol::TopicFetch.new(topic, partition_fetches)]
     end
   end
 end