poseidon 0.0.5.pre1 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6b96a319eb67baf9aa7081d2c2800cf9d24eb672
-  data.tar.gz: fb213c7eb1e31de507c3160bdd6fc6a4fd04084c
+  metadata.gz: a9abab19842081422bfcb903cf47a9dd248202bf
+  data.tar.gz: 94a4d93957d331fd9b1c78e8f32f0eab323135ba
 SHA512:
-  metadata.gz: 1f6daff2a688fafccd154005ef4a3c99c1cb0b3b4cadaf3776695b49607b25f4e2b58863341aa3a15bc47fdcfa40cfa16de0860c3c75d68a28b237185c94c6e7
-  data.tar.gz: 0861e6e8539a62174f01ea7bf9cd3d8c6bec9b77b38488465484e08a670d38713bd8c61636b6496818052f85cf9003eb2ec40e11ae553d28d8ae5b2b22bcb69f
+  metadata.gz: a1590c28e36139583f9fdd6558e7cc2b3497806909c2811fd68184cc5c8b5d3bdfca1c87746901d68cf7531b8d0641490dcb4871e1b583afdf30b8e2dc6084fb
+  data.tar.gz: 1b3f297c8c75ebdcc5bd2560eb56d6b2f8e390a4ffaf6ee3277ace90e8b10b3c77f9abc929e658dddbd60d5dcf2a94f91a9ae418f2e2f84ad02bc2f220d41653

data/.travis.yml

@@ -2,13 +2,13 @@ laguage: ruby
 rvm:
   - 1.9.3
   - 2.0.0
-  - 2.1.2
+  - 2.1
   - ruby-head
   - jruby-19mode
   - jruby-head
-  - rbx-19mode
+  - rbx-2
 matrix:
   allow_failures:
     - rvm: ruby-head
     - rvm: jruby-head
-    - rvm: rbx-19mode
+    - rvm: rbx-2

data/CHANGES.md

@@ -1,6 +1,14 @@
-# 0.0.5 (Unreleased)
-
-* Fix serious bug where we would send messages to the wrong partition [GH-36]. (Thanks @sclasen and @jorgeortiz85 for tracking this down.)
+# 0.0.5
+
+* Add support for negative offsets. [GH-24]
+* Fix serious bug where we would send messages to the wrong partition. [GH-36] (Thanks @sclasen and @jorgeortiz85 for tracking this down.)
+* Better error message when we can't connect to a broker. [GH-42]
+* Handle broker rebalances. [GH-43]
+* PartitionConsumer: Block for messages by default. [GH-48]
+* Add a logger to help debug issues. [GH-51]
+* Add snappy support. [GH-57]
+* Allow `:none` value for `:compression_codec` option. [GH-72]
+* Allow request buffer to accept mixed encodings. [GH-74]
 
 # 0.0.4
 

data/README.md

@@ -30,7 +30,6 @@ producer.send_messages(messages)
 
 More detailed [Poseidon::Producer](http://rubydoc.info/github/bpot/poseidon/Poseidon/Producer) documentation.
 
-
 ### Fetching messages from Kafka
 
 ```ruby
@@ -49,6 +48,10 @@ end
 
 More detailed [Poseidon::PartitionConsumer](http://rubydoc.info/github/bpot/poseidon/Poseidon/PartitionConsumer) documentation.
 
+### Using snappy compression
+
+To use snappy compression in your producers or consumers, install the [snappy](http://rubygems.org/gems/snappy) gem or simply add `gem 'snappy'` to your project's Gemfile.
+
 ## Semantic Versioning
 
 This gem follows [SemVer](http://semver.org). In particular, the public API should not be considered stable and anything may change without warning until Version 1.0.0.  Additionally, for the purposes of the versioning the public API is everything documented in the [public API docs](http://rubydoc.info/github/bpot/poseidon).

data/Rakefile

@@ -2,7 +2,7 @@ require 'bundler/gem_tasks'
 require 'rspec/core/rake_task'
 
 RSpec::Core::RakeTask.new("spec:unit") do |t|
-  t.pattern = 'spec/unit/*_spec.rb'
+  t.pattern = 'spec/unit/**/*_spec.rb'
 end
 
 RSpec::Core::RakeTask.new('spec:integration:simple') do |t|

data/lib/poseidon.rb

@@ -4,6 +4,7 @@ require 'zlib'
 require 'thread'
 require 'set'
 require 'logger'
+require 'stringio'
 
 # Top level Poseidon namespace
 #

data/lib/poseidon/broker_pool.rb

@@ -5,6 +5,15 @@ module Poseidon
   class BrokerPool
     class UnknownBroker < StandardError; end
 
+    # @yieldparam [BrokerPool]
+    def self.open(client_id, seed_brokers, socket_timeout_ms, &block)
+      broker_pool = new(client_id, seed_brokers, socket_timeout_ms)
+
+      yield broker_pool
+    ensure
+      broker_pool.close
+    end
+
     # @param [String] client_id
     def initialize(client_id, seed_brokers, socket_timeout_ms)
       @connections = {}
@@ -45,20 +54,21 @@ module Poseidon
     end
 
     # Closes all open connections to brokers
-    def shutdown
+    def close
       @brokers.values(&:close)
       @brokers = {}
     end
 
+    alias_method :shutdown, :close
+
     private
     def fetch_metadata_from_broker(broker, topics)
       host, port = broker.split(":")
-      c = Connection.new(host, port, @client_id, @socket_timeout_ms)
-      c.topic_metadata(topics)
+      Connection.open(host, port, @client_id, @socket_timeout_ms) do |connection|
+        connection.topic_metadata(topics)
+      end
     rescue Connection::ConnectionFailedError
       return nil
-    ensure
-      c && c.close
     end
 
     def connection(broker_id)

data/lib/poseidon/compression/gzip_codec.rb

@@ -7,8 +7,8 @@ module Poseidon
 
       def self.compress(s)
         io = StringIO.new
-        io.set_encoding("ASCII-8BIT")
-        gz = Zlib::GzipWriter.new io, Zlib::DEFAULT_COMPRESSION, Zlib::DEFAULT_STRATEGY, :encoding => "ASCII-8BIT"
+        io.set_encoding(Encoding::BINARY)
+        gz = Zlib::GzipWriter.new io, Zlib::DEFAULT_COMPRESSION, Zlib::DEFAULT_STRATEGY
         gz.write s
         gz.close
         io.string
@@ -16,7 +16,7 @@ module Poseidon
 
       def self.decompress(s)
         io = StringIO.new(s)
-        Zlib::GzipReader.new(io, :encoding => "ASCII-8BIT").read
+        Zlib::GzipReader.new(io).read
       end
     end
   end

data/lib/poseidon/compression/snappy_codec.rb

@@ -6,12 +6,24 @@ module Poseidon
       end
 
       def self.compress(s)
-        raise "Unimplemented"
+        check!
+        Snappy.deflate(s)
       end
 
       def self.decompress(s)
-        raise "Unimplemented"
+        check!
+        Snappy::Reader.new(StringIO.new(s)).read
       end
+
+      def self.check!
+        @checked ||= begin
+          require 'snappy'
+          true
+        rescue LoadError
+          raise "Snappy compression is not available, please install the 'snappy' gem"
+        end
+      end
+
     end
   end
 end

data/lib/poseidon/connection.rb

@@ -11,6 +11,15 @@ module Poseidon
     API_VERSION = 0
     REPLICA_ID = -1 # Replica id is always -1 for non-brokers
 
+    # @yieldparam [Connection]
+    def self.open(host, port, client_id, socket_timeout_ms, &block)
+      connection = new(host, port, client_id, socket_timeout_ms)
+
+      yield connection
+    ensure
+      connection.close
+    end
+
     attr_reader :host, :port
 
     # Create a new connection

data/lib/poseidon/partition_consumer.rb

@@ -22,13 +22,13 @@ module Poseidon
     # this is a stop-gap.
     #
     def self.consumer_for_partition(client_id, seed_brokers, topic, partition, offset, options = {})
-      broker_pool = BrokerPool.new(client_id, seed_brokers, options[:socket_timeout_ms] || 10_000)
 
-      cluster_metadata = ClusterMetadata.new
-      cluster_metadata.update(broker_pool.fetch_metadata([topic]))
+      broker = BrokerPool.open(client_id, seed_brokers, options[:socket_timeout_ms] || 10_000) do |broker_pool|
+        cluster_metadata = ClusterMetadata.new
+        cluster_metadata.update(broker_pool.fetch_metadata([topic]))
 
-      broker = cluster_metadata.lead_broker_for_partition(topic, partition)
-      broker_pool.shutdown
+        cluster_metadata.lead_broker_for_partition(topic, partition)
+      end
 
       new(client_id, broker.host, broker.port, topic, partition, offset, options)
     end

data/lib/poseidon/producer.rb

@@ -72,15 +72,16 @@ module Poseidon
   class Producer
     # @api private
     VALID_OPTIONS = [
-      :type,
-      :compression_codec,
+      :ack_timeout_ms,
       :compressed_topics,
+      :compression_codec,
+      :max_send_retries,
       :metadata_refresh_interval_ms,
       :partitioner,
-      :max_send_retries,
       :retry_backoff_ms,
       :required_acks,
-      :ack_timeout_ms
+      :socket_timeout_ms,
+      :type,
     ]
 
     # @api private
@@ -163,11 +164,13 @@ module Poseidon
     end
 
     # Closes all open connections to brokers
-    def shutdown
+    def close
       @shutdown = true
-      @producer.shutdown
+      @producer.close
     end
 
+    alias_method :shutdown, :close
+
     private
     def validate_options(options)
       unknown_keys = options.keys - VALID_OPTIONS

data/lib/poseidon/producer_compression_config.rb

@@ -3,15 +3,16 @@ module Poseidon
   class ProducerCompressionConfig
     COMPRESSION_CODEC_MAP = {
       :gzip   => Compression::GzipCodec,
-      :snappy => Compression::SnappyCodec
+      :snappy => Compression::SnappyCodec,
+      :none   => nil
     }
 
     def initialize(compression_codec, compressed_topics)
       if compression_codec
-        @compression_codec = COMPRESSION_CODEC_MAP[compression_codec]
-        if @compression_codec.nil?
-          raise ArgumentError, "Uknown compression codec: '#{compression_codec}' (accepted: #{COMPRESSION_CODEC_MAP.keys.inspect})"
+        unless COMPRESSION_CODEC_MAP.has_key?(compression_codec)
+          raise ArgumentError, "Unknown compression codec: '#{compression_codec}' (accepted: #{COMPRESSION_CODEC_MAP.keys.inspect})"
         end
+        @compression_codec = COMPRESSION_CODEC_MAP[compression_codec]
       else
         @compression_codec = nil
       end

data/lib/poseidon/protocol/request_buffer.rb

@@ -4,13 +4,14 @@ module Poseidon
     #
     # API parallels the primitive types described on the wiki, with some
     # sugar for prepending message sizes and checksums.
-    # (https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-ProtocolPrimitiveTypes) 
+    # (https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-ProtocolPrimitiveTypes)
     class RequestBuffer
       def initialize
-        @s = ''
+        @s = ''.encode(Encoding::BINARY)
       end
 
       def append(string)
+        string = string.dup.force_encoding(Encoding::BINARY)
         @s << string
         nil
       end
@@ -32,7 +33,7 @@ module Poseidon
       end
 
       # Add a string
-      # 
+      #
       # @param [String] string
       def string(string)
         if string.nil?
@@ -53,33 +54,23 @@ module Poseidon
       end
 
       def prepend_crc32
-        ensure_ascii
         checksum_pos = @s.bytesize
         @s += " "
         yield
-        ensure_ascii
         @s[checksum_pos] = [Zlib::crc32(@s[(checksum_pos+1)..-1])].pack("N")
         nil
       end
 
       def prepend_size
-        ensure_ascii
         size_pos = @s.bytesize
         @s += " "
         yield
-        ensure_ascii
         @s[size_pos] = [(@s.bytesize-1) - size_pos].pack("N")
         nil
       end
 
       def to_s
-        ensure_ascii
-      end
-
-      private
-
-      def ensure_ascii
-        @s.force_encoding("ASCII-8BIT")
+        @s
       end
     end
   end

data/lib/poseidon/protocol/response_buffer.rb

@@ -7,44 +7,44 @@ module Poseidon
       end
 
       def int8
-        byte = @s.slice(@pos, 1).unpack("C").first
+        byte = @s.byteslice(@pos, 1).unpack("C").first
         @pos += 1
         byte
       end
 
       def int16
-        short = @s.slice(@pos, 2).unpack("s>").first
+        short = @s.byteslice(@pos, 2).unpack("s>").first
         @pos += 2
         short
       end
 
       def int32
-        int = @s.slice(@pos, 4).unpack("l>").first
+        int = @s.byteslice(@pos, 4).unpack("l>").first
         @pos += 4
         int
       end
 
       def int64
-        long = @s.slice(@pos, 8).unpack("q>").first
+        long = @s.byteslice(@pos, 8).unpack("q>").first
         @pos += 8
         long
       end
 
       def string
         len = int16
-        string = @s.slice(@pos, len)
+        string = @s.byteslice(@pos, len)
         @pos += len
         string
       end
 
       def read(bytes)
-        data = @s.slice(@pos, bytes)
+        data = @s.byteslice(@pos, bytes)
         @pos += bytes
         data
       end
 
       def peek(bytes)
-        @s.slice(@pos, bytes)
+        @s.byteslice(@pos, bytes)
       end
 
       def bytes

data/lib/poseidon/sync_producer.rb

@@ -69,10 +69,12 @@ module Poseidon
       end
     end
 
-    def shutdown
-      @broker_pool.shutdown
+    def close
+      @broker_pool.close
     end
-    
+
+    alias_method :shutdown, :close
+
     private
 
     def ensure_metadata_available_for_topics(messages_to_send)

data/lib/poseidon/version.rb

@@ -1,4 +1,4 @@
 module Poseidon
   # Unstable! API May Change!
-  VERSION = "0.0.5.pre1"
+  VERSION = "0.0.5"
 end

data/poseidon.gemspec

@@ -19,7 +19,8 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
-  gem.add_development_dependency(%q<rspec>, '~> 2.13.0')
+  gem.add_development_dependency(%q<rspec>, '>= 3')
   gem.add_development_dependency(%q<yard>)
   gem.add_development_dependency(%q<simplecov>)
+  gem.add_development_dependency(%q<snappy>)
 end

data/spec/integration/multiple_brokers/consumer_spec.rb

@@ -1,6 +1,6 @@
 require 'integration/multiple_brokers/spec_helper'
 
-describe "consuming with multiple brokers" do
+RSpec.describe "consuming with multiple brokers", :type => :request do
   before(:each) do
     # autocreate the topic by asking for information about it
     c = Connection.new("localhost", 9092, "metadata_fetcher", 10_000)

data/spec/integration/multiple_brokers/metadata_failures_spec.rb

@@ -1,6 +1,6 @@
 require 'integration/multiple_brokers/spec_helper'
 
-describe "handling failures" do
+RSpec.describe "handling failures", :type => :request do
   describe "metadata failures" do
     before(:each) do
       @messages_to_send = [

data/spec/integration/multiple_brokers/rebalance_spec.rb

@@ -1,6 +1,6 @@
 require 'integration/multiple_brokers/spec_helper'
 
-describe "producer handles rebalancing" do
+RSpec.describe "producer handles rebalancing", :type => :request do
   before(:each) do
     # autocreate the topic by asking for information about it
     @c = Connection.new("localhost", 9093, "metadata_fetcher", 10_000)