poseidon_cluster 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bd4a2f60912c1cf42f542630df36bff3fda31f36
-  data.tar.gz: 892927b055dca81c8805ef204cfadab0be1b4d88
+  metadata.gz: 3a9e9d18567496651c7c390954320166410d315f
+  data.tar.gz: a17c4b5192bcd360bd7fe2eda5da3b6d9afcf0a9
 SHA512:
-  metadata.gz: 72d72af7e3c642c93cad4d98c67ae628895a8344497466d7299a5a763c97d95754bb2c7750bd5666d39d79584ba989ec07d57c5767a1a61ff46775778bde8fd0
-  data.tar.gz: 5637f76ec6318cb52b2a353f02324f2c7f1c7e2fd57213ad1c39d6e87f724471a6932a2fc1e9253a21ced77fb01f1273e3fe61ce50ac5807d0727ab5b886cd17
+  metadata.gz: c241fca61a6a6c6b59b32d023250665c939fb3f4c34cdddd4bb9cfbb125a4bab8bc87f405e0e496e2a14e6a8d9d6c6ef9278d85d369ddcde30afbad844fd0536
+  data.tar.gz: 170438cf674768bfc7f4002332f16f1d89d9a0669278e6f801d1dc10e6e0bfd1a9093fcd382bf3ba9adb88491bf11ba348d36e84e803e5a3bb0c2f80079cee2e

data/.coveralls.yml

@@ -0,0 +1 @@
+service_name: travis-ci

data/.gitignore

@@ -3,3 +3,5 @@ kafka*/
 doc/
 .yardoc/
 .bundle/
+pkg/
+coverage/

data/.travis.yml

@@ -1,9 +1,8 @@
 language: ruby
-
 rvm:
   - 2.1.0
   - 2.0.0
   - 1.9.3
+  - jruby-19mode
 env:
   - SLOW=1
-  - SLOW=0

data/Gemfile

@@ -2,4 +2,9 @@ source "https://rubygems.org"
 
 gemspec
 gem "poseidon", git: "https://github.com/dim/poseidon.git"
-gem "snappy"
+gem "coveralls", require: false
+
+platform :jruby do
+  gem 'slyphon-log4j', '= 1.2.15'
+  gem 'slyphon-zookeeper_jar', '= 3.3.5'
+end

data/Gemfile.lock

@@ -7,18 +7,29 @@ GIT
 PATH
   remote: .
   specs:
-    poseidon_cluster (0.0.2)
+    poseidon_cluster (0.0.3)
       poseidon
       zk
 
 GEM
   remote: https://rubygems.org/
   specs:
+    coveralls (0.7.0)
+      multi_json (~> 1.3)
+      rest-client
+      simplecov (>= 0.7)
+      term-ansicolor
+      thor
     diff-lcs (1.2.5)
+    docile (1.1.2)
     little-plugger (1.1.3)
     logging (1.7.2)
       little-plugger (>= 1.1.3)
+    mime-types (2.0)
+    multi_json (1.8.4)
     rake (10.1.1)
+    rest-client (1.6.7)
+      mime-types (>= 1.16)
     rspec (2.14.1)
       rspec-core (~> 2.14.0)
       rspec-expectations (~> 2.14.0)
@@ -27,21 +38,37 @@ GEM
     rspec-expectations (2.14.4)
       diff-lcs (>= 1.1.3, < 2.0)
     rspec-mocks (2.14.4)
-    snappy (0.0.10)
+    simplecov (0.8.2)
+      docile (~> 1.1.0)
+      multi_json
+      simplecov-html (~> 0.8.0)
+    simplecov-html (0.8.0)
+    slyphon-log4j (1.2.15)
+    slyphon-zookeeper_jar (3.3.5-java)
+    term-ansicolor (1.2.2)
+      tins (~> 0.8)
+    thor (0.18.1)
+    tins (0.13.1)
     yard (0.8.7.3)
     zk (1.9.3)
       logging (~> 1.7.2)
       zookeeper (~> 1.4.0)
     zookeeper (1.4.8)
+    zookeeper (1.4.8-java)
+      slyphon-log4j (= 1.2.15)
+      slyphon-zookeeper_jar (= 3.3.5)
 
 PLATFORMS
+  java
   ruby
 
 DEPENDENCIES
   bundler
+  coveralls
   poseidon!
   poseidon_cluster!
   rake
   rspec
-  snappy
+  slyphon-log4j (= 1.2.15)
+  slyphon-zookeeper_jar (= 3.3.5)
   yard

data/README.md

@@ -1,12 +1,12 @@
 # Poseidon Cluster [![Build Status](https://travis-ci.org/bsm/poseidon_cluster.png?branch=master)](https://travis-ci.org/bsm/poseidon_cluster)
 
-Poseidon Cluster is a cluster extenstion the excellent [Poseidon](http://github.com/bpot/poseidon) Ruby client for Kafka 0.8+. It implements the distribution concept of self-rebalancing *Consumer Groups* and supports the consumption of a single topic from multiple instances.
+Poseidon Cluster is a cluster extension the excellent [Poseidon](http://github.com/bpot/poseidon) Ruby client for Kafka 0.8+. It implements the distribution concept of self-rebalancing *Consumer Groups* and supports the consumption of a single topic from multiple instances.
 
 Consumer group instances share a common group name, and each message published to a topic is delivered to one instance within each subscribing consumer group. Consumer instances can be in separate processes or on separate machines.
 
 ## Usage
 
-### Consuming messages
+Launch a consumer group:
 
 ```ruby
 require 'poseidon_cluster'
@@ -18,19 +18,28 @@ consumer = Poseidon::ConsumerGroup.new(
             "my-topic")                               # Topic name
 
 consumer.partitions # => [0, 1, 2, 3] - all partitions of 'my-topic'
-consumer.claimed    # => [0, 1] - partitions this instance is consuming
+consumer.claimed    # => [0, 1] - partitions this instance has claimed
+```
+
+Fetch a bulk of messages, auto-commit the offset:
 
-# Fetch a bulk of messages, auto-commit the offset
+```ruby
 consumer.fetch do |partition, bulk|
   bulk.each do |m|
     puts "Fetched '#{m.value}' at #{m.offset} from #{partition}"
   end
 end
+```
+
+Get the offset for a partition:
 
-# Get the offset for a partition
+```ruby
 consumer.offset(0) # => 320 - current offset from partition 0
+```
 
-# Fetch more, commit manually
+Fetch more messages, commit manually:
+
+```ruby
 consumer.fetch commit: false do |partition, bulk|
   bulk.each do |m|
     puts "Fetched '#{m.value}' at #{m.offset} from #{partition}"
@@ -40,7 +49,17 @@ consumer.fetch commit: false do |partition, bulk|
 end
 ```
 
-For more details and information, please see the [Poseidon::ConsumerGroup](http://rubydoc.info/github/bsm/poseidon_cluster/Poseidon/ConsumerGroup) documentation.
+Initiate a fetch-loop, consume indefinitely:
+
+```ruby
+consumer.fetch_loop do |partition, bulk|
+  bulk.each do |m|
+    puts "Fetched '#{m.value}' at #{m.offset} from #{partition}"
+  end
+end
+```
+
+For more details and information, please see the [Poseidon::ConsumerGroup](http://rubydoc.info/github/bsm/poseidon_cluster/Poseidon/ConsumerGroup) documentation and the [Examples](https://github.com/bsm/poseidon_cluster/tree/master/examples).
 
 ## Running Tests
 

data/examples/consumer_group.rb

@@ -0,0 +1,33 @@
+=begin
+
+ PLEASE NOTE
+
+ This example uses threads, but you could equally use fork or run your
+ consumer groups from completely separate process and from multiple machines.
+
+=end
+require 'poseidon_cluster'
+
+# Create a consumer group
+group1 = Poseidon::ConsumerGroup.new "my-group", ["host1:9092", "host2:9092"], ["host1:2181", "host2:2181"], "my-topic"
+
+# Start consuming "my-topic" in a background thread
+thread1 = Thread.new do
+  group1.fetch_loop do |partition, messages|
+    puts "Consumer #1 fetched #{messages.size} from #{partition}"
+  end
+end
+
+# Create a second consumer group
+group2 = Poseidon::ConsumerGroup.new "my-group", ["host1:9092", "host2:9092"], ["host1:2181", "host2:2181"], "my-topic"
+
+# Now consuming all partitions of "my-topic" in parallel
+thread2 = Thread.new do
+  group2.fetch_loop do |partition, messages|
+    puts "Consumer #2 fetched #{messages.size} from #{partition}"
+  end
+end
+
+# Join threads, loop forever
+[thread1, thread2].each(&:join)
+

data/lib/poseidon/consumer_group.rb

@@ -22,6 +22,7 @@
 # @api public
 class Poseidon::ConsumerGroup
   DEFAULT_CLAIM_TIMEOUT = 10
+  DEFAULT_LOOP_DELAY = 1
 
   # Poseidon::ConsumerGroup::Consumer is internally used by Poseidon::ConsumerGroup.
   # Don't invoke it directly.
@@ -62,7 +63,12 @@ class Poseidon::ConsumerGroup
   # @param [Array<String>] brokers A list of known brokers, e.g. ["localhost:9092"]
   # @param [Array<String>] zookeepers A list of known zookeepers, e.g. ["localhost:2181"]
   # @param [String] topic Topic to operate on
-  # @param [Hash] options Partition consumer options, see Poseidon::PartitionConsumer#initialize
+  # @param [Hash] options Consumer options
+  # @option options [Integer] :max_bytes Maximum number of bytes to fetch. Default: 1048576 (1MB)
+  # @option options [Integer] :max_wait_ms How long to block until the server sends us data. Default: 100 (100ms)
+  # @option options [Integer] :min_bytes Smallest amount of data the server should send us. Default: 0 (Send us data as soon as it is ready)
+  # @option options [Integer] :claim_timeout Maximum number of seconds to wait for a partition claim. Default: 10
+  # @option options [Integer] :loop_delay Number of seconds to delay the next fetch (in #fetch_loop) if nothing was returned. Default: 1
   #
   # @api public
   def initialize(name, brokers, zookeepers, topic, options = {})
@@ -150,6 +156,8 @@ class Poseidon::ConsumerGroup
   # Sorted partitions by broker address (so partitions on the same broker are clustered together)
   # @return [Array<Poseidon::Protocol::PartitionMetadata>] sorted partitions
   def partitions
+    return [] unless topic_metadata
+
     topic_metadata.partitions.sort_by do |part|
       broker = metadata.brokers[part.leader]
       [broker.host, broker.port].join(":")
@@ -170,12 +178,20 @@ class Poseidon::ConsumerGroup
   #
   # @param [Hash] opts
   # @option opts [Boolean] :commit Automatically commit consumer offset (default: true)
+  # @return [Boolean] true if a consumer was checked out, false if none could be claimed
+  #
+  # @example
+  #
+  #   ok = group.checkout do |consumer|
+  #     puts "Checked out consumer for partition #{consumer.partition}"
+  #   end
+  #   ok # => true if the block was run, false otherwise
   #
   # @api public
   def checkout(opts = {})
     @mutex.synchronize do
       consumer = @consumers.shift
-      break unless consumer
+      break false unless consumer
 
       @consumers.push(consumer)
       result = yield(consumer)
@@ -183,8 +199,8 @@ class Poseidon::ConsumerGroup
       unless opts[:commit] == false || result == false
         commit consumer.partition, consumer.offset
       end
+      true
     end
-    nil
   end
 
   # Convenience method to fetch messages from the broker.
@@ -193,10 +209,18 @@ class Poseidon::ConsumerGroup
   # @yield [partition, messages] The processing block
   # @yieldparam [Integer] partition The source partition
   # @yieldparam [Array<Message>] messages The fetched messages
-  # @yieldreturn [Boolean] return false to stop commit
+  # @yieldreturn [Boolean] return false to prevent auto-commit
   #
   # @param [Hash] opts
   # @option opts [Boolean] :commit Automatically commit consumed offset (default: true)
+  # @return [Boolean] true if messages were fetched, false if none could be claimed
+  #
+  # @example
+  #
+  #   ok = group.fetch do |n, messages|
+  #     puts "Fetched #{messages.size} messages for partition #{n}"
+  #   end
+  #   ok # => true if the block was run, false otherwise
   #
   # @api public
   def fetch(opts = {})
@@ -205,6 +229,93 @@ class Poseidon::ConsumerGroup
     end
   end
 
+  # Initializes an infinite fetch loop. This method blocks!
+  #
+  # Will wait for `loop_delay` seconds after each failed fetch. This may happen when there is
+  # no new data or when the consumer hasn't claimed any partitions.
+  #
+  # SPECIAL ATTENTION:
+  # When 'breaking out' of the loop, you must do it before processing the messages, as the
+  # the last offset will not be committed. Please see examples below.
+  #
+  # @yield [partition, messages] The processing block
+  # @yieldparam [Integer] partition The source partition, may be -1 if no partitions are claimed
+  # @yieldparam [Array<Message>] messages The fetched messages
+  # @yieldreturn [Boolean] return false to prevent auto-commit
+  #
+  # @param [Hash] opts
+  # @option opts [Boolean] :commit Automatically commit consumed offset (default: true)
+  # @option opts [Boolean] :loop_delay Delay override in seconds after unsuccessful fetch.
+  #
+  # @example
+  #
+  #   group.fetch_loop do |n, messages|
+  #     puts "Fetched #{messages.size} messages for partition #{n}"
+  #   end
+  #   puts "Done" # => this code is never reached
+  #
+  # @example Stopping the loop (wrong)
+  #
+  #   counts = Hash.new(0)
+  #   group.fetch_loop do |n, messages|
+  #     counts[n] += messages.size
+  #     puts "Status: #{counts.inspect}"
+  #     break if counts[0] > 100
+  #   end
+  #   puts "Result: #{counts.inspect}"
+  #   puts "Offset: #{group.offset(0)}"
+  #
+  #   # Output:
+  #   # Status: {0=>30}
+  #   # Status: {0=>60}
+  #   # Status: {0=>90}
+  #   # Status: {0=>120}
+  #   # Result: {0=>120}
+  #   # Offset: 90      # => Last offset was not committed!
+  #
+  # @example Stopping the loop (correct)
+  #
+  #   counts = Hash.new(0)
+  #   group.fetch_loop do |n, messages|
+  #     break if counts[0] > 100
+  #     counts[n] += messages.size
+  #     puts "Status: #{counts.inspect}"
+  #   end
+  #   puts "Result: #{counts.inspect}"
+  #   puts "Offset: #{group.offset(0)}"
+  #
+  #   # Output:
+  #   # Status: {0=>30}
+  #   # Status: {0=>60}
+  #   # Status: {0=>90}
+  #   # Status: {0=>120}
+  #   # Result: {0=>120}
+  #   # Offset: 120
+  #
+  # @api public
+  def fetch_loop(opts = {})
+    delay = opts[:loop_delay] || options[:loop_delay] || DEFAULT_LOOP_DELAY
+
+    loop do
+      mp = false
+      ok = fetch(opts) do |n, messages|
+        mp = !messages.empty?
+        yield n, messages
+      end
+
+      # Yield over an empty array if nothing claimed,
+      # to allow user to e.g. break out of the loop
+      unless ok
+        yield -1, []
+      end
+
+      # Sleep if either not claimes or nothing returned
+      unless ok && mp
+        sleep delay
+      end
+    end
+  end
+
   protected
 
     # Rebalance algorithm:
@@ -218,18 +329,19 @@ class Poseidon::ConsumerGroup
     def rebalance!
       @mutex.synchronize do
         reload
-        cg  = zk.children(registries[:consumer], watch: true).sort
-        pt  = partitions
-        pos = cg.index(id)
-        n   = pt.size / cg.size
-        n   = 1 if n < 1
+        release_all!
 
-        first = pos*n
-        last  = (pos+1)*n-1
+        cmg = zk.children(registries[:consumer], watch: true).sort
+        ptm = partitions
+        pos = cmg.index(id)
+        break unless ptm.size > pos
 
-        release_all!
-        (pt[first..last] || []).each do |part|
-          consumer = claim!(part.id)
+        num = ptm.size / cmg.size
+        num = 1 if num < 1
+        rng = pos*num..(pos+1)*num-1
+
+        (ptm[rng] || []).each do |pm|
+          consumer = claim!(pm.id)
           @consumers.push(consumer)
         end
       end

data/poseidon_cluster.gemspec

@@ -5,7 +5,7 @@ Gem::Specification.new do |s|
   s.name        = File.basename(__FILE__, '.gemspec')
   s.summary     = "Poseidon cluster extensions"
   s.description = "Cluster extensions for Poseidon, a producer and consumer implementation for Kafka >= 0.8"
-  s.version     = "0.0.2"
+  s.version     = "0.0.3"
 
   s.authors     = ["Black Square Media"]
   s.email       = "info@blacksquaremedia.com"
@@ -22,4 +22,5 @@ Gem::Specification.new do |s|
   s.add_development_dependency "bundler"
   s.add_development_dependency "rspec"
   s.add_development_dependency "yard"
+
 end

data/spec/integration/poseidon/consumer_group_spec.rb

@@ -6,14 +6,31 @@ describe Poseidon::ConsumerGroup, integration: true do
     described_class.new "my-group", ["localhost:29092"], ["localhost:22181"], name, max_bytes: max_bytes
   end
 
-  subject { new_group }
-  after   { zookeeper.rm_rf "/consumers/#{subject.name}" }
+  def stored_offsets
+    { 0 => subject.offset(0), 1 => subject.offset(1) }
+  end
 
+  subject { new_group }
   let(:consumed)  { Hash.new(0) }
   let(:zookeeper) { ::ZK.new("localhost:22181") }
 
-  def stored_offsets
-    { 0 => subject.offset(0), 1 => subject.offset(1) }
+  before :all do
+    producer = Poseidon::Producer.new(["localhost:29092"], "my-producer")
+    payload  = "data" * 10
+    messages = ("aa".."zz").map do |key|
+      Poseidon::MessageToSend.new(TOPIC_NAME, [key, payload].join(":"), key)
+    end
+
+    ok = false
+    100.times do
+      break if (ok = producer.send_messages(messages))
+      sleep(0.1)
+    end
+    pending "Unable to start Kafka instance." unless ok
+  end
+
+  after do
+    zookeeper.rm_rf "/consumers/my-group"
   end
 
   describe "small batches" do
@@ -47,14 +64,16 @@ describe Poseidon::ConsumerGroup, integration: true do
     end
   end
 
-  describe "fuzzing" do
+  describe "multi-thread fuzzing", slow: true do
 
     def in_thread(batch_size, target, qu)
       Thread.new do
-        group = new_group(batch_size)
         sum   = 0
-        while sum < target && qu.size < 676
-          group.fetch {|_, m| sum += m.size; m.size.times { qu << true } }
+        group = new_group(batch_size)
+        group.fetch_loop do |n, m|
+          break if sum > target || qu.size >= 676
+          sum += m.size
+          m.size.times { qu << true }
         end
         group.close
         sum
@@ -64,17 +83,21 @@ describe Poseidon::ConsumerGroup, integration: true do
     it "should consume from multiple sources" do
       q = Queue.new
       a = in_thread(4001, 200, q)
-      b = in_thread(4002,  50, q)
+      b = in_thread(4002,  40, q)
       c = in_thread(4003, 120, q)
-      d = in_thread(4004,  40, q)
+      d = in_thread(4004,  50, q)
       e = in_thread(4005, 400, q)
-      vals = [a, b, c, d, e].map &:value
+      vals = [a, b, c, d, e].map(&:value)
       vals.inject(0, :+).should == 676
+
+      o1, _ = zookeeper.get "/consumers/my-group/offsets/#{TOPIC_NAME}/0"
+      o2, _ = zookeeper.get "/consumers/my-group/offsets/#{TOPIC_NAME}/1"
+      (o1.to_i + o2.to_i).should == 676
     end
 
   end
 
-  describe "multi-process fuzzing", slow: true do
+  describe "multi-process fuzzing", slow: true, java: false do
     before do
       producer = Poseidon::Producer.new(["localhost:29092"], "my-producer")
       payload  = "data" * 10

data/spec/lib/poseidon/consumer_group_spec.rb

@@ -40,6 +40,7 @@ describe Poseidon::ConsumerGroup do
 
   subject { new_group }
   before do
+    Poseidon::ConsumerGroup.any_instance.stub(:sleep)
     Poseidon::BrokerPool.any_instance.stub(:fetch_metadata_from_broker).and_return(metadata)
     Poseidon::Connection.any_instance.stub(:fetch).with{|_, _, req| req[0].partition_fetches[0].partition == 0 }.and_return(fetch_response(10))
     Poseidon::Connection.any_instance.stub(:fetch).with{|_, _, req| req[0].partition_fetches[0].partition == 1 }.and_return(fetch_response(5))
@@ -82,6 +83,14 @@ describe Poseidon::ConsumerGroup do
     subject.partitions.map(&:id).should == [1, 0]
   end
 
+  it "should not fail if topic doesn't exist" do
+    no_topics = Poseidon::Protocol::MetadataResponse.new nil, brokers.dup, []
+    Poseidon::BrokerPool.any_instance.stub(:fetch_metadata_from_broker).and_return(no_topics)
+
+    subject.partitions.should == []
+    subject.claimed.should == []
+  end
+
   it "should return the offset for each partition" do
     subject.offset(0).should == 0
     subject.offset(1).should == 0
@@ -99,8 +108,8 @@ describe Poseidon::ConsumerGroup do
   end
 
   it "should checkout individual partition consumers (atomically)" do
-    subject.checkout {|c| c.partition.should == 1 }
-    subject.checkout {|c| c.partition.should == 0 }
+    subject.checkout {|c| c.partition.should == 1 }.should be_true
+    subject.checkout {|c| c.partition.should == 0 }.should be_true
 
     n = 0
     a = Thread.new do
@@ -135,10 +144,6 @@ describe Poseidon::ConsumerGroup do
       b.claimed.should == [0]
 
       c = new_group
-      subject.claimed.should == [1]
-      b.claimed.should == [0]
-      c.claimed.should == []
-
       b.close
       wait_for { b.claimed.size < 0 }
       wait_for { c.claimed.size > 0 }
@@ -156,15 +161,17 @@ describe Poseidon::ConsumerGroup do
       subject.fetch do |n, msg|
         n.should == 1
         msg.size.should == 5
-      end
+      end.should be_true
+
       subject.fetch do |n, msg|
         n.should == 0
         msg.size.should == 10
-      end
+      end.should be_true
+
       subject.fetch do |n, msg|
         n.should == 1
         msg.size.should == 5
-      end
+      end.should be_true
     end
 
     it "should auto-commit fetched offset" do
@@ -185,6 +192,56 @@ describe Poseidon::ConsumerGroup do
       }.should_not change { subject.offset(1) }
     end
 
+    it "should return false when trying to fetch messages without a claim" do
+      no_topics = Poseidon::Protocol::MetadataResponse.new nil, brokers.dup, []
+      Poseidon::BrokerPool.any_instance.stub fetch_metadata_from_broker: no_topics
+
+      subject.claimed.should == []
+      subject.fetch {|*|  }.should be_false
+    end
+
+    it "should return true even when no messages were fetched" do
+      Poseidon::Connection.any_instance.stub fetch: fetch_response(0)
+      subject.fetch {|*|  }.should be_true
+    end
+
   end
 
+  describe "fetch_loop" do
+
+    it "should fetch indefinitely" do
+      total, cycles = 0, 0
+      subject.fetch_loop do |_, m|
+        total += m.size
+        break if (cycles+=1) > 2
+      end
+      total.should == 20
+      cycles.should == 3
+    end
+
+    it "should delay fetch was unsuccessful" do
+      subject.stub fetch: false
+
+      cycles = 0
+      subject.should_receive(:sleep).with(1)
+      subject.fetch_loop do |n, m|
+        n.should == -1
+        m.should == []
+        break if (cycles+=1) > 1
+      end
+    end
+
+    it "should delay fetch didn't yield any results" do
+      subject.stub(:fetch).and_yield(3, []).and_return(true)
+
+      cycles = 0
+      subject.should_receive(:sleep).with(1)
+      subject.fetch_loop do |n, m|
+        n.should == 3
+        m.should == []
+        break if (cycles+=1) > 1
+      end
+    end
+
+  end
 end

data/spec/spec_helper.rb

@@ -2,6 +2,8 @@ require 'poseidon_cluster'
 require 'rspec'
 require 'fileutils'
 require 'pathname'
+require 'coveralls'
+Coveralls.wear!
 
 TOPIC_NAME  = "my-topic"
 KAFKA_LOCAL = File.expand_path("../kafka_2.8.0-0.8.0", __FILE__)
@@ -21,6 +23,7 @@ end
 RSpec.configure do |c|
   c.include Poseidon::SpecHelper
   c.filter_run_excluding slow: true unless ENV["SLOW"] == "1"
+  c.filter_run_excluding java: false if RUBY_PLATFORM == "java"
 
   c.before :suite do
     kafka_bin = KAFKA_ROOT.join("bin", "kafka-server-start.sh")
@@ -48,22 +51,8 @@ RSpec.configure do |c|
     end
 
     # Start Zookeeper & Kafka
-    $ZOOKP_PID = spawn zookp_bin.to_s, zookp_cfg.to_s, out: '/dev/null' # , err: '/dev/null'
-    $KAFKA_PID = spawn kafka_bin.to_s, kafka_cfg.to_s, out: '/dev/null' #, err: '/dev/null'
-
-    # Produce some fixtures
-    producer = Poseidon::Producer.new(["localhost:29092"], "my-producer")
-    payload  = "data" * 10
-    messages = ("aa".."zz").map do |key|
-      Poseidon::MessageToSend.new(TOPIC_NAME, [key, payload].join(":"), key)
-    end
-
-    ok = false
-    100.times do
-      break if (ok = producer.send_messages(messages))
-      sleep(0.1)
-    end
-    raise "Unable to start Kafka instance." unless ok
+    $ZOOKP_PID = spawn zookp_bin.to_s, zookp_cfg.to_s, out: '/dev/null'
+    $KAFKA_PID = spawn kafka_bin.to_s, kafka_cfg.to_s, out: '/dev/null'
   end
 
   c.after :suite do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: poseidon_cluster
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Black Square Media
@@ -101,12 +101,14 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".coveralls.yml"
 - ".gitignore"
 - ".travis.yml"
 - Gemfile
 - Gemfile.lock
 - README.md
 - Rakefile
+- examples/consumer_group.rb
 - lib/poseidon/cluster.rb
 - lib/poseidon/consumer_group.rb
 - lib/poseidon_cluster.rb