poseidon_cluster 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: bd4a2f60912c1cf42f542630df36bff3fda31f36
+  data.tar.gz: 892927b055dca81c8805ef204cfadab0be1b4d88
+SHA512:
+  metadata.gz: 72d72af7e3c642c93cad4d98c67ae628895a8344497466d7299a5a763c97d95754bb2c7750bd5666d39d79584ba989ec07d57c5767a1a61ff46775778bde8fd0
+  data.tar.gz: 5637f76ec6318cb52b2a353f02324f2c7f1c7e2fd57213ad1c39d6e87f724471a6932a2fc1e9253a21ced77fb01f1273e3fe61ce50ac5807d0727ab5b886cd17

data/.gitignore

@@ -0,0 +1,5 @@
+logs/
+kafka*/
+doc/
+.yardoc/
+.bundle/

data/.travis.yml

@@ -0,0 +1,9 @@
+language: ruby
+
+rvm:
+  - 2.1.0
+  - 2.0.0
+  - 1.9.3
+env:
+  - SLOW=1
+  - SLOW=0

data/Gemfile

@@ -0,0 +1,5 @@
+source "https://rubygems.org"
+
+gemspec
+gem "poseidon", git: "https://github.com/dim/poseidon.git"
+gem "snappy"

data/Gemfile.lock

@@ -0,0 +1,47 @@
+GIT
+  remote: https://github.com/dim/poseidon.git
+  revision: f853e8f07a6f2ff4a520d3f7e1dac030453a18ed
+  specs:
+    poseidon (0.0.4)
+
+PATH
+  remote: .
+  specs:
+    poseidon_cluster (0.0.2)
+      poseidon
+      zk
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    diff-lcs (1.2.5)
+    little-plugger (1.1.3)
+    logging (1.7.2)
+      little-plugger (>= 1.1.3)
+    rake (10.1.1)
+    rspec (2.14.1)
+      rspec-core (~> 2.14.0)
+      rspec-expectations (~> 2.14.0)
+      rspec-mocks (~> 2.14.0)
+    rspec-core (2.14.7)
+    rspec-expectations (2.14.4)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.14.4)
+    snappy (0.0.10)
+    yard (0.8.7.3)
+    zk (1.9.3)
+      logging (~> 1.7.2)
+      zookeeper (~> 1.4.0)
+    zookeeper (1.4.8)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler
+  poseidon!
+  poseidon_cluster!
+  rake
+  rspec
+  snappy
+  yard

data/README.md

@@ -0,0 +1,76 @@
+# Poseidon Cluster [![Build Status](https://travis-ci.org/bsm/poseidon_cluster.png?branch=master)](https://travis-ci.org/bsm/poseidon_cluster)
+
+Poseidon Cluster is a cluster extenstion the excellent [Poseidon](http://github.com/bpot/poseidon) Ruby client for Kafka 0.8+. It implements the distribution concept of self-rebalancing *Consumer Groups* and supports the consumption of a single topic from multiple instances.
+
+Consumer group instances share a common group name, and each message published to a topic is delivered to one instance within each subscribing consumer group. Consumer instances can be in separate processes or on separate machines.
+
+## Usage
+
+### Consuming messages
+
+```ruby
+require 'poseidon_cluster'
+
+consumer = Poseidon::ConsumerGroup.new(
+            "my-group",                               # Group name
+            ["kafka1.host:9092", "kafka2.host:9092"], # Kafka brokers
+            ["kafka1.host:2181", "kafka2.host:2181"], # Zookeepers hosts
+            "my-topic")                               # Topic name
+
+consumer.partitions # => [0, 1, 2, 3] - all partitions of 'my-topic'
+consumer.claimed    # => [0, 1] - partitions this instance is consuming
+
+# Fetch a bulk of messages, auto-commit the offset
+consumer.fetch do |partition, bulk|
+  bulk.each do |m|
+    puts "Fetched '#{m.value}' at #{m.offset} from #{partition}"
+  end
+end
+
+# Get the offset for a partition
+consumer.offset(0) # => 320 - current offset from partition 0
+
+# Fetch more, commit manually
+consumer.fetch commit: false do |partition, bulk|
+  bulk.each do |m|
+    puts "Fetched '#{m.value}' at #{m.offset} from #{partition}"
+  end
+
+  consumer.commit partition, bulk.last.offset+1 unless bulk.empty?
+end
+```
+
+For more details and information, please see the [Poseidon::ConsumerGroup](http://rubydoc.info/github/bsm/poseidon_cluster/Poseidon/ConsumerGroup) documentation.
+
+## Running Tests
+
+The test suite will automatically download, configure and run Kafka locally, you only need a JRE. Run the suite via:
+
+```bash
+bundle exec rake spec
+```
+
+## Licence
+
+```
+Copyright (c) 2014 Black Square Media
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+```

data/Rakefile

@@ -0,0 +1,10 @@
+require 'bundler/gem_tasks'
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec)
+
+require 'yard'
+require 'yard/rake/yardoc_task'
+YARD::Rake::YardocTask.new
+
+task default: :spec

data/lib/poseidon/cluster.rb

@@ -0,0 +1,22 @@
+require 'socket'
+require 'timeout'
+require 'zk'
+require 'poseidon'
+require 'thread'
+
+module Poseidon::Cluster
+  MAX_INT32 = 0x7fffffff
+  @@sem = Mutex.new
+  @@inc = 0
+
+  # @return [Integer] an incremented number
+  # @api private
+  def self.inc!
+    @@sem.synchronize { @@inc += 1; @@inc = 1 if @@inc > MAX_INT32; @@inc }
+  end
+
+end
+
+%w|consumer_group|.each do |name|
+  require "poseidon/#{name}"
+end

data/lib/poseidon/consumer_group.rb

@@ -0,0 +1,276 @@
+# A ConsumerGroup operates on all partitions of a single topic. The goal is to ensure
+# each topic message is consumed only once, no matter of the number of consumer instances within
+# a cluster, as described in: http://kafka.apache.org/documentation.html#distributionimpl.
+#
+# The ConsumerGroup internally creates multiple PartitionConsumer instances. It uses Zookkeper
+# and follows a simple consumer rebalancing algorithms which allows all the consumers
+# in a group to come into consensus on which consumer is consuming which partitions. Each
+# ConsumerGroup can 'claim' 0-n partitions and will consume their messages until another
+# ConsumerGroup instance joins or leaves the cluster.
+#
+# Please note: ConsumerGroups themselves don't implement any threading or concurrency.
+# When consuming messages, they simply round-robin across the claimed partitions. If you wish
+# to parallelize consumption simply create multiple ConsumerGroups instances. The built-in
+# concensus algorithm will automatically rebalance the available partitions between them and you
+# can then decide for yourself if you want to run them in multiple thread or processes, ideally
+# on multiple boxes.
+#
+# Unlike stated in the Kafka documentation, consumer rebalancing is *only* triggered on each
+# addition or removal of consumers within the same group, while the addition of broker nodes
+# and/or partition *does currently not trigger* a rebalancing cycle.
+#
+# @api public
+class Poseidon::ConsumerGroup
+  DEFAULT_CLAIM_TIMEOUT = 10
+
+  # Poseidon::ConsumerGroup::Consumer is internally used by Poseidon::ConsumerGroup.
+  # Don't invoke it directly.
+  #
+  # @api private
+  class Consumer < ::Poseidon::PartitionConsumer
+
+    # @attr_reader [Integer] partition Consumer partition
+    attr_reader :partition
+
+    # @api private
+    def initialize(group, partition, options = {})
+      broker = group.leader(partition)
+      offset = group.offset(partition)
+      super group.id, broker.host, broker.port, group.topic, partition, offset, options
+    end
+
+  end
+
+  # @attr_reader [String] name Group name
+  attr_reader :name
+
+  # @attr_reader [String] topic Topic name
+  attr_reader :topic
+
+  # @attr_reader [Poseidon::BrokerPool] pool Broker pool
+  attr_reader :pool
+
+  # @attr_reader [ZK::Client] zk Zookeeper client
+  attr_reader :zk
+
+  # @attr_reader [Hash] options Consumer options
+  attr_reader :options
+
+  # Create a new consumer group, which processes all partition of the specified topic.
+  #
+  # @param [String] name Group name
+  # @param [Array<String>] brokers A list of known brokers, e.g. ["localhost:9092"]
+  # @param [Array<String>] zookeepers A list of known zookeepers, e.g. ["localhost:2181"]
+  # @param [String] topic Topic to operate on
+  # @param [Hash] options Partition consumer options, see Poseidon::PartitionConsumer#initialize
+  #
+  # @api public
+  def initialize(name, brokers, zookeepers, topic, options = {})
+    @name      = name
+    @topic     = topic
+    @zk        = ::ZK.new(zookeepers.join(","))
+    @options   = options
+    @consumers = []
+    @pool      = ::Poseidon::BrokerPool.new(id, brokers)
+    @mutex     = Mutex.new
+
+    # Register instance
+    registries.each do |_, path|
+      zk.mkdir_p(path)
+    end
+    zk.create(consumer_path, "{}", ephemeral: true)
+    zk.register(registries[:consumer]) {|_| rebalance! }
+
+    # Rebalance
+    rebalance!
+  end
+
+  # @return [String] a globally unique identifier
+  def id
+    @id ||= [name, ::Socket.gethostname, ::Process.pid, ::Time.now.to_i, ::Poseidon::Cluster.inc!].join("-")
+  end
+
+  # @return [Hash<Symbol,String>] registry paths
+  def registries
+    @registries ||= {
+      consumer: "/consumers/#{name}/ids",
+      owner:    "/consumers/#{name}/owners/#{topic}",
+      offset:   "/consumers/#{name}/offsets/#{topic}",
+    }
+  end
+
+  # @return [Poseidon::ClusterMetadata] cluster metadata
+  def metadata
+    @metadata ||= Poseidon::ClusterMetadata.new.tap {|m| m.update pool.fetch_metadata([topic]) }
+  end
+
+  # @return [Poseidon::TopicMetadata] topic metadata
+  def topic_metadata
+    @topic_metadata ||= metadata.metadata_for_topics([topic])[topic]
+  end
+
+  # Reloads metadata/broker/partition information
+  def reload
+    @metadata = @topic_metadata = nil
+    metadata
+    self
+  end
+
+  # Closes the consumer group gracefully, only really useful in tests
+  # @api private
+  def close
+    @mutex.synchronize do
+      release_all!
+      zk.close
+    end
+  end
+
+  # @param [Integer] partition
+  # @return [Poseidon::Protocol::Broker] the leader for the given partition
+  def leader(partition)
+    metadata.lead_broker_for_partition(topic, partition)
+  end
+
+  # @param [Integer] partition
+  # @return [Integer] the latest stored offset for the given partition
+  def offset(partition)
+    data, _ = zk.get offset_path(partition), ignore: :no_node
+    data.to_i
+  end
+
+  # Commits the latest offset for a partition
+  # @param [Integer] partition
+  # @param [Integer] offset
+  def commit(partition, offset)
+    zk.set offset_path(partition), offset.to_s
+  rescue ZK::Exceptions::NoNode
+    zk.create offset_path(partition), offset.to_s, ignore: :node_exists
+  end
+
+  # Sorted partitions by broker address (so partitions on the same broker are clustered together)
+  # @return [Array<Poseidon::Protocol::PartitionMetadata>] sorted partitions
+  def partitions
+    topic_metadata.partitions.sort_by do |part|
+      broker = metadata.brokers[part.leader]
+      [broker.host, broker.port].join(":")
+    end
+  end
+
+  # Partitions currently claimed and consumed by this group instance
+  # @return [Array<Integer>] partition IDs
+  def claimed
+    @consumers.map(&:partition).sort
+  end
+
+  # Checks out a single partition consumer. Round-robins between claimed partitions.
+  #
+  # @yield [consumer] The processing block
+  # @yieldparam [Consumer] consumer The consumer instance
+  # @yieldreturn [Boolean] return false to stop auto-commit
+  #
+  # @param [Hash] opts
+  # @option opts [Boolean] :commit Automatically commit consumer offset (default: true)
+  #
+  # @api public
+  def checkout(opts = {})
+    @mutex.synchronize do
+      consumer = @consumers.shift
+      break unless consumer
+
+      @consumers.push(consumer)
+      result = yield(consumer)
+
+      unless opts[:commit] == false || result == false
+        commit consumer.partition, consumer.offset
+      end
+    end
+    nil
+  end
+
+  # Convenience method to fetch messages from the broker.
+  # Round-robins between claimed partitions.
+  #
+  # @yield [partition, messages] The processing block
+  # @yieldparam [Integer] partition The source partition
+  # @yieldparam [Array<Message>] messages The fetched messages
+  # @yieldreturn [Boolean] return false to stop commit
+  #
+  # @param [Hash] opts
+  # @option opts [Boolean] :commit Automatically commit consumed offset (default: true)
+  #
+  # @api public
+  def fetch(opts = {})
+    checkout(opts) do |consumer|
+      yield consumer.partition, consumer.fetch
+    end
+  end
+
+  protected
+
+    # Rebalance algorithm:
+    #
+    # * let CG be all consumers in the same group that consume topic T
+    # * let PT be all partitions producing topic T
+    # * sort CG
+    # * sort PT (so partitions on the same broker are clustered together)
+    # * let POS be our index position in CG and let N = size(PT)/size(CG)
+    # * assign partitions from POS*N to (POS+1)*N-1
+    def rebalance!
+      @mutex.synchronize do
+        reload
+        cg  = zk.children(registries[:consumer], watch: true).sort
+        pt  = partitions
+        pos = cg.index(id)
+        n   = pt.size / cg.size
+        n   = 1 if n < 1
+
+        first = pos*n
+        last  = (pos+1)*n-1
+
+        release_all!
+        (pt[first..last] || []).each do |part|
+          consumer = claim!(part.id)
+          @consumers.push(consumer)
+        end
+      end
+    end
+
+    # Release all consumer claims
+    def release_all!
+      @consumers.each {|c| release!(c.partition) }
+      @consumers.clear
+    end
+
+    # Claim the ownership of the partition for this consumer
+    # @raise [Timeout::Error]
+    def claim!(partition)
+      path = claim_path(partition)
+      Timeout.timeout(options[:claim_timout] || DEFAULT_CLAIM_TIMEOUT) do
+        sleep(0.01) while zk.create(path, id, ephemeral: true, ignore: :node_exists).nil?
+      end
+      Consumer.new(self, partition, options.dup)
+    end
+
+    # Release ownership of the partition
+    def release!(partition)
+      zk.delete claim_path(partition), ignore: :no_node
+    end
+
+  private
+
+    # @return [String] zookeeper ownership claim path
+    def claim_path(partition)
+      "#{registries[:owner]}/#{partition}"
+    end
+
+    # @return [String] zookeeper offset storage path
+    def offset_path(partition)
+      "#{registries[:offset]}/#{partition}"
+    end
+
+    # @return [String] zookeeper consumer registration path
+    def consumer_path
+      "#{registries[:consumer]}/#{id}"
+    end
+
+end

data/lib/poseidon_cluster.rb

@@ -0,0 +1 @@
+require 'poseidon/cluster'

data/poseidon_cluster.gemspec

@@ -0,0 +1,25 @@
+Gem::Specification.new do |s|
+  s.required_ruby_version = '>= 1.9.1'
+  s.required_rubygems_version = ">= 1.8.0"
+
+  s.name        = File.basename(__FILE__, '.gemspec')
+  s.summary     = "Poseidon cluster extensions"
+  s.description = "Cluster extensions for Poseidon, a producer and consumer implementation for Kafka >= 0.8"
+  s.version     = "0.0.2"
+
+  s.authors     = ["Black Square Media"]
+  s.email       = "info@blacksquaremedia.com"
+  s.homepage    = "https://github.com/bsm/poseidon_cluster"
+
+  s.require_path = 'lib'
+  s.files        = `git ls-files`.split("\n")
+  s.test_files   = `git ls-files -- {test,spec,features}/*`.split("\n")
+
+  s.add_dependency "poseidon"
+  s.add_dependency "zk"
+
+  s.add_development_dependency "rake"
+  s.add_development_dependency "bundler"
+  s.add_development_dependency "rspec"
+  s.add_development_dependency "yard"
+end

data/spec/integration/poseidon/consumer_group_spec.rb

@@ -0,0 +1,144 @@
+require 'spec_helper'
+
+describe Poseidon::ConsumerGroup, integration: true do
+
+  def new_group(max_bytes = 1024*8, name = TOPIC_NAME)
+    described_class.new "my-group", ["localhost:29092"], ["localhost:22181"], name, max_bytes: max_bytes
+  end
+
+  subject { new_group }
+  after   { zookeeper.rm_rf "/consumers/#{subject.name}" }
+
+  let(:consumed)  { Hash.new(0) }
+  let(:zookeeper) { ::ZK.new("localhost:22181") }
+
+  def stored_offsets
+    { 0 => subject.offset(0), 1 => subject.offset(1) }
+  end
+
+  describe "small batches" do
+
+    it "should consume messages from all partitions" do
+      5.times do
+        subject.fetch {|n, msgs| consumed[n] += msgs.size }
+      end
+      consumed.values.inject(0, :+).should < 676
+
+      5.times do
+        subject.fetch {|n, msgs| consumed[n] += msgs.size }
+      end
+      consumed.keys.should =~ [0, 1]
+      consumed.values.inject(0, :+).should == 676
+      consumed.should == stored_offsets
+    end
+
+  end
+
+  describe "large batches" do
+    subject { new_group 1024 * 1024 * 10 }
+
+    it "should consume messages from all partitions" do
+      5.times do
+        subject.fetch {|n, msgs| consumed[n] += msgs.size }
+      end
+      consumed.keys.should =~ [0, 1]
+      consumed.values.inject(0, :+).should == 676
+      consumed.should == stored_offsets
+    end
+  end
+
+  describe "fuzzing" do
+
+    def in_thread(batch_size, target, qu)
+      Thread.new do
+        group = new_group(batch_size)
+        sum   = 0
+        while sum < target && qu.size < 676
+          group.fetch {|_, m| sum += m.size; m.size.times { qu << true } }
+        end
+        group.close
+        sum
+      end
+    end
+
+    it "should consume from multiple sources" do
+      q = Queue.new
+      a = in_thread(4001, 200, q)
+      b = in_thread(4002,  50, q)
+      c = in_thread(4003, 120, q)
+      d = in_thread(4004,  40, q)
+      e = in_thread(4005, 400, q)
+      vals = [a, b, c, d, e].map &:value
+      vals.inject(0, :+).should == 676
+    end
+
+  end
+
+  describe "multi-process fuzzing", slow: true do
+    before do
+      producer = Poseidon::Producer.new(["localhost:29092"], "my-producer")
+      payload  = "data" * 10
+      100.times do
+        messages = (0...1000).map do |i|
+          Poseidon::MessageToSend.new("slow-topic", payload, i.to_s)
+        end
+        producer.send_messages(messages)
+      end
+    end
+
+    it 'should consume correctly' do
+      read, write = IO.pipe
+      pid1 = fork do
+        group = new_group(64*1024, "slow-topic")
+        10.times do
+          5.times { group.fetch {|_, m| write.write "1:#{m.size}\n" }}
+          sleep(1)
+        end
+      end
+      pid2 = fork do
+        group = new_group(32*1024, "slow-topic")
+        5.times do
+          10.times { group.fetch {|_, m| write.write "2:#{m.size}\n" }}
+          sleep(1)
+        end
+      end
+      pid3 = fork do
+        group = new_group(8*1024, "slow-topic")
+        5.times do
+          50.times { group.fetch {|_, m| write.write "3:#{m.size}\n" }}
+        end
+      end
+      Process.wait(pid2)
+
+      pid4 = fork do
+        group = new_group(8*1024, "slow-topic")
+        5.times do
+          50.times { group.fetch {|_, m| write.write "4:#{m.size}\n" }}
+        end
+      end
+      pid5 = fork do
+        group = new_group(32*1024, "slow-topic")
+        8.times do
+          50.times { group.fetch {|_, m| write.write "5:#{m.size}\n" }}
+          sleep(2)
+        end
+      end
+      Process.wait(pid1)
+      Process.wait(pid3)
+      Process.wait(pid4)
+      Process.wait(pid5)
+      write.close
+      raw = read.read
+      read.close
+
+      stats = raw.lines.inject(Hash.new(0)) do |res, line|
+        pid, count = line.chomp.split(":")
+        res[pid.to_i] += count.to_i
+        res
+      end
+      stats.keys.size.should be_within(1).of(4)
+      stats.values.inject(0, :+).should == 100_000
+    end
+
+  end
+end

data/spec/lib/poseidon/cluster_spec.rb

@@ -0,0 +1,15 @@
+require 'spec_helper'
+
+describe Poseidon::Cluster do
+
+  it 'should generate incremented numbers (atomically)' do
+    num = described_class.inc!
+    (described_class.inc! - num).should == 1
+
+    (0...5).map do
+      Thread.new { 100.times { described_class.inc! }}
+    end.each &:join
+    (described_class.inc! - num).should == 502
+  end
+
+end

data/spec/lib/poseidon/consumer_group_spec.rb

@@ -0,0 +1,190 @@
+require 'spec_helper'
+
+describe Poseidon::ConsumerGroup do
+
+  def new_group
+    group = described_class.new "my-group", ["localhost:29092", "localhost:29091"], ["localhost:22181"], TOPIC_NAME
+    groups.push(group)
+    group
+  end
+
+  def fetch_response(n)
+    set = Poseidon::MessageSet.new
+    n.times {|i| set << Poseidon::Message.new(value: "value", key: "key", offset: i) }
+    pfr = Poseidon::Protocol::PartitionFetchResponse.new(0, 0, 100, set)
+    tfr = Poseidon::Protocol::TopicFetchResponse.new(TOPIC_NAME, [pfr])
+    Poseidon::Protocol::FetchResponse.new(nil, [tfr])
+  end
+
+  let :groups do
+    []
+  end
+
+  let :brokers do
+    [ Poseidon::Protocol::Broker.new(1, "localhost", 29092),   # id,host,port
+      Poseidon::Protocol::Broker.new(2, "localhost", 29091), ]
+  end
+
+  let :partitions do
+    [ Poseidon::Protocol::PartitionMetadata.new(0, 0, 1, [1,2], []), # err,id,leader,replicas,isr
+      Poseidon::Protocol::PartitionMetadata.new(0, 1, 2, [1,2], []), ]
+  end
+
+  let :topics do
+    [ Poseidon::TopicMetadata.new(Poseidon::Protocol::TopicMetadataStruct.new(0, TOPIC_NAME, partitions)) ]
+  end
+
+  let :metadata do
+    Poseidon::Protocol::MetadataResponse.new nil, brokers.dup, topics.dup
+  end
+
+  subject { new_group }
+  before do
+    Poseidon::BrokerPool.any_instance.stub(:fetch_metadata_from_broker).and_return(metadata)
+    Poseidon::Connection.any_instance.stub(:fetch).with{|_, _, req| req[0].partition_fetches[0].partition == 0 }.and_return(fetch_response(10))
+    Poseidon::Connection.any_instance.stub(:fetch).with{|_, _, req| req[0].partition_fetches[0].partition == 1 }.and_return(fetch_response(5))
+  end
+  after do
+    subject.zk.rm_rf "/consumers/#{subject.name}"
+    groups.each(&:close)
+  end
+
+  its(:name)       { should == "my-group" }
+  its(:topic)      { should == TOPIC_NAME }
+  its(:pool)       { should be_instance_of(Poseidon::BrokerPool) }
+  its(:id)         { should match(/\Amy-group\-[\w\-\.]+?\-\d{1,5}\-\d{10}\-\d{1,3}\z/) }
+  its(:zk)         { should be_instance_of(ZK::Client::Threaded) }
+
+  its(:claimed)        { should == [0, 1] }
+  its(:metadata)       { should be_instance_of(Poseidon::ClusterMetadata) }
+  its(:topic_metadata) { should be_instance_of(Poseidon::TopicMetadata) }
+  its(:registries)     { should == {
+    consumer: "/consumers/my-group/ids",
+    owner:    "/consumers/my-group/owners/my-topic",
+    offset:   "/consumers/my-group/offsets/my-topic",
+  }}
+
+  its("metadata.brokers.keys") { should =~ [1,2] }
+  its("topic_metadata.partition_count") { should == 2 }
+
+  it "should register with zookeeper" do
+    subject.zk.children("/consumers/my-group/ids").should include(subject.id)
+    stat = subject.zk.stat("/consumers/my-group/ids")
+    stat.ephemeral_owner.should be(0)
+
+    data, stat = subject.zk.get("/consumers/my-group/ids/#{subject.id}")
+    data.should == "{}"
+    stat.num_children.should == 0
+    stat.ephemeral_owner.should > 0
+  end
+
+  it "should sort partitions by leader address" do
+    subject.partitions.map(&:id).should == [1, 0]
+  end
+
+  it "should return the offset for each partition" do
+    subject.offset(0).should == 0
+    subject.offset(1).should == 0
+    subject.offset(2).should == 0
+    subject.fetch {|*| true }
+    subject.offset(0).should == 0
+    subject.offset(1).should == 5
+    subject.offset(2).should == 0
+  end
+
+  it "should return the leader for a partition" do
+    subject.leader(0).should == brokers[0]
+    subject.leader(1).should == brokers[1]
+    subject.leader(2).should be_nil
+  end
+
+  it "should checkout individual partition consumers (atomically)" do
+    subject.checkout {|c| c.partition.should == 1 }
+    subject.checkout {|c| c.partition.should == 0 }
+
+    n = 0
+    a = Thread.new do
+      100.times { subject.checkout {|_| n+=1 } }
+      Thread.pass
+      100.times { subject.checkout {|_| n+=1 } }
+    end
+    b = Thread.new do
+      100.times { subject.checkout {|_| n+=1 } }
+      Thread.pass
+      100.times { subject.checkout {|_| n+=1 } }
+    end
+    [a, b].each &:join
+    n.should == 400
+  end
+
+  describe "rebalance" do
+
+    it "should watch out for new consumers joining/leaving" do
+      subject.should_receive(:rebalance!).twice.and_call_original
+      new_group.should_receive(:rebalance!).once.and_call_original
+      new_group
+    end
+
+    it "should distribute available partitions between consumers" do
+      subject.claimed.should == [0, 1]
+
+      b = new_group
+      wait_for { subject.claimed.size > 0 }
+      wait_for { b.claimed.size > 0 }
+      subject.claimed.should == [1]
+      b.claimed.should == [0]
+
+      c = new_group
+      subject.claimed.should == [1]
+      b.claimed.should == [0]
+      c.claimed.should == []
+
+      b.close
+      wait_for { b.claimed.size < 0 }
+      wait_for { c.claimed.size > 0 }
+
+      subject.claimed.should == [1]
+      b.claimed.should == []
+      c.claimed.should == [0]
+    end
+
+  end
+
+  describe "fetch" do
+
+    it "should return messages from owned partitions" do
+      subject.fetch do |n, msg|
+        n.should == 1
+        msg.size.should == 5
+      end
+      subject.fetch do |n, msg|
+        n.should == 0
+        msg.size.should == 10
+      end
+      subject.fetch do |n, msg|
+        n.should == 1
+        msg.size.should == 5
+      end
+    end
+
+    it "should auto-commit fetched offset" do
+      -> {
+        subject.fetch {|n, _| n.should == 1 }
+      }.should change { subject.offset(1) }.from(0).to(5)
+    end
+
+    it "should skip auto-commits if requested" do
+      -> {
+        subject.fetch(commit: false) {|n, _| n.should == 1 }
+      }.should_not change { subject.offset(1) }
+    end
+
+    it "should skip auto-commits if block results in false" do
+      -> {
+        subject.fetch {|n, _| n.should == 1; false }
+      }.should_not change { subject.offset(1) }
+    end
+
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,75 @@
+require 'poseidon_cluster'
+require 'rspec'
+require 'fileutils'
+require 'pathname'
+
+TOPIC_NAME  = "my-topic"
+KAFKA_LOCAL = File.expand_path("../kafka_2.8.0-0.8.0", __FILE__)
+KAFKA_ROOT  = Pathname.new(ENV["KAFKA_ROOT"] || KAFKA_LOCAL)
+
+module Poseidon::SpecHelper
+
+  def wait_for(&truth)
+    100.times do
+      break if truth.call
+      sleep(0.01)
+    end
+  end
+
+end
+
+RSpec.configure do |c|
+  c.include Poseidon::SpecHelper
+  c.filter_run_excluding slow: true unless ENV["SLOW"] == "1"
+
+  c.before :suite do
+    kafka_bin = KAFKA_ROOT.join("bin", "kafka-server-start.sh")
+    kafka_cfg = KAFKA_ROOT.join("config", "server-poseidon.properties")
+    zookp_bin = KAFKA_ROOT.join("bin", "zookeeper-server-start.sh")
+    zookp_cfg = KAFKA_ROOT.join("config", "zookeeper-poseidon.properties")
+
+    if KAFKA_ROOT.to_s == KAFKA_LOCAL && !kafka_bin.file?
+      puts "---> Downloading Kafka"
+      target = Pathname.new(File.expand_path("../", __FILE__))
+      system("cd #{target} && curl http://www.us.apache.org/dist/kafka/0.8.0/kafka_2.8.0-0.8.0.tar.gz | tar xz") ||
+        raise("Unable to download Kafka")
+
+      kafka_cfg.open("w") do |f|
+        f.write KAFKA_ROOT.join("config", "server.properties").read.sub("=9092", "=29092").sub(":2181", ":22181").sub("/tmp/kafka-logs", "/tmp/kafka-logs-poseidon")
+      end
+      zookp_cfg.open("w") do |f|
+        f.write KAFKA_ROOT.join("config", "zookeeper.properties").read.sub("=2181", "=22181")
+      end
+    end
+
+    # Ensure all required files are present
+    [kafka_bin, zookp_bin, kafka_cfg, zookp_cfg].each do |path|
+      raise "Unable to locate #{path}. File does not exist!" unless path.file?
+    end
+
+    # Start Zookeeper & Kafka
+    $ZOOKP_PID = spawn zookp_bin.to_s, zookp_cfg.to_s, out: '/dev/null' # , err: '/dev/null'
+    $KAFKA_PID = spawn kafka_bin.to_s, kafka_cfg.to_s, out: '/dev/null' #, err: '/dev/null'
+
+    # Produce some fixtures
+    producer = Poseidon::Producer.new(["localhost:29092"], "my-producer")
+    payload  = "data" * 10
+    messages = ("aa".."zz").map do |key|
+      Poseidon::MessageToSend.new(TOPIC_NAME, [key, payload].join(":"), key)
+    end
+
+    ok = false
+    100.times do
+      break if (ok = producer.send_messages(messages))
+      sleep(0.1)
+    end
+    raise "Unable to start Kafka instance." unless ok
+  end
+
+  c.after :suite do
+    Process.kill :TERM, $KAFKA_PID if $KAFKA_PID
+    Process.kill :TERM, $ZOOKP_PID if $ZOOKP_PID
+    FileUtils.rm_rf "/tmp/kafka-logs-poseidon"
+  end
+
+end

metadata

@@ -0,0 +1,142 @@
+--- !ruby/object:Gem::Specification
+name: poseidon_cluster
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- Black Square Media
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-01-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: poseidon
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: zk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Cluster extensions for Poseidon, a producer and consumer implementation
+  for Kafka >= 0.8
+email: info@blacksquaremedia.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- lib/poseidon/cluster.rb
+- lib/poseidon/consumer_group.rb
+- lib/poseidon_cluster.rb
+- poseidon_cluster.gemspec
+- spec/integration/poseidon/consumer_group_spec.rb
+- spec/lib/poseidon/cluster_spec.rb
+- spec/lib/poseidon/consumer_group_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/bsm/poseidon_cluster
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 1.9.1
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 1.8.0
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.0.rc.1
+signing_key: 
+specification_version: 4
+summary: Poseidon cluster extensions
+test_files: []
+has_rdoc: