promiscuous-poseidon_cluster 0.3.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f7653a5f9ee3e08c3ccda22787d2a9da5b752b86
+  data.tar.gz: 6447aadb67ad485dc77e79ea91af4d3b99bd49e1
+SHA512:
+  metadata.gz: 203099eb7d08fa1f80dd197c5e4094352291dc197a4e8137efde19450a4c4224e2cdc23bfdc9dd6789607c0544c326ce329c3c81bbd1d7a30f557f30ae0deddb
+  data.tar.gz: d78680cc99b1dd11fb11183f00fc4186835e8515ade5ae933b10d0a39a55f7bb0f5a99afb2d68e371198659402a20b421b6e9b80f3700006b941e7462a094889

data/.coveralls.yml

@@ -0,0 +1 @@
+service_name: travis-ci

data/.gitignore

@@ -0,0 +1,8 @@
+logs/
+kafka*/
+doc/
+.yardoc/
+.bundle/
+pkg/
+coverage/
+*.log

data/.travis.yml

@@ -0,0 +1,7 @@
+language: ruby
+rvm:
+  - 2.0.0
+  - 2.1.5
+script: bundle exec rake spec:coveralls scenario
+matrix:
+  fast_finish: true

data/Gemfile

@@ -0,0 +1,8 @@
+source "https://rubygems.org"
+
+gemspec
+
+platform :jruby do
+  gem 'slyphon-log4j', '= 1.2.15'
+  gem 'slyphon-zookeeper_jar', '= 3.3.5'
+end

data/Gemfile.lock

@@ -0,0 +1,77 @@
+PATH
+  remote: .
+  specs:
+    poseidon_cluster (0.3.0)
+      poseidon (>= 0.0.5.pre1)
+      zk
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    coveralls (0.7.0)
+      multi_json (~> 1.3)
+      rest-client
+      simplecov (>= 0.7)
+      term-ansicolor
+      thor
+    diff-lcs (1.2.5)
+    docile (1.1.3)
+    little-plugger (1.1.3)
+    logging (1.8.2)
+      little-plugger (>= 1.1.3)
+      multi_json (>= 1.8.4)
+    mime-types (2.3)
+    multi_json (1.10.1)
+    poseidon (0.0.5.pre1)
+    rake (10.3.2)
+    rest-client (1.6.7)
+      mime-types (>= 1.16)
+    rspec (3.0.0)
+      rspec-core (~> 3.0.0)
+      rspec-expectations (~> 3.0.0)
+      rspec-mocks (~> 3.0.0)
+    rspec-core (3.0.0)
+      rspec-support (~> 3.0.0)
+    rspec-expectations (3.0.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.0.0)
+    rspec-its (1.0.1)
+      rspec-core (>= 2.99.0.beta1)
+      rspec-expectations (>= 2.99.0.beta1)
+    rspec-mocks (3.0.0)
+      rspec-support (~> 3.0.0)
+    rspec-support (3.0.0)
+    simplecov (0.8.2)
+      docile (~> 1.1.0)
+      multi_json
+      simplecov-html (~> 0.8.0)
+    simplecov-html (0.8.0)
+    slyphon-log4j (1.2.15)
+    slyphon-zookeeper_jar (3.3.5-java)
+    term-ansicolor (1.3.0)
+      tins (~> 1.0)
+    thor (0.19.1)
+    tins (1.3.0)
+    yard (0.8.7.4)
+    zk (1.9.4)
+      logging (~> 1.8.2)
+      zookeeper (~> 1.4.0)
+    zookeeper (1.4.9)
+    zookeeper (1.4.9-java)
+      slyphon-log4j (= 1.2.15)
+      slyphon-zookeeper_jar (= 3.3.5)
+
+PLATFORMS
+  java
+  ruby
+
+DEPENDENCIES
+  bundler
+  coveralls
+  poseidon_cluster!
+  rake
+  rspec
+  rspec-its
+  slyphon-log4j (= 1.2.15)
+  slyphon-zookeeper_jar (= 3.3.5)
+  yard

data/README.md

@@ -0,0 +1,95 @@
+# Poseidon Cluster [![Build Status](https://travis-ci.org/bsm/poseidon_cluster.png?branch=master)](https://travis-ci.org/bsm/poseidon_cluster) [![Coverage Status](https://coveralls.io/repos/bsm/poseidon_cluster/badge.png?branch=master)](https://coveralls.io/r/bsm/poseidon_cluster?branch=master)
+
+Poseidon Cluster is a cluster extension of the excellent [Poseidon](http://github.com/bpot/poseidon) Ruby client for Kafka 0.8+. It implements the distribution concept of self-rebalancing *Consumer Groups* and supports the consumption of a single topic from multiple instances.
+
+Consumer group instances share a common group name, and each message published to a topic is delivered to one instance within each subscribing consumer group. Consumer instances can be in separate processes or on separate machines.
+
+## Usage
+
+Launch a consumer group:
+
+```ruby
+require 'poseidon_cluster'
+
+consumer = Poseidon::ConsumerGroup.new(
+            "my-group",                               # Group name
+            ["kafka1.host:9092", "kafka2.host:9092"], # Kafka brokers
+            ["kafka1.host:2181", "kafka2.host:2181"], # Zookeepers hosts
+            "my-topic")                               # Topic name
+
+consumer.partitions # => [0, 1, 2, 3] - all partitions of 'my-topic'
+consumer.claimed    # => [0, 1] - partitions this instance has claimed
+```
+
+Fetch a bulk of messages, auto-commit the offset:
+
+```ruby
+consumer.fetch do |partition, bulk|
+  bulk.each do |m|
+    puts "Fetched '#{m.value}' at #{m.offset} from #{partition}"
+  end
+end
+```
+
+Get the offset for a partition:
+
+```ruby
+consumer.offset(0) # => 320 - current offset from partition 0
+```
+
+Fetch more messages, commit manually:
+
+```ruby
+consumer.fetch commit: false do |partition, bulk|
+  bulk.each do |m|
+    puts "Fetched '#{m.value}' at #{m.offset} from #{partition}"
+  end
+
+  consumer.commit partition, bulk.last.offset+1 unless bulk.empty?
+end
+```
+
+Initiate a fetch-loop, consume indefinitely:
+
+```ruby
+consumer.fetch_loop do |partition, bulk|
+  bulk.each do |m|
+    puts "Fetched '#{m.value}' at #{m.offset} from #{partition}"
+  end
+end
+```
+
+For more details and information, please see the [Poseidon::ConsumerGroup](http://rubydoc.info/github/bsm/poseidon_cluster/Poseidon/ConsumerGroup) documentation and the [Examples](https://github.com/bsm/poseidon_cluster/tree/master/examples).
+
+## Running Tests
+
+The test suite will automatically download, configure and run Kafka locally, you only need a JRE. Run the suite via:
+
+```bash
+bundle exec rake spec
+```
+
+## Licence
+
+```
+Copyright (c) 2014 Black Square Media
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+```

data/Rakefile

@@ -0,0 +1,21 @@
+require 'bundler/gem_tasks'
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec)
+
+require 'yard'
+require 'yard/rake/yardoc_task'
+YARD::Rake::YardocTask.new
+
+require 'coveralls/rake/task'
+Coveralls::RakeTask.new
+namespace :spec do
+  task coveralls: [:spec, 'coveralls:push']
+end
+
+desc "Run full integration test scenario"
+task :scenario do
+  load File.expand_path("../scenario/run.rb", __FILE__)
+end
+
+task default: :spec

data/examples/consumer_group.rb

@@ -0,0 +1,33 @@
+=begin
+
+ PLEASE NOTE
+
+ This example uses threads, but you could equally use fork or run your
+ consumer groups from completely separate process and from multiple machines.
+
+=end
+require 'poseidon_cluster'
+
+# Create a consumer group
+group1 = Poseidon::ConsumerGroup.new "my-group", ["host1:9092", "host2:9092"], ["host1:2181", "host2:2181"], "my-topic"
+
+# Start consuming "my-topic" in a background thread
+thread1 = Thread.new do
+  group1.fetch_loop do |partition, messages|
+    puts "Consumer #1 fetched #{messages.size} from #{partition}"
+  end
+end
+
+# Create a second consumer group
+group2 = Poseidon::ConsumerGroup.new "my-group", ["host1:9092", "host2:9092"], ["host1:2181", "host2:2181"], "my-topic"
+
+# Now consuming all partitions of "my-topic" in parallel
+thread2 = Thread.new do
+  group2.fetch_loop do |partition, messages|
+    puts "Consumer #2 fetched #{messages.size} from #{partition}"
+  end
+end
+
+# Join threads, loop forever
+[thread1, thread2].each(&:join)
+

data/lib/poseidon/cluster.rb

@@ -0,0 +1,28 @@
+require 'socket'
+require 'timeout'
+require 'zk'
+require 'poseidon'
+require 'thread'
+
+module Poseidon::Cluster
+  MAX_INT32 = 0x7fffffff
+  @@sem = Mutex.new
+  @@inc = 0
+
+  # @return [Integer] an incremented number
+  # @api private
+  def self.inc!
+    @@sem.synchronize { @@inc += 1; @@inc = 1 if @@inc > MAX_INT32; @@inc }
+  end
+
+  # @return [String] an globally unique identifier
+  # @api private
+  def self.guid
+    [::Socket.gethostname, ::Process.pid, ::Time.now.to_i, inc!].join("-")
+  end
+
+end
+
+%w|consumer_group|.each do |name|
+  require "poseidon/#{name}"
+end

data/lib/poseidon/consumer_group.rb

@@ -0,0 +1,435 @@
+# A ConsumerGroup operates on all partitions of a single topic. The goal is to ensure
+# each topic message is consumed only once, no matter of the number of consumer instances within
+# a cluster, as described in: http://kafka.apache.org/documentation.html#distributionimpl.
+#
+# The ConsumerGroup internally creates multiple PartitionConsumer instances. It uses Zookkeper
+# and follows a simple consumer rebalancing algorithms which allows all the consumers
+# in a group to come into consensus on which consumer is consuming which partitions. Each
+# ConsumerGroup can 'claim' 0-n partitions and will consume their messages until another
+# ConsumerGroup instance joins or leaves the cluster.
+#
+# Please note: ConsumerGroups themselves don't implement any threading or concurrency.
+# When consuming messages, they simply round-robin across the claimed partitions. If you wish
+# to parallelize consumption simply create multiple ConsumerGroups instances. The built-in
+# concensus algorithm will automatically rebalance the available partitions between them and you
+# can then decide for yourself if you want to run them in multiple thread or processes, ideally
+# on multiple boxes.
+#
+# Unlike stated in the Kafka documentation, consumer rebalancing is *only* triggered on each
+# addition or removal of consumers within the same group, while the addition of broker nodes
+# and/or partition *does currently not trigger* a rebalancing cycle.
+#
+# @api public
+class Poseidon::ConsumerGroup
+  DEFAULT_CLAIM_TIMEOUT = 30
+  DEFAULT_LOOP_DELAY = 1
+
+  # Poseidon::ConsumerGroup::Consumer is internally used by Poseidon::ConsumerGroup.
+  # Don't invoke it directly.
+  #
+  # @api private
+  class Consumer < ::Poseidon::PartitionConsumer
+
+    # @attr_reader [Integer] partition consumer partition
+    attr_reader :partition
+
+    # @api private
+    def initialize(group, partition, options = {})
+      broker = group.leader(partition)
+      offset = group.offset(partition)
+      offset = (options[:trail] ? :latest_offset : :earliest_offset) if offset == 0
+      options.delete(:trail)
+      super group.id, broker.host, broker.port, group.topic, partition, offset, options
+    end
+
+  end
+
+  # @param [Integer] pnum number of partitions size
+  # @param [Array<String>] cids consumer IDs
+  # @param [String] id consumer ID
+  # @return [Range, NilClass] selectable range, if any
+  def self.pick(pnum, cids, id)
+    cids = cids.sort
+    pos  = cids.index(id)
+    return unless pos && pos < cids.size
+
+    step = pnum.fdiv(cids.size).ceil
+    frst = pos*step
+    last = (pos+1)*step-1
+    last = pnum-1 if last > pnum-1
+    return if last < 0 || last < frst
+
+    (frst..last)
+  end
+
+  # @attr_reader [String] name Group name
+  attr_reader :name
+
+  # @attr_reader [String] topic Topic name
+  attr_reader :topic
+
+  # @attr_reader [Poseidon::BrokerPool] pool Broker pool
+  attr_reader :pool
+
+  # @attr_reader [ZK::Client] zk Zookeeper client
+  attr_reader :zk
+
+  # @attr_reader [Hash] options Consumer options
+  attr_reader :options
+
+  # Create a new consumer group, which processes all partition of the specified topic.
+  #
+  # @param [String] name Group name
+  # @param [Array<String>] brokers A list of known brokers, e.g. ["localhost:9092"]
+  # @param [Array<String>] zookeepers A list of known zookeepers, e.g. ["localhost:2181"]
+  # @param [String] topic Topic to operate on
+  # @param [Hash] options Consumer options
+  # @option options [Integer] :max_bytes Maximum number of bytes to fetch. Default: 1048576 (1MB)
+  # @option options [Integer] :max_wait_ms How long to block until the server sends us data. Default: 100 (100ms)
+  # @option options [Integer] :min_bytes Smallest amount of data the server should send us. Default: 0 (Send us data as soon as it is ready)
+  # @option options [Integer] :claim_timeout Maximum number of seconds to wait for a partition claim. Default: 10
+  # @option options [Integer] :loop_delay Number of seconds to delay the next fetch (in #fetch_loop) if nothing was returned. Default: 1
+  # @option options [Integer] :socket_timeout_ms broker connection wait timeout in ms. Default: 10000
+  # @option options [Boolean] :register Automatically register instance and start consuming. Default: true
+  # @option options [Boolean] :trail Starts reading messages from the latest partitions offsets and skips 'old' messages . Default: false
+  #
+  # @api public
+  def initialize(name, brokers, zookeepers, topic, options = {})
+    @name       = name
+    @topic      = topic
+    @zk         = ::ZK.new(zookeepers.join(","))
+    # Poseidon::BrokerPool doesn't provide default value for this option
+    # Configuring default value like this isn't beautiful, though.. by kssminus
+    options[:socket_timeout_ms] ||= 10000
+    @options    = options
+    @consumers  = []
+    @pool       = ::Poseidon::BrokerPool.new(id, brokers, options[:socket_timeout_ms])
+    @mutex      = Mutex.new
+    @registered = false
+
+    register! unless options[:register] == false
+  end
+
+  # @return [String] a globally unique identifier
+  def id
+    @id ||= [name, Poseidon::Cluster.guid].join("-")
+  end
+
+  # @return [Hash<Symbol,String>] registry paths
+  def registries
+    @registries ||= {
+      consumer: "/consumers/#{name}/ids",
+      owner:    "/consumers/#{name}/owners/#{topic}",
+      offset:   "/consumers/#{name}/offsets/#{topic}",
+    }
+  end
+
+  # @return [Poseidon::ClusterMetadata] cluster metadata
+  def metadata
+    @metadata ||= Poseidon::ClusterMetadata.new.tap {|m| m.update pool.fetch_metadata([topic]) }
+  end
+
+  # @return [Poseidon::TopicMetadata] topic metadata
+  def topic_metadata
+    @topic_metadata ||= metadata.metadata_for_topics([topic])[topic]
+  end
+
+  # @return [Boolean] true if registered
+  def registered?
+    @registered
+  end
+
+  # @return [Boolean] true if registration was successful, false if already registered
+  def register!
+    return false if registered?
+
+    # Register instance
+    registries.each do |_, path|
+      zk.mkdir_p(path)
+    end
+    zk.create(consumer_path, "{}", ephemeral: true)
+    zk.register(registries[:consumer]) {|_| rebalance! }
+
+    # Rebalance
+    rebalance!
+    @registered = true
+  end
+
+  # Reloads metadata/broker/partition information
+  def reload
+    @metadata = @topic_metadata = nil
+    metadata
+    self
+  end
+
+  # Closes the consumer group gracefully, only really useful in tests
+  # @api private
+  def close
+    @mutex.synchronize { release_all! }
+    zk.close
+  end
+
+  # @param [Integer] partition
+  # @return [Poseidon::Protocol::Broker] the leader for the given partition
+  def leader(partition)
+    metadata.lead_broker_for_partition(topic, partition)
+  end
+
+  # @param [Integer] partition
+  # @return [Integer] the latest stored offset for the given partition
+  def offset(partition)
+    data, _ = zk.get offset_path(partition), ignore: :no_node
+    data.to_i
+  end
+
+  # Commits the latest offset for a partition
+  # @param [Integer] partition
+  # @param [Integer] offset
+  def commit(partition, offset)
+    zk.set offset_path(partition), offset.to_s
+  rescue ZK::Exceptions::NoNode
+    zk.create offset_path(partition), offset.to_s, ignore: :node_exists
+  end
+
+  # Sorted partitions by broker address (so partitions on the same broker are clustered together)
+  # @return [Array<Poseidon::Protocol::PartitionMetadata>] sorted partitions
+  def partitions
+    return [] unless topic_metadata
+
+    topic_metadata.available_partitions.sort_by do |part|
+      broker = metadata.brokers[part.leader]
+      [broker.host, broker.port].join(":")
+    end
+  end
+
+  # Partitions currently claimed and consumed by this group instance
+  # @return [Array<Integer>] partition IDs
+  def claimed
+    @consumers.map(&:partition).sort
+  end
+
+  # Checks out a single partition consumer. Round-robins between claimed partitions.
+  #
+  # @yield [consumer] The processing block
+  # @yieldparam [Consumer] consumer The consumer instance
+  # @yieldreturn [Boolean] return false to stop auto-commit
+  #
+  # @param [Hash] opts
+  # @option opts [Boolean] :commit Automatically commit consumer offset (default: true)
+  # @return [Boolean] true if a consumer was checked out, false if none could be claimed
+  #
+  # @example
+  #
+  #   ok = group.checkout do |consumer|
+  #     puts "Checked out consumer for partition #{consumer.partition}"
+  #   end
+  #   ok # => true if the block was run, false otherwise
+  #
+  # @api public
+  def checkout(opts = {})
+    consumer = nil
+    commit   = @mutex.synchronize do
+      consumer = @consumers.shift
+      return false unless consumer
+
+      @consumers.push consumer
+      yield consumer
+    end
+
+    unless opts[:commit] == false || commit == false
+      commit consumer.partition, consumer.offset
+    end
+    true
+  end
+
+  # Convenience method to fetch messages from the broker.
+  # Round-robins between claimed partitions.
+  #
+  # @yield [partition, messages] The processing block
+  # @yieldparam [Integer] partition The source partition
+  # @yieldparam [Array<Message>] messages The fetched messages
+  # @yieldreturn [Boolean] return false to prevent auto-commit
+  #
+  # @param [Hash] opts
+  # @option opts [Boolean] :commit Automatically commit consumed offset (default: true)
+  # @return [Boolean] true if messages were fetched, false if none could be claimed
+  #
+  # @example
+  #
+  #   ok = group.fetch do |n, messages|
+  #     puts "Fetched #{messages.size} messages for partition #{n}"
+  #   end
+  #   ok # => true if the block was run, false otherwise
+  #
+  # @api public
+  def fetch(opts = {})
+    checkout(opts) do |consumer|
+      yield consumer.partition, consumer.fetch
+    end
+  end
+
+  # Initializes an infinite fetch loop. This method blocks!
+  #
+  # Will wait for `loop_delay` seconds after each failed fetch. This may happen when there is
+  # no new data or when the consumer hasn't claimed any partitions.
+  #
+  # SPECIAL ATTENTION:
+  # When 'breaking out' of the loop, you must do it before processing the messages, as the
+  # the last offset will not be committed. Please see examples below.
+  #
+  # @yield [partition, messages] The processing block
+  # @yieldparam [Integer] partition The source partition, may be -1 if no partitions are claimed
+  # @yieldparam [Array<Message>] messages The fetched messages
+  # @yieldreturn [Boolean] return false to prevent auto-commit
+  #
+  # @param [Hash] opts
+  # @option opts [Boolean] :commit Automatically commit consumed offset (default: true)
+  # @option opts [Boolean] :loop_delay Delay override in seconds after unsuccessful fetch.
+  #
+  # @example
+  #
+  #   group.fetch_loop do |n, messages|
+  #     puts "Fetched #{messages.size} messages for partition #{n}"
+  #   end
+  #   puts "Done" # => this code is never reached
+  #
+  # @example Stopping the loop (wrong)
+  #
+  #   counts = Hash.new(0)
+  #   group.fetch_loop do |n, messages|
+  #     counts[n] += messages.size
+  #     puts "Status: #{counts.inspect}"
+  #     break if counts[0] > 100
+  #   end
+  #   puts "Result: #{counts.inspect}"
+  #   puts "Offset: #{group.offset(0)}"
+  #
+  #   # Output:
+  #   # Status: {0=>30}
+  #   # Status: {0=>60}
+  #   # Status: {0=>90}
+  #   # Status: {0=>120}
+  #   # Result: {0=>120}
+  #   # Offset: 90      # => Last offset was not committed!
+  #
+  # @example Stopping the loop (correct)
+  #
+  #   counts = Hash.new(0)
+  #   group.fetch_loop do |n, messages|
+  #     break if counts[0] > 100
+  #     counts[n] += messages.size
+  #     puts "Status: #{counts.inspect}"
+  #   end
+  #   puts "Result: #{counts.inspect}"
+  #   puts "Offset: #{group.offset(0)}"
+  #
+  #   # Output:
+  #   # Status: {0=>30}
+  #   # Status: {0=>60}
+  #   # Status: {0=>90}
+  #   # Status: {0=>120}
+  #   # Result: {0=>120}
+  #   # Offset: 120
+  #
+  # @api public
+  def fetch_loop(opts = {})
+    delay = opts[:loop_delay] || options[:loop_delay] || DEFAULT_LOOP_DELAY
+
+    loop do
+      mp = false
+      ok = fetch(opts) do |n, messages|
+        mp = !messages.empty?
+        yield n, messages
+      end
+
+      # Yield over an empty array if nothing claimed,
+      # to allow user to e.g. break out of the loop
+      unless ok
+        yield -1, []
+      end
+
+      # Sleep if either not claimes or nothing returned
+      unless ok && mp
+        sleep delay
+      end
+    end
+  end
+
+  protected
+
+    # Rebalance algorithm:
+    #
+    # * let CG be all consumers in the same group that consume topic T
+    # * let PT be all partitions producing topic T
+    # * sort CG
+    # * sort PT (so partitions on the same broker are clustered together)
+    # * let POS be our index position in CG and let N = size(PT)/size(CG)
+    # * assign partitions from POS*N to (POS+1)*N-1
+    def rebalance!
+      return if @pending
+
+      @pending = true
+      @mutex.synchronize do
+        @pending = nil
+
+        release_all!
+        reload
+
+        ids = zk.children(registries[:consumer], watch: true)
+        pms = partitions
+        rng = self.class.pick(pms.size, ids, id)
+
+        pms[rng].each do |pm|
+          if @pending
+            release_all!
+            break
+          end
+
+          consumer = claim!(pm.id)
+          @consumers.push(consumer) if consumer
+        end if rng
+      end
+    end
+
+    # Release all consumer claims
+    def release_all!
+      @consumers.each {|c| release!(c.partition) }
+      @consumers.clear
+    end
+
+  private
+
+    # Claim the ownership of the partition for this consumer
+    # @raise [Timeout::Error]
+    def claim!(partition)
+      path = claim_path(partition)
+      Timeout.timeout options[:claim_timout] || DEFAULT_CLAIM_TIMEOUT do
+        while zk.create(path, id, ephemeral: true, ignore: :node_exists).nil?
+          return if @pending
+          sleep(0.1)
+        end
+      end
+      Consumer.new self, partition, options.dup
+    end
+
+    # Release ownership of the partition
+    def release!(partition)
+      zk.delete claim_path(partition), ignore: :no_node
+    end
+
+    # @return [String] zookeeper ownership claim path
+    def claim_path(partition)
+      "#{registries[:owner]}/#{partition}"
+    end
+
+    # @return [String] zookeeper offset storage path
+    def offset_path(partition)
+      "#{registries[:offset]}/#{partition}"
+    end
+
+    # @return [String] zookeeper consumer registration path
+    def consumer_path
+      "#{registries[:consumer]}/#{id}"
+    end
+
+end