kafka-consumer 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7e0b4f66c8e873f254c213a4d12ec89a2a46134a
-  data.tar.gz: 0dcc6b6cdce939337a51acebc95b9c7d8ac6b757
+  metadata.gz: fd211d48c462dd372ab59425ef6f36b79460abd0
+  data.tar.gz: b13cb4c3d7fcbba2c6c7703eb6ea4e6d72fc030d
 SHA512:
-  metadata.gz: 22f95d7e30a7c4e8c83b1e2ddea5c962a88b1655e19b68df9eadc670286f5a5ae98376b95e7eac9fd85e5f6c13d78e74f009787b5e0eaeb64f4217656a7df104
-  data.tar.gz: f4b831cb109d4ee7b05efa2c61cb3ba00874088e62b03d83af981c10ce411a4dc10535f56a163bc7058f0f5a7d27cac286dff4ef4ad4b359126f757f843583a0
+  metadata.gz: 48d39d21d7370f509b644667ae428c0d8c7ef27883d336ef98e102e4efb2f7f75909959acff27e8b75cfba56b681a1df5f2337f3031e4e6fdfd1bd7ae094536c
+  data.tar.gz: 2de3c13aa95faeb7ed27d7a558ac03c22f5098fb47d28f76125d276608a55d4b6c9e6c1bdba8030601e8bd42f4df43f8a956b24b6690622299f4237dd0db8b74

data/.travis.yml

@@ -0,0 +1,10 @@
+language: ruby
+sudo: false
+
+rvm:
+- "2.0"
+- "2.1"
+- "2.2"
+
+cache:
+- bundler

data/README.md

@@ -11,7 +11,7 @@ instance at a time. It uses Zookeeper watches to be notified of new consumer ins
 online or going offline, which will trigger a redistribition of all the partitions that are consumed.
 
 Periodically, it will commit the last processed offset of every partition to Zookeeper. Whenever a
-new consumer starts, it will resume consumingevery partition at the last committed offset. This implements
+new consumer starts, it will resume consuming every partition at the last committed offset. This implements
 an **at least once guarantee**, so it is possible that you end up consuming the same message more than once.
 It's your responsibility to deal with this if that is a problem for you, e.g. by using idempotent operations.
 
@@ -34,6 +34,14 @@ consumer.each do |message|
 end
 ```
 
+## Notes
+
+- It will spawn a manager thread and two threads per partition. However, your code
+  doesn't have to be thread-safe because every message is yielded to `each`
+  sequentially using a mutex.
+- On my Macbook Pro, I can consume around ~10,000 messages per second from a
+  topic with 64 partitions on a production Kafka cluster.
+
 ## Contributing
 
 1. Fork it ( https://github.com/wvanbergen/kafka-consumer/fork )

data/Rakefile

@@ -30,14 +30,6 @@ namespace :kafka do
     puts
     puts "%d messages consumed in %0.3fs (%0.3f msg/s)" % [counter, duration, counter.to_f / duration]
   end
-
-  namespace :consumer do
-    task :reset do
-      zookeeper = ENV["ZOOKEEPER"] or raise "Specify the ZOOKEEPER connection string."
-      name      = ENV["NAME"]      or raise "Specify NAME to name the consumergroup."
-
-      consumer = Kafka::Consumer.new(name, [], zookeeper: zookeeper)
-      consumer.group.reset_offsets
-    end
-  end
 end
+
+task default: :test

data/kafka-consumer.gemspec

@@ -24,5 +24,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "mocha", "~> 1.0"
 
   spec.add_runtime_dependency "poseidon", "~> 0.0.5"
-  spec.add_runtime_dependency "zookeeper", "~> 1.4"
+  spec.add_runtime_dependency "kazoo-ruby", "~> 0.4.0"
 end

data/lib/kafka-consumer.rb

@@ -0,0 +1 @@
+require 'kafka/consumer'

data/lib/kafka/consumer.rb

@@ -9,8 +9,6 @@ require "kafka/consumer/version"
 
 module Kafka
   class Consumer
-    BACKPRESSURE_MESSAGE_LIMIT = 1000
-
     include Enumerable
 
     attr_reader :subscription,
@@ -18,17 +16,19 @@ module Kafka
         :max_wait_ms, :initial_offset,
         :logger
 
-    def initialize(name, subscription, zookeeper: [], max_wait_ms: 200, initial_offset: :latest_offset, logger: nil)
-      @name, @subscription = name, subscription
+    def initialize(name, subscription, zookeeper: nil, max_wait_ms: 200, initial_offset: :latest_offset, logger: nil)
+      raise ArgumentError, "The consumer's name cannot be empty" if name.nil? || name.empty?
+      raise ArgumentError, "You have to specify a zookeeper connection string" if zookeeper.nil? || zookeeper.empty?
+
+      @name = name
       @max_wait_ms, @initial_offset = max_wait_ms, initial_offset
       @logger = logger || Logger.new($stdout)
 
       @cluster = Kazoo::Cluster.new(zookeeper)
       @group = Kazoo::Consumergroup.new(@cluster, name)
-      @group.create unless @group.exists?
 
-      @instance = @group.instantiate
-      @instance.register(topics)
+      @group.create unless @group.exists?
+      @instance = @group.instantiate(subscription: Kazoo::Subscription.build(subscription)).register
     end
 
     def name
@@ -39,15 +39,12 @@ module Kafka
       instance.id
     end
 
-    def topics
-      @topics ||= begin
-        topic_names = Array(subscription)
-        topic_names.map { |topic_name| cluster.topics.fetch(topic_name) }
-      end
+    def subscription
+      instance.subscription
     end
 
     def partitions
-      topics.flat_map(&:partitions).sort_by { |partition| [partition.leader.id, partition.topic.name, partition.id] }
+      subscription.partitions(@cluster).sort_by { |partition| [partition.preferred_leader.id, partition.topic.name, partition.id] }
     end
 
     def interrupt
@@ -83,9 +80,7 @@ module Kafka
       mutex = Mutex.new
 
       handler = lambda do |message|
-        mutex.synchronize do
-          block.call(message)
-        end
+        mutex.synchronize { block.call(message) }
       end
 
       @consumer_manager = Thread.new do

data/lib/kafka/consumer/version.rb

@@ -1,5 +1,5 @@
 module Kafka
   class Consumer
-    VERSION = "0.1.1"
+    VERSION = "0.1.2"
   end
 end

data/test/test_helper.rb

@@ -1,48 +1,4 @@
 require 'minitest/autorun'
 require 'kafka/consumer'
-require 'kazoo'
 require 'mocha/mini_test'
 require 'pp'
-
-module MockCluster
-  def mock_cluster
-    cluster = Kazoo::Cluster.new('example.com:2181')
-    cluster.stubs(:zk).returns(mock)
-
-    cluster.stubs(:brokers).returns(
-      1 => Kazoo::Broker.new(cluster, 1, 'example.com', 9091),
-      2 => Kazoo::Broker.new(cluster, 2, 'example.com', 9092),
-      3 => Kazoo::Broker.new(cluster, 3, 'example.com', 9093),
-    )
-
-    cluster.stubs(:topics).returns(
-      'test.1' => topic_1 = Kazoo::Topic.new(cluster, 'test.1'),
-      'test.4' => topic_4 = Kazoo::Topic.new(cluster, 'test.4')
-    )
-
-    topic_1.stubs(:partitions).returns([
-      Kazoo::Partition.new(topic_1, 0, replicas: [cluster.brokers[1], cluster.brokers[2]]),
-    ])
-
-    topic_4.stubs(:partitions).returns([
-      Kazoo::Partition.new(topic_4, 0, replicas: [cluster.brokers[2], cluster.brokers[1]]),
-      Kazoo::Partition.new(topic_4, 1, replicas: [cluster.brokers[2], cluster.brokers[3]]),
-      Kazoo::Partition.new(topic_4, 2, replicas: [cluster.brokers[1], cluster.brokers[3]]),
-      Kazoo::Partition.new(topic_4, 3, replicas: [cluster.brokers[3], cluster.brokers[2]]),
-    ])
-
-    topic_1.partitions[0].stubs(:isr).returns([cluster.brokers[1], cluster.brokers[2]])
-    topic_4.partitions[0].stubs(:isr).returns([cluster.brokers[2], cluster.brokers[1]])
-    topic_4.partitions[1].stubs(:isr).returns([cluster.brokers[2], cluster.brokers[3]])
-    topic_4.partitions[2].stubs(:isr).returns([cluster.brokers[1], cluster.brokers[3]])
-    topic_4.partitions[3].stubs(:isr).returns([cluster.brokers[3], cluster.brokers[2]])
-
-    topic_1.partitions[0].stubs(:leader).returns(cluster.brokers[1])
-    topic_4.partitions[0].stubs(:leader).returns(cluster.brokers[2])
-    topic_4.partitions[1].stubs(:leader).returns(cluster.brokers[2])
-    topic_4.partitions[2].stubs(:leader).returns(cluster.brokers[1])
-    topic_4.partitions[3].stubs(:leader).returns(cluster.brokers[3])
-
-    return cluster
-  end
-end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kafka-consumer
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Willem van Bergen
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-09 00:00:00.000000000 Z
+date: 2015-10-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -81,53 +81,41 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.0.5
 - !ruby/object:Gem::Dependency
-  name: zookeeper
+  name: kazoo-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: 0.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: 0.4.0
 description: High-level consumer for Kafka. Implements the Zookeeper-backed consumer
   implementation that offers offset management, load balancing and automatic failovers.
 email:
 - willem@vanbergen.org
-executables:
-- kazoo
+executables: []
 extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .travis.yml
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
-- bin/kazoo
 - kafka-consumer.gemspec
+- lib/kafka-consumer.rb
 - lib/kafka/consumer.rb
 - lib/kafka/consumer/message.rb
 - lib/kafka/consumer/partition_consumer.rb
 - lib/kafka/consumer/version.rb
-- lib/kazoo.rb
-- lib/kazoo/broker.rb
-- lib/kazoo/cli.rb
-- lib/kazoo/cluster.rb
-- lib/kazoo/consumergroup.rb
-- lib/kazoo/partition.rb
-- lib/kazoo/topic.rb
-- lib/kazoo/version.rb
-- test/broker_test.rb
-- test/cluster_test.rb
 - test/partition_distribution_test.rb
-- test/partition_test.rb
 - test/test_helper.rb
-- test/topic_test.rb
 homepage: https://github.com/wvanbergen/kafka-consumer
 licenses:
 - MIT
@@ -153,9 +141,5 @@ signing_key:
 specification_version: 4
 summary: High-level consumer for Kafka
 test_files:
-- test/broker_test.rb
-- test/cluster_test.rb
 - test/partition_distribution_test.rb
-- test/partition_test.rb
 - test/test_helper.rb
-- test/topic_test.rb

data/bin/kazoo

@@ -1,14 +0,0 @@
-#!/usr/bin/env ruby
-$LOAD_PATH.unshift(File.expand_path('../../lib', __FILE__))
-require 'kazoo/cli'
-
-begin
-  ENV["THOR_DEBUG"] = "1"
-  Kazoo::CLI.start(ARGV)
-rescue Thor::UndefinedCommandError, Thor::UnknownArgumentError, Thor::AmbiguousCommandError, Thor::InvocationError => e
-  $stderr.puts(e.message)
-  exit(64)
-rescue Thor::Error => e
-  $stderr.puts(e.message)
-  exit(1)
-end

data/lib/kazoo.rb

@@ -1,19 +0,0 @@
-require 'zookeeper'
-require 'json'
-require 'thread'
-require 'socket'
-require 'securerandom'
-
-module Kazoo
-  Error = Class.new(StandardError)
-  ConsumerInstanceRegistrationFailed = Class.new(Kazoo::Error)
-  PartitionAlreadyClaimed = Class.new(Kazoo::Error)
-  ReleasePartitionFailure = Class.new(Kazoo::Error)
-end
-
-require 'kazoo/cluster'
-require 'kazoo/broker'
-require 'kazoo/topic'
-require 'kazoo/partition'
-require 'kazoo/consumergroup'
-require 'kazoo/version'

data/lib/kazoo/broker.rb

@@ -1,68 +0,0 @@
-module Kazoo
-  class Broker
-    attr_reader :cluster, :id, :host, :port, :jmx_port
-
-    def initialize(cluster, id, host, port, jmx_port: nil)
-      @cluster = cluster
-      @id, @host, @port = id, host, port
-      @jmx_port = jmx_port
-    end
-
-    def led_partitions
-      result, threads, mutex = [], ThreadGroup.new, Mutex.new
-      cluster.partitions.each do |partition|
-        t = Thread.new do
-          select = partition.leader == self
-          mutex.synchronize { result << partition } if select
-        end
-        threads.add(t)
-      end
-      threads.list.each(&:join)
-      result
-    end
-
-    def replicated_partitions
-      result, threads, mutex = [], ThreadGroup.new, Mutex.new
-      cluster.partitions.each do |partition|
-        t = Thread.new do
-          select = partition.replicas.include?(self)
-          mutex.synchronize { result << partition } if select
-        end
-        threads.add(t)
-      end
-      threads.list.each(&:join)
-      result
-    end
-
-    def critical?(replicas: 1)
-      result, threads, mutex = false, ThreadGroup.new, Mutex.new
-      replicated_partitions.each do |partition|
-        t = Thread.new do
-          isr = partition.isr.reject { |r| r == self }
-          mutex.synchronize { result = true if isr.length < replicas }
-        end
-        threads.add(t)
-      end
-      threads.list.each(&:join)
-      result
-    end
-
-    def addr
-      "#{host}:#{port}"
-    end
-
-    def eql?(other)
-      other.is_a?(Kazoo::Broker) && other.cluster == self.cluster && other.id == self.id
-    end
-
-    alias_method :==, :eql?
-
-    def hash
-      [self.cluster, self.id].hash
-    end
-
-    def self.from_json(cluster, id, json)
-      new(cluster, id.to_i, json.fetch('host'), json.fetch('port'), jmx_port: json.fetch('jmx_port', nil))
-    end
-  end
-end

data/lib/kazoo/cli.rb

@@ -1,78 +0,0 @@
-require 'kazoo'
-require 'thor'
-
-module Kazoo
-  class CLI < Thor
-    class_option :zookeeper, :type => :string, :default => ENV['ZOOKEEPER']
-
-    desc "cluster", "Describes the Kafka cluster as registered in Zookeeper"
-    def cluster
-      validate_class_options!
-
-      kafka_cluster.brokers.values.sort_by(&:id).each do |broker|
-        $stdout.puts "#{broker.id}:\t#{broker.addr}\t(hosts #{broker.replicated_partitions.length} partitions, leads #{broker.led_partitions.length})"
-      end
-    end
-
-    desc "topics", "Lists all topics in the cluster"
-    def topics
-      validate_class_options!
-
-      kafka_cluster.topics.values.sort_by(&:name).each do |topic|
-        $stdout.puts topic.name
-      end
-    end
-
-    option :topic, :type => :string
-    desc "partitions", "Lists partitions"
-    def partitions
-      validate_class_options!
-
-      topics = kafka_cluster.topics.values
-      topics.select! { |t| t.name == options[:topic] } if options[:topic]
-      topics.sort_by!(&:name)
-
-      topics.each do |topic|
-        topic.partitions.each do |partition|
-          $stdout.puts "#{partition.topic.name}/#{partition.id}\tReplicas: #{partition.replicas.map(&:id).join(",")}"
-        end
-      end
-    end
-
-    option :replicas, :type => :numeric, :default => 1
-    desc "critical <broker>", "Determine whether a broker is critical"
-    def critical(broker_name)
-      validate_class_options!
-
-      if broker(broker_name).critical?(replicas: options[:replicas])
-        raise Thor::Error, "WARNING: broker #{broker_name} is critical and cannot be stopped safely!"
-      else
-        $stdout.puts "Broker #{broker_name} is non-critical and can be stopped safely."
-      end
-    end
-
-
-    private
-
-    def validate_class_options!
-      if options[:zookeeper].nil? || options[:zookeeper] == ''
-        raise Thor::InvocationError, "Please supply --zookeeper argument, or set the ZOOKEEPER_PEERS environment variable"
-      end
-    end
-
-    def broker(name_or_id)
-      broker = if name_or_id =~ /\A\d+\z/
-        kafka_cluster.brokers[name_or_id.to_i]
-      else
-        kafka_cluster.brokers.values.detect { |b| b.addr == name_or_id } || cluster.brokers.values.detect { |b| b.host == name_or_id }
-      end
-
-      raise Thor::InvocationError, "Broker #{name_or_id.inspect} not found!" if broker.nil?
-      broker
-    end
-
-    def kafka_cluster
-      @kafka_cluster ||= Kazoo::Cluster.new(options[:zookeeper])
-    end
-  end
-end

data/lib/kazoo/cluster.rb

@@ -1,78 +0,0 @@
-module Kazoo
-  class Cluster
-
-    attr_reader :zookeeper
-
-    def initialize(zookeeper)
-      @zookeeper = zookeeper
-      @zk_mutex, @brokers_mutex, @topics_mutex, @consumergroups_mutex = Mutex.new, Mutex.new, Mutex.new, Mutex.new
-    end
-
-    def zk
-      @zk_mutex.synchronize do
-        @zk ||= Zookeeper.new(zookeeper)
-      end
-    end
-
-    def brokers
-      @brokers_mutex.synchronize do
-        @brokers ||= begin
-          brokers = zk.get_children(path: "/brokers/ids")
-          result, threads, mutex = {}, ThreadGroup.new, Mutex.new
-          brokers.fetch(:children).map do |id|
-            t = Thread.new do
-              broker_info = zk.get(path: "/brokers/ids/#{id}")
-              broker = Kazoo::Broker.from_json(self, id, JSON.parse(broker_info.fetch(:data)))
-              mutex.synchronize { result[id.to_i] = broker }
-            end
-            threads.add(t)
-          end
-          threads.list.each(&:join)
-          result
-        end
-      end
-    end
-
-    def consumergroups
-      @consumergroups ||= begin
-        consumers = zk.get_children(path: "/consumers")
-        consumers.fetch(:children).map { |name| Kazoo::Consumergroup.new(self, name) }
-      end
-    end
-
-    def topics
-      @topics_mutex.synchronize do
-        @topics ||= begin
-          topics = zk.get_children(path: "/brokers/topics")
-          result, threads, mutex = {}, ThreadGroup.new, Mutex.new
-          topics.fetch(:children).each do |name|
-            t = Thread.new do
-              topic_info = zk.get(path: "/brokers/topics/#{name}")
-              topic = Kazoo::Topic.from_json(self, name, JSON.parse(topic_info.fetch(:data)))
-              mutex.synchronize { result[name] = topic }
-            end
-            threads.add(t)
-          end
-          threads.list.each(&:join)
-          result
-        end
-      end
-    end
-
-    def partitions
-      topics.values.flat_map(&:partitions)
-    end
-
-    def reset_metadata
-      @topics, @brokers = nil, nil
-    end
-
-    def under_replicated?
-      partitions.any?(&:under_replicated?)
-    end
-
-    def close
-      zk.close
-    end
-  end
-end

data/lib/kazoo/consumergroup.rb

@@ -1,215 +0,0 @@
-module Kazoo
-  class Consumergroup
-    attr_reader :cluster, :name
-
-    def initialize(cluster, name)
-      @cluster, @name = cluster, name
-    end
-
-    def create
-      cluster.zk.create(path: "/consumers/#{name}")
-      cluster.zk.create(path: "/consumers/#{name}/ids")
-      cluster.zk.create(path: "/consumers/#{name}/owners")
-      cluster.zk.create(path: "/consumers/#{name}/offsets")
-    end
-
-    def exists?
-      stat = cluster.zk.stat(path: "/consumers/#{name}")
-      stat.fetch(:stat).exists?
-    end
-
-
-    def instantiate(id: nil)
-      Instance.new(self, id: id)
-    end
-
-    def instances
-      instances = cluster.zk.get_children(path: "/consumers/#{name}/ids")
-      instances.fetch(:children).map { |id| Instance.new(self, id: id) }
-    end
-
-    def watch_instances(&block)
-      cb = Zookeeper::Callbacks::WatcherCallback.create(&block)
-      result = cluster.zk.get_children(path: "/consumers/#{name}/ids", watcher: cb)
-
-      if result.fetch(:rc) != Zookeeper::Constants::ZOK
-        raise Kazoo::Error, "Failed to watch instances. Error code result[:rc]"
-      end
-
-      instances = result.fetch(:children).map { |id| Instance.new(self, id: id) }
-      [instances, cb]
-    end
-
-
-    def watch_partition_claim(partition, &block)
-      cb = Zookeeper::Callbacks::WatcherCallback.create(&block)
-
-      result = cluster.zk.get(path: "/consumers/#{name}/owners/#{partition.topic.name}/#{partition.id}", watcher: cb)
-
-      case result.fetch(:rc)
-      when Zookeeper::Constants::ZNONODE # Nobody is claiming this partition yet
-        [nil, nil]
-      when Zookeeper::Constants::ZOK
-        [Kazoo::Consumergroup::Instance.new(self, id: result.fetch(:data)), cb]
-      else
-        raise Kazoo::Error, "Failed set watch for partition claim of #{partition.topic.name}/#{partition.id}. Error code: #{result.fetch(:rc)}"
-      end
-    end
-
-    def retrieve_offset(partition)
-      result = cluster.zk.get(path: "/consumers/#{name}/offsets/#{partition.topic.name}/#{partition.id}")
-      case result.fetch(:rc)
-        when Zookeeper::Constants::ZOK;
-          result.fetch(:data).to_i
-        when Zookeeper::Constants::ZNONODE;
-          nil
-        else
-          raise Kazoo::Error, "Failed to retrieve offset for partition #{partition.topic.name}/#{partition.id}. Error code: #{result.fetch(:rc)}"
-      end
-    end
-
-    def commit_offset(partition, offset)
-      result = cluster.zk.set(path: "/consumers/#{name}/offsets/#{partition.topic.name}/#{partition.id}", data: (offset + 1).to_s)
-      if result.fetch(:rc) == Zookeeper::Constants::ZNONODE
-        result = cluster.zk.create(path: "/consumers/#{name}/offsets/#{partition.topic.name}")
-        case result.fetch(:rc)
-          when Zookeeper::Constants::ZOK, Zookeeper::Constants::ZNODEEXISTS
-        else
-          raise Kazoo::Error, "Failed to commit offset #{offset} for partition #{partition.topic.name}/#{partition.id}. Error code: #{result.fetch(:rc)}"
-        end
-
-        result = cluster.zk.create(path: "/consumers/#{name}/offsets/#{partition.topic.name}/#{partition.id}", data: (offset + 1).to_s)
-      end
-
-      if result.fetch(:rc) != Zookeeper::Constants::ZOK
-        raise Kazoo::Error, "Failed to commit offset #{offset} for partition #{partition.topic.name}/#{partition.id}. Error code: #{result.fetch(:rc)}"
-      end
-    end
-
-    def reset_offsets
-      result = cluster.zk.get_children(path: "/consumers/#{name}/offsets")
-      raise Kazoo::Error unless result.fetch(:rc) == Zookeeper::Constants::ZOK
-
-      result.fetch(:children).each do |topic|
-        result = cluster.zk.get_children(path: "/consumers/#{name}/offsets/#{topic}")
-        raise Kazoo::Error unless result.fetch(:rc) == Zookeeper::Constants::ZOK
-
-        result.fetch(:children).each do |partition|
-          cluster.zk.delete(path: "/consumers/#{name}/offsets/#{topic}/#{partition}")
-          raise Kazoo::Error unless result.fetch(:rc) == Zookeeper::Constants::ZOK
-        end
-
-        cluster.zk.delete(path: "/consumers/#{name}/offsets/#{topic}")
-        raise Kazoo::Error unless result.fetch(:rc) == Zookeeper::Constants::ZOK
-      end
-    end
-
-    def inspect
-      "#<Kazoo::Consumergroup name=#{name}>"
-    end
-
-    def eql?(other)
-      other.kind_of?(Kazoo::Consumergroup) && cluster == other.cluster && name == other.name
-    end
-
-    alias_method :==, :eql?
-
-    def hash
-      [cluster, name].hash
-    end
-
-    class Instance
-
-      def self.generate_id
-        "#{Socket.gethostname}:#{SecureRandom.uuid}"
-      end
-
-      attr_reader :group, :id
-
-      def initialize(group, id: nil)
-        @group = group
-        @id = id || self.class.generate_id
-      end
-
-      def registered?
-        stat = cluster.zk.stat(path: "/consumers/#{group.name}/ids/#{id}")
-        stat.fetch(:stat).exists?
-      end
-
-      def register(subscription)
-        result = cluster.zk.create(
-          path: "/consumers/#{group.name}/ids/#{id}",
-          ephemeral: true,
-          data: JSON.generate({
-            version: 1,
-            timestamp: Time.now.to_i,
-            pattern: "static",
-            subscription: Hash[*subscription.flat_map { |topic| [topic.name, 1] } ]
-          })
-        )
-
-        if result.fetch(:rc) != Zookeeper::Constants::ZOK
-          raise Kazoo::ConsumerInstanceRegistrationFailed, "Failed to register instance #{id} for consumer group #{group.name}! Error code: #{result.fetch(:rc)}"
-        end
-
-        subscription.each do |topic|
-          stat = cluster.zk.stat(path: "/consumers/#{group.name}/owners/#{topic.name}")
-          unless stat.fetch(:stat).exists?
-            result = cluster.zk.create(path: "/consumers/#{group.name}/owners/#{topic.name}")
-            if result.fetch(:rc) != Zookeeper::Constants::ZOK
-              raise Kazoo::ConsumerInstanceRegistrationFailed, "Failed to register subscription of #{topic.name} for consumer group #{group.name}! Error code: #{result.fetch(:rc)}"
-            end
-          end
-        end
-      end
-
-      def deregister
-        cluster.zk.delete(path: "/consumers/#{group.name}/ids/#{id}")
-      end
-
-      def claim_partition(partition)
-        result = cluster.zk.create(
-          path: "/consumers/#{group.name}/owners/#{partition.topic.name}/#{partition.id}",
-          ephemeral: true,
-          data: id,
-        )
-
-        case result.fetch(:rc)
-        when Zookeeper::Constants::ZOK
-          return true
-        when Zookeeper::Constants::ZNODEEXISTS
-          raise Kazoo::PartitionAlreadyClaimed, "Partition #{partition.topic.name}/#{partition.id} is already claimed!"
-        else
-          raise Kazoo::Error, "Failed to claim partition #{partition.topic.name}/#{partition.id}. Error code: #{result.fetch(:rc)}"
-        end
-      end
-
-      def release_partition(partition)
-        result = cluster.zk.delete(path: "/consumers/#{group.name}/owners/#{partition.topic.name}/#{partition.id}")
-        if result.fetch(:rc) != Zookeeper::Constants::ZOK
-          raise Kazoo::Error, "Failed to release partition #{partition.topic.name}/#{partition.id}. Error code: #{result.fetch(:rc)}"
-        end
-      end
-
-      def inspect
-        "#<Kazoo::Consumergroup::Instance group=#{group.name} id=#{id}>"
-      end
-
-      def hash
-        [group, id].hash
-      end
-
-      def eql?(other)
-        other.kind_of?(Kazoo::Consumergroup::Instance) && group == other.group && id == other.id
-      end
-
-      alias_method :==, :eql?
-
-      private
-
-      def cluster
-        group.cluster
-      end
-    end
-  end
-end

data/lib/kazoo/partition.rb

@@ -1,62 +0,0 @@
-module Kazoo
-  class Partition
-    attr_reader :topic, :id, :replicas
-
-    def initialize(topic, id, replicas: nil)
-      @topic, @id, @replicas = topic, id, replicas
-      @mutex = Mutex.new
-    end
-
-    def cluster
-      topic.cluster
-    end
-
-    def replication_factor
-      replicas.length
-    end
-
-    def leader
-      @mutex.synchronize do
-        refresh_state if @leader.nil?
-        @leader
-      end
-    end
-
-    def isr
-      @mutex.synchronize do
-        refresh_state if @isr.nil?
-        @isr
-      end
-    end
-
-    def under_replicated?
-      isr.length < replication_factor
-    end
-
-    def inspect
-      "#<Kazoo::Partition #{topic.name}/#{id}>"
-    end
-
-    def eql?(other)
-      other.kind_of?(Kazoo::Partition) && topic == other.topic && id == other.id
-    end
-
-    alias_method :==, :eql?
-
-    def hash
-      [topic, id].hash
-    end
-
-    protected
-
-    def refresh_state
-      state_json = cluster.zk.get(path: "/brokers/topics/#{topic.name}/partitions/#{id}/state")
-      set_state(JSON.parse(state_json.fetch(:data)))
-    end
-
-    def set_state(json)
-      @leader = cluster.brokers.fetch(json.fetch('leader'))
-      @isr = json.fetch('isr').map { |r| cluster.brokers.fetch(r) }
-    end
-  end
-end

data/lib/kazoo/topic.rb

@@ -1,46 +0,0 @@
-module Kazoo
-  class Topic
-
-    attr_reader :cluster, :name
-    attr_accessor :partitions
-
-    def initialize(cluster, name, partitions: nil)
-      @cluster, @name, @partitions = cluster, name, partitions
-    end
-
-    def self.from_json(cluster, name, json)
-      topic = new(cluster, name)
-      topic.partitions = json.fetch('partitions').map do |(id, replicas)|
-        topic.partition(id.to_i, replicas: replicas.map { |b| cluster.brokers[b] })
-      end.sort_by(&:id)
-
-      return topic
-    end
-
-    def partition(*args)
-      Kazoo::Partition.new(self, *args)
-    end
-
-    def replication_factor
-      partitions.map(&:replication_factor).min
-    end
-
-    def under_replicated?
-      partitions.any?(:under_replicated?)
-    end
-
-    def inspect
-      "#<Kazoo::Topic #{name}>"
-    end
-
-    def eql?(other)
-      other.kind_of?(Kazoo::Topic) && cluster == other.cluster && name == other.name
-    end
-
-    alias_method :==, :eql?
-
-    def hash
-      [cluster, name].hash
-    end
-  end
-end

data/lib/kazoo/version.rb

@@ -1,3 +0,0 @@
-module Kazoo
-  VERSION = "0.1.1"
-end

data/test/broker_test.rb

@@ -1,45 +0,0 @@
-require 'test_helper'
-
-class BrokerTest < Minitest::Test
-  include MockCluster
-
-  def setup
-    @cluster = mock_cluster
-  end
-
-  def test_broker_critical?
-    refute @cluster.brokers[1].critical?(replicas: 1), "We have 2 in-sync replicas for everything so we can lose 1."
-    assert @cluster.brokers[2].critical?(replicas: 2), "We only have 2 replicas so we can never lose 2."
-
-    # Simulate losing a broker from the ISR for a partition.
-    # This partition lives on broker 1 and 3
-    @cluster.topics['test.4'].partitions[2].expects(:isr).returns([@cluster.brokers[1]])
-
-    assert @cluster.brokers[1].critical?(replicas: 1), "Final remaining broker for this partition's ISR set, cannot lose"
-    refute @cluster.brokers[2].critical?(replicas: 1), "Not related to the under-replicated partitions"
-    refute @cluster.brokers[3].critical?(replicas: 1), "Already down, so not critical"
-  end
-
-  def test_from_json
-    json_payload = '{"jmx_port":9999,"timestamp":"1431719964125","host":"kafka03.example.com","version":1,"port":9092}'
-    broker = Kazoo::Broker.from_json(mock('cluster'), 3, JSON.parse(json_payload))
-
-    assert_equal 3, broker.id
-    assert_equal 'kafka03.example.com', broker.host
-    assert_equal 9092, broker.port
-    assert_equal 9999, broker.jmx_port
-    assert_equal "kafka03.example.com:9092", broker.addr
-  end
-
-  def test_replicated_partitions
-    assert_equal 3, @cluster.brokers[1].replicated_partitions.length
-    assert_equal 4, @cluster.brokers[2].replicated_partitions.length
-    assert_equal 3, @cluster.brokers[3].replicated_partitions.length
-  end
-
-  def test_led_partitions
-    assert_equal 2, @cluster.brokers[1].led_partitions.length
-    assert_equal 2, @cluster.brokers[2].led_partitions.length
-    assert_equal 1, @cluster.brokers[3].led_partitions.length
-  end
-end

data/test/cluster_test.rb

@@ -1,16 +0,0 @@
-require 'test_helper'
-
-class ClusterTest < Minitest::Test
-  include MockCluster
-
-  def setup
-    @cluster = mock_cluster
-  end
-
-  def test_cluster_under_replicated?
-    refute @cluster.under_replicated?
-
-    @cluster.topics['test.4'].partitions[2].expects(:isr).returns([@cluster.brokers[1]])
-    assert @cluster.under_replicated?
-  end
-end

data/test/partition_test.rb

@@ -1,25 +0,0 @@
-require 'test_helper'
-
-class PartitionTest < Minitest::Test
-  include MockCluster
-
-  def setup
-    @cluster = mock_cluster
-  end
-
-  def test_replication_factor
-    assert_equal 2, @cluster.topics['test.1'].partitions[0].replication_factor
-  end
-
-  def test_state
-    partition = @cluster.topics['test.1'].partitions[0]
-    partition.unstub(:leader)
-    partition.unstub(:isr)
-
-    json_payload = '{"controller_epoch":157,"leader":1,"version":1,"leader_epoch":8,"isr":[3,2,1]}'
-    @cluster.zk.expects(:get).with(path: "/brokers/topics/test.1/partitions/0/state").returns(data: json_payload)
-
-    assert_equal 1, partition.leader.id
-    assert_equal [3,2,1], partition.isr.map(&:id)
-  end
-end

data/test/topic_test.rb

@@ -1,40 +0,0 @@
-require 'test_helper'
-
-class TopicTest < Minitest::Test
-  include MockCluster
-
-  def setup
-    @cluster = mock_cluster
-  end
-
-  def test_from_json
-    json_payload = '{"version":1,"partitions":{"2":[1,2,3],"1":[3,1,2],"3":[2,3,1],"0":[3,2,1]}}'
-    topic = Kazoo::Topic.from_json(@cluster, 'test.4', JSON.parse(json_payload))
-
-    assert_equal 4, topic.partitions.length
-    assert_equal [3,2,1], topic.partitions[0].replicas.map(&:id)
-    assert_equal [3,1,2], topic.partitions[1].replicas.map(&:id)
-    assert_equal [1,2,3], topic.partitions[2].replicas.map(&:id)
-    assert_equal [2,3,1], topic.partitions[3].replicas.map(&:id)
-  end
-
-  def test_replication_factor
-    json_payload = '{"version":1,"partitions":{"2":[1,2,3],"1":[3,1,2],"3":[2,3,1],"0":[3,2,1]}}'
-    topic = Kazoo::Topic.from_json(@cluster, 'test.4', JSON.parse(json_payload))
-    assert_equal 3, topic.replication_factor
-
-    json_payload = '{"version":1,"partitions":{"2":[2,3],"1":[2],"3":[2,3,1],"0":[3,2,1]}}'
-    topic = Kazoo::Topic.from_json(@cluster, 'test.4', JSON.parse(json_payload))
-    assert_equal 1, topic.replication_factor
-  end
-
-  def tets_topic_under_replicated?
-    refute @cluster.topics['test.1'].under_replicated?
-    refute @cluster.topics['test.1'].partitions[0].under_replicated?
-
-    @cluster.topics['test.1'].partitions[0].expects(:isr).returns([@cluster.brokers[1]])
-
-    assert @cluster.topics['test.1'].partitions[0].under_replicated?
-    assert @cluster.topics['test.1'].under_replicated?
-  end
-end