kazoo-ruby 0.0.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ec91cf4af491cd9c00ad11b459c64b92884d9289
-  data.tar.gz: 585ba40ab883b2bc70014ae38ece4fb7f4240bcf
+  metadata.gz: 8459c642f0d085f68811488f9d6ab92f186fd296
+  data.tar.gz: 2f187078b0e87d345e71277195750ebecbd3b5b4
 SHA512:
-  metadata.gz: 3f7021a9e7f42e3315d9c7386db8113619c555881535c1ddc44e2c4aec07983acd02d3f49442b3654090376f2d33f3b95a463cc375c5b2e1eab9a9778a054928
-  data.tar.gz: 10bc6041afe7942eeaaa214ef9609ff391e1b8eef6daa723b626f6994bf3cb9dd875c5bf5bf3e68a80c66c7e489c8999ec293c03b5f2f7279cc9935280a0f3c8
+  metadata.gz: e22aa111a85057bd295d80485371c46c84b1306d89160e04e2d2a21b985c407da14c59c4936d49239232f53e0309fb59fd9ca71f8affcc6cab8dd3f428ab55b7
+  data.tar.gz: 570508f6fe5f8d10bb8081b06c43f44b5f20da4e28d57801fcc5c6204f97a1f4557ac7234fd67bc72c7173111d145a42f6a5eabc5479267ccf9eb27095706fac

data/README.md

@@ -1,31 +1,60 @@
 # Kazoo
 
-TODO: Write a gem description
+Ruby library to access and manipulate Kafka metadata in Zookeeper
 
-## Installation
+## Usage
 
-Add this line to your application's Gemfile:
+First, make kazoo available by adding it to your Gemfile
 
-```ruby
-gem 'kazoo'
+``` ruby
+gem 'kazoo-ruby', require: 'kazoo'
 ```
 
-And then execute:
+Now, you can interact with the cluster metadata as follows:
 
-    $ bundle
+``` ruby
+# Connect to the Zookeeper cluster that backs your Kafka cluster
+cluster = Kazoo::Cluster.new('zookeeper1:2181,zookeeper2:2181/chroot')
 
-Or install it yourself as:
+# List the brokers that form the Kafka cluster
+cluster.brokers.each do |id, broker|
+  puts "Broker #{broker.id}: #{broker.addr}"
+end
 
-    $ gem install kazoo
+# Inspect topic/partition metadata
+cluster.topics.each do |name, topic|
+  puts "#{name}: #{topic.partitions.length} partitions"
+end
 
-## Usage
+# List consumers
+cluster.consumergroups.each do |name, group|
+  puts "Consumer #{name}: #{group.instances.length} running instances"
+end
+```
 
-TODO: Write usage instructions here
+## Binary
+
+This gem also comes with a simple `kazoo` binary to inspect your kafka cluster:
+
+```
+# Describe the brokers that compose the cluster
+$ kazoo cluster --zookeeper zk1:2181,zk2:2181,zk3:2181/chroot
+
+# List all topics or partitions in the cluster
+$ kazoo topics --zookeeper zk1:2181,zk2:2181,zk3:2181/chroot
+$ kazoo partitions --topic=access_log --zookeeper zk1:2181,zk2:2181,zk3:2181/chroot
+
+```
 
 ## Contributing
 
-1. Fork it ( https://github.com/[my-github-username]/kazoo/fork )
+1. Fork it ( https://github.com/wvanbergen/kazoo/fork )
 2. Create your feature branch (`git checkout -b my-new-feature`)
 3. Commit your changes (`git commit -am 'Add some feature'`)
 4. Push to the branch (`git push origin my-new-feature`)
 5. Create a new Pull Request
+
+## See also
+
+- [kafka-consumer](https://github.com/wvanbergen/kafka-consumer): a high-level
+  Kafka consumer library that coordinates running instances using Zookeeper.

data/kazoo-ruby.gemspec

@@ -8,8 +8,8 @@ Gem::Specification.new do |spec|
   spec.version       = Kazoo::VERSION
   spec.authors       = ["Willem van Bergen"]
   spec.email         = ["willem@vanbergen.org"]
-  spec.summary       = %q{Library to access Kafka metadata in Zookeeper}
-  spec.description   = %q{Library to access Kafka metadata in Zookeeper}
+  spec.summary       = %q{Library to access and manipulate Kafka metadata in Zookeeper}
+  spec.description   = %q{Library to access and manipulate Kafka metadata in Zookeeper}
   spec.homepage      = "https://github.com/wvanbergen/kazoo"
   spec.license       = "MIT"
 

data/lib/kazoo.rb

@@ -1,12 +1,21 @@
 require 'zookeeper'
 require 'json'
 require 'thread'
+require 'socket'
+require 'securerandom'
 
 module Kazoo
+  Error = Class.new(StandardError)
+
+  NoClusterRegistered = Class.new(Kazoo::Error)
+  ConsumerInstanceRegistrationFailed = Class.new(Kazoo::Error)
+  PartitionAlreadyClaimed = Class.new(Kazoo::Error)
+  ReleasePartitionFailure = Class.new(Kazoo::Error)
 end
 
 require 'kazoo/cluster'
 require 'kazoo/broker'
 require 'kazoo/topic'
 require 'kazoo/partition'
+require 'kazoo/consumergroup'
 require 'kazoo/version'

data/lib/kazoo/broker.rb

@@ -61,6 +61,10 @@ module Kazoo
       [self.cluster, self.id].hash
     end
 
+    def inspect
+      "#<Kazoo::Broker id=#{id} addr=#{addr}>"
+    end
+
     def self.from_json(cluster, id, json)
       new(cluster, id.to_i, json.fetch('host'), json.fetch('port'), jmx_port: json.fetch('jmx_port', nil))
     end

data/lib/kazoo/cli.rb

@@ -3,8 +3,7 @@ require 'thor'
 
 module Kazoo
   class CLI < Thor
-    class_option :zookeeper, :type => :string, :default => ENV['ZOOKEEPER_PEERS']
-    class_option :chroot, :type => :string, :default => ""
+    class_option :zookeeper, :type => :string, :default => ENV['ZOOKEEPER']
 
     desc "cluster", "Describes the Kafka cluster as registered in Zookeeper"
     def cluster
@@ -57,7 +56,7 @@ module Kazoo
 
     def validate_class_options!
       if options[:zookeeper].nil? || options[:zookeeper] == ''
-        raise Thor::InvocationError, "Please supply --zookeeper argument, or set the ZOOKEEPER_PEERS environment variable"
+        raise Thor::InvocationError, "Please supply --zookeeper argument, or set the ZOOKEEPER environment variable"
       end
     end
 
@@ -73,7 +72,7 @@ module Kazoo
     end
 
     def kafka_cluster
-      @kafka_cluster ||= Kazoo::Cluster.new(options[:zookeeper], chroot: options[:chroot])
+      @kafka_cluster ||= Kazoo::Cluster.new(options[:zookeeper])
     end
   end
 end

data/lib/kazoo/cluster.rb

@@ -1,10 +1,10 @@
 module Kazoo
   class Cluster
 
-    attr_reader :zookeeper, :chroot
+    attr_reader :zookeeper
 
-    def initialize(zookeeper, chroot: "")
-      @zookeeper, @chroot = zookeeper, chroot
+    def initialize(zookeeper)
+      @zookeeper = zookeeper
       @zk_mutex, @brokers_mutex, @topics_mutex = Mutex.new, Mutex.new, Mutex.new
     end
 
@@ -17,11 +17,18 @@ module Kazoo
     def brokers
       @brokers_mutex.synchronize do
         @brokers ||= begin
-          brokers = zk.get_children(path: node_with_chroot("/brokers/ids"))
+          brokers = zk.get_children(path: "/brokers/ids")
+
+          if brokers.fetch(:rc) != Zookeeper::Constants::ZOK
+            raise NoClusterRegistered, "No Kafka cluster registered on this Zookeeper location."
+          end
+
           result, threads, mutex = {}, ThreadGroup.new, Mutex.new
           brokers.fetch(:children).map do |id|
             t = Thread.new do
-              broker_info = zk.get(path: node_with_chroot("/brokers/ids/#{id}"))
+              broker_info = zk.get(path: "/brokers/ids/#{id}")
+              raise Kazoo::Error, "Failed to retrieve broker info. Error code: #{broker_info.fetch(:rc)}" unless broker_info.fetch(:rc) == Zookeeper::Constants::ZOK
+
               broker = Kazoo::Broker.from_json(self, id, JSON.parse(broker_info.fetch(:data)))
               mutex.synchronize { result[id.to_i] = broker }
             end
@@ -33,14 +40,25 @@ module Kazoo
       end
     end
 
+    def consumergroups
+      @consumergroups ||= begin
+        consumers = zk.get_children(path: "/consumers")
+        consumers.fetch(:children).map { |name| Kazoo::Consumergroup.new(self, name) }
+      end
+    end
+
     def topics
       @topics_mutex.synchronize do
         @topics ||= begin
-          topics = zk.get_children(path: node_with_chroot("/brokers/topics"))
+          topics = zk.get_children(path: "/brokers/topics")
+          raise Kazoo::Error, "Failed to list topics. Error code: #{topics.fetch(:rc)}" unless topics.fetch(:rc) == Zookeeper::Constants::ZOK
+
           result, threads, mutex = {}, ThreadGroup.new, Mutex.new
           topics.fetch(:children).each do |name|
             t = Thread.new do
-              topic_info = zk.get(path: node_with_chroot("/brokers/topics/#{name}"))
+              topic_info = zk.get(path: "/brokers/topics/#{name}")
+              raise Kazoo::Error, "Failed to get topic info. Error code: #{topic_info.fetch(:rc)}" unless topic_info.fetch(:rc) == Zookeeper::Constants::ZOK
+
               topic = Kazoo::Topic.from_json(self, name, JSON.parse(topic_info.fetch(:data)))
               mutex.synchronize { result[name] = topic }
             end
@@ -64,8 +82,8 @@ module Kazoo
       partitions.any?(&:under_replicated?)
     end
 
-    def node_with_chroot(path)
-      "#{@chroot}#{path}"
+    def close
+      zk.close
     end
   end
 end

data/lib/kazoo/consumergroup.rb

@@ -0,0 +1,215 @@
+module Kazoo
+  class Consumergroup
+    attr_reader :cluster, :name
+
+    def initialize(cluster, name)
+      @cluster, @name = cluster, name
+    end
+
+    def create
+      cluster.zk.create(path: "/consumers/#{name}")
+      cluster.zk.create(path: "/consumers/#{name}/ids")
+      cluster.zk.create(path: "/consumers/#{name}/owners")
+      cluster.zk.create(path: "/consumers/#{name}/offsets")
+    end
+
+    def exists?
+      stat = cluster.zk.stat(path: "/consumers/#{name}")
+      stat.fetch(:stat).exists?
+    end
+
+
+    def instantiate(id: nil)
+      Instance.new(self, id: id)
+    end
+
+    def instances
+      instances = cluster.zk.get_children(path: "/consumers/#{name}/ids")
+      instances.fetch(:children).map { |id| Instance.new(self, id: id) }
+    end
+
+    def watch_instances(&block)
+      cb = Zookeeper::Callbacks::WatcherCallback.create(&block)
+      result = cluster.zk.get_children(path: "/consumers/#{name}/ids", watcher: cb)
+
+      if result.fetch(:rc) != Zookeeper::Constants::ZOK
+        raise Kazoo::Error, "Failed to watch instances. Error code: #{result.fetch(:rc)}"
+      end
+
+      instances = result.fetch(:children).map { |id| Instance.new(self, id: id) }
+      [instances, cb]
+    end
+
+
+    def watch_partition_claim(partition, &block)
+      cb = Zookeeper::Callbacks::WatcherCallback.create(&block)
+
+      result = cluster.zk.get(path: "/consumers/#{name}/owners/#{partition.topic.name}/#{partition.id}", watcher: cb)
+
+      case result.fetch(:rc)
+      when Zookeeper::Constants::ZNONODE # Nobody is claiming this partition yet
+        [nil, nil]
+      when Zookeeper::Constants::ZOK
+        [Kazoo::Consumergroup::Instance.new(self, id: result.fetch(:data)), cb]
+      else
+        raise Kazoo::Error, "Failed set watch for partition claim of #{partition.topic.name}/#{partition.id}. Error code: #{result.fetch(:rc)}"
+      end
+    end
+
+    def retrieve_offset(partition)
+      result = cluster.zk.get(path: "/consumers/#{name}/offsets/#{partition.topic.name}/#{partition.id}")
+      case result.fetch(:rc)
+        when Zookeeper::Constants::ZOK;
+          result.fetch(:data).to_i
+        when Zookeeper::Constants::ZNONODE;
+          nil
+        else
+          raise Kazoo::Error, "Failed to retrieve offset for partition #{partition.topic.name}/#{partition.id}. Error code: #{result.fetch(:rc)}"
+      end
+    end
+
+    def commit_offset(partition, offset)
+      result = cluster.zk.set(path: "/consumers/#{name}/offsets/#{partition.topic.name}/#{partition.id}", data: (offset + 1).to_s)
+      if result.fetch(:rc) == Zookeeper::Constants::ZNONODE
+        result = cluster.zk.create(path: "/consumers/#{name}/offsets/#{partition.topic.name}")
+        case result.fetch(:rc)
+          when Zookeeper::Constants::ZOK, Zookeeper::Constants::ZNODEEXISTS
+        else
+          raise Kazoo::Error, "Failed to commit offset #{offset} for partition #{partition.topic.name}/#{partition.id}. Error code: #{result.fetch(:rc)}"
+        end
+
+        result = cluster.zk.create(path: "/consumers/#{name}/offsets/#{partition.topic.name}/#{partition.id}", data: (offset + 1).to_s)
+      end
+
+      if result.fetch(:rc) != Zookeeper::Constants::ZOK
+        raise Kazoo::Error, "Failed to commit offset #{offset} for partition #{partition.topic.name}/#{partition.id}. Error code: #{result.fetch(:rc)}"
+      end
+    end
+
+    def reset_offsets
+      result = cluster.zk.get_children(path: "/consumers/#{name}/offsets")
+      raise Kazoo::Error unless result.fetch(:rc) == Zookeeper::Constants::ZOK
+
+      result.fetch(:children).each do |topic|
+        result = cluster.zk.get_children(path: "/consumers/#{name}/offsets/#{topic}")
+        raise Kazoo::Error unless result.fetch(:rc) == Zookeeper::Constants::ZOK
+
+        result.fetch(:children).each do |partition|
+          cluster.zk.delete(path: "/consumers/#{name}/offsets/#{topic}/#{partition}")
+          raise Kazoo::Error unless result.fetch(:rc) == Zookeeper::Constants::ZOK
+        end
+
+        cluster.zk.delete(path: "/consumers/#{name}/offsets/#{topic}")
+        raise Kazoo::Error unless result.fetch(:rc) == Zookeeper::Constants::ZOK
+      end
+    end
+
+    def inspect
+      "#<Kazoo::Consumergroup name=#{name}>"
+    end
+
+    def eql?(other)
+      other.kind_of?(Kazoo::Consumergroup) && cluster == other.cluster && name == other.name
+    end
+
+    alias_method :==, :eql?
+
+    def hash
+      [cluster, name].hash
+    end
+
+    class Instance
+
+      def self.generate_id
+        "#{Socket.gethostname}:#{SecureRandom.uuid}"
+      end
+
+      attr_reader :group, :id
+
+      def initialize(group, id: nil)
+        @group = group
+        @id = id || self.class.generate_id
+      end
+
+      def registered?
+        stat = cluster.zk.stat(path: "/consumers/#{group.name}/ids/#{id}")
+        stat.fetch(:stat).exists?
+      end
+
+      def register(subscription)
+        result = cluster.zk.create(
+          path: "/consumers/#{group.name}/ids/#{id}",
+          ephemeral: true,
+          data: JSON.generate({
+            version: 1,
+            timestamp: Time.now.to_i,
+            pattern: "static",
+            subscription: Hash[*subscription.flat_map { |topic| [topic.name, 1] } ]
+          })
+        )
+
+        if result.fetch(:rc) != Zookeeper::Constants::ZOK
+          raise Kazoo::ConsumerInstanceRegistrationFailed, "Failed to register instance #{id} for consumer group #{group.name}! Error code: #{result.fetch(:rc)}"
+        end
+
+        subscription.each do |topic|
+          stat = cluster.zk.stat(path: "/consumers/#{group.name}/owners/#{topic.name}")
+          unless stat.fetch(:stat).exists?
+            result = cluster.zk.create(path: "/consumers/#{group.name}/owners/#{topic.name}")
+            if result.fetch(:rc) != Zookeeper::Constants::ZOK
+              raise Kazoo::ConsumerInstanceRegistrationFailed, "Failed to register subscription of #{topic.name} for consumer group #{group.name}! Error code: #{result.fetch(:rc)}"
+            end
+          end
+        end
+      end
+
+      def deregister
+        cluster.zk.delete(path: "/consumers/#{group.name}/ids/#{id}")
+      end
+
+      def claim_partition(partition)
+        result = cluster.zk.create(
+          path: "/consumers/#{group.name}/owners/#{partition.topic.name}/#{partition.id}",
+          ephemeral: true,
+          data: id,
+        )
+
+        case result.fetch(:rc)
+        when Zookeeper::Constants::ZOK
+          return true
+        when Zookeeper::Constants::ZNODEEXISTS
+          raise Kazoo::PartitionAlreadyClaimed, "Partition #{partition.topic.name}/#{partition.id} is already claimed!"
+        else
+          raise Kazoo::Error, "Failed to claim partition #{partition.topic.name}/#{partition.id}. Error code: #{result.fetch(:rc)}"
+        end
+      end
+
+      def release_partition(partition)
+        result = cluster.zk.delete(path: "/consumers/#{group.name}/owners/#{partition.topic.name}/#{partition.id}")
+        if result.fetch(:rc) != Zookeeper::Constants::ZOK
+          raise Kazoo::Error, "Failed to release partition #{partition.topic.name}/#{partition.id}. Error code: #{result.fetch(:rc)}"
+        end
+      end
+
+      def inspect
+        "#<Kazoo::Consumergroup::Instance group=#{group.name} id=#{id}>"
+      end
+
+      def hash
+        [group, id].hash
+      end
+
+      def eql?(other)
+        other.kind_of?(Kazoo::Consumergroup::Instance) && group == other.group && id == other.id
+      end
+
+      alias_method :==, :eql?
+
+      private
+
+      def cluster
+        group.cluster
+      end
+    end
+  end
+end

data/lib/kazoo/partition.rb

@@ -33,10 +33,26 @@ module Kazoo
       isr.length < replication_factor
     end
 
+    def inspect
+      "#<Kazoo::Partition #{topic.name}/#{id}>"
+    end
+
+    def eql?(other)
+      other.kind_of?(Kazoo::Partition) && topic == other.topic && id == other.id
+    end
+
+    alias_method :==, :eql?
+
+    def hash
+      [topic, id].hash
+    end
+
     protected
 
     def refresh_state
-      state_json = cluster.zk.get(path: cluster.node_with_chroot("/brokers/topics/#{topic.name}/partitions/#{id}/state"))
+      state_json = cluster.zk.get(path: "/brokers/topics/#{topic.name}/partitions/#{id}/state")
+      raise Kazoo::Error, "Failed to get partition state. Error code: #{state_json.fetch(:rc)}" unless state_json.fetch(:rc) == Zookeeper::Constants::ZOK
+
       set_state(JSON.parse(state_json.fetch(:data)))
     end
 

data/lib/kazoo/topic.rb

@@ -28,5 +28,19 @@ module Kazoo
     def under_replicated?
       partitions.any?(:under_replicated?)
     end
+
+    def inspect
+      "#<Kazoo::Topic #{name}>"
+    end
+
+    def eql?(other)
+      other.kind_of?(Kazoo::Topic) && cluster == other.cluster && name == other.name
+    end
+
+    alias_method :==, :eql?
+
+    def hash
+      [cluster, name].hash
+    end
   end
 end

data/lib/kazoo/version.rb

@@ -1,3 +1,3 @@
 module Kazoo
-  VERSION = "0.0.1"
+  VERSION = "0.1.0"
 end

data/test/broker_test.rb

@@ -42,4 +42,16 @@ class BrokerTest < Minitest::Test
     assert_equal 2, @cluster.brokers[2].led_partitions.length
     assert_equal 1, @cluster.brokers[3].led_partitions.length
   end
+
+  def test_inspect
+    assert_equal "#<Kazoo::Broker id=1 addr=example.com:9091>", @cluster.brokers[1].inspect
+  end
+
+  def test_equality
+    b1 = @cluster.brokers[1]
+    b2 = Kazoo::Broker.new(@cluster, 1, 'example.com', 9091)
+
+    assert_equal b1, b2
+    assert_equal b1.hash, b2.hash
+  end
 end

data/test/partition_test.rb

@@ -17,9 +17,24 @@ class PartitionTest < Minitest::Test
     partition.unstub(:isr)
 
     json_payload = '{"controller_epoch":157,"leader":1,"version":1,"leader_epoch":8,"isr":[3,2,1]}'
-    @cluster.zk.expects(:get).with(path: "/brokers/topics/test.1/partitions/0/state").returns(data: json_payload)
+    @cluster.zk.expects(:get).with(path: "/brokers/topics/test.1/partitions/0/state").returns(data: json_payload, rc: 0)
 
     assert_equal 1, partition.leader.id
     assert_equal [3,2,1], partition.isr.map(&:id)
   end
+
+  def test_inspect
+    assert_equal "#<Kazoo::Partition test.1/0>", @cluster.topics['test.1'].partition(0).inspect
+  end
+
+  def test_equality
+    topic = @cluster.topics['test.1']
+    p1 = topic.partition(0)
+    p2 = Kazoo::Partition.new(topic, 0)
+
+    assert_equal p1, p2
+    assert p1 != Kazoo::Partition.new(topic, 1)
+    assert p1 != Kazoo::Partition.new(@cluster.topics['test.4'], 0)
+    assert_equal p1.hash, p2.hash
+  end
 end

data/test/topic_test.rb

@@ -37,4 +37,17 @@ class TopicTest < Minitest::Test
     assert @cluster.topics['test.1'].partitions[0].under_replicated?
     assert @cluster.topics['test.1'].under_replicated?
   end
+
+  def test_inspect
+    assert_equal "#<Kazoo::Topic test.1>", @cluster.topics['test.1'].inspect
+  end
+
+  def test_equality
+    t1 = @cluster.topics['test.1']
+    t2 = Kazoo::Topic.new(@cluster, 'test.1')
+
+    assert_equal t1, t2
+    assert t1 != Kazoo::Topic.new(@cluster, 'test.2')
+    assert_equal t1.hash, t2.hash
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kazoo-ruby
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0
 platform: ruby
 authors:
 - Willem van Bergen
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-12 00:00:00.000000000 Z
+date: 2015-08-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -94,7 +94,7 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '5'
-description: Library to access Kafka metadata in Zookeeper
+description: Library to access and manipulate Kafka metadata in Zookeeper
 email:
 - willem@vanbergen.org
 executables:
@@ -114,6 +114,7 @@ files:
 - lib/kazoo/broker.rb
 - lib/kazoo/cli.rb
 - lib/kazoo/cluster.rb
+- lib/kazoo/consumergroup.rb
 - lib/kazoo/partition.rb
 - lib/kazoo/topic.rb
 - lib/kazoo/version.rb
@@ -145,7 +146,7 @@ rubyforge_project:
 rubygems_version: 2.0.14
 signing_key: 
 specification_version: 4
-summary: Library to access Kafka metadata in Zookeeper
+summary: Library to access and manipulate Kafka metadata in Zookeeper
 test_files:
 - test/broker_test.rb
 - test/cluster_test.rb