kazoo-ruby 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 87213b930132989503c56e226e6755b8c2f51559
-  data.tar.gz: 5c62c5a979ad421634272a4839b746777ed63cc6
+  metadata.gz: 129a9f807dc2c4a376d8941c64f25c652427fac1
+  data.tar.gz: 1f3ea8b05527feaef6b5b6fce8d7ff8eff19f987
 SHA512:
-  metadata.gz: d3a2b95c9e360fdf62c98a68ef752b220e35ddd4d9f8e05081a4eb95176a6e08d753125c47354a98ad171490458d3596199adf8eaea0edfa4768203c244302ab
-  data.tar.gz: e5af8de385e6946a53aa4a276c2708fb772d611cbf6afb99135c8af4c8f3a61969a52baf9c9fca8ea1f75b70c8c602689ec00024b163b748702b4e7411f47dbf
+  metadata.gz: eb6a831e906561296e5c8cb07116a470f4a612ad16b4d92d21d7b64bc3b58d01abf05f10486a4f8d2983b9afd4c1f340fc763bbdd3849dcd935f05675a3702d0
+  data.tar.gz: 560c455ec3ae69a0d2cfcb7f955fec90e4a26eaef79f636632fa40784745537368347ada6f028074a30585760a29bcaf14d3b80d1fce20503846d318820e4f6c

data/lib/kazoo/cli.rb

@@ -1,155 +1,20 @@
 require 'kazoo'
 require 'thor'
 
+require 'kazoo/cli/common'
+require 'kazoo/cli/cluster'
+require 'kazoo/cli/consumers'
+require 'kazoo/cli/topics'
+
 module Kazoo
   class CLI < Thor
-    class_option :zookeeper, type: :string, default: ENV['ZOOKEEPER']
-
-    desc "cluster", "Describes the Kafka cluster as registered in Zookeeper"
-    def cluster
-      validate_class_options!
-
-      kafka_cluster.brokers.values.sort_by(&:id).each do |broker|
-        $stdout.puts "#{broker.id}:\t#{broker.addr}\t(hosts #{broker.replicated_partitions.length} partitions, leads #{broker.led_partitions.length})"
-      end
-    end
-
-    desc "topics", "Lists all topics in the cluster"
-    def topics
-      validate_class_options!
-
-      kafka_cluster.topics.values.sort_by(&:name).each do |topic|
-        $stdout.puts topic.name
-      end
-    end
-
-    desc "create_topic", "Creates a new topic"
-    option :name, type: :string, required: true
-    option :partitions, type: :numeric, required: true
-    option :replication_factor, type: :numeric, required: true
-    def create_topic
-      validate_class_options!
-
-      kafka_cluster.create_topic(options[:name], partitions: options[:partitions], replication_factor: options[:replication_factor])
-    end
-
-    desc "delete_topic", "Removes a topic"
-    option :name, type: :string, required: true
-    def delete_topic
-      validate_class_options!
-
-      kafka_cluster.topics.fetch(options[:name]).destroy
-    end
-
-    option :topic, type: :string
-    desc "partitions", "Lists partitions"
-    def partitions
-      validate_class_options!
-
-      topics = kafka_cluster.topics.values
-      topics.select! { |t| t.name == options[:topic] } if options[:topic]
-      topics.sort_by!(&:name)
-
-      topics.each do |topic|
-        topic.partitions.each do |partition|
-          $stdout.puts "#{partition.topic.name}/#{partition.id}\tReplicas: #{partition.replicas.map(&:id).join(",")}"
-        end
-      end
-    end
-
-    option :replicas, type: :numeric, default: 1
-    desc "critical <broker>", "Determine whether a broker is critical"
-    def critical(broker_name)
-      validate_class_options!
-
-      if broker(broker_name).critical?(replicas: options[:replicas])
-        raise Thor::Error, "WARNING: broker #{broker_name} is critical and cannot be stopped safely!"
-      else
-        $stdout.puts "Broker #{broker_name} is non-critical and can be stopped safely."
-      end
-    end
-
-
-    desc "consumergroups", "Lists the consumergroups registered for this Kafka cluster"
-    def consumergroups
-      validate_class_options!
-
-      kafka_cluster.consumergroups.each do |group|
-        instances = group.instances
-        if instances.length == 0
-          puts "- #{group.name}: inactive"
-        else
-          puts "- #{group.name}: #{instances.length} running instances"
-        end
-      end
-    end
-
-    desc "consumergroup", "Prints information about a consumer group"
-    option :name, type: :string, required: true
-    def consumergroup
-      validate_class_options!
-
-      cg = kafka_cluster.consumergroup(options[:name])
-      raise Kazoo::Error, "Consumergroup #{options[:name]} is not registered in Zookeeper" unless cg.exists?
-
-      puts "Consumer group: #{cg.name}\n"
-
-      if cg.active?
-        puts "Running instances:"
-        cg.instances.each do |instance|
-          puts "- #{instance.id}"
-        end
-      else
-        puts "This consumer group is inactive."
-      end
-    end
-
-    desc "delete_consumergroup", "Removes a consumer group from Zookeeper"
-    option :name, type: :string, required: true
-    def delete_consumergroup
-      validate_class_options!
-
-      cg = kafka_cluster.consumergroup(options[:name])
-      raise Kazoo::Error, "Consumergroup #{options[:name]} is not registered in Zookeeper" unless cg.exists?
-      raise Kazoo::Error, "Cannot remove consumergroup #{cg.name} because it's still active" if cg.active?
-
-      cg.destroy
-    end
-
-    desc "reset_consumergroup", "Resets all the offsets stored for a consumergroup"
-    option :name, type: :string, required: true
-    def reset_consumergroup
-      validate_class_options!
-
-      cg = kafka_cluster.consumergroup(options[:name])
-      raise Kazoo::Error, "Consumergroup #{options[:name]} is not registered in Zookeeper" unless cg.exists?
-      raise Kazoo::Error, "Cannot remove consumergroup #{cg.name} because it's still active" if cg.active?
-
-      cg.reset_all_offsets
-    end
-
-
-    private
-
-    def validate_class_options!
-      if options[:zookeeper].nil? || options[:zookeeper] == ''
-        raise Thor::InvocationError, "Please supply --zookeeper argument, or set the ZOOKEEPER environment variable"
-      end
-    end
-
-    def broker(name_or_id)
-      broker = if name_or_id =~ /\A\d+\z/
-        kafka_cluster.brokers[name_or_id.to_i]
-      else
-        kafka_cluster.brokers.values.detect { |b| b.addr == name_or_id } || cluster.brokers.values.detect { |b| b.host == name_or_id }
-      end
+    desc "cluster SUBCOMMAND ...ARGS", "Inspect the Kafka cluster"
+    subcommand "cluster", Kazoo::CLI::Cluster
 
-      raise Thor::InvocationError, "Broker #{name_or_id.inspect} not found!" if broker.nil?
-      broker
-    end
+    desc "topics SUBCOMMAND ...ARGS", "Manage consumer groups"
+    subcommand "topics", Kazoo::CLI::Topics
 
-    def kafka_cluster
-      @kafka_cluster ||= Kazoo::Cluster.new(options[:zookeeper])
-    end
+    desc "consumers SUBCOMMAND ...ARGS", "Manage consumer groups"
+    subcommand "consumers", Kazoo::CLI::Consumers
   end
 end

data/lib/kazoo/cli/cluster.rb

@@ -0,0 +1,28 @@
+module Kazoo
+  class CLI < Thor
+    class Cluster < Thor
+      include Kazoo::CLI::Common
+
+      desc "brokers", "Describes the brokers in the cluster as registered in Zookeeper"
+      def brokers
+        validate_class_options!
+
+        kafka_cluster.brokers.values.sort_by(&:id).each do |broker|
+          $stdout.puts "#{broker.id}:\t#{broker.addr}\t(hosts #{broker.replicated_partitions.length} partitions, leads #{broker.led_partitions.length})"
+        end
+      end
+
+      option :replicas, type: :numeric, default: 1
+      desc "critical BROKER", "Determine whether a broker is critical"
+      def critical(broker_name)
+        validate_class_options!
+
+        if broker(broker_name).critical?(replicas: options[:replicas])
+          raise Thor::Error, "WARNING: broker #{broker_name} is critical and cannot be stopped safely!"
+        else
+          $stdout.puts "Broker #{broker_name} is non-critical and can be stopped safely."
+        end
+      end
+    end
+  end
+end

data/lib/kazoo/cli/common.rb

@@ -0,0 +1,32 @@
+module Kazoo
+  class CLI < Thor
+    module Common
+      def self.included(base)
+        base.class_option :zookeeper, type: :string, default: ENV['ZOOKEEPER']
+      end
+
+      private
+
+      def validate_class_options!
+        if options[:zookeeper].nil? || options[:zookeeper] == ''
+          raise Thor::InvocationError, "Please supply --zookeeper argument, or set the ZOOKEEPER environment variable"
+        end
+      end
+
+      def broker(name_or_id)
+        broker = if name_or_id =~ /\A\d+\z/
+          kafka_cluster.brokers[name_or_id.to_i]
+        else
+          kafka_cluster.brokers.values.detect { |b| b.addr == name_or_id } || kafka_cluster.brokers.values.detect { |b| b.host == name_or_id }
+        end
+
+        raise Thor::InvocationError, "Broker #{name_or_id.inspect} not found!" if broker.nil?
+        broker
+      end
+
+      def kafka_cluster
+        @kafka_cluster ||= Kazoo::Cluster.new(options[:zookeeper])
+      end
+    end
+  end
+end

data/lib/kazoo/cli/consumers.rb

@@ -0,0 +1,94 @@
+module Kazoo
+  class CLI < Thor
+    class Consumers < Thor
+      include Kazoo::CLI::Common
+
+      desc "list", "Lists the consumergroups registered for this Kafka cluster"
+      def list
+        validate_class_options!
+
+        kafka_cluster.consumergroups.sort_by(&:name).each do |group|
+          instances = group.instances
+          if instances.length == 0
+            puts "- #{group.name}: inactive"
+          else
+            puts "- #{group.name}: #{instances.length} running instances"
+          end
+        end
+      end
+
+      desc "show NAME", "Prints information about a consumer group"
+      def show(name)
+        validate_class_options!
+
+        cg = kafka_cluster.consumergroup(name)
+        raise Kazoo::Error, "Consumergroup #{cg.name} is not registered in Zookeeper" unless cg.exists?
+
+        topics = cg.topics.sort_by(&:name)
+
+        puts "Consumer name: #{cg.name}"
+        puts "Created on: #{cg.created_at}"
+        puts "Topics (#{topics.length}): #{topics.map(&:name).join(', ')}"
+
+        instances = cg.instances
+        if instances.length > 0
+
+          puts
+          puts "Running instances (#{instances.length}):"
+          instances.each do |instance|
+            puts "- #{instance.id}\t(created on #{instance.created_at})"
+          end
+
+          partition_claims = cg.partition_claims
+          if partition_claims.length > 0
+            partitions = partition_claims.keys.sort_by { |p| [p.topic.name, p.id] }
+
+            puts
+            puts "Partition claims (#{partition_claims.length}):"
+            partitions.each do |partition|
+              instance = partition_claims[partition]
+              puts "- #{partition.key}: #{instance.id}"
+            end
+          else
+            puts
+            puts "WARNING: this consumer group is active but hasn't claimed any partitions"
+          end
+
+          unclaimed_partitions = (cg.partitions - partition_claims.keys).sort_by { |p| [p.topic.name, p.id] }
+
+          if unclaimed_partitions.length > 0
+            puts
+            puts "WARNING: this consumergroup has #{unclaimed_partitions.length} unclaimed partitions:"
+            unclaimed_partitions.each do |partition|
+              puts "- #{partition.key}"
+            end
+          end
+        else
+          puts "This consumer group is inactive."
+        end
+      end
+
+      desc "delete NAME", "Removes a consumer group from Zookeeper"
+      def delete(name)
+        validate_class_options!
+
+        cg = kafka_cluster.consumergroup(name)
+        raise Kazoo::Error, "Consumergroup #{cg.name} is not registered in Zookeeper" unless cg.exists?
+        raise Kazoo::Error, "Cannot remove consumergroup #{cg.name} because it's still active" if cg.active?
+
+        cg.destroy
+      end
+
+      desc "reset [NAME]", "Resets all the offsets stored for a consumergroup"
+      def reset(name)
+        validate_class_options!
+
+        cg = kafka_cluster.consumergroup(name)
+        raise Kazoo::Error, "Consumergroup #{cg.name} is not registered in Zookeeper" unless cg.exists?
+        raise Kazoo::Error, "Cannot remove consumergroup #{cg.name} because it's still active" if cg.active?
+
+        cg.reset_all_offsets
+      end
+    end
+  end
+end

data/lib/kazoo/cli/topics.rb

@@ -0,0 +1,43 @@
+module Kazoo
+  class CLI < Thor
+    class Topics < Thor
+      include Kazoo::CLI::Common
+
+      desc "list", "Lists all topics in the cluster"
+      def list
+        validate_class_options!
+
+        kafka_cluster.topics.values.sort_by(&:name).each do |topic|
+          $stdout.puts topic.name
+        end
+      end
+
+      desc "create TOPIC", "Creates a new topic"
+      option :partitions, type: :numeric, required: true
+      option :replication_factor, type: :numeric, required: true
+      def create(name)
+        validate_class_options!
+
+        kafka_cluster.create_topic(name, partitions: options[:partitions], replication_factor: options[:replication_factor])
+      end
+
+      desc "delete TOPIC", "Removes a topic"
+      def delete(name)
+        validate_class_options!
+
+        kafka_cluster.topics.fetch(name).destroy
+      end
+
+      option :topic, type: :string
+      desc "partitions TOPIC", "Lists partitions for a topic"
+      def partitions(topic)
+        validate_class_options!
+
+        topic = kafka_cluster.topics.fetch(topic)
+        topic.partitions.each do |partition|
+          puts "#{partition.key}\tReplicas: #{partition.replicas.map(&:id).join(",")}\tISR: #{partition.isr.map(&:id).join(",")}"
+        end
+      end
+    end
+  end
+end

data/lib/kazoo/consumergroup.rb

@@ -20,6 +20,13 @@ module Kazoo
       stat.fetch(:stat).exists?
     end
 
+    def created_at
+      result = cluster.zk.stat(path: "/consumers/#{name}")
+      raise Kazoo::Error, "Failed to get consumer details. Error code: #{result.fetch(:rc)}" if result.fetch(:rc) != Zookeeper::Constants::ZOK
+
+      Time.at(result.fetch(:stat).mtime / 1000.0)
+    end
+
 
     def instantiate(id: nil)
       Instance.new(self, id: id)
@@ -69,31 +76,88 @@ module Kazoo
       end
     end
 
+    def topics
+      topic_result = cluster.zk.get_children(path: "/consumers/#{name}/owners")
+      case topic_result.fetch(:rc)
+      when Zookeeper::Constants::ZOK
+        topic_result.fetch(:children).map { |topic_name| cluster.topic(topic_name) }
+      when Zookeeper::Constants::ZNONODE
+        []
+      else
+        raise Kazoo::Error, "Failed to get subscribed topics. Result code: #{topic_result.fetch(:rc)}"
+      end
+    end
+
+    def partitions
+      partitions, threads, mutex = [], [], Mutex.new
+      topics.each do |topic|
+        threads << Thread.new do
+          topic_partitions = topic.partitions
+          mutex.synchronize { partitions.concat(topic_partitions) }
+        end
+      end
+
+      threads.each(&:join)
+      return partitions
+    end
+
+    def unclaimed_partitions
+      partitions - partition_claims.keys
+    end
+
+    def partition_claims
+      topic_result = cluster.zk.get_children(path: "/consumers/#{name}/owners")
+      case topic_result.fetch(:rc)
+        when Zookeeper::Constants::ZOK; # continue
+        when Zookeeper::Constants::ZNONODE; return {}
+        else raise Kazoo::Error, "Failed to get partition claims. Result code: #{topic_result.fetch(:rc)}"
+      end
+
+      partition_claims, threads, mutex = {}, [], Mutex.new
+      topic_result.fetch(:children).each do |topic_name|
+        threads << Thread.new do
+          topic = cluster.topic(topic_name)
+
+          partition_result = cluster.zk.get_children(path: "/consumers/#{name}/owners/#{topic.name}")
+          raise Kazoo::Error, "Failed to get partition claims. Result code: #{partition_result.fetch(:rc)}" if partition_result.fetch(:rc) != Zookeeper::Constants::ZOK
+
+          partition_threads = []
+          partition_result.fetch(:children).each do |partition_id|
+            partition_threads << Thread.new do
+              partition = topic.partition(partition_id.to_i)
+
+              claim_result =cluster.zk.get(path: "/consumers/#{name}/owners/#{topic.name}/#{partition.id}")
+              raise Kazoo::Error, "Failed to get partition claims. Result code: #{claim_result.fetch(:rc)}" if claim_result.fetch(:rc) != Zookeeper::Constants::ZOK
+
+              mutex.synchronize { partition_claims[partition] = instantiate(id: claim_result.fetch(:data)) }
+            end
+          end
+          partition_threads.each(&:join)
+        end
+      end
+
+      threads.each(&:join)
+      return partition_claims
+    end
+
     def retrieve_offset(partition)
       result = cluster.zk.get(path: "/consumers/#{name}/offsets/#{partition.topic.name}/#{partition.id}")
       case result.fetch(:rc)
-      when Zookeeper::Constants::ZOK;
-        result.fetch(:data).to_i
-      when Zookeeper::Constants::ZNONODE;
-        nil
-      else
-        raise Kazoo::Error, "Failed to retrieve offset for partition #{partition.topic.name}/#{partition.id}. Error code: #{result.fetch(:rc)}"
+        when Zookeeper::Constants::ZOK; result.fetch(:data).to_i
+        when Zookeeper::Constants::ZNONODE; nil
+        else raise Kazoo::Error, "Failed to retrieve offset for partition #{partition.topic.name}/#{partition.id}. Error code: #{result.fetch(:rc)}"
       end
     end
 
     def retrieve_all_offsets
-      offsets, threads, mutex = {}, [], Mutex.new
-
       topic_result = cluster.zk.get_children(path: "/consumers/#{name}/offsets")
       case topic_result.fetch(:rc)
-      when Zookeeper::Constants::ZOK;
-        # continue
-      when Zookeeper::Constants::ZNONODE;
-        return offsets
-      else
-        raise Kazoo::Error, "Failed to retrieve offset for partition #{partition.topic.name}/#{partition.id}. Error code: #{topic_result.fetch(:rc)}"
+        when Zookeeper::Constants::ZOK; # continue
+        when Zookeeper::Constants::ZNONODE; return {}
+        else raise Kazoo::Error, "Failed to retrieve offset for partition #{partition.topic.name}/#{partition.id}. Error code: #{topic_result.fetch(:rc)}"
       end
 
+      offsets, threads, mutex = {}, [], Mutex.new
       topic_result.fetch(:children).each do |topic_name|
         threads << Thread.new do
           Thread.abort_on_exception = true
@@ -200,6 +264,14 @@ module Kazoo
         end
       end
 
+      def created_at
+        result = cluster.zk.stat(path: "/consumers/#{group.name}/ids/#{id}")
+        raise Kazoo::Error, "Failed to get instance details. Error code: #{result.fetch(:rc)}" if result.fetch(:rc) != Zookeeper::Constants::ZOK
+
+        Time.at(result.fetch(:stat).mtime / 1000.0)
+      end
+
+
       def deregister
         cluster.zk.delete(path: "/consumers/#{group.name}/ids/#{id}")
       end

data/lib/kazoo/partition.rb

@@ -50,6 +50,10 @@ module Kazoo
       "#<Kazoo::Partition #{topic.name}/#{id}>"
     end
 
+    def key
+      "#{topic.name}/#{id}"
+    end
+
     def eql?(other)
       other.kind_of?(Kazoo::Partition) && topic == other.topic && id == other.id
     end

data/lib/kazoo/topic.rb

@@ -4,11 +4,10 @@ module Kazoo
     BLACKLISTED_TOPIC_NAMES = %r{\A\.\.?\z}
 
     attr_reader :cluster, :name
-    attr_accessor :partitions
+    attr_writer :partitions
 
     def initialize(cluster, name)
       @cluster, @name = cluster, name
-      @partitions = []
     end
 
     def self.from_json(cluster, name, json)
@@ -20,6 +19,18 @@ module Kazoo
       return topic
     end
 
+    def partitions
+      @partitions ||= begin
+        result = cluster.zk.get(path: "/brokers/topics/#{name}")
+        raise Kazoo::Error, "Failed to get list of partitions for #{name}. Result code: #{result.fetch(:rc)}" if result.fetch(:rc) != Zookeeper::Constants::ZOK
+
+        partition_json = JSON.parse(result.fetch(:data))
+        partition_json.fetch('partitions').map do |(id, replicas)|
+          partition(id.to_i, replicas: replicas.map { |b| cluster.brokers[b] })
+        end
+      end
+    end
+
     def partition(*args)
       Kazoo::Partition.new(self, *args)
     end

data/lib/kazoo/version.rb

@@ -1,3 +1,3 @@
 module Kazoo
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

data/test/functional/functional_consumergroup_test.rb

@@ -6,9 +6,13 @@ class FunctionalConsumergroupTest < Minitest::Test
     @cluster = Kazoo.connect(zookeeper)
     @cg = Kazoo::Consumergroup.new(@cluster, 'test.kazoo')
     @cg.create
+
+    @topic = @cluster.create_topic("test.4", partitions: 4, replication_factor: 1)
   end
 
   def teardown
+    @topic.destroy
+
     cg = Kazoo::Consumergroup.new(@cluster, 'test.kazoo')
     cg.destroy if cg.exists?
 
@@ -40,6 +44,31 @@ class FunctionalConsumergroupTest < Minitest::Test
     assert_nil @cg.retrieve_offset(partition)
   end
 
+  def test_unclaimed_partitions
+    partition40 = @topic.partition(0)
+    partition41 = @topic.partition(1)
+    partition42 = @topic.partition(2)
+    partition43 = @topic.partition(3)
+
+    refute @cg.active?
+
+    instance1 = @cg.instantiate
+    instance1.register([@topic])
+    instance2 = @cg.instantiate
+    instance2.register([@topic])
+
+    assert @cg.active?
+
+    instance1.claim_partition(partition40)
+    instance2.claim_partition(partition41)
+
+    assert_equal 2, @cg.partition_claims.length
+    assert_equal [@topic], @cg.topics
+    assert_equal @topic.partitions, @cg.partitions
+
+    assert_equal Set[partition42, partition43], Set.new(@cg.unclaimed_partitions)
+  end
+
   def test_retrieve_all_offsets
     topic1 = Kazoo::Topic.new(@cluster, 'test.1')
     partition10 = Kazoo::Partition.new(topic1, 0)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: kazoo-ruby
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Willem van Bergen
@@ -114,6 +114,10 @@ files:
 - lib/kazoo.rb
 - lib/kazoo/broker.rb
 - lib/kazoo/cli.rb
+- lib/kazoo/cli/cluster.rb
+- lib/kazoo/cli/common.rb
+- lib/kazoo/cli/consumers.rb
+- lib/kazoo/cli/topics.rb
 - lib/kazoo/cluster.rb
 - lib/kazoo/consumergroup.rb
 - lib/kazoo/partition.rb