kazoo-ruby 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8459c642f0d085f68811488f9d6ab92f186fd296
-  data.tar.gz: 2f187078b0e87d345e71277195750ebecbd3b5b4
+  metadata.gz: 87213b930132989503c56e226e6755b8c2f51559
+  data.tar.gz: 5c62c5a979ad421634272a4839b746777ed63cc6
 SHA512:
-  metadata.gz: e22aa111a85057bd295d80485371c46c84b1306d89160e04e2d2a21b985c407da14c59c4936d49239232f53e0309fb59fd9ca71f8affcc6cab8dd3f428ab55b7
-  data.tar.gz: 570508f6fe5f8d10bb8081b06c43f44b5f20da4e28d57801fcc5c6204f97a1f4557ac7234fd67bc72c7173111d145a42f6a5eabc5479267ccf9eb27095706fac
+  metadata.gz: d3a2b95c9e360fdf62c98a68ef752b220e35ddd4d9f8e05081a4eb95176a6e08d753125c47354a98ad171490458d3596199adf8eaea0edfa4768203c244302ab
+  data.tar.gz: e5af8de385e6946a53aa4a276c2708fb772d611cbf6afb99135c8af4c8f3a61969a52baf9c9fca8ea1f75b70c8c602689ec00024b163b748702b4e7411f47dbf

data/.gitignore

@@ -7,6 +7,7 @@
 /pkg/
 /spec/reports/
 /tmp/
+/confluent/
 *.bundle
 *.so
 *.o

data/.travis.yml

@@ -1,6 +1,14 @@
 language: ruby
 sudo: false
 
+env:
+  global:
+  - ZOOKEEPER_PEERS=localhost:2181
+  - DEBUG=true
+
+before_script:
+- make confluent/kafka/start
+
 rvm:
 - "2.0"
 - "2.1"

data/Makefile

@@ -0,0 +1,53 @@
+.PHONY: confluent/kafka/* confluent/zookeeper/* confluent/registry/* confluent/start confluent/stop
+
+
+# Confluent platform tasks
+
+confluent/start: confluent/rest/start
+
+confluent/stop: confluent/rest/stop confluent/registry/stop confluent/kafka/stop confluent/zookeeper/stop
+
+# Download & extract tasks
+
+confluent/confluent.tgz:
+	mkdir -p confluent && wget http://packages.confluent.io/archive/1.0/confluent-1.0-2.10.4.tar.gz -O confluent/confluent.tgz
+
+confluent/EXTRACTED: confluent/confluent.tgz
+	tar xzf confluent/confluent.tgz -C confluent --strip-components 1 && mkdir confluent/logs && touch confluent/EXTRACTED
+	echo "delete.topic.enable=true" >> confluent/etc/kafka/server.properties
+
+# Zookeeper tasks
+
+confluent/zookeeper/start: confluent/EXTRACTED
+	nohup confluent/bin/zookeeper-server-start confluent/etc/kafka/zookeeper.properties 2> confluent/logs/zookeeper.err > confluent/logs/zookeeper.out < /dev/null &
+	while ! nc localhost 2181 </dev/null; do echo "Waiting for zookeeper..."; sleep 1; done
+
+confluent/zookeeper/stop: confluent/EXTRACTED
+	confluent/bin/zookeeper-server-stop
+
+# Kafka tasks
+
+confluent/kafka/start: confluent/zookeeper/start confluent/EXTRACTED
+	nohup confluent/bin/kafka-server-start confluent/etc/kafka/server.properties 2> confluent/logs/kafka.err > confluent/logs/kafka.out < /dev/null &
+	while ! nc localhost 9092 </dev/null; do echo "Waiting for Kafka..."; sleep 1; done
+
+confluent/kafka/stop: confluent/EXTRACTED
+	confluent/bin/kafka-server-stop
+
+# schema-registry tasks
+
+confluent/registry/start: confluent/kafka/start confluent/EXTRACTED
+	nohup confluent/bin/schema-registry-start confluent/etc/schema-registry/schema-registry.properties 2> confluent/logs/schema-registry.err > confluent/logs/schema-registry.out < /dev/null &
+	while ! nc localhost 8081 </dev/null; do echo "Waiting for schema registry..."; sleep 1; done
+
+confluent/registry/stop: confluent/EXTRACTED
+	confluent/bin/kafka-server-stop
+
+# REST proxy tasks
+
+confluent/rest/start: confluent/registry/start confluent/EXTRACTED
+	nohup confluent/bin/kafka-rest-start confluent/etc/kafka-rest/kafka-rest.properties 2> confluent/logs/kafka-rest.err > confluent/logs/kafka-rest.out < /dev/null &
+	while ! nc localhost 8082 </dev/null; do echo "Waiting for REST proxy..."; sleep 1; done
+
+confluent/rest/stop: confluent/EXTRACTED
+	confluent/bin/kafka-rest-stop

data/lib/kazoo.rb

@@ -7,10 +7,15 @@ require 'securerandom'
 module Kazoo
   Error = Class.new(StandardError)
 
+  ValidationError = Class.new(Kazoo::Error)
   NoClusterRegistered = Class.new(Kazoo::Error)
   ConsumerInstanceRegistrationFailed = Class.new(Kazoo::Error)
   PartitionAlreadyClaimed = Class.new(Kazoo::Error)
   ReleasePartitionFailure = Class.new(Kazoo::Error)
+
+  def self.connect(zookeeper)
+    Kazoo::Cluster.new(zookeeper)
+  end
 end
 
 require 'kazoo/cluster'

data/lib/kazoo/cli.rb

@@ -3,7 +3,7 @@ require 'thor'
 
 module Kazoo
   class CLI < Thor
-    class_option :zookeeper, :type => :string, :default => ENV['ZOOKEEPER']
+    class_option :zookeeper, type: :string, default: ENV['ZOOKEEPER']
 
     desc "cluster", "Describes the Kafka cluster as registered in Zookeeper"
     def cluster
@@ -23,7 +23,25 @@ module Kazoo
       end
     end
 
-    option :topic, :type => :string
+    desc "create_topic", "Creates a new topic"
+    option :name, type: :string, required: true
+    option :partitions, type: :numeric, required: true
+    option :replication_factor, type: :numeric, required: true
+    def create_topic
+      validate_class_options!
+
+      kafka_cluster.create_topic(options[:name], partitions: options[:partitions], replication_factor: options[:replication_factor])
+    end
+
+    desc "delete_topic", "Removes a topic"
+    option :name, type: :string, required: true
+    def delete_topic
+      validate_class_options!
+
+      kafka_cluster.topics.fetch(options[:name]).destroy
+    end
+
+    option :topic, type: :string
     desc "partitions", "Lists partitions"
     def partitions
       validate_class_options!
@@ -39,7 +57,7 @@ module Kazoo
       end
     end
 
-    option :replicas, :type => :numeric, :default => 1
+    option :replicas, type: :numeric, default: 1
     desc "critical <broker>", "Determine whether a broker is critical"
     def critical(broker_name)
       validate_class_options!
@@ -52,6 +70,65 @@ module Kazoo
     end
 
 
+    desc "consumergroups", "Lists the consumergroups registered for this Kafka cluster"
+    def consumergroups
+      validate_class_options!
+
+      kafka_cluster.consumergroups.each do |group|
+        instances = group.instances
+        if instances.length == 0
+          puts "- #{group.name}: inactive"
+        else
+          puts "- #{group.name}: #{instances.length} running instances"
+        end
+      end
+    end
+
+    desc "consumergroup", "Prints information about a consumer group"
+    option :name, type: :string, required: true
+    def consumergroup
+      validate_class_options!
+
+      cg = kafka_cluster.consumergroup(options[:name])
+      raise Kazoo::Error, "Consumergroup #{options[:name]} is not registered in Zookeeper" unless cg.exists?
+
+      puts "Consumer group: #{cg.name}\n"
+
+      if cg.active?
+        puts "Running instances:"
+        cg.instances.each do |instance|
+          puts "- #{instance.id}"
+        end
+      else
+        puts "This consumer group is inactive."
+      end
+    end
+
+    desc "delete_consumergroup", "Removes a consumer group from Zookeeper"
+    option :name, type: :string, required: true
+    def delete_consumergroup
+      validate_class_options!
+
+      cg = kafka_cluster.consumergroup(options[:name])
+      raise Kazoo::Error, "Consumergroup #{options[:name]} is not registered in Zookeeper" unless cg.exists?
+      raise Kazoo::Error, "Cannot remove consumergroup #{cg.name} because it's still active" if cg.active?
+
+      cg.destroy
+    end
+
+    desc "reset_consumergroup", "Resets all the offsets stored for a consumergroup"
+    option :name, type: :string, required: true
+    def reset_consumergroup
+      validate_class_options!
+
+      cg = kafka_cluster.consumergroup(options[:name])
+      raise Kazoo::Error, "Consumergroup #{options[:name]} is not registered in Zookeeper" unless cg.exists?
+      raise Kazoo::Error, "Cannot remove consumergroup #{cg.name} because it's still active" if cg.active?
+
+      cg.reset_all_offsets
+    end
+
+
     private
 
     def validate_class_options!

data/lib/kazoo/cluster.rb

@@ -47,6 +47,10 @@ module Kazoo
       end
     end
 
+    def consumergroup(name)
+      Kazoo::Consumergroup.new(self, name)
+    end
+
     def topics
       @topics_mutex.synchronize do
         @topics ||= begin
@@ -70,6 +74,17 @@ module Kazoo
       end
     end
 
+    def topic(name)
+      Kazoo::Topic.new(self, name)
+    end
+
+    def create_topic(name, partitions: nil, replication_factor: nil)
+      raise ArgumentError, "partitions must be a positive integer" if Integer(partitions) <= 0
+      raise ArgumentError, "replication_factor must be a positive integer" if Integer(replication_factor) <= 0
+
+      Kazoo::Topic.create(self, name, partitions: Integer(partitions), replication_factor: Integer(replication_factor))
+    end
+
     def partitions
       topics.values.flat_map(&:partitions)
     end
@@ -85,5 +100,42 @@ module Kazoo
     def close
       zk.close
     end
+
+    protected
+
+    def recursive_create(path: nil)
+      raise ArgumentError, "path is a required argument" if path.nil?
+
+      result = zk.stat(path: path)
+      case result.fetch(:rc)
+      when Zookeeper::Constants::ZOK
+        return
+      when Zookeeper::Constants::ZNONODE
+        recursive_create(path: File.dirname(path))
+        result = zk.create(path: path)
+        raise Kazoo::Error, "Failed to create node #{path}. Result code: #{result.fetch(:rc)}" unless result.fetch(:rc) == Zookeeper::Constants::ZOK
+      else
+        raise Kazoo::Error, "Failed to create node #{path}. Result code: #{result.fetch(:rc)}"
+      end
+    end
+
+    def recursive_delete(path: nil)
+      raise ArgumentError, "path is a required argument" if path.nil?
+
+      result = zk.get_children(path: path)
+      raise Kazoo::Error, "Failed to list children of #{path} to delete them. Result code: #{result.fetch(:rc)}" if result.fetch(:rc) != Zookeeper::Constants::ZOK
+
+      threads = []
+      result.fetch(:children).each do |name|
+        threads << Thread.new do
+          Thread.abort_on_exception = true
+          recursive_delete(path: File.join(path, name))
+        end
+        threads.each(&:join)
+      end
+
+      result = zk.delete(path: path)
+      raise Kazoo::Error, "Failed to delete node #{path}. Result code: #{result.fetch(:rc)}" if result.fetch(:rc) != Zookeeper::Constants::ZOK
+    end
   end
 end

data/lib/kazoo/consumergroup.rb

@@ -7,10 +7,12 @@ module Kazoo
     end
 
     def create
-      cluster.zk.create(path: "/consumers/#{name}")
-      cluster.zk.create(path: "/consumers/#{name}/ids")
-      cluster.zk.create(path: "/consumers/#{name}/owners")
-      cluster.zk.create(path: "/consumers/#{name}/offsets")
+      cluster.send(:recursive_create, path: "/consumers/#{name}/ids")
+      cluster.send(:recursive_create, path: "/consumers/#{name}/owners")
+    end
+
+    def destroy
+      cluster.send(:recursive_delete, path: "/consumers/#{name}")
     end
 
     def exists?
@@ -23,9 +25,20 @@ module Kazoo
       Instance.new(self, id: id)
     end
 
+    def active?
+      instances.length > 0
+    end
+
     def instances
-      instances = cluster.zk.get_children(path: "/consumers/#{name}/ids")
-      instances.fetch(:children).map { |id| Instance.new(self, id: id) }
+      result = cluster.zk.get_children(path: "/consumers/#{name}/ids")
+      case result.fetch(:rc)
+      when Zookeeper::Constants::ZOK
+        result.fetch(:children).map { |id| Instance.new(self, id: id) }
+      when Zookeeper::Constants::ZNONODE
+        []
+      else
+        raise Kazoo::Error, "Failed getting a list of runniong instances for #{name}. Error code: #{result.fetch(:rc)}"
+      end
     end
 
     def watch_instances(&block)
@@ -52,58 +65,82 @@ module Kazoo
       when Zookeeper::Constants::ZOK
         [Kazoo::Consumergroup::Instance.new(self, id: result.fetch(:data)), cb]
       else
-        raise Kazoo::Error, "Failed set watch for partition claim of #{partition.topic.name}/#{partition.id}. Error code: #{result.fetch(:rc)}"
+        raise Kazoo::Error, "Failed to set watch for partition claim of #{partition.topic.name}/#{partition.id}. Error code: #{result.fetch(:rc)}"
       end
     end
 
     def retrieve_offset(partition)
       result = cluster.zk.get(path: "/consumers/#{name}/offsets/#{partition.topic.name}/#{partition.id}")
       case result.fetch(:rc)
-        when Zookeeper::Constants::ZOK;
-          result.fetch(:data).to_i
-        when Zookeeper::Constants::ZNONODE;
-          nil
-        else
-          raise Kazoo::Error, "Failed to retrieve offset for partition #{partition.topic.name}/#{partition.id}. Error code: #{result.fetch(:rc)}"
+      when Zookeeper::Constants::ZOK;
+        result.fetch(:data).to_i
+      when Zookeeper::Constants::ZNONODE;
+        nil
+      else
+        raise Kazoo::Error, "Failed to retrieve offset for partition #{partition.topic.name}/#{partition.id}. Error code: #{result.fetch(:rc)}"
       end
     end
 
-    def commit_offset(partition, offset)
-      result = cluster.zk.set(path: "/consumers/#{name}/offsets/#{partition.topic.name}/#{partition.id}", data: (offset + 1).to_s)
-      if result.fetch(:rc) == Zookeeper::Constants::ZNONODE
-        result = cluster.zk.create(path: "/consumers/#{name}/offsets/#{partition.topic.name}")
-        case result.fetch(:rc)
-          when Zookeeper::Constants::ZOK, Zookeeper::Constants::ZNODEEXISTS
-        else
-          raise Kazoo::Error, "Failed to commit offset #{offset} for partition #{partition.topic.name}/#{partition.id}. Error code: #{result.fetch(:rc)}"
-        end
+    def retrieve_all_offsets
+      offsets, threads, mutex = {}, [], Mutex.new
 
-        result = cluster.zk.create(path: "/consumers/#{name}/offsets/#{partition.topic.name}/#{partition.id}", data: (offset + 1).to_s)
+      topic_result = cluster.zk.get_children(path: "/consumers/#{name}/offsets")
+      case topic_result.fetch(:rc)
+      when Zookeeper::Constants::ZOK;
+        # continue
+      when Zookeeper::Constants::ZNONODE;
+        return offsets
+      else
+        raise Kazoo::Error, "Failed to retrieve offset for partition #{partition.topic.name}/#{partition.id}. Error code: #{topic_result.fetch(:rc)}"
       end
 
-      if result.fetch(:rc) != Zookeeper::Constants::ZOK
-        raise Kazoo::Error, "Failed to commit offset #{offset} for partition #{partition.topic.name}/#{partition.id}. Error code: #{result.fetch(:rc)}"
-      end
-    end
+      topic_result.fetch(:children).each do |topic_name|
+        threads << Thread.new do
+          Thread.abort_on_exception = true
+
+          topic = Kazoo::Topic.new(cluster, topic_name)
+          partition_result = cluster.zk.get_children(path: "/consumers/#{name}/offsets/#{topic.name}")
+          raise Kazoo::Error, "Failed to retrieve offsets. Error code: #{partition_result.fetch(:rc)}" if partition_result.fetch(:rc) != Zookeeper::Constants::ZOK
 
-    def reset_offsets
-      result = cluster.zk.get_children(path: "/consumers/#{name}/offsets")
-      raise Kazoo::Error unless result.fetch(:rc) == Zookeeper::Constants::ZOK
+          partition_threads = []
+          partition_result.fetch(:children).each do |partition_id|
+            partition_threads << Thread.new do
+              Thread.abort_on_exception = true
 
-      result.fetch(:children).each do |topic|
-        result = cluster.zk.get_children(path: "/consumers/#{name}/offsets/#{topic}")
-        raise Kazoo::Error unless result.fetch(:rc) == Zookeeper::Constants::ZOK
+              partition = topic.partition(partition_id.to_i)
+              offset_result = cluster.zk.get(path: "/consumers/#{name}/offsets/#{topic.name}/#{partition.id}")
+              raise Kazoo::Error, "Failed to retrieve offsets. Error code: #{offset_result.fetch(:rc)}" if offset_result.fetch(:rc) != Zookeeper::Constants::ZOK
 
-        result.fetch(:children).each do |partition|
-          cluster.zk.delete(path: "/consumers/#{name}/offsets/#{topic}/#{partition}")
-          raise Kazoo::Error unless result.fetch(:rc) == Zookeeper::Constants::ZOK
+              mutex.synchronize { offsets[partition] = offset_result.fetch(:data).to_i }
+            end
+          end
+          partition_threads.each(&:join)
         end
+      end
+
+      threads.each(&:join)
+      return offsets
+    end
 
-        cluster.zk.delete(path: "/consumers/#{name}/offsets/#{topic}")
-        raise Kazoo::Error unless result.fetch(:rc) == Zookeeper::Constants::ZOK
+    def commit_offset(partition, offset)
+      partition_offset_path = "/consumers/#{name}/offsets/#{partition.topic.name}/#{partition.id}"
+      next_offset_data = (offset + 1).to_s
+
+      result = cluster.zk.set(path: partition_offset_path, data: next_offset_data)
+      if result.fetch(:rc) == Zookeeper::Constants::ZNONODE
+        cluster.send(:recursive_create, path: File.dirname(partition_offset_path))
+        result = cluster.zk.create(path: partition_offset_path, data: next_offset_data)
+      end
+
+      if result.fetch(:rc) != Zookeeper::Constants::ZOK
+        raise Kazoo::Error, "Failed to commit offset #{offset} for partition #{partition.topic.name}/#{partition.id}. Error code: #{result.fetch(:rc)}"
       end
     end
 
+    def reset_all_offsets
+      cluster.send(:recursive_delete, path: "/consumers/#{name}/offsets")
+    end
+
     def inspect
       "#<Kazoo::Consumergroup name=#{name}>"
     end

data/lib/kazoo/partition.rb

@@ -33,6 +33,19 @@ module Kazoo
       isr.length < replication_factor
     end
 
+    def validate
+      raise Kazoo::ValidationError, "No replicas defined for #{topic.name}/#{id}" if replicas.length == 0
+      raise Kazoo::ValidationError, "The replicas of #{topic.name}/#{id} should be assigned to different brokers" if replicas.length > replicas.uniq.length
+
+      true
+    end
+
+    def valid?
+      validate
+    rescue Kazoo::ValidationError
+      false
+    end
+
     def inspect
       "#<Kazoo::Partition #{topic.name}/#{id}>"
     end
@@ -47,6 +60,19 @@ module Kazoo
       [topic, id].hash
     end
 
+    def wait_for_leader
+      current_leader = nil
+      while current_leader.nil?
+        current_leader = begin
+          leader
+        rescue Kazoo::Error
+          nil
+        end
+
+        sleep(0.1) if current_leader.nil?
+      end
+    end
+
     protected
 
     def refresh_state

data/lib/kazoo/topic.rb

@@ -1,11 +1,14 @@
 module Kazoo
   class Topic
+    VALID_TOPIC_NAMES = %r{\A[a-zA-Z0-9\\._\\-]+\z}
+    BLACKLISTED_TOPIC_NAMES = %r{\A\.\.?\z}
 
     attr_reader :cluster, :name
     attr_accessor :partitions
 
-    def initialize(cluster, name, partitions: nil)
-      @cluster, @name, @partitions = cluster, name, partitions
+    def initialize(cluster, name)
+      @cluster, @name = cluster, name
+      @partitions = []
     end
 
     def self.from_json(cluster, name, json)
@@ -42,5 +45,117 @@ module Kazoo
     def hash
       [cluster, name].hash
     end
+
+    def exists?
+      stat = cluster.zk.stat(path: "/brokers/topics/#{name}")
+      stat.fetch(:stat).exists?
+    end
+
+    def validate
+      raise Kazoo::ValidationError, "#{name} is not a valid topic name" if VALID_TOPIC_NAMES !~ name
+      raise Kazoo::ValidationError, "#{name} is not a valid topic name" if BLACKLISTED_TOPIC_NAMES =~ name
+      raise Kazoo::ValidationError, "#{name} is too long" if name.length > 255
+      raise Kazoo::ValidationError, "The topic has no partitions defined" if partitions.length == 0
+      partitions.each(&:validate)
+
+      true
+    end
+
+    def valid?
+      validate
+    rescue Kazoo::ValidationError
+      false
+    end
+
+    def create
+      raise Kazoo::Error, "The topic #{name} already exists!" if exists?
+      validate
+
+      result = cluster.zk.create(
+        path: "/brokers/topics/#{name}",
+        data: JSON.dump(version: 1, partitions: partitions_as_json)
+      )
+
+      if result.fetch(:rc) != Zookeeper::Constants::ZOK
+        raise Kazoo::Error, "Failed to create topic #{name}. Error code: #{result.fetch(:rc)}"
+      end
+
+      wait_for_partitions
+    end
+
+    def destroy
+      t = Thread.current
+      cb = Zookeeper::Callbacks::WatcherCallback.create do |event|
+        case event.type
+        when Zookeeper::Constants::ZOO_DELETED_EVENT
+          t.run if t.status == 'sleep'
+        else
+          raise Kazoo::Error, "Unexpected Zookeeper event: #{event.type}"
+        end
+      end
+
+      result = cluster.zk.stat(path: "/brokers/topics/#{name}", watcher: cb)
+      case result.fetch(:rc)
+       when Zookeeper::Constants::ZOK
+        # continue
+      when Zookeeper::Constants::ZNONODE
+        raise Kazoo::Error, "Topic #{name} does not exist!"
+      else
+        raise Kazoo::Error, "Failed to monitor topic"
+      end
+
+
+      result = cluster.zk.create(path: "/admin/delete_topics/#{name}")
+      case result.fetch(:rc)
+      when Zookeeper::Constants::ZOK
+        Thread.stop unless cb.completed?
+      when Zookeeper::Constants::ZNODEEXISTS
+        raise Kazoo::Error, "The topic #{name} is already marked for deletion!"
+      else
+        raise Kazoo::Error, "Failed to delete topic #{name}. Error code: #{result.fetch(:rc)}"
+      end
+    end
+
+    def self.create(cluster, name, partitions: nil, replication_factor: nil)
+      topic = new(cluster, name)
+      topic.send(:sequentially_assign_partitions, partitions, replication_factor)
+      topic.create
+      topic
+    end
+
+    protected
+
+    def wait_for_partitions
+      threads = []
+      partitions.each do |partition|
+        threads << Thread.new do
+          Thread.abort_on_exception = true
+          partition.wait_for_leader
+        end
+      end
+      threads.each(&:join)
+    end
+
+    def sequentially_assign_partitions(partition_count, replication_factor, brokers: nil)
+      brokers = cluster.brokers.values if brokers.nil?
+      raise ArgumentError, "replication_factor should be smaller or equal to the number of brokers" if replication_factor > brokers.length
+
+      # Sequentially assign replicas to brokers. There might be a better way.
+      @partitions = 0.upto(partition_count - 1).map do |partition_index|
+        replicas = 0.upto(replication_factor - 1).map do |replica_index|
+          broker_index = (partition_index + replica_index) % brokers.length
+          brokers[broker_index]
+        end
+
+        self.partition(partition_index, replicas: replicas)
+      end
+    end
+
+    def partitions_as_json
+      partitions.inject({}) do |hash, partition|
+        hash[partition.id] = partition.replicas.map(&:id)
+        hash
+      end
+    end
   end
 end

data/lib/kazoo/version.rb

@@ -1,3 +1,3 @@
 module Kazoo
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/test/functional/functional_consumergroup_test.rb

@@ -0,0 +1,93 @@
+require 'test_helper'
+
+class FunctionalConsumergroupTest < Minitest::Test
+  def setup
+    zookeeper = ENV["ZOOKEEPER_PEERS"] || "127.0.0.1:2181"
+    @cluster = Kazoo.connect(zookeeper)
+    @cg = Kazoo::Consumergroup.new(@cluster, 'test.kazoo')
+    @cg.create
+  end
+
+  def teardown
+    cg = Kazoo::Consumergroup.new(@cluster, 'test.kazoo')
+    cg.destroy if cg.exists?
+
+    @cluster.close
+  end
+
+  def test_create_and_destroy_consumergroup
+    cg = Kazoo::Consumergroup.new(@cluster, 'test.kazoo.2')
+    refute cg.exists?
+
+    cg.create
+    assert cg.exists?
+
+    cg.destroy
+    refute cg.exists?
+  end
+
+  def test_retrieve_and_commit_offsets
+    topic = Kazoo::Topic.new(@cluster, 'test.1')
+    partition = Kazoo::Partition.new(topic, 0)
+
+    assert_nil @cg.retrieve_offset(partition)
+
+    @cg.commit_offset(partition, 1234)
+
+    assert_equal 1234 + 1, @cg.retrieve_offset(partition)
+
+    @cg.reset_all_offsets
+    assert_nil @cg.retrieve_offset(partition)
+  end
+
+  def test_retrieve_all_offsets
+    topic1 = Kazoo::Topic.new(@cluster, 'test.1')
+    partition10 = Kazoo::Partition.new(topic1, 0)
+
+    topic4 = Kazoo::Topic.new(@cluster, 'test.4')
+    partition40 = Kazoo::Partition.new(topic4, 0)
+    partition41 = Kazoo::Partition.new(topic4, 1)
+    partition42 = Kazoo::Partition.new(topic4, 2)
+    partition43 = Kazoo::Partition.new(topic4, 3)
+
+    assert_equal Hash.new, @cg.retrieve_all_offsets
+
+    @cg.commit_offset(partition10, 10)
+    @cg.commit_offset(partition40, 40)
+    @cg.commit_offset(partition41, 41)
+    @cg.commit_offset(partition42, 42)
+    @cg.commit_offset(partition43, 43)
+
+    offsets = @cg.retrieve_all_offsets
+
+    assert_equal 5, offsets.length
+    assert_equal 11, offsets[partition10]
+    assert_equal 41, offsets[partition40]
+    assert_equal 42, offsets[partition41]
+    assert_equal 43, offsets[partition42]
+    assert_equal 44, offsets[partition43]
+
+    @cg.reset_all_offsets
+    assert_equal Hash.new, @cg.retrieve_all_offsets
+  end
+
+  def test_watch_instances
+    topic = Kazoo::Topic.new(@cluster, 'test.1')
+
+    instance1 = @cg.instantiate
+    instance1.register([topic])
+    instance2 = @cg.instantiate
+    instance2.register([topic])
+
+    t = Thread.current
+    instances, cb = @cg.watch_instances { t.run if t.status == 'sleep' }
+    assert_equal Set[instance1, instance2], Set.new(instances)
+
+    Thread.new { instance2.deregister }
+
+    Thread.stop unless cb.completed?
+
+    assert assert_equal Set[instance1], Set.new(@cg.instances)
+    instance1.deregister
+  end
+end

data/test/functional/functional_topic_management_test.rb

@@ -0,0 +1,22 @@
+require 'test_helper'
+
+class FunctionalTopicManagementTest < Minitest::Test
+  def setup
+    zookeeper = ENV["ZOOKEEPER_PEERS"] || "127.0.0.1:2181"
+    @cluster = Kazoo.connect(zookeeper)
+  end
+
+  def test_create_and_delete_topic
+    topic = @cluster.create_topic('test.kazoo', partitions: 8, replication_factor: 1)
+
+    assert @cluster.topics.key?(topic.name)
+    assert topic.partitions.all? { |partition| @cluster.brokers.values.include?(partition.leader) }
+    assert_equal 8, topic.partitions.length
+
+    topic.destroy
+    @cluster.reset_metadata
+
+    refute topic.exists?
+    refute @cluster.topics.key?(topic.name)
+  end
+end

data/test/partition_test.rb

@@ -37,4 +37,10 @@ class PartitionTest < Minitest::Test
     assert p1 != Kazoo::Partition.new(@cluster.topics['test.4'], 0)
     assert_equal p1.hash, p2.hash
   end
+
+  def test_validate
+    partition = Kazoo::Partition.new(@cluster.topics['test.1'], 1, replicas: [@cluster.brokers[1], @cluster.brokers[1]])
+    refute partition.valid?
+    assert_raises(Kazoo::ValidationError) { partition.validate }
+  end
 end

data/test/topic_test.rb

@@ -50,4 +50,45 @@ class TopicTest < Minitest::Test
     assert t1 != Kazoo::Topic.new(@cluster, 'test.2')
     assert_equal t1.hash, t2.hash
   end
+
+  def test_validate
+    t = Kazoo::Topic.new(@cluster, "normal")
+    t.partitions = [t.partition(0, replicas: [@cluster.brokers[1]])]
+    assert t.valid?
+
+    t = Kazoo::Topic.new(@cluster, "invalid/character")
+    t.partitions = [t.partition(0, replicas: [@cluster.brokers[1]])]
+    refute t.valid?
+
+    t = Kazoo::Topic.new(@cluster, "..")
+    t.partitions = [t.partition(0, replicas: [@cluster.brokers[1]])]
+    refute t.valid?
+
+    t = Kazoo::Topic.new(@cluster, "l#{'o' * 253}ng")
+    t.partitions = [t.partition(0, replicas: [@cluster.brokers[1]])]
+    refute t.valid?
+
+    t = Kazoo::Topic.new(@cluster, "normal")
+    t.partitions = [t.partition(0, replicas: [])]
+    refute t.valid?
+  end
+
+  def test_sequentially_assign_partitions
+    topic = Kazoo::Topic.new(@cluster, 'test.new')
+
+    assert_raises(ArgumentError) { topic.send(:sequentially_assign_partitions, 4, 100) }
+
+    topic.send(:sequentially_assign_partitions, 4, 3)
+
+    assert_equal 4, topic.partitions.length
+    assert_equal 3, topic.replication_factor
+    assert topic.partitions.all? { |p| p.replicas.length == 3 }
+    assert topic.valid?
+  end
+
+  def test_partitions_as_json
+    assignment = @cluster.topics['test.1'].send(:partitions_as_json)
+    assert_equal 1, assignment.length
+    assert_equal [1,2], assignment[0]
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kazoo-ruby
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Willem van Bergen
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-13 00:00:00.000000000 Z
+date: 2015-08-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -106,6 +106,7 @@ files:
 - .travis.yml
 - Gemfile
 - LICENSE.txt
+- Makefile
 - README.md
 - Rakefile
 - bin/kazoo
@@ -120,6 +121,8 @@ files:
 - lib/kazoo/version.rb
 - test/broker_test.rb
 - test/cluster_test.rb
+- test/functional/functional_consumergroup_test.rb
+- test/functional/functional_topic_management_test.rb
 - test/partition_test.rb
 - test/test_helper.rb
 - test/topic_test.rb
@@ -150,6 +153,8 @@ summary: Library to access and manipulate Kafka metadata in Zookeeper
 test_files:
 - test/broker_test.rb
 - test/cluster_test.rb
+- test/functional/functional_consumergroup_test.rb
+- test/functional/functional_topic_management_test.rb
 - test/partition_test.rb
 - test/test_helper.rb
 - test/topic_test.rb