kazoo-ruby 0.3.3 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 44f657aaddbff498e1ff0adbbcd6ffec42055e49
-  data.tar.gz: 93b6ccb5d116395408e3296e0b481abd09cb3819
+  metadata.gz: 89757bee4b135ad94b20ef03dd75627928ac6430
+  data.tar.gz: babf18793c68d48949a54fadaa9c26a846cc90df
 SHA512:
-  metadata.gz: 837a4c49281b748f0e2bc5e58c1e5f1ccc06c45c5185f49c700294e8fcb3c538f9353a2f35bc7ad23e84d1a700e0a655129a5cbeb59a512942f19a1371505d13
-  data.tar.gz: dca5c6d3f0ff56a94f0d79d548658659032209e3ad23fa66fbef1f3aa87926cb53552fcc0f854cfb0df3aab3515aacb434a45919df0e66fe4f9af90b2cf9315f
+  metadata.gz: 0fe533d27ff5eceb5cde6b60cb45137382c2498cf1ed97d51e5e59ed8755ec033ccc421915cd4cb16bf9729652d2d2c39675af31eeb1a9a528d468f8c8489625
+  data.tar.gz: d7cf53f56a7c2765d3f85b890e3acab71d5969970fc26ada8247404101d350f51f64b89a16c1987ad3303a2390909f94b8bc38bbbd4a3c56fb0e7e857becc5f7

data/lib/kazoo.rb

@@ -3,6 +3,8 @@ require 'json'
 require 'thread'
 require 'socket'
 require 'securerandom'
+require 'bigdecimal'
+require 'time'
 
 module Kazoo
   Error = Class.new(StandardError)
@@ -14,6 +16,9 @@ module Kazoo
   ConsumerInstanceRegistrationFailed = Class.new(Kazoo::Error)
   PartitionAlreadyClaimed = Class.new(Kazoo::Error)
   ReleasePartitionFailure = Class.new(Kazoo::Error)
+  InvalidSubscription = Class.new(Kazoo::Error)
+  InconsistentSubscriptions = Class.new(Kazoo::Error)
+  NoRunningInstances = Class.new(Kazoo::Error)
 
   def self.connect(zookeeper)
     Kazoo::Cluster.new(zookeeper)
@@ -24,5 +29,7 @@ require 'kazoo/cluster'
 require 'kazoo/broker'
 require 'kazoo/topic'
 require 'kazoo/partition'
+require 'kazoo/replica_assigner'
+require 'kazoo/subscription'
 require 'kazoo/consumergroup'
 require 'kazoo/version'

data/lib/kazoo/broker.rb

@@ -1,4 +1,6 @@
 module Kazoo
+
+  # Kazoo::Broker represents a Kafka broker in a Kafka cluster.
   class Broker
     attr_reader :cluster, :id, :host, :port, :jmx_port
 
@@ -8,45 +10,55 @@ module Kazoo
       @jmx_port = jmx_port
     end
 
+    # Returns a list of all partitions that are currently led by this broker.
     def led_partitions
-      result, threads, mutex = [], ThreadGroup.new, Mutex.new
-      cluster.partitions.each do |partition|
-        t = Thread.new do
+      result, mutex = [], Mutex.new
+      threads = cluster.partitions.map do |partition|
+        Thread.new do
+          Thread.abort_on_exception = true
           select = partition.leader == self
           mutex.synchronize { result << partition } if select
         end
-        threads.add(t)
       end
-      threads.list.each(&:join)
+      threads.each(&:join)
       result
     end
 
+    # Returns a list of all partitions that host a replica on this broker.
     def replicated_partitions
-      result, threads, mutex = [], ThreadGroup.new, Mutex.new
-      cluster.partitions.each do |partition|
-        t = Thread.new do
+      result, mutex = [], Mutex.new
+      threads = cluster.partitions.map do |partition|
+        Thread.new do
+          Thread.abort_on_exception = true
           select = partition.replicas.include?(self)
           mutex.synchronize { result << partition } if select
         end
-        threads.add(t)
       end
-      threads.list.each(&:join)
+      threads.each(&:join)
       result
     end
 
+    # Returns whether this broker is currently considered critical.
+    #
+    # A broker is considered critical if it is the only in sync replica
+    # of any of the partitions it hosts. This means that if this broker
+    # were to go down, the partition woild become unavailable for writes,
+    # and may also lose data depending on the configuration and settings.
     def critical?(replicas: 1)
-      result, threads, mutex = false, ThreadGroup.new, Mutex.new
-      replicated_partitions.each do |partition|
-        t = Thread.new do
+      result, mutex = false, Mutex.new
+      threads = replicated_partitions.map do |partition|
+        Thread.new do
+          Thread.abort_on_exception = true
           isr = partition.isr.reject { |r| r == self }
-          mutex.synchronize { result = true if isr.length < replicas }
+          mutex.synchronize { result = true if isr.length < Integer(replicas) }
         end
-        threads.add(t)
       end
-      threads.list.each(&:join)
+      threads.each(&:join)
       result
     end
 
+    # Returns the address of this broker, i.e. the hostname plus the port
+    # to connect to.
     def addr
       "#{host}:#{port}"
     end
@@ -65,7 +77,10 @@ module Kazoo
       "#<Kazoo::Broker id=#{id} addr=#{addr}>"
     end
 
+    # Instantiates a Kazoo::Broker instance based on the Broker metadata that is stored
+    # in Zookeeper under `/brokers/<id>`.
     def self.from_json(cluster, id, json)
+      raise Kazoo::VersionNotSupported unless json.fetch('version') == 1
       new(cluster, id.to_i, json.fetch('host'), json.fetch('port'), jmx_port: json.fetch('jmx_port', nil))
     end
   end

data/lib/kazoo/cli/consumers.rb

@@ -24,7 +24,7 @@ module Kazoo
         cg = kafka_cluster.consumergroup(name)
         raise Kazoo::Error, "Consumergroup #{cg.name} is not registered in Zookeeper" unless cg.exists?
 
-        topics = cg.topics.sort_by(&:name)
+        topics = cg.subscribed_topics.sort_by(&:name)
 
         puts "Consumer name: #{cg.name}"
         puts "Created on: #{cg.created_at}"
@@ -89,6 +89,41 @@ module Kazoo
 
         cg.reset_all_offsets
       end
+
+      desc "clean-topic-claims [NAME]", "Removes all the topic claim Zookeeper nodes that are not needed for the consumer group's subscription"
+      def clean_topic_claims(name)
+        validate_class_options!
+
+        cg = kafka_cluster.consumergroup(name)
+        raise Kazoo::Error, "Consumergroup #{cg.name} is not registered in Zookeeper" unless cg.exists?
+        raise Kazoo::Error, "Cannot cleanup consumergroup #{cg.name} if it is not running" unless cg.active?
+
+        subscribed_topics = cg.subscribed_topics
+        claimed_topics    = cg.claimed_topics
+        to_clean          = claimed_topics - subscribed_topics
+
+        if to_clean.empty?
+          puts "The consumer group does not have any lingering topic claims."
+        else
+          puts "The following topics were once claimed, but are no longer part of #{cg.name}'s subscriptions:"
+          to_clean.each do |topic|
+            puts "- #{topic.name}"
+          end
+
+          cg.clean_topic_claims
+        end
+      end
+
+      desc "clean-stored-offsets [NAME]", "Removes all stored offsets for topics the consumer group is no longer subscribed to"
+      def clean_stored_offsets(name)
+        validate_class_options!
+
+        cg = kafka_cluster.consumergroup(name)
+        raise Kazoo::Error, "Consumergroup #{cg.name} is not registered in Zookeeper" unless cg.exists?
+        raise Kazoo::Error, "Cannot clean offsets for #{cg.name} if it is not running" unless cg.active?
+
+        cg.clean_stored_offsets
+      end
     end
   end
 end

data/lib/kazoo/cli/topics.rb

@@ -28,7 +28,6 @@ module Kazoo
         kafka_cluster.topics.fetch(name).destroy
       end
 
-      option :topic, type: :string
       desc "partitions TOPIC", "Lists partitions for a topic"
       def partitions(topic)
         validate_class_options!
@@ -38,6 +37,20 @@ module Kazoo
           puts "#{partition.key}\tReplicas: #{partition.replicas.map(&:id).join(",")}\tISR: #{partition.isr.map(&:id).join(",")}"
         end
       end
+
+      option :partitions, type: :numeric, required: true
+      option :replication_factor, type: :numeric, required: false
+      desc "set_partitions TOPIC", "Lists partitions for a topic"
+      def set_partitions(topic)
+        validate_class_options!
+
+        topic = kafka_cluster.topics.fetch(topic)
+        new_partitions = options[:partitions] - topic.partitions.length
+        raise "You can only add partitions to a topic, not remove them" if new_partitions <= 0
+
+        replication_factor = options[:replication_factor] || topic.replication_factor
+        topic.add_partitions(partitions: new_partitions, replication_factor: replication_factor)
+      end
     end
   end
 end

data/lib/kazoo/cluster.rb

@@ -1,6 +1,9 @@
 module Kazoo
-  class Cluster
 
+  # Kazoo::Cluster represents a full Kafka cluster, based on how it is registered in Zookeeper.
+  # It allows you the inspect the brokers of the cluster, the topics and partition metadata,
+  # and the consumergroups that are registered against the cluster.
+  class Cluster
     attr_reader :zookeeper
 
     def initialize(zookeeper)
@@ -8,12 +11,14 @@ module Kazoo
       @zk_mutex, @brokers_mutex, @topics_mutex = Mutex.new, Mutex.new, Mutex.new
     end
 
+    # Returns a zookeeper connection
     def zk
       @zk_mutex.synchronize do
         @zk ||= Zookeeper.new(zookeeper)
       end
     end
 
+    # Returns a hash of all the brokers in the
     def brokers
       @brokers_mutex.synchronize do
         @brokers ||= begin
@@ -23,23 +28,24 @@ module Kazoo
             raise NoClusterRegistered, "No Kafka cluster registered on this Zookeeper location."
           end
 
-          result, threads, mutex = {}, ThreadGroup.new, Mutex.new
-          brokers.fetch(:children).map do |id|
-            t = Thread.new do
+          result, mutex = {}, Mutex.new
+          threads = brokers.fetch(:children).map do |id|
+            Thread.new do
+              Thread.abort_on_exception = true
               broker_info = zk.get(path: "/brokers/ids/#{id}")
               raise Kazoo::Error, "Failed to retrieve broker info. Error code: #{broker_info.fetch(:rc)}" unless broker_info.fetch(:rc) == Zookeeper::Constants::ZOK
 
               broker = Kazoo::Broker.from_json(self, id, JSON.parse(broker_info.fetch(:data)))
               mutex.synchronize { result[id.to_i] = broker }
             end
-            threads.add(t)
           end
-          threads.list.each(&:join)
+          threads.each(&:join)
           result
         end
       end
     end
 
+    # Returns a list of consumer groups that are registered against the Kafka cluster.
     def consumergroups
       @consumergroups ||= begin
         consumers = zk.get_children(path: "/consumers")
@@ -47,62 +53,66 @@ module Kazoo
       end
     end
 
+    # Returns a Kazoo::Consumergroup instance for a given consumer name.
+    #
+    # Note that this doesn't register a new consumer group in Zookeeper; you wil have to call
+    # Kazoo::Consumergroup.create to do that.
     def consumergroup(name)
       Kazoo::Consumergroup.new(self, name)
     end
 
-    def topics
+    # Returns a hash of all the topics in the Kafka cluster, indexed by the topic name.
+    def topics(preload: Kazoo::Topic::DEFAULT_PRELOAD_METHODS)
       @topics_mutex.synchronize do
         @topics ||= begin
           topics = zk.get_children(path: "/brokers/topics")
           raise Kazoo::Error, "Failed to list topics. Error code: #{topics.fetch(:rc)}" unless topics.fetch(:rc) == Zookeeper::Constants::ZOK
-
-          result, threads, mutex = {}, ThreadGroup.new, Mutex.new
-          topics.fetch(:children).each do |name|
-            t = Thread.new do
-              topic_info = zk.get(path: "/brokers/topics/#{name}")
-              raise Kazoo::Error, "Failed to get topic info. Error code: #{topic_info.fetch(:rc)}" unless topic_info.fetch(:rc) == Zookeeper::Constants::ZOK
-
-              topic = Kazoo::Topic.from_json(self, name, JSON.parse(topic_info.fetch(:data)))
-              mutex.synchronize { result[name] = topic }
-            end
-            threads.add(t)
-          end
-          threads.list.each(&:join)
-          result
+          preload_topics_from_names(topics.fetch(:children), preload: preload)
         end
       end
     end
 
+    # Returns a Kazoo::Topic for a given topic name.
     def topic(name)
       Kazoo::Topic.new(self, name)
     end
 
-    def create_topic(name, partitions: nil, replication_factor: nil)
+    # Creates a topic on the Kafka cluster, with the provided number of partitions and
+    # replication factor.
+    def create_topic(name, partitions: nil, replication_factor: nil, config: nil)
       raise ArgumentError, "partitions must be a positive integer" if Integer(partitions) <= 0
       raise ArgumentError, "replication_factor must be a positive integer" if Integer(replication_factor) <= 0
 
-      Kazoo::Topic.create(self, name, partitions: Integer(partitions), replication_factor: Integer(replication_factor))
+      Kazoo::Topic.create(self, name, partitions: Integer(partitions), replication_factor: Integer(replication_factor), config: config)
     end
 
+    # Returns a list of all partitions hosted by the cluster
     def partitions
       topics.values.flat_map(&:partitions)
     end
 
+    # Resets the locally cached list of brokers and topics, which will mean they will be fetched
+    # freshly from Zookeeper the next time they are requested.
     def reset_metadata
-      @topics, @brokers = nil, nil
+      @topics, @brokers, @consumergroups = nil, nil, nil
     end
 
+    # Returns true if any of the partitions hosted by the cluster
     def under_replicated?
       partitions.any?(&:under_replicated?)
     end
 
+    # Closes the zookeeper connection and clears all the local caches.
     def close
       zk.close
+      @zk = nil
+      reset_metadata
     end
 
     protected
 
+    # Recursively creates a node in Zookeeper, by recusrively trying to create its
+    # parent if it doesn not yet exist.
     def recursive_create(path: nil)
       raise ArgumentError, "path is a required argument" if path.nil?
 
@@ -113,29 +123,51 @@ module Kazoo
       when Zookeeper::Constants::ZNONODE
         recursive_create(path: File.dirname(path))
         result = zk.create(path: path)
-        raise Kazoo::Error, "Failed to create node #{path}. Result code: #{result.fetch(:rc)}" unless result.fetch(:rc) == Zookeeper::Constants::ZOK
+
+        case result.fetch(:rc)
+        when Zookeeper::Constants::ZOK, Zookeeper::Constants::ZNODEEXISTS
+          return
+        else
+          raise Kazoo::Error, "Failed to create node #{path}. Result code: #{result.fetch(:rc)}"
+        end
       else
         raise Kazoo::Error, "Failed to create node #{path}. Result code: #{result.fetch(:rc)}"
       end
     end
 
+    # Deletes a node and all of its children from Zookeeper.
     def recursive_delete(path: nil)
       raise ArgumentError, "path is a required argument" if path.nil?
 
       result = zk.get_children(path: path)
       raise Kazoo::Error, "Failed to list children of #{path} to delete them. Result code: #{result.fetch(:rc)}" if result.fetch(:rc) != Zookeeper::Constants::ZOK
 
-      threads = []
-      result.fetch(:children).each do |name|
-        threads << Thread.new do
+      threads = result.fetch(:children).map do |name|
+        Thread.new do
           Thread.abort_on_exception = true
           recursive_delete(path: File.join(path, name))
         end
-        threads.each(&:join)
       end
+      threads.each(&:join)
 
       result = zk.delete(path: path)
       raise Kazoo::Error, "Failed to delete node #{path}. Result code: #{result.fetch(:rc)}" if result.fetch(:rc) != Zookeeper::Constants::ZOK
     end
+
+    private
+
+    def preload_topics_from_names(names, preload: Kazoo::Topic::DEFAULT_PRELOAD_METHODS)
+      result, mutex = {}, Mutex.new
+      threads = names.map do |name|
+        Thread.new do
+          Thread.abort_on_exception = true
+          topic = topic(name)
+          (preload & Kazoo::Topic::ALL_PRELOAD_METHODS).each { |method| topic.send(method) }
+          mutex.synchronize { result[name] = topic }
+        end
+      end
+      threads.each(&:join)
+      result
+    end
   end
 end

data/lib/kazoo/consumergroup.rb

@@ -9,10 +9,12 @@ module Kazoo
     def create
       cluster.send(:recursive_create, path: "/consumers/#{name}/ids")
       cluster.send(:recursive_create, path: "/consumers/#{name}/owners")
+      cluster.reset_metadata
     end
 
     def destroy
       cluster.send(:recursive_delete, path: "/consumers/#{name}")
+      cluster.reset_metadata
     end
 
     def exists?
@@ -27,9 +29,18 @@ module Kazoo
       Time.at(result.fetch(:stat).mtime / 1000.0)
     end
 
+    def instantiate(id: nil, subscription: nil)
+      Instance.new(self, id: id, subscription: subscription)
+    end
+
+    def subscription
+      subscriptions = instances.map(&:subscription).compact
+      raise NoRunningInstances, "Consumergroup #{name} has no running instances; cannot determine subscription" if subscriptions.length == 0
+
+      subscriptions.uniq!
+      raise InconsistentSubscriptions, "Subscriptions of running instances are different from each other" if subscriptions.length != 1
 
-    def instantiate(id: nil)
-      Instance.new(self, id: id)
+      subscriptions.first
     end
 
     def active?
@@ -40,7 +51,7 @@ module Kazoo
       result = cluster.zk.get_children(path: "/consumers/#{name}/ids")
       case result.fetch(:rc)
       when Zookeeper::Constants::ZOK
-        result.fetch(:children).map { |id| Instance.new(self, id: id) }
+        instances_with_subscription(result.fetch(:children))
       when Zookeeper::Constants::ZNONODE
         []
       else
@@ -51,16 +62,18 @@ module Kazoo
     def watch_instances(&block)
       cb = Zookeeper::Callbacks::WatcherCallback.create(&block)
       result = cluster.zk.get_children(path: "/consumers/#{name}/ids", watcher: cb)
-
-      if result.fetch(:rc) != Zookeeper::Constants::ZOK
-        raise Kazoo::Error, "Failed to watch instances. Error code: #{result.fetch(:rc)}"
+      instances = case result.fetch(:rc)
+      when Zookeeper::Constants::ZOK
+        instances_with_subscription(result.fetch(:children))
+      when Zookeeper::Constants::ZNONODE
+        []
+      else
+        raise Kazoo::Error, "Failed getting a list of runniong instances for #{name}. Error code: #{result.fetch(:rc)}"
       end
 
-      instances = result.fetch(:children).map { |id| Instance.new(self, id: id) }
       [instances, cb]
     end
 
-
     def watch_partition_claim(partition, &block)
       cb = Zookeeper::Callbacks::WatcherCallback.create(&block)
 
@@ -76,7 +89,7 @@ module Kazoo
       end
     end
 
-    def topics
+    def claimed_topics
       topic_result = cluster.zk.get_children(path: "/consumers/#{name}/owners")
       case topic_result.fetch(:rc)
       when Zookeeper::Constants::ZOK
@@ -88,6 +101,12 @@ module Kazoo
       end
     end
 
+    def subscribed_topics
+      subscription.topics(cluster)
+    end
+
+    alias_method :topics, :subscribed_topics
+
     def partitions
       partitions, threads, mutex = [], [], Mutex.new
       topics.each do |topic|
@@ -154,35 +173,72 @@ module Kazoo
       case topic_result.fetch(:rc)
         when Zookeeper::Constants::ZOK; # continue
         when Zookeeper::Constants::ZNONODE; return {}
-        else raise Kazoo::Error, "Failed to retrieve offset for partition #{partition.topic.name}/#{partition.id}. Error code: #{topic_result.fetch(:rc)}"
+        else raise Kazoo::Error, "Failed to get topic offsets. Result code: #{topic_result.fetch(:rc)}"
       end
 
-      offsets, threads, mutex = {}, [], Mutex.new
-      topic_result.fetch(:children).each do |topic_name|
-        threads << Thread.new do
+      offsets, mutex = {}, Mutex.new
+      topic_threads = topic_result.fetch(:children).map do |topic_name|
+        Thread.new do
           Thread.abort_on_exception = true
 
-          topic = Kazoo::Topic.new(cluster, topic_name)
+          topic = cluster.topic(topic_name)
           partition_result = cluster.zk.get_children(path: "/consumers/#{name}/offsets/#{topic.name}")
-          raise Kazoo::Error, "Failed to retrieve offsets. Error code: #{partition_result.fetch(:rc)}" if partition_result.fetch(:rc) != Zookeeper::Constants::ZOK
+          raise Kazoo::Error, "Failed to get partition offsets. Result code: #{partition_result.fetch(:rc)}" if partition_result.fetch(:rc) != Zookeeper::Constants::ZOK
 
-          partition_threads = []
-          partition_result.fetch(:children).each do |partition_id|
-            partition_threads << Thread.new do
+          partition_threads = partition_result.fetch(:children).map do |partition_id|
+            Thread.new do
               Thread.abort_on_exception = true
 
               partition = topic.partition(partition_id.to_i)
               offset_result = cluster.zk.get(path: "/consumers/#{name}/offsets/#{topic.name}/#{partition.id}")
-              raise Kazoo::Error, "Failed to retrieve offsets. Error code: #{offset_result.fetch(:rc)}" if offset_result.fetch(:rc) != Zookeeper::Constants::ZOK
+              offset = case offset_result.fetch(:rc)
+              when Zookeeper::Constants::ZOK
+                offset_result.fetch(:data).to_i
+              when Zookeeper::Constants::ZNONODE
+                nil
+              else
+                raise Kazoo::Error, "Failed to retrieve offset for #{partition.key}. Error code: #{offset_result.fetch(:rc)}"
+              end
+              mutex.synchronize { offsets[partition] = offset }
+            end
+          end
+          partition_threads.each(&:join)
+        end
+      end
 
-              mutex.synchronize { offsets[partition] = offset_result.fetch(:data).to_i }
+      topic_threads.each(&:join)
+      return offsets
+    end
+
+    def retrieve_offsets(subscription = self.subscription)
+      subscription = Kazoo::Subscription.build(subscription)
+
+      offsets, mutex = {}, Mutex.new
+      topic_threads = subscription.topics(cluster).map do |topic|
+        Thread.new do
+          Thread.abort_on_exception = true
+
+          partition_threads = topic.partitions.map do |partition|
+            Thread.new do
+              Thread.abort_on_exception = true
+
+              offset_result = cluster.zk.get(path: "/consumers/#{name}/offsets/#{topic.name}/#{partition.id}")
+              offset = case offset_result.fetch(:rc)
+              when Zookeeper::Constants::ZOK
+                offset_result.fetch(:data).to_i
+              when Zookeeper::Constants::ZNONODE
+                nil
+              else
+                raise Kazoo::Error, "Failed to retrieve offset for #{partition.key}. Error code: #{offset_result.fetch(:rc)}"
+              end
+              mutex.synchronize { offsets[partition] = offset }
             end
           end
           partition_threads.each(&:join)
         end
       end
 
-      threads.each(&:join)
+      topic_threads.each(&:join)
       return offsets
     end
 
@@ -192,8 +248,8 @@ module Kazoo
 
       result = cluster.zk.set(path: partition_offset_path, data: next_offset_data)
       if result.fetch(:rc) == Zookeeper::Constants::ZNONODE
-        cluster.send(:recursive_create, path: File.dirname(partition_offset_path))
-        result = cluster.zk.create(path: partition_offset_path, data: next_offset_data)
+        cluster.send(:recursive_create, path: partition_offset_path)
+        result = cluster.zk.set(path: partition_offset_path, data: next_offset_data)
       end
 
       if result.fetch(:rc) != Zookeeper::Constants::ZOK
@@ -205,6 +261,40 @@ module Kazoo
       cluster.send(:recursive_delete, path: "/consumers/#{name}/offsets")
     end
 
+    def clean_topic_claims(subscription = nil)
+      subscription = subscription.nil? ? self.subscription : Kazoo::Subscription.build(subscription)
+
+      threads = claimed_topics.map do |topic|
+        Thread.new do
+          Thread.abort_on_exception = true
+          unless subscription.topics(cluster).include?(topic)
+            cluster.send(:recursive_delete, path: "/consumers/#{name}/owners/#{topic.name}")
+          end
+        end
+      end
+
+      threads.each(&:join)
+    end
+
+    def clean_stored_offsets(subscription = nil)
+      subscription = subscription.nil? ? self.subscription : Kazoo::Subscription.build(subscription)
+
+      topics_result = cluster.zk.get_children(path: "/consumers/#{name}/offsets")
+      raise Kazoo::Error, "Failed to retrieve list of topics. Error code: #{topics_result.fetch(:rc)}" if topics_result.fetch(:rc) != Zookeeper::Constants::ZOK
+
+      threads = topics_result.fetch(:children).map do |topic_name|
+        Thread.new do
+          Thread.abort_on_exception = true
+          topic = cluster.topic(topic_name)
+          unless subscription.topics(cluster).include?(topic)
+            cluster.send(:recursive_delete, path: "/consumers/#{name}/offsets/#{topic.name}")
+          end
+        end
+      end
+
+      threads.each(&:join)
+    end
+
     def inspect
       "#<Kazoo::Consumergroup name=#{name}>"
     end
@@ -219,17 +309,36 @@ module Kazoo
       [cluster, name].hash
     end
 
+    protected
+
+    def instances_with_subscription(instance_ids)
+      instances, threads, mutex = [], [], Mutex.new
+      instance_ids.each do |id|
+        threads << Thread.new do
+          Thread.abort_on_exception = true
+
+          subscription_result = cluster.zk.get(path: "/consumers/#{name}/ids/#{id}")
+          raise Kazoo::Error, "Failed to retrieve subscription for instance. Error code: #{result.fetch(:rc)}" if subscription_result.fetch(:rc) != Zookeeper::Constants::ZOK
+          subscription = Kazoo::Subscription.from_json(subscription_result.fetch(:data))
+          mutex.synchronize { instances << Instance.new(self, id: id, subscription: subscription) }
+        end
+      end
+      threads.each(&:join)
+      instances
+    end
+
     class Instance
 
       def self.generate_id
         "#{Socket.gethostname}:#{SecureRandom.uuid}"
       end
 
-      attr_reader :group, :id
+      attr_reader :group, :id, :subscription
 
-      def initialize(group, id: nil)
+      def initialize(group, id: nil, subscription: nil)
         @group = group
         @id = id || self.class.generate_id
+        @subscription = Kazoo::Subscription.build(subscription) unless subscription.nil?
       end
 
       def registered?
@@ -237,23 +346,21 @@ module Kazoo
         stat.fetch(:stat).exists?
       end
 
-      def register(subscription)
+      def register(subscription_deprecated = nil)
+        # Don't provide the subscription here, but provide it when instantiating the consumer instance.
+        @subscription = Kazoo::Subscription.build(subscription_deprecated) unless subscription_deprecated.nil?
+
         result = cluster.zk.create(
           path: "/consumers/#{group.name}/ids/#{id}",
           ephemeral: true,
-          data: JSON.generate({
-            version: 1,
-            timestamp: Time.now.to_i,
-            pattern: "static",
-            subscription: Hash[*subscription.flat_map { |topic| [topic.name, 1] } ]
-          })
+          data: subscription.to_json,
         )
 
         if result.fetch(:rc) != Zookeeper::Constants::ZOK
           raise Kazoo::ConsumerInstanceRegistrationFailed, "Failed to register instance #{id} for consumer group #{group.name}! Error code: #{result.fetch(:rc)}"
         end
 
-        subscription.each do |topic|
+        subscription.topics(cluster).each do |topic|
           stat = cluster.zk.stat(path: "/consumers/#{group.name}/owners/#{topic.name}")
           unless stat.fetch(:stat).exists?
             result = cluster.zk.create(path: "/consumers/#{group.name}/owners/#{topic.name}")
@@ -262,6 +369,8 @@ module Kazoo
             end
           end
         end
+
+        return self
       end
 
       def created_at
@@ -273,7 +382,12 @@ module Kazoo
 
 
       def deregister
-        cluster.zk.delete(path: "/consumers/#{group.name}/ids/#{id}")
+        result = cluster.zk.delete(path: "/consumers/#{group.name}/ids/#{id}")
+        if result.fetch(:rc) != Zookeeper::Constants::ZOK
+          raise Kazoo::Error, "Failed to deregister instance #{id} for consumer group #{group.name}! Error code: #{result.fetch(:rc)}"
+        end
+
+        return self
       end
 
       def claim_partition(partition)