kafka-consumer 0.0.1 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ad31d35f34d283a73bc8865fc90e1748ecc48ffd
-  data.tar.gz: 58fe93f719be96c3276604973b12887f42eb00f5
+  metadata.gz: 7e0b4f66c8e873f254c213a4d12ec89a2a46134a
+  data.tar.gz: 0dcc6b6cdce939337a51acebc95b9c7d8ac6b757
 SHA512:
-  metadata.gz: 1ac246b560fe2f8ed24f71d9f32a0323f7d79dbbd1643a52e17fb2e524c027166f87dc12ad136ddcee0343c012fe27932a82ed26513a7184bb030a8fc5d4a64b
-  data.tar.gz: 415dc58012fc48dba46a9d5ba34f6ea955fb78a194e3666c9d78f6e2767fdac76fd5decff989d7b4b52b1e31e054b3cca5b49abe337138ed9230ce3f82a92bdc
+  metadata.gz: 22f95d7e30a7c4e8c83b1e2ddea5c962a88b1655e19b68df9eadc670286f5a5ae98376b95e7eac9fd85e5f6c13d78e74f009787b5e0eaeb64f4217656a7df104
+  data.tar.gz: f4b831cb109d4ee7b05efa2c61cb3ba00874088e62b03d83af981c10ce411a4dc10535f56a163bc7058f0f5a7d27cac286dff4ef4ad4b359126f757f843583a0

data/README.md

@@ -17,7 +17,7 @@ It's your responsibility to deal with this if that is a problem for you, e.g. by
 
 ## Usage
 
-First, add `kafka-consumer` to your **Gemfile**, and run `bundle install.
+First, add `kafka-consumer` to your **Gemfile**, and run `bundle install`.
 If your messages are snappy-compressed, add the `snappy` gem as well.
 
 ``` ruby

data/lib/kafka/consumer.rb

@@ -18,12 +18,12 @@ module Kafka
         :max_wait_ms, :initial_offset,
         :logger
 
-    def initialize(name, subscription, zookeeper: [], chroot: '', max_wait_ms: 200, initial_offset: :latest_offset, logger: nil)
+    def initialize(name, subscription, zookeeper: [], max_wait_ms: 200, initial_offset: :latest_offset, logger: nil)
       @name, @subscription = name, subscription
       @max_wait_ms, @initial_offset = max_wait_ms, initial_offset
       @logger = logger || Logger.new($stdout)
 
-      @cluster = Kazoo::Cluster.new(zookeeper, chroot: chroot)
+      @cluster = Kazoo::Cluster.new(zookeeper)
       @group = Kazoo::Consumergroup.new(@cluster, name)
       @group.create unless @group.exists?
 

data/lib/kafka/consumer/version.rb

@@ -1,5 +1,5 @@
 module Kafka
   class Consumer
-    VERSION = "0.0.1"
+    VERSION = "0.1.1"
   end
 end

data/lib/kazoo/cli.rb

@@ -3,8 +3,7 @@ require 'thor'
 
 module Kazoo
   class CLI < Thor
-    class_option :zookeeper, :type => :string, :default => ENV['ZOOKEEPER_PEERS']
-    class_option :chroot, :type => :string, :default => ""
+    class_option :zookeeper, :type => :string, :default => ENV['ZOOKEEPER']
 
     desc "cluster", "Describes the Kafka cluster as registered in Zookeeper"
     def cluster
@@ -73,7 +72,7 @@ module Kazoo
     end
 
     def kafka_cluster
-      @kafka_cluster ||= Kazoo::Cluster.new(options[:zookeeper], chroot: options[:chroot])
+      @kafka_cluster ||= Kazoo::Cluster.new(options[:zookeeper])
     end
   end
 end

data/lib/kazoo/cluster.rb

@@ -1,10 +1,10 @@
 module Kazoo
   class Cluster
 
-    attr_reader :zookeeper, :chroot
+    attr_reader :zookeeper
 
-    def initialize(zookeeper, chroot: "")
-      @zookeeper, @chroot = zookeeper, chroot
+    def initialize(zookeeper)
+      @zookeeper = zookeeper
       @zk_mutex, @brokers_mutex, @topics_mutex, @consumergroups_mutex = Mutex.new, Mutex.new, Mutex.new, Mutex.new
     end
 
@@ -17,11 +17,11 @@ module Kazoo
     def brokers
       @brokers_mutex.synchronize do
         @brokers ||= begin
-          brokers = zk.get_children(path: node_with_chroot("/brokers/ids"))
+          brokers = zk.get_children(path: "/brokers/ids")
           result, threads, mutex = {}, ThreadGroup.new, Mutex.new
           brokers.fetch(:children).map do |id|
             t = Thread.new do
-              broker_info = zk.get(path: node_with_chroot("/brokers/ids/#{id}"))
+              broker_info = zk.get(path: "/brokers/ids/#{id}")
               broker = Kazoo::Broker.from_json(self, id, JSON.parse(broker_info.fetch(:data)))
               mutex.synchronize { result[id.to_i] = broker }
             end
@@ -35,7 +35,7 @@ module Kazoo
 
     def consumergroups
       @consumergroups ||= begin
-        consumers = zk.get_children(path: node_with_chroot("/consumers"))
+        consumers = zk.get_children(path: "/consumers")
         consumers.fetch(:children).map { |name| Kazoo::Consumergroup.new(self, name) }
       end
     end
@@ -43,11 +43,11 @@ module Kazoo
     def topics
       @topics_mutex.synchronize do
         @topics ||= begin
-          topics = zk.get_children(path: node_with_chroot("/brokers/topics"))
+          topics = zk.get_children(path: "/brokers/topics")
           result, threads, mutex = {}, ThreadGroup.new, Mutex.new
           topics.fetch(:children).each do |name|
             t = Thread.new do
-              topic_info = zk.get(path: node_with_chroot("/brokers/topics/#{name}"))
+              topic_info = zk.get(path: "/brokers/topics/#{name}")
               topic = Kazoo::Topic.from_json(self, name, JSON.parse(topic_info.fetch(:data)))
               mutex.synchronize { result[name] = topic }
             end
@@ -71,10 +71,6 @@ module Kazoo
       partitions.any?(&:under_replicated?)
     end
 
-    def node_with_chroot(path)
-      "#{@chroot}#{path}"
-    end
-
     def close
       zk.close
     end

data/lib/kazoo/consumergroup.rb

@@ -7,14 +7,14 @@ module Kazoo
     end
 
     def create
-      cluster.zk.create(path: cluster.node_with_chroot("/consumers/#{name}"))
-      cluster.zk.create(path: cluster.node_with_chroot("/consumers/#{name}/ids"))
-      cluster.zk.create(path: cluster.node_with_chroot("/consumers/#{name}/owners"))
-      cluster.zk.create(path: cluster.node_with_chroot("/consumers/#{name}/offsets"))
+      cluster.zk.create(path: "/consumers/#{name}")
+      cluster.zk.create(path: "/consumers/#{name}/ids")
+      cluster.zk.create(path: "/consumers/#{name}/owners")
+      cluster.zk.create(path: "/consumers/#{name}/offsets")
     end
 
     def exists?
-      stat = cluster.zk.stat(path: cluster.node_with_chroot("/consumers/#{name}"))
+      stat = cluster.zk.stat(path: "/consumers/#{name}")
       stat.fetch(:stat).exists?
     end
 
@@ -24,23 +24,19 @@ module Kazoo
     end
 
     def instances
-      instances = cluster.zk.get_children(path: cluster.node_with_chroot("/consumers/#{name}/ids"))
+      instances = cluster.zk.get_children(path: "/consumers/#{name}/ids")
       instances.fetch(:children).map { |id| Instance.new(self, id: id) }
     end
 
     def watch_instances(&block)
       cb = Zookeeper::Callbacks::WatcherCallback.create(&block)
-      result = cluster.zk.get_children(
-        path: cluster.node_with_chroot("/consumers/#{name}/ids"),
-        watcher: cb,
-      )
-
-      instances = result.fetch(:children).map { |id| Instance.new(self, id: id) }
+      result = cluster.zk.get_children(path: "/consumers/#{name}/ids", watcher: cb)
 
       if result.fetch(:rc) != Zookeeper::Constants::ZOK
         raise Kazoo::Error, "Failed to watch instances. Error code result[:rc]"
       end
 
+      instances = result.fetch(:children).map { |id| Instance.new(self, id: id) }
       [instances, cb]
     end
 
@@ -48,10 +44,7 @@ module Kazoo
     def watch_partition_claim(partition, &block)
       cb = Zookeeper::Callbacks::WatcherCallback.create(&block)
 
-      result = cluster.zk.get(
-        path:    cluster.node_with_chroot("/consumers/#{name}/owners/#{partition.topic.name}/#{partition.id}"),
-        watcher: cb
-      )
+      result = cluster.zk.get(path: "/consumers/#{name}/owners/#{partition.topic.name}/#{partition.id}", watcher: cb)
 
       case result.fetch(:rc)
       when Zookeeper::Constants::ZNONODE # Nobody is claiming this partition yet
@@ -64,7 +57,7 @@ module Kazoo
     end
 
     def retrieve_offset(partition)
-      result = cluster.zk.get(path: cluster.node_with_chroot("/consumers/#{name}/offsets/#{partition.topic.name}/#{partition.id}"))
+      result = cluster.zk.get(path: "/consumers/#{name}/offsets/#{partition.topic.name}/#{partition.id}")
       case result.fetch(:rc)
         when Zookeeper::Constants::ZOK;
           result.fetch(:data).to_i
@@ -76,23 +69,16 @@ module Kazoo
     end
 
     def commit_offset(partition, offset)
-      result = cluster.zk.set(
-        path: cluster.node_with_chroot("/consumers/#{name}/offsets/#{partition.topic.name}/#{partition.id}"),
-        data: (offset + 1).to_s
-      )
-
+      result = cluster.zk.set(path: "/consumers/#{name}/offsets/#{partition.topic.name}/#{partition.id}", data: (offset + 1).to_s)
       if result.fetch(:rc) == Zookeeper::Constants::ZNONODE
-        result = cluster.zk.create(path: cluster.node_with_chroot("/consumers/#{name}/offsets/#{partition.topic.name}"))
+        result = cluster.zk.create(path: "/consumers/#{name}/offsets/#{partition.topic.name}")
         case result.fetch(:rc)
           when Zookeeper::Constants::ZOK, Zookeeper::Constants::ZNODEEXISTS
         else
           raise Kazoo::Error, "Failed to commit offset #{offset} for partition #{partition.topic.name}/#{partition.id}. Error code: #{result.fetch(:rc)}"
         end
 
-        result = cluster.zk.create(
-          path: cluster.node_with_chroot("/consumers/#{name}/offsets/#{partition.topic.name}/#{partition.id}"),
-          data: (offset + 1).to_s
-        )
+        result = cluster.zk.create(path: "/consumers/#{name}/offsets/#{partition.topic.name}/#{partition.id}", data: (offset + 1).to_s)
       end
 
       if result.fetch(:rc) != Zookeeper::Constants::ZOK
@@ -101,19 +87,19 @@ module Kazoo
     end
 
     def reset_offsets
-      result = cluster.zk.get_children(path: cluster.node_with_chroot("/consumers/#{name}/offsets"))
+      result = cluster.zk.get_children(path: "/consumers/#{name}/offsets")
       raise Kazoo::Error unless result.fetch(:rc) == Zookeeper::Constants::ZOK
 
       result.fetch(:children).each do |topic|
-        result = cluster.zk.get_children(path: cluster.node_with_chroot("/consumers/#{name}/offsets/#{topic}"))
+        result = cluster.zk.get_children(path: "/consumers/#{name}/offsets/#{topic}")
         raise Kazoo::Error unless result.fetch(:rc) == Zookeeper::Constants::ZOK
 
         result.fetch(:children).each do |partition|
-          cluster.zk.delete(path: cluster.node_with_chroot("/consumers/#{name}/offsets/#{topic}/#{partition}"))
+          cluster.zk.delete(path: "/consumers/#{name}/offsets/#{topic}/#{partition}")
           raise Kazoo::Error unless result.fetch(:rc) == Zookeeper::Constants::ZOK
         end
 
-        cluster.zk.delete(path: cluster.node_with_chroot("/consumers/#{name}/offsets/#{topic}"))
+        cluster.zk.delete(path: "/consumers/#{name}/offsets/#{topic}")
         raise Kazoo::Error unless result.fetch(:rc) == Zookeeper::Constants::ZOK
       end
     end
@@ -146,13 +132,13 @@ module Kazoo
       end
 
       def registered?
-        stat = cluster.zk.stat(path: cluster.node_with_chroot("/consumers/#{group.name}/ids/#{id}"))
+        stat = cluster.zk.stat(path: "/consumers/#{group.name}/ids/#{id}")
         stat.fetch(:stat).exists?
       end
 
       def register(subscription)
         result = cluster.zk.create(
-          path: cluster.node_with_chroot("/consumers/#{group.name}/ids/#{id}"),
+          path: "/consumers/#{group.name}/ids/#{id}",
           ephemeral: true,
           data: JSON.generate({
             version: 1,
@@ -167,9 +153,9 @@ module Kazoo
         end
 
         subscription.each do |topic|
-          stat = cluster.zk.stat(path: cluster.node_with_chroot("/consumers/#{group.name}/owners/#{topic.name}"))
+          stat = cluster.zk.stat(path: "/consumers/#{group.name}/owners/#{topic.name}")
           unless stat.fetch(:stat).exists?
-            result = cluster.zk.create(path: cluster.node_with_chroot("/consumers/#{group.name}/owners/#{topic.name}"))
+            result = cluster.zk.create(path: "/consumers/#{group.name}/owners/#{topic.name}")
             if result.fetch(:rc) != Zookeeper::Constants::ZOK
               raise Kazoo::ConsumerInstanceRegistrationFailed, "Failed to register subscription of #{topic.name} for consumer group #{group.name}! Error code: #{result.fetch(:rc)}"
             end
@@ -178,12 +164,12 @@ module Kazoo
       end
 
       def deregister
-        cluster.zk.delete(path: cluster.node_with_chroot("/consumers/#{group.name}/ids/#{id}"))
+        cluster.zk.delete(path: "/consumers/#{group.name}/ids/#{id}")
       end
 
       def claim_partition(partition)
         result = cluster.zk.create(
-          path: cluster.node_with_chroot("/consumers/#{group.name}/owners/#{partition.topic.name}/#{partition.id}"),
+          path: "/consumers/#{group.name}/owners/#{partition.topic.name}/#{partition.id}",
           ephemeral: true,
           data: id,
         )
@@ -199,7 +185,7 @@ module Kazoo
       end
 
       def release_partition(partition)
-        result = cluster.zk.delete(path: cluster.node_with_chroot("/consumers/#{group.name}/owners/#{partition.topic.name}/#{partition.id}"))
+        result = cluster.zk.delete(path: "/consumers/#{group.name}/owners/#{partition.topic.name}/#{partition.id}")
         if result.fetch(:rc) != Zookeeper::Constants::ZOK
           raise Kazoo::Error, "Failed to release partition #{partition.topic.name}/#{partition.id}. Error code: #{result.fetch(:rc)}"
         end

data/lib/kazoo/partition.rb

@@ -50,7 +50,7 @@ module Kazoo
     protected
 
     def refresh_state
-      state_json = cluster.zk.get(path: cluster.node_with_chroot("/brokers/topics/#{topic.name}/partitions/#{id}/state"))
+      state_json = cluster.zk.get(path: "/brokers/topics/#{topic.name}/partitions/#{id}/state")
       set_state(JSON.parse(state_json.fetch(:data)))
     end
 

data/lib/kazoo/version.rb

@@ -1,3 +1,3 @@
 module Kazoo
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: kafka-consumer
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.1
 platform: ruby
 authors:
 - Willem van Bergen