promiscuous-poseidon_cluster 0.4.1 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 785249eb37555296aa2d93478182c292850413fe
-  data.tar.gz: 21cf920cade06e1598fee1b47ea19a553a5ab13b
+  metadata.gz: b9fc0d8c24c41642e703279f9feeb80e10c788b8
+  data.tar.gz: 6c35c729adac40669cbb915542810ab107f7dd07
 SHA512:
-  metadata.gz: dc4faf271c725f70d7535e675f97ec27c933b6c63a95c69927738e79bef160e8d92bbcc8229177c25589bdcd5b32dd62af6b082b99d60f8f2a63fe88945758ce
-  data.tar.gz: 7040138c26ff1c65bd4c2600f16c339c488f214830d3307bb7050523652bc671ab01257fa364a2adfb481f6604d03f98f3a8cea8d56a64a71d7369c8e085fe09
+  metadata.gz: 2b7f1477c8fe08586939a90af7a6b7ee4f03a97e59140afb5252d8442f0ad99926bd644848a40fc844116982460a20ded382c0a1ac1493d5c292acb70ca2979a
+  data.tar.gz: d7c875b13cce867d3f43e9969419efc6db74b6fd9e97e3e116c16a3e09b18445cd90564eb44224ca657ef773c5c264d97e95e72b44f4cf9c7777a3133d8bd5fb

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    promiscuous-poseidon_cluster (0.4.1)
+    poseidon_cluster (0.4.1)
       poseidon (>= 0.0.5.pre1)
       zk
 
@@ -65,7 +65,7 @@ PLATFORMS
 DEPENDENCIES
   bundler
   coveralls
-  promiscuous-poseidon_cluster!
+  poseidon_cluster!
   rake
   rspec
   rspec-its

data/lib/poseidon/consumer_group.rb

@@ -110,7 +110,6 @@ class Poseidon::ConsumerGroup
     @consumers  = []
     @pool       = ::Poseidon::BrokerPool.new(id, brokers, options[:socket_timeout_ms])
     @mutex      = Mutex.new
-    @registered = false
 
     register! unless options[:register] == false
   end
@@ -141,7 +140,7 @@ class Poseidon::ConsumerGroup
 
   # @return [Boolean] true if registered
   def registered?
-    @registered
+    !!zk.children(consumer_path, ignore: :no_node)
   end
 
   # @return [Boolean] true if registration was successful, false if already registered
@@ -155,9 +154,7 @@ class Poseidon::ConsumerGroup
     zk.create(consumer_path, "{}", {:ephemeral => true})
     zk.register(registries[:consumer]) { rebalance! }
 
-    # Rebalance
     rebalance!
-    @registered = true
   end
 
   # Reloads metadata/broker/partition information
@@ -172,7 +169,7 @@ class Poseidon::ConsumerGroup
   # Closes the consumer group gracefully, only really useful in tests
   # @api private
   def close
-    @mutex.synchronize { release_all! }
+    synchronize { release_all! }
     zk.close
   end
 
@@ -196,6 +193,8 @@ class Poseidon::ConsumerGroup
     zk.set(offset_path(partition), offset.to_s)
   rescue ZK::Exceptions::NoNode
     zk.create(offset_path(partition), offset.to_s, {:ignore => :node_exists})
+  ensure
+    unlock(offset)
   end
 
   # Sorted partitions by broker address (so partitions on the same broker are clustered together)
@@ -234,20 +233,22 @@ class Poseidon::ConsumerGroup
   #
   # @api public
   def checkout(opts = {})
-    consumer = nil
-    commit   = @mutex.synchronize do
-      consumer = @consumers.shift
-      return false unless consumer
+    register!
+    lock
 
-      @consumers.push consumer
-      yield consumer
-    end
+    @current_consumer = @consumers.shift
+    return false unless @current_consumer
+
+    @consumers.push @current_consumer
+    commit = yield @current_consumer
 
     unless opts[:commit] == false || commit == false
-      commit consumer.partition, consumer.offset
+      commit @current_consumer.partition, @current_consumer.offset
     end
 
     true
+  rescue StandardError
+    unlock
   end
 
   # Convenience method to fetch messages from the broker.
@@ -374,7 +375,12 @@ class Poseidon::ConsumerGroup
     # * let POS be our index position in CG and let N = size(PT)/size(CG)
     # * assign partitions from POS*N to (POS+1)*N-1
     def rebalance!
-      @mutex.synchronize do
+      return if @rebalancing
+
+      @rebalancing = true
+      synchronize do
+        @rebalancing = nil
+
         release_all!
         reload_metadata
 
@@ -385,6 +391,11 @@ class Poseidon::ConsumerGroup
         return if responsible_for.nil?
 
         partition_list[responsible_for].each do |partition|
+          if @rebalancing
+            release_all!
+            break
+          end
+
           if consumer = claim!(partition.id)
             @consumers.push(consumer)
           end
@@ -401,6 +412,23 @@ class Poseidon::ConsumerGroup
     end
 
   private
+    def lock
+      @mutex.lock
+    end
+
+    def unlock(offset=nil)
+      raise "Mutex should be locked, possibly committing out of order" unless  @mutex.locked?
+
+      if offset
+        @mutex.unlock if @current_consumer.offset == offset
+      else
+        @mutex.unlock
+      end
+    end
+
+    def synchronize
+      @mutex.synchronize { yield }
+    end
 
     # Claim the ownership of the partition for this consumer
     def claim!(partition)
@@ -409,6 +437,8 @@ class Poseidon::ConsumerGroup
       created = nil
 
       num_tries.times do
+        return nil if @rebalancing
+
         created = zk.create(claim_path(partition), id, {:ephemeral => true, :ignore => :node_exists})
         break if created
 

data/scenario/scenario.rb

@@ -84,7 +84,7 @@ module Scenario
     sh "cd #{ROOT} && curl http://www.mirrorservice.org/sites/ftp.apache.org/kafka/#{VERSION}/kafka_2.10-#{VERSION}.tgz | tar xz"
   end
 
-  def checkpoint!(timeout = 10)
+  def checkpoint!(timeout = 30)
     puts "--> Verifying #{@@total}"
     timeout.times do
       if numlines > @@total

data/spec/lib/poseidon/consumer_group_spec.rb

@@ -29,7 +29,7 @@ describe Poseidon::ConsumerGroup do
   end
 
   let :zk_client do
-    double "ZK", mkdir_p: nil, get: nil, set: nil, delete: nil, create: "/path", register: nil, children: ["my-group-UNIQUEID"], close: nil
+    double "ZK", mkdir_p: nil, get: nil, set: nil, delete: nil, create: "/path", register: nil, close: nil
   end
 
   let(:group) { described_class.new "my-group", ["localhost:29092", "localhost:29091"], ["localhost:22181"], "mytopic" }
@@ -37,6 +37,7 @@ describe Poseidon::ConsumerGroup do
 
   before do
     allow(ZK).to receive_messages(new: zk_client)
+    allow(zk_client).to receive(:children).and_return(nil, ["my-group-UNIQUEID"])
     allow(Poseidon::Cluster).to receive_messages(guid: "UNIQUEID")
     allow_any_instance_of(Poseidon::ConsumerGroup).to receive(:sleep)
     allow_any_instance_of(Poseidon::PartitionConsumer).to receive_messages(resolve_offset_if_necessary: 0)
@@ -170,6 +171,30 @@ describe Poseidon::ConsumerGroup do
       -> { subject.send :rebalance! }.should change { subject.claimed }.to([0])
     end
 
+    it "should wait for consumer to commit manually all messages before rebalancing" do
+      consumer = nil
+      subject.checkout(commit: false) { |c| consumer = c }
+
+      t = Thread.new { subject.send :rebalance! }
+
+      sleep 0.1
+      t.alive?.should == true
+
+      payloads = consumer.fetch
+      subject.commit(consumer.partition, payloads.shift.offset + 1)
+
+      sleep 0.1
+      t.alive?.should == true
+
+      payloads.each do |payload|
+        subject.commit(consumer.partition, payload.offset + 1)
+      end
+
+      sleep 0.1
+      t.alive?.should == false
+
+      t.join
+    end
   end
 
   describe "fetch" do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: promiscuous-poseidon_cluster
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.4.2
 platform: ruby
 authors:
 - Black Square Media
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-21 00:00:00.000000000 Z
+date: 2015-05-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: poseidon