promiscuous-poseidon_cluster 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f7653a5f9ee3e08c3ccda22787d2a9da5b752b86
-  data.tar.gz: 6447aadb67ad485dc77e79ea91af4d3b99bd49e1
+  metadata.gz: b88185138e6a8e8de7ce363a550b4630f6e95c8f
+  data.tar.gz: b234eaee3e59a7441458f4506a09fd5c5e929e2e
 SHA512:
-  metadata.gz: 203099eb7d08fa1f80dd197c5e4094352291dc197a4e8137efde19450a4c4224e2cdc23bfdc9dd6789607c0544c326ce329c3c81bbd1d7a30f557f30ae0deddb
-  data.tar.gz: d78680cc99b1dd11fb11183f00fc4186835e8515ade5ae933b10d0a39a55f7bb0f5a99afb2d68e371198659402a20b421b6e9b80f3700006b941e7462a094889
+  metadata.gz: 167be7e9484234a3fd4d282567c1230d1d231463be6ed38bb0adfbddfcc91d198de5af2d636caa0a2b588044afbd9e0c43948a6b10d4ffdd0f581104ac334318
+  data.tar.gz: 3952e0a94d4efe8ef020812ba0dd6b9379b4bdbe530cfbcb61da532a2286eaff5a79c6574fe772e5168cd11605bdc80b9b77a2313a83f49cc263e7bf365d445a

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    poseidon_cluster (0.3.0)
+    poseidon_cluster (0.4.0)
       poseidon (>= 0.0.5.pre1)
       zk
 
@@ -22,7 +22,7 @@ GEM
       multi_json (>= 1.8.4)
     mime-types (2.3)
     multi_json (1.10.1)
-    poseidon (0.0.5.pre1)
+    poseidon (0.0.5)
     rake (10.3.2)
     rest-client (1.6.7)
       mime-types (>= 1.16)
@@ -53,13 +53,10 @@ GEM
     thor (0.19.1)
     tins (1.3.0)
     yard (0.8.7.4)
-    zk (1.9.4)
+    zk (1.9.5)
       logging (~> 1.8.2)
       zookeeper (~> 1.4.0)
-    zookeeper (1.4.9)
-    zookeeper (1.4.9-java)
-      slyphon-log4j (= 1.2.15)
-      slyphon-zookeeper_jar (= 3.3.5)
+    zookeeper (1.4.10)
 
 PLATFORMS
   java

data/lib/poseidon/cluster.rb

@@ -1,9 +1,9 @@
 require 'socket'
-require 'timeout'
-require 'zk'
-require 'poseidon'
 require 'thread'
 
+require 'poseidon'
+require 'zk'
+
 module Poseidon::Cluster
   MAX_INT32 = 0x7fffffff
   @@sem = Mutex.new
@@ -12,17 +12,22 @@ module Poseidon::Cluster
   # @return [Integer] an incremented number
   # @api private
   def self.inc!
-    @@sem.synchronize { @@inc += 1; @@inc = 1 if @@inc > MAX_INT32; @@inc }
+    @@sem.synchronize do
+      @@inc += 1;
+
+      if @@inc > MAX_INT32
+        @@inc = 1
+      end
+
+      @@inc
+    end
   end
 
   # @return [String] an globally unique identifier
   # @api private
   def self.guid
-    [::Socket.gethostname, ::Process.pid, ::Time.now.to_i, inc!].join("-")
+    [::Socket.gethostname, ::Process.pid, ::Time.now.nsec, inc!].join('-')
   end
-
 end
 
-%w|consumer_group|.each do |name|
-  require "poseidon/#{name}"
-end
+require 'poseidon/consumer_group'

data/lib/poseidon/consumer_group.rb

@@ -21,7 +21,7 @@
 #
 # @api public
 class Poseidon::ConsumerGroup
-  DEFAULT_CLAIM_TIMEOUT = 30
+  DEFAULT_CLAIM_TIMEOUT = 5
   DEFAULT_LOOP_DELAY = 1
 
   # Poseidon::ConsumerGroup::Consumer is internally used by Poseidon::ConsumerGroup.
@@ -41,7 +41,6 @@ class Poseidon::ConsumerGroup
       options.delete(:trail)
       super group.id, broker.host, broker.port, group.topic, partition, offset, options
     end
-
   end
 
   # @param [Integer] pnum number of partitions size
@@ -97,7 +96,7 @@ class Poseidon::ConsumerGroup
   def initialize(name, brokers, zookeepers, topic, options = {})
     @name       = name
     @topic      = topic
-    @zk         = ::ZK.new(zookeepers.join(","))
+    @zk         = ::ZK.new(zookeepers.join(','), {:thread => :per_callback})
     # Poseidon::BrokerPool doesn't provide default value for this option
     # Configuring default value like this isn't beautiful, though.. by kssminus
     options[:socket_timeout_ms] ||= 10000
@@ -147,8 +146,8 @@ class Poseidon::ConsumerGroup
     registries.each do |_, path|
       zk.mkdir_p(path)
     end
-    zk.create(consumer_path, "{}", ephemeral: true)
-    zk.register(registries[:consumer]) {|_| rebalance! }
+    zk.create(consumer_path, "{}", {:ephemeral => true})
+    zk.register(registries[:consumer]) { rebalance! }
 
     # Rebalance
     rebalance!
@@ -156,10 +155,12 @@ class Poseidon::ConsumerGroup
   end
 
   # Reloads metadata/broker/partition information
-  def reload
-    @metadata = @topic_metadata = nil
+  def reload_metadata
+    @metadata = nil; @topic_metadata = nil
     metadata
-    self
+    topic_metadata
+
+    true
   end
 
   # Closes the consumer group gracefully, only really useful in tests
@@ -178,7 +179,7 @@ class Poseidon::ConsumerGroup
   # @param [Integer] partition
   # @return [Integer] the latest stored offset for the given partition
   def offset(partition)
-    data, _ = zk.get offset_path(partition), ignore: :no_node
+    data, _ = zk.get(offset_path(partition), {:ignore => :no_node})
     data.to_i
   end
 
@@ -186,9 +187,9 @@ class Poseidon::ConsumerGroup
   # @param [Integer] partition
   # @param [Integer] offset
   def commit(partition, offset)
-    zk.set offset_path(partition), offset.to_s
+    zk.set(offset_path(partition), offset.to_s)
   rescue ZK::Exceptions::NoNode
-    zk.create offset_path(partition), offset.to_s, ignore: :node_exists
+    zk.create(offset_path(partition), offset.to_s, {:ignore => :node_exists})
   end
 
   # Sorted partitions by broker address (so partitions on the same broker are clustered together)
@@ -198,7 +199,7 @@ class Poseidon::ConsumerGroup
 
     topic_metadata.available_partitions.sort_by do |part|
       broker = metadata.brokers[part.leader]
-      [broker.host, broker.port].join(":")
+      [broker.host, broker.port].join(':')
     end
   end
 
@@ -239,6 +240,7 @@ class Poseidon::ConsumerGroup
     unless opts[:commit] == false || commit == false
       commit consumer.partition, consumer.offset
     end
+
     true
   end
 
@@ -345,7 +347,7 @@ class Poseidon::ConsumerGroup
       # Yield over an empty array if nothing claimed,
       # to allow user to e.g. break out of the loop
       unless ok
-        yield -1, []
+        yield(-1, [])
       end
 
       # Sleep if either not claimes or nothing returned
@@ -366,28 +368,21 @@ class Poseidon::ConsumerGroup
     # * let POS be our index position in CG and let N = size(PT)/size(CG)
     # * assign partitions from POS*N to (POS+1)*N-1
     def rebalance!
-      return if @pending
-
-      @pending = true
       @mutex.synchronize do
-        @pending = nil
-
         release_all!
-        reload
+        reload_metadata
 
-        ids = zk.children(registries[:consumer], watch: true)
-        pms = partitions
-        rng = self.class.pick(pms.size, ids, id)
+        consumer_ids    = zk.children(registries[:consumer], {:watch => true}).sort
+        partition_list  = partitions
+        responsible_for = self.class.pick(partition_list.size, consumer_ids, id)
 
-        pms[rng].each do |pm|
-          if @pending
-            release_all!
-            break
-          end
+        return if responsible_for.nil?
 
-          consumer = claim!(pm.id)
-          @consumers.push(consumer) if consumer
-        end if rng
+        partition_list[responsible_for].each do |partition|
+          if consumer = claim!(partition.id)
+            @consumers.push(consumer)
+          end
+        end
       end
     end
 
@@ -395,26 +390,33 @@ class Poseidon::ConsumerGroup
     def release_all!
       @consumers.each {|c| release!(c.partition) }
       @consumers.clear
+
+      true
     end
 
   private
 
     # Claim the ownership of the partition for this consumer
-    # @raise [Timeout::Error]
     def claim!(partition)
-      path = claim_path(partition)
-      Timeout.timeout options[:claim_timout] || DEFAULT_CLAIM_TIMEOUT do
-        while zk.create(path, id, ephemeral: true, ignore: :node_exists).nil?
-          return if @pending
-          sleep(0.1)
-        end
+      sleep_duration = 0.1
+      num_tries = ((options[:claim_timout] || DEFAULT_CLAIM_TIMEOUT)/sleep_duration).floor.to_i
+      created = nil
+
+      num_tries.times do
+        created = zk.create(claim_path(partition), id, {:ephemeral => true, :ignore => :node_exists})
+        break if created
+
+        sleep(sleep_duration)
       end
-      Consumer.new self, partition, options.dup
+
+      return nil unless created
+
+      Consumer.new(self, partition, options.dup)
     end
 
     # Release ownership of the partition
     def release!(partition)
-      zk.delete claim_path(partition), ignore: :no_node
+      zk.delete(claim_path(partition), {:ignore => :no_node})
     end
 
     # @return [String] zookeeper ownership claim path
@@ -431,5 +433,4 @@ class Poseidon::ConsumerGroup
     def consumer_path
       "#{registries[:consumer]}/#{id}"
     end
-
 end

data/poseidon_cluster.gemspec

@@ -5,10 +5,10 @@ Gem::Specification.new do |s|
   s.name        = File.basename(__FILE__, '.gemspec')
   s.summary     = "Poseidon cluster extensions"
   s.description = "Cluster extensions for Poseidon, a producer and consumer implementation for Kafka >= 0.8"
-  s.version     = "0.3.0"
+  s.version     = "0.4.0"
 
-  s.authors     = ["Black Square Media"]
-  s.email       = "info@blacksquaremedia.com"
+  s.authors     = ["Black Square Media", "Crowdtap Inc."]
+  s.email       = "ops@crowdtap.com"
   s.homepage    = "https://github.com/promiscuous-io/poseidon_cluster"
 
   s.require_path = 'lib'
@@ -24,5 +24,4 @@ Gem::Specification.new do |s|
   s.add_development_dependency "rspec-its"
   s.add_development_dependency "yard"
   s.add_development_dependency "coveralls"
-
 end

data/spec/lib/poseidon/cluster_spec.rb

@@ -8,12 +8,12 @@ describe Poseidon::Cluster do
 
     (0...5).map do
       Thread.new { 100.times { described_class.inc! }}
-    end.each &:join
+    end.each(&:join)
     (described_class.inc! - num).should == 502
   end
 
   it 'should generate GUIDs' do
-    described_class.guid.should match(/\A[\w\-\.]+?\-\d{1,5}\-\d{10}\-\d{1,3}\z/)
+    described_class.guid.should match(/\A[\w\-\.]+?\-\d{1,5}\-\d{9}\-\d{1,3}\z/)
   end
 
 end

metadata

@@ -1,14 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: promiscuous-poseidon_cluster
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Black Square Media
+- Crowdtap Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-16 00:00:00.000000000 Z
+date: 2015-04-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: poseidon
@@ -124,7 +125,7 @@ dependencies:
         version: '0'
 description: Cluster extensions for Poseidon, a producer and consumer implementation
   for Kafka >= 0.8
-email: info@blacksquaremedia.com
+email: ops@crowdtap.com
 executables: []
 extensions: []
 extra_rdoc_files: []