poseidon_cluster 0.0.5 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 58d3c657309fe7381c8776baa87f54b41f04281f
-  data.tar.gz: 009a15759814e1c92bdec7e903d9a8fb16a4ce4a
+  metadata.gz: 572fd136cb5145934e5c5ee72cd5b679c80284d3
+  data.tar.gz: f376b773d60c642603c45756a5626e3b67927037
 SHA512:
-  metadata.gz: 1fc88b0a4b8f27cb3314db151956242a1c1e3caaa78fa89a17c15907e28a0cfc6ac0bfeadd26829245a01e331ee1112bf6abf2201357418f5bccef4b0ddb933c
-  data.tar.gz: 79d9a39b2ab8a424ed3e0d44419e3c021983afb617637db237ce2ee0134ff18dcaca7494ce65bd535b400cfb1ef816cacf20d0a8c7ebe1c32bd1b05356a699c9
+  metadata.gz: c09941bb0135cbfd85f6d761b3accae3d5e6bf37265a7afc8eb5cc32cf135d5a5f099d558d8bf8281afb7c7901a7a86eeb3dc5bcfe65acd0615d2683e5dc7dad
+  data.tar.gz: ae1cd45aedc7587e7324b34c5a06f9ca1f196a445fc6f2b03cb4a449e396e0713e08102f6d5d85df261ce7320916f9c89398cc48bd702e11c329f4994f350300

data/Gemfile.lock

@@ -7,7 +7,7 @@ GIT
 PATH
   remote: .
   specs:
-    poseidon_cluster (0.0.5)
+    poseidon_cluster (0.1.0)
       poseidon
       zk
 

data/lib/poseidon/consumer_group.rb

@@ -42,6 +42,24 @@ class Poseidon::ConsumerGroup
 
   end
 
+  # @param [Integer] pnum number of partitions size
+  # @param [Array<String>] cids consumer IDs
+  # @param [String] id consumer ID
+  # @return [Range, NilClass] selectable range, if any
+  def self.pick(pnum, cids, id)
+    cids = cids.sort
+    pos  = cids.index(id)
+    return unless pos && pos < cids.size
+
+    step = pnum.fdiv(cids.size).ceil
+    frst = pos*step
+    last = (pos+1)*step-1
+    last = pnum-1 if last > pnum-1
+    return if last < 0 || last < frst
+
+    (frst..last)
+  end
+
   # @attr_reader [String] name Group name
   attr_reader :name
 
@@ -346,19 +364,14 @@ class Poseidon::ConsumerGroup
         reload
         release_all!
 
-        cmg = zk.children(registries[:consumer], watch: true).sort
-        ptm = partitions
-        pos = cmg.index(id)
-        break unless ptm.size > pos
-
-        num = ptm.size / cmg.size
-        num = 1 if num < 1
-        rng = pos*num..(pos+1)*num-1
+        ids = zk.children(registries[:consumer], watch: true)
+        pms = partitions
+        rng = self.class.pick(pms.size, ids, id)
 
-        (ptm[rng] || []).each do |pm|
+        pms[rng].each do |pm|
           consumer = claim!(pm.id)
           @consumers.push(consumer)
-        end
+        end if rng
       end
     end
 

data/poseidon_cluster.gemspec

@@ -5,7 +5,7 @@ Gem::Specification.new do |s|
   s.name        = File.basename(__FILE__, '.gemspec')
   s.summary     = "Poseidon cluster extensions"
   s.description = "Cluster extensions for Poseidon, a producer and consumer implementation for Kafka >= 0.8"
-  s.version     = "0.0.5"
+  s.version     = "0.1.0"
 
   s.authors     = ["Black Square Media"]
   s.email       = "info@blacksquaremedia.com"

data/spec/lib/poseidon/consumer_group_spec.rb

@@ -154,6 +154,25 @@ describe Poseidon::ConsumerGroup do
       c.claimed.should == [0]
     end
 
+    it "should allocate partitions correctly" do
+      subject.claimed.should == [0, 1]
+
+      b = new_group
+      wait_for { subject.claimed.size > 0 }
+      wait_for { b.claimed.size > 0 }
+      subject.claimed.should == [1]
+      b.claimed.should == [0]
+
+      c = new_group
+      b.close
+      wait_for { b.claimed.size < 0 }
+      wait_for { c.claimed.size > 0 }
+
+      subject.claimed.should == [1]
+      b.claimed.should == []
+      c.claimed.should == [0]
+    end
+
   end
 
   describe "fetch" do
@@ -245,4 +264,29 @@ describe Poseidon::ConsumerGroup do
     end
 
   end
+
+  describe "pick" do
+
+    { [3, ["N1", "N2", "N3"], "N1"]       => (0..0),
+      [3, ["N1", "N2", "N3"], "N2"]       => (1..1),
+      [3, ["N1", "N2", "N3"], "N3"]       => (2..2),
+      [4, ["N2", "N4", "N3", "N1"], "N3"] => (2..2),
+      [3, ["N1", "N2", "N3"], "N4"]       => nil,
+      [5, ["N1", "N2", "N3"], "N1"]       => (0..1),
+      [5, ["N1", "N2", "N3"], "N2"]       => (2..3),
+      [5, ["N1", "N2", "N3"], "N3"]       => (4..4),
+      [5, ["N1", "N2", "N3"], "N4"]       => nil,
+      [2, ["N1", "N2"], "N9"]             => nil,
+      [1, ["N1", "N2", "N3"], "N1"]       => (0..0),
+      [1, ["N1", "N2", "N3"], "N2"]       => nil,
+      [1, ["N1", "N2", "N3"], "N3"]       => nil,
+      [5, ["N1", "N2"], "N1"]             => (0..2),
+      [5, ["N1", "N2"], "N2"]             => (3..4),
+    }.each do |args, expected|
+      it "should pick #{expected.inspect} from #{args.inspect}" do
+        described_class.pick(*args).should == expected
+      end
+    end
+
+  end
 end

data/spec/spec_helper.rb

@@ -3,6 +3,7 @@ require 'rspec'
 require 'fileutils'
 require 'pathname'
 require 'coveralls'
+Coveralls.wear_merged!
 
 TOPIC_NAME  = "my-topic"
 KAFKA_LOCAL = File.expand_path("../kafka_2.8.0-0.8.0", __FILE__)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: poseidon_cluster
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.1.0
 platform: ruby
 authors:
 - Black Square Media
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-30 00:00:00.000000000 Z
+date: 2014-02-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: poseidon