infopark-politics 0.7.0 → 0.8.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    OGYyMmNhMjBlZDFiMTIzZTJjZWU0ODg2NjgyYmM4ZTE2NTdiNDZhNw==
+    NmUwNzYzMGNmM2M2MmJkOTliNjFlOTYwY2I1ZmI1NzYxOTA1ZTg1NA==
   data.tar.gz: !binary |-
-    ZTU5ZGU0MDI2YjRiM2FiNjlmNWJlNDU1YzA1NDYxZGQ5NDU1MTg5OA==
+    YTFjZGEzODU3Y2NhOWU1YzQ2NGY0NTRiMTQzZDdlZGRlYjQzZmVlOQ==
 SHA512:
   metadata.gz: !binary |-
-    NzYzZjk0MWVmYzZhNzhmZTFkYjQzM2I2NTViMGUzMWFlN2ZlZTFmNDdlMTQ2
-    Y2RlMTc5ZmU4YzAxNTAxOGNkYjc4N2I0NDM0MDhmYTEzZTg4YmYzYTY5YWZm
-    NjJmMzFjYTkzYzVjMTE0NTZkNTY0NzE1NTZhNzM1MjAzYjk2NTA=
+    Zjk3ZDA3NzU0MmUyMmNkNzEyNjgzN2UzYmQ2ZTkyNThkNjEwNjM4ZTc5MTY4
+    MjA4ZjIxNzgxMjA2ZjRlMGYwYmU1ODAxZTQ4M2Q2YTRjNzAyODlkYWIwOTlm
+    NGQxYzVjYTg4NGY3YWQyNmYyNDYwNmZhNGMwZjYxODIzZjQ3MDI=
   data.tar.gz: !binary |-
-    YTEwM2MyOGEwMjIxNGI1ZmJlMThjY2MxNWJlNzYyZjQ4ZDEwNmVkM2I2NDE5
-    MDQ3OWUxNTUzNzgwZTJjYmVkYTMyNmJjZGU3ZTM2Yzk4NGI4MWMwYWUyYWRl
-    ZDJhM2JlZmY1NzhhOWI0YTI2OWFmMDFiMDMzMTJkZDkyYTFmOGE=
+    YjcyMDNlZTBjNzRlYTcyNDcyZmU5M2U3ODdlN2E3ODM2MjRiZTY3YWI2MzBm
+    Mzk3NDZiZDE5NmNlM2U2N2UxZDcwNDdhYTFlNDk4YzdjMmM1NmY4YTgzMWZh
+    NDVmOGQ4NzAzNjg3ODg0NzUwZTM5Zjg2Y2ExOGFlZjlmNjU0ZTQ=

data/README.md

@@ -3,7 +3,7 @@ Infopark-Politics
 
 Infopark-Politics was derived from mperham-politics. All parts except of the StaticQueueWorker had
 eventually been dropped. The StaticQueueWorker remained for breaking up work into a large amount of
-pieces (called buckets) which then are distributed on n-1 worker nodes where n ist the total number
+pieces (called buckets) which then are distributed on n worker nodes where n ist the total number
 of nodes.
 
 Actually there is no further documentation available…

data/infopark-politics.gemspec

@@ -6,7 +6,7 @@ end
 
 Gem::Specification.new do |gem|
   gem.name = "infopark-politics"
-  gem.version = "0.7.0"
+  gem.version = "0.8.0"
   gem.summary = "Algorithms and Tools for Distributed Computing in Ruby."
   gem.description = ""
   gem.authors = ["Mike Perham", "Tilo Prütz"]
@@ -29,7 +29,6 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency "rdoc", ">= 3.9"
   gem.add_development_dependency "rubygems-tasks", ">=0.2"
 
-  gem.add_runtime_dependency "memcache-client", ">= 1.5.0"
-  gem.add_runtime_dependency "starling", ">= 0.9.8"
+  gem.add_runtime_dependency "dalli"
 end
 

data/lib/politics/static_queue_worker.rb

@@ -5,13 +5,7 @@ require 'uri'
 require 'drb'
 require 'set'
 require 'logger'
-
-begin
-  require 'memcache'
-rescue LoadError => e
-  puts "Unable to load memcache client, please run `sudo gem install memcache-client`: #{e.message}"
-  exit(1)
-end
+require 'dalli'
 
 module Politics
   module StaticQueueWorker
@@ -26,7 +20,6 @@ module Politics
       @group_name = name
       @iteration_length = options[:iteration_length]
       @memcache_client = client_for(Array(options[:servers]))
-      @nominated_at = Time.now
       @dictatorship_length = options[:dictatorship_length]
 
       @buckets = []
@@ -46,7 +39,7 @@ module Politics
     def process_bucket(&block)
       log.debug "start bucket processing"
       raise ArgumentError, "process_bucket requires a block!" unless block_given?
-      unless @memcache_client
+      unless memcache_client
         raise ArgumentError, "You must call register_worker before processing!"
       end
 
@@ -58,21 +51,25 @@ module Politics
         end
         begin
           nominate
-          if leader?
-            log.info { "has been elected leader" }
-            perform_leader_duties
-          else
-            # Get a bucket from the leader and process it
-            begin
-              log.debug "getting bucket request from leader (#{leader_uri}) and processing it"
-              bucket_process(*leader.bucket_request(uri, bucket_request_context), &block)
-            rescue DRb::DRbError => dre
-              log.error { "Error talking to leader: #{dre.message}" }
-              relax until_next_iteration
+
+          if leader? && !(@leader_thread && @leader_thread.alive?)
+            unless (@leader_thread && @leader_thread.alive?)
+              @leader_thread = Thread.new do
+                perform_leader_duties
+              end
             end
           end
-        rescue MemCache::MemCacheError => e
-          log.error { "Unexpected MemCacheError: #{e.message}" }
+
+          # Get a bucket from the leader and process it
+          begin
+            log.debug "getting bucket request from leader (#{leader_uri}) and processing it"
+            bucket_process(*leader.bucket_request(uri, bucket_request_context), &block)
+          rescue DRb::DRbError => dre
+            log.error { "Error talking to leader: #{dre.message}" }
+            relax until_next_iteration
+          end
+        rescue Dalli::DalliError => e
+          log.error { "Unexpected DalliError: #{e.message}" }
           relax until_next_iteration
         end
       end while loop?
@@ -87,34 +84,33 @@ module Politics
       seize_leadership
     end
 
-    def seize_leadership(duration = iteration_length)
-      @memcache_client.set(token, uri, duration)
-      @nominated_at = Time.now + duration - iteration_length
+    def seize_leadership(*args)
+      start_iteration(*args) {|duration| memcache_client.set(token, uri, duration) }
     end
 
     def perform_leader_duties
+      # The DRb thread handles the requests to the leader.
+      # This method performs the bucket managing.
+      log.info { "has been elected leader" }
       before_perform_leader_duties
-      # Drb thread handles requests to leader
-      as_dictator do
-        initialize_buckets
+      # keeping leader state as long as buckets are being initialized
+      as_dictator { initialize_buckets }
+
+      while !buckets.empty?
+        # keeping leader state as long as buckets are available by renominating before
+        # nomination times out
+        as_dictator { update_buckets } unless restart_wanted?
       end
-      # keeping leader state as long as buckets are available by renominating before nomination
-      # times out
-      while !buckets.empty? do
-        log.debug { "relaxes half the time until next iteration" }
-        relax(until_next_iteration / 2)
-        as_dictator do
-          update_buckets unless restart_wanted?
+
+      if restart_wanted?
+        as_dictator { populate_followers_to_stop }
+        # keeping leader state as long as there are followers to stop
+        while !followers_to_stop.empty?
+          relax(until_next_iteration / 2)
+          seize_leadership
         end
+        exit 0
       end
-      as_dictator() {populate_followers_to_stop} if restart_wanted?
-      # keeping leader state as long as there are followers to stop
-      while !followers_to_stop.empty? do
-        relax(until_next_iteration / 2)
-        seize_leadership
-      end
-      exit 0 if restart_wanted?
-      relax until_next_iteration
     end
 
     def populate_followers_to_stop
@@ -149,8 +145,7 @@ module Politics
     end
 
     def until_next_iteration
-      left = iteration_length - (Time.now - @nominated_at)
-      left > 0 ? left : 0
+      [(iteration_end || Time.at(0)) - Time.now, 0].max
     end
 
     def alive?
@@ -173,11 +168,17 @@ module Politics
 
     private
 
+    attr_reader :iteration_end, :memcache_client
+
+    def set_iteration_end(interval = iteration_length)
+      @iteration_end = Time.now + interval
+    end
+
     def before_perform_leader_duties
     end
 
     def restart_wanted?
-      @memcache_client.get(restart_flag)
+      memcache_client.get(restart_flag)
     end
 
     def bucket_process(bucket, sleep_time)
@@ -210,6 +211,8 @@ module Politics
     end
 
     def update_buckets
+      log.debug { "relaxes half the time until next iteration" }
+      relax(until_next_iteration / 2)
     end
 
     def loop?
@@ -227,8 +230,8 @@ module Politics
     def internal_cleanup
       log.debug("uri: #{uri}, leader_uri: #{leader_uri}, until_next_iteration: #{until_next_iteration}")
       if leader?
-        @memcache_client.delete(token)
-        @memcache_client.delete(restart_flag)
+        memcache_client.delete(token)
+        memcache_client.delete(restart_flag)
       end
     end
 
@@ -236,16 +239,6 @@ module Politics
       # for custom use
     end
 
-    def pause_until_expiry(elapsed)
-      pause_time = (iteration_length - elapsed).to_f
-      if pause_time > 0
-        relax(pause_time)
-      else
-        raise ArgumentError, "Negative iteration time left. " +
-            "Assuming the worst and exiting… #{iteration_length}/#{elapsed}"
-      end
-    end
-
     def relax(time)
       sleep time
     end
@@ -254,13 +247,17 @@ module Politics
     # and attempting to add the token with our name attached.
     def nominate
       log.debug("try to nominate")
-      @nominated_at = Time.now
-      @memcache_client.add(token, uri, iteration_length)
+      start_iteration {|duration| memcache_client.add(token, uri, duration) }
+    end
+
+    def start_iteration(duration = iteration_length)
+      yield duration
+      set_iteration_end(duration)
       @leader_uri = nil
     end
 
     def leader_uri
-      @leader_uri ||= @memcache_client.get(token)
+      @leader_uri ||= memcache_client.get(token)
     end
 
     # Check to see if we are leader by looking at the process name
@@ -271,7 +268,7 @@ module Politics
 
     # Easy to mock or monkey-patch if another MemCache client is preferred.
     def client_for(servers)
-      MemCache.new(servers)
+      Dalli::Client.new(servers)
     end
 
     def time_for(&block)
@@ -290,9 +287,7 @@ module Politics
     end
 
     def start_druby_service
-      server = DRb.start_service("druby://#{hostname || ""}:0", self)
-      @uri = DRb.uri
-      register_service
+      @uri = DRb.start_service("druby://#{hostname || ""}:0", self).uri
     end
   end
 end

data/spec/spec_helper.rb

@@ -14,6 +14,7 @@ RSpec.configure do |config|
   # the seed, which is printed after each run.
   #     --seed 1234
   config.order = 'random'
+  config.mock_with(:rspec) {|mocks| mocks.verify_partial_doubles = true }
 end
 
 require_relative 'politics'

data/spec/static_queue_worker_spec.rb

@@ -94,8 +94,7 @@ describe Worker do
   end
 
   it "should return time to next iteration even if nominate was not completed" do
-    worker.until_next_iteration.should > 0
-    worker.until_next_iteration.should <= 10
+    worker.until_next_iteration.should == 0
   end
 
   it "should give access to the uri" do
@@ -127,14 +126,14 @@ describe Worker do
       end
 
       it "should relax until next iteration on MemCache errors during nomination" do
-        worker.should_receive(:nominate).exactly(4).and_raise MemCache::MemCacheError.new("Buh!")
+        worker.should_receive(:nominate).exactly(4).and_raise Dalli::DalliError.new("Buh!")
         worker.should_receive(:relax).with(666).exactly(4).times
 
         worker.start
       end
 
       it "should relax until next iteration on MemCache errors during request for leader" do
-        worker.should_receive(:leader_uri).exactly(4).and_raise(MemCache::MemCacheError.new("Buh"))
+        worker.should_receive(:leader_uri).exactly(4).and_raise(Dalli::DalliError.new("Buh"))
         worker.should_receive(:relax).with(666).exactly(4).times
 
         worker.start
@@ -143,10 +142,14 @@ describe Worker do
       describe "as leader" do
         before do
           worker.stub(:leader?).and_return true
+          allow(worker).to receive(:bucket_process) { sleep 0.5 }
+          allow(worker).to receive(:relax)
+          allow(worker).to receive(:leader).and_return(double('leader', bucket_request: nil))
         end
 
-        it "should do leader duties" do
-          worker.should_receive(:perform_leader_duties).exactly(4).times
+        it "performs leader duties in a separate thread" do
+          # four iterations of 0.5 seconds → two leader threads of one second
+          worker.should_receive(:perform_leader_duties).exactly(2).times { sleep 1 }
           worker.start
         end
       end
@@ -310,52 +313,56 @@ describe Worker do
         worker.stub(:buckets).and_return([])
       end
 
-      it "should populate the followers_to_stop list before evaluating it if restart is wanted" do
-        worker.stub(:restart_wanted?).and_return true
-        worker.stub(:exit)
-        worker.should_receive(:populate_followers_to_stop).ordered.once
-        worker.should_receive(:followers_to_stop).ordered.and_return []
-        worker.perform_leader_duties
-      end
-
-      it "should not populate the followers_to_stop list if restart is not wanted" do
-        worker.stub(:restart_wanted?).and_return false
-        worker.should_not_receive(:populate_followers_to_stop)
-        worker.perform_leader_duties
-      end
-
-      describe "as long as there are followers to stop" do
+      context "when restart is wanted" do
         before do
-          worker.stub(:followers_to_stop).and_return([1], [2], [3], [4], [])
-          worker.stub(:relax)
+          allow(worker).to receive(:restart_wanted?).and_return true
+          allow(worker).to receive(:exit)
         end
 
-        it "should relax half of the time to the next iteration" do
-          worker.stub(:until_next_iteration).and_return(6)
-          worker.should_receive(:relax).with(3).exactly(4).times
+        it "populates the followers_to_stop list before evaluating it" do
+          worker.should_receive(:populate_followers_to_stop).ordered.once
+          worker.should_receive(:followers_to_stop).ordered.and_return []
           worker.perform_leader_duties
         end
 
-        it "should seize the leadership periodically" do
-          worker.should_receive(:seize_leadership).at_least(4).times
-          worker.perform_leader_duties
+        context "as long as there are followers to stop" do
+          before do
+            worker.stub(:followers_to_stop).and_return([1], [2], [3], [4], [])
+            worker.stub(:relax)
+          end
+
+          it "relaxes half of the time to the next iteration" do
+            worker.stub(:until_next_iteration).and_return(6)
+            worker.should_receive(:relax).with(3).exactly(4).times
+            worker.perform_leader_duties
+          end
+
+          it "seizes the leadership periodically" do
+            worker.should_receive(:seize_leadership).at_least(4).times
+            worker.perform_leader_duties
+          end
         end
-      end
 
-      describe "if there are no more followers to stop" do
-        before do
-          worker.stub(:followers_to_stop).and_return([])
+        context "if there are no more followers to stop" do
+          before do
+            worker.stub(:followers_to_stop).and_return([])
+          end
+
+          it "exits" do
+            worker.should_receive(:exit).with(0)
+            worker.perform_leader_duties
+          end
         end
+      end
 
-        it "should relax until next iteration" do
-          worker.stub(:until_next_iteration).and_return(6)
-          worker.should_receive(:relax).with(6).once
-          worker.perform_leader_duties
+      context "when restart is wanted" do
+        before do
+          allow(worker).to receive(:restart_wanted?).and_return false
         end
 
-        it "should exit if restart is wanted" do
-          worker.stub(:restart_wanted?).and_return true
-          worker.should_receive(:exit).with(0)
+        it "does not populate the followers_to_stop list if restart is not wanted" do
+          worker.stub(:restart_wanted?).and_return false
+          worker.should_not_receive(:populate_followers_to_stop)
           worker.perform_leader_duties
         end
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: infopark-politics
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - Mike Perham
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-13 00:00:00.000000000 Z
+date: 2015-03-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -68,33 +68,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.2'
 - !ruby/object:Gem::Dependency
-  name: memcache-client
+  name: dalli
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 1.5.0
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 1.5.0
-- !ruby/object:Gem::Dependency
-  name: starling
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 0.9.8
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 0.9.8
+        version: '0'
 description: ''
 email: tilo@infopark.de
 executables: []