infopark-politics 0.2.15 → 0.3.0

data/lib/politics/static_queue_worker.rb

@@ -2,6 +2,7 @@ require 'socket'
 require 'ipaddr'
 require 'uri'
 require 'drb'
+require 'set'
 
 begin
 require 'net/dns/mdns-sd'
@@ -76,6 +77,7 @@ module Politics
 
       @buckets = []
       @bucket_count = bucket_count
+      @followers_to_stop = Set.new
 
       register_with_bonjour
       log.progname = uri
@@ -89,7 +91,9 @@ module Politics
     def process_bucket(&block)
       log.debug "start bucket processing"
       raise ArgumentError, "process_bucket requires a block!" unless block_given?
-      raise ArgumentError, "You must call register_worker before processing!" unless @memcache_client
+      unless @memcache_client
+        raise ArgumentError, "You must call register_worker before processing!"
+      end
 
       begin
         begin
@@ -144,20 +148,36 @@ module Politics
         log.debug { "relaxes half the time until next iteration" }
         relax(until_next_iteration / 2)
         as_dictator do
-          update_buckets
+          update_buckets unless restart_wanted?
         end
       end
-      handle_leadership_over
+      as_dictator() {populate_followers_to_stop} if restart_wanted?
+      # keeping leader state as long as there are followers to stop
+      while !followers_to_stop.empty? do
+        relax(until_next_iteration / 2)
+        seize_leadership
+      end
+      exit 0 if restart_wanted?
       relax until_next_iteration
     end
 
-    def handle_leadership_over
+    def populate_followers_to_stop
+      @followers_to_stop.replace(find_workers)
+    end
+
+    def followers_to_stop
+      @followers_to_stop.select {|u| DRbObject.new(nil, u).alive? rescue DRb::DRbConnError && false}
     end
 
     def bucket_request(requestor_uri)
       if leader?
         log.debug "delivering bucket request"
-        next_bucket requestor_uri
+        bucket_spec = next_bucket(requestor_uri)
+        if !bucket_spec[0] && @followers_to_stop.include?(requestor_uri)
+          bucket_spec = [:stop, 0]
+          @followers_to_stop.delete(requestor_uri)
+        end
+        bucket_spec
       else
         log.debug "received request for bucket but am not leader - delivering :not_leader"
         [:not_leader, 0]
@@ -191,8 +211,22 @@ module Politics
       DRbObject.new(nil, leader_uri)
     end
 
+    def find_workers
+      workers = []
+      browser = Net::DNS::MDNSSD.browse(mdns_type) do |reply|
+        workers << reply.name unless reply.name == uri
+      end
+      sleep 5
+      browser.stop
+      workers
+    end
+
     private
 
+    def restart_wanted?
+      @memcache_client.get(restart_flag)
+    end
+
     def bucket_process(bucket, sleep_time)
       case bucket
       when nil
@@ -204,6 +238,9 @@ module Politics
         log.warn { "Recv'd NOT_LEADER from peer." }
         relax 1
         @leader_uri = nil
+      when :stop
+        log.info "Received STOP from leader … exiting."
+        exit 0
       else
         log.info { "processing #{bucket}" }
         yield bucket
@@ -230,8 +267,15 @@ module Politics
       "#{group_name}_token"
     end
 
+    def restart_flag
+      "#{group_name}_restart"
+    end
+
     def cleanup
-      @memcache_client.delete(token) if leader?
+      if leader?
+        @memcache_client.delete(token)
+        @memcache_client.delete(restart_flag)
+      end
     end
 
     def pause_until_expiry(elapsed)
@@ -239,7 +283,8 @@ module Politics
       if pause_time > 0
         relax(pause_time)
       else
-        raise ArgumentError, "Negative iteration time left.  Assuming the worst and exiting... #{iteration_length}/#{elapsed}"
+        raise ArgumentError, "Negative iteration time left. " +
+            "Assuming the worst and exiting… #{iteration_length}/#{elapsed}"
       end
     end
 
@@ -277,38 +322,26 @@ module Politics
       Time.now - a
     end
 
+    def mdns_type
+      "_#{group_name}._tcp"
+    end
+
     def register_with_bonjour
       server = DRb.start_service(nil, self)
       @uri = DRb.uri
       @port = URI.parse(DRb.uri).port
 
       # Register our DRb server with Bonjour.
-      name = "#{group_name}-#{local_ip}-#{$$}"
-      type = "_#{group_name}._tcp"
+      name = @uri
       domain = "local"
-      log.debug "register service #{name} of type #{type} within domain #{domain} at port #{@port}"
-      handle = Net::DNS::MDNSSD.register(name, type, domain, @port) do |reply|
+      log.debug "register service #{name} of type #{mdns_type} within domain #{domain} at port #{@port}"
+      handle = Net::DNS::MDNSSD.register(name, mdns_type, domain, @port) do |reply|
         log.debug "registered as #{reply.fullname}"
+        if reply.name != name
+          log.debug "Registered name #{reply.name} differs from requested name #{name} … exiting."
+          handle.stop
+        end
       end
-
-      # ['INT', 'TERM'].each { |signal|
-      #   trap(signal) do
-      #     handle.stop
-      #     server.stop_service
-      #   end
-      # }
-    end
-
-    # http://coderrr.wordpress.com/2008/05/28/get-your-local-ip-address/
-    def local_ip
-      orig, Socket.do_not_reverse_lookup = Socket.do_not_reverse_lookup, true # turn off reverse DNS resolution temporarily
-
-      UDPSocket.open do |s|
-        s.connect '64.233.187.99', 1
-        IPAddr.new(s.addr.last).to_i
-      end
-    ensure
-      Socket.do_not_reverse_lookup = orig
     end
   end
 end

data/spec/static_queue_worker_spec.rb

@@ -124,14 +124,22 @@ describe Worker do
         before do
           @worker.stub!(:leader?).and_return false
           @worker.stub!(:leader_uri).and_return "the leader"
+          @worker.stub!(:leader).and_return(@leader = mock('leader'))
+          @leader.stub!(:bucket_request).and_return([1, 0])
         end
 
         it "should get the bucket to process from the leader at every iteration" do
-          @worker.should_receive(:leader).exactly(4).times.and_return(leader = mock('leader'))
-          leader.should_receive(:bucket_request).with(@worker.uri).exactly(4).times.
+          @worker.should_receive(:leader).exactly(4).times.and_return @leader
+          @leader.should_receive(:bucket_request).with(@worker.uri).exactly(4).times.
               and_return([1, 2])
           @worker.start
         end
+
+        it "should exit on :stop bucket" do
+          @leader.should_receive(:bucket_request).ordered.once.and_return([:stop, 0])
+          @worker.should_receive(:exit).with(0).ordered
+          @worker.start
+        end
       end
     end
   end
@@ -146,6 +154,27 @@ describe Worker do
         @worker.should_receive(:next_bucket).with("requestor").and_return "the bucket"
         @worker.bucket_request("requestor").should == "the bucket"
       end
+
+      describe "when no buckets are left" do
+        before do
+          @worker.stub!(:find_workers).and_return(%w(1 2 3))
+          @worker.populate_followers_to_stop
+          DRbObject.stub(:new).and_return(mock('o', :alive? => true))
+        end
+
+        it "should deliver the :stop bucket if requestor is in followers_to_stop list" do
+          @worker.bucket_request("1").should == [:stop, 0]
+        end
+
+        it "should not deliver the :stop bucket if requestor is not in followers_to_stop list" do
+          @worker.bucket_request("requestor")[0].should be_nil
+        end
+
+        it "should remove the requestor from the followers_to_stop list" do
+          @worker.bucket_request("2")
+          @worker.followers_to_stop.should == %w(1 3)
+        end
+      end
     end
 
     describe "as follower" do
@@ -159,12 +188,27 @@ describe Worker do
     end
   end
 
+  describe "when determining if restart is wanted" do
+    it "should return true if the restart flag is set in memcache" do
+      @@memcache_client.should_receive(:get).with('worker_restart').and_return true
+      @worker.should be_restart_wanted
+    end
+
+    it "should return false if the restart flag is not set in memcache" do
+      @@memcache_client.should_receive(:get).with('worker_restart').and_return false
+      @worker.should_not be_restart_wanted
+      @@memcache_client.should_receive(:get).with('worker_restart').and_return nil
+      @worker.should_not be_restart_wanted
+    end
+  end
+
   describe "when performing leader duties" do
     before do
       @worker.stub!(:until_next_iteration).and_return 0
       @worker.stub!(:leader?).and_return true
       @worker.stub!(:dictatorship_length).and_return 666
       @worker.stub!(:iteration_length).and_return 5
+      @worker.stub!(:find_workers).and_return []
     end
 
     it "should initialize buckets as dictator" do
@@ -195,6 +239,20 @@ describe Worker do
         @worker.should_receive(:seize_leadership).at_least(4).times
         @worker.perform_leader_duties
       end
+
+      it "should seize the leadership periodically even if restart is wanted" do
+        @worker.stub!(:restart_wanted?).and_return true
+        @worker.stub!(:exit)
+        @worker.should_receive(:seize_leadership).at_least(4).times
+        @worker.perform_leader_duties
+      end
+
+      it "should not update buckets if restart is wanted" do
+        @worker.stub!(:restart_wanted?).and_return true
+        @worker.stub!(:exit)
+        @worker.should_not_receive(:update_buckets)
+        @worker.perform_leader_duties
+      end
     end
 
     describe "if there are no more buckets" do
@@ -202,17 +260,55 @@ describe Worker do
         @worker.stub!(:buckets).and_return([])
       end
 
-      it "should relax until next iteration" do
-        @worker.stub!(:until_next_iteration).and_return(6)
-        @worker.should_receive(:relax).with(6).once
+      it "should populate the followers_to_stop list before evaluating it if restart is wanted" do
+        @worker.stub!(:restart_wanted?).and_return true
+        @worker.stub!(:exit)
+        @worker.should_receive(:populate_followers_to_stop).ordered.once
+        @worker.should_receive(:followers_to_stop).ordered.and_return []
         @worker.perform_leader_duties
       end
 
-      it "should call leadership over hook before relaxing" do
-        @worker.should_receive(:handle_leadership_over).ordered
-        @worker.should_receive(:relax).ordered
+      it "should not populate the followers_to_stop list if restart is not wanted" do
+        @worker.stub!(:restart_wanted?).and_return false
+        @worker.should_not_receive(:populate_followers_to_stop)
         @worker.perform_leader_duties
       end
+
+      describe "as long as there are followers to stop" do
+        before do
+          @worker.stub!(:followers_to_stop).and_return([1], [2], [3], [4], [])
+          @worker.stub!(:relax)
+        end
+
+        it "should relax half of the time to the next iteration" do
+          @worker.stub!(:until_next_iteration).and_return(6)
+          @worker.should_receive(:relax).with(3).exactly(4).times
+          @worker.perform_leader_duties
+        end
+
+        it "should seize the leadership periodically" do
+          @worker.should_receive(:seize_leadership).at_least(4).times
+          @worker.perform_leader_duties
+        end
+      end
+
+      describe "if there are no more followers to stop" do
+        before do
+          @worker.stub!(:followers_to_stop).and_return([])
+        end
+
+        it "should relax until next iteration" do
+          @worker.stub!(:until_next_iteration).and_return(6)
+          @worker.should_receive(:relax).with(6).once
+          @worker.perform_leader_duties
+        end
+
+        it "should exit if restart is wanted" do
+          @worker.stub!(:restart_wanted?).and_return true
+          @worker.should_receive(:exit).with(0)
+          @worker.perform_leader_duties
+        end
+      end
     end
   end
 
@@ -324,16 +420,22 @@ describe Worker do
 
   describe "when cleaning up" do
     before do
-      @worker.stub!(:token).and_return('dcc-group')
+      @worker.stub!(:group_name).and_return('the group')
     end
 
     describe "as leader" do
       before do
         @worker.stub!(:leader?).and_return true
+        @@memcache_client.stub!(:delete)
+      end
+
+      it "should remove the leadership token from memcache" do
+        @@memcache_client.should_receive(:delete).with('the group_token')
+        @worker.send(:cleanup)
       end
 
-      it "should remove the leadership token" do
-        @@memcache_client.should_receive(:delete).with('dcc-group')
+      it "should remove the restart wanted flag from memcache" do
+        @@memcache_client.should_receive(:delete).with('the group_restart')
         @worker.send(:cleanup)
       end
     end
@@ -343,10 +445,75 @@ describe Worker do
         @worker.stub!(:leader?).and_return false
       end
 
-      it "should not remove the leadership token" do
+      it "should not remove anything from memcache" do
         @@memcache_client.should_not_receive(:delete)
         @worker.send(:cleanup)
       end
     end
   end
+
+  describe "when finding workers" do
+    before do
+      Net::DNS::MDNSSD.stub(:browse).
+          and_yield(mock('response1', :name => 'w1')).
+          and_yield(mock('response2', :name => 'w2')).
+          and_yield(mock('response3', :name => 'w3')).
+          and_yield(mock('response4', :name => 'w4')).
+          and_return(@browser = mock('browser', :stop => nil))
+      @worker.stub!(:sleep)
+    end
+
+    it "should browse mdns group and return workers found" do
+      @worker.find_workers.should == %w(w1 w2 w3 w4)
+    end
+
+    it "should not add itself to the result list" do
+      @worker.stub!(:uri).and_return('w3')
+      @worker.find_workers.should_not include('w3')
+    end
+
+    it "should stop browser thread after five seconds" do
+      @worker.should_receive(:sleep).with(5).ordered
+      @browser.should_receive(:stop)
+      @worker.find_workers
+    end
+  end
+
+  describe "when populating followers_to_stop" do
+    before do
+      @worker.stub(:find_workers).and_return(%w(a b c))
+      DRbObject.stub(:new).and_return(mock('o', :alive? => true))
+    end
+
+    it "should add all visible workers" do
+      @worker.populate_followers_to_stop
+      @worker.followers_to_stop.should == %w(a b c)
+    end
+  end
+
+  describe "when delivering followers_to_stop" do
+    before do
+      @worker.stub(:find_workers).and_return(%w(a b c))
+      @worker.populate_followers_to_stop
+      DRbObject.stub(:new).and_return(mock('o', :alive? => true))
+    end
+
+    it "should return the actual followers_to_stop" do
+      @worker.followers_to_stop.should == %w(a b c)
+    end
+
+    it "should not deliver entries that are not reachable at the moment" do
+      DRbObject.stub!(:new).with(nil, 'a').and_return(mock('o', :alive? => false))
+      DRbObject.stub!(:new).with(nil, 'b').and_return(x = mock('o'))
+      x.stub(:alive?).and_raise DRb::DRbConnError.new('nix da')
+      @worker.followers_to_stop.should == %w(c)
+    end
+
+    it "should not remove unreachable entries from the list - maybe they reappear" do
+      DRbObject.stub!(:new).with(nil, 'a').and_return(mock('o', :alive? => false))
+      @worker.followers_to_stop.should == %w(b c)
+      DRbObject.stub!(:new).with(nil, 'a').and_return(mock('o', :alive? => true))
+      @worker.followers_to_stop.should == %w(a b c)
+    end
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: infopark-politics
 version: !ruby/object:Gem::Version 
-  version: 0.2.15
+  version: 0.3.0
 platform: ruby
 authors: 
 - Mike Perham