mperham-politics 0.1.0 → 0.2.0

data/History.rdoc

@@ -1,5 +1,9 @@
 = Changelog
 
+== 0.2.0 (2008-10-??)
+
+* Remove BucketWorker based on initial feedback.  Add StaticQueueWorker as a more reliable replacement.
+
 == 0.1.0 (2008-10-07)
 
 * Add BucketWorker and TokenWorker mixins.

data/README.rdoc

@@ -15,9 +15,9 @@ for fault tolerance.  This introduces the problem of coordination between those
 Specifically, how do you keep those processes from stepping on each other's electronic
 toes?  There are several answers:
 
-1. Break the processing into several parts.  Have an individual process "checkout" a part,
-   work on it, and then return the part for processing.  This is a very scalable solution
-   as it allows N workers to work on the same task concurrently.  See the +BucketWorker+ mixin.
+1. Break the processing into N 'buckets'.  Have an individual process fetch a bucket,
+   work on it, and ask for another.  This is a very scalable solution as it allows N workers 
+   to work on different parts of the same task concurrently.  See the +StaticQueueWorker+ mixin.
 1. Elect a leader for a short period of time.  The leader is the process which performs the
    actual processing.  After a length of time, a new leader is elected from the group.  This
    is fault tolerant but not as scalable, as only one process is performing the task at a given
@@ -25,8 +25,13 @@ toes?  There are several answers:
 
 == Dependencies
 
-The BucketWorker mixin uses the Starling queue server as the mechanism to hand out parts.  The
-TokenWorker mixin uses the memcached server as the mechanism to elect a leader.
+StaticQueueWorker mixin
+* memcached - the mechanism to elect a leader amongst a set of peers.
+* DRb - the mechanism to communicate between peers.
+* mDNS - the mechanism to discover peers.
+
+TokenWorker mixin
+* memcached - the mechanism to elect a leader amongst a set of peers.
 
 
 = Author

data/examples/token_worker_example.rb

@@ -0,0 +1,35 @@
+#gem 'mperham-politics'
+require 'politics'
+require 'politics/token_worker'
+
+# Test this example by starting memcached locally and then in two irb sessions, run this:
+#
+#
+# You can then watch as one of them is elected leader.  You can kill the leader and verify
+# the backup process is elected after approximately iteration_length seconds.
+#
+=begin
+ require 'token_worker_example'
+ p = Politics::TokenWorkerExample.new
+ p.start
+=end
+module Politics
+  class TokenWorkerExample
+    include Politics::TokenWorker
+    
+    def initialize
+      register_worker 'token-example', :iteration_length => 10, :servers => memcached_servers
+    end
+    
+    def start
+      process do
+        puts "PID #{$$} processing at #{Time.now}..."
+      end
+    end
+    
+    def memcached_servers
+      ['localhost:11211']
+    end
+
+  end
+end

data/lib/init.rb

@@ -1,4 +1,4 @@
 require 'politics'
 require 'politics/token_worker'
-require 'politics/bucket_worker'
+require 'politics/static_queue_worker'
 require 'politics/discoverable_node'

data/lib/politics/discoverable_node.rb

@@ -42,7 +42,7 @@ module Politics
 		  self.group = group
 		  start_drb
 		  register_with_bonjour(group)
-		  Politics.log "Registered #{self} in group #{group} with RID #{rid}"
+		  Politics::log.info { "Registered #{self} in group #{group} with RID #{rid}" }
 		  sleep 0.5
 		  find_replicas(0)
 		end
@@ -73,7 +73,7 @@ module Politics
         sleep 0.2
         find_replicas(count + 1)
       end
-      puts "Found #{replicas.size} peers: #{replicas.keys.sort.inspect}" if count == 0
+      Politics::log.info { "Found #{replicas.size} peers: #{replicas.keys.sort.inspect}" } if count == 0
       replicas
     end
     

data/lib/politics/static_queue_worker.rb

@@ -0,0 +1,269 @@
+puts 'hello'
+require 'socket'
+require 'ipaddr'
+require 'uri'
+require 'drb'
+
+begin
+require 'net/dns/mdns-sd'
+require 'net/dns/resolv-mdns'
+require 'net/dns/resolv-replace'
+rescue LoadError => e
+  puts "Unable to load net-mdns, please run `sudo gem install net-mdns`: #{e.message}"
+  exit(1)  
+end
+
+begin
+  require 'memcache'
+rescue LoadError => e
+  puts "Unable to load memcache client, please run `sudo gem install memcache-client`: #{e.message}"
+  exit(1)
+end
+
+module Politics
+  
+  # The StaticQueueWorker mixin allows a processing daemon to "lease" or checkout
+  # a portion of a problem space to ensure no other process is processing that same
+  # space at the same time.  The processing space is cut into N "buckets", each of
+  # which is placed in a queue.  Processes then fetch entries from the queue
+  # and process them.  It is up to the application to map the bucket number onto its
+  # specific problem space.
+  #
+  # Note that memcached is used for leader election.  The leader owns the queue during
+  # the iteration period and other peers fetch buckets from the current leader during the
+  # iteration.
+  #
+  # The leader hands out buckets in order.  Once all the buckets have been processed, the
+  # leader returns nil to the processors which causes them to sleep until the end of the
+  # iteration.  Then everyone wakes up, a new leader is elected, and the processing starts
+  # all over again.
+  #
+  # DRb and mDNS are used for peer discovery and communication.
+  #
+  # Example usage:
+  #
+  #  class Analyzer
+  #    include Politics::StaticQueueWorker
+  #    TOTAL_BUCKETS = 16
+  #
+  #    def start
+  #      register_worker(self.class.name, TOTAL_BUCKETS)
+  #      process_bucket do |bucket|
+  #        puts "Analyzing bucket #{bucket} of #{TOTAL_BUCKETS}"
+  #        sleep 5
+  #      end
+  #    end
+  #  end
+  #
+  # Note: process_bucket never returns i.e. this should be the main loop of your processing daemon.
+  #
+  module StaticQueueWorker
+    
+    def self.included(model) #:nodoc:
+      model.class_eval do
+        attr_accessor :group_name, :iteration_length
+      end
+    end
+    
+    # Register this process as able to work on buckets.
+    def register_worker(name, bucket_count, config={})
+      options = { :iteration_length => 60, :servers => ['127.0.0.1:11211'] }
+      options.merge!(config)
+
+      self.group_name = name
+      self.iteration_length = options[:iteration_length]
+      @memcache_client = client_for(Array(options[:servers]))
+
+      @buckets = []
+      @bucket_count = bucket_count
+      initialize_buckets
+
+		  register_with_bonjour
+		  
+		  log.info { "Registered #{self} in group #{group_name} at port #{@port}" }
+    end
+    
+    # Fetch a bucket out of the queue and pass it to the given block to be processed.
+    #
+    # +bucket+::            The bucket number to process, within the range 0...TOTAL_BUCKETS
+    def process_bucket(&block)
+      raise ArgumentError, "process_bucket requires a block!" unless block_given?
+      raise ArgumentError, "You must call register_worker before processing!" unless @memcache_client
+
+      begin
+        nominate
+        if leader?
+          # Drb thread handles leader duties
+          log.info { "#{@uri} has been elected leader" }
+          relax until_next_iteration
+          initialize_buckets          
+        else
+          # Get a bucket from the leader and process it
+          begin
+            bucket_process(*leader.bucket_request, &block)
+          rescue DRb::DRbError => dre
+            log.error { "Error talking to leader: #{dre.message}" }
+            relax until_next_iteration
+          end
+        end
+      end while loop?
+    end
+    
+    def bucket_request
+      if leader?
+        [@buckets.pop, until_next_iteration]
+      else
+        :not_leader
+      end
+    end
+    
+    private
+    
+    def bucket_process(bucket, sleep_time)
+      case bucket
+      when nil
+        # No more buckets to process this iteration
+        log.info { "No more buckets in this iteration, sleeping for #{sleep_time} sec" }
+        sleep sleep_time
+      when :not_leader
+        # Uh oh, race condition?  Invalid any local cache and check again
+        log.warn { "Recv'd NOT_LEADER from peer." }
+        relax 1
+        @leader_uri = nil
+      else
+        log.info { "#{@uri} is processing #{bucket}"}
+        yield bucket
+      end
+    end      
+    
+    def log
+      @logger ||= Logger.new(STDOUT)
+    end
+    
+    def initialize_buckets
+      @buckets.clear
+      @bucket_count.times { |idx| @buckets << idx }
+    end
+
+    def replicas
+      @replicas ||= []
+    end
+    
+    def leader
+      name = leader_uri
+      repl = nil
+      while replicas.empty? or repl == nil
+        repl = replicas.detect { |replica| replica.__drburi == name }
+        unless repl
+          relax 1
+          bonjour_scan do |replica|
+            replicas << replica
+          end
+        end
+      end
+      repl
+    end
+    
+    def until_next_iteration
+      left = iteration_length - (Time.now - @nominated_at)
+      left > 0 ? left : 0
+    end
+    
+    def loop?
+      true
+    end
+    
+    def token
+      "#{group_name}_token"
+    end
+    
+    def cleanup
+      at_exit do
+        @memcache_client.delete(token) if leader?
+      end
+    end
+    
+    def pause_until_expiry(elapsed)
+      pause_time = (iteration_length - elapsed).to_f
+      if pause_time > 0
+        relax(pause_time) 
+      else
+        raise ArgumentError, "Negative iteration time left.  Assuming the worst and exiting... #{iteration_length}/#{elapsed}"
+      end
+    end
+    
+    def relax(time)
+      sleep time
+    end
+    
+    # Nominate ourself as leader by contacting the memcached server
+    # and attempting to add the token with our name attached.
+    def nominate
+      @memcache_client.add(token, @uri, iteration_length)
+      @nominated_at = Time.now
+      @leader_uri = nil
+    end
+
+    def leader_uri
+      @leader_uri ||= @memcache_client.get(token)
+    end
+    
+    # Check to see if we are leader by looking at the process name
+    # associated with the token.
+    def leader?
+      until_next_iteration > 0 && @uri == leader_uri
+    end
+
+    # Easy to mock or monkey-patch if another MemCache client is preferred.
+    def client_for(servers)
+      MemCache.new(servers)
+    end
+
+    def time_for(&block)
+      a = Time.now
+      yield
+      Time.now - a
+    end
+    
+    
+		def register_with_bonjour
+		  server = DRb.start_service(nil, self)
+		  @uri = DRb.uri
+		  @port = URI.parse(DRb.uri).port
+
+		  # Register our DRb server with Bonjour.
+      handle = Net::DNS::MDNSSD.register("#{self.group_name}-#{local_ip}-#{$$}", 
+          "_#{group_name}._tcp", 'local', @port)
+          
+      ['INT', 'TERM'].each { |signal| 
+        trap(signal) do
+          handle.stop
+          server.stop_service
+        end
+      }
+	  end
+		
+		def bonjour_scan
+      Net::DNS::MDNSSD.browse("_#{group_name}._tcp") do |b|
+        Net::DNS::MDNSSD.resolve(b.name, b.type) do |r|
+          drburl = "druby://#{r.target}:#{r.port}"
+          replica = DRbObject.new(nil, drburl)
+          yield replica
+        end
+      end
+	  end
+	  
+    # http://coderrr.wordpress.com/2008/05/28/get-your-local-ip-address/
+    def local_ip
+      orig, Socket.do_not_reverse_lookup = Socket.do_not_reverse_lookup, true # turn off reverse DNS resolution temporarily
+ 
+      UDPSocket.open do |s|
+        s.connect '64.233.187.99', 1
+        IPAddr.new(s.addr.last).to_i
+      end
+    ensure
+      Socket.do_not_reverse_lookup = orig
+    end	  
+    
+  end
+end

data/lib/politics/token_worker.rb

@@ -90,7 +90,7 @@ module Politics
 
         time = 0
         if leader?
-          LOG.info { "I've been elected leader: #{worker_name}" }
+          Politics::log.info { "#{worker_name} elected leader at #{Time.now}" }
           # If we are the master worker, do the work.
           time = time_for do
             result = block.call(*args)
@@ -98,11 +98,16 @@ module Politics
         end
         
         pause_until_expiry(time)
+        reset_state
       end while loop?
     end
 
     private
     
+    def reset_state
+      @leader = nil
+    end
+    
     def verify_registration
       unless self.class.worker_instance
         raise ArgumentError, "Cannot call process without first calling register_worker"
@@ -118,7 +123,7 @@ module Politics
     
     def cleanup
       at_exit do
-        memcache_client.delete(token)
+        memcache_client.delete(token) if leader?
       end
     end
     
@@ -144,8 +149,10 @@ module Politics
     # Check to see if we are leader by looking at the process name
     # associated with the token.
     def leader?
-      master_worker = memcache_client.get(token)
-      worker_name == master_worker
+      @leader ||= begin
+        master_worker = memcache_client.get(token)
+        worker_name == master_worker
+      end
     end
 
     # Easy to mock or monkey-patch if another MemCache client is preferred.
@@ -159,4 +166,4 @@ module Politics
       Time.now - a
     end
   end
-end
+end

data/lib/politics/version.rb

@@ -1,5 +1,5 @@
 module Politics
   module Version
-    STRING = "0.1.0"
+    STRING = "0.2.0"
   end
 end

data/lib/politics.rb

@@ -1,9 +1,12 @@
 require 'logger'
 
 module Politics
-  LOG = Logger.new(STDOUT)
+
+  def self.log=(value)
+    @log = log
+  end
   
-  def self.log(msg)
-    LOG.info(msg)
+  def self.log
+    @log ||= Logger.new(STDOUT)
   end
 end

data/test/static_queue_worker_test.rb

@@ -0,0 +1,42 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+Thread.abort_on_exception = true
+
+class Worker
+  include Politics::StaticQueueWorker
+  def initialize
+    register_worker 'worker', 10, :iteration_length => 10
+  end
+  
+  def start
+    process_bucket do |bucket|
+      sleep 1
+    end
+  end
+end
+
+class StaticQueueWorkerTest < Test::Unit::TestCase
+  
+  context "nodes" do
+    setup do
+      @nodes = []
+      5.times do
+        @nodes << nil
+      end
+    end
+    
+    should "start up" do
+      processes = @nodes.map do
+        fork do
+          ['INT', 'TERM'].each { |signal| 
+            trap(signal) { exit(0) }
+          }
+          Worker.new.start
+        end
+      end
+      sleep 10
+      puts "Terminating"
+      Process.kill('INT', *processes)
+    end
+  end
+end

data/test/test_helper.rb

@@ -14,5 +14,6 @@ rescue LoadError => e
   puts "Please install mocha: `sudo gem install mocha`"
 end
 
+$:.unshift(File.dirname(__FILE__) + '/../lib')
 require File.dirname(__FILE__) + '/../lib/init'
-Politics::LOG.level = Logger::WARN
+#Politics::log.level = Logger::WARN

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: mperham-politics
 version: !ruby/object:Gem::Version 
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors: 
 - Mike Perham
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2008-08-13 00:00:00 -07:00
+date: 2008-10-27 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -19,16 +19,25 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: "0"
+        version: 1.5.0.3
     version: 
 - !ruby/object:Gem::Dependency 
-  name: fiveruns-starling
+  name: starling-starling
   version_requirement: 
   version_requirements: !ruby/object:Gem::Requirement 
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: "0"
+        version: 0.9.8
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: net-mdns
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0.4"
     version: 
 description: Algorithms and Tools for Distributed Computing in Ruby.
 email: mperham@gmail.com
@@ -41,17 +50,18 @@ extra_rdoc_files:
 - History.rdoc
 - LICENSE
 files: 
-- README.rdoc
-- LICENSE
-- History.rdoc
-- Rakefile
 - lib/init.rb
 - lib/politics
-- lib/politics/bucket_worker.rb
 - lib/politics/discoverable_node.rb
+- lib/politics/static_queue_worker.rb
 - lib/politics/token_worker.rb
 - lib/politics/version.rb
 - lib/politics.rb
+- examples/queue_worker_example.rb
+- examples/token_worker_example.rb
+- README.rdoc
+- History.rdoc
+- LICENSE
 has_rdoc: true
 homepage: http://github.com/mperham/politics/
 post_install_message: 
@@ -87,7 +97,6 @@ signing_key:
 specification_version: 2
 summary: Algorithms and Tools for Distributed Computing in Ruby.
 test_files: 
-- test/bucket_worker_test.rb
-- test/political_test.rb
+- test/static_queue_worker_test.rb
 - test/test_helper.rb
 - test/token_worker_test.rb

data/Rakefile

@@ -1,34 +0,0 @@
-require 'echoe'
-
-require File.dirname(__FILE__) << "/lib/politics/version"
-
-Echoe.new 'politics' do |p|
-  p.version = Politics::Version::STRING
-  p.author = "Mike Perham"
-  p.email  = 'mperham@gmail.com'
-  p.project = 'politics'
-  p.summary = "Algorithms and Tools for Distributed Computing in Ruby."
-  p.url = "http://github.com/mperham/politics"
-  p.dependencies = %w(memcache-client)
-  p.development_dependencies = []
-  p.include_rakefile = true
-  p.rubygems_version = nil
-end
-
-
-require 'rake/testtask'
-
-desc "Run tests"
-Rake::TestTask.new do |t|
-  t.libs << ['test', 'lib']
-	t.test_files = FileList['test/*_test.rb']
-end
-
-desc "Create rdoc"
-Rake::RDocTask.new do |rd|
-	rd.main = "README.rdoc"
-	rd.rdoc_files.include("README.rdoc", "History.rdoc", "lib/**/*.rb")
-end
-
-
-task :default => :test

data/lib/politics/bucket_worker.rb

@@ -1,107 +0,0 @@
-begin
-  require 'starling'
-rescue LoadError => e
-  puts "Unable to load starling, please run `sudo gem install starling`: #{e.message}"
-  exit(1)
-end
-
-module Politics
-  
-  # The BucketWorker mixin allows a processing daemon to "lease" or checkout
-  # a portion of a problem space to ensure no other process is processing that same
-  # space at the same time.  The processing space is cut into N "buckets", each of
-  # which is placed in a queue.  Processes then fetch entries from the queue
-  # and process them.  It is up to the application to map the bucket number onto its
-  # specific problem space.
-  #
-  # Note that the Starling queue server is the single point of failure with this
-  # mechanism.  Only you can decide if this is an acceptable tradeoff for your needs.
-  #
-  # Example usage:
-  #
-  #  class Analyzer
-  #    include Politics::BucketWorker
-  #    TOTAL_BUCKETS = 16
-  #
-  #    def start
-  #      register_worker(self.class.name)
-  #      create_buckets(TOTAL_BUCKETS) if master?
-  #      process_bucket do |bucket|
-  #        puts "Analyzing bucket #{bucket} of #{TOTAL_BUCKETS}"
-  #        sleep 5
-  #      end
-  #    end
-  #
-  #    def master?
-  #      # TODO Add your own logic here to denote a 'master' process
-  #      ARGV.include? '-m'
-  #    end
-  #  end
-  #
-  # Note: process_bucket never returns i.e. this should be the main loop of your processing daemon.
-  #
-  module BucketWorker
-    
-    def self.included(model) #:nodoc:
-      model.class_eval do
-        attr_accessor :starling_client, :bucket_count, :queue_name
-      end
-    end
-    
-    # Register this process as able to work on buckets.
-    #
-    # +name+::               The name of the queue to access
-    # +servers+::            The starling server(s) to use, defaults to +['localhost:22122']+
-    def register_worker(name, servers=['localhost:22122'])
-      self.queue_name = name
-      self.starling_client = client_for(Array(servers))
-    end
-    
-    # Create the given number of buckets.  Should ONLY be called by a single worker.
-    # TODO Obviously a major weakness of this algorithm. Is there a cleaner way?
-    #
-    # +bucket_count+::       The number of buckets to create
-    def create_buckets(bucket_count)
-      starling_client.flush(queue_name)
-      bucket_count.times do |count|
-        starling_client.set(queue_name, count)
-      end
-    end
-    
-    # Fetch a bucket out of the queue and pass it to the given block to be processed.
-    # Once processing has completed, it will put the bucket back onto the queue for processing
-    # by a BucketWorker again, possibly immediately, depending on the number of buckets vs
-    # number of workers.
-    #
-    # +bucket+::            The bucket number to process, within the range 0...TOTAL_BUCKETS
-    def process_bucket
-      raise ArgumentError, "process_bucket requires a block!" unless block_given?
-      raise ArgumentError, "You must call register_worker before processing!" unless starling_client
-
-      begin
-        bucket = get_bucket
-        yield bucket
-      ensure
-        push_bucket(bucket)
-      end while loop?
-    end
-    
-    private
-    
-    def get_bucket
-      starling_client.get(queue_name)
-    end
-    
-    def push_bucket(bucket)
-      starling_client.set(queue_name, bucket)
-    end
-    
-    def client_for(servers)
-      Starling.new(servers)
-    end
-    
-    def loop?
-      true
-    end
-  end
-end

data/test/bucket_worker_test.rb

@@ -1,51 +0,0 @@
-require 'test_helper'
-
-class BucketWorkerTest < Test::Unit::TestCase
-  
-  context "bucket workers" do
-    setup do
-      @harness = Class.new
-      @harness.send(:include, Politics::BucketWorker)
-      @harness.any_instance.stubs(:loop?).returns(false)
-      @worker = @harness.new
-    end
-
-    should "have instance property accessors" do
-      assert @worker.bucket_count = 20
-      assert_equal 20, @worker.bucket_count
-    end
-
-    should 'register correctly' do
-      @worker.register_worker('testing')
-      @worker.register_worker('testing', ['localhost:5555,localhost:12121'])
-    end
-    
-    should 'process a bucket' do
-      @worker.register_worker('testing')
-      @worker.starling_client.expects(:get).returns(4)
-      @worker.starling_client.expects(:set).with('testing', 4).returns(nil)
-      processed = false
-      @worker.process_bucket do |bucket|
-        assert_equal 4, bucket
-        processed = true
-      end
-      assert processed
-    end
-
-    should 'not allow processing without block' do
-      assert_raises ArgumentError do
-        @worker.register_worker('hello')
-        @worker.process_bucket
-      end
-    end
-
-    should 'not allow processing without registration' do
-      assert_raises ArgumentError do
-        @worker.process_bucket do
-          fail 'Should not process!'
-        end
-      end
-    end
-
-  end
-end

data/test/political_test.rb

@@ -1,32 +0,0 @@
-require 'rubygems'
-require 'test/unit'
-require 'shoulda'
-require File.dirname(__FILE__) + '/../lib/init'
-
-Thread.abort_on_exception = true
-
-class PoliticalTest < Test::Unit::TestCase
-  
-  context "nodes" do
-    setup do
-      @nodes = []
-      5.times do
-        Object.send(:include, Politics::DiscoverableNode)
-        node = Object.new
-        @nodes << node
-      end
-    end
-    
-    should "start up" do
-      processes = @nodes.map do |node|
-        fork do
-          ['INT', 'TERM'].each { |signal| 
-            trap(signal) { exit(0) }
-          }
-          node.register
-        end
-      end
-      Process.wait
-    end
-  end
-end