archipelago 0.2.5 → 0.2.6

data/README

@@ -4,7 +4,7 @@ It consists of several different parts, that can be used standalone or in conjun
 
 == Dependencies:
 Archipelago::Hashish::BerkeleyHashishProvider:: ruby bdb: http://moulon.inra.fr/ruby/bdb.html
-String:: ruby inline: http://www.zenspider.com/ZSS/Products/RubyInline/
+Archipelago::Client::Base:: rbtree: in this same repository is a patched version, original code is at http://www.geocities.co.jp/SiliconValley-PaloAlto/3388/rbtree/README.html
 
 == Sub packages:
 Archipelago::Disco:: A UDP multicast discovery service useful to find services in your network with a minimum of configuration.

data/TODO

@@ -1,19 +1,4 @@
 
- * Create a failover/redundancy framework
-   * For example: Create a new HashishProvider with built in redundancy, 
-     for example using the Chord project: http://pdos.csail.mit.edu/chord/
-   * Or: Create migration methods that move objects between Chests opon
-     startup and shutdown, and make them keep backups at each others
-     persistence backends.
-   * Or: Create something that stores data the way Chord does (with erasure
-     codes) but doesnt use the same look up mechanism.
-     * Problem: We still have to implement the entire maintenance protocol
-       of Chord (continously checking if our data is safely replicated across
-       the network, continously checking that our data belong with us)
-
- * Replace Raider with some well known near-optimal erasure code, for example
-   Online Codes: http://en.wikipedia.org/wiki/Online_codes
-
  * Make Chest aware about whether transactions have affected it 'for real' ie 
    check whether the instance before the call differs from the instance after 
    the call. Preferably without incurring performance lossage.
@@ -22,11 +7,17 @@
      * This is now done. But it is not yet used to provide intelligence for the
        transaction mechanism. How should it compare dirty state before and after?
 
- * Test the transaction recovery mechanism of Chest.
-
  * Create a memcached-starter that publishes the address to the started memcached
    instance on the Disco network.
 
  * Create a memcached-client that uses Disco instance to find all memcached instances
    in the network and distribute requests among them.
 
+ * Create a file server service that handles really big files within transactions
+   and allows the POSTing and GETing of them via HTTP.
+
+ * Test the ability of Dubloons to reconnect to the proper chest.
+   * If the first chest is disconnected and doesnt reappear.
+   * If the first chest is disconnected and then reappears.
+   * If a new chest that takes responsibility for the Dubloon appears.
+

data/lib/archipelago.rb

@@ -16,15 +16,3 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
 $: << File.dirname(File.expand_path(__FILE__))
-
-require 'archipelago/bitstring'
-require 'archipelago/oneliner'
-require 'archipelago/raider'
-require 'archipelago/disco'
-require 'archipelago/current'
-require 'archipelago/tranny'
-require 'archipelago/treasure'
-require 'archipelago/client'
-require 'archipelago/pirate'
-require 'archipelago/cove'
-require 'archipelago/exxon'

data/lib/archipelago/client.rb

@@ -16,39 +16,65 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
 require 'archipelago/disco'
+require 'rbtree'
 
 module Archipelago
 
   module Client
 
+    #
+    # The initial interval between calling update_services!
+    #
     INITIAL_SERVICE_UPDATE_INTERVAL = 1
+    #
+    # The initial interval will be doubled each time update_services! is called,
+    # but will never be greater than the maximum interval.
+    #
     MAXIMUM_SERVICE_UPDATE_INTERVAL = 60
+    #
+    # The timeout that will be used in the first lookup to ensure that we actually
+    # have any services.
+    #
+    INITIAL_LOOKUP_TIMEOUT = 5
 
     class Base
-      attr_reader :jockey
+      include Archipelago::Disco::Camel
+      attr_reader :jockey, :services, :service_descriptions
+
       #
-      # Initialize an instance using Archipelago::Disco::MC or <i>:jockey</i> if given, 
+      # Initialize a Client using Archipelago::Disco::MC or <i>:jockey</i> if given, 
       # or a new Archipelago::Disco::Jockey if none, that looks for new services 
       # <i>:initial_service_update_interval</i> or INITIAL_SERVICE_UPDATE_INTERVAL,
       # when it starts and never slower than every <i>:maximum_service_update_interval</i> 
       # or MAXIMUM_SERVICE_UPDATE_INTERVAL.
       #
-      def initialize(options = {})
-        setup(options)
+      def setup_client(options = {})
+        setup_jockey(options)
+
+        @initial_service_update_interval = options[:initial_service_update_interval] || INITIAL_SERVICE_UPDATE_INTERVAL
+        @maximum_service_update_interval = options[:maximum_service_update_interval] || MAXIMUM_SERVICE_UPDATE_INTERVAL
+        @service_descriptions = options[:service_descriptions]
+        @initial_lookup_timeout = options[:initial_lookup_timeout] || INITIAL_LOOKUP_TIMEOUT
+        @services = {}
+        @service_descriptions.each do |name, description|
+          t = RBTree.new
+          t.extend(Archipelago::Current::ThreadedCollection)
+          @services[name] = t
+        end
+
+        start_service_updater
+        start_subscriptions
       end
+
       #
-      # Sets up this instance with the given +options+.
+      # Finding our services dynamically.
       #
-      def setup(options = {})
-        @jockey.stop! if defined?(@jockey) && @jockey != Archipelago::Disco::MC
-        if defined?(Archipelago::Disco::MC)
-          @jockey = options[:jockey] || Archipelago::Disco::MC
+      def method_missing(meth, *args)
+        if @services.include?(meth)
+          return @services[meth]
         else
-          @jockey = options[:jockey] || Archipelago::Disco::Jockey.new
+          super
         end
-        
-        @initial_service_update_interval = options[:initial_service_update_interval] || INITIAL_SERVICE_UPDATE_INTERVAL
-        @maximum_service_update_interval = options[:maximum_service_update_interval] || MAXIMUM_SERVICE_UPDATE_INTERVAL
       end
 
       #
@@ -57,26 +83,137 @@ module Archipelago
       #
       def stop!
         @service_update_thread.kill if @service_update_thread
+        stop_subscriptions
         unless defined?(Archipelago::Disco::MC) && @jockey && @jockey == Archipelago::Disco::MC
           @jockey.stop!
         end
       end
 
       #
-      # Override this to do whatever you want to when finding services.
+      # Override this if you want to do something special before or
+      # after calling update_services!
+      #
+      def around_update_services(&block)
+        yield
+      end
+
+      #
+      # Make our @jockey lookup all our services.
       #
-      def update_services!
-        # /moo
+      def update_services!(options = {})
+        timeout = options[:timeout] || 0
+        validate = options[:validate] || false
+        around_update_services do
+          @service_descriptions.each do |name, description|
+            #
+            # This sure sounds inefficient, but hey, listen up:
+            #
+            # * RBTrees are nice and fast when it comes to looking up ordered stuff.
+            # * They are horribly slow in all other ways.
+            #   * As an example: It takes (as of this writing) 10x the time to insert 10k elements in an RBTree
+            #     as it takes to sort 10k elements in an Array.
+            #
+            # This means that using them in Archipelago::Disco::ServiceLocker will be inefficient as hell, since they
+            # are merging and creating new maps all the time. But in here, I expect us to not renew our service lists
+            # more than on average once every MAXIMUM_SERVICE_UPDATE_INTERVAL, so that MAY make it worthwhile to do
+            # the RBTree song and dance in here. Hopefully.
+            #
+            @services[name] = @jockey.lookup(Archipelago::Disco::Query.new(description), timeout)
+            @services[name].convert_to_tree!
+            @services[name].validate! if validate
+          end
+        end
       end
 
       private
 
+      #
+      # Subscribe to our service changes.
+      #
+      def start_subscriptions
+        @service_descriptions.each do |name, description|
+          @jockey.subscribe(:found, 
+                            Archipelago::Disco::Query.new(description), 
+                            object_id) do |record|
+            @services[name][record[:service_id]] = record
+          end
+          @jockey.subscribe(:lost,  
+                            Archipelago::Disco::Query.new(description), 
+                            object_id) do |record|
+            @services[name].delete(record[:service_id])
+          end
+        end
+      end
+
+      #
+      # Stop our subscriptions to our service changes.
+      #
+      def stop_subscriptions
+        @service_descriptions.each do |name, description|
+          @jockey.unsubscribe(:found, 
+                              Archipelago::Disco::Query.new(description), 
+                              object_id)
+          @jockey.subscribe(:lost,  
+                            Archipelago::Disco::Query.new(description), 
+                            object_id)
+        end
+      end
+
+      #
+      # Gets the +n+ smallest keys from services denoted +service_name+ that
+      # are greater than +o+. 
+      #
+      # Will loop to the beginning if the number of elements run out.
+      #
+      def get_least_greater_than(service_name, o, n)
+        rval = []
+        self.send(service_name).each(o) do |id, desc|
+          rval << desc if id > o
+          break if rval.size == n
+        end
+        return rval if rval.size == n
+        fill(self.send(service_name), rval, :each, n)
+       end
+    
+      #
+      # Gets the +n+ values for the largest keys from +hash+ that
+      # are less than +o+.
+      #
+      # Will loop to the end if the number of elements run out.
+      #
+      def get_greatest_less_than(service_name, o, n)
+        rval = []
+        self.send(service_name).reverse_each(o) do |id, desc|
+          rval << desc if id < o
+          break if rval.size == n
+        end
+        return rval if rval.size == n
+        fill(self.send(service_name), rval, :reverse_each, n)
+      end
+
+      #
+      # Will fill +receiver+ up with the return values of +collection+.send(+meth+)
+      # until +receiver+ is of size +n+.
+      #
+      def fill(collection, receiver, meth, n)
+        unless collection.empty?
+          while receiver.size < n
+            collection.send(meth) do |id, desc|
+              receiver << desc
+              break if receiver.size == n
+            end
+          end
+        end
+        return receiver
+      end
+
       #
       # Start a thread looking up existing chests between every
       # +initial+ and +maximum+ seconds.
       #
       def start_service_updater
-        update_services!
+        update_services!(:timeout => @initial_lookup_timeout, :validate => true)
+        @service_update_thread.kill if defined?(@service_update_thread)
         @service_update_thread = Thread.start do
           standoff = @initial_service_update_interval
           loop do

data/lib/archipelago/current.rb

@@ -213,7 +213,7 @@ module Archipelago
         sync_initialize
       end
     end
-    
+
     #
     # Just a convenience empty class with locking functionality.
     #

data/lib/archipelago/disco.rb

@@ -20,9 +20,11 @@ require 'thread'
 require 'ipaddr'
 require 'pp'
 require 'archipelago/current'
+require 'archipelago/hashish'
 require 'drb'
 require 'set'
 require 'digest/sha1'
+require 'forwardable'
 
 module Archipelago
 
@@ -53,7 +55,7 @@ module Archipelago
     # Default pause between trying to validate all services we
     # know about.
     #
-    VALIDATION_INTERVAL = 60
+    VALIDATION_INTERVAL = 30
     #
     # Only save stuff that we KNOW we want.
     #
@@ -71,9 +73,48 @@ module Archipelago
     # The host we are running on.
     #
     HOST = "#{Socket::gethostbyname(Socket::gethostname)[0]}" rescue "localhost"
+
+    #
+    # Anything that has a @jockey that is an Archipelago::Disco::Jockey
+    # can include this for simplicity.
+    #
+    module Camel
+      private
+      #
+      # Setup our @jockey as a Archipelago::Disco::Jockey with given options.
+      #
+      # It will first stop any @jockey we currently have that is NOT the global Archipelago::Disco::MC.
+      #
+      # If +jockey_options+ or +jockey+ are given it will always use a new or given Archipelago::Disco::Jockey,
+      # otherwise it will try to use the global Archipelago::Disco::Jockey instead.
+      #
+      def setup_jockey(options = {})
+        @jockey.stop! if defined?(@jockey) && (!defined?(Archipelago::Disco::MC) || @jockey != Archipelago::Disco::MC)
+
+        @jockey_options ||= {}
+        jockey_options = @jockey_options.merge(options[:jockey_options] || {})
+
+        if options[:jockey]
+          @jockey = options[:jockey]
+          unless jockey_options.empty?
+            @jockey.setup(jockey_options)
+          end
+        else
+          if jockey_options.empty?
+            if defined?(Archipelago::Disco::MC)
+              @jockey = Archipelago::Disco::MC
+            else
+              @jockey = Archipelago::Disco::Jockey.new
+            end
+          else
+            @jockey = Archipelago::Disco::Jockey.new(jockey_options)
+          end
+        end
+      end
+    end
     
     #
-    # A module to simplify publishing services.
+    # A module to simplify publishing of services.
     #
     # If you include it you can use the publish! method
     # at your convenience.
@@ -88,7 +129,7 @@ module Archipelago
     # define <b>@persistence_provider</b> before you call <b>initialize_publishable</b>.
     #
     module Publishable
-
+      include Camel
       #
       # Also add the ClassMethods to +base+.
       #
@@ -113,21 +154,6 @@ module Archipelago
         DRbObject.new(self)._dump(dummy_param)
       end
 
-      #
-      # Will initialize this instance with @service_description and @jockey_options
-      # and merge these with the optionally given <i>:service_description</i> and
-      # <i>:jockey_options</i>.
-      #
-      def initialize_publishable(options = {})
-        @service_description = {
-          :service_id => service_id,
-          :validator => self,
-          :service => self,
-          :class => self.class.name
-        }.merge(options[:service_description] || {})
-        @jockey_options = options[:jockey_options] || {}
-      end
-
       #
       # Create an Archipelago::Disco::Jockey for this instance using @jockey_options
       # or optionally given <i>:jockey_options</i>.
@@ -136,8 +162,10 @@ module Archipelago
       # <i>:service_description</i>.
       #
       def publish!(options = {})
-        @jockey ||= defined?(Archipelago::Disco::MC) ? Archipelago::Disco::MC : Archipelago::Disco::Jockey.new(@jockey_options.merge(options[:jockey_options] || {}))
-        @jockey.publish(Archipelago::Disco::Record.new(@service_description.merge(options[:service_description] || {})))
+        setup_jockey(options)
+        around_publish do
+          @jockey.publish(Archipelago::Disco::Record.new(@service_description.merge(options[:service_description] || {})))
+        end
       end
 
       #
@@ -154,30 +182,104 @@ module Archipelago
       #
       # Stops the publishing of this Publishable.
       #
-      def stop!
-        if valid?
-          @valid = false
-          if defined?(Archipelago::Disco::MC) && @jockey == Archipelago::Disco::MC
-            @jockey.unpublish(self.service_id)
-          else
-            @jockey.stop!
+      def unpublish!
+        if defined?(@jockey)
+          if valid?
+            around_unpublish do
+              @valid = false
+              if defined?(Archipelago::Disco::MC) && @jockey == Archipelago::Disco::MC
+                @jockey.unpublish(self.service_id)
+              else
+                @jockey.stop!
+              end
+            end
           end
         end
       end
-      
+
+      #
+      # Closes the persistence backend of this Publishable.
+      #
+      def close!
+        unpublish!
+        around_close do
+          @persistence_provider.close!
+        end
+      end
+
       #
       # Returns our semi-unique id so that we can be found again.
       #
       def service_id
+        return @service_id ||= @metadata["service_id"]
+      end
+
+      private
+
+      #
+      # Will initialize this instance with @service_description and @jockey_options
+      # and merge these with the optionally given <i>:service_description</i> and
+      # <i>:jockey_options</i>.
+      #
+      def initialize_publishable(options = {})
         #
         # The provider of happy magic persistent hashes of different kinds.
         #
-        @persistence_provider ||= Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(File.expand_path(__FILE__)).parent.join(self.class.name + ".db"))
+        @persistence_provider ||= Archipelago::Hashish::BerkeleyHashishProvider.new(options[:persistence_directory] || Pathname.new(File.expand_path(__FILE__)).parent.join(self.class.name + ".db"))
         #
         # Stuff that didnt fit in any of the other databases.
         #
         @metadata ||= @persistence_provider.get_hashish("metadata")
-        return @metadata["service_id"] ||= Digest::SHA1.hexdigest("#{HOST}:#{Time.new.to_f}:#{self.object_id}:#{rand(1 << 32)}").to_s
+        #
+        # Our service_description that is supposed to define and describe
+        # us in the discovery network.
+        #
+        @service_description = {
+          :service_id => service_id || Digest::SHA1.hexdigest("#{HOST}:#{Time.new.to_f}:#{self.object_id}:#{rand(1 << 32)}").to_s,
+          :validator => self,
+          :service => self,
+          :class => self.class.name
+        }.merge(options[:service_description] || {})
+        #
+        # Our service_id that is supposed to be unique and persistent.
+        #
+        @metadata["service_id"] = @service_description[:service_id]
+        #
+        # Setup our Archipelago::Disco::Jockey.
+        #
+        @jockey_options = options[:jockey_options] || {}
+      end
+
+      #
+      # Override this if you want to do something magical before or after you
+      # get published.
+      #
+      def around_publish(&block)
+        yield
+      end
+
+      #
+      # Override this if you want to do something magical before or after you
+      # get unpublished.
+      #
+      def around_stop(&block)
+        yield
+      end
+
+      #
+      # Override this if you want to do something magical before or after you
+      # get closed.
+      #
+      def around_close(&block)
+        yield
+      end
+
+      #
+      # Override this if you want to do something magical before or after you
+      # get unpublished.
+      #
+      def around_unpublish(&block)
+        yield
       end
 
     end
@@ -196,8 +298,10 @@ module Archipelago
     # A Hash-like description of a service.
     #
     class ServiceDescription
+      extend Forwardable
       IGNORABLE_ATTRIBUTES = Set[:unicast_reply]
       attr_reader :attributes
+      def_delegators :@attributes, :[], :[]=, :each
       #
       # Initialize this service description with a hash
       # that describes its attributes.
@@ -206,14 +310,10 @@ module Archipelago
         @attributes = hash
       end
       #
-      # Forwards as much as possible to our Hash.
+      # Returns whether our @attributes are equal to that of +o+.
       #
-      def method_missing(meth, *args, &block)
-        if @attributes.respond_to?(meth)
-          @attributes.send(meth, *args, &block)
-        else
-          super(*args)
-        end
+      def eql?(o)
+        ServiceDescription === o && @attributes == o.attributes
       end
       #
       # Returns whether this ServiceDescription matches the given +match+.
@@ -272,40 +372,70 @@ module Archipelago
     # A container of services.
     #
     class ServiceLocker
+      extend Forwardable
       attr_reader :hash
-      include Archipelago::Current::Synchronized
       include Archipelago::Current::ThreadedCollection
-      def initialize(hash = nil)
-        super
-        @hash = hash || {}
+      def_delegators :@hash, :[], :size, :empty?, :values, :keys, :include?
+      def initialize(options = {})
+        @hash = options[:hash] || {}
+        @hash.extend(Archipelago::Current::ThreadedCollection)
+        @jockey = options[:jockey]
+      end
+      def each(*args, &block)
+        clone = @hash.clone
+        clone.extend(Archipelago::Current::ThreadedCollection)
+        clone.each(*args, &block)
+      end
+      def reverse_each(*args, &block)
+        clone = @hash.clone
+        clone.extend(Archipelago::Current::ThreadedCollection)
+        clone.reverse_each(*args, &block)
       end
       #
-      # Merge this locker with another.
+      # Set +key+ to +value+.
       #
-      def merge(sd)
-        rval = @hash.clone
-        rval.merge!(sd.hash)
-        ServiceLocker.new(rval)
+      def []=(key, value)
+        existed_before = @hash.include?(key)
+        @hash[key] = value
+        # Notifying AFTER the fact to avoid loops.
+        if @jockey && !existed_before
+          @jockey.instance_eval do notify_subscribers(:found, value) end 
+        end
       end
       #
-      # Forwards as much as possible to our Hash.
+      # Delete +key+.
       #
-      def method_missing(meth, *args, &block)
-        if @hash.respond_to?(meth)
-          synchronize do
-            @hash.send(meth, *args, &block)
-          end
-        else
-          super(meth, *args, &block)
+      def delete(key)
+        value = @hash.delete(key)
+        # Notifying AFTER the fact to avoid loops.
+        @jockey.instance_eval do notify_subscribers(:lost, value) end if @jockey && value
+      end
+      #
+      # Will make this ServiceLocker convert its Hash into an RBTree.
+      #
+      def convert_to_tree!
+        t = RBTree.new
+        @hash.each do |k,v|
+          t[k] = v
         end
+        @hash = t
+      end
+      #
+      # Merge this locker with another.
+      #
+      def merge(sd)
+        rval = @hash.merge(sd.hash)
+        ServiceLocker.new(:hash => rval)
       end
       #
       # Find all containing services matching +match+.
       #
       def get_services(match)
         rval = ServiceLocker.new
-        self.each do |service_id, service_data|
-          rval[service_id] = service_data if service_data.matches?(match) && service_data.valid?
+        self.t_each do |service_id, service_data|
+          if service_data.matches?(match)
+            rval[service_id] = service_data 
+          end
         end
         return rval
       end
@@ -313,9 +443,12 @@ module Archipelago
       # Remove all non-valid services.
       #
       def validate!
-        self.clone.each do |service_id, service_data|
-          self.delete(service_id) unless service_data.valid?
+        self.t_each do |service_id, service_data|
+          unless service_data.valid?
+            self.delete(service_id) 
+          end
         end
+        return self
       end
     end
 
@@ -348,22 +481,38 @@ module Archipelago
       #
       def initialize(options = {})
         @valid = true
-        @remote_services = ServiceLocker.new
-        @local_services = ServiceLocker.new
+        @remote_services = ServiceLocker.new(:jockey => self)
+        @local_services = ServiceLocker.new(:jockey => self)
         @subscribed_services = Set.new
 
         @incoming = Queue.new
         @outgoing = Queue.new
 
         @new_service_semaphore = MonitorMixin::ConditionVariable.new(Archipelago::Current::Lock.new)
+        @service_change_subscribers_by_event_type = {:found => {}, :lost => {}}
+        
+        @validation_interval = options[:validation_interval] || VALIDATION_INTERVAL
 
         setup(options)
+        
+        start!
+      end
 
-        start_listener
-        start_unilistener
-        start_shouter
-        start_picker
-        start_validator(options[:validation_interval] || VALIDATION_INTERVAL)
+      #
+      # Will listen for +event_type+s matching the Query +match+
+      # and do +block+.call with the matching Record.
+      #
+      # Recognized +event_types+: :found, :lost
+      #
+      def subscribe(event_type, match, identity, &block)
+        @service_change_subscribers_by_event_type[event_type][[match, identity]] = block
+      end
+
+      #
+      # Will stop listening for +event_type+ and +match+.
+      #
+      def unsubscribe(event_type, match, identity)
+        @service_change_subscribers_by_event_type[event_type].delete([match, identity])
       end
 
       #
@@ -421,12 +570,12 @@ module Archipelago
       # Clears our local and remote services.
       #
       def clear!
-        @local_services = ServiceLocker.new
-        @remote_services = ServiceLocker.new
+        @local_services = ServiceLocker.new(:jockey => self)
+        @remote_services = ServiceLocker.new(:jockey => self)
       end
 
       #
-      # Stops all the threads in this instance.
+      # Stops all the threads and close all sockets in this instance.
       #
       def stop!
         if @valid
@@ -434,14 +583,24 @@ module Archipelago
           @local_services.each do |service_id, service_description|
             self.unpublish(service_id)
           end
+
           @listener_thread.kill
           @unilistener_thread.kill
-          @validator_thread.kill
+          until @incoming.empty?
+            sleep(0.01)
+          end
+          @listener.close
+          @unilistener.close
           @picker_thread.kill
+
           until @outgoing.empty?
             sleep(0.01)
           end
           @shouter_thread.kill
+          @sender.close
+          @unisender.close
+
+          @validator_thread.kill
         end
       end
 
@@ -459,12 +618,12 @@ module Archipelago
 
         @outgoing << [nil, match]
         known_services = @remote_services.get_services(match).merge(@local_services.get_services(match))
-        return known_services unless known_services.empty?
+        return known_services if timeout == 0 || !known_services.empty?
 
-        @new_service_semaphore.wait(standoff)
+        t = Time.new
+        @new_service_semaphore.wait([standoff, timeout].min)
         standoff *= 2
         
-        t = Time.new
         while Time.new < t + timeout
           known_services = @remote_services.get_services(match).merge(@local_services.get_services(match))
           return known_services unless known_services.empty?
@@ -503,8 +662,45 @@ module Archipelago
         end
       end
 
+      #
+      # Validate all our known services.
+      #
+      def validate!
+        @local_services.validate!
+        @remote_services.validate!
+      end
+
       private
 
+      #
+      # Start all our threads.
+      #
+      def start!(options = {})
+        start_listener
+        start_unilistener
+        start_shouter
+        start_picker
+        start_validator(options[:validation_interval] || @validation_interval)
+      end
+
+      #
+      # Will notify all subscribers to +event_type+ looking for +record+.
+      #
+      def notify_subscribers(event_type, record)
+        @service_change_subscribers_by_event_type[event_type].clone.each do |query_and_identity, proc|
+          query = query_and_identity.first
+          Thread.new do 
+            begin
+              proc.call(record) 
+            rescue Exception => e
+              @service_change_subscribers_by_event_type[event_type].delete(query_and_identity)
+              puts e
+              pp e.backtrace
+            end
+          end if record.matches?(query)
+        end
+      end
+
       #
       # Start the validating thread.
       #
@@ -512,8 +708,7 @@ module Archipelago
         @validator_thread = Thread.new do
           loop do
             begin
-              @local_services.validate!
-              @remote_services.validate!
+              validate!
               sleep(validation_interval)
             rescue Exception => e
               puts e