archipelago 0.1.1 → 0.2.0

data/README

@@ -20,15 +20,15 @@ placed within the pkg/ directory.
 
 To set up an Archipelago::Tranny::Manager do the following (from scripts/tranny.rb):
 
- :include:scripts/tranny.rb
+ :include:script/tranny.rb
 
 To set up an Archipelago::Treasure::Chest do the following (from scripts/chest.rb):
 
- :include:scripts/chest.rb
+ :include:script/chest.rb
 
 To set up an Archipelago::Pirate::Captain do the following (from scripts/pirate.rb):
 
- :include:scripts/pirate.rb
+ :include:script/pirate.rb
 
 To set up a test environment to play around with, run the following
 commands in a few terminals:

data/lib/archipelago/current.rb

@@ -33,6 +33,75 @@ module Archipelago
 
   module Current
 
+    #
+    # Adds a few threaded methods to the normal ruby collections.
+    #
+    # NB: Will work slightly different than the unthreaded ones in certain circumstances.
+    #
+    module ThreadedCollection
+
+      #
+      # Like each, except calls +block+ within a new thread.
+      #
+      def t_each(&block)
+        threads = []
+        self.each do |args|
+          threads << Thread.new do
+            yield(args)
+          end
+        end
+        threads.each do |thread|
+          thread.join
+        end
+      end
+
+      #
+      # Like collect, except calls +block+ within a new thread.
+      #
+      def t_collect(&block)
+        result = []
+        result.extend(Synchronized)
+        self.t_each do |args|
+          result.synchronize do
+            result << yield(args)
+          end
+        end
+        return result
+      end
+
+      #
+      # Like select, except calls +block+ within a new thread.
+      #
+      def t_select(&block)
+        result = []
+        result.extend(Synchronized)
+        self.t_each do |args|
+          matches = yield(args)
+          result.synchronize do
+            result << args
+          end if matches
+        end
+        return result
+      end
+
+      #
+      # Like reject, except calls +block+ within a new thread.
+      #
+      def t_reject(&block)
+        result = []
+        result.extend(Synchronized)
+        self.t_each do |args|
+          matches = yield(args)
+          result.synchronize do
+            result << args
+          end unless matches
+        end
+        return result
+      end
+
+    end
+
+
     #
     # A module that will allow any class to synchronize over any other
     # object.

data/lib/archipelago/disco.rb

@@ -66,6 +66,11 @@ module Archipelago
     #
     THRIFTY_PUBLISHING = false
     
+    #
+    # The host we are running on.
+    #
+    HOST = "#{Socket::gethostbyname(Socket::gethostname)[0]}" rescue "localhost"
+    
     #
     # A module to simplify publishing services.
     #
@@ -138,7 +143,25 @@ module Archipelago
       # We are always valid if we are able to reply.
       #
       def valid?
-        true
+        if defined?(@valid)
+          @valid
+        else
+          @valid = true
+        end
+      end
+
+      #
+      # Stops the publishing of this Publishable.
+      #
+      def stop!
+        if valid?
+          @valid = false
+          if defined?(Archipelago::Disco::MC) && @jockey == Archipelago::Disco::MC
+            @jockey.unpublish(self.service_id)
+          else
+            @jockey.stop!
+          end
+        end
       end
       
       #
@@ -153,12 +176,7 @@ module Archipelago
         # Stuff that didnt fit in any of the other databases.
         #
         @metadata ||= @persistence_provider.get_hashish("metadata")
-        service_id = @metadata["service_id"]
-        unless service_id
-          host = "#{Socket::gethostbyname(Socket::gethostname)[0]}" rescue "localhost"
-          service_id = @metadata["service_id"] ||= Digest::SHA1.hexdigest("#{host}:#{Time.new.to_f}:#{self.object_id}:#{rand(1 << 32)}")
-        end
-        return service_id
+        return @metadata["service_id"] ||= Digest::SHA1.hexdigest("#{HOST}:#{Time.new.to_f}:#{self.object_id}:#{rand(1 << 32)}").to_s
       end
 
     end
@@ -219,6 +237,16 @@ module Archipelago
     class Query < ServiceDescription
     end
 
+    #
+    # A class used to defined removed services.
+    #
+    class UnPublish
+      attr_reader :service_id
+      def initialize(service_id)
+        @service_id = service_id
+      end
+    end
+
     #
     # A class used to define an existing service.
     #
@@ -249,6 +277,7 @@ module Archipelago
     class ServiceLocker
       attr_reader :hash
       include Archipelago::Current::Synchronized
+      include Archipelago::Current::ThreadedCollection
       def initialize(hash = nil)
         super
         @hash = hash || {}
@@ -274,7 +303,7 @@ module Archipelago
             end
           end
         else
-          super(*args)
+          super(meth, *args, &block)
         end
       end
       #
@@ -325,21 +354,35 @@ module Archipelago
       # or if not given THRIFTY_REPLYING. Otherwise will reply with multicasts.
       #
       def initialize(options = {})
-        @thrifty_caching = options.include?(:thrifty_caching) ? options[:thrifty_caching] : THRIFTY_CACHING
-        @thrifty_replying = options.include?(:thrifty_replying) ? options[:thrifty_replying] : THRIFTY_REPLYING
-        @thrifty_publishing = options.include?(:thrifty_publishing) ? options[:thrifty_publishing] : THRIFTY_PUBLISHING
-        @lookup_timeout = options[:lookup_timeout] || LOOKUP_TIMEOUT
-        @initial_lookup_standoff = options[:initial_lookup_standoff] || INITIAL_LOOKUP_STANDOFF
-
+        @valid = true
         @remote_services = ServiceLocker.new
         @local_services = ServiceLocker.new
         @subscribed_services = Set.new
 
         @incoming = Queue.new
         @outgoing = Queue.new
-        
+
         @new_service_semaphore = MonitorMixin::ConditionVariable.new(Archipelago::Current::Lock.new)
-        
+
+        setup(options)
+
+        start_listener
+        start_unilistener
+        start_shouter
+        start_picker
+        start_validator(options[:validation_interval] || VALIDATION_INTERVAL)
+      end
+
+      #
+      # Sets up this instance according to the given +options+.
+      #
+      def setup(options = {})
+        @thrifty_caching = options.include?(:thrifty_caching) ? options[:thrifty_caching] : THRIFTY_CACHING
+        @thrifty_replying = options.include?(:thrifty_replying) ? options[:thrifty_replying] : THRIFTY_REPLYING
+        @thrifty_publishing = options.include?(:thrifty_publishing) ? options[:thrifty_publishing] : THRIFTY_PUBLISHING
+        @lookup_timeout = options[:lookup_timeout] || LOOKUP_TIMEOUT
+        @initial_lookup_standoff = options[:initial_lookup_standoff] || INITIAL_LOOKUP_STANDOFF
+
         @listener = UDPSocket.new
         @unilistener = UDPSocket.new
         
@@ -371,29 +414,40 @@ module Archipelago
             raise e
           end
         end
-        @unicast_address = "#{Socket::gethostbyname(Socket::gethostname)[0]}:#{this_port}" rescue "localhost:#{this_port}"
+        @unicast_address = "#{HOST}:#{this_port}"
         
         @sender = UDPSocket.new
         @sender.connect(options[:address] || ADDRESS, options[:port] || PORT)
 
         @unisender = UDPSocket.new
+      end
 
-        start_listener
-        start_unilistener
-        start_shouter
-        start_picker
-        start_validator(options[:validation_interval] || VALIDATION_INTERVAL)
+      #
+      # Clears our local and remote services.
+      #
+      def clear!
+        @local_services = ServiceLocker.new
+        @remote_services = ServiceLocker.new
       end
 
       #
       # Stops all the threads in this instance.
       #
       def stop!
-        @listener_thread.kill
-        @unilistener_thread.kill
-        @shouter_thread.kill
-        @picker_thread.kill
-        @validator_thread.kill
+        if @valid
+          @valid = false
+          @local_services.each do |service_id, service_description|
+            self.unpublish(service_id)
+          end
+          @listener_thread.kill
+          @unilistener_thread.kill
+          @validator_thread.kill
+          @picker_thread.kill
+          until @outgoing.empty?
+            sleep(0.01)
+          end
+          @shouter_thread.kill
+        end
       end
 
       #
@@ -443,6 +497,17 @@ module Archipelago
         end
       end
 
+      #
+      # Removes the service with given +service_id+ from the published services.
+      #
+      def unpublish(service_id)
+        @local_services.delete(service_id)
+        @new_service_semaphore.broadcast
+        unless @thrifty_publishing
+          @outgoing << [nil, UnPublish.new(service_id)]
+        end
+      end
+
       private
 
       #
@@ -523,8 +588,8 @@ module Archipelago
       end
 
       #
-      # Start the thread picking incoming Records and Queries and
-      # handling them properly
+      # Start the thread picking incoming Records, Queries and UnPublishes and
+      # handling them properly.
       #
       def start_picker
         @picker_thread = Thread.new do
@@ -544,6 +609,8 @@ module Archipelago
                   @remote_services[data[:service_id]] = data
                   @new_service_semaphore.broadcast
                 end
+              elsif Archipelago::Disco::UnPublish === data
+                @remote_services.delete(data.service_id)
               end
             rescue Exception => e
               puts e
@@ -566,6 +633,12 @@ module Archipelago
 
     end
 
+    #
+    # The default Archipelago::Disco::Jockey that is always available for lookups is Archipelago::Disco::MC.
+    #
+    # If you really need to you can customize it by defining MC_OPTIONS before loading disco.rb, and if you REALLY
+    # need to you can disable it completely by setting MC_DISABLED to true.
+    #
     MC = Jockey.new(defined?(MC_OPTIONS) ? MC_OPTIONS : {}) unless defined?(MC_DISABLED) && MC_DISABLED
 
   end

data/lib/archipelago/hashish.rb

@@ -48,6 +48,13 @@ module Archipelago
         @lock = Archipelago::Current::Lock.new
       end
       #
+      # Close the @content_db and @timestamps_db behind this BerkeleyHashish
+      #
+      def close!
+        @content_db.close
+        @timestamps_db.close
+      end
+      #
       # Returns a deep ( Marshal.load(Marshal.dump(o)) ) clone
       # of the object represented by +key+.
       #
@@ -70,12 +77,16 @@ module Archipelago
       #
       # Insert +value+ under +key+.
       #
+      # Will call <b>value.save_hook(old_value)</b> and send
+      # it a block that does the actual saving if 
+      # <b>value.respond_to?(:save_hook)</b>.
+      #
       def []=(key, value)
         @lock.synchronize_on(key) do
 
           @content[key] = value
 
-          write_to_db(key, Marshal.dump(key), Marshal.dump(value))
+          write_to_db(key, Marshal.dump(key), Marshal.dump(value), value)
 
           return value
 
@@ -85,13 +96,19 @@ module Archipelago
       # Stores whatever is under +key+ if it is not the same as
       # whats in the persistent db.
       #
+      # Will call <b>value.save_hook(old_value)</b> and send
+      # it a block that does the actual saving if 
+      # <b>value.respond_to?(:save_hook)</b> and
+      # a save is actually performed.
+      #
       def store_if_changed(key)
         @lock.synchronize_on(key) do
           
           serialized_key = Marshal.dump(key)
-          serialized_value = Marshal.dump(@content[key])
+          value = @content[key]
+          serialized_value = Marshal.dump(value)
           
-          write_to_db(key, serialized_key, serialized_value) if @content_db[serialized_key] != serialized_value
+          write_to_db(key, serialized_key, serialized_value, value) if @content_db[serialized_key] != serialized_value
 
         end
       end
@@ -115,6 +132,19 @@ module Archipelago
         end
       end
       #
+      # Will do +callable+.call(key, value) for each
+      # key-and-value pair in this Hashish.
+      #
+      # NB: This is totaly thread-unsafe, only do this
+      # for management or rescue!
+      #
+      def each(callable)
+        @content_db.each do |serialized_key, serialized_value|
+          key = Marshal.load(serialized_key)
+          callable.call(key, self.[](key))
+        end
+      end
+      #
       # Delete +key+ and its value and timestamp.
       #
       def delete(key)
@@ -136,7 +166,27 @@ module Archipelago
       # Write +key+, serialized as +serialized_key+ and 
       # +serialized_value+ to the db.
       #
-      def write_to_db(key, serialized_key, serialized_value)
+      # Will call <b>value.save_hook(old_value)</b> and send
+      # it a block that does the actual saving if 
+      # <b>value.respond_to?(:save_hook)</b>.
+      #
+      def write_to_db(key, serialized_key, serialized_value, value)
+        if value.respond_to?(:save_hook)
+          old_serialized_value = @content_db[serialized_key]
+          old_value = old_serialized_value ? Marshal.load(old_serialized_value) : nil
+          value.save_hook(old_value) do
+            do_write_to_db(key, serialized_key, serialized_value)            
+          end
+        else
+          do_write_to_db(key, serialized_key, serialized_value)
+        end
+      end
+
+      #
+      # Actually writes +key+ serialized as +serialized_key+ an
+      # +serialized_value+ to the db. Used by <b>write_to_db</b>.
+      #
+      def do_write_to_db(key, serialized_key, serialized_value)
         now = Time.now
 
         @content_db[serialized_key] = serialized_value
@@ -170,6 +220,8 @@ module Archipelago
       def initialize(env_path)
         env_path.mkpath
         @env = BDB::Env.open(env_path, BDB::CREATE | BDB::INIT_MPOOL)
+        @berkeley_hashishes = []
+        @bdb_dbs = []
       end
       #
       # Returns a cleverly cached (but slightly inefficient)
@@ -177,20 +229,31 @@ module Archipelago
       # using +name+.
       #
       def get_cached_hashish(name)
-        BerkeleyHashish.new(name, @env)
+        hashish = BerkeleyHashish.new(name, @env)
+        @berkeley_hashishes << hashish
+        return hashish
       end
       #
       # Returns a normal hash-like instance using +name+.
       #
       def get_hashish(name)
-        @env.open_db(BDB::HASH, name, nil, BDB::CREATE | BDB::NOMMAP)
+        db = @env.open_db(BDB::HASH, name, nil, BDB::CREATE | BDB::NOMMAP)
+        @bdb_dbs << db
+        return db
       end
       #
-      # Removes the persistent files of this instance.
+      # Closes databases opened by this instance and removes the persistent files.
       #
       def unlink
-        p = Pathname.new(@env.home)
-        p.rmtree if p.exist?
+        @berkeley_hashishes.each do |h|
+          h.close!
+        end
+        @bdb_dbs.each do |d|
+          d.close
+        end
+        home = Pathname.new(@env.home)
+        @env.close
+        home.rmtree if home.exist?
       end
     end
     

data/lib/archipelago/pirate.rb

@@ -78,22 +78,37 @@ module Archipelago
       # of required classes and modules at the chest.
       #
       def initialize(options = {})
-        @treasure_map = defined?(Archipelago::Disco::MC) ? Archipelago::Disco::MC : Archipelago::Disco::Jockey.new(options[:jockey_options] || {})
+        setup(options)
+
+        @transaction = nil
+
+        start_service_updater(options[:initial_service_update_interval] || INITIAL_SERVICE_UPDATE_INTERVAL,
+                              options[:maximum_service_update_interval] || MAXIMUM_SERVICE_UPDATE_INTERVAL)
+
+      end
+
+      #
+      # Sets up this instance with the given +options+.
+      #
+      def setup(options = {})
+        @treasure_map ||= nil
+        if defined?(Archipelago::Disco::MC)
+          @treasure_map.stop! if @treasure_map && @treasure_map != Archipelago::Disco::MC
+          @treasure_map = options.include?(:jockey_options) ? Archipelago::Disco::Jockey.new(options[:jockey_options]) : Archipelago::Disco::MC
+        else
+          @treasure_map.stop! if @treasure_map
+          @treasure_map = Archipelago::Disco::Jockey.new(options[:jockey_options] || {})
+        end
 
         @chest_description = CHEST_DESCRIPTION.merge(options[:chest_description] || {})
         @tranny_description = TRANNY_DESCRIPTION.merge(options[:tranny_description] || {})
-        @jockey_options = options[:jockey_options] || {}
 
-        @chest_eval_files = options[:chest_eval_files] || []
+        @chest_eval_files ||= []
+        @chest_eval_files += options[:chest_eval_files] || []
 
-        @chests_having_evaluated = {}
-
-        start_service_updater(options[:initial_service_update_interval] || INITIAL_SERVICE_UPDATE_INTERVAL,
-                              options[:maximum_service_update_interval] || MAXIMUM_SERVICE_UPDATE_INTERVAL)
+        @chests_having_evaluated ||= {}
 
         @yar_counter = 0
-
-        @transaction = nil
       end
       
       #
@@ -199,10 +214,36 @@ module Archipelago
       end
 
       #
-      # Stops the service update thread for this Pirate.
+      # Stops the service update thread for this Pirate and also unpublishes and/or
+      # stops the Jockey.
       #
       def stop!
         @service_update_thread.kill
+        unless defined?(Archipelago::Disco::MC) && @treasure_map && @treasure_map == Archipelago::Disco::MC
+          @treasure_map.stop!
+        end
+      end
+
+      #
+      # Will do +callable+.call(key, value)
+      # for each key-and-value pair in this database network.
+      #
+      # NB: This is totaly thread-unsafe, only do this
+      # for management or rescue!
+      #
+      def each(callable)
+        @chests.t_each do |service_id, chest|
+          chest[:service].each(callable)
+        end
+      end
+
+      #
+      # Does an immediate update of our service lists.
+      #
+      def update_services!
+        @chests = @treasure_map.lookup(Archipelago::Disco::Query.new(@chest_description), 0)
+        evaluate_in_chests
+        @trannies = @treasure_map.lookup(Archipelago::Disco::Query.new(@tranny_description), 0)
       end
 
       private
@@ -271,9 +312,7 @@ module Archipelago
       # +initial+ and +maximum+ seconds.
       #
       def start_service_updater(initial, maximum)
-        @chests = @treasure_map.lookup(Archipelago::Disco::Query.new(@chest_description), 0)
-        evaluate_in_chests
-        @trannies = @treasure_map.lookup(Archipelago::Disco::Query.new(@tranny_description), 0)
+        update_services!
         @service_update_thread = Thread.start do
           standoff = initial
           loop do
@@ -281,9 +320,7 @@ module Archipelago
               sleep(standoff)
               standoff *= 2
               standoff = maximum if standoff > maximum
-              @chests = @treasure_map.lookup(Archipelago::Disco::Query.new(@chest_description), 0)
-              evaluate_in_chests
-              @trannies = @treasure_map.lookup(Archipelago::Disco::Query.new(@tranny_description), 0)
+              update_services!
             rescue Exception => e
               puts e
               pp e.backtrace
@@ -291,6 +328,7 @@ module Archipelago
           end
         end
       end
+
     end
 
   end