archipelago 0.2.5 → 0.2.6

data/lib/archipelago/pirate.rb

@@ -73,7 +73,6 @@ module Archipelago
     # Archipelago::Treasure:Dubloons in them.
     #
     class Captain < Archipelago::Client::Base
-      attr_reader :chests, :trannies
       #
       # Will look for Archipelago::Treasure::Chests matching <i>:chest_description</i> or CHEST_DESCRIPTION and 
       # Archipelago::Tranny::Managers matching <i>:tranny_description</i> or TRANNY_DESCRIPTION.
@@ -82,29 +81,28 @@ module Archipelago
       # of required classes and modules at the chest.
       #
       def initialize(options = {})
-        super(options)
-
         @transaction = nil
-
-        start_service_updater
-
+        setup(options)
       end
 
       #
       # Sets up this instance with the given +options+.
       #
       def setup(options = {})
-        super(options)
-
-        @chest_description = CHEST_DESCRIPTION.merge(options[:chest_description] || {})
-        @tranny_description = TRANNY_DESCRIPTION.merge(options[:tranny_description] || {})
-
         @chest_eval_files ||= []
         @chest_eval_files += options[:chest_eval_files] || []
-
         @chests_having_evaluated ||= {}
 
         @yar_counter = 0
+
+        options.merge!({
+                         :service_descriptions => {
+                           :chests => CHEST_DESCRIPTION.merge(options[:chest_description] || {}),
+                           :trannies => TRANNY_DESCRIPTION.merge(options[:tranny_description] || {})
+                         }
+                       })
+
+        setup_client(options)
       end
       
       #
@@ -150,11 +148,11 @@ module Archipelago
       # Return a clone of this instance bound to a newly created transaction.
       #
       def begin
-        raise NoTransactionManagerAvailableException.new(self) if @trannies.empty?
+        raise NoTransactionManagerAvailableException.new(self) if self.trannies.empty?
 
         rval = self.clone
         rval.instance_eval do
-          @transaction = @trannies.values.first[:service].begin
+          @transaction = self.trannies.values.first[:service].begin
         end
 
         return rval
@@ -175,21 +173,20 @@ module Archipelago
       #
       # Will abort! the transaction if any exception is raised.
       #
-      def transaction(&block) #:yields: transaction
-        raise NoTransactionManagerAvailableException.new(self) if @trannies.empty?
+      def transaction(&block) #:yields: a clone of this Archipelago::Pirate::Captain with the given transaction as default @transaction.
+        raise NoTransactionManagerAvailableException.new(self) if self.trannies.empty?
 
-        @transaction = @trannies.values.first[:service].begin
+        my_clone = self.begin
+        transa = my_clone.active_transaction
         begin
           begin
-            return yield(@transaction)
+            return yield(my_clone)
           ensure
-            raise CommitFailedException.new(self, @transaction) unless @transaction.commit! == :commited
+            raise CommitFailedException.new(my_clone, transa) unless transa.commit! == :commited
           end
         rescue Exception => e
-          @transaction.abort! unless @transaction.state == :aborted
+          transa.abort! unless transa.state == :aborted
           raise e
-        ensure
-          @transaction = nil
         end
       end
 
@@ -204,17 +201,27 @@ module Archipelago
       #
       # Commit the transaction we are a member of and forget about it.
       #
+      # Returns the new state of the transaction.
+      #
       def commit!
-        @transaction.commit!
-        @transaction = nil
+        begin
+          return @transaction.commit!
+        ensure
+          @transaction = nil
+        end
       end
 
       #
       # Abort the transaction we are a member of and forget about it.
       #
+      # Returns the new state of the transaction.
+      #
       def abort!
-        @transaction.abort!
-        @transaction = nil
+        begin
+          @transaction.abort!
+        ensure
+          @transaction = nil
+        end
       end
 
       #
@@ -233,34 +240,36 @@ module Archipelago
       # for management or rescue!
       #
       def each(callable)
-        @chests.t_each do |service_id, chest|
+        self.chests.t_each do |service_id, chest|
           chest[:service].each(callable)
         end
       end
 
       #
-      # Does an immediate update of our service lists.
+      # Get the chest responsible for +key+.
       #
-      def update_services!
-        @chests = @jockey.lookup(Archipelago::Disco::Query.new(@chest_description), 0)
-        evaluate_in_chests
-        @trannies = @jockey.lookup(Archipelago::Disco::Query.new(@tranny_description), 0)
+      def responsible_chest(key)
+        raise NoRemoteDatabaseAvailableException.new(self) if self.chests.empty?
+
+        return get_least_greater_than(:chests, Digest::SHA1.hexdigest(Marshal.dump(key)), 1).first
+      end
+
+      #
+      # Gets the successor of +service_id+ in the array of services.
+      #
+      def successor(service_id)
+        return nil if self.chests.empty?
+        return get_least_greater_than(:chests, service_id, 1).first
       end
 
       private
 
       #
-      # Get the chest responsible for +key+.
+      # Does an immediate update of our service lists.
       #
-      def responsible_chest(key)
-        raise NoRemoteDatabaseAvailableException.new(self) if @chests.empty?
-
-        key_id = Digest::SHA1.hexdigest(Marshal.dump(key))
-        sorted_chest_ids = @chests.keys.sort
-        sorted_chest_ids.each do |id|
-          return @chests[id] if id > key_id
-        end
-        return @chests[sorted_chest_ids.first]
+      def around_update_services(&block)
+        yield
+        evaluate_in_chests
       end
 
       #
@@ -271,7 +280,7 @@ module Archipelago
         #
         # For all chests
         #
-        @chests.values.each do |chest|
+        self.chests.values.each do |chest|
           #
           # Ensure that this chest has a Set of evaluated files
           #

data/lib/archipelago/sanitation.rb

@@ -0,0 +1,268 @@
+# Archipelago - a distributed computing toolkit for ruby
+# Copyright (C) 2006  Martin Kihlgren <zond at troja dot ath dot cx>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+require 'archipelago/client'
+require 'archipelago/dump'
+require 'monitor'
+require 'drb'
+
+module Archipelago
+
+  module Sanitation
+
+    #
+    # Description of the <i>Dump::Site</i>s that can
+    # store our data.
+    #
+    SITE_DESCRIPTION = {
+      :class => 'Archipelago::Dump::Site'
+    }
+
+    #
+    # The minimum size of data that is reasonable to
+    # recover without too much overhead.
+    #
+    MINIMUM_RECOVERABLE_SIZE = 128
+    #
+    # The minimum number of chunks we want to spread out
+    # to ensure us against server failure.
+    #
+    MINIMUM_NR_OF_CHUNKS = 14
+    #
+    # The minimum ratio of redundancy (used diskspace / data size)
+    # we want to use.
+    #
+    MINIMUM_REDUNDANCY_RATIO = 2
+    #
+    # The extra bytes used by metadata in all check block chunks.
+    #
+    METADATA_OVERHEAD = 8
+
+    #
+    # Raised when you try to do stuff without any remote database
+    # available.
+    #
+    class NoRemoteDatabaseAvailableException < RuntimeError
+      def initialize(officer)
+        super("#{officer} can not find any remote database for you")
+      end
+    end
+
+    #
+    # Raised when you try to fetch data that we find traces of
+    # but are unable to fully restore.
+    #
+    class NotEnoughDataException < RuntimeError
+      def initialize(officer, key)
+        super("#{officer} can not find enough data to restore the value for #{key}")
+      end
+    end
+
+    #
+    # The client class for the redundant Archipelago::Dump network.
+    #
+    # Keeps track of our sites and writes and reads data.
+    #
+    # Also keeps track of all the redundancy work needed, but lets Site do the
+    # work.
+    #
+    class Officer < Archipelago::Client::Base
+      def initialize(options = {})
+        setup(options)
+      end
+
+      def setup(options = {})
+        @minimum_recoverable_size = options[:minimum_recoverable_size] || MINIMUM_RECOVERABLE_SIZE
+        @minimum_nr_of_chunks = options[:minimum_nr_of_chunks] || MINIMUM_NR_OF_CHUNKS
+        @minimum_redundancy_ratio = options[:minimum_redundancy_ratio] || MINIMUM_REDUNDANCY_RATIO
+        @metadata_overhead = options[:metadata_overhead] || METADATA_OVERHEAD
+
+        options.merge!({
+                         :service_descriptions => {
+                           :sites => Archipelago::Disco::Query.new(SITE_DESCRIPTION.merge(options[:site_description] || {}))
+                         }
+                       })
+        setup_client(options)
+      end
+
+      #
+      # Write +key+ and +value+ into the site network with a good level of redundancy etc.
+      #
+      # The key should must be a SHA1 hash.
+      #
+      # Optionally the timestamp +t+ can be provided, but it defaults to now.
+      #
+      def []=(key, value, t = [Time.now.to_i].pack("I"))
+        super_string = Oneliner::SuperString.new(value)
+        nr_of_needed_chunks = @minimum_nr_of_chunks / @minimum_redundancy_ratio
+        chunk_size = (super_string.size / nr_of_needed_chunks) + @metadata_overhead
+        chunk_size = @minimum_recoverable_size / nr_of_needed_chunks if chunk_size < @minimum_recoverable_size / nr_of_needed_chunks
+
+        dump_hash = responsible_sites(key)
+        super_string.encode(8)
+        dump_hash.t_each do |dump_id, nr_of_chunks_needed|
+          self.sites[dump_id][:service].insert!(key, 
+                                                (0...nr_of_chunks_needed).collect do |nr_of_chunks_needed|
+                                                  super_string.encode(chunk_size)
+                                                end,
+                                                t)
+        end
+      end
+
+      def delete!(key)
+        dump_hash = responsible_sites(key)
+        dump_hash.t_each do |dump_id, nr_of_chunks_available|
+          self.sites[dump_id][:service].delete!(key)
+        end
+      end
+
+      #
+      # Get the data for +key+ in the site network.
+      #
+      # The key must be a SHA1 hash.
+      #
+      def [](key)
+        fetch(key).first
+      end
+
+      #
+      # Returns {service_id => nr_of_chunks_it_should_have} 
+      # where sum(nr_of_chunks_it_should_have) == +n+
+      # from self.sites having service_id > +key+.
+      #
+      # Will loop to the beginning if the number of elements run out.
+      #
+      def responsible_sites(key)
+        raise NoRemoteDatabaseAvailableException.new(self) if self.sites.empty?
+
+        rval = {}
+        rval.extend(Archipelago::Current::ThreadedCollection)
+        get_least_greater_than(:sites, key, @minimum_nr_of_chunks).each do |desc|
+          rval[desc[:service_id]] ||= 0
+          rval[desc[:service_id]] += 1
+        end
+        return rval
+      end
+
+      #
+      # Returns whether the key belongs at the service with given id.
+      #
+      def belongs_at?(service_id, key)
+        responsible_sites(key).include?(service_id)
+      end
+
+      #
+      # Ensures that all the dumps responsible for +key+
+      # has chunks for that key without changing the timestamp
+      # for +key+.
+      #
+      def redistribute(key)
+        value, timestamp = fetch(key)
+        #
+        # Even if fetch didnt raise the exception we must, cause this is serious business.
+        #
+        raise NotEnoughDataException.new(self, key) if value.nil?
+        self.[]=(key, value, timestamp)
+      end
+
+      #
+      # Returns whether +service_id1+ and +service_id2+ would come in that
+      # order in the site array if both existed.
+      #
+      def next_to?(service_id1, service_id2)
+        if self.sites.include?(service_id1)
+          return get_least_greater_than(:sites, service_id1, 1).first[:service_id] <= service_id2
+        elsif self.sites.include?(service_id2)
+          return get_greatest_less_than(:sites, service_id2, 1).first[:service_id] >= service_id1
+        else 
+          return false
+        end
+      end
+
+      #
+      # Returns the site after the first one that has keys that
+      # will be stored in the site identified by +service_id+.
+      #
+      def second_master_to(service_id)
+        return get_greatest_less_than(:sites, service_id, @minimum_nr_of_chunks - 1).first[:service_id]
+      end
+
+      #
+      # Gets the predecessor of +service_id+ in the array of services.
+      #
+      def predecessor(service_id)
+        return get_greatest_less_than(:sites, service_id, 1).first[:service_id]
+      end
+
+      private
+
+      #
+      # Returns [the value for +key+, the timestamp for the value].
+      #
+      def fetch(key)
+        dump_hash = responsible_sites(key)
+        dump_ids = dump_hash.keys
+        newest_timestamp = "\000\000\000\000"
+        threads = []
+        rval = Oneliner::SuperString.new
+        lock = Archipelago::Current::Lock.new
+
+        dump_hash.t_each do |dump_id, nr_of_chunks_available|
+          site = self.sites[dump_id][:service]
+          begin
+            chunks = site.fetch(key)
+            lock.mon_synchronize do
+              while chunks.size > 0
+                t, data = chunks.shift
+                if t > newest_timestamp
+                  rval = Oneliner::SuperString.new
+                  newest_timestamp = t
+                end
+                
+                if t == newest_timestamp
+                  rval.decode!(data)
+                end
+              end
+            end
+          rescue DRb::DRbConnError => e
+            self.sites.delete(dump_id)
+          end
+        end
+
+        if rval.decode_done?
+          return [rval.to_s, newest_timestamp]
+        else
+          raise NotEnoughDataException.new(self, key) if newest_timestamp != "\000\000\000\000"
+          return [nil, nil]
+        end
+
+      end
+
+    end
+
+    #
+    # The default Archipelago::Sanitation::Officer that is always available for lookups is Archipelago::Sanitation::CLEANER.
+    #
+    # If you really need to you can customize it by defining CLEANER_OPTIONS before loading sanitation.rb, and if you REALLY
+    # need to you can disable it completely by setting CLEANER_DISABLED to true.
+    #
+    CLEANER = Officer.new(defined?(CLEANER_OPTIONS) ? CLEANER_OPTIONS : {}) unless defined?(CLEANER_DISABLED) && CLEANER_DISABLED
+
+
+  end
+
+end

data/lib/archipelago/tranny.rb

@@ -99,7 +99,7 @@ module Archipelago
 
       #
       # Will use a BerkeleyHashishProvider using tranny_manager.db in the same dir to get its hashes
-      # if not <i>:persistence_provider</i> is given.
+      # if not <i>:persistence_directory</i> is given.
       #
       # Will create Transactions timing out after <i>:transaction_timeout</i> seconds or TRANSACTION_TIMEOUT
       # if none is given.
@@ -107,13 +107,11 @@ module Archipelago
       # Will use Archipelago::Disco::Publishable by calling <b>initialize_publishable</b> with +options+.
       #
       def initialize(options = {})
-        @persistence_provider = options[:persistence_provider] || Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(File.expand_path(__FILE__)).parent.join("tranny_manager.db"))
-
         initialize_publishable(options)
 
         @transaction_timeout = options[:transaction_timeout] || TRANSACTION_TIMEOUT
 
-        @db = @persistence_provider.get_cached_hashish("db")
+        @db = @persistence_provider.get_cached_hashish(:name => "db")
       end
       
       #