archipelago 0.2.2 → 0.2.3

data/README

@@ -13,27 +13,33 @@ Archipelago::Hashish:: A hash-like tool that provides transparent persistence.
 Archipelago::Treasure:: A distributed object database where the objects never leave the database, instead you do your operations upon references to the objects. It has support for serializably isolated transactions with optimistic concurrency control using Archipelago::Tranny or any transaction manager with similar semantics.
 Archipelago::Pirate:: A client tool to allocate Archipelago::Treasure::Chests for different keys and act like an almost normal local Hash for providing distributed and scaleable object database facilities to any ruby application.
 
-== Examples:
+== Usage:
+
+To use archipelago in the simplest and most obvious way, just run the script/services.rb script.
 
-To build a ruby gem from these sources do 'rake gem'. The gem will be
-placed within the pkg/ directory.
+You can run this on any number of machines in your local network, and they will all communicate through their own
+discovery services.
 
-To set up an Archipelago::Tranny::Manager do the following (from scripts/tranny.rb):
+Then you instantiate an Archipelago::Pirate::Captain anywhere in the network. This instance can be used basically as
+a normal Hash - most everything will be transparently hidden from you.
 
- :include:script/tranny.rb
+Everything you put in this instance will be persistently stored in the network.
 
-To set up an Archipelago::Treasure::Chest do the following (from scripts/chest.rb):
+== Examples:
 
- :include:script/chest.rb
+To run a transaction manager and a database server, just do the following (from script/services.rb):
 
-To set up an Archipelago::Pirate::Captain do the following (from scripts/pirate.rb):
+ :include:script/services.rb
+
+To set up an Archipelago::Pirate::Captain do the following (from script/pirate.rb):
 
  :include:script/pirate.rb
 
-To set up a test environment to play around with, run the following
+Or you can run script/console, which starts an irb and loads script/pirate.rb.
+
+So, to set up a test environment to play around with in a few simple steps, run the following
 commands in a few terminals:
 
-* script/tranny.rb /tmp/tranny
-* script/chest.rb /tmp/chest1
-* script/chest.rb /tmp/chest2
-* script/console
+ script/services.rb /tmp/services1
+ script/services.rb /tmp/services2
+ script/console

data/TODO

@@ -6,6 +6,9 @@
    check whether the instance before the call differs from the instance after 
    the call. Preferably without incurring performance lossage.
 
+ * Make Dubloon proxy targets able to provide a dirty_state method that decides
+   whether they are to be considered dirty, clean or auto-select.
+
  * Test the transaction recovery mechanism of Chest.
 
  * Make Archipelago::Treasure::Dubloons work after an Archipelago::Treasure::Chest

data/lib/archipelago/current.rb

@@ -36,47 +36,58 @@ module Archipelago
     #
     # Adds a few threaded methods to the normal ruby collections.
     #
+    # The only method your class has to implement to use this module is <b>each(&block)</b>.
+    #
     # NB: Will work slightly different than the unthreaded ones in certain circumstances.
     #
     module ThreadedCollection
 
       #
-      # Like each, except calls +block+ within a new thread.
+      # Like each, except calls +block+ or +callable+ within a new thread.
       #
-      def t_each(&block)
+      def t_each(callable = nil, &block)
+        raise "You have to provide either callable or block" if callable.nil? && block.nil?
+
         threads = []
+
         self.each do |args|
           threads << Thread.new do
-            yield(args)
+            call_helper(callable, args, &block)
           end
         end
+
         threads.each do |thread|
           thread.join
         end
       end
 
       #
-      # Like collect, except calls +block+ within a new thread.
+      # Like collect, except calls +block+ or +callable+ within a new thread.
       #
-      def t_collect(&block)
+      def t_collect(callable = nil, &block)
+        raise "You have to provide either callable or block" if callable.nil? && block.nil?
+
         result = []
         result.extend(Synchronized)
         self.t_each do |args|
+          new_value = call_helper(callable, args, &block)
           result.synchronize do
-            result << yield(args)
+            result << new_value
           end
         end
         return result
       end
 
       #
-      # Like select, except calls +block+ within a new thread.
+      # Like select, except calls +block+ or +callable+ within a new thread.
       #
-      def t_select(&block)
+      def t_select(callable = nil, &block)
+        raise "You have to provide either callable or block" if callable.nil? && block.nil?
+
         result = []
         result.extend(Synchronized)
         self.t_each do |args|
-          matches = yield(args)
+          matches = call_helper(callable, args, &block)
           result.synchronize do
             result << args
           end if matches
@@ -85,13 +96,15 @@ module Archipelago
       end
 
       #
-      # Like reject, except calls +block+ within a new thread.
+      # Like reject, except calls +block+ or +callable+ within a new thread.
       #
-      def t_reject(&block)
+      def t_reject(callable = nil, &block)
+        raise "You have to provide either callable or block" if callable.nil? && block.nil?
+        
         result = []
         result.extend(Synchronized)
         self.t_each do |args|
-          matches = yield(args)
+          matches = call_helper(callable, args, &block)
           result.synchronize do
             result << args
           end unless matches
@@ -99,6 +112,20 @@ module Archipelago
         return result
       end
 
+      private
+
+      def call_helper(o = nil, args = nil, &block)
+        if o
+          if Array === args
+            return o.call(*args)
+          else
+            return o.call(args)
+          end
+        else
+          return yield(args)
+        end
+      end
+
     end
 
 

data/lib/archipelago/disco.rb

@@ -209,11 +209,7 @@ module Archipelago
       #
       def method_missing(meth, *args, &block)
         if @attributes.respond_to?(meth)
-          if block
-            @attributes.send(meth, *args, &block)
-          else
-            @attributes.send(meth, *args)
-          end
+          @attributes.send(meth, *args, &block)
         else
           super(*args)
         end
@@ -296,11 +292,7 @@ module Archipelago
       def method_missing(meth, *args, &block)
         if @hash.respond_to?(meth)
           synchronize do
-            if block
-              @hash.send(meth, *args, &block)
-            else
-              @hash.send(meth, *args)
-            end
+            @hash.send(meth, *args, &block)
           end
         else
           super(meth, *args, &block)

data/lib/archipelago/hashish.rb

@@ -62,8 +62,19 @@ module Archipelago
         return Marshal.load(@content_db[Marshal.dump(key)])
       end
       #
+      # Returns true if this BerkeleyHashish include +key+.
+      #
+      def include?(key)
+        @content.include?(key) || !@content_db[Marshal.dump(key)].nil?
+      end
+      #
       # Simply get the value for the +key+.
       #
+      # Will call <b>value.load_hook</b> and send it
+      # a block that does the actuall insertion of the value
+      # into the live hash if <b>value.respond_to?(:load_hook)</b>
+      # if the value didnt exist in the live hash yet.
+      #
       def [](key)
         @lock.synchronize_on(key) do
 
@@ -94,7 +105,8 @@ module Archipelago
       end
       #
       # Stores whatever is under +key+ if it is not the same as
-      # whats in the persistent db.
+      # whats in the persistent db - if the +key+ actually has
+      # a representation in the db.
       #
       # Will call <b>value.save_hook(old_value)</b> and send
       # it a block that does the actual saving if 
@@ -105,11 +117,13 @@ module Archipelago
         @lock.synchronize_on(key) do
           
           serialized_key = Marshal.dump(key)
+
           value = @content[key]
           serialized_value = Marshal.dump(value)
-          
-          write_to_db(key, serialized_key, serialized_value, value) if @content_db[serialized_key] != serialized_value
+          old_serialized_value = @content_db[serialized_key]
 
+          write_to_db(key, serialized_key, serialized_value, value) if old_serialized_value && old_serialized_value != serialized_value
+          
         end
       end
       #
@@ -198,13 +212,23 @@ module Archipelago
       # Read +key+ from db and if it is found
       # put it in the cache Hash.
       #
+      # Will call <b>value.load_hook</b> and send it
+      # a block that does the actuall insertion of the value
+      # into the live hash if <b>value.respond_to?(:load_hook)</b>.
+      #
       def get_from_db(key)
         serialized_key = Marshal.dump(key)
         serialized_value = @content_db[serialized_key]
         return nil unless serialized_value
         
         value = Marshal.load(serialized_value)
-        @content[key] = value
+        if value.respond_to?(:load_hook)
+          value.load_hook do
+            @content[key] = value
+          end
+        else
+          @content[key] = value
+        end
         return value
       end
     end
@@ -242,15 +266,21 @@ module Archipelago
         return db
       end
       #
-      # Closes databases opened by this instance and removes the persistent files.
+      # Closes databases opened by this instance.
       #
-      def unlink
+      def close!
         @berkeley_hashishes.each do |h|
           h.close!
         end
         @bdb_dbs.each do |d|
           d.close
         end
+      end
+      #
+      # Closes databases opened by this instance and removes the persistent files.
+      #
+      def unlink!
+        close!
         home = Pathname.new(@env.home)
         @env.close
         home.rmtree if home.exist?

data/lib/archipelago/pirate.rb

@@ -51,6 +51,15 @@ module Archipelago
       end
     end
 
+    #
+    # Raised when the transaction block failed to commit the transaction in the end.
+    #
+    class CommitFailedException < RuntimeError
+      def initialize(pirate, transaction)
+        super("#{pirate} failed to commit #{transaction}")
+      end
+    end
+
     INITIAL_SERVICE_UPDATE_INTERVAL = 1
     MAXIMUM_SERVICE_UPDATE_INTERVAL = 60
     CHEST_DESCRIPTION = {
@@ -111,6 +120,13 @@ module Archipelago
         @yar_counter = 0
       end
       
+      #
+      # Returns true if this Captain includes the given +key+, optionally within a +transaction+.
+      #
+      def include?(key, transaction = nil)
+        responsible_chest(key)[:service].include?(key, transaction || @transaction)
+      end
+      
       #
       # Get a value from the distributed database network using a +key+,
       # optionally within a +transaction+.
@@ -157,6 +173,13 @@ module Archipelago
         return rval
       end
 
+      #
+      # Returns our active transaction, if any.
+      #
+      def active_transaction
+        @transaction
+      end
+
       #
       # Execute +block+ within a transaction.
       #
@@ -171,11 +194,10 @@ module Archipelago
         @transaction = @trannies.values.first[:service].begin
         begin
           yield(@transaction)
-          @transaction.commit! if @transaction.state == :active
+          raise CommitFailedException.new(self, @transaction) unless @transaction.commit! == :commited
         rescue Exception => e
           @transaction.abort! unless @transaction.state == :aborted
-          puts e
-          pp e.backtrace
+          raise e
         ensure
           @transaction = nil
         end
@@ -254,7 +276,7 @@ module Archipelago
       def responsible_chest(key)
         raise NoRemoteDatabaseAvailableException.new(self) if @chests.empty?
 
-        key_id = Digest::SHA1.new(Marshal.dump(key)).to_s
+        key_id = Digest::SHA1.hexdigest(Marshal.dump(key))
         sorted_chest_ids = @chests.keys.sort
         sorted_chest_ids.each do |id|
           return @chests[id] if id > key_id

data/lib/archipelago/tranny.rb

@@ -218,6 +218,12 @@ module Archipelago
         end
       end
       #
+      # Implemented to allow comparison between proxies.
+      #
+      def ==(o)
+        eql?(o)
+      end
+      #
       # Forwards everything to our Transaction and remembers
       # returnvalue if necessary.
       #

data/lib/archipelago/treasure.rb

@@ -94,6 +94,19 @@ module Archipelago
     #
     # A proxy to something in the chest.
     #
+    # This will do a very efficient masquerade as the object it proxies. When asked
+    # for its class or any other attribute it will forward the query to the proxied object.
+    #
+    # It can also be a part of a transaction, either because it was fetched from the chest
+    # within a transaction, or because it is the return value of a Dubloon#join call.
+    #
+    # In this case all forwarded methods will also be within the same transaction, and
+    # any change to the proxied object will be inside that transaction.
+    #
+    # If the proxied object itself needs to handle transaction semantics it can implement
+    # the <b>with_transaction(transaction, &block)</b> method, which will wrap the
+    # method call itself within the home Chest of the proxied object.
+    #
     class Dubloon
       #
       # Remove all methods so that we look like our target.
@@ -328,6 +341,20 @@ module Archipelago
         end
       end
 
+      #
+      # Returns true if this Chest includes the given +key+, optionally within a +transaction+.
+      #
+      def include?(key, transaction = nil)
+        join!(transaction)
+
+        if transaction
+          return @snapshot_by_transaction[transaction].include?(key)
+        else
+          return @db.include?(key)
+        end
+        
+      end
+
       #
       # Return the contents of this chest using a given +key+ and +transaction+.
       #
@@ -402,6 +429,10 @@ module Archipelago
       # Call an instance +method+ on whatever this chest holds at +key+
       # with any +transaction+ and +args+.
       #
+      # If a +transaction+ is provided <b>and</b> the value for the +key+
+      # <b>respond_to?(:with_transaction)</b> then the actual method call
+      # will be wrapped within the block sent to <b>with_transaction(transaction, &block)</b>.
+      #
       def call_instance_method(key, method, transaction, *arguments, &block)
         if transaction
           return call_with_transaction(key, method, transaction, *arguments, &block)
@@ -602,7 +633,10 @@ module Archipelago
       end
 
       #
-      # Call a method within a transaction.
+      # Call a +method+ within a +transaction+.
+      #
+      # If the object we want to run the +method+ on <b>respond_to?(:with_transaction)</b>
+      # then we will execute the actual +method+ within a block sent to the <b>with_transaction</b> method.
       #
       def call_with_transaction(key, method, transaction, *arguments, &block)
         assert_transaction(transaction)
@@ -617,7 +651,15 @@ module Archipelago
         raise UnknownObjectException.new(self, key, transaction) unless instance
 
         begin
-          return execute(instance, method, *arguments, &block)
+          if instance.respond_to?(:with_transaction)
+            return_value = nil
+            instance.with_transaction(transaction) do
+              return_value = instance.send(method, *arguments, &block)
+            end
+            return return_value
+          else
+            return instance.send(method, *arguments, &block)
+          end
         ensure
           #
           # Make sure we remember when this object was last changed according
@@ -630,17 +672,6 @@ module Archipelago
         end
       end
 
-      #
-      # Execute +m+ with arguments +a+ and block +b+ on +o+.
-      #
-      def execute(o, m, *a, &b)
-        if b
-          return o.send(m, *a, &b)
-        else
-          return o.send(m, *a)
-        end
-      end
-
       #
       # Call a method outside any transaction (ie inside a transaction of its own).
       #
@@ -650,7 +681,7 @@ module Archipelago
         raise UnknownObjectException(self, key) unless instance
 
         begin
-          return execute(instance, method, *arguments, &block)
+          return instance.send(method, *arguments, &block)
         ensure
           @db.store_if_changed(key)
         end

data/tests/current_test.rb

@@ -1,6 +1,22 @@
 
 require File.join(File.dirname(__FILE__), 'test_helper')
 
+class HashCollector
+  attr_accessor :epa
+  def call(k,v)
+    @epa ||= {}
+    @epa[k] = v
+  end
+end
+
+class ArrayCollector
+  attr_accessor :epa
+  def call(e)
+    @epa ||= []
+    @epa << e
+  end
+end
+
 class CurrentTest < Test::Unit::TestCase
 
   def test_synchronized
@@ -24,6 +40,24 @@ class CurrentTest < Test::Unit::TestCase
     end
   end
 
+  def test_lock_on
+    t = true
+
+    a = "hej"
+    a.extend(Archipelago::Current::Synchronized)
+    a.lock_on("epa")
+    Thread.new do
+      a.lock_on("epa")
+      t = false
+    end
+    Thread.pass
+    assert(t)
+    a.unlock_on("epa")
+    assert_within(0.5) do
+      !t
+    end
+  end
+
   def test_threaded_collection
     a = Array(10)
     a.extend(Archipelago::Current::ThreadedCollection)
@@ -51,6 +85,16 @@ class CurrentTest < Test::Unit::TestCase
     b = {}
     a.t_each do |k,v| b[k] = v end
     assert_equal(b,a)
+
+    c = HashCollector.new
+    a.t_each(c)
+    assert_equal(a, c.epa)
+
+    a = [1,2,3,4,5,6,7]
+    a.extend(Archipelago::Current::ThreadedCollection)
+    c = ArrayCollector.new
+    a.t_each(c)
+    assert_equal(a.sort, c.epa.sort)
   end
 
 end

data/tests/pirate_test.rb

@@ -25,14 +25,36 @@ class PirateTest < Test::Unit::TestCase
   def teardown
     @p.stop!
     @c.stop!
-    @c.persistence_provider.unlink
+    @c.persistence_provider.unlink!
     @c2.stop!
-    @c2.persistence_provider.unlink
+    @c2.persistence_provider.unlink!
     @tm.stop!
-    @tm.persistence_provider.unlink
+    @tm.persistence_provider.unlink!
     DRb.stop_service
   end
 
+  def test_include
+    assert(!@p.include?("blabla"))
+    @p["blabla"] = "kissobjas"
+    assert(@p.include?("blabla"))
+    t = @tm.begin
+    assert(!@p.include?("hehu"))
+    assert(!@p.include?("hehu", t))
+    @p["hehu", t] = "brunte"
+    assert(!@p.include?("hehu"))
+    assert(@p.include?("hehu", t))
+    t.commit!
+    assert(@p.include?("hehu"))
+    t = @tm.begin
+    assert(!@p.include?("hehu2"))
+    assert(!@p.include?("hehu2", t))
+    @p["hehu2", t] = "brunte"
+    assert(!@p.include?("hehu2"))
+    assert(@p.include?("hehu2", t))
+    t.abort!
+    assert(!@p.include?("hehu2"))
+  end
+
   def test_each
     @p["oj"] = "bla"
     @p["brunt"] = "ja"
@@ -89,6 +111,40 @@ class PirateTest < Test::Unit::TestCase
     assert_equal(s1, s2)
   end
 
+  def test_transaction
+    p2 = Archipelago::Pirate::Captain.new(:chest_description => {:class => "TestChest"}, 
+                                          :tranny_description => {:class => "TestManager"})
+    assert_within(10) do
+      p2.chests.keys.sort == [@c.service_id, @c2.service_id].sort
+    end
+    assert_within(10) do
+      p2.trannies.keys == [@tm.service_id]
+    end
+    trans = nil
+    assert_raise(Archipelago::Pirate::CommitFailedException) do
+      @p.transaction do |trans|
+        assert_equal(trans.transaction_id, @p.active_transaction.transaction_id)
+        @p["hehu"] = "haha"
+        assert(!p2.include?("hehu"))
+        assert(!p2.include?("hehu", nil))
+        assert_equal(nil, p2.active_transaction)
+        trans2 = nil
+        p2.transaction do |trans2|
+          assert_equal(trans2.transaction_id, p2.active_transaction.transaction_id)
+          p2["hehu"] = "hoj"
+          assert_equal("haha", @p["hehu"])
+          assert_equal("hoj", @p["hehu", trans2])
+        end
+        assert_equal(:commited, trans2.state)
+        assert_equal(:active, trans.state)
+        assert_equal("hoj", p2["hehu"])
+        assert_equal("haha", p2["hehu", trans])
+      end
+    end
+    assert_equal(:aborted, trans.state)
+    assert_equal("hoj", @p["hehu"])
+  end
+
   def test_write_read_transaction
     $T = true
     @p["hej"] = "haha"

data/tests/tranny_test.rb

@@ -9,7 +9,7 @@ class TrannyTest < Test::Unit::TestCase
   end
 
   def teardown
-    @tm.persistence_provider.unlink
+    @tm.persistence_provider.unlink!
     DRb.stop_service
   end
 

data/tests/treasure_benchmark.rb

@@ -9,7 +9,7 @@ class TreasureBenchmark < Test::Unit::TestCase
   end
 
   def teardown
-    @c.persistence_provider.unlink
+    @c.persistence_provider.unlink!
     DRb.stop_service
   end
 

data/tests/treasure_test.rb

@@ -7,6 +7,17 @@ class A < String
     yield
     $BURKMAT2 += 1
   end
+  def load_hook(&block)
+    $BURKMAT3 += 1
+    yield
+    $BURKMAT4 += 1
+  end
+  def with_transaction(transaction, &block)
+    $BURKMAT7 = transaction
+    $BURKMAT5 += 1
+    yield
+    $BURKMAT6 += 1
+  end
 end
 
 class TreasureTest < Test::Unit::TestCase
@@ -18,15 +29,29 @@ class TreasureTest < Test::Unit::TestCase
     @tm = TestManager.new(:persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(__FILE__).parent.join("tranny1.db")))
     $BURKMAT = 0
     $BURKMAT2 = 0
+    $BURKMAT3 = 0
+    $BURKMAT4 = 0
+    $BURKMAT5 = 0
+    $BURKMAT6 = 0
+    $BURKMAT7 = nil
   end
 
   def teardown
-    @c.persistence_provider.unlink
-    @c2.persistence_provider.unlink
-    @tm.persistence_provider.unlink
+    @c.persistence_provider.unlink!
+    @c2.persistence_provider.unlink!
+    @tm.persistence_provider.unlink!
     DRb.stop_service
   end
 
+  def test_with_transaction
+    t = @tm.begin
+    @c["hehu"] = A.new("kiss")
+    @c["hehu", t].upcase!
+    assert_equal(1, $BURKMAT5)
+    assert_equal(1, $BURKMAT6)
+    assert(!$BURKMAT7.nil?)
+  end
+
   def test_each
     @c["oj"] = "bla"
     @c["brunt"] = "ja"
@@ -37,6 +62,37 @@ class TreasureTest < Test::Unit::TestCase
     assert_equal({"oj" => "bla", "brunt" => "ja"}, h)
   end
 
+  def test_include
+    assert(!@c.include?("blabla"))
+    @c["blabla"] = "kissobjas"
+    assert(@c.include?("blabla"))
+    t = @tm.begin
+    assert(!@c.include?("hehu"))
+    assert(!@c.include?("hehu", t))
+    @c["hehu", t] = "brunte"
+    assert(!@c.include?("hehu"))
+    assert(@c.include?("hehu", t))
+    t.commit!
+    assert(@c.include?("hehu"))
+    t = @tm.begin
+    assert(!@c.include?("hehu2"))
+    assert(!@c.include?("hehu2", t))
+    @c["hehu2", t] = "brunte"
+    assert(!@c.include?("hehu2"))
+    assert(@c.include?("hehu2", t))
+    t.abort!
+    assert(!@c.include?("hehu2"))
+  end
+
+  def test_around_load
+    @c["brunis"] = A.new("hirr")
+    @c.persistence_provider.close!
+    @c = TestChest.new(:persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(__FILE__).parent.join("chest.db")))
+    a = @c["brunis"]
+    assert_equal(1, $BURKMAT3)
+    assert_equal(1, $BURKMAT4)
+  end
+
   def test_around_save
     s = A.new("hehu")
     @c["oj"] = s
@@ -61,7 +117,9 @@ class TreasureTest < Test::Unit::TestCase
 
   def test_store_load_update
     s = "hehu"
+    assert(!@c.include?("oj"))
     @c["oj"] = "hehu"
+    assert(@c.include?("oj"))
     oj1 = @c["oj"]
     assert_equal(oj1, @c["oj"])
     assert_equal("hehu", oj1)

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.8.11
 specification_version: 1
 name: archipelago
 version: !ruby/object:Gem::Version 
-  version: 0.2.2
-date: 2006-11-28 00:00:00 +01:00
+  version: 0.2.3
+date: 2006-11-29 00:00:00 +01:00
 summary: A set of tools for distributed computing in ruby.
 require_paths: 
 - lib