archipelago 0.2.5 → 0.2.6

data/tests/dump_test.rb

@@ -0,0 +1,71 @@
+
+require File.join(File.dirname(__FILE__), 'test_helper')
+
+class DumpTest < Test::Unit::TestCase
+
+  def setup
+    @c = Archipelago::Sanitation::Officer.new(:initial_lookup_timeout => 0)
+    @d = Archipelago::Dump::Site.new(:officer => @c,
+                                     :persistence_directory => Pathname.new(__FILE__).parent.join("site.db"))
+  end
+
+  def teardown
+    @d.instance_eval do @persistence_provider.unlink! end
+    @c.stop!
+  end
+
+  def test_insert_fetch
+    @d.insert!("key", ["value", "value1"])
+
+    assert_equal(["value", "value1"].sort,
+                 @d.fetch("key").collect do |e| e.last end.sort)
+
+    @d.insert!("key2", ["value2"])
+    @d.insert!("key2", ["value3"])
+    assert_equal(["value2"],
+                 @d.fetch("key2").collect do |e| e.last end.sort)
+
+    @d.insert!("key2", ["value3"], "hehu")
+    assert_equal(["value3"],
+                 @d.fetch("key2").collect do |e| e.last end.sort)
+
+    @d.insert!("key", ["value2", "value3"], "apap")
+    assert_equal(["value2", "value3"].sort,
+                 @d.fetch("key").collect do |e| e.last end.sort)
+
+    @d.delete!("key")
+    @d.delete!("key2")
+    assert_equal([],
+                 @d.fetch("key").collect do |e| e.last end)
+    assert_equal([],
+                 @d.fetch("key2").collect do |e| e.last end)
+  end
+
+  def test_too_few
+    @d.insert!("a", ["b","c"])
+    assert_equal(["b","c"].sort, 
+                 @d.fetch("a").collect do |e| e.last end.sort)
+    @d.insert!("a", ["d","c","b"])
+    assert_equal(["b","c","d"].sort,
+                 @d.fetch("a").collect do |e| e.last end.sort)
+  end
+
+  def test_too_many
+    @d.insert!("a", ["b","c"])
+    assert_equal(["b","c"].sort, 
+                 @d.fetch("a").collect do |e| e.last end.sort)
+    @d.insert!("a", ["d"])
+    assert_equal(["d"],
+                 @d.fetch("a").collect do |e| e.last end)
+  end
+
+  def test_wrong_timestamp
+    @d.insert!("a", ["b","c"], "epap")
+    assert_equal(["b","c"].sort, 
+                 @d.fetch("a").collect do |e| e.last end.sort)
+    @d.insert!("a", ["x", "y"], "epao")
+    assert_equal(["x","y"].sort,
+                 @d.fetch("a").collect do |e| e.last end.sort)
+  end
+
+end

data/tests/pirate_test.rb

@@ -4,33 +4,72 @@ require File.join(File.dirname(__FILE__), 'test_helper')
 class PirateTest < Test::Unit::TestCase
 
   def setup
-    DRb.start_service("druby://localhost:#{rand(1000) + 5000}")
+    @j = Archipelago::Disco::Jockey.new
     @p = Archipelago::Pirate::Captain.new(:chest_description => {:class => "TestChest"}, 
-                                          :tranny_description => {:class => "TestManager"})
-    @c = TestChest.new(:persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(__FILE__).parent.join("chest.db")))
-    @c.publish!
-    @c2 = TestChest.new(:persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(__FILE__).parent.join("chest2.db")))
-    @c2.publish!
-    @tm = TestManager.new(:persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(__FILE__).parent.join("tranny.db")))
+                                          :tranny_description => {:class => "TestManager"},
+                                          :jockey => @j,
+                                          :initial_lookup_timeout => 0)
+    @c = TestChest.new(:captain => @p,
+                       :persistence_directory => Pathname.new(__FILE__).parent.join("chest.db"))
+    @c.publish!(:jockey => @j)
+    assert(@j.instance_eval do 
+             @service_change_subscribers_by_event_type 
+           end[:lost].include?([@p.instance_eval do 
+                                  @service_descriptions 
+                                end[:chests], @c.service_id]))
+    assert(@j.instance_eval do 
+             @service_change_subscribers_by_event_type 
+           end[:found].include?([@p.instance_eval do 
+                                  @service_descriptions 
+                                end[:chests], @c.service_id]))
+    assert_within(10) do
+      @p.update_services!
+      @p.chests.keys.include?(@c.service_id)
+    end
+    assert_equal(@c.service_id, @c.instance_eval do @predecessor_id end)
+
+    @c2 = TestChest.new(:captain => @p,
+                        :persistence_directory => Pathname.new(__FILE__).parent.join("chest2.db"))
+    @c2.publish!(:jockey => @j)
+    assert_within(10) do
+      @p.update_services!
+      @p.chests.keys.include?(@c2.service_id)
+    end
+    assert_within(10) do
+      @c2.service_id == @c.instance_eval do @predecessor_id end
+    end
+    assert_within(10) do
+      @c.service_id == @c2.instance_eval do @predecessor_id end
+    end
+
+    @tm = TestManager.new(:captain => @p,
+                          :persistence_directory => Pathname.new(__FILE__).parent.join("tranny.db"))
     @tm.publish!
-    @p.update_services!
     assert_within(10) do
+      @p.update_services!
+      @p.trannies.keys.include?(@tm.service_id)
+    end
+
+
+    assert_within(10) do
+      @p.update_services!
       @p.chests.keys.sort == [@c.service_id, @c2.service_id].sort
     end
     assert_within(10) do
+      @p.update_services!
       @p.trannies.keys == [@tm.service_id]
     end
   end
 
   def teardown
-    @p.stop!
-    @c.stop!
+    @c.close!
     @c.persistence_provider.unlink!
-    @c2.stop!
+    @c2.close!
     @c2.persistence_provider.unlink!
-    @tm.stop!
+    @tm.close!
     @tm.persistence_provider.unlink!
-    DRb.stop_service
+    @p.stop!
+    @j.stop!
   end
 
   def test_include
@@ -122,18 +161,18 @@ class PirateTest < Test::Unit::TestCase
     end
     trans = nil
     assert_raise(Archipelago::Pirate::CommitFailedException) do
-      @p.transaction do |trans|
-        assert_equal(trans.transaction_id, @p.active_transaction.transaction_id)
-        @p["hehu"] = "haha"
+      @p.transaction do |copy|
+        trans = copy.active_transaction
+        copy["hehu"] = "haha"
         assert(!p2.include?("hehu"))
         assert(!p2.include?("hehu", nil))
         assert_equal(nil, p2.active_transaction)
         trans2 = nil
-        p2.transaction do |trans2|
-          assert_equal(trans2.transaction_id, p2.active_transaction.transaction_id)
-          p2["hehu"] = "hoj"
-          assert_equal("haha", @p["hehu"])
-          assert_equal("hoj", @p["hehu", trans2])
+        p2.transaction do |copy2|
+          trans2 = copy2.active_transaction
+          copy2["hehu"] = "hoj"
+          assert_equal("haha", copy["hehu"])
+          assert_equal("hoj", copy["hehu", trans2])
         end
         assert_equal(:commited, trans2.state)
         assert_equal(:active, trans.state)
@@ -145,6 +184,20 @@ class PirateTest < Test::Unit::TestCase
     assert_equal("hoj", @p["hehu"])
   end
 
+  def test_wrong_chest_exception
+    chests = [@c, @c2].sort do |a,b|
+      a.service_id <=> b.service_id
+    end
+    begin
+      $DO_ASSERT_MINE = true
+      assert_raise(Archipelago::Treasure::WrongChestException) do
+        chests.last.call_instance_method(chests.last.service_id.next, :bruhuf, nil)
+      end
+    ensure
+      $DO_ASSERT_MINE = false
+    end
+  end
+
   def test_write_read_transaction
     $T = true
     @p["hej"] = "haha"

data/tests/sanitation_benchmark.rb

@@ -0,0 +1,50 @@
+
+require File.join(File.dirname(__FILE__), 'test_helper')
+
+class SanitationBenchmark < Test::Unit::TestCase
+
+  def setup
+    @d = Archipelago::Dump::Site.new(:persistence_directory => Pathname.new(__FILE__).parent.join("site.db"))
+    @d.publish!
+    @c = Archipelago::Sanitation::Officer.new
+    assert_within(10) do 
+      @c.update_services!
+      @c.sites.keys == [@d.service_id]
+    end
+  end
+
+  def teardown
+    @d.close!
+    @d.instance_eval do @persistence_provider.unlink! end
+    @c.stop!
+  end
+
+  def test_set_get
+    do_set_get(8)
+    do_set_get(128)
+    do_set_get(512)
+    do_set_get(1024)
+  end
+
+  private
+
+  def do_set_get(size)
+    k = "apa"
+    v = " " * size
+    bm("Sanitation#[]= (#{size})", :n => 100) do
+      @c[k] = v
+      k = k.next
+    end
+    k = "apa"
+    bm("Sanitation#[] (#{size})", :n => 100) do
+      t = @c[k]
+      k = k.next
+    end
+    k = "apa"
+    bm("Sanitation#delete", :n => 100) do
+      t = @c.delete!(k)
+      k = k.next
+    end
+  end
+
+end

data/tests/sanitation_test.rb

@@ -0,0 +1,219 @@
+
+require File.join(File.dirname(__FILE__), 'test_helper')
+
+class SanitationTest < Test::Unit::TestCase
+
+  def setup
+    @c = Archipelago::Sanitation::Officer.new(:initial_lookup_timeout => 0)
+    @d = Archipelago::Dump::Site.new(:officer => @c,
+                                     :persistence_directory => Pathname.new(__FILE__).parent.join("site.db"))
+    @d.publish!
+    begin
+      assert_within(10) do 
+        @c.update_services!
+        @c.sites.keys == [@d.service_id]
+      end
+    rescue Test::Unit::AssertionFailedError => e
+      puts "#{@c.sites.keys.inspect} != #{[@d.service_id].inspect}"
+      raise e
+    end
+  end
+
+  def teardown
+    @d.close!
+    @d.instance_eval do @persistence_provider.unlink! end
+    @c.stop!
+  end
+
+  def test_sanitation_backed_chest
+    chest = Archipelago::Treasure::Chest.new(:persistence_directory => Pathname.new(__FILE__).parent.join("site_chest.db"),
+                                             :officer => @c)
+    chest.publish!
+    begin
+      assert_equal(nil, chest["klibb"])
+      assert_equal("kladd", chest["klibb"] = "kladd")
+      assert_equal("kladd", chest["klibb"])
+      assert_equal("KLADD", chest["klibb"].upcase!)
+      klibb = chest["klibb"]
+      assert_equal("kladd", klibb.downcase)
+      assert_equal("kladd", klibb.downcase!)
+      assert_equal("KLADD", klibb.upcase)
+      assert_equal("kladd", klibb)
+      @d.close!
+      @c.update_services!(:validate => true)
+      assert_raise(Archipelago::Sanitation::NoRemoteDatabaseAvailableException) do
+        klibb.upcase!
+      end
+    ensure
+      chest.close!
+      chest.instance_eval do @persistence_provider.unlink! end
+    end
+  end
+
+  def test_missing_bits
+    s1 = Oneliner::SuperString.new("brappa")
+    @d.insert!("a", [s1.encode(9)], "abab")
+    assert_raise(Archipelago::Sanitation::NotEnoughDataException) do
+      @c["a"]
+    end
+  end
+
+
+  def test_get_set
+    @c["hej"] = "hoho"
+    assert_equal("hoho", @c["hej"])
+    @c.delete!("hej")
+    assert_equal(@c["hej"], nil)
+  end
+
+  def test_multi
+    #
+    # Setup the network
+    #
+    @d.close!
+    dumps = []
+    cleaner2 = Archipelago::Sanitation::Officer.new(:minimum_nr_of_chunks => 3,
+                                                    :initial_lookup_timeout => 0)
+    begin
+      10.times do |n|
+        dumps[n] = Archipelago::Dump::Site.new(:officer => cleaner2,
+                                               :service_description => {:service_id => n.to_s},
+                                               :persistence_directory => Pathname.new(__FILE__).parent.join("master_test_#{n}.db"))
+        dumps[n].publish!
+      end
+      #
+      # Wait until it is alive
+      #
+      assert_within(30) do 
+        cleaner2.update_services!
+        dumps.collect do |d| d.service_id end.sort == cleaner2.sites.keys.sort
+      end
+      #
+      # Perform checks on all dumps
+      #
+      10.times do |n|
+        if n < 9
+          assert(dumps[n].instance_eval do right_before?(dumps[n+1].service_id) end, 
+                 "#{dumps[n].service_id} is supposed to be right_before? #{dumps[n+1].service_id}, but isnt. services are #{cleaner2.sites.keys.inspect}")
+          assert(dumps[n+1].instance_eval do right_after?(dumps[n].service_id) end, 
+                 "#{dumps[n+1].service_id} is supposed to be right_after? #{dumps[n].service_id}, but isnt. services are #{cleaner2.sites.keys.inspect}")
+        else
+          assert(dumps[n].instance_eval do right_before?(dumps[0].service_id) end, 
+                 "#{dumps[n].service_id} is supposed to be right_before? #{dumps[0].service_id}, but isnt. services are #{cleaner2.sites.keys.inspect}")
+          assert(dumps[0].instance_eval do right_after?(dumps[n].service_id) end, 
+                 "#{dumps[0].service_id} is supposed to be right_after? #{dumps[n].service_id}, but isnt. services are #{cleaner2.sites.keys.inspect}")
+        end
+        if n > 0
+          assert_equal(dumps[n - 1].service_id, cleaner2.predecessor(dumps[n].service_id))
+        else
+          assert_equal(dumps[9].service_id, cleaner2.predecessor(dumps[0].service_id))
+        end
+      end
+      #
+      # Check that only fresh data is returned
+      #
+      s1 = Oneliner::SuperString.new("brappa")
+      s2 = Oneliner::SuperString.new("brappa2")
+      assert_equal(0, dumps[0].db.size)
+      assert_equal(0, dumps[0].db.size)
+      dumps[0].insert!("a", [s1.encode(20)], "aaaa")
+      dumps[1].insert!("a", [s2.encode(200)], "aaab")
+      assert(cleaner2.responsible_sites("a").include?(dumps[0].service_id))
+      assert(cleaner2.responsible_sites("a").include?(dumps[1].service_id))
+      assert_equal(s2.to_s, cleaner2["a"])
+      #
+      # Check the utility methods of sanitation
+      #
+      assert_equal([dumps[9].service_id, dumps[0].service_id, dumps[1].service_id].sort,
+                   cleaner2.instance_eval do
+                     get_greatest_less_than(:sites, dumps[2].service_id, 3)
+                   end.collect do |d| d[:service_id] end.sort)
+      assert_equal([dumps[8].service_id, dumps[9].service_id, dumps[0].service_id].sort,
+                   cleaner2.instance_eval do
+                     get_least_greater_than(:sites, dumps[7].service_id, 3)
+                   end.collect do |d| d[:service_id] end.sort)
+      assert_equal(dumps[3].service_id,
+                   cleaner2.second_master_to(dumps[4].service_id))
+      assert_equal(dumps[9].service_id,
+                   cleaner2.second_master_to(dumps[0].service_id))
+      #
+      # Check the recovery methods
+      #
+      assert_equal({
+                     "6"=>0,
+                     "7"=>0,
+                     "8"=>0,
+                     "9"=>0,
+                     "0"=>1,
+                     "1"=>1,
+                     "2"=>0,
+                     "3"=>0,
+                     "4"=>0,
+                     "5"=>0
+                   },
+                   chunks_by_id(cleaner2, "a"))
+      cleaner2.redistribute("a")
+      healthy_chunks = {
+        "6"=>0,
+        "7"=>0,
+        "8"=>0,
+        "9"=>0,
+        "0"=>1,
+        "1"=>1,
+        "2"=>1,
+        "3"=>0,
+        "4"=>0,
+        "5"=>0
+      }
+      assert_equal(healthy_chunks,
+                   chunks_by_id(cleaner2, "a"))
+      dumps[1].close!
+      cleaner2.update_services!(:validate => true)
+      dumps[0].instance_eval do lost_peer({:service_id => dumps[1].service_id}) end
+      dumps[2].instance_eval do lost_peer({:service_id => dumps[1].service_id}) end
+      assert_equal({"6"=>0, "7"=>0, "8"=>0, "9"=>0, "0"=>1, "2"=>1, "3"=>1, "4"=>0, "5"=>0},
+                   chunks_by_id(cleaner2, "a"))
+
+      dumps[1] = Archipelago::Dump::Site.new(:officer => cleaner2,
+                                             :service_description => {:service_id => "1"},
+                                             :persistence_directory => Pathname.new(__FILE__).parent.join("master_test_1.db"))
+      dumps[1].publish!
+      begin
+        assert_within(20) do
+          cleaner2.update_services!
+          ["0","1","2","3","4","5","6","7","8","9"].sort == cleaner2.sites.keys.sort
+        end
+      rescue Test::Unit::AssertionFailedError => e
+        pp chunks_by_id(cleaner2, "a")
+        raise e
+      end
+      assert_equal({"0"=>1, "1"=>1, "2"=>1},
+                   cleaner2.responsible_sites("a"))
+      dumps[3].instance_eval do @edge_check_thread.wakeup end
+      cleaner2.update_services!(:validate => true)
+      begin
+        assert_within(20) do
+          chunks_by_id(cleaner2, "a") == healthy_chunks
+        end
+      rescue Test::Unit::AssertionFailedError => e
+        pp chunks_by_id(cleaner2, "a")
+        raise e
+      end
+    ensure
+      dumps.extend(Archipelago::Current::ThreadedCollection)
+      dumps.t_each do |dump|
+        dump.close!
+        dump.instance_eval do @persistence_provider.unlink! end
+      end
+    end
+  end
+
+  private
+
+  def chunks_by_id(officer, key)
+    officer.sites.values.inject({}) do |result, service_desc| 
+      result.merge({service_desc[:service_id] => service_desc[:service].fetch(key).size})
+    end
+  end
+
+end