archipelago 0.2.4 → 0.2.5

data/README

@@ -4,6 +4,7 @@ It consists of several different parts, that can be used standalone or in conjun
 
 == Dependencies:
 Archipelago::Hashish::BerkeleyHashishProvider:: ruby bdb: http://moulon.inra.fr/ruby/bdb.html
+String:: ruby inline: http://www.zenspider.com/ZSS/Products/RubyInline/
 
 == Sub packages:
 Archipelago::Disco:: A UDP multicast discovery service useful to find services in your network with a minimum of configuration.
@@ -31,7 +32,7 @@ To run a transaction manager and a database server, just do the following (from
 
  :include:script/services.rb
 
-To set up an Archipelago::Pirate::Captain do the following (from script/pirate.rb):
+To set up an Archipelago::Pirate::Captain and load some debugging overloads do the following (from script/pirate.rb):
 
  :include:script/pirate.rb
 

data/TODO

@@ -5,12 +5,22 @@
    * Or: Create migration methods that move objects between Chests opon
      startup and shutdown, and make them keep backups at each others
      persistence backends.
+   * Or: Create something that stores data the way Chord does (with erasure
+     codes) but doesnt use the same look up mechanism.
+     * Problem: We still have to implement the entire maintenance protocol
+       of Chord (continously checking if our data is safely replicated across
+       the network, continously checking that our data belong with us)
+
+ * Replace Raider with some well known near-optimal erasure code, for example
+   Online Codes: http://en.wikipedia.org/wiki/Online_codes
 
  * Make Chest aware about whether transactions have affected it 'for real' ie 
    check whether the instance before the call differs from the instance after 
    the call. Preferably without incurring performance lossage.
    * For example: Make Dubloon proxy targets able to provide a dirty_state 
      method that decide whether they are to be considered dirty, clean or auto-select.
+     * This is now done. But it is not yet used to provide intelligence for the
+       transaction mechanism. How should it compare dirty state before and after?
 
  * Test the transaction recovery mechanism of Chest.
 

data/lib/archipelago/client.rb

@@ -0,0 +1,99 @@
+# Archipelago - a distributed computing toolkit for ruby
+# Copyright (C) 2006  Martin Kihlgren <zond at troja dot ath dot cx>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+require 'archipelago/disco'
+
+module Archipelago
+
+  module Client
+
+    INITIAL_SERVICE_UPDATE_INTERVAL = 1
+    MAXIMUM_SERVICE_UPDATE_INTERVAL = 60
+
+    class Base
+      attr_reader :jockey
+      #
+      # Initialize an instance using Archipelago::Disco::MC or <i>:jockey</i> if given, 
+      # or a new Archipelago::Disco::Jockey if none, that looks for new services 
+      # <i>:initial_service_update_interval</i> or INITIAL_SERVICE_UPDATE_INTERVAL,
+      # when it starts and never slower than every <i>:maximum_service_update_interval</i> 
+      # or MAXIMUM_SERVICE_UPDATE_INTERVAL.
+      #
+      def initialize(options = {})
+        setup(options)
+      end
+      #
+      # Sets up this instance with the given +options+.
+      #
+      def setup(options = {})
+        @jockey.stop! if defined?(@jockey) && @jockey != Archipelago::Disco::MC
+        if defined?(Archipelago::Disco::MC)
+          @jockey = options[:jockey] || Archipelago::Disco::MC
+        else
+          @jockey = options[:jockey] || Archipelago::Disco::Jockey.new
+        end
+        
+        @initial_service_update_interval = options[:initial_service_update_interval] || INITIAL_SERVICE_UPDATE_INTERVAL
+        @maximum_service_update_interval = options[:maximum_service_update_interval] || MAXIMUM_SERVICE_UPDATE_INTERVAL
+      end
+
+      #
+      # Stops the service update thread for this Pirate and also unpublishes and/or
+      # stops the Jockey.
+      #
+      def stop!
+        @service_update_thread.kill if @service_update_thread
+        unless defined?(Archipelago::Disco::MC) && @jockey && @jockey == Archipelago::Disco::MC
+          @jockey.stop!
+        end
+      end
+
+      #
+      # Override this to do whatever you want to when finding services.
+      #
+      def update_services!
+        # /moo
+      end
+
+      private
+
+      #
+      # Start a thread looking up existing chests between every
+      # +initial+ and +maximum+ seconds.
+      #
+      def start_service_updater
+        update_services!
+        @service_update_thread = Thread.start do
+          standoff = @initial_service_update_interval
+          loop do
+            begin
+              sleep(standoff)
+              standoff *= 2
+              standoff = @maximum_service_update_interval if standoff > @maximum_service_update_interval
+              update_services!
+            rescue Exception => e
+              puts e
+              pp e.backtrace
+            end
+          end
+        end
+      end
+    end
+
+  end
+
+end

data/lib/archipelago/cove.rb

@@ -0,0 +1,68 @@
+# Archipelago - a distributed computing toolkit for ruby
+# Copyright (C) 2006  Martin Kihlgren <zond at troja dot ath dot cx>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+require 'archipelago/disco'
+require 'archipelago/hashish'
+require 'archipelago/exxon'
+require 'archipelago/raider'
+
+module Archipelago
+
+  module Cove
+
+    class Tanker
+
+      #
+      # The Tanker can be published.
+      #
+      include Archipelago::Disco::Publishable
+
+      def initialize(options = {})
+        #
+        # The provider of happy magic persistent hashes of different kinds.
+        #
+        @persistence_provider = options[:persistence_provider] || Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(File.expand_path(__FILE__)).parent.join("cove_tanker.db"))
+
+        @db = @persistence_provider.get_hashish("db")
+
+        @valdez = options[:valdez] || Archipelago::Exxon::HAZELWOOD
+
+        #
+        # Use the given options to initialize the publishable
+        # instance variables.
+        #
+        initialize_publishable(options)
+
+      end
+
+      def store_part(key, split_part)
+        @db[key] = Marshal.dump(split_part)
+      end
+
+      def get_part(key)
+        if @db.include?(key)
+          return Marshal.load(@db[key])
+        else
+          return nil
+        end
+      end
+
+    end
+
+  end
+
+end

data/lib/archipelago/current.rb

@@ -134,9 +134,12 @@ module Archipelago
     # object.
     #
     module Synchronized 
+
       include MonitorMixin
+
       alias :lock :mon_enter
       alias :unlock :mon_exit
+
       #
       # We dont care about lock ownership.
       #

data/lib/archipelago/disco.rb

@@ -22,6 +22,7 @@ require 'pp'
 require 'archipelago/current'
 require 'drb'
 require 'set'
+require 'digest/sha1'
 
 module Archipelago
 
@@ -252,7 +253,7 @@ module Archipelago
       #
       def initialize(hash)
         raise "Record must have a :service_id" unless hash.include?(:service_id)
-        raise "Record must have a :validator" unless hash.include?(:validator)
+        raise "Record must have a :validator" unless hash.include?(:validator) && hash[:validator].respond_to?(:valid?)
         super(hash)
       end
       #
@@ -532,17 +533,9 @@ module Archipelago
               recipient, data = @outgoing.pop
               if recipient
                 address, port = recipient.split(/:/)
-                if $DEBUG
-                  puts "sending to #{address}:#{port}:"
-                  puts Marshal.dump(data)
-                end
                 @unisender.send(Marshal.dump(data), 0, address, port.to_i)
               else
                 begin
-                  if $DEBUG
-                    puts "broadcasting to #{@multiaddress}:#{@multiport}:"
-                    puts Marshal.dump(data)
-                  end
                   @sender.write(Marshal.dump(data))
                 rescue Errno::ECONNREFUSED => e
                   retry

data/lib/archipelago/exxon.rb

@@ -0,0 +1,138 @@
+# Archipelago - a distributed computing toolkit for ruby
+# Copyright (C) 2006  Martin Kihlgren <zond at troja dot ath dot cx>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+require 'archipelago/client'
+
+module Archipelago
+
+  module Exxon
+
+    TANKER_DESCRIPTION = {
+      :class => 'Archipelago::Cove::Tanker'
+    }
+
+    PARTS = 3
+
+    #
+    # Raised when you try to do stuff without any remote database
+    # available.
+    #
+    class NoRemoteDatabaseAvailableException < RuntimeError
+      def initialize(valdez)
+        super("#{valdez} can not find any remote database for you")
+      end
+    end
+
+    class Valdez < Archipelago::Client::Base
+      attr_reader :tankers
+      def initialize(options = {})
+        super(options)
+
+        start_service_updater
+      end
+
+      def setup(options = {})
+        super(options)
+
+        @parts = options[:parts] || PARTS
+        @tanker_description = TANKER_DESCRIPTION.merge(options[:tanker_description] || {})
+      end
+
+      def []=(key, value)
+        split_parts = Archipelago::Raider.split(value, @parts)
+        followers = responsible_tankers(key, split_parts.size)
+        0.upto(followers.size - 1) do |n|
+          followers[n].store_part(key, split_parts[n])
+        end
+      end
+
+      def [](key)
+        return_value = nil
+        
+        split_parts = []
+        followers = responsible_tankers(key, (1 << @parts) - 1)
+        while split_parts.size < @parts
+          split_parts << followers.shift.get_part(key)
+        end
+        begin
+          return_value = Archipelago::Raider.join(*split_parts.compact)
+        rescue Archipelago::Raider::NotEnoughBulkError => e
+          if followers.empty?
+            raise e
+          else
+            split_parts << followers.shift.get_part(key)
+            retry
+          end
+        end
+        
+        return return_value
+      end
+
+      def update_services!
+        @tankers = @jockey.lookup(Archipelago::Disco::Query.new(@tanker_description), 0)
+      end
+
+      def responsible_tankers(key, n)
+        raise NoRemoteDatabaseAvailableException.new(self) if @tankers.empty?
+
+        responsible_ids = []
+
+        key_id = Digest::SHA1.hexdigest(Marshal.dump(key))
+        sorted_tanker_ids = @tankers.keys.sort
+        0.upto(sorted_tanker_ids.size - 1) do |index|
+          id = sorted_tanker_ids[index]
+          if id > key_id
+            responsible_ids = get_some(sorted_tanker_ids, index, n)
+          end
+        end
+        responsible_ids = get_some(sorted_tanker_ids, 0, n)
+
+        return responsible_ids.collect do |id|
+          @tankers[id][:service]
+        end
+      end
+
+      def responsible_tanker(key)
+        return responsible_tankers(key, 1).first
+      end
+
+      private
+
+      def get_some(array, index, n)
+        rval = []
+        while rval.size < n
+          index = 0 if index > array.size - 1
+          rval << array[index]
+          index += 1
+        end
+        return rval
+      end
+      
+    end
+
+    #
+    # The default Archipelago::Exxon::Valdez that is always available for lookups is Archipelago::Exxon::HAZELWOOD.
+    #
+    # If you really need to you can customize it by defining HAZELWOOD_OPTIONS before loading disco.rb, and if you REALLY
+    # need to you can disable it completely by setting HAZELWOOD_DISABLED to true.
+    #
+    HAZELWOOD = Valdez.new(defined?(HAZELWOOD_OPTIONS) ? HAZELWOOD_OPTIONS : {}) unless defined?(HAZELWOOD_DISABLED) && HAZELWOOD_DISABLED
+
+
+  end
+
+end

data/lib/archipelago/hashish.rb

@@ -115,15 +115,28 @@ module Archipelago
       #
       def store_if_changed(key)
         @lock.synchronize_on(key) do
-          
-          serialized_key = Marshal.dump(key)
 
           value = @content[key]
-          serialized_value = Marshal.dump(value)
-          old_serialized_value = @content_db[serialized_key]
 
-          write_to_db(key, serialized_key, serialized_value, value) if old_serialized_value && old_serialized_value != serialized_value
-          
+          if value.respond_to?(:dirty?)
+
+            if value.dirty?
+              serialized_value = Marshal.dump(value)
+              serialized_key = Marshal.dump(key)
+              write_to_db(key, serialized_key, serialized_value, value)
+              value.is_clean! if value.respond_to?(:is_clean!)
+            end
+    
+          else
+
+            serialized_value = Marshal.dump(value)
+            serialized_key = Marshal.dump(key)
+            old_serialized_value = @content_db[serialized_key]
+            
+            write_to_db(key, serialized_key, serialized_value, value) if old_serialized_value && old_serialized_value != serialized_value
+            
+          end
+
         end
       end
       #

data/lib/archipelago/oneline.rb

@@ -0,0 +1,641 @@
+# Archipelago - a distributed computing toolkit for ruby
+# Copyright (C) 2006  Martin Kihlgren <zond at troja dot ath dot cx>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+require 'rubygems'
+require 'inline'
+require 'pp'
+
+#
+# Adds a few methods to String, some using the inline gem.
+#
+class String
+
+  E = [0.1, 0.01]
+  Q = [6, 3]
+  THRESHOLDS = [1024]
+
+  class << self
+    private
+    #
+    # Get an F value for given epsilon value
+    #
+    def get_f(e)
+      Math.log(e ** 2 / 4) / Math.log(1 - (e / 2))
+    end
+    #
+    # Get a p distribution with a given name and epsilon value.
+    #
+    # Will return a C variable definition for this distribution.
+    #
+    def _get_p_distribution(name, e)
+      f = get_f(e)
+      rval = "double #{name}[] = {0.0,"
+      p1 = 1 - ( (1 + (1 / f) ) / (1 + e) )
+      1.upto(f) do |n|
+        if n == 1
+          rval << "#{p1}"
+        else
+          rval << "#{( ( 1 - p1 ) * f ) / ( ( f - 1 ) * n * ( n - 1 ) )}"
+        end
+        rval << "," unless n + 1 > f
+      end
+      rval << "};\n"
+      return rval
+    end
+    #
+    # Get all the p distributions we want.
+    #
+    # Will return a C variable definition for this distribution.
+    #
+    def get_p_distribution
+      rval = ""
+      0.upto(E.size - 1) do |n|
+        rval << _get_p_distribution("P#{n}", E[n])
+      end
+      return rval
+    end
+  end
+
+  inline do |builder|
+    #
+    # We need math.h
+    #
+    builder.include("<math.h>")
+    #
+    # We need sys/time.h
+    #
+    builder.include("<sys/time.h>")
+    #
+    # Define the p distribution array so that it is easily accessible.
+    #
+    builder.prefix(get_p_distribution)
+    #
+    # Define a sane random number generator.
+    #
+    builder.prefix("#define RAND (((double)random())/((double)RAND_MAX+1))")
+    builder.prefix("#define RANDOM(n) (random() % (n))")
+    #
+    # Get a random degree.
+    #
+    # Will define as many get_degree{n} functions as we need.
+    #
+    degree_methods = ""
+    0.upto(E.size - 1) do |n|
+      degree_methods += <<EOC
+static long
+get_degree#{n}(size)
+{
+  long rval = 1;
+  double d = RAND - P#{n}[rval];
+  while (d > 0.0) {
+    rval ++;
+    d = d - P#{n}[rval]; 
+  }
+  if (rval > size)
+    return size;
+  else
+    return rval;
+}
+EOC
+    end
+    builder.prefix(degree_methods)
+    #
+    # Get the function to produce random degrees for us.
+    #
+    # Will go through our THRESHOLDS and return the proper degree-method.
+    #
+    get_degree_func = <<EOC
+long (*get_degree_func(long size))(long)
+{
+EOC
+    0.upto(THRESHOLDS.size - 1) do |n|
+      get_degree_func += <<EOC
+  if (size < #{THRESHOLDS[n]})
+    return &get_degree#{n};
+EOC
+    end
+    get_degree_func += <<EOC
+  return &get_degree#{THRESHOLDS.size};
+EOC
+    get_degree_func += <<EOC
+}
+EOC
+    builder.prefix(get_degree_func)
+    #
+    # Check if a certain block is known.
+    #
+    builder.prefix <<EOC
+static char
+get_is_known(VALUE decode_status, long block_number)
+{
+  long byte_in_status = block_number / 8;
+  char bit_in_byte = 1 << (block_number % 8);
+  return ((RSTRING(decode_status)->ptr[byte_in_status]) & bit_in_byte) == bit_in_byte;
+}
+EOC
+    #
+    # Set that a certain block is known.
+    #
+    builder.prefix <<EOC
+static void
+set_is_known(VALUE decode_status, long block_number)
+{
+  long byte_in_status = block_number / 8;
+  char bit_in_byte = 1 << (block_number % 8);
+  rb_str_modify(decode_status);
+  RSTRING(decode_status)->ptr[byte_in_status] = RSTRING(decode_status)->ptr[byte_in_status] | bit_in_byte;
+}
+EOC
+    #
+    # Returns whether this String is decoded enough.
+    #
+    # Will first go through our decode_status one byte a time
+    # and then check the relevant bits of the last byte.
+    #
+    builder.c <<EOC
+static VALUE
+_oneline_done(VALUE decode_status)
+{
+  long tmp;
+  if (decode_status == Qnil)
+    return Qfalse;
+  for (tmp = 0; tmp < (RSTRING(decode_status)->len - 1); tmp++) {
+    if (( (unsigned char) RSTRING(decode_status)->ptr[tmp] ) != 255) {
+      return Qfalse;
+    }
+  }
+  for (tmp = 0; tmp < (RSTRING(self)->len % 8); tmp++) {
+    if (! ( (unsigned char) RSTRING(self)->ptr[tmp] ) & (1 << tmp)) {
+      return Qfalse;
+    }
+  }
+  return Qtrue;
+}
+EOC
+    #
+    # Set a data or aux block with a known value.
+    #
+    # Will both update ourselves or our aux_blocks depending on index,
+    # then set the decode_status so that this byte is officially known.
+    #
+    builder.prefix <<EOC
+static void
+set_value(VALUE aux_blocks, long index, char value, VALUE self, VALUE decode_status)
+{
+  if (index < RSTRING(self)->len) {
+    rb_str_modify(self);
+    RSTRING(self)->ptr[index] = value;
+    set_is_known(decode_status, index);
+  } else {
+    rb_str_modify(aux_blocks);
+    RSTRING(aux_blocks)->ptr[index - RSTRING(self)->len] = value;
+    set_is_known(decode_status, index);
+  }
+}
+EOC
+    #
+    # Get a data or aux block.
+    #
+    # Gets from ourselves or our aux_blocks depending on the index.
+    #
+    builder.prefix <<EOC
+static char
+get_value(VALUE aux_blocks, long index, VALUE self)
+{
+  if (index < RSTRING(self)->len) {
+    return RSTRING(self)->ptr[index];
+  } else {
+    return RSTRING(aux_blocks)->ptr[index - RSTRING(self)->len];
+  }
+}
+EOC
+    #
+    # Decode the given chunk until we dont get any new
+    # data from it.
+    #
+    builder.c <<EOC
+static VALUE
+_decode_chunk(VALUE chunk, VALUE decode_status, VALUE aux_blocks)
+{
+  long seed = ( (long *) RSTRING(chunk)->ptr )[0];
+  long size = ( (long *) RSTRING(chunk)->ptr )[1];
+  long tmp, tmp2;
+  char got_new_block = 0;
+  long (*degree_func)() = get_degree_func(size);
+
+  // Go through the chunk.
+  for (tmp = 8; tmp < RSTRING(chunk)->len; tmp++) {
+    long degree;  
+
+    // Seed with our seed and the index in the chunk.
+    srandom(seed + tmp);
+
+    //printf("looking at check block %d(%d):", tmp, RSTRING(chunk)->ptr[tmp]);
+
+    // Get the degree of this chunk.
+    degree = degree_func(size);
+    // If it is degree 1 then we can deduce the data/aux block immediately.
+    if (degree == 1) {
+      long block_nr;
+
+      // Seed with our seed, the chunk index and the degree.
+      srandom(seed + tmp + degree);
+
+      // This block can be either from ourselves or from our aux blocks.
+      block_nr = RANDOM(size + RSTRING(aux_blocks)->len);
+
+      //printf(" %d", block_nr);
+
+      // If it is not already known, then set it as known. Otherwise just log about its state.
+      if (!get_is_known(decode_status, block_nr)) {
+        set_value(aux_blocks, block_nr, RSTRING(chunk)->ptr[tmp], self, decode_status);
+        got_new_block = 1;
+        //printf(" => [%d]=%d", block_nr, RSTRING(chunk)->ptr[tmp]);
+      } else {
+        //printf(" [%d]=%d", block_nr, RSTRING(chunk)->ptr[tmp]);
+      }
+    } else {
+      // If it is NOT degree 1, then we go through all the blocks it is xored from 
+      // and see if we have exactly one missing.
+      long missing_blocks;
+      long missing_block;
+      // Create the xor_sum with this block of the chunk.
+      char xor_sum = RSTRING(chunk)->ptr[tmp];
+
+      for (missing_blocks = tmp2 = degree; tmp2 > 0; tmp2--) {
+        long block_nr;
+
+        srandom(seed + tmp + tmp2);
+
+        block_nr = RANDOM(size + RSTRING(aux_blocks)->len);
+
+        //printf(" %d", block_nr);
+
+        // If the block we are looking at is known, then we decrement missing_blocks
+        // and continue with our xor_sum.
+        if (get_is_known(decode_status, block_nr)) {
+          missing_blocks--;
+          xor_sum = xor_sum ^ get_value(aux_blocks, block_nr, self);
+          //printf("!");
+        } else {
+          missing_block = block_nr;
+          //printf("?");
+        }
+      }
+      // If exactly one block is missing, set is as the final xor_sum.
+      if (missing_blocks == 1) {
+        set_value(aux_blocks, missing_block, xor_sum, self, decode_status);
+        got_new_block = 1;
+        //printf(" => [%d]=%d", missing_block, xor_sum);
+      }
+    }
+    //printf("\\n");
+    fflush(NULL);
+  }
+
+  return got_new_block ? Qtrue : Qfalse;
+}
+EOC
+    #
+    # Get a check block from a message.
+    #
+    builder.prefix <<EOC
+static char
+get_check_block(VALUE self, long seed, long index, VALUE aux_blocks, long (*degree_func)(long))
+{
+  long degree;
+  int empty_block = 1;
+  char block;
+  char found_block;
+
+  // Seed with our seed and the index of this block in the chunk.
+  srandom(seed + index);
+
+  //printf("creating check block %d:", index);
+
+  // Find [degree] blocks.
+  for (degree = degree_func(RSTRING(self)->len); degree > 0; degree--) {
+    long block_number;
+    char found_block;
+
+    // Seed with our seed, the index in the chunk, and the degree.
+    srandom(seed + index + degree);
+
+    block_number = RANDOM(RSTRING(self)->len + RSTRING(aux_blocks)->len);
+    // Depending on whether the block is in our data or our own, debug about it
+    // and remember what it was.
+    if (block_number < RSTRING(self)->len) {
+      found_block = RSTRING(self)->ptr[block_number];
+      //printf(" d[%d]=%d", block_number, RSTRING(self)->ptr[block_number]);
+    } else {
+      found_block = RSTRING(aux_blocks)->ptr[block_number - RSTRING(self)->len];
+      //printf(" a[%d]=%d", block_number - RSTRING(self)->len, RSTRING(aux_blocks)->ptr[block_number - RSTRING(self)->len]);
+    }
+    // Create the block by xoring.
+    if (empty_block) {
+      block = found_block;
+      empty_block = 0;
+    } else {
+      block = block ^ found_block;
+    }
+  }
+  //printf(" => %d\\n", block);
+  return block;
+}
+EOC
+    #
+    # Fill our aux blocks with the proper data.
+    #
+    builder.c <<EOC
+static void
+_fill_aux_blocks(long size, long q)
+{
+  char aux_blocks[size];
+  char initialized_aux_blocks[size];
+  long tmp, tmp2;
+
+  for (tmp = 0; tmp < size; tmp++)
+    initialized_aux_blocks[tmp] = 0;
+
+  srandom(RSTRING(self)->len);
+
+  // For each block of our data.
+  for (tmp = 0; tmp < RSTRING(self)->len; tmp++) {
+    //printf("creating aux blocks for d[%d](%d):", tmp, RSTRING(self)->ptr[tmp]);
+    // Create q blocks.
+    for (tmp2 = 0; tmp2 < q; tmp2++) {
+      long aux_block_nr = RANDOM(size);
+      //printf(" [%d]", aux_block_nr);
+      // If this aux block is already initialized, just xor to it, otherwise set it.
+      if (initialized_aux_blocks[aux_block_nr]) {
+        aux_blocks[aux_block_nr] = aux_blocks[aux_block_nr] ^ RSTRING(self)->ptr[tmp];
+      } else {
+        aux_blocks[aux_block_nr] = RSTRING(self)->ptr[tmp];
+        initialized_aux_blocks[aux_block_nr] = 1;
+      }
+      //printf(" => %d", aux_blocks[aux_block_nr]);
+    }
+    //printf("\\n");
+  }
+  rb_ivar_set(self, rb_intern("@_aux_blocks"), rb_str_new(aux_blocks, size));
+}
+EOC
+    #
+    # Returns a chunk of coded data for a given message.
+    #
+    builder.c <<EOC
+static VALUE
+_get_chunk(long chunk_size, VALUE aux_blocks) 
+{
+  unsigned long tmp;
+  long seed;
+  struct timeval tmp_timeval;
+  char chunk[chunk_size];
+  long (*degree_func)() = get_degree_func(RSTRING(self)->len);
+
+  // Seed with timeofday
+  gettimeofday(&tmp_timeval, NULL);
+  srandom(tmp_timeval.tv_usec);
+  // Get the seed for this chunk.
+  seed = random();
+
+  ( (long *) chunk) [0] = seed;
+  ( (long *) chunk) [1] = RSTRING(self)->len;
+
+  // Get the requested number of check blocks and append to this chunk.
+  for (tmp = 8; tmp < chunk_size; tmp++) {
+    chunk[tmp] = get_check_block(self, seed, tmp, aux_blocks, degree_func);
+  }
+
+  return rb_str_new(chunk, chunk_size);
+}
+EOC
+    #
+    # Create the @_aux_hash so that we know what data blocks each aux block represents.
+    #
+    builder.c <<EOC
+static void
+_fill_aux_hash(long q, long size)
+{
+  long tmp, tmp2;
+  VALUE aux_ary = rb_ary_new();
+  VALUE set_func = rb_intern("[]=");
+  VALUE get_func = rb_intern("[]");
+  VALUE push_func = rb_intern("<<");
+  VALUE size_func = rb_intern("size");
+
+  srandom(RSTRING(self)->len);
+
+  // Go through all our blocks
+  for (tmp = 0; tmp < RSTRING(self)->len; tmp++) {
+    // And for each block q aux blocks.
+    for (tmp2 = 0; tmp2 < q; tmp2++) {
+      // And build an Array of the data blocks each aux block represents.
+      long aux_block_nr = RANDOM(size);
+      VALUE data_blocks_for_this_aux_block;
+      if ((data_blocks_for_this_aux_block = rb_funcall(aux_ary, get_func, 1, INT2NUM(aux_block_nr))) == Qnil)
+        rb_funcall(aux_ary, set_func, 2, INT2NUM(aux_block_nr), (data_blocks_for_this_aux_block = rb_ary_new()));
+      rb_funcall(data_blocks_for_this_aux_block, push_func, 1, INT2NUM(tmp));
+    }
+  }
+  
+  // Then go through all our blocks again.
+  for (tmp = 0; tmp < size; tmp++) {
+    // Fetch the data blocks for this aux block.
+    VALUE data_blocks_for_this_aux_block = rb_funcall(aux_ary, get_func, 1, INT2NUM(tmp));
+    // Count them.
+    long number_of_data_blocks = NUM2INT(rb_funcall(data_blocks_for_this_aux_block, size_func, 0));
+    // Create a String that is 4 times the size.
+    VALUE data_block_string = rb_str_new(0, number_of_data_blocks * 4);
+    // And go through all q blocks per data block, again.
+    for (tmp2 = 0; tmp2 < number_of_data_blocks; tmp2++) {
+      // And make the String contain the data blocks for this aux block, as longs.
+      rb_str_modify(data_block_string);
+      ( (long *) RSTRING(data_block_string)->ptr )[tmp2] = (long) NUM2INT(rb_funcall(data_blocks_for_this_aux_block, 
+                                                                                     get_func, 
+                                                                                     1, 
+                                                                                     INT2NUM(tmp2)));
+    }
+    rb_funcall(aux_ary, set_func, 2, INT2NUM(tmp), data_block_string);
+  }
+  rb_ivar_set(self, rb_intern("@_aux_hash"), aux_ary);
+}
+EOC
+    #
+    # Try to decode our data and aux blocks until we dont get any more data.
+    #
+    builder.c <<EOC
+static VALUE
+_aux_decode(VALUE decode_status, VALUE aux_blocks, VALUE aux_hash)
+{
+  long tmp, tmp2;
+  char got_new_block = 0;
+  VALUE get_func = rb_intern("[]");
+
+  // Go through all aux blocks.
+  for (tmp = 0; tmp < RSTRING(aux_blocks)->len; tmp++) {
+    // Only check those that are actually known.
+    if (get_is_known(decode_status, RSTRING(self)->len + tmp)) {
+
+      //printf("looking at aux block %d(%d): ", tmp, RSTRING(aux_blocks)->ptr[tmp]);
+
+      // Fetch the String defining what data blocks this aux block is made out of.
+      VALUE data_block_string = rb_funcall(aux_hash, get_func, 1, INT2NUM(tmp));
+
+      // If it is only one data block, then its easy.
+      if (RSTRING(data_block_string)->len == 4) {
+        long block_nr = ( (long *) RSTRING(data_block_string)->ptr )[0];
+
+        //printf(" %d", block_nr);
+
+        // If it is not already known, then make it known.
+        if (!get_is_known(decode_status, block_nr)) {
+          set_value(aux_blocks, block_nr, RSTRING(aux_blocks)->ptr[tmp], self, decode_status);
+          got_new_block = 1;
+          //printf(" => [%d]=%d", block_nr, RSTRING(aux_blocks)->ptr[tmp]);
+        } else {
+          //printf(" [%d]=%d", block_nr, RSTRING(aux_blocks)->ptr[tmp]);
+        }
+      } else {
+        // If it is several data blocks, then we just go through them all to see if only
+        // one of them is unknown.
+        long missing_blocks;
+        long missing_block;
+        // Initialize the xor_sum as this aux_block.
+        char xor_sum = RSTRING(aux_blocks)->ptr[tmp];
+
+        for (missing_blocks = (tmp2 = ((RSTRING(data_block_string)->len / 4) - 1)) + 1; tmp2 > -1; tmp2--) {
+          long block_nr = ( (long *) RSTRING(data_block_string)->ptr )[tmp2];
+
+          //printf(" %d", block_nr);
+
+          // If it is a known data block, decrease missing_blocks and append to the xor_sum.
+          if (get_is_known(decode_status, block_nr)) {
+            missing_blocks--;
+            xor_sum = xor_sum ^ get_value(aux_blocks, block_nr, self);
+            //printf("!(%d)->(%d)", get_value(aux_blocks, block_nr, self), xor_sum);
+          } else {
+            missing_block = block_nr;
+            //printf("?");
+          }
+        }
+        // If only one data block was unknown, then just set it to the built up xor_sum.
+        if (missing_blocks == 1) {
+          set_value(aux_blocks, missing_block, xor_sum, self, decode_status);
+          got_new_block = 1;
+          //printf(" => [%d]=%d", missing_block, xor_sum);
+        }
+      }  
+    }
+  }
+  //printf("\\n");
+  fflush(NULL);
+
+  return got_new_block ? Qtrue : Qfalse;
+}
+EOC
+  end
+
+  def oneline_encode(chunk_size)
+    ensure_aux_blocks
+    return _get_chunk(chunk_size, @_aux_blocks)
+  end
+
+  def oneline_decode(chunk)
+
+    needed_size = chunk[4..7].unpack("L").first
+    if size == 0
+      concat("\000" * needed_size)
+    elsif size != needed_size
+      raise "The argument to oneline_decode must be a chunk produced by oneline_encode from the same source"
+    end
+
+    @_decode_status ||= "\000" * (((needed_size + get_aux_blocks) / 8) + 1)
+    @_known_chunks ||= []
+    @_aux_blocks ||= "\000" * get_aux_blocks
+
+    found_new_block = _decode_chunk(chunk, @_decode_status, @_aux_blocks)
+    do_decode if found_new_block
+
+    @_known_chunks.unshift(chunk)
+
+    nil while do_decode
+
+    return oneline_done
+  end
+
+  def oneline_done
+    return nil unless @_decode_status
+    return _oneline_done(@_decode_status)
+  end
+
+  def decode_status
+    return nil unless @_decode_status
+    @_decode_status.unpack("b*").first
+  end
+
+  def decode_level
+    ones = 0
+    decode_status[0..(size + @_aux_blocks.size)].split(//).each do |s|
+      ones += 1 if s == "1"
+    end
+    ones.to_f / (size + @_aux_blocks.size).to_f
+  end
+
+  private
+
+  def get_aux_blocks
+    (self.size * (get_q * get_e * 0.55)).ceil
+  end
+
+  def get_q
+    0.upto(THRESHOLDS.size - 1) do |n|
+      if size < THRESHOLDS[n]
+        return Q[n]
+      end
+    end
+    return Q.last
+  end
+
+  def do_decode
+    found_new_block = false
+    @_known_chunks.each do |known_chunk|
+      found_new_block = _decode_chunk(known_chunk, @_decode_status, @_aux_blocks) || found_new_block
+    end
+    ensure_aux_hash
+    found_new_block = _aux_decode(@_decode_status, @_aux_blocks, @_aux_hash) || found_new_block
+    return found_new_block
+  end
+
+  def get_e
+    0.upto(THRESHOLDS.size - 1) do |n|
+      if size < THRESHOLDS[n]
+        return E[n]
+      end
+    end
+    return E.last
+  end
+
+  def ensure_aux_blocks
+    _fill_aux_blocks(get_aux_blocks, get_q) unless @_aux_blocks
+  end
+
+  def ensure_aux_hash
+    _fill_aux_hash(get_q, get_aux_blocks) unless @_aux_hash
+  end
+    
+end

data/lib/archipelago/pirate.rb

@@ -18,6 +18,7 @@
 require 'archipelago/disco'
 require 'archipelago/tranny'
 require 'archipelago/treasure'
+require 'archipelago/client'
 require 'pp'
 require 'drb'
 require 'digest/sha1'
@@ -60,8 +61,6 @@ module Archipelago
       end
     end
 
-    INITIAL_SERVICE_UPDATE_INTERVAL = 1
-    MAXIMUM_SERVICE_UPDATE_INTERVAL = 60
     CHEST_DESCRIPTION = {
       :class => 'Archipelago::Treasure::Chest'
     }
@@ -73,12 +72,8 @@ module Archipelago
     # The class that actually keeps track of the Archipelago::Treasure:Chests and the
     # Archipelago::Treasure:Dubloons in them.
     #
-    class Captain
-      attr_reader :chests, :treasure_map, :trannies
-      #
-      # Initialize an instance using an Archipelago::Disco::Jockey with <i>:jockey_options</i> 
-      # if given, that looks for new services <i>:initial_service_update_interval</i> or INITIAL_SERVICE_UPDATE_INTERVAL,
-      # when it starts and never slower than every <i>:maximum_service_update_interval</i> or MAXIMUM_SERVICE_UPDATE_INTERVAL.
+    class Captain < Archipelago::Client::Base
+      attr_reader :chests, :trannies
       #
       # Will look for Archipelago::Treasure::Chests matching <i>:chest_description</i> or CHEST_DESCRIPTION and 
       # Archipelago::Tranny::Managers matching <i>:tranny_description</i> or TRANNY_DESCRIPTION.
@@ -87,12 +82,11 @@ module Archipelago
       # of required classes and modules at the chest.
       #
       def initialize(options = {})
-        setup(options)
+        super(options)
 
         @transaction = nil
 
-        start_service_updater(options[:initial_service_update_interval] || INITIAL_SERVICE_UPDATE_INTERVAL,
-                              options[:maximum_service_update_interval] || MAXIMUM_SERVICE_UPDATE_INTERVAL)
+        start_service_updater
 
       end
 
@@ -100,14 +94,7 @@ module Archipelago
       # Sets up this instance with the given +options+.
       #
       def setup(options = {})
-        @treasure_map ||= nil
-        if defined?(Archipelago::Disco::MC)
-          @treasure_map.stop! if @treasure_map && @treasure_map != Archipelago::Disco::MC
-          @treasure_map = options.include?(:jockey_options) ? Archipelago::Disco::Jockey.new(options[:jockey_options]) : Archipelago::Disco::MC
-        else
-          @treasure_map.stop! if @treasure_map
-          @treasure_map = Archipelago::Disco::Jockey.new(options[:jockey_options] || {})
-        end
+        super(options)
 
         @chest_description = CHEST_DESCRIPTION.merge(options[:chest_description] || {})
         @tranny_description = TRANNY_DESCRIPTION.merge(options[:tranny_description] || {})
@@ -193,8 +180,11 @@ module Archipelago
 
         @transaction = @trannies.values.first[:service].begin
         begin
-          yield(@transaction)
-          raise CommitFailedException.new(self, @transaction) unless @transaction.commit! == :commited
+          begin
+            return yield(@transaction)
+          ensure
+            raise CommitFailedException.new(self, @transaction) unless @transaction.commit! == :commited
+          end
         rescue Exception => e
           @transaction.abort! unless @transaction.state == :aborted
           raise e
@@ -235,17 +225,6 @@ module Archipelago
         'yar!'
       end
 
-      #
-      # Stops the service update thread for this Pirate and also unpublishes and/or
-      # stops the Jockey.
-      #
-      def stop!
-        @service_update_thread.kill
-        unless defined?(Archipelago::Disco::MC) && @treasure_map && @treasure_map == Archipelago::Disco::MC
-          @treasure_map.stop!
-        end
-      end
-
       #
       # Will do +callable+.call(key, value)
       # for each key-and-value pair in this database network.
@@ -263,9 +242,9 @@ module Archipelago
       # Does an immediate update of our service lists.
       #
       def update_services!
-        @chests = @treasure_map.lookup(Archipelago::Disco::Query.new(@chest_description), 0)
+        @chests = @jockey.lookup(Archipelago::Disco::Query.new(@chest_description), 0)
         evaluate_in_chests
-        @trannies = @treasure_map.lookup(Archipelago::Disco::Query.new(@tranny_description), 0)
+        @trannies = @jockey.lookup(Archipelago::Disco::Query.new(@tranny_description), 0)
       end
 
       private
@@ -329,30 +308,16 @@ module Archipelago
         end
       end
 
-      #
-      # Start a thread looking up existing chests between every
-      # +initial+ and +maximum+ seconds.
-      #
-      def start_service_updater(initial, maximum)
-        update_services!
-        @service_update_thread = Thread.start do
-          standoff = initial
-          loop do
-            begin
-              sleep(standoff)
-              standoff *= 2
-              standoff = maximum if standoff > maximum
-              update_services!
-            rescue Exception => e
-              puts e
-              pp e.backtrace
-            end
-          end
-        end
-      end
-
     end
 
+    #
+    # The default Archipelago::Pirate::Captain that is always available for lookups is Archipelago::Pirate::BLACKBEARD.
+    #
+    # If you really need to you can customize it by defining BLACKBEARD_OPTIONS before loading disco.rb, and if you REALLY
+    # need to you can disable it completely by setting BLACKBEARD_DISABLED to true.
+    #
+    BLACKBEARD = Captain.new(defined?(BLACKBEARD_OPTIONS) ? BLACKBEARD_OPTIONS : {}) unless defined?(BLACKBEARD_DISABLED) && BLACKBEARD_DISABLED
+
   end
 
 end

data/lib/archipelago.rb

@@ -17,8 +17,14 @@
 
 $: << File.dirname(File.expand_path(__FILE__))
 
+require 'archipelago/bitstring'
+require 'archipelago/oneliner'
+require 'archipelago/raider'
 require 'archipelago/disco'
 require 'archipelago/current'
 require 'archipelago/tranny'
 require 'archipelago/treasure'
+require 'archipelago/client'
 require 'archipelago/pirate'
+require 'archipelago/cove'
+require 'archipelago/exxon'

data/script/services.rb

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 
 if ARGV.size < 1
-  puts "Usage: #{$0} DB-PATH [EXTRA-LIBS]"
+  puts "Usage: #{$0} DB-PATH [DRB-URI [EXTRA-LIBS]]"
   exit 1
 end
 
@@ -9,16 +9,22 @@ $: << File.join(File.dirname(__FILE__), "..", "lib")
 
 require 'archipelago/treasure'
 require 'archipelago/tranny'
+require 'archipelago/cove'
 
-ARGV[1..-1].each do |lib|
-  load lib
+if ARGV.size > 1
+  DRb.start_service(ARGV[1])
+  ARGV[2..-1].each do |lib|
+    load lib
+  end
+else
+  DRb.start_service
 end
 
-DRb.start_service
-
 t = Archipelago::Tranny::Manager.new(:persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(File.join(ARGV[0], "tranny"))))
 t.publish!
 c = Archipelago::Treasure::Chest.new(:persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(File.join(ARGV[0], "chest"))))
 c.publish!
+tank = Archipelago::Cove::Tanker.new(:persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(File.join(ARGV[0], "tanker"))))
+tank.publish!
 
 DRb.thread.join

data/tests/test_helper.rb

@@ -3,6 +3,7 @@ home = File.expand_path(File.dirname(__FILE__))
 $: << File.join(home, "..", "lib")
 
 MC_DISABLED = true unless defined?(MC_ENABLED) && MC_ENABLED
+BLACKBEARD_DISABLED = true unless defined?(BLACKBEARD_ENABLED) && BLACKBEARD_ENABLED
 
 require 'pp'
 require 'drb'

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.8.11
 specification_version: 1
 name: archipelago
 version: !ruby/object:Gem::Version 
-  version: 0.2.4
-date: 2006-12-02 00:00:00 +01:00
+  version: 0.2.5
+date: 2006-12-20 00:00:00 +01:00
 summary: A set of tools for distributed computing in ruby.
 require_paths: 
 - lib
@@ -29,9 +29,13 @@ authors:
 - Martin Kihlgren
 files: 
 - lib/archipelago.rb
+- lib/archipelago/client.rb
+- lib/archipelago/cove.rb
 - lib/archipelago/current.rb
 - lib/archipelago/disco.rb
+- lib/archipelago/exxon.rb
 - lib/archipelago/hashish.rb
+- lib/archipelago/oneline.rb
 - lib/archipelago/pirate.rb
 - lib/archipelago/tranny.rb
 - lib/archipelago/treasure.rb
@@ -46,12 +50,10 @@ files:
 - tests/tranny_test.rb
 - tests/treasure_benchmark.rb
 - tests/treasure_test.rb
-- script/chest.rb
 - script/console
 - script/overloads.rb
 - script/pirate.rb
 - script/services.rb
-- script/tranny.rb
 - GPL-2
 - TODO
 - README

data/script/chest.rb

@@ -1,21 +0,0 @@
-#!/usr/bin/env ruby
-
-if ARGV.size < 1
-  puts "Usage: #{$0} DB-PATH [EXTRA-LIBS]"
-  exit 1
-end
-
-$: << File.join(File.dirname(__FILE__), "..", "lib")
-
-require 'archipelago/treasure'
-
-ARGV[1..-1].each do |lib|
-  load lib
-end
-
-DRb.start_service
-
-c = Archipelago::Treasure::Chest.new(:persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(ARGV[0])))
-c.publish!
-
-DRb.thread.join

data/script/tranny.rb

@@ -1,21 +0,0 @@
-#!/usr/bin/env ruby
-
-if ARGV.size < 1
-  puts "Usage: #{$0} DB-PATH [EXTRA-LIBS]"
-  exit 1
-end
-
-$: << File.join(File.dirname(__FILE__), "..", "lib")
-
-require 'archipelago/treasure'
-
-ARGV[1..-1].each do |lib|
-  load lib
-end
-
-DRb.start_service
-
-t = Archipelago::Tranny::Manager.new(:persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(ARGV[0])))
-t.publish!
-
-DRb.thread.join