oneliner 0.2.6

data/lib/oneliner.rb

@@ -0,0 +1,21 @@
+# Archipelago - a distributed computing toolkit for ruby
+# Copyright (C) 2006  Martin Kihlgren <zond at troja dot ath dot cx>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+$: << File.dirname(File.expand_path(__FILE__))
+
+require 'oneliner_ext'
+require 'oneliner/superstring'

data/lib/oneliner/superstring.rb

@@ -0,0 +1,385 @@
+# Archipelago - a distributed computing toolkit for ruby
+# Copyright (C) 2006  Martin Kihlgren <zond at troja dot ath dot cx>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+require 'set'
+require 'digest/sha2'
+
+module Oneliner
+  
+  #
+  # A String subclass providing Online Code functionality.
+  #
+  class SuperString < String
+
+    E = 0.01
+    Q = 3
+    F = (Math.log((E ** 2) / 4) / Math.log(1 - (E / 2))).abs.to_i
+    P = [0, (1 - ((1 + (1 / F)) / (1 + E)))]
+    2.upto(F) do |i|
+      P << (((1 - P[1]) * F) / ((F - 1) * i * (i - 1)))
+    end
+
+    #
+    # Will return a String containing +requested_size+ nr of bytes
+    # as an online coded chunk of blocks for this SuperString.
+    #
+    def encode(requested_size)
+      raise "requested size is too small for metadata (8 bytes)" unless requested_size > 7
+
+      ensure_encode_format
+      srand
+      seed = rand(Context::INT_MAX)
+      context = Context.new
+      context.seed(seed)
+      
+      rval = [self.size].pack("i")
+      rval << [seed].pack("i")
+
+      blocks = []
+      wanted_blocks = ((requested_size - 8) * 8) / @block_size
+      1.upto(wanted_blocks) do |n|
+        blocks << generate_check_block(context, n - 1)
+      end
+
+      return rval + compact(blocks)
+    end
+
+    #
+    # Will try to decode this SuperString using the given +chunk+ and all
+    # formerly given chunks.
+    #
+    # Returns whether decoding is done.
+    #
+    def decode!(chunk)
+      raise "#{chunk.inspect} is too small for metadata (8 bytes)" unless chunk.size > 7
+
+      @decode_done = nil
+
+      context = Context.new
+
+      requested_size = chunk[0..3].unpack("i").first
+
+      ensure_decode_format(requested_size)
+
+      the_seed = chunk[4..7].unpack("i").first
+
+      chunk_data = build_chunk_data(chunk, context, the_seed)
+      
+      @known_nr_of_blocks += chunk_data.size
+      @known_chunks.unshift(chunk_data)
+
+      if @known_nr_of_blocks > @nr_of_blocks
+
+        nil while do_decode(context)
+        
+        if decode_done?
+          self.replace(compact(@blocks[0...@nr_of_data_blocks])[0...requested_size])
+          return true
+        else
+          return false
+        end
+        
+      else
+        return false
+      end
+    end
+
+    #
+    # Returns whether decoding is done.
+    #
+    def decode_done?
+      return false unless defined?(@decode_done) && defined?(@nr_of_data_blocks)
+
+      unless @decode_done
+        data = @blocks[0...@nr_of_data_blocks]
+        @decode_done = data.compact.size == data.size
+      end
+      return @decode_done
+    end
+
+    private
+
+    #
+    # Misc stuff
+    #
+
+    #
+    # Builds a Hash with index of each block in the +chunk+
+    # paired with the block itself and the source blocks for 
+    # that block determined using +context+ and +the_seed+.
+    #
+    def build_chunk_data(chunk, context, the_seed)
+      blocks_to_return = {}
+
+      context.seed(the_seed)
+
+      expand(chunk[8..-1]).each_with_index do |block, index|
+        this_degree = get_degree(context)
+        these_blocks = []
+        this_degree.times do |m|
+          these_blocks << context.random(@nr_of_blocks)
+        end
+
+        blocks_to_return[index] = [block, these_blocks]
+      end
+
+      return blocks_to_return
+    end
+
+    #
+    # Split +s+ up into @block_size (in bits if less than 8, otherwise closed matching nr of bytes) pieces
+    # and return them in an array.
+    #
+    def expand(s)
+      if @block_size < 8
+        rval = s.unpack("b*").first
+        rval = rval.gsub(/(.{#{@block_size},#{@block_size}})/, "\\1#{"0" * (8 - @block_size)}")
+        rval += "0" * (8 - (rval.size % 8)) if (rval.size % 8) > 0
+        rval = [rval].pack("b*")
+        return rval.split(//)
+      else
+        bytes_per_block = @block_size / 8
+        rval = s
+        rval += "\000" * (bytes_per_block - (rval.size % bytes_per_block)) if (rval.size % bytes_per_block) > 0
+        return rval.hack(bytes_per_block)
+      end
+    end
+    
+    #
+    # Glue +blocks+ together into a String.
+    #
+    def compact(blocks)
+      if @block_size < 8
+        rval = blocks.join
+        rval = rval.unpack("b*")
+        rval = rval.first.gsub(/(.{#{@block_size},#{@block_size}}).{#{8 - @block_size},#{8 - @block_size}}/,
+                               "\\1")
+        rval += "0" * (8 - (rval.size % 8)) if (rval.size % 8) > 0
+        return [rval].pack("b*")
+      else
+        return blocks.join
+      end
+    end
+
+    #
+    # Determine and save the nr of aux_blocks we want for this SuperString.
+    #
+    def set_nr_of_aux_blocks
+      @nr_of_aux_blocks ||= (0.55 * Q * E * @nr_of_data_blocks).ceil
+    end
+
+    #
+    # Determine and save the block_size we want for this SuperString. 
+    #
+    def set_block_size
+      unless defined?(@block_size)
+        if size < (256 + 128)
+          @block_size = 1
+        else
+          @block_size = size / 256
+        end
+      end
+      return @block_size
+    end
+
+    #
+    # Determine and save the nr_of_blocks we want for this SuperString.
+    #
+    def set_nr_of_blocks
+      @nr_of_blocks ||= @blocks.size
+    end
+
+    #
+    # Decoding stuff
+    #
+
+    def do_decode(context)
+      found_new_block = false
+      @known_chunks.clone.each_with_index do |chunk_data, i|
+        this_chunk_found_new_block, this_chunk_informative = decode_chunk(chunk_data)
+        found_new_block = found_new_block || this_chunk_found_new_block
+        @known_chunks.delete(i) unless this_chunk_informative
+      end
+      return aux_decode || found_new_block
+    end
+
+    def aux_decode
+      got_new_block = false
+
+      @nr_of_data_blocks.upto(@nr_of_blocks - 1) do |i|
+
+        aux_block = @blocks[i]
+        if aux_block
+
+          source_blocks = @aux_hash[i]
+          if source_blocks.size == 1
+            block_nr = source_blocks.first
+
+            if @blocks[block_nr].nil?
+              @blocks[block_nr] = aux_block
+              got_new_block = true
+              @blocks[i] = nil
+            end
+          else
+            missing_blocks = source_blocks.size
+            missing_block = nil
+            xor_sum = aux_block
+            source_blocks.each do |block|
+              source_block = @blocks[block]
+              if source_block
+                missing_blocks -= 1
+                xor_sum ^= source_block
+              else
+                missing_block = block
+              end
+            end
+
+            if missing_blocks == 1
+              @blocks[missing_block] = xor_sum
+              got_new_block = true
+              @blocks[i] = nil
+            end
+          end
+        end
+      end
+
+      return got_new_block
+    end
+
+    def decode_chunk(blocks)
+      got_new_block = false
+      got_informative_block = false
+
+      blocks.clone.each do |index, block_data|
+
+        block, block_numbers = block_data
+
+        got_informative_block = true
+        
+        if block_numbers.size == 1
+          block_nr = block_numbers.first
+          
+          if @blocks[block_nr].nil?
+            @blocks[block_nr] = block
+            got_new_block = true
+            blocks.delete(index)
+          end
+        else
+          if decode_multiple(block_data, @blocks, @nr_of_blocks)
+            got_new_block = true
+            blocks.delete(index)
+          end
+        end
+        
+      end
+      
+      return [got_new_block, got_informative_block]
+    end
+    
+    def generate_aux_hash
+      rval = Array.new(@nr_of_blocks)
+      
+      context = Context.new
+      context.seed(@nr_of_data_blocks)
+
+      0.upto(@nr_of_data_blocks - 1) do |i|
+        1.upto(Q) do
+          aux_block_nr = @nr_of_data_blocks + context.random(@nr_of_aux_blocks)
+          (rval[aux_block_nr] ||= []) << i
+        end
+      end
+
+      return rval
+    end
+
+    def ensure_decode_format(requested_size)
+      unless defined?(@blocks)
+        self.concat("\000" * requested_size)
+        set_block_size
+        @nr_of_data_blocks = expand(self).size
+        set_nr_of_aux_blocks
+        @blocks = Array.new(@nr_of_data_blocks)
+        @blocks += Array.new(@nr_of_aux_blocks)
+        set_nr_of_blocks
+        @aux_hash = generate_aux_hash
+        @known_chunks = []
+        @known_nr_of_blocks = 0
+        @known_block_nrs_by_seed = {}
+      end
+    end
+
+    def ensure_size(requested_size)
+      if size == 0
+        concat("\000" * requested_size)
+        @decoded_blocks = Set.new
+      elsif size != requested_size
+        raise "size of #{self} (#{size}) is wrong, should be #{requested_size}"
+      end
+    end
+
+    #
+    # Encoding stuff
+    #
+
+    def generate_check_block(context, n)
+      degree = get_degree(context)
+
+      block_nr = context.random(@nr_of_blocks)
+      rval = @blocks[block_nr]
+
+      2.upto(degree) do
+        block_nr = context.random(@nr_of_blocks)
+        rval ^= @blocks[block_nr]
+      end
+      return rval
+    end
+    
+    def generate_aux_blocks
+      rval = Array.new(@nr_of_aux_blocks)
+
+      context = Context.new
+      context.seed(@nr_of_data_blocks)
+
+      b_size = nil
+      @blocks.each_with_index do |b, i|
+        1.upto(Q) do
+          aux_block_nr = context.random(@nr_of_aux_blocks)
+          if rval[aux_block_nr].nil?
+            rval[aux_block_nr] = b
+          else
+            rval[aux_block_nr] ^= b
+          end
+        end
+      end
+
+      return rval
+    end
+
+    def ensure_encode_format
+      unless defined?(@blocks)
+        set_block_size
+        @blocks = expand(self)
+        @nr_of_data_blocks = @blocks.size
+        set_nr_of_aux_blocks
+        @blocks += generate_aux_blocks
+        @nr_of_blocks = @blocks.size
+      end
+    end
+
+  end
+
+end

data/tests/superstring_benchmark.rb

@@ -0,0 +1,60 @@
+
+require File.join(File.dirname(__FILE__), 'test_helper')
+
+class SuperStringTest < Test::Unit::TestCase
+
+  def test_decoding
+    puts "required for 512 blocks: #{required(1, 64)}"
+    puts "required for 1024 blocks: #{required(1, 128)}"
+    puts "required for 2048 blocks: #{required(1, 256)}"
+    puts "required for 3064 blocks: #{required(1, 127 + 256)}"
+  end
+
+  def test_speed
+    7.upto(20) do |n|
+      decode(1 << n)
+    end
+  end
+
+  private
+
+  def randstr(size)
+    rval = ""
+    size.times do 
+      rval += rand(128).chr
+    end
+    rval
+  end
+
+  def decode(n)
+    s2 = Oneliner::SuperString.new(randstr(n))
+    bm("#{n} bytes in 40% chunks", :n => 10) do
+      s = Oneliner::SuperString.new
+      nil while !s.decode!(s2.encode((n * 0.4).to_i + 8))
+    end
+    bm("#{n} bytes in 5% chunks", :n => 10) do
+      s = Oneliner::SuperString.new
+      nil while !s.decode!(s2.encode((n * 0.05).to_i + 8))
+    end
+  end
+
+  def required(n, size)
+    total_req = 0
+    n.times do
+      s = Oneliner::SuperString.new
+      s2 = Oneliner::SuperString.new(" " * size)
+      total_req += measure_required(s, s2)
+    end
+    return total_req / n.to_f
+  end
+
+  def measure_required(dest, source)
+    dest.decode!(source.encode(source.size))
+    while (!dest.decode!(source.encode((source.size.to_f * 0.01).ceil + 8)))
+      print "."
+      STDOUT.flush
+    end
+    return dest.instance_eval do @known_nr_of_blocks end.to_f / source.instance_eval do @nr_of_data_blocks end.to_f
+  end
+
+end