abundance 1.0.9 → 1.1.0

data/lib/abundance.rb

@@ -1,11 +1,12 @@
 # This class provides a mean to parallelize the execution of your program processes.
+# It is a process queue, named the Garden, with concurrent workers, named Rows, all this getting orchestrated by a Gardener.
 # 
-# Its caracteristics are:
-# * concurent
+# Its characteristics are:
+# * concurrent
 # * non-blocking
 # * simple
 # * pure ruby
-# * no dependency installation
+# * serialization friendly 
 # 
 # It:
 # * scales to multi core
@@ -50,7 +51,7 @@ class Abundance
   # with its garden supplied as a block.  The invocation block must include
   # the +grow+ class method and a preceeding optional initialisation section that may include and +init_status+ return message.
   # === Parameters
-  # * :seed_size = allowed seed size in bytes
+  # * :wheelbarrow = the socket size for the garden communication packets, in bytes, range from 1024 to 8192, defaults to 8192
   # * :rows = garden rows number, the number of concurent threads
   # * :init_timeout = allow to pause execution to allow for larger gardens to initialize
   # === Example
@@ -81,7 +82,7 @@ class Abundance
   # 
   #  gardener.close
   
-  def Abundance.gardener(options={:seed_size => 8192, :rows => 2, :init_timeout => 2},&gardener_block)
+  def Abundance.gardener(options={:wheelbarrow => 8192, :rows => 2, :init_timeout => 2},&gardener_block)
     return Gardener.new(options,gardener_block)
   end
   

data/lib/garden.rb

@@ -220,8 +220,8 @@ class Garden
       @pids = []
       rows.times do
         row_port = Toolshed.available_port
-        @socket_client_perm = Toolshed.socket_client_perm
         @pids << fork do
+          @socket_client_perm = Toolshed.socket_client_perm
           @seed_all = false
           @socket_server = Toolshed.socket_server(row_port)
           t1 = Thread.new do

data/lib/gardener.rb

@@ -23,15 +23,18 @@ class Gardener
   # As part of the Abundance lib, Gardener is not initialized directly, 
   # but rather through Abundance.gardener.
   # === Parameters
-  # * :seed_size = allowed seed size in bytes
+  # * :wheelbarrow = the socket size for the garden communication packets, in bytes, range from 1024 to 8192, defaults to 8192
   # * :rows = garden rows number, the number of concurent threads
   # * :init_timeout = allow to pause execution to allow for larger gardens to initialize
   # === Example
-  #  gardener = Gardener.new({:seed_size => 1024, :rows => 6, :init_timeout}) { your_special_garden function }
+  #  gardener = Gardener.new({:wheelbarrow => 1024, :rows => 6, :init_timeout}) { your_special_garden function }
   # 
   
   def initialize(options,gardener_block)
-    Toolshed::block_size = options[:seed_size]
+    Toolshed::block_size = if options[:wheelbarrow].nil? || options[:wheelbarrow] > 8192 then 8192
+                          elsif options[:wheelbarrow] < 1024 then 1024
+                          else options[:wheelbarrow]
+                          end
     Toolshed::garden_port = Toolshed.available_port
     
     @garden = Garden.new
@@ -69,7 +72,7 @@ class Gardener
   # === Parameter
   # * _command_ = a ruby expression or object
   # === Example
-  #   result = gardener.seed("pref local")  # =>  [{:success=>true, :message=>["row pref changed to local"], :seed=>"pref local", :pid=>14915},
+  #   result = gardener.seed_all("pref local")  # =>  [{:success=>true, :message=>["row pref changed to local"], :seed=>"pref local", :pid=>14915},
   #                                               {:success=>true, :message=>["row pref changed to local"], :seed=>"pref local", :pid=>14913}]
   def seed_all(command)
     seed = [@garden_rows.pids.size, command]

data/lib/toolshed.rb

@@ -14,7 +14,9 @@ module Toolshed
   require 'socket'
   UDP_HOST = 'localhost'
   @@start_port = 50000
+  @@sessions = []
   
+  # The Toolshed.available_port method scans for a an available UDP port and returns its value.
   def Toolshed.available_port
     port = @@start_port + 1
     catch :scan_port do
@@ -34,62 +36,122 @@ module Toolshed
     return port
   end
   
+  # The Toolshed.socket_client_perm method creates a client socket permanently connected to the main server socket.  Returns the client socket object.
   def Toolshed.socket_client_perm
     socket = UDPSocket.new
     socket.connect(UDP_HOST,@@garden_port)
     return socket
   end
   
+  # The Toolshed.socket_client_temp method creates a client socket available for changing connections to multiple socket servers. Returns the client socket object.
   def Toolshed.socket_client_temp
     UDPSocket.new
   end
   
+  # The Toolshed.socket_server method creates a permanent main server socket.  Returns the server socket object.
   def Toolshed.socket_server(port)
     socket_server = UDPSocket.new
     socket_server.bind(nil,port)
     return socket_server
   end
   
+  # The Toolshed::block_size= method sets the UDP block size for socket operations.
   def Toolshed::block_size=(block_size)
     @@block_size = block_size
   end
   
+  # The Toolshed::block_size method gets and returns the UDP block size for socket operations.
   def Toolshed::block_size
     @@block_size
   end
   
+  # The Toolshed::garden_port= sets the UDP socket port for the garden server.
   def Toolshed::garden_port=(garden_port)
     @@garden_port = garden_port
   end
   
+  # The Toolshed::garden_port gets the UDP socket port of the garden server.
   def Toolshed::garden_port
     @@garden_port
   end
   
+  # The +socket_client_perm_duplex+ method is used as the main Row loop send/receive method and for all gardener's send/receive
   def socket_client_perm_duplex(command,data)
-    @socket_client_perm.send(Marshal.dump([command,data]),0)
-    recv_block,address = @socket_client_perm.recvfrom(@@block_size)
+    block_splitter([command,data]) do |block|
+      @socket_client_perm.send(block,0)
+    end
+    recv_block,address = block_filter { @socket_client_perm.recvfrom(@@block_size) }
     return Marshal.load(recv_block)
   end
   
   def socket_client_perm_send(command,data)
-    @socket_client_perm.send(Marshal.dump([command,data]),0)
+    block_splitter([command,data]) do |block|
+      @socket_client_perm.send(block,0)
+    end
   end
   
+  # The +socket_client_temp+ method is used by the Garden for connecting with the Row idle socket for quick messages. 
   def socket_client_temp(command,data,port)
     @socket_client_temp.connect(UDP_HOST,port)
-    @socket_client_temp.send(Marshal.dump([command,data]),0)
+    block_splitter([command,data]) do |block|
+      @socket_client_temp.send(block,0)
+    end
   end
   
+  # The +socket_server_recv+ method is used by the Garden permanent socket and by the Row idle socket to wait for incomming messages.
   def socket_server_recv
-    block,address = @socket_server.recvfrom(@@block_size)
+    block,address = block_filter { @socket_server.recvfrom(@@block_size) }
     clientport = address[1]; clientname = address[2]; clientaddr = address[3] 
     command, data = Marshal.load(block)
     return command, data, clientport, clientname, clientaddr
   end
   
+  # The +socket_server_send+ method is used by the Garden to answer back to socket clients.
   def socket_server_send(command,data,clientaddr,clientport)
-    @socket_server.send(Marshal.dump([command,data]), 0, clientaddr, clientport)
+    block_splitter([command,data]) do |block|
+      @socket_server.send(block, 0, clientaddr, clientport)
+    end
+  end
+  
+  private 
+  # The +block_splitter+ method is used internally by the Toolshed method to split message into acceptable UDP block size.  Its operating as a block method, sending a message chunk on each iteration.
+  def block_splitter(data)
+    data_string = Marshal.dump(data)
+    if data_string.size >= @@block_size
+      parts = data_string.size / (@@block_size - @@block_size/32) + 1
+      yield Marshal.dump([:block_head,Process.pid,parts])
+      parts.times do |num|
+        part = [data_string[0,@@block_size - @@block_size/32]]; data_string[0,@@block_size - @@block_size/32] = ''
+        yield Marshal.dump([:block_part,Process.pid,num, part])
+      end
+    else
+      yield data_string
+    end
+  end
+  
+  # The +block_filter+ method is used internally by the Toolshed method to filter splitted messages blocks, buffering them until the full message can be reconstructed.
+  def block_filter
+    loop do
+      block,address = yield
+      block_array = Marshal.load(block)
+      if block_array[0] == :block_head
+        @@sessions[block_array[1]] = {} if @@sessions[block_array[1]].nil?
+        @@sessions[block_array[1]][:size] = block_array[2]
+        @@sessions[block_array[1]][:address] = address
+      elsif block_array[0] == :block_part
+        @@sessions[block_array[1]] = {} if @@sessions[block_array[1]].nil?
+        @@sessions[block_array[1]][:data] = [] if @@sessions[block_array[1]][:data].nil?
+        @@sessions[block_array[1]][:data][block_array[2]] = block_array[3]
+      else
+        return block,address
+      end
+      if ! @@sessions[block_array[1]].nil? && ! @@sessions[block_array[1]][:data].nil? && @@sessions[block_array[1]][:data].size == @@sessions[block_array[1]][:size]
+        block = @@sessions[block_array[1]][:data].join
+        address = @@sessions[block_array[1]][:address]
+        @@sessions[block_array[1]] = nil
+        return block,address
+      end
+    end
   end
   
 end

data/test/tc_high_api.rb

@@ -0,0 +1,147 @@
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+require 'test/unit'
+require 'abundance'
+
+class TestHighAPI < Test::Unit::TestCase
+  
+  def test_abundance_monothread
+    @rows = 1
+    set_gardener
+    reality_check
+  end
+  
+  def test_abundance_quadthread
+    @rows = 4
+    set_gardener
+    reality_check
+  end
+  
+  def test_abundance_hexthread
+    @rows = 16
+    set_gardener
+    reality_check
+  end
+  
+  def teardown
+    final = @g.close
+    assert_kind_of(Hash,final)
+    assert_equal(3,final.size)
+    assert_not_nil(final[:seeds])
+    assert_not_nil(final[:sprouts])
+    assert_not_nil(final[:crops])
+  end
+  
+  private
+  
+  def set_gardener
+    @g = Abundance.gardener(:wheelbarrow => 8192, :rows => @rows, :init_timeout => 3) do
+      Abundance.init_status(true,Process.pid)
+      Abundance.grow do |seed|
+        seed.crop(true, "gardener: #{seed.sprout}")
+      end      
+    end
+  end
+  
+  def reality_check
+    assert_instance_of(Gardener,@g)
+    
+    check_init          
+    check_seed_harvest      # leaves no crops in the queue
+    check_full_harvest         # also leaves no crops in the queue
+    check_deep_harvest    # may leave crap behind, so needs to come last of the harvest
+    check_growth
+    check_seed_all
+    
+  end
+  
+  def check_init
+    @g.init_status.each do |init|
+      assert_not_nil(init[:message])
+      assert_not_nil(init[:success])
+      assert_not_nil(init[:pid])
+    
+      assert_not_equal(Process.pid,init[:message])
+      assert_equal(init[:message],init[:pid])
+    end
+  end
+  
+  def check_seed_harvest
+    id = @g.seed(Process.pid)
+    assert_kind_of(Integer,id)
+    
+    answer = @g.harvest(id)
+    assert_kind_of(Hash,answer)
+    assert_equal(Process.pid,answer[:seed])
+    assert_equal(id,answer[:id])
+    assert_equal(true,answer[:success])
+    assert_equal("gardener: #{Process.pid}",answer[:message])
+  end
+  
+  def check_full_harvest
+    queue_items = {}
+    25.times do |num|
+      queue_items[num] = @g.seed(num)
+    end
+    full_crop = @g.harvest(:full_crop)
+    assert_equal(25,full_crop.size)
+    results = []
+    queue_items.each do |num,id|
+      success = false
+      full_crop.each do |crop|
+        success = true if crop[:id] == id && crop[:seed] == num && crop[:message] == "gardener: #{num}"
+      end
+      results << success
+    end
+    assert(results[0] == true && results.uniq.size == 1)
+  end
+  
+  def check_growth
+    progress = @g.growth(:progress)
+    assert_kind_of(String,progress)
+    assert(progress.to_f >= 0 && progress.to_f <= 1)
+    
+    seeds_growth = @g.growth(:seed)
+    assert_kind_of(Integer,seeds_growth)
+    sprouts_growth = @g.growth(:sprout)
+    assert_kind_of(Integer,sprouts_growth)
+    crops_growth = @g.growth(:crop)
+    assert_kind_of(Integer,crops_growth)
+  end
+  
+  def check_deep_harvest
+    queue_items = {}
+    25.times do |num|
+      queue_items[num] = @g.seed(num)
+    end
+    
+    all = @g.harvest(:all)
+    assert_kind_of(Hash,all)
+    assert_equal(25,all[:seeds].size + all[:sprouts].size + all[:crops].size)
+    seeds_harvest = @g.harvest(:seed); assert_kind_of(Array,seeds_harvest)
+    sprouts_harvest = @g.harvest(:sprout); assert_kind_of(Array,sprouts_harvest)
+    crops_harvest = @g.harvest(:crop); assert_kind_of(Array,crops_harvest)
+    results = []
+    queue_items.each do |num,id|
+      success = false
+      seeds_harvest.each do |seed|
+        success = true if seed[:id] == id && seed[:seed] == num
+      end
+      sprouts_harvest.each do |sprout|
+        success = true if sprout[:id] == id && sprout[:seed] == num
+      end
+      crops_harvest.each do |crop|
+        success = true if crop[:id] == id && crop[:seed] == num && crop[:message] == "gardener: #{num}"
+      end
+      results << success
+    end
+    assert(results[0] == true && results.uniq.size == 1)
+  end
+  
+  def check_seed_all
+    all = @g.seed_all("all")
+    assert_equal(@rows,all.size)
+    all.map! { |seed| seed[:message] == "gardener: all" }
+    assert( all.uniq.size == 1 && all[0] == true )
+  end
+  
+end

data/test/tc_robustness.rb

@@ -0,0 +1,67 @@
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+require 'test/unit'
+require 'abundance'
+
+class TestHighAPI < Test::Unit::TestCase
+  
+  def test_abundance_robust
+    @seed_size = 8192
+    @rows = 8
+    @init_timeout = 3
+    hyper_gardener
+    seed_3000
+    seed_1000x10
+  end
+  
+  def teardown
+    @g.close
+  end
+
+  private
+  
+  def seed_3000
+    seed = {:jo => 'ker', :lo => 'ver'}
+    3000.times do
+      @g.seed(seed)
+    end
+    crop = @g.harvest(:full_crop)
+    assert_crop(crop)
+  end
+  
+  def seed_1000x10
+    (1..1000).each do |num1|
+      (1..10).each do |num2|
+        @g.seed([num1,num2])
+      end
+    end
+    crop = @g.harvest(:full_crop)
+    assert_crop(crop)
+  end
+  
+  def assert_crop(crop)
+    assert_kind_of(Array,crop)
+    assert_kind_of(Hash,crop[0])
+    assert( ! crop[0][:success].nil? && (crop[0][:success] == true || crop[0][:success] == false))
+    assert(crop[0][:seed].class == Array || crop[0][:seed].class == Hash)
+    assert_kind_of(String,crop[0][:message])
+    assert_kind_of(Numeric,crop[0][:id])
+  end
+
+  def hyper_gardener
+    @g = Abundance.gardener(:wheelbarrow => @seed_size, :rows => @rows, :init_timeout => @init_timeout) do
+      Abundance.init_status(true,Process.pid)
+      Abundance.grow do |seed|
+        sprout = seed.sprout
+        if sprout.is_a?(Hash)
+          seed.crop(true, "gardener: #{sprout[:jo]} - #{sprout[:lo]}")
+        elsif sprout.is_a?(Array)
+          result = sprout[0] ** sprout[1]
+          seed.crop(true, "gardener: #{result.to_s}")
+        else
+          seed.crop(true, "????????????????")
+        end
+      end      
+    end
+  end
+  
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: abundance
 version: !ruby/object:Gem::Version 
-  version: 1.0.9
+  version: 1.1.0
 platform: ruby
 authors: 
 - Louis-Philippe Perron
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2008-12-15 00:00:00 -05:00
+date: 2008-12-23 00:00:00 -05:00
 default_executable: 
 dependencies: []
 
@@ -27,6 +27,8 @@ files:
 - lib/gardener.rb
 - lib/seed.rb
 - lib/toolshed.rb
+- test/tc_high_api.rb
+- test/tc_robustness.rb
 has_rdoc: true
 homepage: http://abundance.rubyforge.org/
 post_install_message: