abundance 1.2.3 → 1.2.4

data/lib/abundance.rb

@@ -2,10 +2,10 @@
 # Based on the low maintenance Gardener,Garden,Seed natural design pattern.
 # 
 # Its for:
-# * forking otherwise blocking loops in a non-blocking fashion
+# * running otherwise blocking loops in a non-blocking fashion
 # * a simple abstraction for loopback communication with a forked execution
 # * concurrent batch processing
-# * scaling process intensive computations to multi-core parallel execution
+# * scaling process intensive computations for SMP execution
 # 
 # And not:
 # * a replacement for Thread.new invocations
@@ -14,7 +14,7 @@
 # Its initial implementation uses:
 # * pure ruby
 # * standard forks as mean to parallel non-blocking execution
-# * fast UDP loopback sockets for process fork communication.
+# * fast loopback sockets for process fork communication.
 # * serialization friendly communication with process forks
 # * a tier queuing fork, as a packet control middle man between process forks and the non-blocking process client
 # * an elemental namespace: a process queue, named the Garden, with concurrent workers, named Rows, all this getting orchestrated by a Gardener.

data/lib/garden.rb

@@ -56,8 +56,6 @@ class Garden
           ask_for_init_status(message_block)
         when :init_crop
           answer_init_status(message_block)
-        when :seed_all
-          seed_for_all_rows(message_block)
         when :seed_all_crop
           special_crop_seed_all(message_block)
         when :close

data/lib/garden_cycles.rb

@@ -5,7 +5,7 @@ class Garden
   module Cycles
     
     def set_my_containers
-      @quit = false; @full_crop = false; @do_init = nil; @seed_all = nil; @init_all_crop = []
+      @quit = false; @full_crop = false; @do_init = nil; @seed_all_message_block = nil; @init_all_crop = []
       @harvest = []; @rows_socket_paths = []; @init_done = []; @seed_all_done = []; @seed_all_crop = []
       @seeds = []; @sprouts = []; @crops = []; @id = 0
     end
@@ -13,10 +13,10 @@ class Garden
     def seed_if_row_available
        catch :fill_rows do
          loop do
-           if ! @seed_all.nil? && ! @rows_socket_paths.empty? && @seed_all_done.size != @seed_all[0]
+           if ! @seed_all_message_block.nil? && ! @rows_socket_paths.empty? && @seed_all_done.size != @seed_all_message_block[1]
              row_socket_path = @rows_socket_paths.shift
              unless @seed_all_done.include?( row_socket_path )
-               socket_send(:seed_all,:garden,@seed_all[1],row_socket_path)
+               socket_send([:seed,:all,@seed_all_message_block[2],row_socket_path])
                @seed_all_done << row_socket_path
              else
                @rows_socket_paths << row_socket_path
@@ -24,7 +24,7 @@ class Garden
            elsif ! @do_init.nil? && ! @rows_socket_paths.empty? && @init_done.size != @do_init
              row_socket_path = @rows_socket_paths.shift
              unless @init_done.include?( row_socket_path )
-               socket_send(:init,:garden,'init_status',row_socket_path)
+               socket_send([:seed,:init,'init_status',row_socket_path])
                @init_done << row_socket_path
              else
                @rows_socket_paths << row_socket_path
@@ -33,11 +33,11 @@ class Garden
              seed = @seeds.shift
              @sprouts[seed[:id]] = seed
              row_socket_path = @rows_socket_paths.shift
-             socket_send(:sprout,:garden,seed,row_socket_path)
+             socket_send([:seed,:sprout,seed,row_socket_path])
            elsif @quit && ! @rows_socket_paths.empty?
              seed = nil
              row_socket_path = @rows_socket_paths.shift
-             socket_send(:quit,:garden,seed,row_socket_path)
+             socket_send([:seed,:quit,seed,row_socket_path])
            else
              throw :fill_rows
            end               
@@ -46,29 +46,42 @@ class Garden
      end
      
      def place_seed_in_queue(message_block)
-       @id += 1; @seeds << {:id => @id , :seed => message_block[2]}
-       socket_send(message_block[0],:garden,@id,message_block[3])
+       case message_block[1]
+       when :one
+         @id += 1; @seeds << {:id => @id , :seed => message_block[2]}
+         socket_send([message_block[0],:garden,@id,message_block[3]])
+       else
+         @seed_all_message_block = Array.new(message_block)
+       end
      end
      
      def this_row_is_available(message_block)
        if @quit
-         message_block[0] = :quit; seed = nil
+         message_block = [:row, :quit, nil, message_block[3]]
+       elsif ! @seed_all_message_block.nil? && @seed_all_done.size != @seed_all_message_block[1] && ! @seed_all_done.include?( message_block[3] )
+         message_block = [:row, :all, @seed_all_message_block[2], message_block[3]]
+         @seed_all_done << message_block[3]
+       elsif ! @do_init.nil? && @init_done.size != @do_init && ! @init_done.include?( message_block[3] )
+         message_block = [:row, :init, 'init_status', message_block[3]]
+         @init_done << message_block[3]
        elsif @seeds.empty?
-         seed = nil; @rows_socket_paths << message_block[2]
+         @rows_socket_paths << message_block[2]
+         message_block = [:row, :wait, nil, message_block[3]]
        else
          seed = @seeds.shift; @sprouts[seed[:id]] = seed
+         message_block = [:row, :sprout, seed, message_block[3]]
        end
-       socket_send(message_block[0],:garden,seed,message_block[3])
+       socket_send(message_block)
      end
      
      def save_crop_for(message_block)
        @sprouts[message_block[2][:id]] = nil
        @crops[message_block[2][:id]] = message_block[2]
        if @harvest[message_block[2][:id]]
-         socket_send(message_block[0],:garden,message_block[2], @harvest[message_block[2][:id]][:client_socket_path]) 
+         socket_send([message_block[0],:garden,message_block[2], @harvest[message_block[2][:id]][:client_socket_path]]) 
          @crops[message_block[2][:id]] = @harvest[message_block[2][:id]] = nil
        elsif @full_crop && @seeds.compact.empty? && @sprouts.compact.empty?
-         socket_send(message_block[0],:garden,@crops.compact,@mem_client_socket_path)
+         socket_send([message_block[0],:garden,@crops.compact,@mem_client_socket_path])
          @crops.clear; @full_crop = false
        end
      end
@@ -78,32 +91,32 @@ class Garden
        when :progress
          value = @crops.size.to_f / (@crops.size + @sprouts.compact.size + @seeds.size)
          value = 1 if value.nan?; progress = sprintf( "%.2f", value)
-         socket_send(message_block[0],:garden,progress,message_block[3])
+         socket_send([message_block[0],:garden,progress,message_block[3]])
        when :seed
-         socket_send(message_block[0],:garden,@seeds.size,message_block[3])
+         socket_send([message_block[0],:garden,@seeds.size,message_block[3]])
        when :sprout
-         socket_send(message_block[0],:garden,@sprouts.compact.size,message_block[3])
+         socket_send([message_block[0],:garden,@sprouts.compact.size,message_block[3]])
        when :crop
-         socket_send(message_block[0],:garden,@crops.size,message_block[3])
+         socket_send([message_block[0],:garden,@crops.size,message_block[3]])
        else
-         socket_send(message_block[0],:garden,false,message_block[3])
+         socket_send([message_block[0],:garden,false,message_block[3]])
        end
      end
      
      def harvest_some(message_block)
        case message_block[2]
        when :all
-         socket_send(message_block[0],:garden,{:seeds => @seeds, :sprouts => @sprouts.compact, :crops => @crops.compact},message_block[3])
+         socket_send([message_block[0],:garden,{:seeds => @seeds, :sprouts => @sprouts.compact, :crops => @crops.compact},message_block[3]])
        when :seed
-         socket_send(message_block[0],:garden,@seeds,message_block[3])
+         socket_send([message_block[0],:garden,@seeds,message_block[3]])
        when :sprout
-         socket_send(message_block[0],:garden,@sprouts.compact,message_block[3])
+         socket_send([message_block[0],:garden,@sprouts.compact,message_block[3]])
        when :crop
-         socket_send(message_block[0],:garden,@crops.compact,message_block[3])
+         socket_send([message_block[0],:garden,@crops.compact,message_block[3]])
          @crops.clear
        when :full_crop
          if @seeds.compact.empty? && @sprouts.compact.empty?
-           socket_send(message_block[0],:garden,@crops.compact,message_block[3])
+           socket_send([message_block[0],:garden,@crops.compact,message_block[3]])
            @crops.clear
          else
            @full_crop = true
@@ -112,13 +125,13 @@ class Garden
        else
          if message_block[2].is_a? Integer
            if @crops[message_block[2]]
-             socket_send(message_block[0],:garden,@crops[message_block[2]],message_block[3])
+             socket_send([message_block[0],:garden,@crops[message_block[2]],message_block[3]])
              @crops[message_block[2]] = nil
            else
              @harvest[message_block[2]] = {:client_socket_path => message_block[3]}
            end
          else
-           socket_send(message_block[0],:garden,false,message_block[3])
+           socket_send([message_block[0],:garden,false,message_block[3]])
          end
        end
      end
@@ -131,21 +144,16 @@ class Garden
      def answer_init_status(message_block)
        @init_all_crop << message_block[2]
        if @init_all_crop.size == @do_init
-         socket_send(message_block[0],:garden,@init_all_crop, @init_return[:client_socket_path])
+         socket_send([message_block[0],:garden,@init_all_crop, @init_return[:client_socket_path]])
          @init_return = Hash.new; @init_done = Array.new; @do_init = nil; @init_all_crop = Array.new
        end
      end
      
-     def seed_for_all_rows(message_block)
-       @seed_all = message_block[2]
-       @seed_all_return = {:client_socket_path => message_block[3], :data => []}
-     end
-     
      def special_crop_seed_all(message_block)
        @seed_all_crop << message_block[2]
-       if @seed_all_crop.size == @seed_all[0]
-         socket_send(message_block[0],:garden,@seed_all_crop, @seed_all_return[:client_socket_path])
-         @seed_all = nil; @seed_all_return = Hash.new; @seed_all_done = Array.new; @seed_all_crop = Array.new
+       if @seed_all_crop.size == @seed_all_message_block[1]
+         socket_send([message_block[0],:garden,@seed_all_crop, @seed_all_message_block[3]])
+         @seed_all_message_block = nil; @seed_all_done = Array.new; @seed_all_crop = Array.new
        end
      end
      
@@ -157,7 +165,7 @@ class Garden
        else
          @seeds_pid.delete(message_block[2][:pid].to_i)
          if @seeds_pid.empty?
-           socket_send(:close,:garden,{:seeds => @seeds, :sprouts => @sprouts.compact, :crops => @crops.compact}, @mem_client_socket_path)
+           socket_send([:close,:garden,{:seeds => @seeds, :sprouts => @sprouts.compact, :crops => @crops.compact}, @mem_client_socket_path])
            exit
          end
        end

data/lib/garden_rows.rb

@@ -51,32 +51,43 @@ class Garden
           t2 = Thread.new do
             loop do
               if $seed.nil?
-                message_block = socket_duplex(:row,:row,my_socket_path)
-                if message_block[0] == :quit
-                  pid = Process.pid
-                  socket_send(:close,:row,{:level => :seed, :pid => pid})
-                  exit
-                end
-                $seed = message_block[2]
-                if $seed.nil?
+                message_block = socket_duplex([:row,:row,my_socket_path,@garden_path])
+                case message_block[1]
+                when :sprout
+                  $seed = message_block[2]
+                when :all
+                  @seed_all = true
+                  $seed = {:id => Process.pid, :seed => message_block[2]}
+                when :wait
                   message_block = socket_recv
-                  case message_block[0]
+                  case message_block[1]
                   when :sprout
                     $seed = message_block[2]
-                  when :seed_all
+                  when :all
                     @seed_all = true
                     $seed = {:id => Process.pid, :seed => message_block[2]}
                   when :init
                     $init = {:seed => 'init_status', :message => 'No Init Message', :id => Process.pid} if $init.nil?
-                    socket_send(:init_crop,:row,$init)
+                    socket_send([:init_crop,:row,$init,@garden_path])
+                  when :quit
+                    pid = Process.pid
+                    socket_send([:close,:row,{:level => :seed, :pid => pid},@garden_path])
+                    exit
                   end
+                when :init
+                  $init = {:seed => 'init_status', :message => 'No Init Message', :id => Process.pid} if $init.nil?
+                  socket_send([:init_crop,:row,$init,@garden_path])
+                when :quit
+                  pid = Process.pid
+                  socket_send([:close,:row,{:level => :seed, :pid => pid},@garden_path])
+                  exit
                 end
               elsif ! $seed[:success].nil?
                 if @seed_all
-                  socket_send(:seed_all_crop,:row,$seed)
+                  socket_send([:seed_all_crop,:row,$seed,@garden_path])
                   @seed_all = false
                 else
-                  socket_send(:crop,:row,$seed)
+                  socket_send([:crop,:row,$seed,@garden_path])
                 end
                 $seed = nil;
               else

data/lib/gardener.rb

@@ -48,7 +48,7 @@ class Gardener
   # === Example
   #   puts gardener.init_status.inspect   # => [{:message=>"init ok", :success=>true, :pid=>4760}, {:message=>"init failed", :success=>false, :pid=>4761}] 
   def init_status
-    message_block = socket_duplex(:init,:gardener,@garden_rows.pids.size)
+    message_block = socket_duplex([:init,:gardener,@garden_rows.pids.size,@garden_path])
     message_block[2].map! do |row|
       {:success => row[:success], :message => row[:message], :pid => row[:id]}
     end
@@ -62,7 +62,7 @@ class Gardener
   #  id_seed_1 = gardener.seed("system 'ruby -v'")
   
   def seed(data)
-    message_block = socket_duplex(:seed,:gardener,data)
+    message_block = socket_duplex([:seed,:one,data,@garden_path])
     return message_block[2]
   end
   
@@ -74,8 +74,7 @@ class Gardener
   #   result = gardener.seed_all("pref local")  # =>  [{:success=>true, :message=>["row pref changed to local"], :seed=>"pref local", :pid=>14915},
   #                                               {:success=>true, :message=>["row pref changed to local"], :seed=>"pref local", :pid=>14913}]
   def seed_all(data)
-    seed = [@garden_rows.pids.size, data]
-    message_block = socket_duplex(:seed_all,:gardener,seed)
+    message_block = socket_duplex([:seed,@garden_rows.pids.size,data,@garden_path])
     message_block[2].map! do |row|
       {:success => row[:success], :message => row[:message], :pid => row[:id]}
     end
@@ -94,7 +93,7 @@ class Gardener
   #  puts "progress is now #{progress}"  # => progress is now 0.75
   
   def growth(data=:progress)
-    message_block = socket_duplex(:growth,:gardener,data)
+    message_block = socket_duplex([:growth,:gardener,data,@garden_path])
     return message_block[2]
   end
   
@@ -113,7 +112,7 @@ class Gardener
   #  seed1_result = gardener.harvest(id_seed_1)
   
   def harvest(data)
-    message_block = socket_duplex(:harvest,:gardener,data)
+    message_block = socket_duplex([:harvest,:gardener,data,@garden_path])
     return message_block[2]
   end
   
@@ -123,7 +122,7 @@ class Gardener
   #  final_harvest = gardener.close
   
   def close
-    message_block = socket_duplex(:close,:gardener,{:level => :garden, :pid => @garden_rows.pids})
+    message_block = socket_duplex([:close,:gardener,{:level => :garden, :pid => @garden_rows.pids},@garden_path])
     return message_block[2]
   end
   

data/lib/toolshed.rb

@@ -52,8 +52,8 @@ module Toolshed
   # * _command_ = command part of the sent packet
   # * _data_ = data part of the sent packet
   # * _server_socket_path_ = a UNIXServer socket path for the packets to be sent to
-  def socket_send(command,option,data,server_socket_path=@garden_path)
-    send_block(command,option,data,server_socket_path)
+  def socket_send(message_block)
+    send_block(message_block)
   end
   
   # The +socket_duplex+ method open a UNIXSocket and send packets to a UNIXServer socket, then wait for loopback communication from destination and return results like +socket_recv+.
@@ -61,8 +61,8 @@ module Toolshed
   # * _command_ = command part of the sent packet
   # * _data_ = data part of the sent packet
   # * _server_socket_path_ = a UNIXServer socket path for the packets to be sent to
-  def socket_duplex(command,option,data,server_socket_path=@garden_path)
-    send_block(command,option,data,server_socket_path)
+  def socket_duplex(message_block)
+    send_block(message_block)
     Marshal.load(recv_whole_block)
   end
   
@@ -121,10 +121,10 @@ module Toolshed
   # * _command_ = command part of the sent packet
   # * _data_ = data part of the sent packet
   # * _server_socket_path_ = the UNIXServer socket path to send to 
-  def send_block(command,option,data,server_socket_path)
+  def send_block(message_block)
     begin
-      client = UNIXSocket.open(server_socket_path)
-      client.send(Marshal.dump([command,option,data,@my_socket_path]),0)
+      client = UNIXSocket.open(message_block[3])
+      client.send(Marshal.dump(message_block[0..2] + [@my_socket_path]),0)
       client.close
     rescue Errno::EADDRINUSE
       retry

data/test/tc_burst.rb

@@ -0,0 +1,84 @@
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+require 'test/unit'
+require 'abundance'
+
+class TestHighAPI < Test::Unit::TestCase
+  
+  def test_burst1
+    check_init
+    seed_1000x10
+    check_init
+    seed_2000
+    check_seed_all
+    seed_2000
+    check_crop
+  end
+  
+  
+  def setup
+    @g = Abundance.gardener(:wheelbarrow => 124, :rows => 8, :init_timeout => 2) do
+      Abundance.init_status(true,Process.pid)
+      Abundance.grow do |seed|
+        sprout = seed.sprout
+        if sprout.is_a?(Hash)
+          seed.crop(true, "gardener: #{sprout[:jo]} - #{sprout[:lo]}")
+        elsif sprout.is_a?(Array)
+          result = sprout[0] ** sprout[1]
+          seed.crop(true, "gardener: #{result.to_s}")
+        else
+          seed.crop(true, "????????????????")
+        end
+      end      
+    end
+  end
+  
+  def teardown
+    @g.close
+  end
+  
+  private
+  
+  def seed_1000x10
+    (1..1000).each do |num1|
+      (1..10).each do |num2|
+        @g.seed([num1,num2])
+      end
+    end
+  end
+  
+  def seed_2000
+    seed = {:jo => 'ker', :lo => 'ver'}
+    2000.times do
+      @g.seed(seed)
+    end
+  end
+  
+  def check_init
+    @g.init_status.each do |init|
+      assert_not_nil(init[:message])
+      assert_not_nil(init[:success])
+      assert_not_nil(init[:pid])
+    
+      assert_not_equal(Process.pid,init[:message])
+      assert_equal(init[:message],init[:pid])
+    end
+  end
+  
+  def check_seed_all
+    all = @g.seed_all([1000,1000])
+    assert_equal(8,all.size)
+    all.map! { |seed| seed[:message] == "gardener: #{(1000**1000).to_s}" }
+    assert( all.uniq.size == 1 && all[0] == true )
+  end
+  
+  def check_crop
+    crop = @g.harvest(:full_crop)
+    assert_equal(14000,crop.size)
+    assert_kind_of(Array,crop)
+    assert_kind_of(Hash,crop[0])
+    assert( ! crop[0][:success].nil? && (crop[0][:success] == true || crop[0][:success] == false))
+    assert(crop[0][:seed].class == Array || crop[0][:seed].class == Hash)
+    assert_kind_of(String,crop[0][:message])
+    assert_kind_of(Numeric,crop[0][:id])
+  end
+end

data/test/ts_abundance.rb

@@ -2,4 +2,5 @@ $:.unshift File.join(File.dirname(__FILE__), "..", "lib")
 require 'test/unit'
 require 'test/tc_high_api'
 require 'test/tc_robustness'
-require 'test/tc_multi_gardener'
+require 'test/tc_multi_gardener'
+require 'test/tc_burst'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: abundance
 version: !ruby/object:Gem::Version 
-  version: 1.2.3
+  version: 1.2.4
 platform: ruby
 authors: 
 - Louis-Philippe Perron
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-01-14 00:00:00 -05:00
+date: 2009-01-16 00:00:00 -05:00
 default_executable: 
 dependencies: []
 
@@ -29,6 +29,7 @@ files:
 - lib/gardener.rb
 - lib/seed.rb
 - lib/toolshed.rb
+- test/tc_burst.rb
 - test/tc_high_api.rb
 - test/tc_multi_gardener.rb
 - test/tc_robustness.rb